From e0ad0f14c272b3c8e2f559dfe3f7fd66283211cf Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 6 Oct 2026 13:09:29 -0700 Subject: [PATCH 001/336] simx: deliver same-cycle channel events in the order they were sent LinkedList::push_back inserted before the current tail, because insert() places a node ahead of its position: pushing 1, 2, 3 produced 2, 3, 1. The timing wheel and the zero-delay event list are LinkedLists, so two packets sent into one SimChannel in the same cycle with the same delay arrived reversed. Models almost never send twice per channel per cycle, which kept it hidden; a two-wide fetch prototype exposed it as a younger instruction issuing ahead of its producer. push_back now appends at end(). The header also includes so it compiles on its own. tests/unittest/linked_list checks the order and fails against the old header. SimX legs of all 31 model_parity cases: identical retired instructions; cycles move by at most 0.7%, except rt_raycast +1.7%, toward rtlsim (gap 2.4% -> 0.8% against a current rtlsim run). rtlsim and the perf gate do not use this code. Co-Authored-By: Claude Opus 5.5 --- sw/common/linked_list.h | 3 +- tests/unittest/Makefile | 2 +- tests/unittest/linked_list/Makefile | 10 ++++++ tests/unittest/linked_list/main.cpp | 55 +++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 tests/unittest/linked_list/Makefile create mode 100644 tests/unittest/linked_list/main.cpp diff --git a/sw/common/linked_list.h b/sw/common/linked_list.h index f75a6b8dae..38f175ed54 100644 --- a/sw/common/linked_list.h +++ b/sw/common/linked_list.h @@ -14,6 +14,7 @@ #pragma once #include +#include #include namespace vortex { @@ -238,7 +239,7 @@ class LinkedList { } void push_back(T *obj) { - insert(iterator(tail_), obj); + insert(end(), obj); } void push_front(T *obj) { diff --git a/tests/unittest/Makefile b/tests/unittest/Makefile index b344a6ff1a..0c561d642e 100644 --- a/tests/unittest/Makefile +++ b/tests/unittest/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../..) include $(ROOT_DIR)/config.mk -PROJECTS := vx_malloc dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt +PROJECTS := vx_malloc linked_list dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt all: $(foreach p,$(PROJECTS),$(MAKE) -C $(p) &&) true diff --git a/tests/unittest/linked_list/Makefile b/tests/unittest/linked_list/Makefile new file mode 100644 index 0000000000..293746789e --- /dev/null +++ b/tests/unittest/linked_list/Makefile @@ -0,0 +1,10 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := linked_list + +SRC_DIR := $(VORTEX_HOME)/tests/unittest/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +include ../common.mk diff --git a/tests/unittest/linked_list/main.cpp b/tests/unittest/linked_list/main.cpp new file mode 100644 index 0000000000..9434ed22bd --- /dev/null +++ b/tests/unittest/linked_list/main.cpp @@ -0,0 +1,55 @@ +#include +#include +#include + +using namespace vortex; + +struct Node { + int value; + LinkedListNode hook; +}; + +using List = LinkedList; + +static bool expect(const List& list, const std::vector& want, const char* what) { + std::vector got; + for (auto& n : list) { + got.push_back(n.value); + } + if (got == want) + return true; + printf("Error: %s: got {", what); + for (size_t i = 0; i < got.size(); ++i) printf("%s%d", i ? ", " : "", got[i]); + printf("}, expected {"); + for (size_t i = 0; i < want.size(); ++i) printf("%s%d", i ? ", " : "", want[i]); + printf("}\n"); + return false; +} + +int main() { + Node n[6] = {{1, {}}, {2, {}}, {3, {}}, {4, {}}, {5, {}}, {6, {}}}; + List list; + bool ok = true; + + // The simulator's timing wheel delivers same-cycle events in push_back order. + list.push_back(&n[0]); + list.push_back(&n[1]); + list.push_back(&n[2]); + ok &= expect(list, {1, 2, 3}, "push_back"); + + list.push_front(&n[3]); + ok &= expect(list, {4, 1, 2, 3}, "push_front"); + + list.insert(++list.begin(), &n[4]); + ok &= expect(list, {4, 5, 1, 2, 3}, "insert before position"); + + list.erase(list.begin()); + list.pop_back(); + ok &= expect(list, {5, 1, 2}, "erase/pop_back"); + + list.push_back(&n[5]); + ok &= expect(list, {5, 1, 2, 6}, "push_back after erase"); + + printf("%s\n", ok ? "PASSED!" : "FAILED!"); + return ok ? 0 : 1; +} From cbd9cbee8c35d87fdeb017ac6f1de9f173ce343a Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 7 Oct 2026 18:51:08 -0700 Subject: [PATCH 002/336] sw, tests: make device index math portable to RV64 On RV64 an index computed in uint32_t wraps at 32 bits, so LLVM cannot widen it into a 64-bit induction variable or fold it into a pointer increment, and re-emits zero-extend + shift + add for every access (sgemm's unrolled body: 90 instructions on RV64 vs 35 on RV32). Indices that feed pointer arithmetic are now size_t, which on RV32 is the same type as uint32_t, so RV32 kernels compile to identical code; vm_stress uses int, where size_t moved the loop exit onto a divergent per-thread pointer compare. RV64 correctness: - vx_dxa.h: keep the wgather result (lane 0 carries the shared-memory address) at register width instead of truncating it to 32 bits. - vx_graphics.h: compute the OM aperture address in uintptr_t and pass it to the export at register width; on RV32 the old expression was a 64-bit sum whose dead carry cost an instruction per fragment. - vx_intrinsics.h, vx_tensor.h, packld_box: pack-load strides and ldm steps are size_t; the address unit reads the full register. - vx_serial.S: keep sp 16-byte aligned across the callback (frames 32/64 bytes on RV32/RV64); select the XLEN path with __riscv_xlen, as LOAD_IMMEDIATE64 in src/common.h now does. - vx_start.S: __dso_handle is pointer-sized on RV64. - vx_syscalls.c: use newlib's prototypes for _lseek/_read/_write/_sbrk. - dotproduct2: don't pass uninitialized upper bytes through the shuffle. Kernels whose RV64 cycles got worse with a wider index are left as they were: sgemv, demo, probe, fence, dogfood, dotproduct, softmax, vm_test, mpi_dotproduct, mpi_put_dotproduct, sgemm_tcu, sgemm_tcu_mx, sgemm_tcu_sp_mx and sgemm_tcu_wg. In most of them LLVM's loop strength reduction moves the loop exit onto a per-thread pointer compare, which needs predication every iteration. RV64 simx cycles: sgemm -55%, mstress -28%, sgemm_tcu_wg_mx -22%, sgemm2_dxa -21%, stencil3d -20%, sgemm_v1 -18%, sgemm2_v1 -16%, dxa_copy_mcast -16%, sgemm2 -12%, sgemm2_dxa_mcast -12%, jacobi -10%, sgemm2_tcu -6%; no measured kernel is slower (mpi_sgemm, mpi_blocked_sgemm and the graphics pipeline kernels print no cycle counts and were checked for correctness only). On RV32 only the OM export path and vx_serial's frame change; the affected graphics perf-gate baselines are regenerated with --update-baselines (om -1.3%/-1.9%, draw3d -0.4% cycles on rtlsim). Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/graphics.json | 20 +-- sw/common/gfx_sw.h | 4 +- sw/gfx/gfx_frontend_k.h | 28 ++-- sw/kernel/include/vx_dxa.h | 144 +++++++++--------- sw/kernel/include/vx_graphics.h | 7 +- sw/kernel/include/vx_intrinsics.h | 4 +- sw/kernel/include/vx_tensor.h | 14 +- sw/kernel/src/common.h | 2 +- sw/kernel/src/vx_serial.S | 12 +- sw/kernel/src/vx_start.S | 6 + sw/kernel/src/vx_syscalls.c | 15 +- tests/mpi/mpi_blocked_sgemm/kernel.cpp | 4 +- tests/mpi/mpi_sgemm/kernel.cpp | 4 +- tests/regression/async_barrier/kernel.cpp | 14 +- tests/regression/basic/kernel.cpp | 6 +- tests/regression/dotproduct2/kernel.cpp | 4 +- tests/regression/dxa_copy_mcast/kernel.cpp | 12 +- tests/regression/dxa_kmajor_check/kernel.cpp | 6 +- .../regression/gfx_binsort_kernel/kernel.cpp | 14 +- .../regression/gfx_pipeline_kernel/kernel.cpp | 40 ++--- tests/regression/gfx_setup_kernel/kernel.cpp | 18 +-- tests/regression/jacobi/kernel.cpp | 8 +- tests/regression/module_reload/kernel.cpp | 6 +- tests/regression/mstress/kernel.cpp | 10 +- tests/regression/packld/kernel.cpp | 2 +- tests/regression/packld_box/kernel.cpp | 8 +- tests/regression/raycast/render.h | 6 +- tests/regression/reopen/kernel.cpp | 6 +- tests/regression/sgemm/kernel.cpp | 4 +- tests/regression/sgemm2/kernel.cpp | 22 +-- tests/regression/sgemm2_dxa/kernel.cpp | 30 ++-- tests/regression/sgemm2_dxa_mcast/kernel.cpp | 14 +- tests/regression/sgemm2_tcu/kernel.cpp | 26 ++-- tests/regression/sgemm2_v1/kernel.cpp | 22 +-- tests/regression/sgemm_tcu_wg_dxa/kernel.cpp | 8 +- tests/regression/sgemm_tcu_wg_mx/kernel.cpp | 22 +-- tests/regression/sgemm_tcu_wg_sp/kernel.cpp | 24 +-- tests/regression/sgemm_v1/kernel.cpp | 4 +- tests/regression/stencil3d/kernel.cpp | 2 +- tests/regression/vm_stress/kernel.cpp | 4 +- 40 files changed, 307 insertions(+), 299 deletions(-) diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 39103db55e..3ca43a64f7 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,8 +1,8 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 531658, - "instrs": 204102 + "cycles": 529553, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", @@ -12,8 +12,8 @@ }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 148046, - "instrs": 51475 + "cycles": 147521, + "instrs": 51232 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", @@ -23,8 +23,8 @@ }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 531210, - "instrs": 204102 + "cycles": 529126, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", @@ -34,8 +34,8 @@ }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 192077, - "instrs": 69648 + "cycles": 189486, + "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", @@ -45,8 +45,8 @@ }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 755028, - "instrs": 278544 + "cycles": 740760, + "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", diff --git a/sw/common/gfx_sw.h b/sw/common/gfx_sw.h index 503b9721f9..96405a48d8 100644 --- a/sw/common/gfx_sw.h +++ b/sw/common/gfx_sw.h @@ -399,11 +399,11 @@ static inline __attribute__((always_inline)) bool om_sample_rmw( // stride gather. static inline __attribute__((always_inline)) uintptr_t msaa_color_addr( const om_state_t& s, uint32_t samples, uint32_t x, uint32_t y, uint32_t sample) { - return (uintptr_t)(s.cbuf_base + (uint64_t)y * s.cbuf_pitch + (uint64_t)(x * samples + sample) * om_color_bpp(s.color_format)); + return (uintptr_t)(s.cbuf_base + (uint64_t)y * s.cbuf_pitch + (uint64_t)((uintptr_t)x * samples + sample) * om_color_bpp(s.color_format)); } static inline __attribute__((always_inline)) uintptr_t msaa_depth_addr( const om_state_t& s, uint32_t samples, uint32_t x, uint32_t y, uint32_t sample) { - return (uintptr_t)(s.zbuf_base + (uint64_t)y * s.zbuf_pitch + (uint64_t)(x * samples + sample) * om_depth_bpp(s.depth_format)); + return (uintptr_t)(s.zbuf_base + (uint64_t)y * s.zbuf_pitch + (uint64_t)((uintptr_t)x * samples + sample) * om_depth_bpp(s.depth_format)); } // Box resolve of one pixel's S color samples → a single ARGB8888 value diff --git a/sw/gfx/gfx_frontend_k.h b/sw/gfx/gfx_frontend_k.h index 9e557ab7e3..e2ee763c97 100644 --- a/sw/gfx/gfx_frontend_k.h +++ b/sw/gfx/gfx_frontend_k.h @@ -85,8 +85,8 @@ __kernel void expand_k(expand_arg_t* __UNIFORM__ arg) { const uint32_t nv = arg->num_varyings; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t i = gid; i < n; i += gstride) { - const uint8_t* rec = recs + (size_t)i * stride; + for (size_t i = gid; i < n; i += gstride) { + const uint8_t* rec = recs + i * stride; const float* pos = reinterpret_cast(rec); setup_vertex_t v; v.pos[0] = pos[0]; v.pos[1] = pos[1]; v.pos[2] = pos[2]; v.pos[3] = pos[3]; @@ -156,7 +156,7 @@ __kernel void setup_k(pipe_arg_t* __UNIFORM__ arg) { // traffic and removes the slot_prim/slot_bbox scratch entirely. uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { + for (size_t t = gid; t < ntri; t += gstride) { rast_prim_t pr[SETUP_MAX_SUB]; setup_bbox_t bb[SETUP_MAX_SUB]; keep[t] = pipe_clip_and_setup(&verts[3 * t], W, H, arg->cull_mode, vpp, pr, bb); @@ -183,7 +183,7 @@ __kernel void setup_k(pipe_arg_t* __UNIFORM__ arg) { // here equals keep[t] from SETUP (same pure function, same inputs). uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { + for (size_t t = gid; t < ntri; t += gstride) { rast_prim_t pr[SETUP_MAX_SUB]; setup_bbox_t bb[SETUP_MAX_SUB]; uint32_t kept = pipe_clip_and_setup(&verts[3 * t], W, H, arg->cull_mode, vpp, pr, bb); @@ -210,12 +210,12 @@ __kernel void setup_k(pipe_arg_t* __UNIFORM__ arg) { for (uint32_t c = 0; c < 4; ++c) { fw[2 + c] = pv[4 + c]; } for (uint32_t c = 0; c < 6; ++c) { fw[6 + c] = pv[10 + c]; } } - uint32_t w = offset[t]; + size_t w = offset[t]; for (uint32_t s = 0; s < kept; ++s) { prim[w] = pr[s]; bbox[w] = bb[s]; if (flat) { - uint32_t* dst = flat + (size_t)w * GFX_FS_FLAT_WORDS; + uint32_t* dst = flat + w * GFX_FS_FLAT_WORDS; for (uint32_t c = 0; c < GFX_FS_FLAT_WORDS; ++c) { dst[c] = fw[c]; } } ++w; @@ -250,7 +250,7 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; pipe_bin_range(bbox[i], bL, bR, bT, bB); bcount[i] = (uint32_t)((bR - bL + 1) * (bB - bT + 1)); } @@ -275,9 +275,9 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; pipe_bin_range(bbox[i], bL, bR, bT, bB); - uint32_t w = boffset[i]; + size_t w = boffset[i]; for (int by = bT; by <= bB; ++by) for (int bx = bL; bx <= bR; ++bx) keys[w++] = ((uint32_t)(by * arg->bin_cols + bx) << PIPE_PRIM_BITS) | i; @@ -289,11 +289,11 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { uint32_t K = meta[1]; uint32_t kchunk = (K + T - 1) / T; uint32_t klo = pipe_umin(tid * kchunk, K), khi = pipe_umin(klo + kchunk, K); - for (uint32_t b = lo; b < hi; ++b) thist[tid * B + b] = 0; + for (uint32_t b = lo; b < hi; ++b) thist[(size_t)tid * B + b] = 0; __syncthreads(); - for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> PIPE_PRIM_BITS; if (b >= lo && b < hi) thist[tid * B + b]++; } + for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> PIPE_PRIM_BITS; if (b >= lo && b < hi) thist[(size_t)tid * B + b]++; } __syncthreads(); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[(size_t)t * B + b]; bincount[b] = s; } } break; case PIPE_STAGE_BBASE: { @@ -341,11 +341,11 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { auto pids = reinterpret_cast(tilebuf + B * sizeof(rast_bin_header_t)); uint32_t kchunk = (K + T - 1) / T; uint32_t klo = pipe_umin(tid * kchunk, K), khi = pipe_umin(klo + kchunk, K); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[(size_t)t * B + b]; thist[(size_t)t * B + b] = run; run += c; } } __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t key = keys[k], b = key >> PIPE_PRIM_BITS; - if (b >= lo && b < hi) pids[thist[tid * B + b]++] = key & PIPE_PRIM_MASK; + if (b >= lo && b < hi) pids[thist[(size_t)tid * B + b]++] = key & PIPE_PRIM_MASK; } } break; diff --git a/sw/kernel/include/vx_dxa.h b/sw/kernel/include/vx_dxa.h index dbbc031e37..79babb18b5 100644 --- a/sw/kernel/include/vx_dxa.h +++ b/sw/kernel/include/vx_dxa.h @@ -55,10 +55,10 @@ inline void vx_dxa_issue_1d_wg(uint32_t desc_slot, const void* smem_addr, uint32_t coord0) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)0u); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, x0\n\t" : @@ -73,10 +73,10 @@ inline void vx_dxa_issue_2d_wg(uint32_t desc_slot, uint32_t coord0, uint32_t coord1) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, x0\n\t" : @@ -92,14 +92,14 @@ inline void vx_dxa_issue_3d_wg(uint32_t desc_slot, uint32_t coord1, uint32_t coord2) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)0u, - (size_t)0u, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)0u, + (size_t)0u, + (size_t)0u); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -115,14 +115,14 @@ inline void vx_dxa_issue_4d_wg(uint32_t desc_slot, uint32_t coord2, uint32_t coord3) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)coord3, - (size_t)0u, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)coord3, + (size_t)0u, + (size_t)0u); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -139,14 +139,14 @@ inline void vx_dxa_issue_5d_wg(uint32_t desc_slot, uint32_t coord3, uint32_t coord4) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)coord3, - (size_t)coord4, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)coord3, + (size_t)coord4, + (size_t)0u); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -164,14 +164,14 @@ inline void vx_dxa_issue_1d_multicast_wg(uint32_t desc_slot, uint32_t coord0, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)0u); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)0, - (size_t)0, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)0u); + const size_t a1 = vx_wgather((size_t)0, + (size_t)0, + (size_t)0, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -187,14 +187,14 @@ inline void vx_dxa_issue_2d_multicast_wg(uint32_t desc_slot, uint32_t coord1, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)0, - (size_t)0, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)0, + (size_t)0, + (size_t)0, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -211,14 +211,14 @@ inline void vx_dxa_issue_3d_multicast_wg(uint32_t desc_slot, uint32_t coord2, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)0, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)0, + (size_t)0, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -236,14 +236,14 @@ inline void vx_dxa_issue_4d_multicast_wg(uint32_t desc_slot, uint32_t coord3, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)coord3, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)coord3, + (size_t)0, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -262,14 +262,14 @@ inline void vx_dxa_issue_5d_multicast_wg(uint32_t desc_slot, uint32_t coord4, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)coord3, - (size_t)coord4, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)coord3, + (size_t)coord4, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : diff --git a/sw/kernel/include/vx_graphics.h b/sw/kernel/include/vx_graphics.h index 0eb7b094a6..67664edf5b 100644 --- a/sw/kernel/include/vx_graphics.h +++ b/sw/kernel/include/vx_graphics.h @@ -68,8 +68,9 @@ inline unsigned vx_tex(unsigned stage, unsigned u, unsigned v, unsigned lod) { // XBITS/YBITS/RECORD_SHIFT come from the OM DCRs; the runtime programs them and // passes them to the kernel, so the shader just shifts and adds. #define VX_OM_APERTURE_ADDR_RT(xbits, ybits, record_shift, x, y, face, rt) \ - ((VX_MEM_OM_BASE_ADDR) + \ - (((((uint32_t)(rt) << 1 | (uint32_t)(face)) << ((xbits) + (ybits))) \ + ((uintptr_t)(VX_MEM_OM_BASE_ADDR) + \ + (uintptr_t)(((((uint32_t)(rt) << 1 | (uint32_t)(face)) \ + << ((xbits) + (ybits))) \ | ((uint32_t)(y) << (xbits)) \ | (uint32_t)(x)) << (record_shift))) @@ -84,7 +85,7 @@ inline unsigned vx_tex(unsigned stage, unsigned u, unsigned v, unsigned lod) { // into one ordinary store per word; the LSU never learns that OM exists. #define vx_om_export(addr, color, depth, mask) \ __asm__ volatile (".insn r4 %0, 3, %1, x0, %2, %3, %4" \ - :: "i"(RISCV_CUSTOM1), "i"(mask), "r"(addr), "r"(color), "r"(depth)) + :: "i"(RISCV_CUSTOM1), "i"(mask), "r"((uintptr_t)(addr)), "r"(color), "r"(depth)) // The three record shapes. #define vx_om_export_color(addr, color) vx_om_export(addr, color, 0, 1) diff --git a/sw/kernel/include/vx_intrinsics.h b/sw/kernel/include/vx_intrinsics.h index 11ee4081a8..255a6ec392 100644 --- a/sw/kernel/include/vx_intrinsics.h +++ b/sw/kernel/include/vx_intrinsics.h @@ -602,7 +602,7 @@ inline void vx_barrier_expect_tx(int barrier_id, int count) { // | (MEM8[base + 2 * stride] << 16) // | (MEM8[base + 3 * stride] << 24)); __attribute__((always_inline)) -inline float vx_packlb_f(const void* base, uint32_t stride) { +inline float vx_packlb_f(const void* base, size_t stride) { float out; __asm__ volatile ( ".insn r %1, 1, 4, %0, %2, %3" : "=f"(out) : "i"(RISCV_CUSTOM0), "r"(base), "r"(stride) : "memory" @@ -616,7 +616,7 @@ inline float vx_packlb_f(const void* base, uint32_t stride) { // fd = bitcast((MEM16[base + 0 * stride] << 0) // | (MEM16[base + 1 * stride] << 16)); __attribute__((always_inline)) -inline float vx_packlh_f(const void* base, uint32_t stride) { +inline float vx_packlh_f(const void* base, size_t stride) { float out; __asm__ volatile ( ".insn r %1, 2, 4, %0, %2, %3" : "=f"(out) : "i"(RISCV_CUSTOM0), "r"(base), "r"(stride) : "memory" diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index 4106404b6b..618a95b030 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -87,7 +87,7 @@ namespace detail { } } - static __attribute__((always_inline)) D pack_row(const Type *base, uint32_t ldm) { + static __attribute__((always_inline)) D pack_row(const Type *base, size_t ldm) { static_assert(sizeof(D) % sizeof(Type) == 0, "D must be a multiple of Type in size"); if constexpr (sizeof(Type) == 1 && sizeof(D) == 4) { // 4 × 1-byte strided loads → single pack-load byte instruction @@ -368,7 +368,7 @@ struct wmma_context { k_bases[0] = base; if constexpr (num_k_groups >= 2) { asm volatile("" : "+r"(k_bases[0])); // prevent reverse strength reduction - auto k_ldm_step = k_stride * (uint32_t)ldm; + size_t k_ldm_step = k_stride * ldm; detail::unroll_for([&](auto i) { k_bases[i + 1] = k_bases[i] + k_ldm_step; }); @@ -393,7 +393,7 @@ struct wmma_context { n_bases[0] = base; if constexpr (num_n_groups >= 2) { asm volatile("" : "+r"(n_bases[0])); // prevent reverse strength reduction - auto n_ldm_step = n_stride * (uint32_t)ldm; + size_t n_ldm_step = n_stride * ldm; detail::unroll_for([&](auto i) { n_bases[i + 1] = n_bases[i] + n_ldm_step; }); @@ -864,7 +864,7 @@ struct wgmma_context { uint32_t k_in_elem = (lane % tcK) * i_ratio; auto load_reg = [&](uint32_t m_blk, uint32_t k_blk) { - uint32_t elem_off; + size_t elem_off; if (ldm == 0) { // Block-major SMEM: blocks contiguous, k-block outer. constexpr uint32_t block_elems = is_sparse ? sparse_blk_elems : a_blk_elems; @@ -873,7 +873,7 @@ struct wgmma_context { + k_in_elem; } else { // Row-major SMEM: standard row*ldm + col. - elem_off = (m_blk * tcM + i_in) * uint32_t(ldm) + elem_off = (m_blk * tcM + i_in) * ldm + (k_blk * k_row_elems + k_in_elem); } return *reinterpret_cast( @@ -893,14 +893,14 @@ struct wgmma_context { uint32_t m_blk = r / a_reg_k_steps; uint32_t k_blk = r % a_reg_k_steps; if constexpr (k_steps == 1) { - uint32_t elem_off; + size_t elem_off; if (ldm == 0) { elem_off = m_blk * a_blk_elems + i_in * (fedpK * i_ratio) + k_blk * k_row_elems + k_in_elem; } else { - elem_off = (m_blk * tcM + i_in) * uint32_t(ldm) + elem_off = (m_blk * tcM + i_in) * ldm + (k_blk * k_row_elems + k_in_elem); } dst.data[r] = *reinterpret_cast( diff --git a/sw/kernel/src/common.h b/sw/kernel/src/common.h index af05d63fe2..6256141855 100644 --- a/sw/kernel/src/common.h +++ b/sw/kernel/src/common.h @@ -13,7 +13,7 @@ #define RISCV_CUSTOM0 0x0B -#ifdef VX_CFG_XLEN_64 +#if __riscv_xlen == 64 #define LOAD_IMMEDIATE64(rd, imm) \ li t0, (imm >> 32); \ slli t0, t0, 32; \ diff --git a/sw/kernel/src/vx_serial.S b/sw/kernel/src/vx_serial.S index 6d6d8ba827..0cea0704a7 100644 --- a/sw/kernel/src/vx_serial.S +++ b/sw/kernel/src/vx_serial.S @@ -18,8 +18,8 @@ .type vx_serial, @function .global vx_serial vx_serial: -#if (VX_CFG_XLEN == 64) - addi sp, sp, -56 +#if __riscv_xlen == 64 + addi sp, sp, -64 sd ra, 48(sp) sd s5, 40(sp) sd s4, 32(sp) @@ -28,7 +28,7 @@ vx_serial: sd s1, 8(sp) sd s0, 0(sp) #else - addi sp, sp, -28 + addi sp, sp, -32 sw ra, 24(sp) sw s5, 20(sp) sw s4, 16(sp) @@ -53,7 +53,7 @@ label_join: .insn r RISCV_CUSTOM0, 3, 0, x0, s5, x0 # join s5 addi s0, s0, 1 # index++ blt s0, s2, label_loop # loop back -#if (VX_CFG_XLEN == 64) +#if __riscv_xlen == 64 ld ra, 48(sp) ld s5, 40(sp) ld s4, 32(sp) @@ -61,7 +61,7 @@ label_join: ld s2, 16(sp) ld s1, 8(sp) ld s0, 0(sp) - addi sp, sp, 56 + addi sp, sp, 64 #else lw ra, 24(sp) lw s5, 20(sp) @@ -70,6 +70,6 @@ label_join: lw s2, 8(sp) lw s1, 4(sp) lw s0, 0(sp) - addi sp, sp, 28 + addi sp, sp, 32 #endif ret diff --git a/sw/kernel/src/vx_start.S b/sw/kernel/src/vx_start.S index 60d47d8125..5467253caa 100644 --- a/sw/kernel/src/vx_start.S +++ b/sw/kernel/src/vx_start.S @@ -261,5 +261,11 @@ init_tls_all: .section .data .weak __dso_handle +#if __riscv_xlen == 64 + .p2align 3 +__dso_handle: + .dword 0 +#else __dso_handle: .long 0 +#endif diff --git a/sw/kernel/src/vx_syscalls.c b/sw/kernel/src/vx_syscalls.c index b96ff6f17f..bef330323e 100644 --- a/sw/kernel/src/vx_syscalls.c +++ b/sw/kernel/src/vx_syscalls.c @@ -17,6 +17,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -28,21 +29,21 @@ int _fstat(int file, struct stat *st) { return -1; } int _isatty(int file) { return 0; } -int _lseek(int file, int ptr, int dir) { return 0; } +off_t _lseek(int file, off_t ptr, int dir) { return 0; } int _open(const char *name, int flags, int mode) { return -1; } -int _read(int file, char *ptr, int len) { return -1; } +ssize_t _read(int file, void *ptr, size_t len) { return -1; } -caddr_t _sbrk(int incr) { +void *_sbrk(ptrdiff_t incr) { __asm__ __volatile__("ebreak"); return 0; } -int _write(int file, char *ptr, int len) { - int i; - for (i = 0; i < len; ++i) { - vx_putchar(*ptr++); +ssize_t _write(int file, const void *ptr, size_t len) { + const char *p = (const char *)ptr; + for (size_t i = 0; i < len; ++i) { + vx_putchar(*p++); } return len; } diff --git a/tests/mpi/mpi_blocked_sgemm/kernel.cpp b/tests/mpi/mpi_blocked_sgemm/kernel.cpp index 4634bae635..001651da5f 100644 --- a/tests/mpi/mpi_blocked_sgemm/kernel.cpp +++ b/tests/mpi/mpi_blocked_sgemm/kernel.cpp @@ -5,13 +5,13 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x; int row = blockIdx.y; TYPE sum(0); - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/mpi/mpi_sgemm/kernel.cpp b/tests/mpi/mpi_sgemm/kernel.cpp index 4634bae635..001651da5f 100644 --- a/tests/mpi/mpi_sgemm/kernel.cpp +++ b/tests/mpi/mpi_sgemm/kernel.cpp @@ -5,13 +5,13 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x; int row = blockIdx.y; TYPE sum(0); - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/regression/async_barrier/kernel.cpp b/tests/regression/async_barrier/kernel.cpp index 8fc25a4b61..0a1b3aa082 100644 --- a/tests/regression/async_barrier/kernel.cpp +++ b/tests/regression/async_barrier/kernel.cpp @@ -24,11 +24,11 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto local_B0 = local_A0 + blockDim.x * blockDim.y; auto local_u32 = reinterpret_cast(local_B0 + blockDim.x * blockDim.y); - auto size = arg->size; - auto tile_size = arg->tile_size; + size_t size = arg->size; + size_t tile_size = arg->tile_size; - auto g_row = blockIdx.x * blockDim.x + threadIdx.x; - auto g_col = blockIdx.y * blockDim.y + threadIdx.y; + size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; + size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; auto l_row = threadIdx.x; auto l_col = threadIdx.y; @@ -40,7 +40,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t token_compute; - for (uint32_t k = 0; k < size; k += tile_size) { + for (size_t k = 0; k < size; k += tile_size) { if (k > 0) { bar1.wait(token_compute); } @@ -52,7 +52,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { TYPE my_val = local_A0[l_row * tile_size + l_col]; TYPE my_val_b = local_B0[l_row * tile_size + l_col]; - for (uint32_t iter = 0; iter < tile_size; ++iter) { + for (size_t iter = 0; iter < tile_size; ++iter) { my_val = sqrt(my_val * my_val + 0.01f); my_val_b = sqrt(my_val_b * my_val_b + 0.01f); } @@ -60,7 +60,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { bar1.wait(token_load); - for (uint32_t j = 0; j < tile_size; ++j) { + for (size_t j = 0; j < tile_size; ++j) { sum += local_A0[l_row * tile_size + j] * local_B0[j * tile_size + l_col]; } token_compute = bar1.arrive(); diff --git a/tests/regression/basic/kernel.cpp b/tests/regression/basic/kernel.cpp index 5e8f51a124..31eb93f18d 100644 --- a/tests/regression/basic/kernel.cpp +++ b/tests/regression/basic/kernel.cpp @@ -3,13 +3,13 @@ int main() { kernel_arg_t* __UNIFORM__ arg = (kernel_arg_t*)csr_read(VX_CSR_MSCRATCH); - uint32_t count = arg->count; + size_t count = arg->count; int32_t* src_ptr = (int32_t*)arg->src_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t offset = vx_core_id() * count; + size_t offset = vx_core_id() * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset + i] = src_ptr[offset + i]; } diff --git a/tests/regression/dotproduct2/kernel.cpp b/tests/regression/dotproduct2/kernel.cpp index c10a20c2bf..bf7ead6823 100644 --- a/tests/regression/dotproduct2/kernel.cpp +++ b/tests/regression/dotproduct2/kernel.cpp @@ -5,8 +5,8 @@ #define VX_CFG_NUM_THREADS 4 #endif -static inline size_t bits_from_f32(float x) { size_t u; __builtin_memcpy(&u, &x, 4); return u; } -static inline float f32_from_bits(size_t u) { float x; __builtin_memcpy(&x, &u, 4); return x; } +static inline size_t bits_from_f32(float x) { uint32_t u; __builtin_memcpy(&u, &x, 4); return u; } +static inline float f32_from_bits(size_t u) { uint32_t b = u; float x; __builtin_memcpy(&x, &b, 4); return x; } static inline float warp_reduce_sum(float x, uint32_t warp_size) { int clamp = warp_size - 1; diff --git a/tests/regression/dxa_copy_mcast/kernel.cpp b/tests/regression/dxa_copy_mcast/kernel.cpp index d56b2558c5..c25a792719 100644 --- a/tests/regression/dxa_copy_mcast/kernel.cpp +++ b/tests/regression/dxa_copy_mcast/kernel.cpp @@ -14,9 +14,9 @@ constexpr uint32_t kDescSrc = 0; __kernel void kernel_main(kernel_arg_t* arg) { - const uint32_t cta_id = get_local_group_id(); - const uint32_t tile_rows = arg->tile_rows; - const uint32_t tile_cols = arg->tile_cols; + const size_t cta_id = get_local_group_id(); + const size_t tile_rows = arg->tile_rows; + const size_t tile_cols = arg->tile_cols; const uint32_t num_recv = arg->num_recv; auto shmem = reinterpret_cast(__local_mem()); @@ -42,9 +42,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Store the received tile to dst region for host verification. // Layout: dst[cta_id * tile_elems + r * tile_cols + c] auto dst = reinterpret_cast(arg->dst_addr); - const uint32_t tile_elems = tile_rows * tile_cols; - const uint32_t l_col = threadIdx.x; - for (uint32_t r = 0; r < tile_rows; ++r) { + const size_t tile_elems = tile_rows * tile_cols; + const size_t l_col = threadIdx.x; + for (size_t r = 0; r < tile_rows; ++r) { if (l_col < tile_cols) { dst[cta_id * tile_elems + r * tile_cols + l_col] = shmem[r * tile_cols + l_col]; diff --git a/tests/regression/dxa_kmajor_check/kernel.cpp b/tests/regression/dxa_kmajor_check/kernel.cpp index de381140da..d1bcc86f8c 100644 --- a/tests/regression/dxa_kmajor_check/kernel.cpp +++ b/tests/regression/dxa_kmajor_check/kernel.cpp @@ -14,7 +14,7 @@ constexpr uint32_t kDescA = 0; constexpr uint32_t kDescB = 1; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - const uint32_t tid = threadIdx.x; + const size_t tid = threadIdx.x; const uint32_t cta_id = blockIdx.x; // 1D grid — one CTA per (M-tile, N-tile) pair const uint32_t grid_n = arg->N / arg->tileN; const uint32_t cta_row = cta_id / grid_n; @@ -37,8 +37,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // Copy both SMEM regions to dst[cta_id * cta_bytes ..] auto pDst = reinterpret_cast(arg->dst_addr) + cta_id * arg->cta_bytes; - const uint32_t total = arg->cta_bytes; - for (uint32_t i = tid; i < total; i += blockDim.x) { + const size_t total = arg->cta_bytes; + for (size_t i = tid; i < total; i += blockDim.x) { pDst[i] = A_smem[i]; // A_smem is followed by B_smem in memory } __syncthreads(); diff --git a/tests/regression/gfx_binsort_kernel/kernel.cpp b/tests/regression/gfx_binsort_kernel/kernel.cpp index 53e11bdc03..0791bdb204 100644 --- a/tests/regression/gfx_binsort_kernel/kernel.cpp +++ b/tests/regression/gfx_binsort_kernel/kernel.cpp @@ -34,9 +34,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto meta = reinterpret_cast(arg->meta_addr); const uint32_t n = arg->num_prims; - const uint32_t tid = threadIdx.x; - const uint32_t T = blockDim.x; - const uint32_t B = BINSORT_NUM_BINS; + const size_t tid = threadIdx.x; + const size_t T = blockDim.x; + const size_t B = BINSORT_NUM_BINS; switch (arg->stage) { @@ -69,7 +69,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { __syncthreads(); if (tid == 0) { uint32_t acc = 0; - for (uint32_t t = 0; t < T; ++t) { uint32_t v = tsum[t]; tsum[t] = acc; acc += v; } + for (size_t t = 0; t < T; ++t) { uint32_t v = tsum[t]; tsum[t] = acc; acc += v; } meta[0] = acc; offset[n] = acc; } __syncthreads(); @@ -82,7 +82,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { for (uint32_t i = lo; i < hi; ++i) { if (count[i] == 0) continue; // culled / empty bbox int bL, bR, bT, bB; bin_range(prims[i], bL, bR, bT, bB); - uint32_t w = offset[i]; + size_t w = offset[i]; for (int by = bT; by <= bB; ++by) for (int bx = bL; bx <= bR; ++bx) keys[w++] = ((uint32_t)(by * BINSORT_BIN_COLS + bx) << BINSORT_PRIM_BITS) | i; @@ -98,7 +98,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> BINSORT_PRIM_BITS; if (b >= lo && b < hi) thist[tid * B + b]++; } __syncthreads(); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (size_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } } break; case BINSORT_STAGE_BASE: { // single CTA: bin-base scan + headers @@ -123,7 +123,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t kchunk = (P + T - 1) / T; uint32_t klo = umin(tid * kchunk, P), khi = umin(klo + kchunk, P); // owned hist columns -> stable write cursors (base + thread-exclusive prefix) - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (size_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t key = keys[k], b = key >> BINSORT_PRIM_BITS; diff --git a/tests/regression/gfx_pipeline_kernel/kernel.cpp b/tests/regression/gfx_pipeline_kernel/kernel.cpp index 765cfc8b16..1d17eb3703 100644 --- a/tests/regression/gfx_pipeline_kernel/kernel.cpp +++ b/tests/regression/gfx_pipeline_kernel/kernel.cpp @@ -62,9 +62,9 @@ __kernel void setup_k(kernel_arg_t* __UNIFORM__ arg) { switch (arg->stage) { case PIPE_STAGE_SETUP: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < ntri; t += gstride) { rast_prim_t pr[SETUP_MAX_SUB]; setup_bbox_t bb[SETUP_MAX_SUB]; uint32_t kept = clip_and_setup(&verts[3 * t], W, H, pr, bb); @@ -91,12 +91,12 @@ __kernel void setup_k(kernel_arg_t* __UNIFORM__ arg) { } break; case PIPE_STAGE_EMIT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { - uint32_t w = offset[t]; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < ntri; t += gstride) { + size_t w = offset[t]; for (uint32_t s = 0; s < keep[t]; ++s) { - uint32_t slot = t * SETUP_MAX_SUB + s; + size_t slot = t * SETUP_MAX_SUB + s; prim[w] = slot_prim[slot]; bbox[w] = slot_bbox[slot]; ++w; @@ -120,17 +120,17 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { auto tilebuf = reinterpret_cast(arg->tilebuf_addr); auto meta = reinterpret_cast(arg->meta_addr); - const uint32_t tid = threadIdx.x; - const uint32_t T = blockDim.x; - const uint32_t B = PIPE_NUM_BINS; + const size_t tid = threadIdx.x; + const size_t T = blockDim.x; + const size_t B = PIPE_NUM_BINS; switch (arg->stage) { case PIPE_STAGE_BCOUNT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; bin_range(bbox[i], bL, bR, bT, bB); bcount[i] = (uint32_t)((bR - bL + 1) * (bB - bT + 1)); } @@ -152,12 +152,12 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { } break; case PIPE_STAGE_BEMIT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; bin_range(bbox[i], bL, bR, bT, bB); - uint32_t w = boffset[i]; + size_t w = boffset[i]; for (int by = bT; by <= bB; ++by) for (int bx = bL; bx <= bR; ++bx) keys[w++] = ((uint32_t)(by * PIPE_BIN_COLS + bx) << PIPE_PRIM_BITS) | i; @@ -173,7 +173,7 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> PIPE_PRIM_BITS; if (b >= lo && b < hi) thist[tid * B + b]++; } __syncthreads(); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (size_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } } break; case PIPE_STAGE_BBASE: { // single CTA: tilebuf headers @@ -205,7 +205,7 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { auto pids = reinterpret_cast(tilebuf + nb * sizeof(rast_tile_header_t)); uint32_t kchunk = (K + T - 1) / T; uint32_t klo = umin(tid * kchunk, K), khi = umin(klo + kchunk, K); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (size_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t key = keys[k], b = key >> PIPE_PRIM_BITS; diff --git a/tests/regression/gfx_setup_kernel/kernel.cpp b/tests/regression/gfx_setup_kernel/kernel.cpp index 657311e37a..d20c43dd83 100644 --- a/tests/regression/gfx_setup_kernel/kernel.cpp +++ b/tests/regression/gfx_setup_kernel/kernel.cpp @@ -38,9 +38,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { switch (arg->stage) { case SETUP_STAGE_SETUP: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < n; t += gstride) { + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < n; t += gstride) { clip_tri_t sub[SETUP_MAX_SUB]; int ns = clip_near(verts[3 * t + 0], verts[3 * t + 1], verts[3 * t + 2], sub); uint32_t kept = 0; @@ -49,7 +49,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { setup_bbox_t bbox{}; if (setup_triangle(sub[s].v[0], sub[s].v[1], sub[s].v[2], W, H, SETUP_NEAR, SETUP_FAR, prim, bbox, arg->cull_mode)) { - uint32_t slot = t * SETUP_MAX_SUB + kept; + size_t slot = t * SETUP_MAX_SUB + kept; slot_prim[slot] = prim; slot_bbox[slot] = bbox; slot_vtx[slot] = sub[s]; @@ -75,12 +75,12 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { } break; case SETUP_STAGE_EMIT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < n; t += gstride) { - uint32_t w = offset[t]; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < n; t += gstride) { + size_t w = offset[t]; for (uint32_t s = 0; s < keep[t]; ++s) { - uint32_t slot = t * SETUP_MAX_SUB + s; + size_t slot = t * SETUP_MAX_SUB + s; out_prim[w] = slot_prim[slot]; out_bbox[w] = slot_bbox[slot]; out_vtx[w] = slot_vtx[slot]; diff --git a/tests/regression/jacobi/kernel.cpp b/tests/regression/jacobi/kernel.cpp index 5b16c63cf9..9e10782d50 100644 --- a/tests/regression/jacobi/kernel.cpp +++ b/tests/regression/jacobi/kernel.cpp @@ -6,15 +6,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto x_old = reinterpret_cast(arg->src1_addr); auto b = reinterpret_cast(arg->src2_addr); auto x_new = reinterpret_cast(arg->dst_addr); - auto n = arg->size; + size_t n = arg->size; - uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + size_t i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= n) return; - uint32_t index = i * n; + size_t index = i * n; double sum = 0.0; - for (uint32_t j = 0; j < n; j++) { + for (size_t j = 0; j < n; j++) { if (j != i) sum += A[index + j] * x_old[j]; } diff --git a/tests/regression/module_reload/kernel.cpp b/tests/regression/module_reload/kernel.cpp index 1ebcfeddae..0491f994d1 100644 --- a/tests/regression/module_reload/kernel.cpp +++ b/tests/regression/module_reload/kernel.cpp @@ -3,13 +3,13 @@ int main() { kernel_arg_t* __UNIFORM__ arg = (kernel_arg_t*)csr_read(VX_CSR_MSCRATCH); - uint32_t count = arg->count; + size_t count = arg->count; int32_t* src_ptr = (int32_t*)arg->src_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t offset = vx_core_id() * count; + size_t offset = vx_core_id() * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset + i] = src_ptr[offset + i]; } diff --git a/tests/regression/mstress/kernel.cpp b/tests/regression/mstress/kernel.cpp index f3b5d2d212..8a66f2047c 100644 --- a/tests/regression/mstress/kernel.cpp +++ b/tests/regression/mstress/kernel.cpp @@ -2,18 +2,18 @@ #include "common.h" __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - uint32_t stride = arg->stride; + size_t stride = arg->stride; uint32_t* addr_ptr = (uint32_t*)arg->src0_addr; float* src_ptr = (float*)arg->src1_addr; float* dst_ptr = (float*)arg->dst_addr; - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t offset = gid * stride; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t offset = gid * stride; - for (uint32_t i = 0; i < stride; ++i) { + for (size_t i = 0; i < stride; ++i) { float value = 0.0f; for (uint32_t j = 0; j < NUM_LOADS; ++j) { - uint32_t addr = offset + i + j; + size_t addr = offset + i + j; uint32_t index = addr_ptr[addr]; value *= src_ptr[index]; } diff --git a/tests/regression/packld/kernel.cpp b/tests/regression/packld/kernel.cpp index 0d3bef2215..5473c06d22 100644 --- a/tests/regression/packld/kernel.cpp +++ b/tests/regression/packld/kernel.cpp @@ -14,7 +14,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_lb = reinterpret_cast(arg->dst_lb_addr); auto dst_lh = reinterpret_cast(arg->dst_lh_addr); - uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + size_t tid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t stride = 1; // byte stride between consecutive elements for (uint32_t p = 0; p < NUM_POINTS; ++p) { diff --git a/tests/regression/packld_box/kernel.cpp b/tests/regression/packld_box/kernel.cpp index b407b12518..e41ef52e7f 100644 --- a/tests/regression/packld_box/kernel.cpp +++ b/tests/regression/packld_box/kernel.cpp @@ -15,7 +15,7 @@ // Register spilling silently erases the bug by doing proper NaN-boxing. // Uncomment the two lines to see all the tests pass. __attribute__((always_inline)) -inline uint64_t packlb_bits(const void* base, uint32_t stride) { +inline uint64_t packlb_bits(const void* base, size_t stride) { uint64_t bits; __asm__ volatile ( ".insn r %1, 1, 4, ft0, %2, %3\n\t" @@ -28,7 +28,7 @@ inline uint64_t packlb_bits(const void* base, uint32_t stride) { } __attribute__((always_inline)) -inline uint64_t packlh_bits(const void* base, uint32_t stride) { +inline uint64_t packlh_bits(const void* base, size_t stride) { uint64_t bits; __asm__ volatile ( ".insn r %1, 2, 4, ft0, %2, %3\n\t" @@ -53,8 +53,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_lb = reinterpret_cast(arg->dst_lb_addr); auto dst_lh = reinterpret_cast(arg->dst_lh_addr); - uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t stride = 1; // byte stride between consecutive elements + size_t tid = blockIdx.x * blockDim.x + threadIdx.x; + size_t stride = 1; // byte stride between consecutive elements for (uint32_t p = 0; p < NUM_POINTS; ++p) { // 4 bytes at consecutive addresses → one packed float (PACKLB) diff --git a/tests/regression/raycast/render.h b/tests/regression/raycast/render.h index f793a6e946..b75d87fc84 100644 --- a/tests/regression/raycast/render.h +++ b/tests/regression/raycast/render.h @@ -78,7 +78,7 @@ void BVHIntersect(const ray_t &ray, const tri_t *triBuffer, ray_hit_t *hit) { uint32_t stack[BVH_STACK_SIZE]; - uint32_t stackPtr = 0; + size_t stackPtr = 0; stack[stackPtr++] = 0; // Push root node index while (stackPtr != 0) { @@ -86,7 +86,7 @@ void BVHIntersect(const ray_t &ray, const bvh_node_t &node = bvhBuffer[nodeIdx]; if (node.isLeaf()) { // Intersect leaf triangles - for (uint32_t i = 0; i < node.triCount; ++i) { + for (size_t i = 0; i < node.triCount; ++i) { uint32_t triIdx = triIdxBuffer[node.leftFirst + i]; float dist; float3_t bcoords; @@ -149,7 +149,7 @@ void TLASIntersect(const ray_t &ray, const tri_t *triBuffer, ray_hit_t *hit) { uint32_t stack[BVH_STACK_SIZE]; - uint32_t stackPtr = 0; + size_t stackPtr = 0; stack[stackPtr++] = tlas_root; while (stackPtr != 0) { diff --git a/tests/regression/reopen/kernel.cpp b/tests/regression/reopen/kernel.cpp index 1ebcfeddae..0491f994d1 100644 --- a/tests/regression/reopen/kernel.cpp +++ b/tests/regression/reopen/kernel.cpp @@ -3,13 +3,13 @@ int main() { kernel_arg_t* __UNIFORM__ arg = (kernel_arg_t*)csr_read(VX_CSR_MSCRATCH); - uint32_t count = arg->count; + size_t count = arg->count; int32_t* src_ptr = (int32_t*)arg->src_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t offset = vx_core_id() * count; + size_t offset = vx_core_id() * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset + i] = src_ptr[offset + i]; } diff --git a/tests/regression/sgemm/kernel.cpp b/tests/regression/sgemm/kernel.cpp index bafb73cb91..7b79bbfb11 100644 --- a/tests/regression/sgemm/kernel.cpp +++ b/tests/regression/sgemm/kernel.cpp @@ -5,14 +5,14 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x * blockDim.x + threadIdx.x; int row = blockIdx.y * blockDim.y + threadIdx.y; TYPE sum(0); #pragma unroll 8 - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/regression/sgemm2/kernel.cpp b/tests/regression/sgemm2/kernel.cpp index 17a8817f9a..afe6c336e8 100644 --- a/tests/regression/sgemm2/kernel.cpp +++ b/tests/regression/sgemm2/kernel.cpp @@ -6,15 +6,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto B_ptr = reinterpret_cast(arg->B_addr); auto C_ptr = reinterpret_cast(arg->C_addr); - auto size = arg->size; - auto tile_size = arg->tile_size; - auto chunk_k = arg->chunk_k; + size_t size = arg->size; + size_t tile_size = arg->tile_size; + size_t chunk_k = arg->chunk_k; // Global and local thread coordinates. - auto g_row = blockIdx.x * blockDim.x + threadIdx.x; - auto g_col = blockIdx.y * blockDim.y + threadIdx.y; - auto l_row = threadIdx.x; - auto l_col = threadIdx.y; + size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; + size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; + size_t l_row = threadIdx.x; + size_t l_col = threadIdx.y; // Shared memory layout: // local_A: [tile_size × chunk_k], row-major (row = l_row, col = k_local) @@ -26,19 +26,19 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { TYPE sum(0); - for (uint32_t k = 0; k < size; k += chunk_k) { + for (size_t k = 0; k < size; k += chunk_k) { // Cooperative load: each thread loads chunk_k/tile_size elements per tile. // Thread (l_row, l_col) loads columns l_col, l_col+tile_size, ... of its row. - for (uint32_t kk = l_col; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_col; kk < chunk_k; kk += tile_size) { local_A[l_row * chunk_k + kk] = A_ptr[g_row * size + (k + kk)]; } - for (uint32_t kk = l_row; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_row; kk < chunk_k; kk += tile_size) { local_B[kk * tile_size + l_col] = B_ptr[(k + kk) * size + g_col]; } __syncthreads(); - for (uint32_t j = 0; j < chunk_k; ++j) { + for (size_t j = 0; j < chunk_k; ++j) { sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; } diff --git a/tests/regression/sgemm2_dxa/kernel.cpp b/tests/regression/sgemm2_dxa/kernel.cpp index f152f52275..7693034844 100644 --- a/tests/regression/sgemm2_dxa/kernel.cpp +++ b/tests/regression/sgemm2_dxa/kernel.cpp @@ -11,9 +11,9 @@ constexpr uint32_t kDescB = 1; // Accumulate partial sum from shared memory tiles. // shA layout: [tile_size × chunk_k], row-major (row = l_row, col = k) // shB layout: [chunk_k × tile_size], row-major (row = k, col = l_col) -static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, uint32_t tile_size, - uint32_t chunk_k, uint32_t l_row, uint32_t l_col) { - for (uint32_t k = 0; k < chunk_k; ++k) { +static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, size_t tile_size, + size_t chunk_k, size_t l_row, size_t l_col) { + for (size_t k = 0; k < chunk_k; ++k) { sum += shA[l_row * chunk_k + k] * shB[k * tile_size + l_col]; } } @@ -21,25 +21,25 @@ static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, __kernel void kernel_main(kernel_arg_t* arg) { auto C = reinterpret_cast(arg->C_addr); - const uint32_t size = arg->size; - const uint32_t tile_size = arg->tile_size; - const uint32_t chunk_k = arg->chunk_k; + const size_t size = arg->size; + const size_t tile_size = arg->tile_size; + const size_t chunk_k = arg->chunk_k; const uint32_t mode = arg->mode; // Global and local thread coordinates. - const uint32_t row_base = blockIdx.x * tile_size; - const uint32_t col_base = blockIdx.y * tile_size; - const uint32_t l_row = threadIdx.x; - const uint32_t l_col = threadIdx.y; - const uint32_t g_row = row_base + l_row; - const uint32_t g_col = col_base + l_col; + const size_t row_base = blockIdx.x * tile_size; + const size_t col_base = blockIdx.y * tile_size; + const size_t l_row = threadIdx.x; + const size_t l_col = threadIdx.y; + const size_t g_row = row_base + l_row; + const size_t g_col = col_base + l_col; TYPE sum(0); // SMEM tile layout: A tile is [tile_size x chunk_k], B tile is [chunk_k x tile_size]. - const uint32_t tile_elems_a = tile_size * chunk_k; - const uint32_t tile_elems_b = chunk_k * tile_size; - const uint32_t stage_elems = tile_elems_a + tile_elems_b; + const size_t tile_elems_a = tile_size * chunk_k; + const size_t tile_elems_b = chunk_k * tile_size; + const size_t stage_elems = tile_elems_a + tile_elems_b; // const uint32_t stage_count = (mode == 2) ? 2u : 1u; // Allocate shared memory for tile buffers. diff --git a/tests/regression/sgemm2_dxa_mcast/kernel.cpp b/tests/regression/sgemm2_dxa_mcast/kernel.cpp index 1127865e94..7552459835 100644 --- a/tests/regression/sgemm2_dxa_mcast/kernel.cpp +++ b/tests/regression/sgemm2_dxa_mcast/kernel.cpp @@ -26,16 +26,16 @@ constexpr uint32_t kDescB = 1; __kernel void kernel_main(kernel_arg_t* arg) { auto C = reinterpret_cast(arg->C_addr); - const uint32_t size = arg->size; - const uint32_t tile_size = arg->tile_size; - const uint32_t chunk_k = arg->chunk_k; + const size_t size = arg->size; + const size_t tile_size = arg->tile_size; + const size_t chunk_k = arg->chunk_k; const uint32_t mc_group_size = arg->mc_group_size; // CTA position. blockIdx.x walks rows; blockIdx.y walks column-blocks. // 4 CTAs (mc_group_size) in adjacent blockIdx.x values are co-resident // (= one local group) and share one B column-block via multicast. - const uint32_t row = blockIdx.x; - const uint32_t col_blk = blockIdx.y; + const size_t row = blockIdx.x; + const size_t col_blk = blockIdx.y; // SMEM tile layout: A row [chunk_k] then B column-block [chunk_k × tile_size]. auto shmem = reinterpret_cast(__local_mem()); @@ -68,9 +68,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { local_B.arrive_and_wait(); // ── Compute: 1 row × tile_size cols ─────────────────────────────────────── - const uint32_t l_col = threadIdx.x; // 0..tile_size-1 (single warp) + const size_t l_col = threadIdx.x; // 0..tile_size-1 (single warp) TYPE sum(0); - for (uint32_t k = 0; k < chunk_k; ++k) { + for (size_t k = 0; k < chunk_k; ++k) { sum += shA[k] * shB[k * tile_size + l_col]; } diff --git a/tests/regression/sgemm2_tcu/kernel.cpp b/tests/regression/sgemm2_tcu/kernel.cpp index ff7c717dd0..3cc808bf0d 100644 --- a/tests/regression/sgemm2_tcu/kernel.cpp +++ b/tests/regression/sgemm2_tcu/kernel.cpp @@ -11,16 +11,16 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pB = reinterpret_cast(arg->B_addr); auto pC = reinterpret_cast(arg->C_addr); - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; uint32_t cta_M = arg->cta_M; uint32_t tid = threadIdx.x; uint32_t num_threads = blockDim.x; // num_warps * VX_CFG_NUM_THREADS uint32_t warp_rank = tid / VX_CFG_NUM_THREADS; - uint32_t tile_row = blockIdx.y * cta_M; - uint32_t tile_col = blockIdx.x * ctx::tileN; + size_t tile_row = blockIdx.y * cta_M; + size_t tile_col = blockIdx.x * ctx::tileN; // Shared memory: A [cta_M x tileK] row-major, B [tileK x tileN] row-major auto smem = reinterpret_cast(__local_mem()); @@ -32,23 +32,23 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); - for (uint32_t k = 0; k < K; k += ctx::tileK) { + for (size_t k = 0; k < K; k += ctx::tileK) { // Cooperatively load A tile [cta_M x tileK] from DRAM (row-major). uint32_t a_size = cta_M * ctx::tileK; - for (uint32_t i = 0; i < a_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::tileK; - uint32_t c = idx % ctx::tileK; + for (size_t i = 0; i < a_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::tileK; + size_t c = idx % ctx::tileK; A_smem[r * ctx::tileK + c] = pA[(tile_row + r) * K + (k + c)]; } // Cooperatively load B tile [tileK x tileN] from DRAM (col-major source) // and store row-major in shared memory. uint32_t b_size = ctx::tileK * ctx::tileN; - for (uint32_t i = 0; i < b_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::tileN; - uint32_t c = idx % ctx::tileN; + for (size_t i = 0; i < b_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::tileN; + size_t c = idx % ctx::tileN; B_smem[r * ctx::tileN + c] = pB[(tile_col + c) * K + (k + r)]; } diff --git a/tests/regression/sgemm2_v1/kernel.cpp b/tests/regression/sgemm2_v1/kernel.cpp index e26687a322..9f32b87f3d 100644 --- a/tests/regression/sgemm2_v1/kernel.cpp +++ b/tests/regression/sgemm2_v1/kernel.cpp @@ -6,15 +6,15 @@ void kernel_body(kernel_arg_t *arg) { auto B_ptr = reinterpret_cast(arg->B_addr); auto C_ptr = reinterpret_cast(arg->C_addr); - auto size = arg->size; - auto tile_size = arg->tile_size; - auto chunk_k = arg->chunk_k; + size_t size = arg->size; + size_t tile_size = arg->tile_size; + size_t chunk_k = arg->chunk_k; // Global and local thread coordinates. - auto g_row = blockIdx.x * blockDim.x + threadIdx.x; - auto g_col = blockIdx.y * blockDim.y + threadIdx.y; - auto l_row = threadIdx.x; - auto l_col = threadIdx.y; + size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; + size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; + size_t l_row = threadIdx.x; + size_t l_col = threadIdx.y; // Shared memory layout: // local_A: [tile_size × chunk_k], row-major (row = l_row, col = k_local) @@ -27,19 +27,19 @@ void kernel_body(kernel_arg_t *arg) { TYPE sum(0); - for (uint32_t k = 0; k < size; k += chunk_k) { + for (size_t k = 0; k < size; k += chunk_k) { // Cooperative load: each thread loads chunk_k/tile_size elements per tile. // Thread (l_row, l_col) loads columns l_col, l_col+tile_size, ... of its row. - for (uint32_t kk = l_col; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_col; kk < chunk_k; kk += tile_size) { local_A[l_row * chunk_k + kk] = A_ptr[g_row * size + (k + kk)]; } - for (uint32_t kk = l_row; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_row; kk < chunk_k; kk += tile_size) { local_B[kk * tile_size + l_col] = B_ptr[(k + kk) * size + g_col]; } __syncthreads(); - for (uint32_t j = 0; j < chunk_k; ++j) { + for (size_t j = 0; j < chunk_k; ++j) { sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; } diff --git a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp index 05dfd78059..281bd1aa77 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp @@ -99,10 +99,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #ifdef SW_LOAD_A // Cooperative load A row-major (matches DXA row-major layout). uint32_t a_size = cta_M * ctx::tileK; - for (uint32_t i = 0; i < a_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::tileK; - uint32_t c = idx % ctx::tileK; + for (size_t i = 0; i < a_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::tileK; + size_t c = idx % ctx::tileK; A_smem[r * ctx::tileK + c] = pA[(tile_row + r) * K + (k + c)]; } #endif diff --git a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp index f370bbd583..34bb9e5c9c 100644 --- a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp @@ -19,15 +19,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pBTensorScale = reinterpret_cast(arg->B_tensor_scale_addr); #endif - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; uint32_t tid = threadIdx.x; uint32_t num_threads = blockDim.x; uint32_t warp_rank = tid / VX_CFG_NUM_THREADS; uint32_t num_warps = num_threads / VX_CFG_NUM_THREADS; uint32_t cta_m = num_warps * ctx::xtileM; - uint32_t tile_row = blockIdx.y * cta_m; - uint32_t tile_col = blockIdx.x * ctx::xtileN; + size_t tile_row = blockIdx.y * cta_m; + size_t tile_col = blockIdx.x * ctx::xtileN; uint32_t num_k_tiles = K / ctx::tileK; auto A_smem = reinterpret_cast(__local_mem()); @@ -37,18 +37,18 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); - for (uint32_t k = 0; k < K; k += ctx::tileK) { + for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; uint32_t a_bytes = cta_m * ctx::tileK; - for (uint32_t idx = tid; idx < a_bytes; idx += num_threads) { - uint32_t row = idx / ctx::tileK; - uint32_t col = idx % ctx::tileK; + for (size_t idx = tid; idx < a_bytes; idx += num_threads) { + size_t row = idx / ctx::tileK; + size_t col = idx % ctx::tileK; A_smem[idx] = pA[(tile_row + row) * K + k + col]; } uint32_t b_bytes = ctx::xtileN * ctx::tileK; - for (uint32_t idx = tid; idx < b_bytes; idx += num_threads) { - uint32_t col = idx / ctx::tileK; - uint32_t k_byte = idx % ctx::tileK; + for (size_t idx = tid; idx < b_bytes; idx += num_threads) { + size_t col = idx / ctx::tileK; + size_t k_byte = idx % ctx::tileK; B_smem[idx] = pB[(tile_col + col) * K + k + k_byte]; } __syncthreads(); diff --git a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp index 8c60045c2e..f1872cbed6 100644 --- a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp @@ -27,8 +27,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); // compressed A auto pB = reinterpret_cast(arg->B_addr); // dense B - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; uint32_t tid = threadIdx.x; uint32_t lane = tid % VX_CFG_NUM_THREADS; @@ -38,8 +38,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // CTA tile dimensions uint32_t cta_M = num_warps * ctx::xtileM; - uint32_t tile_row = blockIdx.y * cta_M; - uint32_t tile_col = blockIdx.x * ctx::xtileN; + size_t tile_row = blockIdx.y * cta_M; + size_t tile_col = blockIdx.x * ctx::xtileN; auto smem_base = reinterpret_cast(__local_mem()); uint32_t smem_b_off = @@ -51,28 +51,28 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); - uint32_t a_sp_stride = K / 2; + size_t a_sp_stride = K / 2; uint32_t num_k_tiles = K / ctx::tileK; uint32_t meta_words_per_tile = ctx::wg_meta_total_bytes / 4; - for (uint32_t k = 0; k < K; k += ctx::tileK) { + for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; // Each warp loads its compressed A tile into its private TBUF section. auto A_smem_w = reinterpret_cast( smem_base + warp_rank * per_warp_section); - for (uint32_t i = lane; i < smem_a_elems; i += VX_CFG_NUM_THREADS) { - uint32_t r = i / (ctx::tileK / 2); - uint32_t c = i % (ctx::tileK / 2); + for (size_t i = lane; i < smem_a_elems; i += VX_CFG_NUM_THREADS) { + size_t r = i / (ctx::tileK / 2); + size_t c = i % (ctx::tileK / 2); A_smem_w[i] = pA[(tile_row + warp_rank * ctx::xtileM + r) * a_sp_stride + (k / 2 + c)]; } // Cooperative load: K-major B, matching the TBUF's strided descriptor. - for (uint32_t i = tid; i < smem_b_elems; i += num_threads) { - uint32_t r = i / ctx::xtileN; - uint32_t c = i % ctx::xtileN; + for (size_t i = tid; i < smem_b_elems; i += num_threads) { + size_t r = i / ctx::xtileN; + size_t c = i % ctx::xtileN; B_smem[c * ctx::tileK + r] = pB[(k + r) * N + (tile_col + c)]; } diff --git a/tests/regression/sgemm_v1/kernel.cpp b/tests/regression/sgemm_v1/kernel.cpp index 4634bae635..001651da5f 100644 --- a/tests/regression/sgemm_v1/kernel.cpp +++ b/tests/regression/sgemm_v1/kernel.cpp @@ -5,13 +5,13 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x; int row = blockIdx.y; TYPE sum(0); - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/regression/stencil3d/kernel.cpp b/tests/regression/stencil3d/kernel.cpp index bab24af116..9c0c0c7bf4 100644 --- a/tests/regression/stencil3d/kernel.cpp +++ b/tests/regression/stencil3d/kernel.cpp @@ -5,7 +5,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x * blockDim.x + threadIdx.x; int row = blockIdx.y * blockDim.y + threadIdx.y; diff --git a/tests/regression/vm_stress/kernel.cpp b/tests/regression/vm_stress/kernel.cpp index c1df67de35..dc241710a1 100644 --- a/tests/regression/vm_stress/kernel.cpp +++ b/tests/regression/vm_stress/kernel.cpp @@ -14,8 +14,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t bias = phys_ptr[task_id % arg->phys_words]; for (uint32_t k = 0; k < arg->pages_per_task; ++k) { - uint32_t page = ((task_id * arg->pages_per_task + k) * arg->stride_pages) % arg->total_pages; - uint32_t word = page * WORDS_PER_PAGE + (task_id % WORDS_PER_PAGE); + int page = ((task_id * arg->pages_per_task + k) * arg->stride_pages) % arg->total_pages; + int word = page * WORDS_PER_PAGE + (task_id % WORDS_PER_PAGE); dst_ptr[word] = src_ptr[word] + bias; } } From 577a4afea074ea35f9c02beb5dad9ff7abf787cf Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 9 Oct 2026 18:57:06 -0700 Subject: [PATCH 003/336] rtu: carry full-width scene and payload pointers on RV64 On RV64 the trace lost the upper half of both warp-uniform pointers. SimX narrowed the scene base to 32 bits, so a scene above 4 GiB was read from the wrong address and every ray missed. SimX and the RTL both carried the payload pointer as 32 bits and wrote it back through the single VX_RT_PAYLOAD_PTR_LO slot, so a callback shader got a truncated payload pointer. vx_rt_wtrace took uint32_t pointers and narrowed the gathered config, and every tests/raytracing kernel masked its device addresses to 32 bits, which kept all of it hidden: device buffers normally sit below 4 GiB. - The high word goes in slot VX_RT_PAYLOAD_PTR_LO + 1, which already sat unused next to the low word. It is not a new VX_types define: the perf-gate fingerprint hashes VX_types, so a define would mark every perf baseline stale without moving a cycle. - SimX keeps the scene base at the device address width and the payload at full width, and writes the high word on RV64. - RTL widens payload_ptr to XLEN across the RTU bus, the arm staging RAM and the slot descriptor. On RV64 the record write-back sends one more word (the high half) before the hit attribute; STATUS stays last. At XLEN=32 the logic is unchanged. - vx_raytrace.h takes uintptr_t pointers, keeps the gathered config at register width, and adds vx_rt_get_payload(), which reads both words on RV64. - The tests/raytracing kernels pass full-width pointers. On a 64-bit device rt_smoke_chs places its scene and payload above 4 GiB; it fails before this change (miss, payload never written) and passes after. - ci/testcases/raytracing.yaml adds rt_suite-x64, the suite on RV64 simx. - hw/unittest gfx and rtu_core tops follow the wider arm word. RV32: all 34 ray-tracing kernels compile to identical code; the three rt_raycast perf_gate cases reproduce their baseline cycles and instruction counts exactly, the rtlsim suite and model_parity pass, and every perf_gate config hash is unchanged. RV64: the simx suite passes 34/34, and rtlsim passes 31/32 plus rt_smoke_chs above 4 GiB. rt_raycast fails pixel comparison on RV64 rtlsim identically before and after this change (210/768 pixels); that is a separate, pre-existing RTL defect, so rt_suite-x64 stays simx-only until it is fixed. The extra RV64 write-back word moves RV64 rtlsim rt_raycast cycles by 0% to +5.5% depending on shape (NT4 +5.1%, NT16 0.0%, 2-core +0.7%, 8x8 +5.5%) against two added cycles per record; RV64 rtlsim has no perf gate. Co-Authored-By: Claude Opus 5.5 --- VX_types.toml | 1 + ci/testcases/raytracing.yaml | 17 ++++++++-- docs/designs/ray_tracing_architecture.md | 2 +- hw/rtl/rtu/VX_rtu_bus_arb.sv | 2 +- hw/rtl/rtu/VX_rtu_bus_if.sv | 2 +- hw/rtl/rtu/VX_rtu_bus_slice.sv | 2 +- hw/rtl/rtu/VX_rtu_core.sv | 34 ++++++++++--------- hw/rtl/rtu/VX_rtu_pkg.sv | 19 ++++++----- hw/rtl/rtu/VX_rtu_unit.sv | 2 +- hw/unittest/gfx/VX_gfx_top.sv | 2 +- hw/unittest/rtu_core/VX_rtu_core_top.sv | 2 +- sim/simx/rtu/rtu_core.cpp | 6 ++-- sim/simx/rtu/rtu_types.h | 2 +- sim/simx/rtu/rtu_unit.cpp | 22 +++++++++--- sim/simx/rtu/rtu_unit.h | 7 ++-- sw/kernel/include/vx_raytrace.h | 19 ++++++++--- tests/raytracing/rt_bvh_multinode/kernel.cpp | 4 +-- tests/raytracing/rt_raycast/kernel.cpp | 4 +-- tests/raytracing/rt_smoke/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_ahs/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp | 4 +-- .../raytracing/rt_smoke_ahs_custom/kernel.cpp | 12 +++---- tests/raytracing/rt_smoke_ahs_geom/kernel.cpp | 12 +++---- .../raytracing/rt_smoke_ahs_mixed/kernel.cpp | 4 +-- .../rt_smoke_async_batch/kernel.cpp | 8 ++--- tests/raytracing/rt_smoke_bvh6/kernel.cpp | 4 +-- .../raytracing/rt_smoke_bvh_basic/kernel.cpp | 4 +-- .../rt_smoke_bvh_instanced/kernel.cpp | 4 +-- .../rt_smoke_bvh_multilevel/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_chs/common.h | 2 +- tests/raytracing/rt_smoke_chs/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_chs/main.cpp | 26 +++++++++++--- .../raytracing/rt_smoke_cull_back/kernel.cpp | 4 +-- .../raytracing/rt_smoke_cull_mask/kernel.cpp | 4 +-- .../raytracing/rt_smoke_deep_stack/kernel.cpp | 4 +-- .../raytracing/rt_smoke_deep_tlas/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_fat_leaf/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_host_cfg/kernel.cpp | 4 +-- .../raytracing/rt_smoke_inst_flags/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_is/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_miss/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_multitri/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_proc/kernel.cpp | 4 +-- .../raytracing/rt_smoke_recursive/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_reform/kernel.cpp | 4 +-- .../raytracing/rt_smoke_reform_mw/kernel.cpp | 4 +-- .../raytracing/rt_smoke_reform_sbt/kernel.cpp | 2 +- tests/raytracing/rt_smoke_sbt/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_shadow/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_tlas/kernel.cpp | 4 +-- .../rt_smoke_tlas_builder/kernel.cpp | 4 +-- 51 files changed, 204 insertions(+), 145 deletions(-) diff --git a/VX_types.toml b/VX_types.toml index 495afbf7cc..c00dd29521 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -442,6 +442,7 @@ VX_RT_STATUS = 26 # per-lane VX_RT_STS_*; WAIT completes on # Every slot is written by the RTU and read by the shader; nothing else writes the # window. The payload pointer rides the arm doorbell (warp-uniform) and the # hitAttribute rides the CONTINUE, so both reach the RTU without a window write. +# On RV64 the payload pointer's upper word follows in slot VX_RT_PAYLOAD_PTR_LO + 1. VX_RT_PAYLOAD_PTR_LO = 27 # the tracing shader's payload; the callbacks read it VX_RT_HIT_ATTR_0 = 29 # user hitAttributeEXT (an IS shader returns it) diff --git a/ci/testcases/raytracing.yaml b/ci/testcases/raytracing.yaml index 6ccbc74045..f4b42836d4 100644 --- a/ci/testcases/raytracing.yaml +++ b/ci/testcases/raytracing.yaml @@ -1,8 +1,7 @@ # Test cases for `raytracing` (RTU), ported from regression.sh.in raytracing(). # -# simx is the only backend that validates the RTU end-to-end today; rtlsim / -# opae / xrt RTU coverage lands when the RTL side comes online. xlen is pinned to -# 32 (the RTU SimX-parity reference build); 64-bit RTU coverage is a follow-on. +# xlen defaults to 32 (the RTU SimX-parity reference build); rt_suite-x64 runs +# the suite on RV64. category: raytracing defaults: xlen: @@ -30,6 +29,18 @@ tests: target: run-{driver} configs: -DVX_CFG_EXT_RTU_ENABLE +# The same suite on RV64. There rt_smoke_chs puts its scene and payload above +# 4 GiB, so a trace that narrows either pointer misses or loses its payload. +- id: rt_suite-x64 + via: make-run + xlen: + - 64 + drivers: + - simx + dir: tests/raytracing + target: run-{driver} + configs: -DVX_CFG_EXT_RTU_ENABLE + # Multi-core RTU coverage: same canary kernel (rt_smoke) under two 4-core # topologies so a topology-only break is a single failure. -n64 = full device # capacity (cores x warps x threads); 64 rays in flight through the cluster's diff --git a/docs/designs/ray_tracing_architecture.md b/docs/designs/ray_tracing_architecture.md index e1590aae68..efd89a0fbb 100644 --- a/docs/designs/ray_tracing_architecture.md +++ b/docs/designs/ray_tracing_architecture.md @@ -176,7 +176,7 @@ state as base+count bursts, so each span's members must abut. | 17..22 | object-space ray `origin`/`dir` (staged on an AHS/IS yield) | RTU writes | | 23 / 24 / 25 | candidate `cb_type` / `sbt_idx` / `cb_handle` | RTU writes | | 26 | **STATUS** (`VX_RT_STS_*`) | RTU writes **last** | -| 27 / 29 | payload pointer / user `hitAttribute` | RTU writes (payload at arm; attr on commit) | +| 27 (+28 on RV64) / 29 | payload pointer (low / high word) / user `hitAttribute` | RTU writes (payload with every record; attr on commit) | **STATUS is deliberately last.** A parked WAIT completes on the address-match write to slot 26, so every other result word must already be in the window when diff --git a/hw/rtl/rtu/VX_rtu_bus_arb.sv b/hw/rtl/rtu/VX_rtu_bus_arb.sv index f619044b3c..ec01860259 100644 --- a/hw/rtl/rtu/VX_rtu_bus_arb.sv +++ b/hw/rtl/rtu/VX_rtu_bus_arb.sv @@ -68,7 +68,7 @@ module VX_rtu_bus_arb import VX_gpu_pkg::*, VX_rtu_pkg::*; #( localparam SRC_IN_W = 1; localparam ARM_DATAW = NW_WIDTH + NUM_LANES - + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + 32 + TAG_WIDTH; + + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + `VX_CFG_XLEN + TAG_WIDTH; localparam REQ_DATAW = 1 + NW_WIDTH + NUM_LANES * 32 + NUM_LANES * RTU_CB_ACTION_BITS; // the win payload carries the sel-stripped tag on the core side localparam WIN_DATAW = 1 + NW_WIDTH + RTU_SLOT_BITS diff --git a/hw/rtl/rtu/VX_rtu_bus_if.sv b/hw/rtl/rtu/VX_rtu_bus_if.sv index 23b017e826..2f259fa299 100644 --- a/hw/rtl/rtu/VX_rtu_bus_if.sv +++ b/hw/rtl/rtu/VX_rtu_bus_if.sv @@ -69,7 +69,7 @@ interface VX_rtu_bus_if import VX_gpu_pkg::*, VX_rtu_pkg::*; #( logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] scene_base; // device address of the scene logic [15:0] flags; // VX_RT_FLAG_* logic [15:0] cull_mask; - logic [31:0] payload_ptr; // the RTU stages it for the callbacks + logic [`VX_CFG_XLEN-1:0] payload_ptr; // the RTU stages it for the callbacks logic [TAG_WIDTH-1:0] tag; } arm_data_t; diff --git a/hw/rtl/rtu/VX_rtu_bus_slice.sv b/hw/rtl/rtu/VX_rtu_bus_slice.sv index d2601157a0..a85e27711c 100644 --- a/hw/rtl/rtu/VX_rtu_bus_slice.sv +++ b/hw/rtl/rtu/VX_rtu_bus_slice.sv @@ -54,7 +54,7 @@ module VX_rtu_bus_slice import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // parameter cannot read $bits of a hierarchical interface member, so size // the elastic buffers from the field formula instead. localparam ARM_DATAW = SRC_WIDTH + NW_WIDTH + NUM_LANES - + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + 32 + TAG_WIDTH; + + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + `VX_CFG_XLEN + TAG_WIDTH; localparam REQ_DATAW = 1 + SRC_WIDTH + NW_WIDTH + NUM_LANES * 32 + NUM_LANES * RTU_CB_ACTION_BITS; localparam WIN_DATAW = 1 + NW_WIDTH + RTU_SLOT_BITS diff --git a/hw/rtl/rtu/VX_rtu_core.sv b/hw/rtl/rtu/VX_rtu_core.sv index f7bd8af8ea..425c167f7a 100644 --- a/hw/rtl/rtu/VX_rtu_core.sv +++ b/hw/rtl/rtu/VX_rtu_core.sv @@ -147,11 +147,11 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( reg [NUM_SLOTS-1:0][2:0] tstate; reg [NUM_SLOTS-1:0][NUM_LANES-1:0] req_mask; - reg [NUM_SLOTS-1:0][TAG_WIDTH-1:0] req_tag; - reg [NUM_SLOTS-1:0][STG_IDX_W-1:0] req_stg; // the staging entry it launched from - reg [NUM_SLOTS-1:0][NW_WIDTH-1:0] req_wid; // its warp, for the window write - reg [NUM_SLOTS-1:0][31:0] req_payload; // warp-uniform; staged for the callbacks - reg [NUM_SLOTS-1:0] is_cand; + reg [NUM_SLOTS-1:0][TAG_WIDTH-1:0] req_tag; + reg [NUM_SLOTS-1:0][STG_IDX_W-1:0] req_stg; // the staging entry it launched from + reg [NUM_SLOTS-1:0][NW_WIDTH-1:0] req_wid; // its warp, for the window write + reg [NUM_SLOTS-1:0][`VX_CFG_XLEN-1:0] req_payload; // warp-uniform; staged for the callbacks + reg [NUM_SLOTS-1:0] is_cand; reg [NUM_SLOTS-1:0][NUM_LANES-1:0][RTU_CB_ACTION_BITS-1:0] cont_action; // ── the scheduler ───────────────────────────────────────────────── @@ -306,7 +306,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( ); // warp-uniform scalars of a staged trace (read once, at the claim) - localparam SCLW = NUM_LANES + TAG_WIDTH + 32 + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16; + localparam SCLW = NUM_LANES + TAG_WIDTH + `VX_CFG_XLEN + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16; wire [SCLW-1:0] stg_scl_rdata; wire [SLOT_W-1:0] fill_slot_pick; wire [STG_IDX_W-1:0] fill_stg_pick; @@ -332,11 +332,11 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( .raddr (fill_stg_pick), .rdata (stg_scl_rdata) ); - wire [NUM_LANES-1:0] scl_mask; - wire [TAG_WIDTH-1:0] scl_tag; - wire [31:0] scl_payload; + wire [NUM_LANES-1:0] scl_mask; + wire [TAG_WIDTH-1:0] scl_tag; + wire [`VX_CFG_XLEN-1:0] scl_payload; wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] scl_scene; - wire [15:0] scl_flags, scl_cull; + wire [15:0] scl_flags, scl_cull; assign {scl_mask, scl_tag, scl_payload, scl_scene, scl_flags, scl_cull} = stg_scl_rdata; // ── the fill engine ─────────────────────────────────────────────── @@ -481,7 +481,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // 7..12 object ray (candidate only) — the staging RAM's beat rows // 13,14 cb_type (flag flops), sbt (window row) // 15 cb_handle (0) - // then payload (descriptor), hitAttribute (window row), status LAST + // then payload (descriptor, XLEN/32 words), hitAttribute (window row), status LAST localparam [2:0] WB_IDLE = 3'd0, WB_RD1 = 3'd1, WB_CAP1 = 3'd2, @@ -495,9 +495,10 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire [RTU_IDX_BITS-1:0] n_attrs = is_cand[ws] ? RTU_IDX_BITS'(RTU_RES_CAND) : RTU_IDX_BITS'(RTU_RES_HIT); - wire wr_payload = (wr_idx == n_attrs); - wire wr_attr = (wr_idx == (n_attrs + RTU_IDX_BITS'(1))); - wire wr_status = (wr_idx == (n_attrs + RTU_IDX_BITS'(2))); + wire wr_pl_hi = (RTU_PAYLOAD_WORDS > 1) && (wr_idx == (n_attrs + RTU_IDX_BITS'(1))); + wire wr_payload = (wr_idx == n_attrs) || wr_pl_hi; + wire wr_attr = (wr_idx == (n_attrs + RTU_IDX_BITS'(RTU_PAYLOAD_WORDS))); + wire wr_status = (wr_idx == (n_attrs + RTU_IDX_BITS'(RTU_PAYLOAD_WORDS + 1))); wire wr_hitspan = (wr_idx < RTU_IDX_BITS'(RTU_RES_HIT)); wire wr_objray = is_cand[ws] && !wr_hitspan && (wr_idx < RTU_IDX_BITS'(13)); wire wr_cbtype = is_cand[ws] && (wr_idx == RTU_IDX_BITS'(13)); @@ -553,7 +554,8 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( if (wr_status) begin win_word[i] = status_word[i]; end else if (wr_payload) begin - win_word[i] = req_payload[ws]; + win_word[i] = wr_pl_hi ? req_payload[ws][`VX_CFG_XLEN-1 -: 32] + : req_payload[ws][31:0]; end else if (wr_attr) begin win_word[i] = sch_attrv[wctx[i]] ? wb_d1[i*32 +: 32] : 32'd0; end else if (wr_hitspan) begin @@ -591,7 +593,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( assign rtu_bus_w.win_data.mask = (is_cand[ws] && !wr_uniform) ? wr_cb_mask : req_mask[ws]; assign rtu_bus_w.win_data.slot = wr_status ? RTU_SLOT_BITS'(RTU_STATUS_SLOT) - : wr_payload ? RTU_SLOT_BITS'(RTU_PAYLOAD_SLOT) + : wr_payload ? RTU_SLOT_BITS'(RTU_PAYLOAD_SLOT) + RTU_SLOT_BITS'(wr_pl_hi) : wr_attr ? RTU_SLOT_BITS'(RTU_ATTR_SLOT) : RTU_SLOT_BITS'(RTU_RES_BASE) + RTU_SLOT_BITS'(wr_idx); diff --git a/hw/rtl/rtu/VX_rtu_pkg.sv b/hw/rtl/rtu/VX_rtu_pkg.sv index d21d89be69..0eb4a8a5a9 100644 --- a/hw/rtl/rtu/VX_rtu_pkg.sv +++ b/hw/rtl/rtu/VX_rtu_pkg.sv @@ -264,21 +264,22 @@ package VX_rtu_pkg; // // hit attrs [RES_BASE, +RES_HIT) written by every response // candidate [RES_BASE, +RES_CAND) + object ray, cb_type, sbt, handle - // payload PAYLOAD_SLOT written once, at arm (warp-uniform) + // payload PAYLOAD_SLOT(+1) XLEN/32 words, written per record (warp-uniform) // hit attr ATTR_SLOT the IS shader's hitAttribute, on commit // status STATUS_SLOT written LAST // // Status is last by design: writing it is what completes the warp's parked // WAIT, so every slot the warp may then read must already be in place. // ───────────────────────────────────────────────────────────────── - localparam RTU_RAY_BEATS = 8; // origin[3], dir[3], t_min, t_max - localparam RTU_RES_BASE = `VX_RT_HIT_T; - localparam RTU_RES_HIT = 7; // hit_t, hit_u, hit_v, prim, instance, geometry, custom - localparam RTU_RES_CAND = 16; // + object ray (6), cb_type, cb_sbt_idx, cb_handle - localparam RTU_STATUS_SLOT = `VX_RT_STATUS; - localparam RTU_PAYLOAD_SLOT = `VX_RT_PAYLOAD_PTR_LO; - localparam RTU_ATTR_SLOT = `VX_RT_HIT_ATTR_0; - localparam RTU_IDX_BITS = `CLOG2(RTU_RES_CAND + 1); + localparam RTU_RAY_BEATS = 8; // origin[3], dir[3], t_min, t_max + localparam RTU_RES_BASE = `VX_RT_HIT_T; + localparam RTU_RES_HIT = 7; // hit_t, hit_u, hit_v, prim, instance, geometry, custom + localparam RTU_RES_CAND = 16; // + object ray (6), cb_type, cb_sbt_idx, cb_handle + localparam RTU_STATUS_SLOT = `VX_RT_STATUS; + localparam RTU_PAYLOAD_SLOT = `VX_RT_PAYLOAD_PTR_LO; + localparam RTU_PAYLOAD_WORDS = `VX_CFG_XLEN / 32; + localparam RTU_ATTR_SLOT = `VX_RT_HIT_ATTR_0; + localparam RTU_IDX_BITS = `CLOG2(RTU_RES_CAND + 1); // ───────────────────────────────────────────────────────────────── // Window store — the per-slot result record, one NUM_LANES*32 row per word, diff --git a/hw/rtl/rtu/VX_rtu_unit.sv b/hw/rtl/rtu/VX_rtu_unit.sv index 07bc926c74..50781fcaf0 100644 --- a/hw/rtl/rtu/VX_rtu_unit.sv +++ b/hw/rtl/rtu/VX_rtu_unit.sv @@ -204,7 +204,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( assign rtu_bus_w.arm_data.wid = wid; assign rtu_bus_w.arm_data.mask = execute_if.data.header.tmask; assign rtu_bus_w.arm_data.scene_base = execute_if.data.rs1_data[CFG_L1][`VX_CFG_MEM_ADDR_WIDTH-1:0]; - assign rtu_bus_w.arm_data.payload_ptr = execute_if.data.rs1_data[CFG_L2][31:0]; + assign rtu_bus_w.arm_data.payload_ptr = execute_if.data.rs1_data[CFG_L2]; assign rtu_bus_w.arm_data.flags = execute_if.data.rs1_data[CFG_L3][15:0]; assign rtu_bus_w.arm_data.cull_mask = execute_if.data.rs1_data[CFG_L3][31:16]; assign rtu_bus_w.arm_data.tag = ($bits(rtu_bus_w.arm_data.tag))'(execute_if.data.header.uuid); diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index 53386bfd77..51aed0b6d5 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -99,7 +99,7 @@ module VX_gfx_top import VX_gpu_pkg::* input wire [NC-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] rtu_arm_scene_base, input wire [NC-1:0][15:0] rtu_arm_flags, input wire [NC-1:0][15:0] rtu_arm_cull_mask, - input wire [NC-1:0][31:0] rtu_arm_payload_ptr, + input wire [NC-1:0][`VX_CFG_XLEN-1:0] rtu_arm_payload_ptr, input wire [NC-1:0][RTU_REQ_TAG_WIDTH-1:0] rtu_arm_tag, output wire [NC-1:0] rtu_arm_ready, diff --git a/hw/unittest/rtu_core/VX_rtu_core_top.sv b/hw/unittest/rtu_core/VX_rtu_core_top.sv index 33452de81b..f4405346dd 100644 --- a/hw/unittest/rtu_core/VX_rtu_core_top.sv +++ b/hw/unittest/rtu_core/VX_rtu_core_top.sv @@ -31,7 +31,7 @@ module VX_rtu_core_top import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // Channel widths (mirror VX_rtu_bus_slice / VX_rtu_bus_if). parameter SRC_WIDTH = `UP(`CLOG2(NUM_SRCS)), parameter ARM_DATAW = SRC_WIDTH + NW_WIDTH + NUM_LANES - + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + 32 + TAG_WIDTH, + + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + `VX_CFG_XLEN + TAG_WIDTH, parameter REQ_DATAW = 1 + SRC_WIDTH + NW_WIDTH + NUM_LANES * 32 + NUM_LANES * RTU_CB_ACTION_BITS, parameter WIN_DATAW = 1 + NW_WIDTH + RTU_SLOT_BITS diff --git a/sim/simx/rtu/rtu_core.cpp b/sim/simx/rtu/rtu_core.cpp index 75492a860c..0e2db91fba 100644 --- a/sim/simx/rtu/rtu_core.cpp +++ b/sim/simx/rtu/rtu_core.cpp @@ -463,14 +463,14 @@ class RtuCore::Impl { } void bind_context(uint32_t ci, uint32_t slot, uint32_t lane, - uint32_t scene_root) { + uint64_t scene_root) { Context& cx = contexts_[ci]; cx = Context{}; cx.valid = true; cx.slot = slot; cx.lane = lane; - cx.base_line = uint64_t(scene_root) & kRtuLineMask; - cx.byte_off = uint32_t(uint64_t(scene_root) - cx.base_line); + cx.base_line = scene_root & kRtuLineMask; + cx.byte_off = uint32_t(scene_root - cx.base_line); // The ray opens on the reciprocal (1/dir) setup and the scene-header fetch, // which cost front-end states like any other step, then drain the reciprocal // pipeline before the first node can be tested. diff --git a/sim/simx/rtu/rtu_types.h b/sim/simx/rtu/rtu_types.h index 8d8858bce2..6e16c3f627 100644 --- a/sim/simx/rtu/rtu_types.h +++ b/sim/simx/rtu/rtu_types.h @@ -66,7 +66,7 @@ struct RtuReq { uint32_t slot_idx = 0; // Per-lane ray descriptor snapshot (TRACE_NEW only). - std::array scene_root = {}; + std::array scene_root = {}; std::array origin_x = {}; std::array origin_y = {}; std::array origin_z = {}; diff --git a/sim/simx/rtu/rtu_unit.cpp b/sim/simx/rtu/rtu_unit.cpp index 5b6a93ad8a..2d8cef624f 100644 --- a/sim/simx/rtu/rtu_unit.cpp +++ b/sim/simx/rtu/rtu_unit.cpp @@ -75,6 +75,10 @@ namespace { inline uint32_t fp_src_bits(const instr_trace_t* trace, uint32_t src, uint32_t t) { return static_cast(trace->src_data[src].at(t).u64 & 0xffffffffu); } + +// The scene base rides the arm doorbell as a device address, MEM_ADDR_WIDTH wide. +constexpr uint64_t kSceneAddrMask = (VX_CFG_MEM_ADDR_WIDTH >= 64) + ? ~uint64_t(0) : ((uint64_t(1) << VX_CFG_MEM_ADDR_WIDTH) - 1); } // namespace instr_trace_t* RtuUnit::process_wait(instr_trace_t* trace, uint32_t block_id) { @@ -260,7 +264,7 @@ void RtuUnit::apply_response(const RtuRsp& rsp) { auto& lregs = wregs.at(t); // Written by the RTU, not by any instruction: the payload pointer rode the // arm and the hitAttribute rode the CONTINUE that accepted this hit. - lregs[VX_RT_PAYLOAD_PTR_LO] = trace_payload_.at(rsp.warp_id); + stage_payload_ptr(lregs, rsp.warp_id); lregs[VX_RT_HIT_ATTR_0] = rsp.hit_attr[t]; lregs[VX_RT_HIT_T] = float_to_bits(rsp.hit_t[t]); lregs[VX_RT_HIT_BARY_U] = float_to_bits(rsp.hit_bary_u[t]); @@ -280,6 +284,14 @@ void RtuUnit::apply_response(const RtuRsp& rsp) { } } +void RtuUnit::stage_payload_ptr(RtuWindow::LaneRegs& lregs, uint32_t wid) const { + uint64_t payload = trace_payload_.at(wid); + lregs[VX_RT_PAYLOAD_PTR_LO] = uint32_t(payload); +#ifdef VX_CFG_XLEN_64 + lregs[VX_RT_PAYLOAD_PTR_LO + 1] = uint32_t(payload >> 32); +#endif +} + void RtuUnit::apply_callback_payload(const RtuRsp& rsp) { // Stage candidate-hit attrs + cb_type + cb_handle into the RTU regs // for the lanes whose rays yielded, so the dispatcher's vx_rt_get @@ -295,7 +307,7 @@ void RtuUnit::apply_callback_payload(const RtuRsp& rsp) { // The callback shaders read the tracing shader's payload pointer, and they // are a different shader — it cannot reach them any other way. A closest-hit // shader likewise reads the accepted hit's attribute. - lregs[VX_RT_PAYLOAD_PTR_LO] = trace_payload_.at(rsp.warp_id); + stage_payload_ptr(lregs, rsp.warp_id); lregs[VX_RT_HIT_ATTR_0] = rsp.hit_attr[t]; lregs[VX_RT_HIT_T] = float_to_bits(rsp.hit_t[t]); lregs[VX_RT_HIT_BARY_U] = float_to_bits(rsp.hit_bary_u[t]); @@ -437,7 +449,7 @@ instr_trace_t* RtuUnit::process_trace_uop(instr_trace_t* trace, uint32_t block_i case 0: { // GP config uop: the pool slot was claimed at issue (trace2_reserve_slot), // so this uop has no backpressure source. Unpack the lane-packed config - // (lane0=scene, lane1=payload, lane2=flags, lane3=cull — the implicit + // (lane1=scene, lane2=payload, lane3=flags|cull — the implicit // vx_wgather layout) and stage it. int32_t slot = trace_slot_.at(wid); assert(slot >= 0 && "TRACE uop0 issued without a reserved pool slot"); @@ -448,12 +460,12 @@ instr_trace_t* RtuUnit::process_trace_uop(instr_trace_t* trace, uint32_t block_i uint32_t flagscull = static_cast(cfg.at(3).u); // Warp-uniform: it rides the arm doorbell, so it is staged here, not written // into the window. The RTU writes the payload pointer back with the record. - trace_payload_.at(wid) = static_cast(cfg.at(2).u); + trace_payload_.at(wid) = cfg.at(2).u; trace_flags_.at(wid) = flagscull & 0xffffu; trace_cull_.at(wid) = flagscull >> 16; for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { if (!trace->tmask.test(t)) continue; - trace_scene_.at(wid)[t] = static_cast(cfg.at(1).u); + trace_scene_.at(wid)[t] = uint64_t(cfg.at(1).u) & kSceneAddrMask; trace->dst_data[t].u = uint32_t(slot); // handle returns early } return trace; diff --git a/sim/simx/rtu/rtu_unit.h b/sim/simx/rtu/rtu_unit.h index ab807f324c..978e7ea511 100644 --- a/sim/simx/rtu/rtu_unit.h +++ b/sim/simx/rtu/rtu_unit.h @@ -156,6 +156,9 @@ class RtuUnit { // code reads the right payload before issuing CONTINUE. void apply_callback_payload(const RtuRsp& rsp); + // The payload pointer is XLEN wide; the window holds it as 32-bit words. + void stage_payload_ptr(RtuWindow::LaneRegs& lregs, uint32_t wid) const; + // Representative slot of a candidate rsp (first active lane's cb_handle). static uint32_t candidate_slot(const RtuRsp& rsp); @@ -225,9 +228,9 @@ class RtuUnit { VX_CFG_NUM_WARPS> trace_ray_; // Warp-uniform half of the ray, staged by the config uop: it rides the arm // doorbell in HW, so it is not a window write either. - std::array, + std::array, VX_CFG_NUM_WARPS> trace_scene_; - std::array trace_payload_{}; + std::array trace_payload_{}; std::array trace_flags_{}; std::array trace_cull_{}; // The slot handle each lane's outstanding candidate came from, mirrored here diff --git a/sw/kernel/include/vx_raytrace.h b/sw/kernel/include/vx_raytrace.h index 0ff732b88c..baf3fe98da 100644 --- a/sw/kernel/include/vx_raytrace.h +++ b/sw/kernel/include/vx_raytrace.h @@ -50,6 +50,17 @@ extern "C" { : "memory"); \ __v; }) +// vx_rt_get_payload — the tracing shader's payload pointer, as the RTU staged it +// for this record (VX_RT_PAYLOAD_PTR_LO, plus the next slot on RV64). +static inline __attribute__((always_inline)) +uintptr_t vx_rt_get_payload(uint32_t tok) { + uintptr_t ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, tok); +#if __riscv_xlen == 64 + ptr |= (uintptr_t)vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO + 1, tok) << 32; +#endif + return ptr; +} + // vx_rt_cb_ret — release the lane's parked context in the RTU with one of // VX_RT_CB_{ACCEPT,IGNORE,TERMINATE}, handing back the intersection shader's // computed hit distance and attribute in the same instruction. The callback @@ -141,13 +152,13 @@ _Static_assert(offsetof(vx_hit_t, instance_custom) == 6 * sizeof(float), "vx_hit // across the warp's lanes, and the f0..f7 ray window is per-lane. // scene_ptr / payload_ptr / ray_flags / cull_mask : per-trace warp-uniform // config, lane-packed into one register via an IMPLICIT vx_wgather -// (lane0=scene, lane1=payload, lane2=flags, lane3=cull) — pure register +// (lane1=scene, lane2=payload, lane3=flags|cull) — pure register // domain, no memory traffic; hoists out of a bounce loop when invariant. // ray : the per-thread geometry, pinned into the f0..f7 caller-saved window. // Returns the async ray handle in rd. Non-blocking: the RTU traverses // while the kernel runs independent work; vx_rt_wait is the sync point. static inline __attribute__((always_inline)) -uint32_t vx_rt_wtrace(uint32_t scene_ptr, uint32_t payload_ptr, +uint32_t vx_rt_wtrace(uintptr_t scene_ptr, uintptr_t payload_ptr, uint32_t ray_flags, uint32_t cull_mask, const vx_ray_t* ray) { // Pack config into the GATHERED operands (not the wgather self slot): the @@ -156,7 +167,7 @@ uint32_t vx_rt_wtrace(uint32_t scene_ptr, uint32_t payload_ptr, // lane3={flags,cull}; lane0 (self) unused. Keeps scene valid even when lane 0 // is masked (callback/recursion-narrowed traces). uint32_t flags_cull = (ray_flags & 0xffffu) | (cull_mask << 16); - uint32_t cfg = (uint32_t)vx_wgather(0u, scene_ptr, payload_ptr, flags_cull); + size_t cfg = vx_wgather(0u, scene_ptr, payload_ptr, flags_cull); register float r0 __asm__("f0") = ray->origin[0]; register float r1 __asm__("f1") = ray->origin[1]; register float r2 __asm__("f2") = ray->origin[2]; @@ -328,7 +339,7 @@ void vx_rt_get_objray(vx_objray_t* out) { // have independent work, it calls trace/wait separately and the compiler // schedules that work into the gap. static inline __attribute__((always_inline)) -uint32_t vx_rt_wtrace_sync(uint32_t scene_ptr, uint32_t payload_ptr, +uint32_t vx_rt_wtrace_sync(uintptr_t scene_ptr, uintptr_t payload_ptr, uint32_t ray_flags, uint32_t cull_mask, const vx_ray_t* ray, vx_hit_t* hit) { uint32_t h = vx_rt_wtrace(scene_ptr, payload_ptr, ray_flags, cull_mask, ray); diff --git a/tests/raytracing/rt_bvh_multinode/kernel.cpp b/tests/raytracing/rt_bvh_multinode/kernel.cpp index 0b31652e27..12eb9ff64a 100644 --- a/tests/raytracing/rt_bvh_multinode/kernel.cpp +++ b/tests/raytracing/rt_bvh_multinode/kernel.cpp @@ -29,8 +29,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { { r[3], r[4], r[5] }, r[6], r[7] }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_raycast/kernel.cpp b/tests/raytracing/rt_raycast/kernel.cpp index 9b8013146c..0ffc6baf7d 100644 --- a/tests/raytracing/rt_raycast/kernel.cpp +++ b/tests/raytracing/rt_raycast/kernel.cpp @@ -57,8 +57,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Fire the ray on the RTU (v2 single-issue trace). vx_ray_t ray = { {ro.x, ro.y, ro.z}, {rd.x, rd.y, rd.z}, 0.001f, 1e30f }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke/kernel.cpp b/tests/raytracing/rt_smoke/kernel.cpp index 7271651877..dd265ad464 100644 --- a/tests/raytracing/rt_smoke/kernel.cpp +++ b/tests/raytracing/rt_smoke/kernel.cpp @@ -43,8 +43,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // One trace + one wait. cull_mask = 0xff = no culling (the default). // payload = 0 (smoke test reads no payload). - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_ahs/kernel.cpp b/tests/raytracing/rt_smoke_ahs/kernel.cpp index 0019a747e0..0acb3685fc 100644 --- a/tests/raytracing/rt_smoke_ahs/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs/kernel.cpp @@ -44,8 +44,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Fire ray + wait. Ray flags = 0 (no OPAQUE override; per-triangle flags // drive AHS). No payload. Any-hit candidates return to this warp; resume // with the decided action until a terminal status ends traversal. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp b/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp index 1953f25848..467439e216 100644 --- a/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp @@ -43,8 +43,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Fire ray + wait for terminal. Ray flags = 0 (no OPAQUE override; // per-triangle flags drive AHS). No payload. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp b/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp index cfbe1c7e94..a476597782 100644 --- a/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp @@ -33,20 +33,20 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Ray flags = 0: the non-opaque triangle drives the opacity classifier so // the walker yields an AHS candidate callback. The candidate-capture buffer - // rides the trace as the payload pointer (VX_RT_PAYLOAD_PTR_LO), so the + // rides the trace as the payload pointer (vx_rt_get_payload), so the // loop can locate it from the window. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t cand_lo = (uint32_t)(arg->cand_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, cand_lo, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t capture_ptr = arg->cand_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, capture_ptr, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the candidate instance attributes and the capture buffer pointer // (staged as the trace payload) from the register window, then stash. - uint32_t cand_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t cand_ptr = vx_rt_get_payload(sts); uint32_t cand_custom = vx_rt_get_attr(VX_RT_HIT_INSTANCE_CUSTOM, sts); uint32_t cand_inst = vx_rt_get_attr(VX_RT_HIT_INSTANCE_ID, sts); - uint32_t* cand = (uint32_t*)(uintptr_t)cand_ptr; + uint32_t* cand = (uint32_t*)cand_ptr; cand[0] = cand_custom; // cand->cand_instance_custom cand[1] = cand_inst; // cand->cand_instance_id sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); diff --git a/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp b/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp index f3bf38da34..9e7b7c7b43 100644 --- a/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp @@ -33,20 +33,20 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Ray flags = 0: the non-opaque triangle drives the opacity classifier so // the walker yields an AHS candidate callback. The candidate-capture buffer - // rides the trace as the payload pointer (VX_RT_PAYLOAD_PTR_LO), so the + // rides the trace as the payload pointer (vx_rt_get_payload), so the // loop can locate it from the window. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t cand_lo = (uint32_t)(arg->cand_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, cand_lo, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t capture_ptr = arg->cand_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, capture_ptr, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the candidate geometry/instance attributes and the capture buffer // pointer (staged as the trace payload) from the register window, stash. - uint32_t cand_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t cand_ptr = vx_rt_get_payload(sts); uint32_t cand_geom = vx_rt_get_attr(VX_RT_HIT_GEOMETRY_INDEX, sts); uint32_t cand_inst = vx_rt_get_attr(VX_RT_HIT_INSTANCE_ID, sts); - uint32_t* cand = (uint32_t*)(uintptr_t)cand_ptr; + uint32_t* cand = (uint32_t*)cand_ptr; cand[0] = cand_geom; // cand->cand_geometry cand[1] = cand_inst; // cand->cand_instance_id sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); diff --git a/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp b/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp index a5525d56b0..33ca412327 100644 --- a/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp @@ -37,8 +37,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmax, }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_async_batch/kernel.cpp b/tests/raytracing/rt_smoke_async_batch/kernel.cpp index f7a0f9de1c..a9ce39b2db 100644 --- a/tests/raytracing/rt_smoke_async_batch/kernel.cpp +++ b/tests/raytracing/rt_smoke_async_batch/kernel.cpp @@ -41,10 +41,10 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Issue RTU_ASYNC_NUM_BATCH traces back-to-back. No WAIT in // between — each ray runs async in the cluster's RtuCore. - uint32_t h0 = vx_rt_wtrace((uint32_t)(arg->scene_addr[0] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); - uint32_t h1 = vx_rt_wtrace((uint32_t)(arg->scene_addr[1] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); - uint32_t h2 = vx_rt_wtrace((uint32_t)(arg->scene_addr[2] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); - uint32_t h3 = vx_rt_wtrace((uint32_t)(arg->scene_addr[3] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h0 = vx_rt_wtrace(arg->scene_addr[0], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h1 = vx_rt_wtrace(arg->scene_addr[1], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h2 = vx_rt_wtrace(arg->scene_addr[2], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h3 = vx_rt_wtrace(arg->scene_addr[3], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); // Drain in declared order. Each WAIT blocks until its TERMINAL // delivers and applies the matching hit attrs. diff --git a/tests/raytracing/rt_smoke_bvh6/kernel.cpp b/tests/raytracing/rt_smoke_bvh6/kernel.cpp index 2b81b595bb..ef33e3ba4a 100644 --- a/tests/raytracing/rt_smoke_bvh6/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh6/kernel.cpp @@ -29,8 +29,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp b/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp index 1fecb230e1..c6a731bb9b 100644 --- a/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp @@ -26,8 +26,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp b/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp index 5dc276e80a..4344855937 100644 --- a/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp @@ -23,8 +23,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp b/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp index 0154389871..00001a3025 100644 --- a/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp @@ -23,8 +23,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_chs/common.h b/tests/raytracing/rt_smoke_chs/common.h index e6abfdea09..d501295419 100644 --- a/tests/raytracing/rt_smoke_chs/common.h +++ b/tests/raytracing/rt_smoke_chs/common.h @@ -47,7 +47,7 @@ typedef struct { typedef struct { uint64_t scene_addr; uint64_t results_addr; - uint64_t payload_addr; // CHS reads VX_RT_PAYLOAD_PTR_LO/HI + uint64_t payload_addr; // CHS reads it back with vx_rt_get_payload uint32_t reserved; uint32_t reserved2; float ray_origin[3]; diff --git a/tests/raytracing/rt_smoke_chs/kernel.cpp b/tests/raytracing/rt_smoke_chs/kernel.cpp index c555e3f6a6..bd164b4bc2 100644 --- a/tests/raytracing/rt_smoke_chs/kernel.cpp +++ b/tests/raytracing/rt_smoke_chs/kernel.cpp @@ -37,17 +37,17 @@ __kernel void kernel_main(kernel_arg_t* arg) { // The trace stages the payload pointer the CHS loop body reads via // vx_gfx_get. Opt into CHS dispatch. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, VX_RT_FLAG_ENABLE_CHS, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, VX_RT_FLAG_ENABLE_CHS, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the committed hit_t bits and the payload pointer from the window, // write MAGIC ^ hit_t, then resume: DONE drains the CHS-shaded hit. uint32_t hit_t_bits = vx_rt_get_attr(VX_RT_HIT_T, sts); - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); - *(uint32_t*)(uintptr_t)payload_ptr = RTU_CHS_MAGIC ^ hit_t_bits; + uintptr_t payload_ptr = vx_rt_get_payload(sts); + *(uint32_t*)payload_ptr = RTU_CHS_MAGIC ^ hit_t_bits; sts = vx_rt_continue(h, VX_RT_CB_DONE, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_chs/main.cpp b/tests/raytracing/rt_smoke_chs/main.cpp index c5ee60896d..6e3b95c3ab 100644 --- a/tests/raytracing/rt_smoke_chs/main.cpp +++ b/tests/raytracing/rt_smoke_chs/main.cpp @@ -16,11 +16,13 @@ // Scene: one opaque triangle at z=5. Single lane fires a ray that // hits it. The kernel opts in to CHS via VX_RT_FLAG_ENABLE_CHS and // publishes a per-ray payload buffer pointer through -// VX_RT_PAYLOAD_PTR_LO. RtuCore commits the hit, then queues a +// VX_RT_PAYLOAD_PTR_LO/HI. RtuCore commits the hit, then queues a // CHS yield through the existing reformation/CB_YIELD path. The // dispatcher reads hit_t + payload pointer, writes MAGIC ^ hit_t_bits // to the payload, and exits via vx_rt_cb_ret(VX_RT_CB_DONE) + mret. // Oracle: status=HIT, payload[0] = MAGIC ^ bits(5.0f). +// On a 64-bit device the scene and the payload sit above 4 GiB, so the trace +// and the payload read-back must carry full-width pointers. #include #include @@ -52,6 +54,10 @@ vx_queue_h queue = nullptr; vx_module_h module_ = nullptr; vx_kernel_h kernel = nullptr; kernel_arg_t kernel_arg = {}; +bool high_mem = false; + +constexpr uint64_t HIGH_SCENE_ADDR = 0x140000000ull; +constexpr uint64_t HIGH_PAYLOAD_ADDR = 0x140010000ull; void cleanup() { if (device) { @@ -65,11 +71,23 @@ void cleanup() { } } +static vx_result_t create_buffer(uint64_t high_addr, uint64_t size, + uint32_t flags, vx_buffer_h* buf) { + if (high_mem) { + return vx_buffer_reserve(device, high_addr, size, flags, buf); + } + return vx_buffer_create(device, size, flags, buf); +} + int main(int /*argc*/, char* /*argv*/[]) { RT_CHECK(vx_device_open(0, &device)); vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; RT_CHECK(vx_queue_create(device, &qi, &queue)); + uint64_t isa_flags = 0; + RT_CHECK(vx_device_query(device, VX_CAPS_ISA_FLAGS, &isa_flags)); + high_mem = (VX_ISA_ARCH(isa_flags) == 64); + // Scene: 1 opaque triangle at z=5. std::vector scene_bytes(RTU_SCENE_HDR_BYTES + RTU_TRI_STRIDE_BYTES, 0); uint32_t* hdr = reinterpret_cast(scene_bytes.data()); @@ -83,7 +101,7 @@ int main(int /*argc*/, char* /*argv*/[]) { *tri_flags = RTU_TRI_FLAG_OPAQUE; uint32_t scene_bytes_sz = (uint32_t)scene_bytes.size(); - RT_CHECK(vx_buffer_create(device, scene_bytes_sz, VX_MEM_READ, &scene_buffer)); + RT_CHECK(create_buffer(HIGH_SCENE_ADDR, scene_bytes_sz, VX_MEM_READ, &scene_buffer)); RT_CHECK(vx_buffer_address(scene_buffer, &kernel_arg.scene_addr)); uint32_t res_size = sizeof(rtu_result_t); @@ -93,8 +111,8 @@ int main(int /*argc*/, char* /*argv*/[]) { // Payload buffer: 4 bytes. Pre-init with a sentinel so we can tell // "CHS never ran" from "CHS ran and wrote 0". uint32_t payload_init = 0xdeadbeef; - RT_CHECK(vx_buffer_create(device, sizeof(uint32_t), - VX_MEM_READ_WRITE, &payload_buffer)); + RT_CHECK(create_buffer(HIGH_PAYLOAD_ADDR, sizeof(uint32_t), + VX_MEM_READ_WRITE, &payload_buffer)); RT_CHECK(vx_buffer_address(payload_buffer, &kernel_arg.payload_addr)); RT_CHECK(vx_enqueue_write(queue, payload_buffer, 0, &payload_init, sizeof(payload_init), 0, nullptr, nullptr)); diff --git a/tests/raytracing/rt_smoke_cull_back/kernel.cpp b/tests/raytracing/rt_smoke_cull_back/kernel.cpp index 25d4c99be3..492bccdf0a 100644 --- a/tests/raytracing/rt_smoke_cull_back/kernel.cpp +++ b/tests/raytracing/rt_smoke_cull_back/kernel.cpp @@ -33,8 +33,8 @@ static inline uint32_t fire_ray(uint64_t scene_addr, ray.dir[2] = dir[2]; ray.tmin = tmin; ray.tmax = tmax; - uint32_t scene_lo = (uint32_t)(scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, + uintptr_t scene_ptr = scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE | VX_RT_FLAG_CULL_BACK_FACING, 0xffu, &ray); return vx_rt_wait(h, hit); diff --git a/tests/raytracing/rt_smoke_cull_mask/kernel.cpp b/tests/raytracing/rt_smoke_cull_mask/kernel.cpp index 4e8ce8137b..38723d6ad5 100644 --- a/tests/raytracing/rt_smoke_cull_mask/kernel.cpp +++ b/tests/raytracing/rt_smoke_cull_mask/kernel.cpp @@ -36,12 +36,12 @@ __kernel void kernel_main(kernel_arg_t* arg) { ray.tmin = arg->tmin; ray.tmax = arg->tmax; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uintptr_t scene_ptr = arg->scene_addr; rtu_result_t* results = (rtu_result_t*)((uintptr_t)arg->results_addr); // Each ray: fire+wait with a distinct cull mask, read results. for (uint32_t i = 0; i < RTU_NUM_RAYS; ++i) { - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, arg->ray_cull_mask[i], &ray); + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, arg->ray_cull_mask[i], &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); results[tid].rays[i].status = sts; diff --git a/tests/raytracing/rt_smoke_deep_stack/kernel.cpp b/tests/raytracing/rt_smoke_deep_stack/kernel.cpp index 3f7da83385..f83047fcd8 100644 --- a/tests/raytracing/rt_smoke_deep_stack/kernel.cpp +++ b/tests/raytracing/rt_smoke_deep_stack/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp b/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp index ef23db8f9f..68c1941e68 100644 --- a/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp +++ b/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp b/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp index 3f7da83385..f83047fcd8 100644 --- a/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp +++ b/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_host_cfg/kernel.cpp b/tests/raytracing/rt_smoke_host_cfg/kernel.cpp index b508ae0760..cc94b8342b 100644 --- a/tests/raytracing/rt_smoke_host_cfg/kernel.cpp +++ b/tests/raytracing/rt_smoke_host_cfg/kernel.cpp @@ -27,8 +27,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_inst_flags/kernel.cpp b/tests/raytracing/rt_smoke_inst_flags/kernel.cpp index 2d078fc2cb..39ff1830e2 100644 --- a/tests/raytracing/rt_smoke_inst_flags/kernel.cpp +++ b/tests/raytracing/rt_smoke_inst_flags/kernel.cpp @@ -36,8 +36,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Ray flags = 0: the per-instance FORCE_NO_OPAQUE flag alone drives the // opacity classification, so the opaque triangle yields an AHS callback. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_is/kernel.cpp b/tests/raytracing/rt_smoke_is/kernel.cpp index 0d273775b1..32b56087bd 100644 --- a/tests/raytracing/rt_smoke_is/kernel.cpp +++ b/tests/raytracing/rt_smoke_is/kernel.cpp @@ -32,19 +32,19 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the candidate's callback type + payload pointer from the window. // Write MAGIC for a procedural candidate, a ~MAGIC sentinel otherwise. uint32_t cb_type = vx_rt_get_attr(VX_RT_CB_TYPE, sts); - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t payload_ptr = vx_rt_get_payload(sts); uint32_t val = (cb_type == VX_RT_CB_TYPE_PROC) ? (uint32_t)RTU_IS_MAGIC : ~(uint32_t)RTU_IS_MAGIC; - *(uint32_t*)(uintptr_t)payload_ptr = val; + *(uint32_t*)payload_ptr = val; sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_miss/kernel.cpp b/tests/raytracing/rt_smoke_miss/kernel.cpp index fba90e7b70..b120f863bf 100644 --- a/tests/raytracing/rt_smoke_miss/kernel.cpp +++ b/tests/raytracing/rt_smoke_miss/kernel.cpp @@ -35,15 +35,15 @@ __kernel void kernel_main(kernel_arg_t* arg) { // The trace stages the payload pointer the MISS loop body reads via // vx_gfx_get. Opt into MISS dispatch. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, VX_RT_FLAG_ENABLE_MISS, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, VX_RT_FLAG_ENABLE_MISS, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the payload pointer from the window, write MAGIC, then resume. - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); - *(uint32_t*)(uintptr_t)payload_ptr = RTU_MISS_MAGIC; + uintptr_t payload_ptr = vx_rt_get_payload(sts); + *(uint32_t*)payload_ptr = RTU_MISS_MAGIC; sts = vx_rt_continue(h, VX_RT_CB_DONE, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_multitri/kernel.cpp b/tests/raytracing/rt_smoke_multitri/kernel.cpp index 53b94f3172..bbc13fbff0 100644 --- a/tests/raytracing/rt_smoke_multitri/kernel.cpp +++ b/tests/raytracing/rt_smoke_multitri/kernel.cpp @@ -28,8 +28,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_proc/kernel.cpp b/tests/raytracing/rt_smoke_proc/kernel.cpp index 98e1c4a59d..17cf1d0967 100644 --- a/tests/raytracing/rt_smoke_proc/kernel.cpp +++ b/tests/raytracing/rt_smoke_proc/kernel.cpp @@ -35,9 +35,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uintptr_t scene_ptr = arg->scene_addr; // procedural primitive → IS decides (flags = 0), no payload. - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_recursive/kernel.cpp b/tests/raytracing/rt_smoke_recursive/kernel.cpp index 3cc63860c0..521d7496c1 100644 --- a/tests/raytracing/rt_smoke_recursive/kernel.cpp +++ b/tests/raytracing/rt_smoke_recursive/kernel.cpp @@ -29,7 +29,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { uint32_t tid = blockIdx.x; if (tid != 0) return; - uint32_t sub_scene = (uint32_t)(arg->sub_scene_addr & 0xffffffffu); + uintptr_t sub_scene = arg->sub_scene_addr; vx_ray_t ray = { { arg->ray_origin[0], arg->ray_origin[1], arg->ray_origin[2] }, @@ -37,11 +37,11 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uintptr_t scene_ptr = arg->scene_addr; // payload pointer for the CHS to write sub_status into; enable CHS for // the parent ray (so its closest hit returns a candidate). - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, VX_RT_FLAG_ENABLE_CHS, + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, VX_RT_FLAG_ENABLE_CHS, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); @@ -51,7 +51,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { vx_hit_t sub_hit; uint32_t sub_h = vx_rt_wtrace(sub_scene, 0u, 0u, 0xffu, &ray); uint32_t sub_status = vx_rt_wait(sub_h, &sub_hit); - *(volatile uint32_t*)(uintptr_t)payload = sub_status; + *(volatile uint32_t*)payload = sub_status; sts = vx_rt_continue(h, VX_RT_CB_DONE, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_reform/kernel.cpp b/tests/raytracing/rt_smoke_reform/kernel.cpp index fde753bffa..2339db5a09 100644 --- a/tests/raytracing/rt_smoke_reform/kernel.cpp +++ b/tests/raytracing/rt_smoke_reform/kernel.cpp @@ -36,8 +36,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_reform_mw/kernel.cpp b/tests/raytracing/rt_smoke_reform_mw/kernel.cpp index ed2fca2489..ff89e784d2 100644 --- a/tests/raytracing/rt_smoke_reform_mw/kernel.cpp +++ b/tests/raytracing/rt_smoke_reform_mw/kernel.cpp @@ -35,8 +35,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp b/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp index f260332c21..a7a213c3ca 100644 --- a/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp +++ b/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp @@ -32,7 +32,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { // One shared (warp-uniform) scene; the per-lane ray aims at tri `tid`, so // each lane gets a distinct sbt_idx from the tri it hits — divergence rides // the ray, not the scene pointer. - uint32_t scene_addr = (uint32_t)(arg->scene_base_addr & 0xffffffffu); + uintptr_t scene_addr = arg->scene_base_addr; float ox = (float)tid * RTU_TRI_SPACING + RTU_RAY_XOFF; vx_ray_t ray = { diff --git a/tests/raytracing/rt_smoke_sbt/kernel.cpp b/tests/raytracing/rt_smoke_sbt/kernel.cpp index d9b407b5f1..bcd4ad849c 100644 --- a/tests/raytracing/rt_smoke_sbt/kernel.cpp +++ b/tests/raytracing/rt_smoke_sbt/kernel.cpp @@ -46,9 +46,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { }; // The trace stages the payload pointer the SBT lookup reads via vx_gfx_get. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { @@ -56,9 +56,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { // cb_type slot, load the selected magic, write it to the payload. uint32_t cb_type = vx_rt_get_attr(VX_RT_CB_TYPE, sts); uint32_t sbt_idx = vx_rt_get_attr(VX_RT_HIT_SBT_IDX, sts); - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t payload_ptr = vx_rt_get_payload(sts); uint32_t magic = sbt[sbt_idx * 4 + (cb_type - 1)]; - *(uint32_t*)(uintptr_t)payload_ptr = magic; + *(uint32_t*)payload_ptr = magic; sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_shadow/kernel.cpp b/tests/raytracing/rt_smoke_shadow/kernel.cpp index 6b5e4b21d0..5723306124 100644 --- a/tests/raytracing/rt_smoke_shadow/kernel.cpp +++ b/tests/raytracing/rt_smoke_shadow/kernel.cpp @@ -27,8 +27,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; // Shadow-ray flags: opaque-force + terminate-on-first-hit. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE | VX_RT_FLAG_TERMINATE_ON_FIRST_HIT, 0xffu, &ray); vx_hit_t hit; diff --git a/tests/raytracing/rt_smoke_tlas/kernel.cpp b/tests/raytracing/rt_smoke_tlas/kernel.cpp index 653139aa08..1f20dd3b2b 100644 --- a/tests/raytracing/rt_smoke_tlas/kernel.cpp +++ b/tests/raytracing/rt_smoke_tlas/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp b/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp index d083e6b25d..833667843f 100644 --- a/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp +++ b/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); From 9e90dab10ac17113edf04bab33abe539ae5592d1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 9 Oct 2026 19:37:00 -0700 Subject: [PATCH 004/336] rtu: NaN-box single-precision slot reads into FP registers on RV64 On RV64 with the D extension an FP register is 64 bits, and a single-precision value in it must have its upper word all ones; anything else reads back as the canonical NaN. The core-side RTU unit zero-extended every slot read, so the hit t/u/v that vx_rt_wait (GETWF) lands in f0..f2 arrived as NaN, and the shading that used them went wrong on every hit. rt_raycast failed pixel comparison on RV64 rtlsim (210/768 pixels, maxdiff 172) while simx passed: SimX already NaN-boxes GETWF, as the LSU does for flw and the TCU for its results. The result path now NaN-boxes when the destination is an FP register, the same test the LSU uses; GP reads and RV32 are unchanged. rt_suite-x64 now also runs on rtlsim. RV64 rtlsim: rt_raycast passes with 0/768 mismatched pixels at NT4, NT16, 2-core and 8x8, and its retired instruction count (46136 at NT4) now matches simx. The ray-tracing CI category passes at xlen 64 on simx and rtlsim, and at xlen 32 on rtlsim (suite, model_parity, perf_gate unchanged). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/raytracing.yaml | 4 +++- hw/rtl/rtu/VX_rtu_unit.sv | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/ci/testcases/raytracing.yaml b/ci/testcases/raytracing.yaml index f4b42836d4..734ef894cf 100644 --- a/ci/testcases/raytracing.yaml +++ b/ci/testcases/raytracing.yaml @@ -30,13 +30,15 @@ tests: configs: -DVX_CFG_EXT_RTU_ENABLE # The same suite on RV64. There rt_smoke_chs puts its scene and payload above -# 4 GiB, so a trace that narrows either pointer misses or loses its payload. +# 4 GiB, so a trace that narrows either pointer misses or loses its payload, and +# the FP hit-window reads must come back NaN-boxed. - id: rt_suite-x64 via: make-run xlen: - 64 drivers: - simx + - rtlsim dir: tests/raytracing target: run-{driver} configs: -DVX_CFG_EXT_RTU_ENABLE diff --git a/hw/rtl/rtu/VX_rtu_unit.sv b/hw/rtl/rtu/VX_rtu_unit.sv index 50781fcaf0..cfeb22a315 100644 --- a/hw/rtl/rtu/VX_rtu_unit.sv +++ b/hw/rtl/rtu/VX_rtu_unit.sv @@ -391,8 +391,21 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // write back zero. sfu_result_t rsp_data_out; assign rsp_data_out.header = s1_header; +`ifdef VX_CFG_XLEN_64 +`ifdef VX_CFG_EXT_F_ENABLE + // A single-precision slot read into a 64-bit FP register must be NaN-boxed. + wire rsp_is_float = (get_reg_type(s1_header.rd) == REG_TYPE_F); +`else + wire rsp_is_float = 1'b0; +`endif +`endif for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rsp_data - assign rsp_data_out.data[i] = `VX_CFG_XLEN'(s1_from_ram ? core_rdata[i] : 32'd0); + wire [31:0] rsp_word = s1_from_ram ? core_rdata[i] : 32'd0; + `ifdef VX_CFG_XLEN_64 + assign rsp_data_out.data[i] = rsp_is_float ? {32'hffffffff, rsp_word} : 64'(rsp_word); + `else + assign rsp_data_out.data[i] = rsp_word; + `endif end assign result_if.valid = s1_valid; From 127b03927eee29d5f31aef1039ec09896cb4dd05 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 03:14:20 -0700 Subject: [PATCH 005/336] gfx: store the depth plane modulo 2^32 instead of converting out of range Setup converts the screen-space depth plane Z = A'x + B'y + C' to Q7.24 with a plain float-to-int cast. C' is the plane at the screen origin, so a steep plane on a triangle away from the origin puts it past Q7.24's +-128 range even though the depth at every covered pixel is in range. That conversion is undefined: RISC-V fcvt saturates (wrong depth on the device), x86 cvttss2si returns INT_MIN. gfx_pipeline_kernel failed on primbuf[41] at both xlen 32 and 64 with dev C' = INT_MAX against ref C' = INT_MIN. Late-Z and the raster early-Z (SimX and RTL) all evaluate the plane mod 2^32, so a term stored modulo 2^32 gives the exact depth at the covered pixels. PlaneToFixed keeps the plain cast for in-range terms, wraps an out-of-range term through int64, and maps a non-finite or absurdly steep term (past 2^39, which would overflow int64) to 0 so every input is defined on host and device. gfx_setup_kernel, gfx_binsort_kernel and gfx_pipeline_kernel compare the device front end byte for byte against the host reference but were in no CI category; they now run in graphics at xlen 32 and 64. gfx_pipeline_kernel passes at xlen 32 and 64 (failed at both before). graphics simx 38/38 at xlen 32 and 64, vulkan simx 13/13 at xlen 32 and 64, graphics rtlsim 56/56 at xlen 32 including every perf_gate case unchanged. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/graphics.yaml | 5 +++++ sw/common/gfx_setup.h | 24 +++++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/ci/testcases/graphics.yaml b/ci/testcases/graphics.yaml index 7214c48ba0..493d96b9fd 100644 --- a/ci/testcases/graphics.yaml +++ b/ci/testcases/graphics.yaml @@ -547,6 +547,11 @@ tests: - {id: gfx_pipeline_tex, via: make-run, drivers: [simx], dir: tests/graphics/gfx_pipeline_tex, target: "run-{driver}"} - {id: gfx_pipeline_om, via: make-run, drivers: [simx], dir: tests/graphics/gfx_pipeline_om, target: "run-{driver}"} - {id: gfx_pipeline_raster,via: make-run, drivers: [simx], dir: tests/graphics/gfx_pipeline_raster,target: "run-{driver}"} +# The front-end setup, bin-sort and full pipeline kernels, each checked byte for +# byte against the host reference built from the same sw/gfx and sw/common headers. +- {id: gfx_setup_kernel, via: make-run, drivers: [simx], dir: tests/regression/gfx_setup_kernel, target: "run-{driver}"} +- {id: gfx_binsort_kernel, via: make-run, drivers: [simx], dir: tests/regression/gfx_binsort_kernel, target: "run-{driver}"} +- {id: gfx_pipeline_kernel, via: make-run, drivers: [simx], dir: tests/regression/gfx_pipeline_kernel, target: "run-{driver}"} # ── FF/SW routing matrix ───────────────────────────────────────────────────── # Each fixed-function unit dropped from the build (NO_TEX / NO_OM / NO_RASTER); diff --git a/sw/common/gfx_setup.h b/sw/common/gfx_setup.h index 3698cd84f7..7b9d619d00 100644 --- a/sw/common/gfx_setup.h +++ b/sw/common/gfx_setup.h @@ -155,6 +155,24 @@ static inline bool EdgeEquation(vec3f edges[3], const vec4f& v0, return true; } +// A depth-plane term in Q7.24, stored modulo 2^32. The constant term is the plane +// at the screen origin, which leaves Q7.24's +-128 range for a steep plane even +// though the depth at every covered pixel is in range, and an out-of-range float +// to int conversion is undefined (RISC-V saturates, x86 yields INT_MIN). Late-Z +// and early-Z both evaluate the plane mod 2^32, so the wrapped term is exact. A +// term past 2^39 (non-finite, or a degenerate sliver) has no usable depth. +static inline FloatA PlaneToFixed(float f) { + float scaled = f * 16777216.0f; // 2^24 + float mag = __builtin_fabsf(f); + if (mag < 128.0f) { + return FloatA::make((int32_t)scaled); + } + if (!(mag < 549755813888.0f)) { // 2^39: scaled would overflow int64 + return FloatA::make(0); + } + return FloatA::make((int32_t)(uint32_t)(uint64_t)(int64_t)scaled); +} + // Normalize the edge matrix by 1/maxVal and convert to Q15.16. static inline void EdgeToFixed(vec3e_t out[3], const vec3f in[3]) { float maxVal = __builtin_fabsf(in[0].x); @@ -271,9 +289,9 @@ static inline bool setup_triangle(const setup_vertex_t& v0, // C' evaluated so an integer-pixel MAC samples Z at the pixel center // (X+0.5, Y+0.5) — matching the +0.5 half-pixel offset the coverage edges use. float zc = ps0.z - za * ps0.x - zb * ps0.y + 0.5f * (za + zb); - out_prim.attribs.z.x = FloatA(za); - out_prim.attribs.z.y = FloatA(zb); - out_prim.attribs.z.z = FloatA(zc); + out_prim.attribs.z.x = PlaneToFixed(za); + out_prim.attribs.z.y = PlaneToFixed(zb); + out_prim.attribs.z.z = PlaneToFixed(zc); } // Perspective-correct varyings: premultiply each colour/texcoord by the // vertex's 1/w and carry a separate 1/w plane; the FS interpolates every plane From afd7acb2cc3aeccd035eb4289ef618220783c440 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 03:24:52 -0700 Subject: [PATCH 006/336] kernel: check vx_printf formats, and print vx_putint without a float detour vx_printf and vx_vprintf carried no format attribute, so a kernel could pass a 64-bit value, a pointer or a size_t under a %d and the compiler said nothing; on RV64 that reads the wrong varargs slot. They are now declared format(printf, ...), so -Wall -Werror rejects a mismatched call. tinyprintf already implements the l/ll/z/j/t/h modifiers and %p the checker asks for. Every current caller (vx_spawn.c, tests/kernel/*, tests/regression/printf) is clean at xlen 32 and 64; a deliberately mismatched call is rejected at both. vx_putint copied its int argument into a float before itoa, so any value past 2^24 printed rounded (16777217 printed as 16777216). It now stays an int. Co-Authored-By: Claude Opus 5.5 --- sw/kernel/include/vx_print.h | 4 ++-- sw/kernel/src/vx_print.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sw/kernel/include/vx_print.h b/sw/kernel/include/vx_print.h index 696d297759..6df557009f 100644 --- a/sw/kernel/include/vx_print.h +++ b/sw/kernel/include/vx_print.h @@ -20,8 +20,8 @@ extern "C" { #endif -int vx_vprintf(const char* format, va_list va); -int vx_printf(const char * format, ...); +int vx_vprintf(const char* format, va_list va) __attribute__((format(printf, 1, 0))); +int vx_printf(const char * format, ...) __attribute__((format(printf, 1, 2))); void vx_putchar(int c); void vx_putint(int value, int base); diff --git a/sw/kernel/src/vx_print.c b/sw/kernel/src/vx_print.c index e23f0719ad..1056353031 100644 --- a/sw/kernel/src/vx_print.c +++ b/sw/kernel/src/vx_print.c @@ -43,7 +43,7 @@ typedef struct { static void __putint_cb(const putint_arg_t* arg) { char tmp[33]; - float value = arg->value; + int value = arg->value; int base = arg->base; itoa(value, tmp, base); for (int i = 0; i < 33; ++i) { From 2a8e29d92d9412a9e8604be7e0e8e1b0f571dba1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 03:37:39 -0700 Subject: [PATCH 007/336] sgemm_tcu_wg_dxa: widen the SW_LOAD_B index math, build SW_LOAD_A + SW_LOAD_B The SW_LOAD_B cooperative load still computed its indices in uint32_t, the pattern the RV64 index-portability change replaced in the SW_LOAD_A loop next to it; it now uses size_t (the same type as uint32_t on RV32). RV64 simx at the dxa sgemm_tcu_wg_dxa-1 config, 8 warps x 32 threads: SW_LOAD_A + SW_LOAD_B 28832 vs 29141 cycles (-1.1%), SW_LOAD_B alone 23167 vs 23160 (flat). With both loads in software no DXA transfer is issued, is_dxa_warp goes unused, and -Werror rejected the build at xlen 32 and 64; it is now [[maybe_unused]], like the descriptor ids above it. RV32 code is identical with SW_LOAD_B, and the default (DXA) build CI runs is identical at xlen 32 and 64. Co-Authored-By: Claude Opus 5.5 --- tests/regression/sgemm_tcu_wg_dxa/kernel.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp index 281bd1aa77..7b2596dabb 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp @@ -75,7 +75,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fill_fragment(fragC, 0); // Only the first warp in the CTA issues DXA commands. - const bool is_dxa_warp = (get_sub_group_id() == 0); + [[maybe_unused]] const bool is_dxa_warp = (get_sub_group_id() == 0); // Stage the tile at K-offset k into the {A_smem, B_smem} buffer guarded // by barrier id `bar`. All state is passed by value in registers. @@ -109,10 +109,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #ifdef SW_LOAD_B // Cooperative load B block-major (matches DXA BlockMajor LAYOUT / bbuf). uint32_t b_size = ctx::tileK * ctx::xtileN; - for (uint32_t i = 0; i < b_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::xtileN; - uint32_t c = idx % ctx::xtileN; + for (size_t i = 0; i < b_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::xtileN; + size_t c = idx % ctx::xtileN; B_smem[ctx::b_blockmajor_idx(r, c)] = pB[(k + r) * N + (tile_col + c)]; } #endif From 5d224d8f68a030d5fd9dc218b53db0165ead5f5e Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 06:27:34 -0700 Subject: [PATCH 008/336] tests: widen the index math of demo, fence, dogfood, softmax and sgemm_tcu These five kernels kept uint32_t indices in cbd9cbee8 because widening them made RV64 slower: LLVM's LoopTermFold moved each loop's uniform exit test onto a per-thread pointer, which needs divergence predication every iteration. llvm_vortex a81a5957f406 keeps a uniform exit uniform on targets with branch divergence, and with it the size_t form is the faster one on RV64 simx: demo -20.4%, fence -20.6%, dogfood -14.6%, softmax -10.7%, sgemm_tcu -8.6% cycles. RV32 code is identical (size_t is uint32_t there). This depends on that compiler change: with the current toolchain the same kernels are 1.4-3.8% slower on RV64 (demo +1.8%, fence +1.4%, dogfood +1.5%, softmax +3.8%, sgemm_tcu +2.1%). It must land after the toolchain prebuilt carrying a81a5957f406, together with the perf baselines that compiler moves. The other kernels left narrow in cbd9cbee8 stay so: sgemv (+5.0%), dotproduct (+1.5%), sgemm_tcu_mx (+3.7%), sgemm_tcu_sp_mx (+1.8%) and sgemm_tcu_wg (+1.3%) are still slower widened with the new compiler, and vm_test is unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/regression/demo/kernel.cpp | 8 +-- tests/regression/dogfood/kernel.cpp | 90 +++++++++++++-------------- tests/regression/fence/kernel.cpp | 8 +-- tests/regression/sgemm_tcu/kernel.cpp | 10 +-- tests/regression/softmax/kernel.cpp | 10 +-- 5 files changed, 63 insertions(+), 63 deletions(-) diff --git a/tests/regression/demo/kernel.cpp b/tests/regression/demo/kernel.cpp index 2199d70fa0..a4f5467bb4 100644 --- a/tests/regression/demo/kernel.cpp +++ b/tests/regression/demo/kernel.cpp @@ -11,11 +11,11 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // the flat tid = y*bdx+x reconstruction. uint32_t gx = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gy = blockIdx.y * blockDim.y + threadIdx.y; - uint32_t gid = gy * arg->dim_x + gx; - uint32_t count = arg->task_size; - uint32_t offset = gid * count; + size_t gid = gy * arg->dim_x + gx; + size_t count = arg->task_size; + size_t offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset+i] = src0_ptr[offset+i] + src1_ptr[offset+i]; } } diff --git a/tests/regression/dogfood/kernel.cpp b/tests/regression/dogfood/kernel.cpp index 889a063f50..a5ffa38a22 100644 --- a/tests/regression/dogfood/kernel.cpp +++ b/tests/regression/dogfood/kernel.cpp @@ -10,14 +10,14 @@ inline float __ieee754_sqrtf (float x) { } void kernel_iadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { int32_t a = src0_ptr[offset+i]; int32_t b = src1_ptr[offset+i]; int32_t c = a + b; @@ -26,14 +26,14 @@ void kernel_iadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_imul(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b; @@ -42,14 +42,14 @@ void kernel_imul(kernel_arg_t* __UNIFORM__ arg) { } void kernel_idiv(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -58,14 +58,14 @@ void kernel_idiv(kernel_arg_t* __UNIFORM__ arg) { } void kernel_idiv_mul(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -76,14 +76,14 @@ void kernel_idiv_mul(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { float a = src0_ptr[offset+i]; float b = src1_ptr[offset+i]; float c = a + b; @@ -92,14 +92,14 @@ void kernel_fadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fsub(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a - b; @@ -108,14 +108,14 @@ void kernel_fsub(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fmul(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b; @@ -124,14 +124,14 @@ void kernel_fmul(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fmadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b + b; @@ -140,14 +140,14 @@ void kernel_fmadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fmsub(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b - b; @@ -156,14 +156,14 @@ void kernel_fmsub(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fnmadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c =-a * b - b; @@ -172,14 +172,14 @@ void kernel_fnmadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fnmsub(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c =-a * b + b; @@ -188,14 +188,14 @@ void kernel_fnmsub(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fnmadd_madd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c =-a * b - b; @@ -206,14 +206,14 @@ void kernel_fnmadd_madd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fdiv(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -222,14 +222,14 @@ void kernel_fdiv(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fdiv2(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -240,14 +240,14 @@ void kernel_fdiv2(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fsqrt(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = __ieee754_sqrtf(a * b); @@ -256,14 +256,14 @@ void kernel_fsqrt(kernel_arg_t* __UNIFORM__ arg) { } void kernel_ftoi(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -273,14 +273,14 @@ void kernel_ftoi(kernel_arg_t* __UNIFORM__ arg) { } void kernel_ftou(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (uint32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -290,14 +290,14 @@ void kernel_ftou(kernel_arg_t* __UNIFORM__ arg) { } void kernel_itof(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -307,14 +307,14 @@ void kernel_itof(kernel_arg_t* __UNIFORM__ arg) { } void kernel_utof(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -328,14 +328,14 @@ inline float fclamp(float a, float b, float c) { } void kernel_fclamp(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; dst_ptr[offset+i] = fclamp(1.0f, a, b); @@ -347,14 +347,14 @@ inline int iclamp(int a, int b, int c) { } void kernel_iclamp(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int*)arg->src0_addr; auto src1_ptr = (int*)arg->src1_addr; auto dst_ptr = (int*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; dst_ptr[offset+i] = iclamp(1, a, b); @@ -362,14 +362,14 @@ void kernel_iclamp(kernel_arg_t* __UNIFORM__ arg) { } void kernel_trigo(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { - uint32_t j = offset + i; + for (size_t i = 0; i < count; ++i) { + size_t j = offset + i; auto a = src0_ptr[j]; auto b = src1_ptr[j]; auto c = a * b; diff --git a/tests/regression/fence/kernel.cpp b/tests/regression/fence/kernel.cpp index 8d0c8576ea..1e7a6ba47f 100644 --- a/tests/regression/fence/kernel.cpp +++ b/tests/regression/fence/kernel.cpp @@ -2,14 +2,14 @@ #include "common.h" __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - uint32_t count = arg->task_size; + size_t count = arg->task_size; int32_t* src0_ptr = (int32_t*)arg->src0_addr; int32_t* src1_ptr = (int32_t*)arg->src1_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t offset = gid * count; + for (size_t i = 0; i < count; ++i) { dst_ptr[offset+i] = src0_ptr[offset+i] + src1_ptr[offset+i]; } diff --git a/tests/regression/sgemm_tcu/kernel.cpp b/tests/regression/sgemm_tcu/kernel.cpp index 32cfd94d35..b3f8c8d2d9 100644 --- a/tests/regression/sgemm_tcu/kernel.cpp +++ b/tests/regression/sgemm_tcu/kernel.cpp @@ -11,16 +11,16 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pB = reinterpret_cast(arg->B_addr); auto pC = reinterpret_cast(arg->C_addr); - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; ctx::fragment_a fragA; ctx::fragment_b fragB; ctx::fragment_acc fragC; // calculate tile row & column based on block index - uint32_t tile_row = blockIdx.y * ctx::tileM; - uint32_t tile_col = blockIdx.x * ctx::tileN; + size_t tile_row = blockIdx.y * ctx::tileM; + size_t tile_col = blockIdx.x * ctx::tileN; // Initialize accumulator tile to zero ctx::fill_fragment(fragC, 0); @@ -29,7 +29,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t cycles = 0; #endif - for (uint32_t i = 0; i < K; i += ctx::tileK) { + for (size_t i = 0; i < K; i += ctx::tileK) { auto pTileA = pA + tile_row * K + i; auto pTileB = pB + tile_col * K + i; diff --git a/tests/regression/softmax/kernel.cpp b/tests/regression/softmax/kernel.cpp index 92fec90970..4d2a0ea5ac 100644 --- a/tests/regression/softmax/kernel.cpp +++ b/tests/regression/softmax/kernel.cpp @@ -5,30 +5,30 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto src0_ptr = reinterpret_cast(arg->src0_addr); auto dst_ptr = reinterpret_cast(arg->dst_addr); - uint32_t num_cols = arg->num_cols; + size_t num_cols = arg->num_cols; uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; if (row >= arg->num_rows) return; - uint32_t offset = row * num_cols; + size_t offset = row * num_cols; // 1. Find the max value TYPE max = 0.0; - for (uint32_t i = 0; i < num_cols; i++) { + for (size_t i = 0; i < num_cols; i++) { auto curr = src0_ptr[offset + i]; if (curr > max) max = curr; } // 2. Subtract max and compute exp TYPE sum = 0.0; - for (uint32_t i = 0; i < num_cols; i++) { + for (size_t i = 0; i < num_cols; i++) { auto computed = exp(src0_ptr[offset + i] - max); src0_ptr[offset + i] = computed; sum += computed; } // 3. Normalize - for (uint32_t i = 0; i < num_cols; i++) { + for (size_t i = 0; i < num_cols; i++) { dst_ptr[offset + i] = src0_ptr[offset + i] / sum; } } From 6c26b424ae66eccd522bcfd89e360254ce8f377e Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 15:50:48 -0700 Subject: [PATCH 009/336] rtu: carry the full scene address through the ray log and rt_replay Code merged from master after the RTU pointer widening narrowed the scene again on RV64: - vx_rt_wtrace pinned the gathered trace config in a uint32_t x31 register, truncating the scene base it had just gathered at full width; - the SimX ray log stored scene_root in 32 bits, so a snapshot of a scene above 4 GiB read the wrong memory, and rt_replay replayed it against that address; - rt_smoke_ahs_multi masked its scene pointer to the low word. The ray-log record keeps its 112-byte size (scene_root takes one reserved word); the format version goes from 1 to 2 since the field offsets change. Co-Authored-By: Claude Opus 5.5 --- sim/simx/rtu/rtu_raylog.cpp | 10 +++++----- sim/simx/rtu/rtu_raylog.h | 6 +++--- sw/kernel/include/vx_raytrace.h | 2 +- tests/raytracing/rt_replay/common.h | 10 +++++----- tests/raytracing/rt_replay/kernel.cpp | 2 +- tests/raytracing/rt_replay/main.cpp | 6 +++--- tests/raytracing/rt_smoke_ahs_multi/kernel.cpp | 4 ++-- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sim/simx/rtu/rtu_raylog.cpp b/sim/simx/rtu/rtu_raylog.cpp index 4281972b50..42592e10ec 100644 --- a/sim/simx/rtu/rtu_raylog.cpp +++ b/sim/simx/rtu/rtu_raylog.cpp @@ -158,13 +158,13 @@ class Logger { static constexpr uint64_t kInstTableSpan = 64 * 1024; static constexpr uint64_t kMaxSceneBytes = 1ull << 30; - void snapshot_scene(uint32_t root) { + void snapshot_scene(uint64_t root) { if (ram_ == nullptr || full() || !snapped_.insert(root).second) return; const RAM& ram = *ram_; uint32_t scene_bytes = 0; for (int i = 0; i < 4; ++i) scene_bytes |= uint32_t(ram[root + 8 + i]) << (8 * i); const uint64_t lo = (root > kInstTableSpan ? root - kInstTableSpan : 0) & kRtuLineMask; - const uint64_t hi = uint64_t(root) + std::min(scene_bytes, kMaxSceneBytes); + const uint64_t hi = root + std::min(scene_bytes, kMaxSceneBytes); uint64_t added = 0; for (uint64_t a = lo; a < hi; a += VX_CFG_MEM_BLOCK_SIZE) { if (image_.count(a)) continue; @@ -184,12 +184,12 @@ class Logger { std::fwrite(&r, sizeof(r), 1, fp_); ++added; } - std::fprintf(stderr, "[rtu-raylog] scene 0x%x: %u bytes, snapshot %llu lines\n", - root, scene_bytes, (unsigned long long)added); + std::fprintf(stderr, "[rtu-raylog] scene 0x%llx: %u bytes, snapshot %llu lines\n", + (unsigned long long)root, scene_bytes, (unsigned long long)added); } const RAM* ram_ = nullptr; - std::unordered_set snapped_; + std::unordered_set snapped_; FILE* fp_ = nullptr; unsigned long long max_rays_ = 4ull << 20; unsigned long long every_ = 1; diff --git a/sim/simx/rtu/rtu_raylog.h b/sim/simx/rtu/rtu_raylog.h index d9b284cee1..b73b2d834b 100644 --- a/sim/simx/rtu/rtu_raylog.h +++ b/sim/simx/rtu/rtu_raylog.h @@ -47,7 +47,7 @@ class RAM; namespace rtu { namespace raylog { constexpr uint32_t kMagic = 0x4C525856; // "VXRL" -constexpr uint32_t kVersion = 1; +constexpr uint32_t kVersion = 2; enum RecType : uint32_t { REC_LINE = 1, @@ -80,7 +80,7 @@ struct RaylogRay { uint32_t info; uint32_t seq; uint32_t epoch; - uint32_t scene_root; + uint64_t scene_root; uint32_t ray_flags; uint32_t cull_mask; uint32_t warp_lane; // lane | warp << 8 | slot << 16 @@ -97,7 +97,7 @@ struct RaylogRay { uint32_t inst_id; uint32_t inst_custom; uint32_t hit_attr; - uint32_t reserved[3]; + uint32_t reserved[2]; }; static_assert(sizeof(RaylogRay) == 112, "RaylogRay layout"); diff --git a/sw/kernel/include/vx_raytrace.h b/sw/kernel/include/vx_raytrace.h index ea9e63e5bb..fe092afd1a 100644 --- a/sw/kernel/include/vx_raytrace.h +++ b/sw/kernel/include/vx_raytrace.h @@ -184,7 +184,7 @@ uint32_t vx_rt_wtrace(uintptr_t scene_ptr, uintptr_t payload_ptr, // the field references stable across the long register-binding list. // The RTU reads the packed config lanes regardless of the thread mask, so it // reads them from the warp-gather register itself (see __VX_WGATHER_IN). - register uint32_t cfg_reg __asm__("x31") = cfg; + register size_t cfg_reg __asm__("x31") = cfg; __asm__ volatile (".insn r %[op], 7, 0, %[hnd], %[cfg], x0" : [hnd]"=r"(handle) : [op]"i"(RISCV_CUSTOM1), [cfg]"r"(cfg_reg), diff --git a/tests/raytracing/rt_replay/common.h b/tests/raytracing/rt_replay/common.h index 8f94077e7b..e696d13641 100644 --- a/tests/raytracing/rt_replay/common.h +++ b/tests/raytracing/rt_replay/common.h @@ -23,10 +23,9 @@ // One ray as the kernel consumes it. Each warp's rays share scene / flags / // cull: those ride the warp-uniform half of vx_rt_wtrace. typedef struct { - uint32_t scene; + uint64_t scene; uint32_t flags; uint32_t cull; - uint32_t pad; float origin[3]; float dir[3]; float tmin; @@ -57,7 +56,7 @@ typedef struct { #ifdef __cplusplus // Ray-log stream layout (mirror of sim/simx/rtu/rtu_raylog.h). #define RAYLOG_MAGIC 0x4C525856u -#define RAYLOG_VERSION 1u +#define RAYLOG_VERSION 2u #define RAYLOG_REC_LINE 1u #define RAYLOG_REC_RAY 2u #define RAYLOG_INFO_REPLAYABLE (1u << 8) @@ -69,11 +68,12 @@ struct RaylogHeader { struct RaylogRay { uint32_t type, info, seq, epoch; - uint32_t scene_root, ray_flags, cull_mask, warp_lane; + uint64_t scene_root; + uint32_t ray_flags, cull_mask, warp_lane; uint32_t origin[3], dir[3]; uint32_t tmin, tmax; uint32_t status, hit_t, hit_u, hit_v, prim, geom, inst_id, inst_custom, hit_attr; - uint32_t reserved[3]; + uint32_t reserved[2]; }; static_assert(sizeof(RaylogRay) == 112, "RaylogRay layout"); #endif diff --git a/tests/raytracing/rt_replay/kernel.cpp b/tests/raytracing/rt_replay/kernel.cpp index a6cf6922c5..24b6434c95 100644 --- a/tests/raytracing/rt_replay/kernel.cpp +++ b/tests/raytracing/rt_replay/kernel.cpp @@ -27,7 +27,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { vx_ray_t ray = { {r.origin[0], r.origin[1], r.origin[2]}, {r.dir[0], r.dir[1], r.dir[2]}, r.tmin, r.tmax }; - uint32_t h = vx_rt_wtrace(r.scene, 0u, r.flags, r.cull, &ray); + uint32_t h = vx_rt_wtrace((uintptr_t)r.scene, 0u, r.flags, r.cull, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); uint32_t yielded = 0; diff --git a/tests/raytracing/rt_replay/main.cpp b/tests/raytracing/rt_replay/main.cpp index e326bcecfe..cf0dd113c7 100644 --- a/tests/raytracing/rt_replay/main.cpp +++ b/tests/raytracing/rt_replay/main.cpp @@ -164,9 +164,9 @@ bool matches(const RaylogRay& ref, const replay_result_t& got) { } void print_ray(FILE* fp, uint64_t idx, const RaylogRay& r, const replay_result_t& g, const char* cat) { - fprintf(fp, "%llu seq=%u cat=%s epoch=%u scene=0x%x flags=0x%x cull=0x%x" + fprintf(fp, "%llu seq=%u cat=%s epoch=%u scene=0x%llx flags=0x%x cull=0x%x" " o=(%a,%a,%a) d=(%a,%a,%a) tmin=%a tmax=%a\n", - (unsigned long long)idx, r.seq, cat, r.epoch, r.scene_root, r.ray_flags, r.cull_mask, + (unsigned long long)idx, r.seq, cat, r.epoch, (unsigned long long)r.scene_root, r.ray_flags, r.cull_mask, fbits(r.origin[0]), fbits(r.origin[1]), fbits(r.origin[2]), fbits(r.dir[0]), fbits(r.dir[1]), fbits(r.dir[2]), fbits(r.tmin), fbits(r.tmax)); @@ -338,7 +338,7 @@ int main(int argc, char** argv) { // Group by the warp-uniform trace config; pad each group to whole warps // with copies of its last ray (their results are dropped). - std::map, std::vector> groups; + std::map, std::vector> groups; for (size_t k = pos; k < end; ++k) { const RaylogRay& r = rays[sel[k]]; groups[{r.scene_root, r.ray_flags & ~(VX_RT_FLAG_ENABLE_CHS | VX_RT_FLAG_ENABLE_MISS), r.cull_mask}].push_back(sel[k]); diff --git a/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp b/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp index fdbe122e8d..96caf8da71 100644 --- a/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp @@ -35,8 +35,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { rtu_result_t* res = (rtu_result_t*)((uintptr_t)arg->results_addr) + tid; uint32_t n = 0; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { From 78dbecfa79315c2ace2febbb04fbfbe6cd288195 Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 6 Oct 2026 02:12:40 -0700 Subject: [PATCH 010/336] config: hardware profiles shared by every flow A profile (profiles/.json) names a complete machine: per-core knobs, chip-level knobs, the platform around it, the measurements it is calibrated against, and the sources of every value. gen_config.py layers it between the VX_config.toml defaults and explicit -D flags, validates each knob against the TOML (unknown knobs, enum values, bool/int types, required sources) and folds it into CONFIGS with --merge-configs, one -D per macro so -Werror builds never see a redefinition. Profiles inherit with "extends". config.mk does the merge once, so every flow that includes it -- simulators, runtimes, kernels, tests and the synthesis Makefiles -- builds the same machine without per-flow changes. configure --profile sets a build directory's default and bakes it into the generated headers and the config stamp; blackbox.sh --profile selects one per run. Without a profile the generated flags and headers are byte-identical to before. Adds profiles/rtx_5080.json (GB203 at base clock) with the knobs the model has today. Co-Authored-By: Claude Opus 5.5 --- ci/blackbox.sh | 3 +- ci/gen_config.py | 147 +++++++++++++++++++++++++++++++++++++++++ config.mk.in | 11 +++ configure | 16 ++++- profiles/rtx_5080.json | 45 +++++++++++++ 5 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 profiles/rtx_5080.json diff --git a/ci/blackbox.sh b/ci/blackbox.sh index f4072e9ada..21215167a0 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -19,7 +19,7 @@ ROOT_DIR=$SCRIPT_DIR/.. show_usage() { echo "Vortex BlackBox Test Driver v1.0" - echo "Usage: $0 [[--clusters=#n] [--cores=#n] [--warps=#n] [--threads=#n] [--l2cache] [--l3cache] [[--driver=#name] [--target=#name] [--app=#app] [--args=#args] [--debug=#level] [--scope] [--vcd] [--saif] [--perf=#class] [--vcd_file=#file] [--saif_file=#file] [--log=logfile] [--nohup] [--help]]" + echo "Usage: $0 [[--profile=#name] [--clusters=#n] [--cores=#n] [--warps=#n] [--threads=#n] [--l2cache] [--l3cache] [[--driver=#name] [--target=#name] [--app=#app] [--args=#args] [--debug=#level] [--scope] [--vcd] [--saif] [--perf=#class] [--vcd_file=#file] [--saif_file=#file] [--log=logfile] [--nohup] [--help]]" } show_help() @@ -66,6 +66,7 @@ parse_args() { for i in "$@"; do case $i in --driver=*) DRIVER=${i#*=} ;; + --profile=*) export PROFILE=${i#*=} ;; --target=*) TARGET=${i#*=} ;; --app=*) APP=${i#*=} ;; --clusters=*) CONFIGS=$(add_option "$CONFIGS" "-DVX_CFG_NUM_CLUSTERS=${i#*=}") ;; diff --git a/ci/gen_config.py b/ci/gen_config.py index 1dd4c83ced..ea3436c49d 100755 --- a/ci/gen_config.py +++ b/ci/gen_config.py @@ -39,6 +39,9 @@ definitions that can be overridden from -D flags. - Resolved mode (-r/--resolved): fully evaluates expressions and emits resolved constants (always enabled for cflags). + - Hardware profiles (--profile): a named machine in profiles/.json layered + between the TOML defaults and explicit -D flags. --merge-configs prints the + caller's CONFIGS with the profile folded in, one -D per macro. """ from __future__ import annotations @@ -46,6 +49,7 @@ import argparse import ast import copy +import json import os import re import shlex @@ -468,6 +472,124 @@ def set_enum(name: str, val: Any) -> None: return overrides, explicit +# ----------------------------- +# Hardware profiles +# ----------------------------- + +PROFILE_SECTIONS = ("core", "chip", "platform") +PROFILE_FIELDS = {"name", "extends", "description", "reference", "sources"} | set(PROFILE_SECTIONS) + + +@dataclass +class Profile: + name: str + chain: List[str] # base first + values: Dict[str, Any] # VX_CFG_* knob -> literal value + sections: Dict[str, Set[str]] # section -> knobs it sets + reference: Dict[str, Any] + sources: Any + + +def _profile_path(name: str, profiles_dir: str) -> str: + if name.endswith(".json") and os.path.isfile(name): + return name + return os.path.join(profiles_dir, f"{name}.json") + + +def load_profile(name: str, profiles_dir: str, _seen: Tuple[str, ...] = ()) -> Profile: + if name in _seen: + raise ValueError(f"Profile inheritance cycle: {' -> '.join(_seen + (name,))}") + path = _profile_path(name, profiles_dir) + if not os.path.isfile(path): + raise ValueError(f"Unknown profile '{name}' (no {path})") + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + unknown = set(data.keys()) - PROFILE_FIELDS + if unknown: + raise ValueError(f"Profile '{name}': unknown field(s) {sorted(unknown)} (allowed: {sorted(PROFILE_FIELDS)})") + + if "extends" in data: + base = load_profile(data["extends"], profiles_dir, _seen + (name,)) + else: + base = Profile(name="", chain=[], values={}, sections={s: set() for s in PROFILE_SECTIONS}, + reference={}, sources=None) + + values = dict(base.values) + sections = {s: set(base.sections[s]) for s in PROFILE_SECTIONS} + for sec in PROFILE_SECTIONS: + for k, v in data.get(sec, {}).items(): + for other in PROFILE_SECTIONS: + sections[other].discard(k) + values[k] = v + sections[sec].add(k) + reference = dict(base.reference) + reference.update(data.get("reference", {})) + sources = data.get("sources", base.sources) + return Profile(name=data.get("name", name), chain=base.chain + [name], values=values, + sections=sections, reference=reference, sources=sources) + + +def validate_profile(prof: Profile, toml_defs: Dict[str, Any], enums: Dict[str, EnumSpec], + read_only: Set[str]) -> None: + if not prof.sources: + raise ValueError(f"Profile '{prof.name}': 'sources' is required (where every value comes from)") + for k, v in prof.values.items(): + if k in read_only: + raise ValueError(f"Profile '{prof.name}': {k} is a read-only [[param]]/[[builtin]] symbol") + if k not in toml_defs and k not in enums: + raise ValueError(f"Profile '{prof.name}': unknown knob {k} (not in VX_config.toml)") + if not _has_public_scope(k): + raise ValueError(f"Profile '{prof.name}': {k} is a private TOML local") + if k in enums: + if v not in enums[k].values: + raise ValueError(f"Profile '{prof.name}': invalid {k}={v!r} (allowed: {enums[k].values})") + elif k.endswith("_ENABLE"): + if not isinstance(v, bool): + raise ValueError(f"Profile '{prof.name}': {k} must be true or false") + elif isinstance(v, bool) or not isinstance(v, (int, str)): + raise ValueError(f"Profile '{prof.name}': {k} must be an integer or a string literal") + elif isinstance(v, str) and _is_expr_string(v): + raise ValueError(f"Profile '{prof.name}': {k} must be a literal; expressions belong in VX_config.toml") + + +def profile_defines(prof: Profile) -> List[str]: + toks: List[str] = [f"-DVX_PROFILE={prof.name}"] + for k in sorted(prof.values): + v = prof.values[k] + if isinstance(v, bool): + toks.append(f"-D{k}" if v else f"-D{k[:-len('_ENABLE')]}_DISABLE") + else: + toks.append(f"-D{k}={v}") + return toks + + +def _canonical_define(tok: str, enums: Dict[str, EnumSpec]) -> Optional[str]: + # The knob a -D token sets, so X_DISABLE and X_ENABLE, or X_ and X=, + # are recognised as the same macro. + defs = _parse_defines([tok]) + if not defs: + return None + d = defs[0] + if d.name.endswith("_DISABLE"): + return d.name[:-len("_DISABLE")] + "_ENABLE" + if d.name in enums: + return d.name + if d.value is None and "_" in d.name: + base, _ = d.name.rsplit("_", 1) + if base in enums: + return base + return d.name + + +def merge_configs(prof: Profile, cflags: str, enums: Dict[str, EnumSpec]) -> str: + explicit = shlex.split(cflags) if cflags else [] + given = {_canonical_define(t, enums) for t in explicit} + given.discard(None) + merged = [t for t in profile_defines(prof) if _canonical_define(t, enums) not in given] + # The caller's flags pass through verbatim, quoting included. + return " ".join(merged + ([cflags.strip()] if cflags.strip() else [])) + + # ----------------------------- # Dialect helpers # ----------------------------- @@ -1489,6 +1611,11 @@ def main(argv: List[str]) -> int: ap.add_argument("--output", "-o", default=None) ap.add_argument("--resolved", "-r", action="store_true", help="Emit resolved constants for cpp/verilog (cflags is always resolved)") + ap.add_argument("--profile", default="", + help="hardware profile: profiles/.json next to the config, or a .json path") + ap.add_argument("--profiles-dir", default=None, help="profile directory (default: /profiles)") + ap.add_argument("--merge-configs", action="store_true", + help="print --cflags with the profile folded in (explicit -D flags win) and exit") args, unknown = ap.parse_known_args(argv) hex_meta = _scan_hex_literals(args.config) @@ -1501,6 +1628,26 @@ def main(argv: List[str]) -> int: toml_defs, layout = _flatten_with_layout(toml_data) + prof: Optional[Profile] = None + if args.profile and args.profile != "none": + profiles_dir = args.profiles_dir or os.path.join(os.path.dirname(os.path.abspath(args.config)), "profiles") + prof = load_profile(args.profile, profiles_dir) + validate_profile(prof, toml_defs, enums, params_set | set(builtins.keys())) + + if args.merge_configs: + out = merge_configs(prof, args.cflags, enums) if prof else args.cflags + sys.stdout.write(out.strip() + "\n") + return 0 + + if prof is not None: + if args.format == "cflags" or args.resolved: + # Resolved outputs see the profile as the lowest-priority overrides. + args.cflags = merge_configs(prof, args.cflags, enums) + else: + # Unresolved headers carry the profile as their overridable defaults. + for k, v in prof.values.items(): + toml_defs[k] = v + defs: List[Define] = [] defs += _parse_defines_from_cflags(args.cflags) defs += _parse_defines(unknown) diff --git a/config.mk.in b/config.mk.in index 8fe66bbd0d..1dc8f4efbd 100644 --- a/config.mk.in +++ b/config.mk.in @@ -55,3 +55,14 @@ SIM_COMMON_DIR ?= $(VORTEX_HOME)/sim/common # V80 runtime install prefix; only consulted by the aved backend when it # targets real hardware or the vendor simulator. VRT_HOME ?= /usr/local + +# Hardware profile (profiles/.json), folded into CONFIGS once here so every +# flow that includes config.mk builds the same machine. Explicit -D flags in +# CONFIGS win over the profile. +PROFILE ?= @PROFILE@ +ifneq ($(strip $(PROFILE)),) +override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --merge-configs --cflags='$(CONFIGS)') +ifneq ($(.SHELLSTATUS),0) +$(error invalid hardware profile '$(PROFILE)') +endif +endif diff --git a/configure b/configure index f9340d12ee..c5e1e21147 100755 --- a/configure +++ b/configure @@ -77,7 +77,7 @@ copy_files() { continue fi mkdir -p "$dest_dir" - sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g" "$file" > "$dest_file" + sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g; s|@PROFILE@|$PROFILE|g" "$file" > "$dest_file" # apply permissions to bash scripts read -r firstline < "$dest_file" if [[ "$firstline" =~ ^#!.*bash ]]; then @@ -140,6 +140,7 @@ default_xlen=32 default_tooldir=$HOME/tools default_osversion=$(detect_osversion) default_prefix=$CURRENT_DIR/install +default_profile= # load default configuration parameters from existing config.mk if [ -f "config.mk" ]; then @@ -152,6 +153,7 @@ if [ -f "config.mk" ]; then TOOLDIR\ ?*) default_tooldir=${value//\?=/} ;; OSVERSION\ ?*) default_osversion=${value//\?=/} ;; PREFIX\ ?*) default_prefix=${value//\?=/} ;; + PROFILE\ ?*) default_profile=${value//\?=/} ;; esac done < config.mk fi @@ -161,6 +163,7 @@ XLEN=${XLEN:=$default_xlen} TOOLDIR=${TOOLDIR:=$default_tooldir} OSVERSION=${OSVERSION:=$default_osversion} PREFIX=${PREFIX:=$default_prefix} +PROFILE=${PROFILE-$default_profile} # parse command line arguments usage() { @@ -169,6 +172,7 @@ usage() { echo " --tooldir= Set the TOOLDIR path (default: $HOME/tools)" echo " --osversion= Set the OS Version (default: $(detect_osversion))" echo " --prefix=, --installdir= Set installation directory" + echo " --profile= Default hardware profile, profiles/.json (default: none)" exit 1 } while [[ "$#" -gt 0 ]]; do @@ -177,12 +181,19 @@ while [[ "$#" -gt 0 ]]; do --tooldir=*) TOOLDIR="${1#*=}" ;; --osversion=*) OSVERSION="${1#*=}" ;; --prefix=*|--installdir=*) PREFIX="${1#*=}" ;; + --profile=*) PROFILE="${1#*=}" ;; -h|--help) usage ;; *) echo "Unknown parameter passed: $1"; usage ;; esac shift done +# check the profile before anything is generated from it +if [ -n "$PROFILE" ] && [ ! -f "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/profiles/$PROFILE.json" ]; then + echo "Error: unknown profile '$PROFILE' (no profiles/$PROFILE.json)" + exit -1 +fi + # check OS if [ "$OSVERSION" == "unsupported" ]; then echo "Error: Unsupported OS." @@ -202,7 +213,7 @@ source "$SOURCE_DIR/VERSION" # stamp file tracks config parameter changes; touched only when params change CONFIG_STAMP="$CURRENT_DIR/.config.stamp" -CONFIG_SIG="XLEN=$XLEN TOOLDIR=$TOOLDIR OSVERSION=$OSVERSION PREFIX=$PREFIX SOURCE_DIR=$SOURCE_DIR BUILDDIR=$CURRENT_DIR" +CONFIG_SIG="XLEN=$XLEN TOOLDIR=$TOOLDIR OSVERSION=$OSVERSION PREFIX=$PREFIX PROFILE=$PROFILE SOURCE_DIR=$SOURCE_DIR BUILDDIR=$CURRENT_DIR" if [ ! -f "$CONFIG_STAMP" ] || [ "$(cat "$CONFIG_STAMP" 2>/dev/null)" != "$CONFIG_SIG" ]; then echo "$CONFIG_SIG" > "$CONFIG_STAMP" fi @@ -238,6 +249,7 @@ for toml in "$SOURCE_DIR"/*.toml; do extra="" case "$name" in VX_types) extra="--resolved" ;; + VX_config) [ -n "$PROFILE" ] && extra="--profile=$PROFILE" ;; esac gen_header "$toml" "$CURRENT_DIR/hw/$name.vh" verilog "$extra" gen_header "$toml" "$CURRENT_DIR/sw/$name.h" cpp "$extra" diff --git a/profiles/rtx_5080.json b/profiles/rtx_5080.json new file mode 100644 index 0000000000..8dc5906364 --- /dev/null +++ b/profiles/rtx_5080.json @@ -0,0 +1,45 @@ +{ + "name": "rtx_5080", + "description": "NVIDIA GeForce RTX 5080 (GB203, Blackwell, sm_120) at its 2295 MHz base clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 48, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_DCACHE_MSHR_SIZE": 24 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 84, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 67108864, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_L2_MSHR_SIZE": 192 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 2295, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000 + }, + "reference": { + "clock_mhz": 2295, + "l1_hit_cycles": 44.1, + "l2_hit_cycles": 354.4, + "dram_cycles": 802, + "dram_peak_bytes_per_cycle": 423, + "stream_efficiency": { "4MiB": 0.795, "16MiB": 0.894, "64MiB": 0.919 }, + "sgemm_cycles": { "128": 24193, "256": 45476, "512": 270396, "1024": 1845287 }, + "stream_cycles": { "4MiB": 25569, "16MiB": 90741, "64MiB": 427878 } + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "84 SMs, 48 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA RTX Blackwell GPU Architecture whitepaper, App. B; cudaGetDeviceProperties on the user's RTX 5080)", + "DCACHE_SIZE": "128 KB L1/shared per SM (whitepaper App. B)", + "L2_SIZE, L2_NUM_WAYS": "64 MB L2 (whitepaper App. B); 16 ways as in GPGPU-Sim's Ampere/Hopper L2 configs", + "L2_NUM_BANKS": "32 banks as the nearest power of two to the 48 L2 slices measured by arXiv 2507.10789", + "DCACHE_MSHR_SIZE, L2_MSHR_SIZE": "384 L1 MSHR entries per SM (24 x 16 banks) and 192 per L2 slice, GPGPU-Sim SM86_RTX3070 (-gpgpu_cache:dl1 A:384:48, dl2 A:192:4)", + "PLATFORM_MEMORY_NUM_BANKS": "8 x 32-bit GDDR7 devices = 32 independent channels (whitepaper App. B)", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", + "reference": "Nsight Compute on the user's RTX 5080, clocks locked to base, cold L2, 2026-10-05 (docs/reports/simx_vs_rtx5080_mlp_report.md)" + } +} From 7b91e0e94a367cbf99a9b8d0ce204d14eae53cd4 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 22:17:16 -0700 Subject: [PATCH 011/336] build: restore the zero-warnings policy everywhere Warnings had become non-fatal again: a June RTU change re-added Verilator -Wno-fatal to rtlsim after it had been removed, the AFU simulators copied it, and several C/C++ trees (OpenCL/HIP host code, unit tests, kernel library) never had -Werror. Warnings then accumulated unseen. Every C/C++ build (simx, rtlsim, xrtsim, opaesim, avedsim, runtime, kernel library, hw/sw unit tests, OpenCL/HIP/Vulkan apps) is now -Werror and Verilator warnings are fatal, with the code fixed rather than silenced: - RTL: VX_find_first, VX_onehot_encoder (MODEL1) and VX_pipe_buffer rewritten loop-free so Verilator -O0 does not report UNOPTFLAT (each proven equivalent to the original with an sv2v+yosys miter, with negative controls); width fixes in the tex bus arbiter, RTU scheduler config, PLRU replacement, the no-F CSR data path and the TFR MX scale; unused cluster/AFU ports removed or tied off; the early-Z y-pitch add widened explicitly (WIDTHEXPAND), and the RTU scheduler's issue block reads the memory grant instead of mem_fire, which is derived from the block's own mem_issue output (UNOPTFLAT). - Host code: 21 OpenCL apps, HIP, Vulkan, the kernel library and the DXA/FMA testbenches; -Wno-maybe-uninitialized/-Wno-array-bounds dropped; MAKEFLAGS cleared around verilator --build (jobserver warning). - vx_spawn.c printed uint32_t with %d, which the vx_printf format attribute rejects under GCC's newlib (uint32_t is unsigned long there). - The riscv-tests benchmark patch cast the unsuffixed COUT address straight to a pointer (-Wint-to-pointer-cast on RV64), and the riscv-tests stamp did not depend on the patch, so an edited patch was never re-applied. Remaining exceptions compile code that is not ours: the yosys gate-level netlist sims, the cln28hpc vendor macro testbench, and the FireSim driver, whose upstream bridge and platform objects share our CXXFLAGS. Verified: from-clean build of every group at XLEN 32 and 64 with 0 warnings; all 252 CI Verilator commands verilate with 0 warnings; the CI matrix cells run so far (perf_gate, amo, cache, config1 on rtlsim/simx) build warning-free. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_cluster.sv | 2 - hw/rtl/Vortex.sv | 1 - hw/rtl/afu/opae/vortex_afu.sv | 3 + hw/rtl/cache/VX_cache_repl.sv | 1 + hw/rtl/core/VX_csr_data.sv | 2 +- hw/rtl/fpu/VX_fma_unit_rtl.sv | 6 +- hw/rtl/libs/VX_find_first.sv | 42 ++-- hw/rtl/libs/VX_onehot_encoder.sv | 49 +++-- hw/rtl/libs/VX_pipe_buffer.sv | 6 +- hw/rtl/raster/VX_raster_earlyz.sv | 2 +- hw/rtl/rtu/VX_rtu_scheduler.sv | 31 +-- hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv | 4 + hw/rtl/tex/VX_tex_bus_arb.sv | 27 ++- hw/unittest/common.mk | 8 +- hw/unittest/dxa_core/main.cpp | 6 +- hw/unittest/fma_unit/main.cpp | 3 + miscs/patches/riscv-benchmarks.patch | 2 +- sim/avedsim/Makefile | 8 +- sim/opaesim/Makefile | 10 +- sim/rtlsim/Makefile | 14 +- sim/simx/Makefile | 2 +- sim/xrtsim/Makefile | 10 +- sw/kernel/Makefile | 1 + sw/kernel/src/vx_spawn.c | 7 +- sw/kernel/src/vx_syscalls.c | 45 +++- sw/runtime/simx/Makefile | 2 +- tests/hip/common.mk | 2 +- tests/hip/sgemm/main.cpp | 4 +- tests/hip/vecadd/main.cpp | 4 +- tests/opencl/bfs/CLHelper.h | 15 +- tests/opencl/bfs/main.cc | 38 ++-- tests/opencl/bfs/util.h | 2 + .../blackscholes/oclBlackScholes_launcher.cpp | 5 +- tests/opencl/blackscholes/shrUtils.cpp | 57 ++--- tests/opencl/cfd/main.cc | 12 +- tests/opencl/common.mk | 9 +- tests/opencl/conv3/main.cc | 2 +- tests/opencl/dotproduct/main.cc | 1 - tests/opencl/dotproduct/shrUtils.cpp | 57 ++--- tests/opencl/guassian/clutils.cpp | 30 +-- tests/opencl/guassian/main.cc | 20 +- tests/opencl/image_get/main.cc | 9 +- tests/opencl/image_int/main.cc | 17 +- tests/opencl/image_linear/main.cc | 11 +- tests/opencl/kmeans/kmeans_clustering.c | 6 +- tests/opencl/kmeans/main.cc | 17 +- tests/opencl/kmeans/read_input.c | 16 +- tests/opencl/kmeans/rmse.c | 6 +- tests/opencl/lbm/args.c | 5 +- tests/opencl/lbm/lbm.c | 21 +- tests/opencl/lbm/ocl.c | 2 +- tests/opencl/lbm/parboil_opencl.c | 204 ++---------------- tests/opencl/myocyte/myocyte_model.h | 4 +- tests/opencl/nearn/clutils.cpp | 31 +-- tests/opencl/nearn/main.cc | 10 +- tests/opencl/oclprintf/main.cc | 2 +- tests/opencl/psort/main.cc | 2 +- tests/opencl/saxpy/main.cc | 6 +- tests/opencl/sfilter/main.cc | 6 +- tests/opencl/spmv/args.c | 5 +- tests/opencl/spmv/convert_dataset.c | 17 +- tests/opencl/spmv/file.c | 43 +++- tests/opencl/spmv/main.cc | 15 +- tests/opencl/spmv/mmio.c | 12 +- tests/opencl/spmv/ocl.c | 2 +- tests/opencl/spmv/parboil_opencl.c | 81 ++----- tests/opencl/srad/main.cc | 4 +- tests/opencl/stencil/args.c | 5 +- tests/opencl/stencil/file.c | 10 +- tests/opencl/stencil/main.cc | 18 +- tests/opencl/stencil/ocl.c | 2 +- tests/opencl/stencil/parboil_opencl.c | 80 ++----- tests/opencl/transpose/main.cc | 5 +- tests/opencl/transpose/shrUtils.cpp | 57 ++--- tests/riscv/common.mk | 5 +- tests/unittest/common.mk | 2 +- tests/vulkan/common.mk | 2 +- tests/vulkan/subgroup_vs/main.c | 2 +- 78 files changed, 593 insertions(+), 701 deletions(-) diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index fa68f07398..86fc979410 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -43,8 +43,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif `ifdef VX_CFG_VM_ENABLE - // Device MMU sideband from the top DCR surface. - input wire [`VX_CFG_XLEN-1:0] mmu_satp, // Device-level walker: the L2 TLB's miss bus, exported to the device. VX_tlb_bus_if.master dev_ptw_if, // Walker PTE fetches, entering as an L2 client when this cluster's L2 diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 762e1e0200..2b4b49df89 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -346,7 +346,6 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `endif `ifdef VX_CFG_VM_ENABLE - .mmu_satp (mmu_satp), .dev_ptw_if (per_cluster_dev_ptw_if[cluster_id]), .dev_ptw_mem_if (per_cluster_ptw_mem_if[cluster_id]), .mmu_flush_req (mmu_flush_req), diff --git a/hw/rtl/afu/opae/vortex_afu.sv b/hw/rtl/afu/opae/vortex_afu.sv index e9ba15f39e..61ad02a8e4 100644 --- a/hw/rtl/afu/opae/vortex_afu.sv +++ b/hw/rtl/afu/opae/vortex_afu.sv @@ -356,6 +356,9 @@ module vortex_afu import ccip_if_pkg::*; import local_mem_cfg_pkg::*; import VX_ .axil_s (cp_axil), .axi_host (cp_axi_host), .axi_dev (cp_axi_dev), + // CCI-P has no AXI host port, so there are no drain counters to report. + .dbg_host_w_counts ('0), + .dbg_host_r_counts ('0), .gpu_if (cp_gpu_if), .irq (cp_interrupt) ); diff --git a/hw/rtl/cache/VX_cache_repl.sv b/hw/rtl/cache/VX_cache_repl.sv index 5aa92f7d5a..b639ac33ac 100644 --- a/hw/rtl/cache/VX_cache_repl.sv +++ b/hw/rtl/cache/VX_cache_repl.sv @@ -108,6 +108,7 @@ module VX_cache_repl #( if (REPL_POLICY == `CS_REPL_PLRU) begin : g_plru // Pseudo Least Recently Used replacement policy `UNUSED_VAR (repl_line) // read uses repl_line_n (look-ahead) + `UNUSED_VAR (repl_valid) // state advances on lookup hits only localparam LRU_WIDTH = `UP(NUM_WAYS-1); wire [LRU_WIDTH-1:0] plru_rdata; diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index 51efaa9762..f09c17bb7e 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -338,7 +338,7 @@ import VX_fpu_pkg::*; end end `else - `UNUSED_VAR (write_sel[WR_FFLAGS +: 3]) + `UNUSED_VAR ({write_sel[WR_FCSR], write_sel[WR_FRM], write_sel[WR_FFLAGS]}) `endif `ifdef VX_CFG_VM_ENABLE diff --git a/hw/rtl/fpu/VX_fma_unit_rtl.sv b/hw/rtl/fpu/VX_fma_unit_rtl.sv index cddf2b4889..02b986d826 100644 --- a/hw/rtl/fpu/VX_fma_unit_rtl.sv +++ b/hw/rtl/fpu/VX_fma_unit_rtl.sv @@ -115,9 +115,9 @@ module VX_fma_unit_rtl import VX_gpu_pkg::*, VX_fpu_pkg::*; #( // DAZ: flush a subnormal operand to signed zero when subnormals are disabled. localparam DAZ = (SUBNORM_ENABLE == 0); - function automatic [FLOAT_BITS-1:0] daz_f(input [FLOAT_BITS-1:0] v); - daz_f = (DAZ && (v[FLOAT_BITS-2:MAN_BITS] == '0) && (|v[MAN_BITS-1:0])) - ? {v[FLOAT_BITS-1], {(FLOAT_BITS-1){1'b0}}} : v; + function automatic [FLOAT_BITS-1:0] daz_f(input [FLOAT_BITS-1:0] val); + daz_f = (DAZ && (val[FLOAT_BITS-2:MAN_BITS] == '0) && (|val[MAN_BITS-1:0])) + ? {val[FLOAT_BITS-1], {(FLOAT_BITS-1){1'b0}}} : val; endfunction wire [FLOAT_BITS-1:0] op_a = daz_f(dataa); diff --git a/hw/rtl/libs/VX_find_first.sv b/hw/rtl/libs/VX_find_first.sv index b7fc3e2596..92ad0f2d20 100644 --- a/hw/rtl/libs/VX_find_first.sv +++ b/hw/rtl/libs/VX_find_first.sv @@ -28,32 +28,30 @@ module VX_find_first #( localparam TL = (1 << LOGN) - 1; localparam TN = (1 << (LOGN+1)) - 1; - wire s_n [TN]; - wire [DATAW-1:0] d_n [TN]; - - for (genvar i = 0; i < N; ++i) begin : g_fill - assign s_n[TL+i] = REVERSE ? valid_in[N-1-i] : valid_in[i]; - assign d_n[TL+i] = REVERSE ? data_in[N-1-i] : data_in[i]; - end - - if (TL < (TN-N)) begin : g_padding - for (genvar i = TL+N; i < TN; ++i) begin : g_i - assign s_n[i] = 0; - assign d_n[i] = '0; + reg valid_r; + reg [DATAW-1:0] data_r; + + // Binary tree: node k reduces children 2k+1 (first) and 2k+2; leaves start at TL. + // The tree is local to one block so no node variable feeds itself. + always @(*) begin : g_scan + reg [TN-1:0] s_n; + reg [TN*DATAW-1:0] d_n; + s_n = '0; + d_n = '0; + for (integer i = 0; i < N; ++i) begin + s_n[TL+i] = REVERSE ? valid_in[N-1-i] : valid_in[i]; + d_n[(TL+i)*DATAW +: DATAW] = REVERSE ? data_in[N-1-i] : data_in[i]; end - end - - for (genvar j = 0; j < LOGN; ++j) begin : g_scan - localparam I = 1 << j; - for (genvar i = 0; i < I; ++i) begin : g_i - localparam K = I+i-1; - assign s_n[K] = s_n[2*K+2] | s_n[2*K+1]; - assign d_n[K] = s_n[2*K+1] ? d_n[2*K+1] : d_n[2*K+2]; + for (integer k = TL-1; k >= 0; --k) begin + s_n[k] = s_n[2*k+2] | s_n[2*k+1]; + d_n[k*DATAW +: DATAW] = s_n[2*k+1] ? d_n[(2*k+1)*DATAW +: DATAW] : d_n[(2*k+2)*DATAW +: DATAW]; end + valid_r = s_n[0]; + data_r = d_n[0 +: DATAW]; end - assign valid_out = s_n[0]; - assign data_out = d_n[0]; + assign valid_out = valid_r; + assign data_out = data_r; endmodule `TRACING_ON diff --git a/hw/rtl/libs/VX_onehot_encoder.sv b/hw/rtl/libs/VX_onehot_encoder.sv index 273a9cfe26..2bbf58d377 100644 --- a/hw/rtl/libs/VX_onehot_encoder.sv +++ b/hw/rtl/libs/VX_onehot_encoder.sv @@ -39,31 +39,38 @@ module VX_onehot_encoder #( end else if (MODEL == 1) begin : g_model1 localparam M = 1 << LN; - wire [M-1:0] addr [LN]; - wire [M-1:0] v [LN+1]; - - // base case, also handle padding for non-power of two inputs - assign v[0] = REVERSE ? (M'(data_in) << (M - N)) : M'(data_in); - - for (genvar lvl = 1; lvl < (LN+1); ++lvl) begin : g_scan_l - localparam SN = 1 << (LN - lvl); - localparam SI = M / SN; - for (genvar s = 0; s < SN; ++s) begin : g_scan_s - wire [1:0] vs = {v[lvl-1][s*SI+(SI>>1)], v[lvl-1][s*SI]}; - assign v[lvl][s*SI] = (| vs); - if (lvl == 1) begin : g_lvl_1 - assign addr[lvl-1][s*SI +: lvl] = vs[!REVERSE]; - end else begin : g_lvl_n - assign addr[lvl-1][s*SI +: lvl] = { - vs[!REVERSE], - addr[lvl-2][s*SI +: lvl-1] | addr[lvl-2][s*SI+(SI>>1) +: lvl-1] - }; + + reg [LN-1:0] addr_r; + reg valid_r; + + // Level lvl merges pairs of level lvl-1 groups of size 2^(lvl-1); each + // level has its own slice of v/addr, local to one block, so no + // variable feeds itself. + always @(*) begin : g_scan + reg [(LN+1)*M-1:0] v; + reg [LN*M-1:0] addr; + reg [1:0] vs; + v = '0; + addr = '0; + // base case, also handle padding for non-power of two inputs + v[0 +: M] = REVERSE ? (M'(data_in) << (M - N)) : M'(data_in); + for (integer lvl = 1; lvl <= LN; ++lvl) begin + for (integer s = 0; s < (1 << (LN - lvl)); ++s) begin + vs = {v[(lvl-1)*M + s*(1 << lvl) + (1 << (lvl-1))], v[(lvl-1)*M + s*(1 << lvl)]}; + v[lvl*M + s*(1 << lvl)] = (| vs); + addr[(lvl-1)*M + s*(1 << lvl) + (lvl-1)] = vs[!REVERSE]; + for (integer b = 0; b < lvl-1; ++b) begin + addr[(lvl-1)*M + s*(1 << lvl) + b] = addr[(lvl-2)*M + s*(1 << lvl) + b] + | addr[(lvl-2)*M + s*(1 << lvl) + (1 << (lvl-1)) + b]; + end end end + addr_r = addr[(LN-1)*M +: LN]; + valid_r = v[LN*M]; end - assign data_out = addr[LN-1][LN-1:0]; - assign valid_out = v[LN][0]; + assign data_out = addr_r; + assign valid_out = valid_r; end else if (MODEL == 2 && REVERSE == 0) begin : g_model2 diff --git a/hw/rtl/libs/VX_pipe_buffer.sv b/hw/rtl/libs/VX_pipe_buffer.sv index 0847c0bbe9..e72e8d2b35 100644 --- a/hw/rtl/libs/VX_pipe_buffer.sv +++ b/hw/rtl/libs/VX_pipe_buffer.sv @@ -40,7 +40,7 @@ module VX_pipe_buffer #( assign data_out = data_in; end else begin : g_register wire [DEPTH:0] valid; - wire ready [DEPTH+1]; + wire [DEPTH-1:0] ready; wire [DEPTH:0][DATAW-1:0] data; assign valid[0] = valid_in; @@ -48,7 +48,8 @@ module VX_pipe_buffer #( assign ready_in = ready[0]; for (genvar i = 0; i < DEPTH; ++i) begin : g_pipe_regs - assign ready[i] = (ready[i+1] || ~valid[i+1]); + // Stage i advances when any later stage has a hole or the output drains. + assign ready[i] = ready_out || ~(& valid[DEPTH:i+1]); VX_pipe_register #( .DATAW (1 + DATAW), .RESETW (1 + RESETW) @@ -63,7 +64,6 @@ module VX_pipe_buffer #( assign valid_out = valid[DEPTH]; assign data_out = data[DEPTH]; - assign ready[DEPTH] = ready_out; end endmodule diff --git a/hw/rtl/raster/VX_raster_earlyz.sv b/hw/rtl/raster/VX_raster_earlyz.sv index 59ae4e853d..2cb0529650 100644 --- a/hw/rtl/raster/VX_raster_earlyz.sv +++ b/hw/rtl/raster/VX_raster_earlyz.sv @@ -141,7 +141,7 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX assign pix_cand_n[p] = zbits[31] ? 24'd0 : (|zbits[30:24]) ? 24'(OM_DEPTH_MASK) : zbits[23:0]; - wire [31:0] y_pitch = quad_ypitch[q] + (i[1] ? dcrs.zbuf_pitch : 32'd0); + wire [31:0] y_pitch = quad_ypitch[q] + (i[1] ? 32'(dcrs.zbuf_pitch) : 32'd0); wire [W_ADDR_BITS-1:0] baddr = {dcrs.zbuf_addr, 4'b0} + W_ADDR_BITS'(px); assign pix_waddr_n[p] = baddr + W_ADDR_BITS'(y_pitch[31:2]); `UNUSED_VAR (y_pitch) diff --git a/hw/rtl/rtu/VX_rtu_scheduler.sv b/hw/rtl/rtu/VX_rtu_scheduler.sv index 8477773ae5..46f1676251 100644 --- a/hw/rtl/rtu/VX_rtu_scheduler.sv +++ b/hw/rtl/rtu/VX_rtu_scheduler.sv @@ -119,8 +119,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( localparam LINES = FLAT ? RTU_FLAT_LINES : RTU_NODE_LINES; localparam LB = `CLOG2(LINES + 1); localparam BUF_BITS = LINES * LINE_BITS; - localparam IDXW = `CLOG2(`UP(RTU_BVH_WIDTH)); localparam NODE_W = `UP(RTU_BVH_WIDTH); + localparam IDXW = `LOG2UP(NODE_W); localparam ADDRW = `VX_CFG_MEM_ADDR_WIDTH; `STATIC_ASSERT(((NUM_LANES * NUM_SLOTS) == NUM_CTX), @@ -461,8 +461,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wire recip_obj; wire recip_valid_out, recip_last_out; wire [CTX_TAG_W-1:0] recip_tag_out; - wire [1:0] recip_axis_out; - wire [31:0] recip_result; wire [95:0] recip_rdata; // ── short stack (BVH only) ──────────────────────────────────────── @@ -792,6 +790,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign ins_shift[oc] = ~ins_le[oc-1]; end end else begin : g_no_ins_slots + `UNUSED_VAR ({coll_ordt_sel, coll_ordcnt_sel}) assign {ins_le, ins_here, ins_shift} = '0; end wire coll_pushable = box_hit && (box_childoff != 32'd0); @@ -847,6 +846,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wire [1:0] recip_axis; wire [31:0] recip_din = recip_obj ? word_q.obj_d[recip_axis] : ray_q.dir[recip_axis]; if (!FLAT) begin : g_recip_pe + wire [1:0] recip_axis_out; + wire [31:0] recip_result; VX_rtu_recip #( .LATENCY (RTU_FDIV_LAT), .DSP_SEED (`VX_CFG_RTU_RECIP_DSP_SEED) @@ -890,8 +891,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign recip_valid_out = 1'b0; assign recip_last_out = 1'b0; assign recip_tag_out = '0; - assign recip_axis_out = '0; - assign recip_result = '0; assign recip_rdata = '0; `UNUSED_VAR ({recip_issue, recip_obj, recip_din, recip_q}) end @@ -1292,7 +1291,11 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [LVLW-1:0] path_wlvl_r; reg [PATHW-1:0] path_wdata_r; - wire mem_fire = x_valid && mem_issue && mem_req_ready; + // The issue block below reads the grant only where it has just set + // mem_issue, so it uses mem_grant: reading mem_fire would feed its own + // output back into itself (a combinational loop to Verilator). + wire mem_grant = x_valid && mem_req_ready; + wire mem_fire = mem_grant && mem_issue; wire [RTU_CHILD_BITS-1:0] last_child = node.n_children - RTU_CHILD_BITS'(1); @@ -1355,7 +1358,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_HDR_REQ: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_HDR_WAIT; end else begin wake_self = 1'b1; @@ -1400,7 +1403,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_REQ0: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_RSP0; end else begin wake_self = 1'b1; @@ -1435,7 +1438,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_REQN: begin mem_issue = 1'b1; mem_fslot = word_x.f_idx; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_RSPN; end else begin wake_self = 1'b1; @@ -1598,7 +1601,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_LTRI_REQ0: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.f_total = tri_rec_lines; word_n.cstate = CS_LTRI_RSP0; end else begin @@ -1619,7 +1622,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_LTRI_REQN: begin mem_issue = 1'b1; mem_fslot = word_x.f_idx; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_LTRI_RSPN; end else begin wake_self = 1'b1; @@ -1758,7 +1761,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_INST_REQ: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.f_total = inst_lines; word_n.cstate = CS_INST_RSP0; end else begin @@ -1777,7 +1780,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_INST_REQN: begin mem_issue = 1'b1; mem_fslot = word_x.f_idx; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_INST_RSPN; end else begin wake_self = 1'b1; @@ -1862,7 +1865,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_BHDR_REQ: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_BHDR_WAIT; end else begin wake_self = 1'b1; diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv index dec2159349..12953c50f3 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv @@ -55,6 +55,10 @@ module VX_tcu_tfr_shared_mul import VX_tcu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) `ifndef VX_CFG_TCU_MX_ENABLE `UNUSED_PARAM (SF) +`else + if (!`VX_CFG_TCU_FP8_ENABLED && !`VX_CFG_TCU_FP4_ENABLED) begin : g_sf_unused + `UNUSED_VAR ({sf_a, sf_b}) + end `endif `UNUSED_VAR ({clk, req_id, valid_in}) diff --git a/hw/rtl/tex/VX_tex_bus_arb.sv b/hw/rtl/tex/VX_tex_bus_arb.sv index 319570f9c6..5041b81a11 100644 --- a/hw/rtl/tex/VX_tex_bus_arb.sv +++ b/hw/rtl/tex/VX_tex_bus_arb.sv @@ -32,6 +32,8 @@ module VX_tex_bus_arb import VX_gpu_pkg::*, VX_tex_pkg::*; #( ); localparam LOG_NUM_REQS = `ARB_SEL_BITS(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(`CDIV(`MAX(NUM_INPUTS, NUM_OUTPUTS), SEL_COUNT))); localparam REQ_DATAW = TAG_WIDTH + NUM_LANES * (1 + 2 * 32 + TEX_LODF_BITS) + TEX_STAGE_BITS; localparam RSP_DATAW = TAG_WIDTH + NUM_LANES * 32; @@ -43,7 +45,7 @@ module VX_tex_bus_arb import VX_gpu_pkg::*, VX_tex_pkg::*; #( wire [NUM_OUTPUTS-1:0] req_valid_out; wire [NUM_OUTPUTS-1:0][REQ_DATAW-1:0] req_data_out; - wire [NUM_OUTPUTS-1:0][`UP(LOG_NUM_REQS)-1:0] req_sel_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] req_sel_out; wire [NUM_OUTPUTS-1:0] req_ready_out; for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_req_data_in @@ -72,15 +74,20 @@ module VX_tex_bus_arb import VX_gpu_pkg::*, VX_tex_pkg::*; #( for (genvar i = 0; i < NUM_OUTPUTS; ++i) begin : g_bus_out_if wire [TAG_WIDTH-1:0] req_tag_out; - VX_bits_insert #( - .N (TAG_WIDTH), - .S (LOG_NUM_REQS), - .POS (TAG_SEL_IDX) - ) bits_insert ( - .data_in (req_tag_out), - .ins_in (req_sel_out[i]), - .data_out (bus_out_if[i].req_data.tag) - ); + if (NUM_INPUTS > NUM_OUTPUTS) begin : g_req_tag_sel_out + VX_bits_insert #( + .N (TAG_WIDTH), + .S (LOG_NUM_REQS), + .POS (TAG_SEL_IDX) + ) bits_insert ( + .data_in (req_tag_out), + .ins_in (req_sel_out[i]), + .data_out (bus_out_if[i].req_data.tag) + ); + end else begin : g_req_tag_out + `UNUSED_VAR (req_sel_out) + assign bus_out_if[i].req_data.tag = req_tag_out; + end assign bus_out_if[i].req_valid = req_valid_out[i]; assign {req_tag_out, bus_out_if[i].req_data.mask, bus_out_if[i].req_data.coords, bus_out_if[i].req_data.lod, bus_out_if[i].req_data.stage} = req_data_out[i]; assign req_ready_out[i] = bus_out_if[i].req_ready; diff --git a/hw/unittest/common.mk b/hw/unittest/common.mk index 3729fadce2..25ca8e25ee 100644 --- a/hw/unittest/common.mk +++ b/hw/unittest/common.mk @@ -3,8 +3,8 @@ DESTDIR ?= . CONFIGS += PARAMS += -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) @@ -67,8 +67,10 @@ endif all: $(DESTDIR)/$(PROJECT) +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' -MMD -o ../$@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' -MMD -o ../$@ run: $(DESTDIR)/$(PROJECT) $(DESTDIR)/$(PROJECT) $(OPTS) diff --git a/hw/unittest/dxa_core/main.cpp b/hw/unittest/dxa_core/main.cpp index 91b74ff863..6f796cd0aa 100644 --- a/hw/unittest/dxa_core/main.cpp +++ b/hw/unittest/dxa_core/main.cpp @@ -19,6 +19,7 @@ #include #include #include "VVX_dxa_core_top.h" +#include "VX_config.h" #include "verilated.h" #ifdef VCD_OUTPUT #include "verilated_vcd_c.h" @@ -93,9 +94,10 @@ int main(int argc, char** argv) { dut->dcr_req_valid = 0; dut->dcr_req_rw = 0; dut->dxa_req_valid = 0; - dut->gmem_req_ready = ~0ULL; // all ready + auto all_ready = [](uint32_t ports) { return (ports >= 32) ? ~0u : ((1u << ports) - 1); }; + dut->gmem_req_ready = all_ready(VX_CFG_NUM_DXA_CORES); dut->gmem_rsp_valid = 0; - dut->lmem_req_ready = ~0ULL; // all ready + dut->lmem_req_ready = all_ready(VX_CFG_SOCKET_SIZE); reset_dut(); diff --git a/hw/unittest/fma_unit/main.cpp b/hw/unittest/fma_unit/main.cpp index 2ff05c8a90..7f0c07b672 100644 --- a/hw/unittest/fma_unit/main.cpp +++ b/hw/unittest/fma_unit/main.cpp @@ -74,6 +74,9 @@ void encode(int op, uint32_t a, uint32_t b, uint32_t c, case FMSUB: op_type = OP_MADD; fmt = 2; break; // a*b - c case FNMADD: op_type = OP_NMADD; fmt = 0; break; // -(a*b) - c case FNMSUB: op_type = OP_NMADD; fmt = 2; break; // -(a*b) + c + default: + std::fprintf(stderr, "encode: unknown op %d\n", op); + std::abort(); } } diff --git a/miscs/patches/riscv-benchmarks.patch b/miscs/patches/riscv-benchmarks.patch index 3f4011d2c7..4db8b67a94 100644 --- a/miscs/patches/riscv-benchmarks.patch +++ b/miscs/patches/riscv-benchmarks.patch @@ -120,7 +120,7 @@ index 2a2a8f3..e967138 100644 - syscall(SYS_write, HTIF_DEV_CONSOLE, (uintptr_t)&c, HTIF_CONSOLE_CMD_PUTC); - + unsigned tid = read_csr(mhartid) & (VX_MEM_IO_COUT_SLOTS - 1); -+ *(volatile char*)(VX_MEM_IO_COUT_ADDR + tid) = (char)ch; ++ *(volatile char*)(uintptr_t)(VX_MEM_IO_COUT_ADDR + tid) = (char)ch; return 0; } diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index 539c3922ef..6dd3b52bf2 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -10,8 +10,8 @@ XRTSIM_DIR := $(VORTEX_HOME)/sim/xrtsim AFU_DIR := $(RTL_DIR)/afu/aved AFU_COMMON_DIR := $(RTL_DIR)/afu/common -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -213,8 +213,10 @@ $(DESTDIR)/vortex.json: $(CONFIG_FILE) $(DESTDIR)/scope.json: $(DESTDIR)/vortex.json $(SCRIPT_DIR)/scope.py $^ -o $@ +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SCOPE_JSON) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/opaesim/Makefile b/sim/opaesim/Makefile index 5fc6e6aaee..61fc9aa0d8 100644 --- a/sim/opaesim/Makefile +++ b/sim/opaesim/Makefile @@ -6,8 +6,8 @@ CONFIG_FILE := $(DESTDIR)/opaesim_config.stamp SRC_DIR := $(VORTEX_HOME)/sim/opaesim AFU_DIR := $(RTL_DIR)/afu/opae -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -141,7 +141,7 @@ RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic -VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO -Wno-fatal +VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) @@ -216,8 +216,10 @@ $(DESTDIR)/vortex.json: $(CONFIG_FILE) $(DESTDIR)/scope.json: $(DESTDIR)/vortex.json $(SCRIPT_DIR)/scope.py $^ -o $@ +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SCOPE_JSON) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index 9856159b26..f6b8ca3cf8 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -5,8 +5,8 @@ DESTDIR ?= $(CURDIR) CONFIG_FILE = $(DESTDIR)/rtlsim_config.stamp SRC_DIR = $(VORTEX_HOME)/sim/rtlsim -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -132,9 +132,7 @@ TOP = rtlsim_shim VL_FLAGS = --exe VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic -# Keep lint warnings visible but non-fatal: newer Verilator surfaces latent -# UNUSED*/PINCONNECTEMPTY warnings in existing IP that older releases did not. -VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO -Wno-fatal +VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) @@ -201,11 +199,13 @@ PROJECT := rtlsim all: $(DESTDIR)/$(PROJECT) +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SRC_DIR)/main.cpp $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) $(SRC_DIR)/main.cpp -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) $(SRC_DIR)/main.cpp -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ $(DESTDIR)/lib$(PROJECT).so: $(SRCS) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '-shared $(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '-shared $(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 63581b4b8f..a9638e5ac7 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -17,7 +17,7 @@ CONFIG_FILE = $(DESTDIR)/simx_config.stamp SRC_DIR = $(VORTEX_HOME)/sim/simx CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -fPIC CXXFLAGS += -I$(SRC_DIR) -I$(SRC_DIR)/dxa -I$(SRC_DIR)/tex -I$(SRC_DIR)/om -I$(SRC_DIR)/raster -I$(SRC_DIR)/rtu -I$(SRC_DIR)/kmu -I$(SRC_DIR)/mem -I$(SRC_DIR)/dtm -I$(SW_COMMON_DIR) -I$(SIM_COMMON_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(THIRD_PARTY_DIR) CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include CXXFLAGS += -I$(THIRD_PARTY_DIR)/ramulator/ext/spdlog/include diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index b2e9be960a..231d745806 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -7,8 +7,8 @@ SRC_DIR := $(VORTEX_HOME)/sim/xrtsim AFU_DIR := $(RTL_DIR)/afu/xrt AFU_COMMON_DIR := $(RTL_DIR)/afu/common -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -141,7 +141,7 @@ RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic -VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO -Wno-fatal +VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) @@ -216,8 +216,10 @@ $(DESTDIR)/vortex.json: $(CONFIG_FILE) $(DESTDIR)/scope.json: $(DESTDIR)/vortex.json $(SCRIPT_DIR)/scope.py $^ -o $@ +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SCOPE_JSON) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sw/kernel/Makefile b/sw/kernel/Makefile index a00ea753a8..668b6f8c18 100644 --- a/sw/kernel/Makefile +++ b/sw/kernel/Makefile @@ -29,6 +29,7 @@ DP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objdump CP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy CFLAGS += -O3 -mcmodel=medany -fno-exceptions -fdata-sections -ffunction-sections -D__VORTEX__ +CFLAGS += -Wall -Wextra -Wfatal-errors -Werror CFLAGS += -I$(INC_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') CFLAGS += $(XCONFIGS) diff --git a/sw/kernel/src/vx_spawn.c b/sw/kernel/src/vx_spawn.c index af8d9c2c65..fecb6fdfb7 100644 --- a/sw/kernel/src/vx_spawn.c +++ b/sw/kernel/src/vx_spawn.c @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -144,7 +145,7 @@ static void __attribute__ ((noinline)) process_thread_groups() { for (uint32_t group_id = start_group; group_id < end_group; group_id += group_stride) { blockIdx.x = group_id % gridDim_x; blockIdx.y = (group_id / gridDim_x) % gridDim_y; - blockIdx.z = group_id / (gridDim_x * gridDim_y); + blockIdx.z = group_id / gridDim_xy; callback((void*)arg); } } @@ -155,7 +156,7 @@ static void __attribute__ ((noinline)) process_thread_groups_stub() { uint32_t remaining_mask = targs->remaining_mask; uint32_t warp_id = vx_warp_id(); uint32_t group_warp_id = warp_id % warps_per_group; - uint32_t threads_mask = (group_warp_id == warps_per_group-1) ? remaining_mask : -1; + uint32_t threads_mask = (group_warp_id == warps_per_group-1) ? remaining_mask : ~0u; // activate threads vx_tmc(threads_mask); @@ -192,7 +193,7 @@ int vx_spawn_threads(uint32_t dimension, // check group size uint32_t threads_per_core = warps_per_core * threads_per_warp; if (threads_per_core < group_size) { - vx_printf("error: group_size > threads_per_core (%d,%d)\n", group_size, threads_per_core); + vx_printf("error: group_size > threads_per_core (%" PRIu32 ",%" PRIu32 ")\n", group_size, threads_per_core); return -1; } diff --git a/sw/kernel/src/vx_syscalls.c b/sw/kernel/src/vx_syscalls.c index bef330323e..4f34771e32 100644 --- a/sw/kernel/src/vx_syscalls.c +++ b/sw/kernel/src/vx_syscalls.c @@ -23,24 +23,51 @@ extern "C" { #endif -int _close(int file) { return -1; } +int _close(int file) { + (void)file; + return -1; +} -int _fstat(int file, struct stat *st) { return -1; } +int _fstat(int file, struct stat *st) { + (void)file; + (void)st; + return -1; +} -int _isatty(int file) { return 0; } +int _isatty(int file) { + (void)file; + return 0; +} -off_t _lseek(int file, off_t ptr, int dir) { return 0; } +off_t _lseek(int file, off_t ptr, int dir) { + (void)file; + (void)ptr; + (void)dir; + return 0; +} -int _open(const char *name, int flags, int mode) { return -1; } +int _open(const char *name, int flags, int mode) { + (void)name; + (void)flags; + (void)mode; + return -1; +} -ssize_t _read(int file, void *ptr, size_t len) { return -1; } +ssize_t _read(int file, void *ptr, size_t len) { + (void)file; + (void)ptr; + (void)len; + return -1; +} void *_sbrk(ptrdiff_t incr) { + (void)incr; __asm__ __volatile__("ebreak"); return 0; } ssize_t _write(int file, const void *ptr, size_t len) { + (void)file; const char *p = (const char *)ptr; for (size_t i = 0; i < len; ++i) { vx_putchar(*p++); @@ -48,7 +75,11 @@ ssize_t _write(int file, const void *ptr, size_t len) { return len; } -int _kill(int pid, int sig) { return -1; } +int _kill(int pid, int sig) { + (void)pid; + (void)sig; + return -1; +} int _getpid() { return vx_hart_id(); diff --git a/sw/runtime/simx/Makefile b/sw/runtime/simx/Makefile index 65ec2c4c7a..39c40cc70f 100644 --- a/sw/runtime/simx/Makefile +++ b/sw/runtime/simx/Makefile @@ -6,7 +6,7 @@ CONFIG_FILE := $(DESTDIR)/simx_rt_config.stamp SRC_DIR := $(VORTEX_HOME)/sw/runtime/simx CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -fPIC CXXFLAGS += -I$(INC_DIR) -I../common -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SIM_DIR)/simx -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) -I$(RT_COMMON_DIR) CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) diff --git a/tests/hip/common.mk b/tests/hip/common.mk index 0b315896d2..77c80baa6e 100644 --- a/tests/hip/common.mk +++ b/tests/hip/common.mk @@ -83,7 +83,7 @@ POCL_CC_FLAGS += POCL_VORTEX_BINTOOL="$(VX_BINTOOL)" POCL_CC_FLAGS += POCL_VORTEX_CFLAGS="$(VX_CFLAGS)" POCL_CC_FLAGS += POCL_VORTEX_LDFLAGS="$(VX_LDFLAGS)" -HIPCC_FLAGS += -std=c++17 +HIPCC_FLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror HIPCC_FLAGS += $(CONFIGS) # The embedded .hipInfo OFFLOAD_TRIPLE selects the primary SPIR-V pointer # width. hipcc rewrites it to match XLEN so the emitted SPIR-V is diff --git a/tests/hip/sgemm/main.cpp b/tests/hip/sgemm/main.cpp index 26de4d7233..99b63c0da8 100644 --- a/tests/hip/sgemm/main.cpp +++ b/tests/hip/sgemm/main.cpp @@ -89,8 +89,8 @@ int main(int argc, char** argv) { srand(50); for (uint32_t i = 0; i < size_sq; ++i) { - h_a[i] = static_cast(rand()) / RAND_MAX; - h_b[i] = static_cast(rand()) / RAND_MAX; + h_a[i] = static_cast(rand()) / static_cast(RAND_MAX); + h_b[i] = static_cast(rand()) / static_cast(RAND_MAX); } printf("Allocate device buffers\n"); diff --git a/tests/hip/vecadd/main.cpp b/tests/hip/vecadd/main.cpp index 9532b98eec..5e8e8b7e4b 100644 --- a/tests/hip/vecadd/main.cpp +++ b/tests/hip/vecadd/main.cpp @@ -71,8 +71,8 @@ int main(int argc, char** argv) { srand(50); for (uint32_t i = 0; i < size; ++i) { - h_a[i] = static_cast(rand()) / RAND_MAX; - h_b[i] = static_cast(rand()) / RAND_MAX; + h_a[i] = static_cast(rand()) / static_cast(RAND_MAX); + h_b[i] = static_cast(rand()) / static_cast(RAND_MAX); } printf("Allocate device buffers\n"); diff --git a/tests/opencl/bfs/CLHelper.h b/tests/opencl/bfs/CLHelper.h index e4106845bd..eafce0d076 100755 --- a/tests/opencl/bfs/CLHelper.h +++ b/tests/opencl/bfs/CLHelper.h @@ -254,7 +254,6 @@ free(allPlatforms);*/ // read kernel binary from file uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; if (0 != read_kernel_file("kernel.cl", &kernel_bin, &kernel_size)) std::abort(); @@ -374,7 +373,7 @@ free(allPlatforms);*/ void _clRelease() { char errorFlag = false; - for (int nKernel = 0; nKernel < oclHandles.kernel.size(); nKernel++) { + for (size_t nKernel = 0; nKernel < oclHandles.kernel.size(); nKernel++) { if (oclHandles.kernel[nKernel] != NULL) { cl_int resultCL = clReleaseKernel(oclHandles.kernel[nKernel]); if (resultCL != CL_SUCCESS) { @@ -511,7 +510,9 @@ cl_mem _clCreateAndCpyPinnedMem(int size, float *h_mem_source) { throw(string("excpetion in _clCreateAndCpyMem() -> clEnqueueMapBuffer")); #endif int element_number = size / sizeof(float); +#ifdef _OPENMP #pragma omp parallel for +#endif for (int i = 0; i < element_number; i++) { h_mem_pinned[i] = h_mem_source[i]; } @@ -690,8 +691,8 @@ void _clInvokeKernel(int kernel_id, int work_items, // cannot be divided by work_group_size work_items = work_items + (work_group_size - (work_items % work_group_size)); - size_t local_work_size[] = {work_group_size, 1}; - size_t global_work_size[] = {work_items, 1}; + size_t local_work_size[] = {static_cast(work_group_size), 1}; + size_t global_work_size[] = {static_cast(work_items), 1}; oclHandles.cl_status = clEnqueueNDRangeKernel( oclHandles.queue, oclHandles.kernel[kernel_id], work_dim, 0, global_work_size, local_work_size, 0, 0, NULL); @@ -757,8 +758,10 @@ void _clInvokeKernel(int kernel_id, int work_items, void _clInvokeKernel2D(int kernel_id, int range_x, int range_y, int group_x, int group_y) { cl_uint work_dim = WORK_DIM; - size_t local_work_size[] = {group_x, group_y}; - size_t global_work_size[] = {range_x, range_y}; + size_t local_work_size[] = {static_cast(group_x), + static_cast(group_y)}; + size_t global_work_size[] = {static_cast(range_x), + static_cast(range_y)}; //cl_event e[1]; /*if(work_items%work_group_size != 0) //process situations that work_items cannot be divided by work_group_size diff --git a/tests/opencl/bfs/main.cc b/tests/opencl/bfs/main.cc index cd55f5b3fb..c676a40c53 100755 --- a/tests/opencl/bfs/main.cc +++ b/tests/opencl/bfs/main.cc @@ -29,10 +29,9 @@ struct Node { //--date: 26/01/2011 //--note: width is changed to the new_width //---------------------------------------------------------- -void run_bfs_cpu(int no_of_nodes, Node *h_graph_nodes, int edge_list_size, - int *h_graph_edges, char *h_graph_mask, - char *h_updating_graph_mask, char *h_graph_visited, - int *h_cost_ref) { +void run_bfs_cpu(int no_of_nodes, Node *h_graph_nodes, int *h_graph_edges, + char *h_graph_mask, char *h_updating_graph_mask, + char *h_graph_visited, int *h_cost_ref) { char stop; int k = 0; do { @@ -186,8 +185,9 @@ int main(int argc, char *argv[]) { int no_of_nodes; int edge_list_size; FILE *fp; - Node *h_graph_nodes; - char *h_graph_mask, *h_updating_graph_mask, *h_graph_visited; + Node *h_graph_nodes = NULL; + char *h_graph_mask = NULL, *h_updating_graph_mask = NULL, + *h_graph_visited = NULL; if (argc < 2) { printf("graph file missing!\n"); @@ -206,17 +206,22 @@ int main(int argc, char *argv[]) { printf("Reading File completed!\n"); + auto read_int = [fp](int *value) { + if (fscanf(fp, "%d", value) != 1) { + fclose(fp); + throw(std::string("error parsing graph file")); + } + }; + int source = 0; - fscanf(fp, "%d", &no_of_nodes); + read_int(&no_of_nodes); - int num_of_blocks = 1; int num_of_threads_per_block = no_of_nodes; // Make execution Parameters according to the number of nodes // Distribute threads across multiple Blocks if necessary if (no_of_nodes > MAX_THREADS_PER_BLOCK) { - num_of_blocks = (int)ceil(no_of_nodes / (double)MAX_THREADS_PER_BLOCK); num_of_threads_per_block = MAX_THREADS_PER_BLOCK; } work_group_size = num_of_threads_per_block; @@ -229,7 +234,8 @@ int main(int argc, char *argv[]) { int start, edgeno; // initalize the memory for (int i = 0; i < no_of_nodes; i++) { - fscanf(fp, "%d %d", &start, &edgeno); + read_int(&start); + read_int(&edgeno); h_graph_nodes[i].starting = start; h_graph_nodes[i].no_of_edges = edgeno; h_graph_mask[i] = false; @@ -237,17 +243,17 @@ int main(int argc, char *argv[]) { h_graph_visited[i] = false; } // read the source node from the file - fscanf(fp, "%d", &source); + read_int(&source); source = 0; // set the source node as true in the mask h_graph_mask[source] = true; h_graph_visited[source] = true; - fscanf(fp, "%d", &edge_list_size); + read_int(&edge_list_size); int id, cost; int *h_graph_edges = (int *)malloc(sizeof(int) * edge_list_size); for (int i = 0; i < edge_list_size; i++) { - fscanf(fp, "%d", &id); - fscanf(fp, "%d", &cost); + read_int(&id); + read_int(&cost); h_graph_edges[i] = id; } @@ -278,7 +284,7 @@ int main(int argc, char *argv[]) { source = 0; h_graph_mask[source] = true; h_graph_visited[source] = true; - run_bfs_cpu(no_of_nodes, h_graph_nodes, edge_list_size, h_graph_edges, + run_bfs_cpu(no_of_nodes, h_graph_nodes, h_graph_edges, h_graph_mask, h_updating_graph_mask, h_graph_visited, h_cost_ref); //--------------------------------------------------------- @@ -290,7 +296,7 @@ int main(int argc, char *argv[]) { free(h_updating_graph_mask); free(h_graph_visited); } catch (std::string msg) { - printf("--cambine: exception in main ->%s\n", msg); + printf("--cambine: exception in main ->%s\n", msg.c_str()); // release host memory free(h_graph_nodes); free(h_graph_mask); diff --git a/tests/opencl/bfs/util.h b/tests/opencl/bfs/util.h index 4d1d3e27b9..0941d18071 100755 --- a/tests/opencl/bfs/util.h +++ b/tests/opencl/bfs/util.h @@ -35,7 +35,9 @@ template void verify_array(const datatype *cpuResults, const datatype *gpuResults, const int size){ char passed = true; +#ifdef _OPENMP #pragma omp parallel for +#endif for (int i=0; i MAX_RELATIVE_ERROR){ passed = false; diff --git a/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp b/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp index 41209e3637..19b89b6c73 100644 --- a/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp +++ b/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp @@ -37,9 +37,8 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return CL_SUCCESS; } -extern "C" void initBlackScholes(cl_context cxGPUContext, cl_command_queue cqParamCommandQueue, const char **argv){ +extern "C" void initBlackScholes(cl_context cxGPUContext, cl_command_queue cqParamCommandQueue, const char ** /*argv*/){ cl_int ciErrNum; - size_t kernelLength; /*shrLog("...loading BlackScholes.cl\n"); char *cPathAndName = shrFindFilePath("BlackScholes.cl", argv[0]); @@ -50,8 +49,6 @@ extern "C" void initBlackScholes(cl_context cxGPUContext, cl_command_queue cqPar shrLog("...creating BlackScholes program\n"); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; - cl_device_id device_id = oclGetFirstDev(cxGPUContext); ciErrNum = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); shrCheckError(ciErrNum, CL_SUCCESS); diff --git a/tests/opencl/blackscholes/shrUtils.cpp b/tests/opencl/blackscholes/shrUtils.cpp index cf0d2c3e7e..fb4bea436f 100644 --- a/tests/opencl/blackscholes/shrUtils.cpp +++ b/tests/opencl/blackscholes/shrUtils.cpp @@ -203,7 +203,6 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { static FILE* pFileStream0 = NULL; static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; char cFileMode [3]; // if the sample log file is closed and the call incudes a "write-to-file", open file for writing @@ -318,12 +317,12 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print string to console if flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console + printf ("\n !!! Error # %i at ", iErrNum); // console } // print string to file if flagged if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file + fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file } } @@ -347,14 +346,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // character is not '%', so print it verbatim to console and/or files as flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = putc(*pStr, stdout); // console + putc(*pStr, stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } @@ -399,16 +398,26 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { case '%': // special handling for string of %%%% { + // collapse each "%%" to a literal '%', as printf would + std::string sLiteral; + for (size_t i = 0; i < sFormatSpec.size(); ++i) + { + sLiteral += sFormatSpec[i]; + if ((sFormatSpec[i] == '%') && (i + 1 < sFormatSpec.size()) && (sFormatSpec[i + 1] == '%')) + { + ++i; + } + } if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str()); // console + fputs(sLiteral.c_str(), stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file + fputs(sLiteral.c_str(), pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file + fputs(sLiteral.c_str(), pFileStream1); // master log file } } continue; @@ -420,14 +429,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list cArg = va_arg(vaArgList, char*); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console + printf(sFormatSpec.c_str(), cArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file } } continue; @@ -439,14 +448,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list iArg = va_arg(vaArgList, int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console + printf(sFormatSpec.c_str(), iArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file } } continue; @@ -460,14 +469,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list uiArg = va_arg(vaArgList, unsigned int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console + printf(sFormatSpec.c_str(), uiArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file } } continue; @@ -484,14 +493,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list dArg = va_arg(vaArgList, double); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console + printf(sFormatSpec.c_str(), dArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file } } continue; @@ -501,14 +510,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print arg of unknown/unsupported type to console and/or file if flagged if (iLogMode & LOGCONSOLE) // console { - szNumWritten = putc(*pStr, stdout); + putc(*pStr, stdout); } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } diff --git a/tests/opencl/cfd/main.cc b/tests/opencl/cfd/main.cc index 330d68ec66..1d8d11c741 100644 --- a/tests/opencl/cfd/main.cc +++ b/tests/opencl/cfd/main.cc @@ -158,9 +158,8 @@ static inline float compute_pressure(float density, float density_energy, float static inline float compute_speed_of_sound(float density, float pressure) { return sqrtf(GAMMA * pressure / density); } -static inline void compute_flux_contribution(float density, FLOAT3 momentum, float density_energy, - float pressure, FLOAT3 velocity, - FLOAT3* fc_momentum_x, FLOAT3* fc_momentum_y, +static inline void compute_flux_contribution(FLOAT3 momentum, float density_energy, float pressure, + FLOAT3 velocity, FLOAT3* fc_momentum_x, FLOAT3* fc_momentum_y, FLOAT3* fc_momentum_z, FLOAT3* fc_density_energy) { fc_momentum_x->x = velocity.x * momentum.x + pressure; fc_momentum_x->y = velocity.x * momentum.y; @@ -216,7 +215,7 @@ static void cpu_compute_flux(int nelr, const int* esse, const float* normals, float pressure_i = compute_pressure(density_i, density_energy_i, speed_sqd_i); float speed_of_sound_i = compute_speed_of_sound(density_i, pressure_i); FLOAT3 fc_i_momentum_x, fc_i_momentum_y, fc_i_momentum_z, fc_i_density_energy; - compute_flux_contribution(density_i, momentum_i, density_energy_i, pressure_i, velocity_i, + compute_flux_contribution(momentum_i, density_energy_i, pressure_i, velocity_i, &fc_i_momentum_x, &fc_i_momentum_y, &fc_i_momentum_z, &fc_i_density_energy); float flux_i_density = 0.0f; @@ -244,7 +243,7 @@ static void cpu_compute_flux(int nelr, const int* esse, const float* normals, speed_sqd_nb = compute_speed_sqd(velocity_nb); pressure_nb = compute_pressure(density_nb, density_energy_nb, speed_sqd_nb); speed_of_sound_nb = compute_speed_of_sound(density_nb, pressure_nb); - compute_flux_contribution(density_nb, momentum_nb, density_energy_nb, pressure_nb, velocity_nb, + compute_flux_contribution(momentum_nb, density_energy_nb, pressure_nb, velocity_nb, &fc_nb_momentum_x, &fc_nb_momentum_y, &fc_nb_momentum_z, &fc_nb_density_energy); factor = -normal_len * smoothing_coefficient * 0.5f * @@ -352,8 +351,7 @@ int main(int argc, char** argv) { ff_momentum.x = h_ff_variable[VAR_MOMENTUM + 0]; ff_momentum.y = h_ff_variable[VAR_MOMENTUM + 1]; ff_momentum.z = h_ff_variable[VAR_MOMENTUM + 2]; - compute_flux_contribution(h_ff_variable[VAR_DENSITY], ff_momentum, - h_ff_variable[VAR_DENSITY_ENERGY], ff_pressure, ff_velocity, + compute_flux_contribution(ff_momentum, h_ff_variable[VAR_DENSITY_ENERGY], ff_pressure, ff_velocity, &h_ff_fc_momentum_x, &h_ff_fc_momentum_y, &h_ff_fc_momentum_z, &h_ff_fc_density_energy); } diff --git a/tests/opencl/common.mk b/tests/opencl/common.mk index e35673e00e..84774d3e2d 100644 --- a/tests/opencl/common.mk +++ b/tests/opencl/common.mk @@ -51,6 +51,7 @@ VX_CFLAGS += -Xclang -target-feature -Xclang +xvortex VX_CFLAGS += -Xclang -target-feature -Xclang +zicond VX_LDFLAGS += -fuse-ld=lld VX_CFLAGS += -mllvm -disable-loop-idiom-all # disable memset/memcpy loop replacement +VX_CFLAGS += -Wall -Wextra -Wfatal-errors -Werror VX_LDFLAGS += -Wl,-z,norelro #VX_CFLAGS += -mllvm -vortex-branch-divergence=0 #VX_CFLAGS += -mllvm -debug -mllvm -print-after-all @@ -59,10 +60,11 @@ VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T$(VORTEX_HOME)/sw/kernel/scripts/link VX_BINTOOL += OBJCOPY=$(LLVM_PATH)/bin/llvm-objcopy $(VORTEX_HOME)/sw/kernel/scripts/vxbin.py -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -CXXFLAGS += -Wno-deprecated-declarations -Wno-unused-parameter -Wno-narrowing +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror CXXFLAGS += -pthread CXXFLAGS += -I$(POCL_PATH)/include +# The tests are written against the OpenCL 1.2 host API. +CXXFLAGS += -DCL_TARGET_OPENCL_VERSION=120 CXXFLAGS += $(CONFIGS) POCL_CC_FLAGS += LLVM_PREFIX=$(LLVM_PATH) POCL_VORTEX_BINTOOL="$(VX_BINTOOL)" POCL_VORTEX_CFLAGS="$(VX_CFLAGS)" POCL_VORTEX_LDFLAGS="$(VX_LDFLAGS)" @@ -99,8 +101,9 @@ all: $(PROJECT) %.cpp.o: $(SRC_DIR)/%.cpp $(CXX) $(CXXFLAGS) -c $< -o $@ +# Host C sources share the C++ flags except the language standard. %.c.o: $(SRC_DIR)/%.c - $(CC) $(CXXFLAGS) -c $< -o $@ + $(CC) $(filter-out -std=c++%,$(CXXFLAGS)) -c $< -o $@ $(VORTEX_KN_PATH)/libvortex.a: $(MAKE) -C $(VORTEX_KN_PATH) diff --git a/tests/opencl/conv3/main.cc b/tests/opencl/conv3/main.cc index 1220dabdb2..44bfe64886 100644 --- a/tests/opencl/conv3/main.cc +++ b/tests/opencl/conv3/main.cc @@ -180,7 +180,7 @@ int main (int argc, char **argv) { // Create kernel kernel = CL_CHECK2(clCreateKernel(program, KERNEL_NAME, &_err)); - size_t global_size[2] = {size, size}; + size_t global_size[2] = {static_cast(size), static_cast(size)}; size_t local_size[2] = {1, 1}; // Set kernel arguments diff --git a/tests/opencl/dotproduct/main.cc b/tests/opencl/dotproduct/main.cc index 789b41491b..fe4418228b 100644 --- a/tests/opencl/dotproduct/main.cc +++ b/tests/opencl/dotproduct/main.cc @@ -165,7 +165,6 @@ int main(int argc, char **argv) // Create the program shrLog("clCreateProgramWithSource...\n"); - cl_int binary_status; cl_program program; program = clCreateProgramWithSource( diff --git a/tests/opencl/dotproduct/shrUtils.cpp b/tests/opencl/dotproduct/shrUtils.cpp index cf0d2c3e7e..fb4bea436f 100644 --- a/tests/opencl/dotproduct/shrUtils.cpp +++ b/tests/opencl/dotproduct/shrUtils.cpp @@ -203,7 +203,6 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { static FILE* pFileStream0 = NULL; static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; char cFileMode [3]; // if the sample log file is closed and the call incudes a "write-to-file", open file for writing @@ -318,12 +317,12 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print string to console if flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console + printf ("\n !!! Error # %i at ", iErrNum); // console } // print string to file if flagged if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file + fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file } } @@ -347,14 +346,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // character is not '%', so print it verbatim to console and/or files as flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = putc(*pStr, stdout); // console + putc(*pStr, stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } @@ -399,16 +398,26 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { case '%': // special handling for string of %%%% { + // collapse each "%%" to a literal '%', as printf would + std::string sLiteral; + for (size_t i = 0; i < sFormatSpec.size(); ++i) + { + sLiteral += sFormatSpec[i]; + if ((sFormatSpec[i] == '%') && (i + 1 < sFormatSpec.size()) && (sFormatSpec[i + 1] == '%')) + { + ++i; + } + } if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str()); // console + fputs(sLiteral.c_str(), stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file + fputs(sLiteral.c_str(), pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file + fputs(sLiteral.c_str(), pFileStream1); // master log file } } continue; @@ -420,14 +429,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list cArg = va_arg(vaArgList, char*); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console + printf(sFormatSpec.c_str(), cArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file } } continue; @@ -439,14 +448,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list iArg = va_arg(vaArgList, int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console + printf(sFormatSpec.c_str(), iArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file } } continue; @@ -460,14 +469,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list uiArg = va_arg(vaArgList, unsigned int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console + printf(sFormatSpec.c_str(), uiArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file } } continue; @@ -484,14 +493,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list dArg = va_arg(vaArgList, double); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console + printf(sFormatSpec.c_str(), dArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file } } continue; @@ -501,14 +510,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print arg of unknown/unsupported type to console and/or file if flagged if (iLogMode & LOGCONSOLE) // console { - szNumWritten = putc(*pStr, stdout); + putc(*pStr, stdout); } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } diff --git a/tests/opencl/guassian/clutils.cpp b/tests/opencl/guassian/clutils.cpp index 2881028489..53a25ff70d 100755 --- a/tests/opencl/guassian/clutils.cpp +++ b/tests/opencl/guassian/clutils.cpp @@ -71,9 +71,6 @@ static cl_uint numPlatforms; static cl_device_id* devices = NULL; static cl_uint* numDevices = NULL; -//! The chosen OpenCL platform -static cl_platform_id platform = NULL; - //! The chosen OpenCL device static cl_device_id device = NULL; @@ -82,11 +79,6 @@ static cl_context context = NULL; //! OpenCL command queue static cl_command_queue commandQueue = NULL; -static cl_command_queue commandQueueProf = NULL; -static cl_command_queue commandQueueNoProf = NULL; - -//! Global status of events -static bool eventsEnabled = false; //------------------------------------------------------- // Initialization and Cleanup @@ -260,9 +252,7 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) } -cl_context cl_init_context(int platform, int dev,int quiet) { - int printInfo=1; - if (platform >= 0 && dev >= 0) printInfo = 0; +cl_context cl_init_context(int /*platform*/, int /*dev*/, int /*quiet*/) { cl_int status; // Used to iterate through the platforms and devices, respectively @@ -401,7 +391,6 @@ cl_context cl_init_context(int platform, int dev,int quiet) { numDevices[0] = 1; devices = (cl_device_id*)malloc(sizeof(cl_device_id)*numDevices[0]); - int platform_touse = 0; int device_touse = 0; status = clGetPlatformIDs(numPlatforms, platforms, NULL); @@ -644,7 +633,11 @@ cl_mem cl_allocImage(size_t height, size_t width, char type, cl_mem_flags flags) allocationSize += height*width*elemSize; // Create the image - mem = clCreateImage2D(context, flags, &format, width, height, 0, NULL, &status); + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE2D; + desc.image_width = width; + desc.image_height = height; + mem = clCreateImage(context, flags, &format, &desc, NULL, &status); //cl_errChk(status, "creating image", true); if(status != CL_SUCCESS) { @@ -726,7 +719,7 @@ void cl_copyBufferToHost(void* dst, cl_mem src, size_t mem_size, cl_bool blockin void cl_copyBufferToImage(cl_mem buffer, cl_mem image, int height, int width) { size_t origin[3] = {0, 0, 0}; - size_t region[3] = {width, height, 1}; + size_t region[3] = {static_cast(width), static_cast(height), 1}; cl_int status; status = clEnqueueCopyBufferToImage(commandQueue, buffer, image, 0, @@ -833,12 +826,9 @@ Compile Opencl source file into a cl_program. The cl_program will be made into a \param compileoptions Compilation options \param verbosebuild Switch to enable verbose Output */ -cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbosebuild ) +cl_program cl_compileProgram(char* /*kernelPath*/, char* compileoptions, bool verbosebuild ) { cl_int status; - FILE *fp = NULL; - char *source = NULL; - long int size; /*printf("\t%s\n", kernelPath); @@ -879,7 +869,6 @@ cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbos // read kernel binary from file uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgramReturn; // Create the program object @@ -1421,8 +1410,7 @@ char* catStringWithInt(const char* string, int integer) { char tmp[6]; - strcpy(eventStr, string); - strncat(eventStr, itoa_portable(integer, tmp, 10), 5); + snprintf(eventStr, strLen, "%s%s", string, itoa_portable(integer, tmp, 10)); return eventStr; } diff --git a/tests/opencl/guassian/main.cc b/tests/opencl/guassian/main.cc index fbb99576e4..a06ebbbdcd 100755 --- a/tests/opencl/guassian/main.cc +++ b/tests/opencl/guassian/main.cc @@ -31,7 +31,10 @@ int main(int argc, char *argv[]) { if (size == 0) { fp = fopen(filename, "r"); - fscanf(fp, "%d", &size); + if (fscanf(fp, "%d", &size) != 1) { + printf("Error reading matrix size from %s\n", filename); + exit(1); + } printf("using %dx%d input matrix\n", size, size); @@ -200,8 +203,9 @@ void ForwardSub(cl_context context, float *a, float *b, float *m, int size, writeMB = (float)(sizeof(float) * size * (size + size + 1) / 1e6); // 3. Determine block sizes - size_t globalWorksizeFan1[1] = {size}; - size_t globalWorksizeFan2[2] = {size, size}; + size_t globalWorksizeFan1[1] = {static_cast(size)}; + size_t globalWorksizeFan2[2] = {static_cast(size), + static_cast(size)}; size_t localWorksizeFan1[1] = {1}; size_t localWorksizeFan2[2] = {1, 1}; @@ -425,7 +429,10 @@ void InitMat(FILE *fp, int size, float *ary, int nrow, int ncol) { for (i = 0; i < nrow; i++) { for (j = 0; j < ncol; j++) { - fscanf(fp, "%f", ary + size * i + j); + if (fscanf(fp, "%f", ary + size * i + j) != 1) { + printf("Error reading input matrix\n"); + exit(1); + } } } } @@ -438,7 +445,10 @@ void InitAry(FILE *fp, float *ary, int ary_size) { int i; for (i = 0; i < ary_size; i++) { - fscanf(fp, "%f", &ary[i]); + if (fscanf(fp, "%f", &ary[i]) != 1) { + printf("Error reading input array\n"); + exit(1); + } } } /*------------------------------------------------------ diff --git a/tests/opencl/image_get/main.cc b/tests/opencl/image_get/main.cc index 8ceae1749f..3ec48fb636 100644 --- a/tests/opencl/image_get/main.cc +++ b/tests/opencl/image_get/main.cc @@ -21,9 +21,12 @@ static int read_kernel_file(const char* fn, uint8_t** data, size_t* size) { cl_device_id device_id = NULL; cl_context context = NULL; cl_command_queue queue = NULL; cl_program program = NULL; cl_kernel kernel = NULL; cl_mem image = NULL, out_buf = NULL; static void cleanup() { - if (queue) clReleaseCommandQueue(queue); if (kernel) clReleaseKernel(kernel); - if (program) clReleaseProgram(program); if (image) clReleaseMemObject(image); - if (out_buf) clReleaseMemObject(out_buf); if (context) clReleaseContext(context); + if (queue) { clReleaseCommandQueue(queue); } + if (kernel) { clReleaseKernel(kernel); } + if (program) { clReleaseProgram(program); } + if (image) { clReleaseMemObject(image); } + if (out_buf) { clReleaseMemObject(out_buf); } + if (context) { clReleaseContext(context); } } int main(int argc, char** argv) { diff --git a/tests/opencl/image_int/main.cc b/tests/opencl/image_int/main.cc index 2665fa6ee5..af04df8148 100644 --- a/tests/opencl/image_int/main.cc +++ b/tests/opencl/image_int/main.cc @@ -23,10 +23,13 @@ cl_device_id device_id = NULL; cl_context context = NULL; cl_command_queue queue cl_program program = NULL; cl_kernel kernel = NULL; cl_sampler sampler = NULL; cl_mem src_image = NULL, dst_image = NULL; static void cleanup() { - if (queue) clReleaseCommandQueue(queue); if (kernel) clReleaseKernel(kernel); - if (program) clReleaseProgram(program); if (sampler) clReleaseSampler(sampler); - if (src_image) clReleaseMemObject(src_image); if (dst_image) clReleaseMemObject(dst_image); - if (context) clReleaseContext(context); + if (queue) { clReleaseCommandQueue(queue); } + if (kernel) { clReleaseKernel(kernel); } + if (program) { clReleaseProgram(program); } + if (sampler) { clReleaseSampler(sampler); } + if (src_image) { clReleaseMemObject(src_image); } + if (dst_image) { clReleaseMemObject(dst_image); } + if (context) { clReleaseContext(context); } } int main(int argc, char** argv) { @@ -66,7 +69,11 @@ int main(int argc, char** argv) { int errors=0; for (size_t i=0;i(n_points), 1, 1}; /// Ke Wang adjustable local group size 2013/08/07 10:37:33 size_t local_work_size = BLOCK_SIZE; // work group size is defined by // RD_WG_SIZE_0 or RD_WG_SIZE_0_0 @@ -335,10 +332,10 @@ int kmeansOCL(float **feature, /* in: [npoints][nfeatures] */ float **clusters, int *new_centers_len, float **new_centers) { int delta = 0; - int i, j, k; + int i, j; cl_int err = 0; - size_t global_work[3] = {n_points, 1, 1}; + size_t global_work[3] = {static_cast(n_points), 1, 1}; /// Ke Wang adjustable local group size 2013/08/07 10:37:33 size_t local_work_size = BLOCK_SIZE2; // work group size is defined by diff --git a/tests/opencl/kmeans/read_input.c b/tests/opencl/kmeans/read_input.c index f70e7fb300..1dfc2df346 100755 --- a/tests/opencl/kmeans/read_input.c +++ b/tests/opencl/kmeans/read_input.c @@ -126,7 +126,6 @@ int setup(int argc, char **argv) { int best_nclusters = 0; int nfeatures = 16; /* small self-contained default */ int npoints = 512; /* small self-contained default */ - float len; float **features; float **cluster_centres = NULL; @@ -192,8 +191,11 @@ int setup(int argc, char **argv) { fprintf(stderr, "Error: no such file (%s)\n", filename); exit(1); } - fread(&npoints, 1, sizeof(int), infile); - fread(&nfeatures, 1, sizeof(int), infile); + if (fread(&npoints, 1, sizeof(int), infile) != sizeof(int) || + fread(&nfeatures, 1, sizeof(int), infile) != sizeof(int)) { + fprintf(stderr, "Error: failed to read header of (%s)\n", filename); + exit(1); + } // allocate space for features[][] and read attributes of all objects buf = (float *)malloc(npoints * nfeatures * sizeof(float)); @@ -202,7 +204,11 @@ int setup(int argc, char **argv) { for (i = 1; i < npoints; i++) { features[i] = features[i - 1] + nfeatures; } - fread(buf, 1, npoints * nfeatures * sizeof(float), infile); + if (fread(buf, 1, npoints * nfeatures * sizeof(float), infile) != + npoints * nfeatures * sizeof(float)) { + fprintf(stderr, "Error: failed to read features of (%s)\n", filename); + exit(1); + } fclose(infile); } else { FILE *infile; @@ -320,8 +326,6 @@ int setup(int argc, char **argv) { } } - len = (float)((max_nclusters - min_nclusters + 1) * nloops); - printf("Number of Iteration: %d\n", nloops); // printf("Time for I/O: %.5fsec\n", io_timing); // printf("Time for Entire Clustering: %.5fsec\n", cluster_timing); diff --git a/tests/opencl/kmeans/rmse.c b/tests/opencl/kmeans/rmse.c index 03d614a624..2f6cee3c61 100755 --- a/tests/opencl/kmeans/rmse.c +++ b/tests/opencl/kmeans/rmse.c @@ -42,7 +42,7 @@ int find_nearest_point(float *pt, /* [nfeatures] */ float **pts, /* [npts][nfeatures] */ int npts) { - int index, i; + int index = 0, i; float max_dist=FLT_MAX; /* find the cluster center id with min distance to pt */ @@ -70,11 +70,13 @@ float rms_err (float **feature, /* [npoints][nfeatures] */ float ret; /* return value */ /* calculate and sum the sqaure of euclidean distance*/ +#ifdef _OPENMP #pragma omp parallel for \ shared(feature,cluster_centres) \ firstprivate(npoints,nfeatures,nclusters) \ private(i, nearest_cluster_index) \ - schedule (static) + schedule (static) +#endif for (i=0; iasync_markers != NULL) && - (timers->async_markers->timerID != INVALID_TIMERID); -} - -static struct pb_async_time_marker_list * -get_last_async(struct pb_TimerSet* timers) -{ - /* Find the last event recorded thus far */ - struct pb_async_time_marker_list * last_event = timers->async_markers; - if(last_event != NULL && last_event->timerID != INVALID_TIMERID) { - while(last_event->next != NULL && - last_event->next->timerID != INVALID_TIMERID) - last_event = last_event->next; - return last_event; - } else - return NULL; -} - -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - -static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = label; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - - -/* Assumes that all recorded events have completed */ -static pb_Timestamp record_async_times(struct pb_TimerSet* tset) -{ - struct pb_async_time_marker_list * next_interval = NULL; - struct pb_async_time_marker_list * last_marker = get_last_async(tset); - pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; - - for(next_interval = tset->async_markers; next_interval != last_marker; - next_interval = next_interval->next) { - cl_ulong command_start=0, command_end=0; - cl_int ciErrNum = CL_SUCCESS; - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_start, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting first EventProfilingInfo: %d\n", ciErrNum); - } - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->next->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_end, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting second EventProfilingInfo: %d\n", ciErrNum); - } - - pb_Timestamp interval = (pb_Timestamp) (((double)(command_end - command_start)) / 1e3); - tset->timers[next_interval->timerID].elapsed += interval; - if (next_interval->label != NULL) { - struct pb_SubTimer *subtimer = tset->sub_timer_list[next_interval->timerID]->subtimer_list; - while (subtimer != NULL) { - if ( strcmp(subtimer->label, next_interval->label) == 0) { - subtimer->timer.elapsed += interval; - break; - } - subtimer = subtimer->next; - } - } - total_async_time += interval; - next_interval->timerID = INVALID_TIMERID; - } - - if(next_interval != NULL) - next_interval->timerID = INVALID_TIMERID; - - return total_async_time; -} - -static void -accumulate_time(pb_Timestamp *accum, - pb_Timestamp start, - pb_Timestamp end) -{ -//#if _POSIX_VERSION >= 200112L - *accum += end - start; -//#else -//# error "Timestamps not implemented for this system" -//#endif -} - -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - void pb_ResetTimer(struct pb_Timer *timer) { @@ -222,6 +40,7 @@ pb_ResetTimer(struct pb_Timer *timer) void pb_StartTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER if (timer->state != pb_Timer_STOPPED) { fputs("Ignoring attempt to start a running timer\n", stderr); @@ -245,6 +64,8 @@ pb_StartTimer(struct pb_Timer *timer) void pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER unsigned int numNotStopped = 0x3; // 11 @@ -287,6 +108,7 @@ pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) void pb_StopTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -315,6 +137,8 @@ pb_StopTimer(struct pb_Timer *timer) } void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -364,6 +188,7 @@ void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) double pb_GetElapsedTime(struct pb_Timer *timer) { + (void)timer; /*double ret; #ifndef DISABLE_PARBOIL_TIMER @@ -384,6 +209,7 @@ pb_GetElapsedTime(struct pb_Timer *timer) void pb_InitializeTimerSet(struct pb_TimerSet *timers) { + (void)timers; /*#ifndef DISABLE_PARBOIL_TIMER int n; @@ -406,6 +232,9 @@ void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { void pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + (void)timers; + (void)label; + (void)pb_Category; /*#ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc @@ -440,6 +269,8 @@ pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Categ void pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) { + (void)timers; + (void)timer; #if 0 #ifndef DISABLE_PARBOIL_TIMER @@ -560,6 +391,9 @@ pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) void pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category) { + (void)timers; + (void)label; + (void)category; #if 0 #ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; @@ -708,6 +542,7 @@ pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID cat void pb_PrintTimerSet(struct pb_TimerSet *timers) { + (void)timers; #if 0 #ifndef DISABLE_PARBOIL_TIMER pb_Timestamp wall_end = 0; //get_time(); @@ -1198,6 +1033,7 @@ pb_GetPlatform(struct pb_PlatformParam *platform) { pb_Context* pb_InitOpenCLContext(struct pb_Parameters* parameters) { + (void)parameters; #if 0 pb_Platform* ps = pb_GetPlatform(parameters->platform); if (!ps) return NULL; @@ -1254,11 +1090,13 @@ pb_InitOpenCLContext(struct pb_Parameters* parameters) { void pb_ReleaseOpenCLContext(pb_Context* c) { + (void)c; pb_ReleasePlatforms(); } void pb_PrintPlatformInfo(pb_Context* c) { + (void)c; /*pb_Platform** ps = pb_GetPlatforms(); if (!ps) { fprintf (stderr, "No platform found"); @@ -1370,7 +1208,7 @@ pb_sig_int(char* c, int* p, int sz) { void pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; + unsigned int i; unsigned long long int s = 0; for (i = 0; i < sz; i++) s += p[i]; printf ("[Signature] %s = %lld\n", c, s); diff --git a/tests/opencl/myocyte/myocyte_model.h b/tests/opencl/myocyte/myocyte_model.h index e4212ecc55..2aff0285f7 100644 --- a/tests/opencl/myocyte/myocyte_model.h +++ b/tests/opencl/myocyte/myocyte_model.h @@ -922,7 +922,7 @@ kernel_ecc( fp timeinst, case 2: V_hold = -55; V_test = 0; - if(timeinst>0.5 & timeinst<200.5){ + if((timeinst>0.5) & (timeinst<200.5)){ V_clamp = V_test; } else{ @@ -951,7 +951,7 @@ kernel_ecc( fp timeinst, //========================================================================================================================================================================================================200 void -kernel_cam( fp timeinst, +kernel_cam( fp /*timeinst*/, GLOBAL fp *d_initvalu, GLOBAL fp *d_finavalu, int valu_offset, diff --git a/tests/opencl/nearn/clutils.cpp b/tests/opencl/nearn/clutils.cpp index b411b19b15..8cd8065195 100755 --- a/tests/opencl/nearn/clutils.cpp +++ b/tests/opencl/nearn/clutils.cpp @@ -71,9 +71,6 @@ static cl_uint numPlatforms; static cl_device_id* devices = NULL; static cl_uint* numDevices = NULL; -//! The chosen OpenCL platform -static cl_platform_id platform = NULL; - //! The chosen OpenCL device static cl_device_id device = NULL; @@ -82,11 +79,6 @@ static cl_context context = NULL; //! OpenCL command queue static cl_command_queue commandQueue = NULL; -static cl_command_queue commandQueueProf = NULL; -static cl_command_queue commandQueueNoProf = NULL; - -//! Global status of events -static bool eventsEnabled = false; //------------------------------------------------------- // Initialization and Cleanup @@ -259,9 +251,7 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return 0; } -cl_context cl_init_context(int platform, int dev,int quiet) { - int printInfo=1; - if (platform >= 0 && dev >= 0) printInfo = 0; +cl_context cl_init_context(int /*platform*/, int /*dev*/, int /*quiet*/) { cl_int status; // Used to iterate through the platforms and devices, respectively @@ -400,7 +390,6 @@ cl_context cl_init_context(int platform, int dev,int quiet) { numDevices[0] = 1; devices = (cl_device_id*)malloc(sizeof(cl_device_id)*numDevices[0]); - int platform_touse = 0; int device_touse = 0; status = clGetPlatformIDs(numPlatforms, platforms, NULL); @@ -643,7 +632,12 @@ cl_mem cl_allocImage(size_t height, size_t width, char type, cl_mem_flags flags) allocationSize += height*width*elemSize; // Create the image - mem = clCreateImage2D(context, flags, &format, width, height, 0, NULL, &status); + cl_image_desc desc; + memset(&desc, 0, sizeof(desc)); + desc.image_type = CL_MEM_OBJECT_IMAGE2D; + desc.image_width = width; + desc.image_height = height; + mem = clCreateImage(context, flags, &format, &desc, NULL, &status); //cl_errChk(status, "creating image", true); if(status != CL_SUCCESS) { @@ -725,7 +719,7 @@ void cl_copyBufferToHost(void* dst, cl_mem src, size_t mem_size, cl_bool blockin void cl_copyBufferToImage(cl_mem buffer, cl_mem image, int height, int width) { size_t origin[3] = {0, 0, 0}; - size_t region[3] = {width, height, 1}; + size_t region[3] = {static_cast(width), static_cast(height), 1}; cl_int status; status = clEnqueueCopyBufferToImage(commandQueue, buffer, image, 0, @@ -832,12 +826,9 @@ Compile Opencl source file into a cl_program. The cl_program will be made into a \param compileoptions Compilation options \param verbosebuild Switch to enable verbose Output */ -cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbosebuild ) +cl_program cl_compileProgram(char* /*kernelPath*/, char* compileoptions, bool verbosebuild ) { cl_int status; - FILE *fp = NULL; - char *source = NULL; - long int size; /*printf("\t%s\n", kernelPath); @@ -878,7 +869,6 @@ cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbos // read kernel binary from file uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgramReturn; // Create the program object @@ -1420,8 +1410,7 @@ char* catStringWithInt(const char* string, int integer) { char tmp[6]; - strcpy(eventStr, string); - strncat(eventStr, itoa_portable(integer, tmp, 10), 5); + snprintf(eventStr, strLen, "%s%s", string, itoa_portable(integer, tmp, 10)); return eventStr; } diff --git a/tests/opencl/nearn/main.cc b/tests/opencl/nearn/main.cc index 973dcd9c8c..ea8238e167 100755 --- a/tests/opencl/nearn/main.cc +++ b/tests/opencl/nearn/main.cc @@ -209,8 +209,6 @@ int loadData(char *filename, std::vector &records, /**Main processing **/ - int q = 0; - flist = fopen(filename, "r"); while (!feof(flist)) { /** @@ -232,7 +230,13 @@ int loadData(char *filename, std::vector &records, while (!feof(fp)) { Record record; LatLong latLong; - fgets(record.recString, 49, fp); + if (fgets(record.recString, 49, fp) == NULL) { + if (ferror(fp)) { + printf("error reading a db\n"); + exit(1); + } + break; + } fgetc(fp); // newline if (feof(fp)) break; diff --git a/tests/opencl/oclprintf/main.cc b/tests/opencl/oclprintf/main.cc index c23e6dec08..73dc536716 100644 --- a/tests/opencl/oclprintf/main.cc +++ b/tests/opencl/oclprintf/main.cc @@ -147,7 +147,7 @@ int main (int argc, char **argv) { CL_CHECK(clEnqueueWriteBuffer(commandQueue, a_memobj, CL_TRUE, 0, nbytes, h_a, 0, NULL, NULL)); printf("Execute the kernel\n"); - size_t global_work_size[1] = {size}; + size_t global_work_size[1] = {static_cast(size)}; size_t local_work_size[1] = {1}; auto time_start = std::chrono::high_resolution_clock::now(); CL_CHECK(clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL)); diff --git a/tests/opencl/psort/main.cc b/tests/opencl/psort/main.cc index 8ecfdc5239..0e72599ece 100644 --- a/tests/opencl/psort/main.cc +++ b/tests/opencl/psort/main.cc @@ -167,7 +167,7 @@ int main (int argc, char **argv) { CL_CHECK(clEnqueueWriteBuffer(commandQueue, a_memobj, CL_TRUE, 0, nbytes, h_a, 0, NULL, NULL)); printf("Execute the kernel\n"); - size_t global_work_size[1] = {size}; + size_t global_work_size[1] = {static_cast(size)}; size_t local_work_size[1] = {1}; auto time_start = std::chrono::high_resolution_clock::now(); CL_CHECK(clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL)); diff --git a/tests/opencl/saxpy/main.cc b/tests/opencl/saxpy/main.cc index 2397c720e4..2cb4b71ba8 100644 --- a/tests/opencl/saxpy/main.cc +++ b/tests/opencl/saxpy/main.cc @@ -53,8 +53,8 @@ _ret; \ }) -void pfn_notify(const char *errinfo, const void *private_info, size_t cb, - void *user_data) { +void pfn_notify(const char *errinfo, const void * /*private_info*/, size_t /*cb*/, + void * /*user_data*/) { fprintf(stderr, "OpenCL Error (via pfn_notify): %s\n", errinfo); } @@ -198,7 +198,7 @@ int main(int argc, char **argv) { CL_CHECK(clSetKernelArg(kernel, 2, sizeof(factor), &factor)); size_t global_offset[1] = {0}; - size_t global_work_size[1] = {size}; + size_t global_work_size[1] = {static_cast(size)}; size_t local_work_size[1] = {1}; printf("initialize buffers\n"); diff --git a/tests/opencl/sfilter/main.cc b/tests/opencl/sfilter/main.cc index 97cfb689e7..c56d26ffdc 100644 --- a/tests/opencl/sfilter/main.cc +++ b/tests/opencl/sfilter/main.cc @@ -56,8 +56,8 @@ _ret; \ }) -void pfn_notify(const char *errinfo, const void *private_info, size_t cb, - void *user_data) { +void pfn_notify(const char *errinfo, const void * /*private_info*/, size_t /*cb*/, + void * /*user_data*/) { fprintf(stderr, "OpenCL Error (via pfn_notify): %s\n", errinfo); } @@ -220,7 +220,7 @@ int main(int argc, char **argv) { CL_CHECK(clSetKernelArg(kernel, 11, sizeof(m8), (&m8))); size_t global_offset[2] = {1, 1}; - size_t global_work_size[2] = {size - 2, size - 2}; + size_t global_work_size[2] = {static_cast(size - 2), static_cast(size - 2)}; size_t local_work_size[2] = {1, 1}; // {size-2,1} printf("enqueue write buffer\n"); diff --git a/tests/opencl/spmv/args.c b/tests/opencl/spmv/args.c index 9d751e2940..42637fe1c5 100644 --- a/tests/opencl/spmv/args.c +++ b/tests/opencl/spmv/args.c @@ -225,7 +225,7 @@ read_platform_param(char *str) /* The platform name is either the entire string, or all characters before * the separator */ - int name_length = separator_index == -1 ? strlen(str) : separator_index; + int name_length = separator_index == -1 ? (int)strlen(str) : separator_index; char *name_str = (char *)malloc(name_length + 1); memcpy(name_str, str, name_length); name_str[name_length] = 0; @@ -410,6 +410,8 @@ parse_input_files(struct argparse *ap, struct pb_Parameters *params) static result parse_end_options(struct argparse *ap, struct pb_Parameters *params) { + (void)ap; + (void)params; return ARGPARSE_DONE; } @@ -500,7 +502,6 @@ is_last_option(struct option *op) static int pb_ParseParameters (struct pb_Parameters *ret, int *_argc, char **argv) { - char *err_message; struct argparse ap; /* Each argument */ diff --git a/tests/opencl/spmv/convert_dataset.c b/tests/opencl/spmv/convert_dataset.c index aba9c3b3de..8aab7b32d4 100644 --- a/tests/opencl/spmv/convert_dataset.c +++ b/tests/opencl/spmv/convert_dataset.c @@ -83,7 +83,6 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, FILE *f; int nz; int i; - float *val; mat_entry *entries; row_stats *stats; int rows, cols; @@ -126,10 +125,16 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, for (i = 0; i < nz; i++, cur_i++) { if (!binary) { - fscanf(f, "%d %d %f\n", &entries[cur_i].row, &entries[cur_i].col, - &entries[cur_i].val); + if (fscanf(f, "%d %d %f\n", &entries[cur_i].row, &entries[cur_i].col, + &entries[cur_i].val) != 3) { + printf("Could not read matrix entry %d.\n", i); + exit(1); + } } else { - fscanf(f, "%d %d\n", &entries[cur_i].row, &entries[cur_i].col); + if (fscanf(f, "%d %d\n", &entries[cur_i].row, &entries[cur_i].col) != 2) { + printf("Could not read matrix entry %d.\n", i); + exit(1); + } entries[cur_i].val = 1.0; } entries[cur_i].row--; @@ -216,7 +221,7 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, if (debug_level >= 1) { printf("Padding data....%d rows, %d groups\n", rows, *nz_count_len); } - int pad_to, total_padding = 0, pack_to; + int pad_to = 0, total_padding = 0, pack_to; pad_rows *= pack_size; // change padding to account for packed items for (i = 0; i < rows; i++) { // record JDS to real row number @@ -281,7 +286,7 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, */ irow = 0; // keep track of which row we are in inside the fubini-ed array int idata = 0; // position within final data array - int entry_index, j; + int entry_index; int ipack; // used in internal loop for writing packed values mat_entry entry; while (1) { diff --git a/tests/opencl/spmv/file.c b/tests/opencl/spmv/file.c index a98451e56b..23709bad19 100644 --- a/tests/opencl/spmv/file.c +++ b/tests/opencl/spmv/file.c @@ -28,27 +28,50 @@ void inputData(char* fName, int* len, int* depth, int* dim,int *nzcnt_len,int *p exit(-1); } - fscanf(fid, "%d %d %d %d %d\n",len,depth,nzcnt_len,dim,pad); + if (fscanf(fid, "%d %d %d %d %d\n",len,depth,nzcnt_len,dim,pad) != 5) + { + fprintf(stderr, "Error reading input file header\n"); + exit(-1); + } int _len=len[0]; int _depth=depth[0]; int _dim=dim[0]; - int _pad=pad[0]; int _nzcnt_len=nzcnt_len[0]; *h_data = (float *) malloc(_len * sizeof (float)); - fread (*h_data, sizeof (float), _len, fid); + if (fread (*h_data, sizeof (float), _len, fid) != (size_t)_len) + { + fprintf(stderr, "Error reading input file data\n"); + exit(-1); + } *h_indices = (int *) malloc(_len * sizeof (int)); - fread (*h_indices, sizeof (int), _len, fid); + if (fread (*h_indices, sizeof (int), _len, fid) != (size_t)_len) + { + fprintf(stderr, "Error reading input file indices\n"); + exit(-1); + } *h_ptr = (int *) malloc(_depth * sizeof (int)); - fread (*h_ptr, sizeof (int), _depth, fid); + if (fread (*h_ptr, sizeof (int), _depth, fid) != (size_t)_depth) + { + fprintf(stderr, "Error reading input file row pointers\n"); + exit(-1); + } *h_perm = (int *) malloc(_dim * sizeof (int)); - fread (*h_perm, sizeof (int), _dim, fid); + if (fread (*h_perm, sizeof (int), _dim, fid) != (size_t)_dim) + { + fprintf(stderr, "Error reading input file permutation\n"); + exit(-1); + } *h_nzcnt = (int *) malloc(_nzcnt_len * sizeof (int)); - fread (*h_nzcnt, sizeof (int), _nzcnt_len, fid); + if (fread (*h_nzcnt, sizeof (int), _nzcnt_len, fid) != (size_t)_nzcnt_len) + { + fprintf(stderr, "Error reading input file nonzero counts\n"); + exit(-1); + } fclose (fid); } @@ -56,7 +79,11 @@ void inputData(char* fName, int* len, int* depth, int* dim,int *nzcnt_len,int *p void input_vec(char *fName,float *h_vec,int dim) { FILE* fid = fopen(fName, "rb"); - fread (h_vec, sizeof (float), dim, fid); + if (fread (h_vec, sizeof (float), dim, fid) != (size_t)dim) + { + fprintf(stderr, "Error reading input vector file\n"); + exit(-1); + } fclose(fid); } diff --git a/tests/opencl/spmv/main.cc b/tests/opencl/spmv/main.cc index 0be7565cd0..fada19c156 100644 --- a/tests/opencl/spmv/main.cc +++ b/tests/opencl/spmv/main.cc @@ -50,7 +50,7 @@ static float* read_output_file(const char* filename, int* out_size) { return NULL; } // Read the float data - if (fread(floats, sizeof(float), size, file) != size) { + if (fread(floats, sizeof(float), size, file) != (size_t)size) { fclose(file); free(floats); perror("Error reading floats from file"); @@ -109,15 +109,6 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return CL_SUCCESS; } -static int generate_vector(float *x_vector, int dim) { - srand(54321); - int i; - for (i = 0; i < dim; i++) { - x_vector[i] = (rand() / (float)RAND_MAX); - } - return 0; -} - int main(int argc, char **argv) { struct pb_TimerSet timers; struct pb_Parameters *parameters; @@ -146,7 +137,6 @@ int main(int argc, char **argv) { } cl_device_id clDevice = (cl_device_id)pb_context->clDeviceId; - cl_platform_id clPlatform = (cl_platform_id)pb_context->clPlatformId; cl_context clContext = (cl_context)pb_context->clContext; cl_command_queue clCommandQueue = clCreateCommandQueue( clContext, clDevice, CL_QUEUE_PROFILING_ENABLE, &clStatus); @@ -159,7 +149,6 @@ int main(int argc, char **argv) { // clCreateProgramWithSource(clContext,1,clSource,NULL,&clStatus); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgram; clStatus = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); @@ -169,7 +158,7 @@ int main(int argc, char **argv) { CHECK_ERROR("clCreateProgramWithSource") char clOptions[50]; - sprintf(clOptions, ""); + clOptions[0] = '\0'; clStatus = clBuildProgram(clProgram, 1, &clDevice, clOptions, NULL, NULL); CHECK_ERROR("clBuildProgram") diff --git a/tests/opencl/spmv/mmio.c b/tests/opencl/spmv/mmio.c index a356471851..c5eb01eb6f 100644 --- a/tests/opencl/spmv/mmio.c +++ b/tests/opencl/spmv/mmio.c @@ -72,7 +72,12 @@ int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_, for (i=0; i #endif -#include "perfmon.h" cl_context *clContextPtr; cl_command_queue *clCommandQueuePtr; @@ -56,46 +55,6 @@ get_last_async(struct pb_TimerSet* timers) return NULL; } -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) { cl_int ciErrNum = CL_SUCCESS; @@ -128,7 +87,7 @@ fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); /* valid event handle now aquired: insert the event record */ (*new_event)->label = label; (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); + ciErrNum = clEnqueueMarkerWithWaitList(*clCommandQueuePtr, 0, NULL, (cl_event *)(*new_event)->marker); if (ciErrNum != CL_SUCCESS) { fprintf(stderr, "Error Enqueueing Marker!\n"); } @@ -142,7 +101,6 @@ static pb_Timestamp record_async_times(struct pb_TimerSet* tset) struct pb_async_time_marker_list * next_interval = NULL; struct pb_async_time_marker_list * last_marker = get_last_async(tset); pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; for(next_interval = tset->async_markers; next_interval != last_marker; next_interval = next_interval->next) { @@ -193,18 +151,6 @@ accumulate_time(pb_Timestamp *accum, //#endif } -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - void pb_ResetTimer(struct pb_Timer *timer) { @@ -222,6 +168,7 @@ pb_ResetTimer(struct pb_Timer *timer) void pb_StartTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER if (timer->state != pb_Timer_STOPPED) { fputs("Ignoring attempt to start a running timer\n", stderr); @@ -245,6 +192,8 @@ pb_StartTimer(struct pb_Timer *timer) void pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER unsigned int numNotStopped = 0x3; // 11 @@ -287,6 +236,7 @@ pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) void pb_StopTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -315,6 +265,8 @@ pb_StopTimer(struct pb_Timer *timer) } void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -364,6 +316,7 @@ void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) double pb_GetElapsedTime(struct pb_Timer *timer) { + (void)timer; /*double ret; #ifndef DISABLE_PARBOIL_TIMER @@ -384,6 +337,7 @@ pb_GetElapsedTime(struct pb_Timer *timer) void pb_InitializeTimerSet(struct pb_TimerSet *timers) { + (void)timers; /*#ifndef DISABLE_PARBOIL_TIMER int n; @@ -405,7 +359,10 @@ void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { } void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { +pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + (void)timers; + (void)label; + (void)pb_Category; /*#ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc @@ -440,6 +397,8 @@ pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Categ void pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) { + (void)timers; + (void)timer; #if 0 #ifndef DISABLE_PARBOIL_TIMER @@ -732,8 +691,9 @@ pb_PrintTimerSet(struct pb_TimerSet *timers) maxSubLength = 0; while (sub != NULL) { // Find longest SubTimer label - if (strlen(sub->label) > maxSubLength) { - maxSubLength = strlen(sub->label); + int labelLength = (int)strlen(sub->label); + if (labelLength > maxSubLength) { + maxSubLength = labelLength; } sub = sub->next; } @@ -1194,6 +1154,7 @@ pb_GetPlatform(struct pb_PlatformParam *platform) { pb_Context* pb_InitOpenCLContext(struct pb_Parameters* parameters) { + (void)parameters; #if 0 pb_Platform* ps = pb_GetPlatform(parameters->platform); if (!ps) return NULL; @@ -1235,11 +1196,13 @@ pb_InitOpenCLContext(struct pb_Parameters* parameters) { void pb_ReleaseOpenCLContext(pb_Context* c) { + (void)c; pb_ReleasePlatforms(); } void pb_PrintPlatformInfo(pb_Context* c) { + (void)c; /*pb_Platform** ps = pb_GetPlatforms(); if (!ps) { fprintf (stderr, "No platform found"); @@ -1351,7 +1314,7 @@ pb_sig_int(char* c, int* p, int sz) { void pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; + unsigned int i; unsigned long long int s = 0; for (i = 0; i < sz; i++) s += p[i]; printf ("[Signature] %s = %lld\n", c, s); diff --git a/tests/opencl/srad/main.cc b/tests/opencl/srad/main.cc index 3ef2cfa1a1..c15d1deb8f 100644 --- a/tests/opencl/srad/main.cc +++ b/tests/opencl/srad/main.cc @@ -129,7 +129,7 @@ static void parse_args(int argc, char** argv) { // Serial CPU reference: identical single-precision SRAD math over the same // (extracted) image. Mirrors extract -> [stats, srad, srad2]*niter -> compress. -static void srad_cpu(std::vector& I, int Nr, int Nc, long Ne, int niter, +static void srad_cpu(std::vector& I, int Nr, long Ne, int niter, float lambda, long NeROI, const std::vector& iN, const std::vector& iS, const std::vector& jE, const std::vector& jW) { @@ -342,7 +342,7 @@ int main(int argc, char** argv) { // CPU golden reference over the same seeded input. std::vector h_ref(h_image); - srad_cpu(h_ref, Nr, Nc, Ne, niter, lambda, NeROI, iN, iS, jE, jW); + srad_cpu(h_ref, Nr, Ne, niter, lambda, NeROI, iN, iS, jE, jW); // Compare with a mixed absolute/relative float tolerance (exp/log + iterated // accumulation across host and device libm differ by a few ULP). diff --git a/tests/opencl/stencil/args.c b/tests/opencl/stencil/args.c index 9d751e2940..42637fe1c5 100644 --- a/tests/opencl/stencil/args.c +++ b/tests/opencl/stencil/args.c @@ -225,7 +225,7 @@ read_platform_param(char *str) /* The platform name is either the entire string, or all characters before * the separator */ - int name_length = separator_index == -1 ? strlen(str) : separator_index; + int name_length = separator_index == -1 ? (int)strlen(str) : separator_index; char *name_str = (char *)malloc(name_length + 1); memcpy(name_str, str, name_length); name_str[name_length] = 0; @@ -410,6 +410,8 @@ parse_input_files(struct argparse *ap, struct pb_Parameters *params) static result parse_end_options(struct argparse *ap, struct pb_Parameters *params) { + (void)ap; + (void)params; return ARGPARSE_DONE; } @@ -500,7 +502,6 @@ is_last_option(struct option *op) static int pb_ParseParameters (struct pb_Parameters *ret, int *_argc, char **argv) { - char *err_message; struct argparse ap; /* Each argument */ diff --git a/tests/opencl/stencil/file.c b/tests/opencl/stencil/file.c index a9b39b1935..4b44376df6 100644 --- a/tests/opencl/stencil/file.c +++ b/tests/opencl/stencil/file.c @@ -26,9 +26,13 @@ void inputData(char* fName, int* nx, int* ny, int* nz) exit(-1); } - fread(nx, sizeof(int ),1,fid); - fread(ny, sizeof(int ),1,fid); - fread(nz, sizeof(int ),1,fid); + if (fread(nx, sizeof(int ),1,fid) != 1 || + fread(ny, sizeof(int ),1,fid) != 1 || + fread(nz, sizeof(int ),1,fid) != 1) + { + fprintf(stderr, "Error reading input file\n"); + exit(-1); + } fclose (fid); } diff --git a/tests/opencl/stencil/main.cc b/tests/opencl/stencil/main.cc index 0018d00d70..30dee81bd0 100644 --- a/tests/opencl/stencil/main.cc +++ b/tests/opencl/stencil/main.cc @@ -35,7 +35,10 @@ static int read_data(float *A0, int nx,int ny,int nz,FILE *fp) { for(k=0;kclPlatformId; clContext = (cl_context) pb_context->clContext; clDevice = (cl_device_id) pb_context->clDeviceId; @@ -208,7 +209,6 @@ int main(int argc, char** argv) { //cl_program clProgram = clCreateProgramWithSource(clContext,1,clSource,NULL,&clStatus); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgram; clStatus = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); @@ -241,7 +241,10 @@ int main(int argc, char** argv) { h_Anext=(float*)malloc(sizeof(float)*size); pb_SwitchToTimer(&timers, pb_TimerID_IO); FILE *fp = fopen(parameters->inpFiles[0], "rb"); - read_data(h_A0, nx,ny,nz, fp); + if (read_data(h_A0, nx,ny,nz, fp) != 0) { + fclose(fp); + return -1; + } fclose(fp); memcpy (h_Anext,h_A0,sizeof(float)*size); @@ -264,9 +267,8 @@ int main(int argc, char** argv) { //only use 1D thread block int tx = 128; size_t block[3] = {1,1,1}; // {tx,1,1} - size_t grid[3] = {(nx-2+tx-1)/tx*tx,ny-2,nz-2}; + size_t grid[3] = {static_cast((nx-2+tx-1)/tx*tx),static_cast(ny-2),static_cast(nz-2)}; //size_t grid[3] = {nx-2,ny-2,nz-2}; - size_t offset[3] = {1,1,1}; printf("grid size in x/y/z = %ld %ld %ld\n",grid[0],grid[1],grid[2]); printf("block size in x/y/z = %ld %ld %ld\n",block[0],block[1],block[2]); diff --git a/tests/opencl/stencil/ocl.c b/tests/opencl/stencil/ocl.c index 9ce9a2f5d2..08b3aa06b5 100644 --- a/tests/opencl/stencil/ocl.c +++ b/tests/opencl/stencil/ocl.c @@ -27,7 +27,7 @@ char* readFile(const char* fileName) } size_t res = fread(buffer,1,size,fp); - if(res != size) + if(res != (size_t)size) { printf("Error 3!\n"); fclose(fp); diff --git a/tests/opencl/stencil/parboil_opencl.c b/tests/opencl/stencil/parboil_opencl.c index 7348dc2c0d..3caf4967de 100644 --- a/tests/opencl/stencil/parboil_opencl.c +++ b/tests/opencl/stencil/parboil_opencl.c @@ -56,46 +56,6 @@ get_last_async(struct pb_TimerSet* timers) return NULL; } -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) { cl_int ciErrNum = CL_SUCCESS; @@ -128,7 +88,7 @@ fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); /* valid event handle now aquired: insert the event record */ (*new_event)->label = label; (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); + ciErrNum = clEnqueueMarkerWithWaitList(*clCommandQueuePtr, 0, NULL, (cl_event *)(*new_event)->marker); if (ciErrNum != CL_SUCCESS) { fprintf(stderr, "Error Enqueueing Marker!\n"); } @@ -142,7 +102,6 @@ static pb_Timestamp record_async_times(struct pb_TimerSet* tset) struct pb_async_time_marker_list * next_interval = NULL; struct pb_async_time_marker_list * last_marker = get_last_async(tset); pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; for(next_interval = tset->async_markers; next_interval != last_marker; next_interval = next_interval->next) { @@ -193,18 +152,6 @@ accumulate_time(pb_Timestamp *accum, //#endif } -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - void pb_ResetTimer(struct pb_Timer *timer) { @@ -222,6 +169,7 @@ pb_ResetTimer(struct pb_Timer *timer) void pb_StartTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER if (timer->state != pb_Timer_STOPPED) { fputs("Ignoring attempt to start a running timer\n", stderr); @@ -245,6 +193,8 @@ pb_StartTimer(struct pb_Timer *timer) void pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER unsigned int numNotStopped = 0x3; // 11 @@ -287,6 +237,7 @@ pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) void pb_StopTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -315,6 +266,8 @@ pb_StopTimer(struct pb_Timer *timer) } void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -364,6 +317,7 @@ void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) double pb_GetElapsedTime(struct pb_Timer *timer) { + (void)timer; /*double ret; #ifndef DISABLE_PARBOIL_TIMER @@ -384,6 +338,7 @@ pb_GetElapsedTime(struct pb_Timer *timer) void pb_InitializeTimerSet(struct pb_TimerSet *timers) { + (void)timers; /*#ifndef DISABLE_PARBOIL_TIMER int n; @@ -405,7 +360,10 @@ void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { } void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { +pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + (void)timers; + (void)label; + (void)pb_Category; /*#ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc @@ -440,6 +398,8 @@ pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Categ void pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) { + (void)timers; + (void)timer; #if 0 #ifndef DISABLE_PARBOIL_TIMER @@ -732,8 +692,9 @@ pb_PrintTimerSet(struct pb_TimerSet *timers) maxSubLength = 0; while (sub != NULL) { // Find longest SubTimer label - if (strlen(sub->label) > maxSubLength) { - maxSubLength = strlen(sub->label); + int labelLength = (int)strlen(sub->label); + if (labelLength > maxSubLength) { + maxSubLength = labelLength; } sub = sub->next; } @@ -1194,6 +1155,7 @@ pb_GetPlatform(struct pb_PlatformParam *platform) { pb_Context* pb_InitOpenCLContext(struct pb_Parameters* parameters) { + (void)parameters; #if 0 pb_Platform* ps = pb_GetPlatform(parameters->platform); if (!ps) return NULL; @@ -1235,11 +1197,13 @@ pb_InitOpenCLContext(struct pb_Parameters* parameters) { void pb_ReleaseOpenCLContext(pb_Context* c) { + (void)c; pb_ReleasePlatforms(); } void pb_PrintPlatformInfo(pb_Context* c) { + (void)c; /*pb_Platform** ps = pb_GetPlatforms(); if (!ps) { fprintf (stderr, "No platform found"); @@ -1351,7 +1315,7 @@ pb_sig_int(char* c, int* p, int sz) { void pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; + unsigned int i; unsigned long long int s = 0; for (i = 0; i < sz; i++) s += p[i]; printf ("[Signature] %s = %lld\n", c, s); diff --git a/tests/opencl/transpose/main.cc b/tests/opencl/transpose/main.cc index 93c3a4fdbd..8a2148b4c3 100644 --- a/tests/opencl/transpose/main.cc +++ b/tests/opencl/transpose/main.cc @@ -62,7 +62,7 @@ static double transposeGPU(const char* kernelName, bool useLocalMem, cl_uint ci { cl_mem d_odata[MAX_GPU_COUNT]; cl_mem d_idata[MAX_GPU_COUNT]; - cl_kernel ckKernel[MAX_GPU_COUNT]; + cl_kernel ckKernel[MAX_GPU_COUNT] = {}; size_t szGlobalWorkSize[2]; size_t szLocalWorkSize[2]; @@ -317,7 +317,6 @@ int runTest( const int argc, const char** argv) // (const char **)&source, &program_length, &ciErrNum); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; ciErrNum = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); if (ciErrNum != CL_SUCCESS) { @@ -349,7 +348,7 @@ int runTest( const int argc, const char** argv) //double sharedCopyTime = transposeGPU("shared_copy", true, ciDeviceCount, h_idata, h_odata, size_x, size_y); #endif - double naiveTime = transposeGPU("transpose_naive", false, ciDeviceCount, h_idata, h_odata, size_x, size_y); + transposeGPU("transpose_naive", false, ciDeviceCount, h_idata, h_odata, size_x, size_y); //double optimizedTime = transposeGPU("transpose", true, ciDeviceCount, h_idata, h_odata, size_x, size_y); #ifdef GPU_PROFILING diff --git a/tests/opencl/transpose/shrUtils.cpp b/tests/opencl/transpose/shrUtils.cpp index 23e4488d27..89c0b7169b 100644 --- a/tests/opencl/transpose/shrUtils.cpp +++ b/tests/opencl/transpose/shrUtils.cpp @@ -203,7 +203,6 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { static FILE* pFileStream0 = NULL; static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; char cFileMode [3]; // if the sample log file is closed and the call incudes a "write-to-file", open file for writing @@ -318,12 +317,12 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print string to console if flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console + printf ("\n !!! Error # %i at ", iErrNum); // console } // print string to file if flagged if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file + fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file } } @@ -347,14 +346,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // character is not '%', so print it verbatim to console and/or files as flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = putc(*pStr, stdout); // console + putc(*pStr, stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } @@ -399,16 +398,26 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { case '%': // special handling for string of %%%% { + // collapse each "%%" to a literal '%', as printf would + std::string sLiteral; + for (size_t i = 0; i < sFormatSpec.size(); ++i) + { + sLiteral += sFormatSpec[i]; + if ((sFormatSpec[i] == '%') && (i + 1 < sFormatSpec.size()) && (sFormatSpec[i + 1] == '%')) + { + ++i; + } + } if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str()); // console + fputs(sLiteral.c_str(), stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file + fputs(sLiteral.c_str(), pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file + fputs(sLiteral.c_str(), pFileStream1); // master log file } } continue; @@ -420,14 +429,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list cArg = va_arg(vaArgList, char*); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console + printf(sFormatSpec.c_str(), cArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file } } continue; @@ -439,14 +448,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list iArg = va_arg(vaArgList, int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console + printf(sFormatSpec.c_str(), iArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file } } continue; @@ -460,14 +469,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list uiArg = va_arg(vaArgList, unsigned int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console + printf(sFormatSpec.c_str(), uiArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file } } continue; @@ -484,14 +493,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list dArg = va_arg(vaArgList, double); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console + printf(sFormatSpec.c_str(), dArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file } } continue; @@ -501,14 +510,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print arg of unknown/unsupported type to console and/or file if flagged if (iLogMode & LOGCONSOLE) // console { - szNumWritten = putc(*pStr, stdout); + putc(*pStr, stdout); } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } diff --git a/tests/riscv/common.mk b/tests/riscv/common.mk index ef9ec7332b..b3c3a9413d 100644 --- a/tests/riscv/common.mk +++ b/tests/riscv/common.mk @@ -41,8 +41,9 @@ BENCH_LIST := median qsort rsort towers memcpy multiply dhrystone spmv # (VX_MEM_IO_EXIT_CODE, VX_MEM_IO_COUT_*) via the patched crt.S + # syscalls.c, so the stamp must invalidate when those values change. # Listing $(ROOT_DIR)/sw/VX_types.h as a prerequisite forces a rebuild -# after a memory-map shift (e.g. a COUT region size change). -$(RISCV_TESTS_STAMP): $(ROOT_DIR)/sw/VX_types.h +# after a memory-map shift (e.g. a COUT region size change), and listing +# the patch re-applies an edited patch to a fresh clone. +$(RISCV_TESTS_STAMP): $(ROOT_DIR)/sw/VX_types.h $(VORTEX_HOME)/miscs/patches/riscv-benchmarks.patch rm -rf $(RISCV_TESTS_DIR) git clone $(RISCV_TESTS_REPO) $(RISCV_TESTS_DIR) cd $(RISCV_TESTS_DIR) && git checkout --quiet $(RISCV_TESTS_COMMIT) && git submodule update --init --recursive diff --git a/tests/unittest/common.mk b/tests/unittest/common.mk index 4a0139e677..07887cbafd 100644 --- a/tests/unittest/common.mk +++ b/tests/unittest/common.mk @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../../..) -CXXFLAGS += -std=c++17 -Wall -Wextra -pedantic -Wfatal-errors +CXXFLAGS += -std=c++17 -Wall -Wextra -pedantic -Wfatal-errors -Werror CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SW_COMMON_DIR) CXXFLAGS += $(CONFIGS) diff --git a/tests/vulkan/common.mk b/tests/vulkan/common.mk index a3a6453258..ccdaeabd90 100644 --- a/tests/vulkan/common.mk +++ b/tests/vulkan/common.mk @@ -70,7 +70,7 @@ CONFIGS += -DVX_CFG_EXT_A_ENABLE GLSLC ?= glslc CC ?= cc -CFLAGS += -O2 -Wall +CFLAGS += -O2 -Wall -Wextra -Wfatal-errors -Werror LDLIBS += -lvulkan # One SPIR-V module per shader source, stage kept in the name so a diff --git a/tests/vulkan/subgroup_vs/main.c b/tests/vulkan/subgroup_vs/main.c index ae42a7f810..1538dafc3d 100644 --- a/tests/vulkan/subgroup_vs/main.c +++ b/tests/vulkan/subgroup_vs/main.c @@ -443,7 +443,7 @@ main(int argc, char **argv) return 1; } - struct sg_report r = { 0, 0, 0, 0 }; + struct sg_report r = { 0 }; int rc = render_subgroup(dev, queue, qf, &mp, vs, fs, &r); vkDestroyShaderModule(dev, vs, NULL); From 57f28d2f31e549226ca2d5f59c44de50c02f2671 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 22:17:50 -0700 Subject: [PATCH 012/336] gen_config: accept enum flag overrides whose value contains '_' A bare -D_ override was recognised by splitting the flag at its last underscore, so a value that itself contains one (e.g. -DVX_CFG_PLATFORM_MEMORY_TYPE_HBM2_A100) never matched its enum: the generator silently kept the default, and the profile merge treated the flag and the profile's entry as different knobs and passed both. The enum is now matched by name prefix and the rest checked against its declared values, in both places. Output is byte-identical to before for all 247 CONFIGS strings in ci/testcases at both XLENs. Co-Authored-By: Claude Opus 5.5 --- ci/gen_config.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/ci/gen_config.py b/ci/gen_config.py index ea3436c49d..316c8d0a97 100755 --- a/ci/gen_config.py +++ b/ci/gen_config.py @@ -313,6 +313,13 @@ class EnumSpec: values: List[Any] +def _enum_of_flag(name: str, enums: Dict[str, EnumSpec]) -> Optional[str]: + # The enum a bare -D_ flag selects. A value may itself contain + # '_', so the enum name is matched as a prefix instead of splitting the flag. + owners = [e for e in enums if name.startswith(e + "_")] + return max(owners, key=len) if owners else None + + def _load_enums(toml_data: Dict[str, Any]) -> Dict[str, EnumSpec]: enums = toml_data.get("enum", None) if enums is None: @@ -454,10 +461,10 @@ def set_enum(name: str, val: Any) -> None: set_enum(d.name, _scalar(d.value)) continue - if "_" in d.name and d.value is None: - base, suffix = d.name.rsplit("_", 1) - if base in enums: - set_enum(base, _scalar(suffix)) + if d.value is None: + base = _enum_of_flag(d.name, enums) + if base is not None: + set_enum(base, _scalar(d.name[len(base) + 1:])) overrides[d.name] = True explicit.add(d.name) continue @@ -574,9 +581,9 @@ def _canonical_define(tok: str, enums: Dict[str, EnumSpec]) -> Optional[str]: return d.name[:-len("_DISABLE")] + "_ENABLE" if d.name in enums: return d.name - if d.value is None and "_" in d.name: - base, _ = d.name.rsplit("_", 1) - if base in enums: + if d.value is None: + base = _enum_of_flag(d.name, enums) + if base is not None: return base return d.name From 4b924f24d0f4ad3e7e398c61bd8ee947a11f8524 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 22:18:07 -0700 Subject: [PATCH 013/336] caps: report exact warp, core and cluster counts GPU_DEV_CAPS encoded threads, warps, socket/cluster sizes and clusters as 3-bit log2 fields, so a machine that is not a power of two (48 warps, 84 or 108 cores) reported the wrong shape to the runtime. The spare bits [63:42] now carry exact warps (7 bits), sockets per cluster (8) and clusters (7), produced identically by the RTL CP regfile and the C++ CP model. The decoder prefers them and falls back to the log2 fields only when they read zero, so devices built before this change keep reporting correctly. Verified: every simx/rtlsim test in the matrix so far decodes these caps (NUM_WARPS/NUM_CORES drive the kernel launch), and the A100-shaped 108-core SimX run reports 108 cores. Co-Authored-By: Claude Opus 5.5 --- docs/designs/command_processor.md | 11 +++++++++-- hw/rtl/cp/VX_cp_axil_regfile.sv | 7 ++++++- sim/common/cmd_processor.cpp | 8 +++++++- sw/runtime/common/caps.h | 20 ++++++++++++++------ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index c7f7e93962..118c5dbdcb 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -419,10 +419,17 @@ CP regfile is the only producer on every target. | `[33:26]` | 8 | log2 local_mem_size | `VX_CAPS_LOCAL_MEM_SIZE` | | `[36:34]` | 3 | log2 num_mem_banks | `VX_CAPS_NUM_MEM_BANKS` | | `[41:37]` | 5 | mem_bank_size, biased by 20 | `VX_CAPS_MEM_BANK_SIZE` | -| `[63:42]` | 22 | reserved | — | +| `[48:42]` | 7 | warps/core | `VX_CAPS_NUM_WARPS` | +| `[56:49]` | 8 | cluster_size | (only via `NUM_CORES`) | +| `[63:57]` | 7 | num_clusters | `VX_CAPS_NUM_CLUSTERS` | + +The exact counts in `[63:42]` let a machine that is not a power of two +(e.g. 48 warps, 84 cores) report its true shape; the decoder prefers them and +falls back to the log2 fields only when they read zero, as on devices built +before they existed. `VX_CAPS_NUM_CORES` is derived, not a field: -`num_clusters × cluster_size × socket_size`. `CACHE_LINE_SIZE`, +`num_clusters × cluster_size × socket_size`, from the exact counts. `CACHE_LINE_SIZE`, `GLOBAL_MEM_SIZE`, `CLOCK_RATE`, `PEAK_MEM_BW` and the VM ids are not encoded here — `decode_caps` returns false and the backend resolves them. diff --git a/hw/rtl/cp/VX_cp_axil_regfile.sv b/hw/rtl/cp/VX_cp_axil_regfile.sv index db425cbb2c..806a996da9 100644 --- a/hw/rtl/cp/VX_cp_axil_regfile.sv +++ b/hw/rtl/cp/VX_cp_axil_regfile.sv @@ -128,8 +128,13 @@ module VX_cp_axil_regfile localparam int GPU_BANK_ADDR_W = `VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - `CLOG2(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + `STATIC_ASSERT(`VX_CFG_NUM_WARPS < 128 && `VX_CFG_NUM_CLUSTERS < 128 && GPU_CLUSTER_SIZE < 256, + ("GPU_DEV_CAPS exact-count fields overflow")) + wire [63:0] gpu_dev_caps = { - 22'b0, + 7'(`VX_CFG_NUM_CLUSTERS), + 8'(GPU_CLUSTER_SIZE), + 7'(`VX_CFG_NUM_WARPS), 5'(GPU_BANK_ADDR_W - 20), 3'($clog2(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS)), 8'(`VX_CFG_LMEM_ENABLED ? `VX_CFG_LMEM_LOG_SIZE : 0), diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 85083ef49d..0972440954 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -26,6 +26,9 @@ constexpr unsigned cp_clog2(uint64_t n) { } uint64_t gpu_dev_caps() { const unsigned cluster_size = VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE; + static_assert(VX_CFG_NUM_WARPS < 128 && VX_CFG_NUM_CLUSTERS < 128 + && VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE < 256, + "GPU_DEV_CAPS exact-count fields overflow"); const unsigned bank_addr_w = VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - cp_clog2(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); return (uint64_t(VX_ISA_IMPL_ID) & 0xFF) @@ -37,7 +40,10 @@ uint64_t gpu_dev_caps() { | ((uint64_t(cp_clog2(VX_CFG_ISSUE_WIDTH)) & 0x7) << 23) | ((uint64_t(VX_CFG_LMEM_ENABLED ? VX_CFG_LMEM_LOG_SIZE : 0) & 0xFF) << 26) | ((uint64_t(cp_clog2(VX_CFG_PLATFORM_MEMORY_NUM_BANKS)) & 0x7) << 34) - | ((uint64_t(bank_addr_w - 20) & 0x1F) << 37); + | ((uint64_t(bank_addr_w - 20) & 0x1F) << 37) + | ((uint64_t(VX_CFG_NUM_WARPS) & 0x7F) << 42) + | ((uint64_t(cluster_size) & 0xFF) << 49) + | ((uint64_t(VX_CFG_NUM_CLUSTERS) & 0x7F) << 57); } uint64_t gpu_isa_caps() { return (uint64_t(VX_CFG_MISA_EXT) << 32) diff --git a/sw/runtime/common/caps.h b/sw/runtime/common/caps.h index ca30664017..ddb1def115 100644 --- a/sw/runtime/common/caps.h +++ b/sw/runtime/common/caps.h @@ -37,6 +37,14 @@ static constexpr uint32_t CP_REG_GPU_DEV_CAPS_HI = 0x01C; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_LO = 0x020; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_HI = 0x024; +// Warps, sockets per cluster and clusters carry an exact count, so a machine +// that is not a power of two reports truly; the field is zero on devices +// built before it existed, which report only the 3-bit log2 field. +inline uint64_t exact_or_log2(uint64_t dev_caps, uint32_t exact_lsb, uint64_t exact_mask, uint32_t log2_lsb) { + uint64_t exact = (dev_caps >> exact_lsb) & exact_mask; + return exact ? exact : (1ull << ((dev_caps >> log2_lsb) & 0x7)); +} + // Decode a VX_CAPS_* id from the two CP capability words. Returns true // and writes *out when the id is encoded in the words; returns false // for ids the caller must resolve itself (CACHE_LINE_SIZE, @@ -51,20 +59,20 @@ inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, *out = 1ull << ((dev_caps >> 8) & 0x7); return true; case VX_CAPS_NUM_WARPS: - *out = 1ull << ((dev_caps >> 11) & 0x7); + *out = exact_or_log2(dev_caps, 42, 0x7F, 11); return true; case VX_CAPS_NUM_CORES: { - uint32_t socket_size = 1u << ((dev_caps >> 14) & 0x7); - uint32_t cluster_size = 1u << ((dev_caps >> 17) & 0x7); - uint32_t num_clusters = 1u << ((dev_caps >> 20) & 0x7); - *out = uint64_t(num_clusters) * cluster_size * socket_size; + uint64_t socket_size = 1ull << ((dev_caps >> 14) & 0x7); + uint64_t cluster_size = exact_or_log2(dev_caps, 49, 0xFF, 17); + uint64_t num_clusters = exact_or_log2(dev_caps, 57, 0x7F, 20); + *out = num_clusters * cluster_size * socket_size; return true; } case VX_CAPS_SOCKET_SIZE: *out = 1ull << ((dev_caps >> 14) & 0x7); return true; case VX_CAPS_NUM_CLUSTERS: - *out = 1ull << ((dev_caps >> 20) & 0x7); + *out = exact_or_log2(dev_caps, 57, 0x7F, 20); return true; case VX_CAPS_ISSUE_WIDTH: *out = 1ull << ((dev_caps >> 23) & 0x7); From 5f6d3e147d99c17f97a6f619b2ca57052f8d8d27 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 22:22:43 -0700 Subject: [PATCH 014/336] runtime: VORTEX_VM_MEGAPAGE_HEAP maps the heap with superpages With 4 KB pages a working set beyond the TLB reach thrashes: sgemm at N=512 on a VM-enabled RTX 5080-shaped machine ran 18x slower (1% L1 TLB hit rate, the memory port blocked behind 2 requests parked per in-flight page). GPU drivers back allocations with 2 MB pages. VORTEX_VM_MEGAPAGE_HEAP=1 maps the whole heap once with superpages at its identity VA, which the page walker and the L2 TLB's megapage array already support, so a buffer costs no TLB entry per 4 KB (in that study VM then cost 1-4% instead of 18x). Per-buffer access flags are not enforced in this mode and it cannot be combined with VORTEX_RANDOMIZE_VA; the default mapping is unchanged. Adds vm-megapage-heap (sgemm, simx + rtlsim). Verified: vm-megapage-heap passes on simx (XLEN 64); the rtlsim variant runs in the CI matrix vm cell. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/vm.yaml | 11 ++++++++++ docs/designs/virtual_memory_subsystem.md | 4 ++++ sw/runtime/common/vm.cpp | 26 ++++++++++++++++++++++++ sw/runtime/common/vm.h | 3 +++ 4 files changed, 44 insertions(+) diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index 738eb3851f..2c4d2b7c2b 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -170,6 +170,17 @@ tests: target: run-{driver} vars: OPTS: -t4 -n4 -p1 +# The heap mapped once with superpages (VORTEX_VM_MEGAPAGE_HEAP): every buffer +# translates through the TLBs' megapage path at its identity VA. +- id: vm-megapage-heap + via: make-run + drivers: + - simx + - rtlsim + dir: tests/regression/sgemm + target: run-{driver} + env: + VORTEX_VM_MEGAPAGE_HEAP: 1 - id: tlb-concurrent via: make-run drivers: diff --git a/docs/designs/virtual_memory_subsystem.md b/docs/designs/virtual_memory_subsystem.md index 5a667d44a0..8b6a34a6e4 100644 --- a/docs/designs/virtual_memory_subsystem.md +++ b/docs/designs/virtual_memory_subsystem.md @@ -653,6 +653,10 @@ MMU moved out of the core there is no `sysmem_perf` slot to carry it, so the in-core view reads zero on rtlsim / FPGA (§13). Testing knobs: `VORTEX_RANDOMIZE_VA=0|1` and `VORTEX_VA_SEED=N` (§8.4); +`VORTEX_VM_MEGAPAGE_HEAP=1` maps the whole heap once with superpages at its +identity VA, as GPU drivers back allocations with 2 MB pages, so a buffer costs +no TLB entry per 4 KB page (per-buffer access flags are not enforced, and it +cannot be combined with `VORTEX_RANDOMIZE_VA`); `VX_CFG_VM_ADDR_MODE=BARE` in `CONFIGS` builds the VM hardware but runs untranslated. diff --git a/sw/runtime/common/vm.cpp b/sw/runtime/common/vm.cpp index 3c539fd521..0922d8cfd1 100644 --- a/sw/runtime/common/vm.cpp +++ b/sw/runtime/common/vm.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +60,12 @@ VMManager::VMManager(DeviceMemIO* dev_io) std::cout << "[VM] Virtual address randomization ENABLED (seed=0x" << std::hex << seed << std::dec << ")" << std::endl; } + + const char* megapage_env = std::getenv("VORTEX_VM_MEGAPAGE_HEAP"); + megapage_heap_ = (megapage_env != nullptr && std::atoi(megapage_env) != 0); + if (megapage_heap_) { + std::cout << "[VM] Heap mapped with superpages" << std::endl; + } } VMManager::~VMManager() { @@ -114,6 +121,22 @@ int VMManager::init() { GLOBAL_MEM_SIZE - VX_MEM_PAGE_TABLE_BASE_ADDR), { return err; }); + if (megapage_heap_) { + // Superpages over the whole heap, as GPU drivers back allocations with + // 2 MB pages: a buffer keeps its identity VA and no longer costs a TLB + // entry per 4 KB. Per-buffer access flags are not enforced in this mode. + if (randomize_va_) { + std::cerr << "[VM] VORTEX_VM_MEGAPAGE_HEAP and VORTEX_RANDOMIZE_VA are mutually exclusive" << std::endl; + return -1; + } + uint64_t heap_end = VX_MEM_PAGE_TABLE_BASE_ADDR; +#if VX_VM_ADDR_MODE == SV32 + heap_end = std::min(heap_end, 0x100000000ULL); +#endif + CHECK_ERR(install_identity_map(ALLOC_BASE_ADDR, heap_end - ALLOC_BASE_ADDR), { + return err; + }); + } } // install_identity_map() already flushed; an extra flush here is a // cheap no-op if nothing else dirtied the shadow. @@ -183,6 +206,9 @@ uint64_t VMManager::map_p2v(uint64_t ppn, uint32_t flags) { int VMManager::phy_to_virt_map(uint64_t size, uint64_t* dev_pAddr, uint32_t flags) { if (!need_trans(*dev_pAddr)) return 0; + if (megapage_heap_ && *dev_pAddr >= ALLOC_BASE_ADDR && *dev_pAddr < VX_MEM_PAGE_TABLE_BASE_ADDR) { + return 0; // identity-mapped by init() + } uint64_t init_pAddr = *dev_pAddr; // Round up: a sub-page allocation still needs one PTE. A plain diff --git a/sw/runtime/common/vm.h b/sw/runtime/common/vm.h index e922a580aa..581ecd43fe 100644 --- a/sw/runtime/common/vm.h +++ b/sw/runtime/common/vm.h @@ -126,6 +126,9 @@ class VMManager { // Randomized VA allocation (VORTEX_RANDOMIZE_VA / VORTEX_VA_SEED). std::mt19937_64 rng_; bool randomize_va_; + + // Heap mapped once with superpages (VORTEX_VM_MEGAPAGE_HEAP). + bool megapage_heap_; }; } // namespace vortex From 4dbb6154a749c22a6633c00d060a54a173a1759c Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:17:22 -0700 Subject: [PATCH 015/336] simx: widen the cache config's input count and pipeline latency Cache::Config stored the input count and the pipeline latency in uint8_t, so any machine whose L2 sees more than 255 inputs (an 84-132 core chip: 16 L1 requests per socket) or whose L2 latency exceeds 255 cycles failed to build with a narrowing error. The RTL caches take both as unbounded parameters; the model now matches. Verified: the 84-core rtx5080 profile builds and runs vecadd on simx; the a100/h100/v100/rtx1080 profiles build (1,728 L2 inputs on a100). Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/cache.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index b3ffe9c8be..70ad9e6e3f 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -35,12 +35,12 @@ class Cache : public SimObject { uint8_t A; // log2 associativity uint8_t B; // log2 number of banks uint8_t addr_width; // word address bits - uint8_t num_inputs; // number of inputs + uint16_t num_inputs; // number of inputs uint8_t mem_ports; // memory ports bool write_back; // is write-back bool write_reponse; // enable write response uint16_t mshr_size; // MSHR buffer size - uint8_t latency; // pipeline latency + uint16_t latency; // pipeline latency uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache }; From 325d685dbe4ab62985001da5916ef61948093c58 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:17:30 -0700 Subject: [PATCH 016/336] lsu: declare the LMEM window parameters only when LMEM exists With local memory disabled (e.g. the tinygpu profile) LMEM_BLK_BASE was never read, and Verilator's UNUSEDPARAM warning, now fatal, stopped the rtlsim build. No CI configuration disables LMEM, so the zero-warnings sweep did not reach it. The window parameters and their static check now sit under the same VX_CFG_LMEM_ENABLE guard as their only use. Verified: 1 core, 2 warps x 2 threads, LMEM disabled builds warning-free on rtlsim and passes sgemm -n16 (15,240 instructions, as on simx). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_lsu_slice.sv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_slice.sv index dba90d2856..affb932298 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_slice.sv @@ -42,9 +42,11 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( // window: the LMEM base is asserted aligned to the capacity, so the prefix // compare is exact and needs no end address -- which is not representable // when the window ends at the top of the address space. +`ifdef VX_CFG_LMEM_ENABLE localparam LMEM_BLK_LSB = `VX_CFG_LMEM_LOG_SIZE - MEM_ASHIFT; localparam [MEM_ADDRW-1:0] LMEM_BLK_BASE = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> MEM_ASHIFT); `STATIC_ASSERT (MEM_ADDRW > LMEM_BLK_LSB, ("LMEM window wider than the block address space")) +`endif `UNUSED_PARAM (CORE_ID) `UNUSED_SPARAM (INSTANCE_ID) From 1642289411ea1d2c51828ea92593e7d2c483daf7 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:21:31 -0700 Subject: [PATCH 017/336] tests: guard the global index in vecadd, relu, dropout and minimal vx_device_max_occupancy_grid sizes a 1D block to a whole core and rounds the grid up, so grid x block exceeds the problem size whenever a core has more threads than there are points. These four kernels indexed with the global id unguarded, so the surplus threads read and wrote past their buffers. On a core with 1,024 threads vecadd -n256 overwrote its kernel-argument block and later warps faulted on a float loaded as the destination pointer ("misaligned memory access", every app on the 20-132 core profiles). Each kernel now returns when the index is past num_points, the size the host already passes, as any kernel on a rounded-up grid must. The other tests that launch through the helper were audited: they guard the index, use a grid-stride loop, or size the problem as an exact multiple of the block. Verified: the four pass on simx at the default config and at 32 warps x 32 threads, including sizes that do not divide the block (vecadd -n64/-n100/ -n256/-n16384, relu -n64/-n100, dropout -n1000/-n1024, minimal -n1); the unguarded vecadd reproduces the fault at 32x32 -n256. The guard adds instructions, so the perf-gate baselines are re-recorded after this series. Co-Authored-By: Claude Opus 5.5 --- tests/regression/dropout/kernel.cpp | 3 +++ tests/regression/minimal/kernel.cpp | 3 +++ tests/regression/relu/kernel.cpp | 3 +++ tests/regression/vecadd/kernel.cpp | 3 +++ 4 files changed, 12 insertions(+) diff --git a/tests/regression/dropout/kernel.cpp b/tests/regression/dropout/kernel.cpp index 95e77d8189..c9cd35b5fe 100644 --- a/tests/regression/dropout/kernel.cpp +++ b/tests/regression/dropout/kernel.cpp @@ -10,6 +10,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto multiplier = arg->multiplier; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= arg->num_points) { + return; + } float rand_value = RandomFloat(WangHash(gid)); TYPE scaled_value = src0_ptr[gid] * multiplier; dst_ptr[gid] = (rand_value < dropout_p) ? 0.0 : scaled_value; diff --git a/tests/regression/minimal/kernel.cpp b/tests/regression/minimal/kernel.cpp index 0d04cd2563..9cdf8cf52b 100644 --- a/tests/regression/minimal/kernel.cpp +++ b/tests/regression/minimal/kernel.cpp @@ -7,5 +7,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_ptr = reinterpret_cast(arg->dst_addr); uint32_t idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= arg->num_points) { + return; + } dst_ptr[idx] = MINIMAL_MAGIC | idx; } diff --git a/tests/regression/relu/kernel.cpp b/tests/regression/relu/kernel.cpp index 55335af652..b70730b79a 100644 --- a/tests/regression/relu/kernel.cpp +++ b/tests/regression/relu/kernel.cpp @@ -6,6 +6,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_ptr = reinterpret_cast(arg->dst_addr); uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= arg->num_points) { + return; + } TYPE value = src0_ptr[gid]; dst_ptr[gid] = (value < 0) ? TYPE(0) : value; } diff --git a/tests/regression/vecadd/kernel.cpp b/tests/regression/vecadd/kernel.cpp index a84972e50e..3660c77e59 100644 --- a/tests/regression/vecadd/kernel.cpp +++ b/tests/regression/vecadd/kernel.cpp @@ -6,5 +6,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto src1_ptr = reinterpret_cast(arg->src1_addr); auto dst_ptr = reinterpret_cast(arg->dst_addr); uint32_t idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= arg->num_points) { + return; + } dst_ptr[idx] = src0_ptr[idx] + src1_ptr[idx]; } From 997ee24e094da45151d0dccaa729312442adf659 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:50:54 -0700 Subject: [PATCH 018/336] cache: admit LLC fills against the AMO writeback queue's room The LLC AMO commit engine queues each result in a 4-entry writeback queue that drains through the bank's core-request slot, last in the input priority. Its sizing assumed commit_busy serializes AMOs, but commit_busy only gates core requests: a burst of fills for different lines, each line's chain ending in one AMO, replays an AMO per fill while the fill/replay stream holds the input slot, so nothing drains and the fifth result overflows the queue (amo:zacas:rtlsim, exposed by the per-channel DRAM model's tighter fill bursts; latent on master). - The engine counts the AMOs still able to reach the queue (S0, S0..stC, the compute stage) and flags when the queue cannot take one more; the bank admits a fill only while it can. A fill is admitted only between replay chains and a chain carries at most one AMO, so replays never wait mid-chain, and the drain still fires between chains as before. The flag is registered (the sum grows by at most one a cycle, so it tests one entry early). - req_input_defer missed the cycle a missing AMO finalizes at S1 (the S0 pulse has passed, the MSHR probe does not see the entry yet), letting a same-line request link behind it. A registered S1 allocation term closes it, in both the LLC and passthrough guards, and the MSHR asserts that no request links behind a pending AMO at the LLC. - chain_stall now paces any AMO behind a busy compute stage, not only a same-line one: a core AMO admitted while a chain's head leaves through the forward port could otherwise be overwritten by the chain's storing AMO. - A flush no longer starts while AMO results are still queued (bank_empty now includes commit_busy). SimX models no writeback queue; cycles change only when a fill would overflow it. Verified: amo and cache cells on rtlsim, XLEN 32 and 64, all pass (zacas included). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_amo.sv | 69 ++++++++++++++++++++++++++++------- hw/rtl/cache/VX_cache_bank.sv | 15 +++++--- hw/rtl/cache/VX_cache_mshr.sv | 5 +++ 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/hw/rtl/cache/VX_cache_amo.sv b/hw/rtl/cache/VX_cache_amo.sv index 23c33a60cd..63a857d330 100644 --- a/hw/rtl/cache/VX_cache_amo.sv +++ b/hw/rtl/cache/VX_cache_amo.sv @@ -101,8 +101,9 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // commit outputs (tied off when IS_LLC=0) output wire amo_hit_st1, // AMO commits locally at S1 output wire commit_busy, // commit in flight - output wire chain_stall, // pace same-line chained AMO + output wire chain_stall, // pace a request behind the compute stage output wire wb_pending, // writeback request live + output wire wbq_full, // no queue room for another chain's AMO output wire [WORD_WIDTH-1:0] rsp_data, // response word on amo_hit_st1 // Read forward: a request reading the AMO'd word while the result is still // queued/settling must observe those bytes (replays are admitted during @@ -152,6 +153,18 @@ module VX_cache_amo import VX_gpu_pkg::*; #( end wire st0_ld_alloc_any = mshr_allocate_st0 && ~pipe_stall && ~amo_st0.amo_valid && ~rw_st0; wire st0_amo_alloc_any = mshr_allocate_st0 && ~pipe_stall && amo_st0.amo_valid; + // The allocation one stage on: at S1 it finalizes, and the MSHR probe + // only sees it the cycle after, so this covers the cycle in between. + reg st1_ld_alloc_r, st1_amo_alloc_r; + always @(posedge clk) begin + if (reset) begin + st1_ld_alloc_r <= 1'b0; + st1_amo_alloc_r <= 1'b0; + end else if (~pipe_stall) begin + st1_ld_alloc_r <= mshr_allocate_st0 && ~amo_st0.amo_valid && ~rw_st0; + st1_amo_alloc_r <= mshr_allocate_st0 && amo_st0.amo_valid; + end + end if (IS_LLC != 0) begin : g_commit // ---------------------------------------------------------------- @@ -184,8 +197,8 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // bounded by the core-port fan-in (<= 4 for this cache family), not the // full line. Sizing to that bound (min 2, cap 4) halves the coalesce/ // forward/enqueue fanout vs a full-line queue; the cap is a constant - // here because the engine has no NUM_REQS parameter, and the overflow - // assertion guards it if a wider core-port config ever exceeds 4. + // here because the engine has no NUM_REQS parameter. Replays bypass + // commit_busy, so fills are admitted against queue room (wbq_full). localparam WBQ_TARGETS = WORDS_PER_LINE * NUM_LANES; localparam WBQ_SIZE = (WBQ_TARGETS < 4) ? ((WBQ_TARGETS < 2) ? 2 : WBQ_TARGETS) : 4; localparam WBQ_CNTW = `CLOG2(WBQ_SIZE+1); @@ -228,9 +241,9 @@ module VX_cache_amo import VX_gpu_pkg::*; #( wire post_wb_valid = (post_wb_age != 2'd0); // Compute stage: S1 latches the aligned operands, the RMW ALU + the - // re-align shift run the next cycle, off the S1 critical path. AMO - // commits are serialized by commit_busy (the bank holds off core - // requests), so the stage holds at most one operation; the old operand + // re-align shift run the next cycle, off the S1 critical path. The + // stage holds at most one operation (commit_busy serializes core AMOs, + // chain_stall paces any AMO behind a busy stage); the old operand // is byte-forwarded from the writeback queue when a prior AMO to the same // bytes has not yet reached the array (see line_lane_st1). reg cmp_valid; @@ -750,7 +763,8 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // line AMOs back to back); those are paced instead by chain_stall. assign commit_busy = do_store_st0 || amo_inflight || do_store_st1 || cmp_valid || wb_pending_r; // Pace any same-line request sitting behind an in-flight compute by one - // cycle, so the result lands in wb_data_r and forwards cleanly. Gated on + // cycle, so the result lands in wb_data_r and forwards cleanly, and any + // AMO, which needs the compute stage free. Gated on // cmp_valid (an AMO is computing), so it never fires for baseline traffic. // The address match is registered from the look-ahead operands: both only // move on ~pipe_stall, so a held match stays valid across a stall. The @@ -765,7 +779,34 @@ module VX_cache_amo import VX_gpu_pkg::*; #( chain_match_r <= (cmp_addr_n == res_addr_n); end end - assign chain_stall = cmp_valid && valid_st1 && is_creq_st1 && chain_match_r; + assign chain_stall = cmp_valid && valid_st1 && is_creq_st1 && (chain_match_r || amo_st1.amo_valid); + + // AMOs that may reach the queue: at S0, between S0 and stC, and in the + // compute stage. A fill is admitted only while one more fits. The sum + // grows by at most one a cycle (one AMO enters S0), so the registered + // flag tests one entry early and stays exact a cycle later. + localparam AMO_CNTW = `CLOG2(PIPE_EX + 2); + wire st0_amo = amo_st0.amo_valid && valid_st0 && is_creq_st0; + wire st1_amo = amo_st1.amo_valid && valid_st1 && is_creq_st1; + wire amo_cnt_inc = st0_amo && ~pipe_stall; + wire amo_cnt_dec = st1_amo && ~pipe_stall; + reg [AMO_CNTW-1:0] amo_pipe_cnt; + reg wbq_full_r; + always @(posedge clk) begin + if (reset) begin + amo_pipe_cnt <= '0; + wbq_full_r <= 1'b0; + end else begin + if (amo_cnt_inc != amo_cnt_dec) begin + amo_pipe_cnt <= amo_cnt_inc ? (amo_pipe_cnt + AMO_CNTW'(1)) + : (amo_pipe_cnt - AMO_CNTW'(1)); + end + wbq_full_r <= (32'(wbq_count) + 32'(amo_pipe_cnt) + 32'(cmp_valid) + 32'(st0_amo) + 1) >= WBQ_SIZE; + end + end + assign wbq_full = wbq_full_r; + `RUNTIME_ASSERT (~(amo_cnt_dec && ~amo_cnt_inc && (amo_pipe_cnt == '0)), + ("%t: AMO pipe count underflow", $time)) // Invariants: a store-bearing AMO is only ever accepted into a free // compute stage (the queue absorbs different-line writebacks behind it), @@ -808,8 +849,9 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // covers it from S0 onward. Consumes only the registered probe (see // the header block) so the CAM stays out of core_req_ready. assign req_input_defer = core_req_valid - && (st0_amo_alloc_any || persist_amo_r + && (st0_amo_alloc_any || st1_amo_alloc_r || persist_amo_r || (fresh_r ? mshr_any_amo : probe_amo_hit_r)); + `UNUSED_VAR (st1_ld_alloc_r) `UNUSED_VAR (amo_st0) // only amo_valid/amo_op are consumed at S0 `UNUSED_VAR (is_replay_st0) @@ -860,13 +902,13 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // Age-ordering guards, off the registered probe (see the header // block): an incoming AMO holds behind pending line-filling requests, - // an incoming load holds behind pending AMO passthroughs. The S0 - // alloc pulses catch the window between admit and allocate. + // an incoming load holds behind pending AMO passthroughs. The S0/S1 + // alloc terms catch the window between admit and the probe. wire amo_input_defer = core_req_valid && core_req_is_amo - && (st0_ld_alloc_any || persist_ld_r + && (st0_ld_alloc_any || st1_ld_alloc_r || persist_ld_r || (fresh_r ? mshr_any_ld : probe_ld_hit_r)); wire load_input_defer = core_req_valid && ~core_req_is_amo && ~core_req_rw - && (st0_amo_alloc_any || persist_amo_r + && (st0_amo_alloc_any || st1_amo_alloc_r || persist_amo_r || (fresh_r ? mshr_any_amo : probe_amo_hit_r)); assign req_input_defer = amo_input_defer || load_input_defer; @@ -875,6 +917,7 @@ module VX_cache_amo import VX_gpu_pkg::*; #( assign commit_busy = 1'b0; assign chain_stall = 1'b0; assign wb_pending = 1'b0; + assign wbq_full = 1'b0; assign rsp_data = '0; // no local commit window: loads are held at the input while a // forwarded AMO is pending on the line, so no read forward exists. diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index f62c57214f..28576fcd39 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -188,7 +188,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( amo_req_t replay_amo; // AMO engine interconnect (tied to 0 when the bank carries no AMO logic). - wire amo_hit_st1, amo_commit_busy, amo_chain_stall, amo_wb_pending; + wire amo_hit_st1, amo_commit_busy, amo_chain_stall, amo_wb_pending, amo_wbq_full; wire [`CS_WORD_WIDTH-1:0] amo_rsp_data; wire [WORD_SIZE-1:0] amo_rd_fwd_mask; wire [`CS_WORD_WIDTH-1:0] amo_rd_fwd_data; @@ -282,7 +282,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( end assign pipe_inflight = st0.req.valid || st1.req.valid || (| commit_valid); end - wire no_pending_req = ~pipe_inflight && mreq_queue_empty; + // Queued AMO results are still writes to land in the array. + wire no_pending_req = ~pipe_inflight && mreq_queue_empty && ~amo_commit_busy; VX_cache_flush #( .BANK_ID (BANK_ID), @@ -350,8 +351,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( && (replay_rw || st0.req.rw) && (replay_wsel == st0.req.word_idx); + // A fill starts a replay chain, and a chain carries at most one AMO (no + // request enters behind a pending AMO), so a fill waits until the AMO + // writeback queue has room for it; the queue drains meanwhile. wire replay_mux = replay_valid && ~fwd_head && ~replay_link_hold; - wire fill_mux = mem_rsp_valid && ~fwd_pending && ~fill_inflight && ~replay_link_hold; + wire fill_mux = mem_rsp_valid && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full; wire replay_grant = ~init_valid; wire replay_enable = replay_grant && replay_mux; @@ -369,7 +373,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire creq_enable = creq_grant && (amo_creq_path || amo_wb_path); assign replay_ready = replay_grant && ~fwd_head && ~replay_link_hold && ~(!WRITEBACK && replay_rw && mreq_queue_alm_full) && ~pipe_stall; - assign mem_rsp_ready = fill_grant && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; + assign mem_rsp_ready = fill_grant && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign core_req_ready = creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; @@ -997,6 +1001,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .commit_busy (amo_commit_busy), .chain_stall (amo_chain_stall), .wb_pending (amo_wb_pending), + .wbq_full (amo_wbq_full), .rsp_data (amo_rsp_data), .rd_fwd_mask (amo_rd_fwd_mask), .rd_fwd_data (amo_rd_fwd_data), @@ -1015,7 +1020,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .req_input_defer (req_input_defer) ); end else begin : g_no_amo - assign {amo_hit_st1, amo_commit_busy, amo_wb_pending, amo_chain_stall} = '0; + assign {amo_hit_st1, amo_commit_busy, amo_wb_pending, amo_chain_stall, amo_wbq_full} = '0; assign {amo_rd_fwd_mask, amo_rd_fwd_data} = '0; assign {amo_rsp_data, amo_wb_addr, amo_wb_word_idx, amo_wb_byteen} = '0; assign {amo_wb_data, amo_wb_tag, amo_wb_idx, amo_wb_attr} = '0; diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index fdbdcaa787..22605fd148 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -373,6 +373,11 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // in arrival order like any request; the probe outputs alone provide // the same-line ordering guard. assign amo_mask = (AMO_PASSTHRU != 0) ? amo_table : '0; + // At the LLC no request links behind a pending AMO: the bank defers + // every request to a line while an AMO to it waits here. + `RUNTIME_ASSERT (~((AMO_PASSTHRU == 0) && finalize_valid && ~finalize_is_release && finalize_is_pending + && valid_table[finalize_previd] && persisted_table[finalize_previd] && amo_table[finalize_previd]), + ("%t: *** %s request linked behind a pending AMO: id=%0d", $time, INSTANCE_ID, finalize_previd)) wire [MSHR_SIZE-1:0] probe_ld, probe_amo; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_probe_matches From 843536c48df669cba58332271ebfdc6bad9e52d4 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:51:53 -0700 Subject: [PATCH 019/336] sim: redesign the DRAM model: independent channels, refresh, configurable type The shared Ramulator wrapper (sim/common/dram_sim) capped every simulator's memory far below the configured device: - one injection queue for all channels accepted one request per DRAM cycle in total, and one response was drained per core cycle device-wide, so the whole memory delivered at most about one line per cycle; - each 64 B line was split into four 16 B requests, four bursts' worth of command bandwidth for one line; - Ramulator's AllBank refresh manager iterates over a "rank" level, which HBM2 and GDDR6 do not have, so refresh never fired; - the DRAM clock was a raw -DMEM_CLOCK_RATIO define (default 1, the core clock), unrelated to the configured platform. DramSim now models one independent controller per platform memory bank, in simx, rtlsim, opaesim (which also gets per-bank queues) and xrtsim alike: the caller passes the bank as the channel and the address within it; a channel accepts one transaction per DRAM cycle and drains its own responses; a transaction is one burst (64 B HBM2, 32 B GDDR6) with the A100 HBM interleave or GPGPU-Sim's RTX 3070 GDDR layout and IPOLY bank hash; refresh is issued per channel. VX_config.toml selects it: VX_CFG_PLATFORM_MEMORY_TYPE HBM2 | HBM2_A100 | GDDR6 VX_CFG_PLATFORM_MEMORY_CLOCK_RATE DRAM clock (MHz); with PLATFORM_CLOCK_RATE it sets the DRAM cycles per core cycle VX_CFG_PLATFORM_MEMORY_LATENCY fixed read latency (core cycles) VX_CFG_PLATFORM_MEMORY_WINDOW requests a channel holds in flight The refresh manager and Ramulator's GDDR6 device (shipped but not built upstream) are C++20 like Ramulator, so they live in a companion library, third_party/ramulator_ext -> libramulator-vortex.so, built by third_party/Makefile and part of the CI third-party cache key. tests/unittest/dram_sim checks the properties the old model broke: bandwidth scales with the channel count (40 channels = 40x one), the fixed latency adds exactly, the window bounds requests in flight, a two-burst block answers once, refresh stalls reads (fails with the AllBank manager), and the A100 and RTX 5080 shapes land on their targets: A100 open-row/row-conflict reads take 216/257 core cycles (GPGPU-Sim SM80: ~215/~255) and stream 952 of 1,373 B/cycle; GDDR6 at the 5080 shape streams 393 of its 418 B/cycle peak (the card's 960 GB/s at the 2,295 MHz base clock). Every rtlsim/simx cycle count moves; the perf baselines are re-recorded in the same series. Verified: tests/unittest/dram_sim passes (12 checks); every simulator builds with the new model at XLEN 32 and 64; the CI matrix reruns on the series. Co-Authored-By: Claude Opus 5.5 --- .github/actions/setup-vortex/action.yml | 2 +- .gitignore | 4 + VX_config.toml | 9 + docs/codebase.md | 3 +- docs/designs/simx_simulator_architecture.md | 4 +- docs/simulation.md | 21 + sim/avedsim/Makefile | 2 +- sim/common/dram_sim.cpp | 495 +++++++++++++----- sim/common/dram_sim.h | 40 +- sim/opaesim/Makefile | 2 +- sim/opaesim/opae_sim.cpp | 34 +- sim/rtlsim/Makefile | 2 +- sim/rtlsim/processor.cpp | 12 +- sim/simx/Makefile | 2 +- sim/simx/constants.h | 4 - sim/simx/mem/memory.cpp | 16 +- sim/simx/mem/memory.h | 5 +- sim/simx/processor.cpp | 9 +- sim/xrtsim/Makefile | 2 +- sim/xrtsim/xrt_sim.cpp | 22 +- tests/unittest/Makefile | 2 +- tests/unittest/dram_sim/Makefile | 16 + tests/unittest/dram_sim/main.cpp | 228 ++++++++ third_party/Makefile | 20 +- third_party/ramulator_ext/channel_refresh.cpp | 65 +++ 25 files changed, 821 insertions(+), 200 deletions(-) create mode 100644 tests/unittest/dram_sim/Makefile create mode 100644 tests/unittest/dram_sim/main.cpp create mode 100644 third_party/ramulator_ext/channel_refresh.cpp diff --git a/.github/actions/setup-vortex/action.yml b/.github/actions/setup-vortex/action.yml index b27d3c1305..a73448a887 100644 --- a/.github/actions/setup-vortex/action.yml +++ b/.github/actions/setup-vortex/action.yml @@ -49,7 +49,7 @@ runs: uses: actions/cache@v4 with: path: third_party - key: ${{ runner.os }}-thirdparty-${{ hashFiles('third_party/Makefile', '.gitmodules') }} + key: ${{ runner.os }}-thirdparty-${{ hashFiles('third_party/Makefile', 'third_party/ramulator_ext/**', '.gitmodules') }} # Publish WHERE the toolchain cache above was restored to. The `tools` cache # path is relative to the workspace, but nothing downstream could know that: diff --git a/.gitignore b/.gitignore index dbcea3d6e3..136b7f2186 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ hw/syn/libs/asap7/verilog/ hw/syn/libs/asap7/metadata/ hw/syn/libs/asap7/.installed_* hw/syn/libs/asap7/install.sh + +# Built by third_party/Makefile +third_party/ramulator_ext/*.o +third_party/ramulator_ext/*.so diff --git a/VX_config.toml b/VX_config.toml index a3336551a8..8d331ff67b 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -79,6 +79,14 @@ VX_CFG_PLATFORM_MEMORY_DATA_SIZE = 64 VX_CFG_PLATFORM_MEMORY_INTERLEAVE = 1 VX_CFG_PLATFORM_MEMORY_PEAK_BW = 460000 # 460 GB/s VX_CFG_PLATFORM_CLOCK_RATE = 400 +# DRAM the simulators model behind each memory bank (one channel per bank), +# its clock (MHz), the fixed read latency added for the PHY, controller and +# interconnect (core cycles), and the requests a channel holds in flight +# (0 = unbounded). +VX_CFG_PLATFORM_MEMORY_TYPE = "HBM2" +VX_CFG_PLATFORM_MEMORY_CLOCK_RATE = 1000 +VX_CFG_PLATFORM_MEMORY_LATENCY = 0 +VX_CFG_PLATFORM_MEMORY_WINDOW = 0 [alu] VX_CFG_NUM_ALU_LANES = "expr: $VX_CFG_SIMD_WIDTH" @@ -456,6 +464,7 @@ VX_CFG_XLEN = [32, 64] VX_CFG_FLEN = [32, 64] VX_CFG_FPU_TYPE = ["DPI", "DSP", "FPNEW", "STD"] VX_CFG_TCU_TYPE = ["DPI", "DSP", "BHF", "TFR", "FPNEW"] +VX_CFG_PLATFORM_MEMORY_TYPE = ["HBM2", "HBM2_A100", "GDDR6"] [[param]] VX_CFG_DCACHE_NUM_REQS = "int" diff --git a/docs/codebase.md b/docs/codebase.md index e355ad3045..257af03e13 100644 --- a/docs/codebase.md +++ b/docs/codebase.md @@ -74,6 +74,7 @@ The directory/file layout of the Vortex codebase is as follows: - `runtime`: driver API tests - `mpi`: multi-process tests - `third_party`: external library submodules (cvfpu, softfloat, hardfloat, - ramulator, cocogfx) + ramulator, cocogfx), plus `ramulator_ext` (Vortex's Ramulator extensions: + per-channel refresh and the GDDR6 device) - `perf`: performance analysis resources - `miscs`: miscellaneous resources diff --git a/docs/designs/simx_simulator_architecture.md b/docs/designs/simx_simulator_architecture.md index 9913211a13..9da381bc39 100644 --- a/docs/designs/simx_simulator_architecture.md +++ b/docs/designs/simx_simulator_architecture.md @@ -118,7 +118,9 @@ atomics, threads, locks, or deferred-call plumbing — enforced by - `cache.{cpp,h}` — `Cache`: tags + MSHRs + replacement **and line data**; embeds the AMO unit. - `cache_cluster` — shared L2/L3 wrapper. -- `memory.{cpp,h}` — `Memory`: bottom-of-hierarchy DRAM (ramulator2); +- `memory.{cpp,h}` — `Memory`: bottom-of-hierarchy DRAM, one channel per + platform memory bank timed by `sim/common/dram_sim` (ramulator2, shared with + rtlsim/opaesim/xrtsim; see [Configuring the DRAM Model](../simulation.md#configuring-the-dram-model)); `attach_ram(RAM*)` owns the image; carries data in `MemRsp`; exposes `set_pre_send_hook` for SST mirroring. - `amo/amo_unit.{cpp,h}` — AMO RMW in the cache hierarchy (see diff --git a/docs/simulation.md b/docs/simulation.md index 29c6d3a2de..4b9d67fb8e 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -138,6 +138,27 @@ PERF: instrs=363180, cycles=53108, IPC=6.838518 > to compare configurations relative to one another, not as fixed targets to > reproduce. +### Configuring the DRAM Model + +The simulators (simx, rtlsim, opaesim, xrtsim) all model the device memory +the same way: one independent DRAM channel per platform memory bank, timed by +[Ramulator](https://github.com/CMU-SAFARI/ramulator2), with its refresh. These +`VX_config.toml` keys select it, overridable through `CONFIGS` or a profile: + +| Key | Default | Meaning | +|---|---|---| +| `VX_CFG_PLATFORM_MEMORY_TYPE` | `HBM2` | `HBM2` (JEDEC 2 Gbps), `HBM2_A100` (GPGPU-Sim's A100 timing) or `GDDR6` (x16 channels, 32 B bursts) | +| `VX_CFG_PLATFORM_MEMORY_NUM_BANKS` | 2 | memory banks, and so DRAM channels | +| `VX_CFG_PLATFORM_MEMORY_CLOCK_RATE` | 1000 | DRAM clock in MHz (GDDR6: the data clock, half the transfer rate) | +| `VX_CFG_PLATFORM_CLOCK_RATE` | 400 | core clock in MHz; with the DRAM clock it sets how many DRAM cycles a core cycle spans | +| `VX_CFG_PLATFORM_MEMORY_LATENCY` | 0 | fixed read latency added for the PHY, controller and interconnect, in core cycles | +| `VX_CFG_PLATFORM_MEMORY_WINDOW` | 0 | requests a channel holds in flight (0 = unbounded) | + +```bash +$ CONFIGS="-DVX_CFG_PLATFORM_MEMORY_TYPE=GDDR6 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_CLOCK_RATE=8000" \ + ./ci/blackbox.sh --driver=simx --app=vecadd +``` + ## Additional Quick Start Scenarios Running Vortex simulators with different configurations and drivers is supported. For example: diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index 6dd3b52bf2..dfe8ab9bb4 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -25,7 +25,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state -pthread # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE diff --git a/sim/common/dram_sim.cpp b/sim/common/dram_sim.cpp index cbb881e713..e6f7f1a0c5 100644 --- a/sim/common/dram_sim.cpp +++ b/sim/common/dram_sim.cpp @@ -13,7 +13,14 @@ #include "dram_sim.h" #include "util.h" +#include +#include #include +#include +#include +#include +#include +#include DISABLE_WARNING_PUSH DISABLE_WARNING_UNUSED_PARAMETER @@ -27,146 +34,360 @@ DISABLE_WARNING_POP using namespace vortex; +namespace { + +// IPOLY(5) bank hash over 16 banks (Rau, "Pseudo-randomly interleaved memory", +// ISCA 1991), GPGPU-Sim's -dram_bnk_indexing_policy 2: the bank XORed with a +// function of the row, so streams whose bases differ by a power of two land in +// different banks instead of closing each other's rows. +uint32_t ipoly16(uint64_t row, uint32_t bank) { + auto bit = [row](int i) { return uint32_t(row >> i) & 1u; }; + uint32_t i0 = bit(11) ^ bit(10) ^ bit(9) ^ bit(8) ^ bit(6) ^ bit(4) ^ bit(3) ^ bit(0); + uint32_t i1 = bit(12) ^ bit(8) ^ bit(7) ^ bit(6) ^ bit(5) ^ bit(3) ^ bit(1) ^ bit(0); + uint32_t i2 = bit(9) ^ bit(8) ^ bit(7) ^ bit(6) ^ bit(4) ^ bit(2) ^ bit(1); + uint32_t i3 = bit(10) ^ bit(9) ^ bit(8) ^ bit(7) ^ bit(5) ^ bit(3) ^ bit(2); + return bank ^ (i0 | (i1 << 1) | (i2 << 2) | (i3 << 3)); +} + +} + class DramSim::Impl { private: - struct mem_req_t { - uint64_t addr; - bool is_write; - ResponseCallback callback; - void* arg; - }; - - struct mem_rsp_t { - ResponseCallback callback; - void* arg; - }; - - Ramulator::IFrontEnd* ramulator_frontend_; - Ramulator::IMemorySystem* ramulator_memorysystem_; - uint32_t cpu_channel_size_; - uint64_t cpu_cycles_; - uint32_t scaled_dram_cycles_; - static const uint32_t tick_cycles_ = 1000; - static const uint32_t dram_channel_size_ = 16; // 128 bits - std::queue pending_reqs_; - std::queue pending_rsps_; - - void handle_pending_responses() { - if (pending_rsps_.empty()) - return; - auto& rsp = pending_rsps_.front(); - if (rsp.callback(rsp.arg)) { - pending_rsps_.pop(); - } - } - - void handle_pending_requests() { - if (pending_reqs_.empty()) - return; - auto& req = pending_reqs_.front(); - auto req_type = req.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; - std::function callback = nullptr; - if (req.callback) { - callback = [this, req_callback = std::move(req.callback), req_arg = std::move(req.arg)](Ramulator::Request& /*dram_req*/) { - this->pending_rsps_.push({req_callback, req_arg}); - }; - } - if (ramulator_frontend_->receive_external_requests(req_type, req.addr, 0, callback)) { - // Ramulator does not handle write responses, so we handle the callback ourselves. - if (req.is_write && req.callback) { - pending_rsps_.push({req.callback, req.arg}); - } - pending_reqs_.pop(); - } - } + // One request (a block); it completes when all its transactions have. + struct request_t { + ResponseCallback callback; + void* arg; + uint32_t pending; + }; + + struct mem_tx_t { + uint64_t addr; + bool is_write; + std::shared_ptr request; + }; + + struct mem_rsp_t { + std::shared_ptr request; + uint64_t ready_cycle; + }; + + // GDDR6 holds a fixed 16 Gb per device, so its row count shrinks as the + // channel count grows (Ramulator only knows the refresh timing of 4/8/16 Gb + // devices); rows alias above that capacity, which only affects row hits. + static constexpr uint32_t gddr6_density_mb_ = 16384; + + Config config_; + uint32_t tx_size_; + uint32_t row_mask_; + Ramulator::IFrontEnd* ramulator_frontend_; + Ramulator::IMemorySystem* ramulator_memorysystem_; + uint32_t ch_bits_; + uint64_t cycle_; + // DRAM ticks owed, in units of 1/core_clock: each core cycle adds + // dram_clock, each DRAM tick takes core_clock. + uint64_t clock_credit_; + // A channel accepts one transaction per DRAM cycle and returns its + // responses independently of the others; write acknowledgements do not wait + // behind read data. + std::vector> pending_txs_; + std::vector> read_rsps_; + std::vector> write_rsps_; + std::vector inflight_; + + // Ramulator address of a transaction: the channel in the low bits, then the + // fields in the order the RoBaRaCoCh mapper slices them. + uint64_t dram_addr(uint32_t channel, uint64_t addr) const { + uint64_t t; + if (config_.type == Type::GDDR6) { + // GPGPU-Sim's RTX 3070 layout (-gpgpu_mem_addr_mapping): 32 B bursts, + // two column bits, one bank bit, four column bits, three bank bits, + // row; the low bank bit selects the bank group. + uint64_t col = ((addr >> 5) & 0x3) | (((addr >> 8) & 0xf) << 2); + uint64_t row = (addr >> 15) & row_mask_; + uint32_t bank = ipoly16(row, uint32_t(((addr >> 7) & 0x1) | (((addr >> 12) & 0x7) << 1))); + t = row; + t = (t << 2) | (bank >> 2); + t = (t << 2) | (bank & 0x3); + t = (t << 6) | col; + } else { + // The A100 HBM interleave: a stream alternates column, pseudo-channel + // and column again at 64 B granules, then changes bank group every 4 KB + // and bank every 16 KB, with 2 KB rows. HBM2_A100 uses 2 of the 4 banks + // of a group (16 per channel), as GPGPU-Sim models the A100; HBM2 all 4. + uint32_t bank_bits = (config_.type == Type::HBM2_A100) ? 1 : 2; + uint64_t col = (((addr >> 8) & 0xf) << 1) | ((addr >> 6) & 0x1); + uint64_t pc = (addr >> 7) & 0x1; + uint64_t bg = (addr >> 12) & 0x3; + uint64_t bank = (addr >> 14) & ((1u << bank_bits) - 1); + uint64_t row = (addr >> (14 + bank_bits)) & row_mask_; + t = row; + t = (t << 2) | bank; + t = (t << 2) | bg; + t = (t << 1) | pc; + t = (t << 5) | col; + } + t = (t << ch_bits_) | channel; + return t * tx_size_; + } + + static void set_hbm2_device(YAML::Node& dram, const Config& config, uint32_t pow2_channels) { + dram["impl"] = "HBM2"; + dram["org"]["preset"] = "HBM2_8Gb"; + dram["org"]["density"] = 8192; + dram["org"]["channel"] = pow2_channels; + // A transaction is one burst: 2 x 256-bit prefetch = 64 B. + dram["org"]["channel_width"] = 256; + dram["timing"]["preset"] = "HBM2_2Gbps"; + if (config.type == Type::HBM2_A100) { + // GPGPU-Sim SM80_A100 HBM timing in DRAM cycles. One 64 B transaction + // is two of its 32 B atoms, so the same-bank-group spacing is twice its + // tCCDL. + auto timing = dram["timing"]; + timing["nCL"] = 22; + timing["nRCDRD"] = 22; + timing["nRCDWR"] = 22; + timing["nRP"] = 22; + timing["nRAS"] = 50; + timing["nRC"] = 72; + timing["nWR"] = 19; + timing["nRTPS"] = 7; + timing["nRTPL"] = 7; + timing["nCWL"] = 4; + timing["nCCDS"] = 2; + timing["nCCDL"] = 8; + timing["nRRDS"] = 7; + timing["nRRDL"] = 7; + } + } + + static void set_gddr6_device(YAML::Node& dram, const Config& config, uint32_t pow2_channels) { + // x16 channels with 2 KB pages: 512 columns of 16 bits, 4 bank groups of 4 + // banks; a 32 B transaction is a 16-beat burst (8 prefetched columns over a + // 32-bit internal channel width). + dram["impl"] = "GDDR6"; + dram["org"]["dq"] = 16; + dram["org"]["channel"] = pow2_channels; + dram["org"]["bankgroup"] = 4; + dram["org"]["bank"] = 4; + dram["org"]["column"] = 512; + dram["org"]["row"] = uint32_t((uint64_t(gddr6_density_mb_) << 20) / (uint64_t(pow2_channels) * 4 * 4 * 512 * 16)); + dram["org"]["density"] = gddr6_density_mb_; + dram["org"]["channel_width"] = 32; + // The transfer rate is twice the data clock. Bus timings are in clocks of + // the 32 B burst; array timings are GPGPU-Sim's RTX 3070 values + // (-gpgpu_dram_timing_opt) held in nanoseconds; refresh is JEDEC GDDR6. + auto timing = dram["timing"]; + timing["rate"] = 2 * config.dram_clock; + timing["nBL"] = 8; + timing["nCCDS"] = 8; + timing["nCCDL"] = 12; + timing["tCL"] = 6.8f; + timing["tRCDRD"] = 6.8f; + timing["tRCDWD"] = 6.8f; + timing["tRP"] = 6.8f; + timing["tRAS"] = 15.7f; + timing["tRC"] = 22.3f; + timing["tWR"] = 6.8f; + timing["tRTP"] = 1.2f; + timing["tCWL"] = 2.3f; + timing["tRRDS"] = 3.5f; + timing["tRRDL"] = 3.5f; + timing["tWTRS"] = 2.8f; + timing["tWTRL"] = 2.8f; + timing["tFAW"] = 16.0f; + timing["tRFC"] = 120.0f; + timing["tRFCpb"] = 60.0f; + timing["tREFI"] = 1900.0f; + timing["tRREFD"] = 8.0f; + } + + // Retires the ready head responses of `rsps`; a request completes with its + // last transaction. + void drain(std::deque& rsps, uint32_t ch) { + while (!rsps.empty()) { + auto& rsp = rsps.front(); + if (rsp.ready_cycle > cycle_) { + break; + } + auto& request = *rsp.request; + if (request.pending == 1 && request.callback && !request.callback(request.arg)) { + break; + } + if (--request.pending == 0) { + --inflight_[ch]; + } + rsps.pop_front(); + } + } + + void handle_pending_responses() { + for (uint32_t ch = 0; ch < config_.num_channels; ++ch) { + this->drain(read_rsps_[ch], ch); + this->drain(write_rsps_[ch], ch); + } + } + + void handle_pending_requests() { + for (uint32_t ch = 0; ch < config_.num_channels; ++ch) { + auto& txs = pending_txs_[ch]; + if (txs.empty()) { + continue; + } + auto& tx = txs.front(); + auto req_type = tx.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; + std::function callback = nullptr; + if (!tx.is_write) { + callback = [this, ch, request = tx.request](Ramulator::Request& /*dram_req*/) { + this->read_rsps_[ch].push_back({request, this->cycle_ + 1 + this->config_.latency}); + }; + } + if (ramulator_frontend_->receive_external_requests(req_type, tx.addr, 0, callback)) { + // Ramulator does not handle write responses, so we handle the callback ourselves. + if (tx.is_write) { + write_rsps_[ch].push_back({tx.request, cycle_}); + } + txs.pop(); + } + } + } public: - Impl(uint32_t num_channels, uint32_t channel_size, float clock_ratio) { - YAML::Node dram_config; - dram_config["Frontend"]["impl"] = "GEM5"; - dram_config["MemorySystem"]["impl"] = "GenericDRAM"; - dram_config["MemorySystem"]["clock_ratio"] = 1; - dram_config["MemorySystem"]["DRAM"]["impl"] = "HBM2"; - dram_config["MemorySystem"]["DRAM"]["org"]["preset"] = "HBM2_8Gb"; - dram_config["MemorySystem"]["DRAM"]["org"]["density"] = 8192; - dram_config["MemorySystem"]["DRAM"]["org"]["channel"] = num_channels; - dram_config["MemorySystem"]["DRAM"]["timing"]["preset"] = "HBM2_2Gbps"; - dram_config["MemorySystem"]["Controller"]["impl"] = "Generic"; - dram_config["MemorySystem"]["Controller"]["Scheduler"]["impl"] = "FRFCFS"; - dram_config["MemorySystem"]["Controller"]["RefreshManager"]["impl"] = "AllBank"; - dram_config["MemorySystem"]["Controller"]["RowPolicy"]["impl"] = "OpenRowPolicy"; - // Per-command DRAM trace recording is a debugging aid only: it writes an - // entry for every DRAM command and dominates simulation time (100s of MB - // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env - // var so default simx/rtlsim runs stay fast. - if (const char* trace_path = getenv("DRAM_TRACE")) { - YAML::Node draw_plugin; - draw_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; - draw_plugin["ControllerPlugin"]["path"] = trace_path[0] ? trace_path : "./trace/ramulator.log"; - dram_config["MemorySystem"]["Controller"]["plugins"].push_back(draw_plugin); - } - dram_config["MemorySystem"]["AddrMapper"]["impl"] = "RoBaRaCoCh"; - - ramulator_frontend_ = Ramulator::Factory::create_frontend(dram_config); - ramulator_memorysystem_ = Ramulator::Factory::create_memory_system(dram_config); - ramulator_frontend_->connect_memory_system(ramulator_memorysystem_); - ramulator_memorysystem_->connect_frontend(ramulator_frontend_); - - cpu_channel_size_ = channel_size; - scaled_dram_cycles_ = static_cast(clock_ratio * tick_cycles_); - this->reset(); - } - - ~Impl() { - std::ofstream nullstream("ramulator.stats.log"); - auto original_buf = std::cout.rdbuf(); - std::cout.rdbuf(nullstream.rdbuf()); - ramulator_frontend_->finalize(); - ramulator_memorysystem_->finalize(); - std::cout.rdbuf(original_buf); - } - - void reset() { - cpu_cycles_ = 0; - pending_reqs_ = std::queue(); - pending_rsps_ = std::queue(); - } - - void tick() { - cpu_cycles_ += tick_cycles_; - this->handle_pending_responses(); - while (cpu_cycles_ >= scaled_dram_cycles_) { - this->handle_pending_requests(); - ramulator_memorysystem_->tick(); - cpu_cycles_ -= scaled_dram_cycles_; - } - } - - void send_request(uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { - if (cpu_channel_size_ > dram_channel_size_) { - uint32_t n = cpu_channel_size_ / dram_channel_size_; - for (uint32_t i = 0; i < n; ++i) { - uint64_t dram_byte_addr = (addr / cpu_channel_size_) * dram_channel_size_ + (i * dram_channel_size_); - if (i == 0) { - pending_reqs_.push({dram_byte_addr, is_write, response_cb, arg}); - } else { - pending_reqs_.push({dram_byte_addr, is_write, nullptr, nullptr}); - } - } - } else if (cpu_channel_size_ < dram_channel_size_) { - uint64_t dram_byte_addr = (addr / cpu_channel_size_) * dram_channel_size_; - pending_reqs_.push({dram_byte_addr, is_write, response_cb, arg}); - } else { - uint64_t dram_byte_addr = addr; - pending_reqs_.push({dram_byte_addr, is_write, response_cb, arg}); - } - } + Impl(const Config& config) : config_(config) { + uint32_t pow2_channels = 1; + ch_bits_ = 0; + while (pow2_channels < config.num_channels) { + pow2_channels <<= 1; + ++ch_bits_; + } + + YAML::Node dram; + if (config.type == Type::GDDR6) { + set_gddr6_device(dram, config, pow2_channels); + tx_size_ = 32; + row_mask_ = dram["org"]["row"].as() - 1; + } else { + set_hbm2_device(dram, config, pow2_channels); + tx_size_ = 64; + row_mask_ = 0x7fff; + } + + YAML::Node dram_config; + dram_config["Frontend"]["impl"] = "GEM5"; + dram_config["MemorySystem"]["impl"] = "GenericDRAM"; + dram_config["MemorySystem"]["clock_ratio"] = 1; + dram_config["MemorySystem"]["DRAM"] = dram; + dram_config["MemorySystem"]["Controller"]["impl"] = "Generic"; + dram_config["MemorySystem"]["Controller"]["Scheduler"]["impl"] = "FRFCFS"; + // libramulator-vortex: Ramulator's AllBank refresh never fires on rank-less HBM2/GDDR6. + dram_config["MemorySystem"]["Controller"]["RefreshManager"]["impl"] = "ChannelRefresh"; + dram_config["MemorySystem"]["Controller"]["RowPolicy"]["impl"] = "OpenRowPolicy"; + // Per-command DRAM trace recording is a debugging aid only: it writes an + // entry for every DRAM command and dominates simulation time (100s of MB + // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env + // var so default simx/rtlsim runs stay fast. + if (const char* trace_path = getenv("DRAM_TRACE")) { + YAML::Node draw_plugin; + draw_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; + draw_plugin["ControllerPlugin"]["path"] = trace_path[0] ? trace_path : "./trace/ramulator.log"; + dram_config["MemorySystem"]["Controller"]["plugins"].push_back(draw_plugin); + } + dram_config["MemorySystem"]["AddrMapper"]["impl"] = "RoBaRaCoCh"; + + ramulator_frontend_ = Ramulator::Factory::create_frontend(dram_config); + ramulator_memorysystem_ = Ramulator::Factory::create_memory_system(dram_config); + ramulator_frontend_->connect_memory_system(ramulator_memorysystem_); + ramulator_memorysystem_->connect_frontend(ramulator_frontend_); + + pending_txs_.resize(config.num_channels); + read_rsps_.resize(config.num_channels); + write_rsps_.resize(config.num_channels); + inflight_.resize(config.num_channels, 0); + this->reset(); + } + + ~Impl() { + std::ofstream nullstream("ramulator.stats.log"); + auto original_buf = std::cout.rdbuf(); + std::cout.rdbuf(nullstream.rdbuf()); + ramulator_frontend_->finalize(); + ramulator_memorysystem_->finalize(); + std::cout.rdbuf(original_buf); + } + + void reset() { + cycle_ = 0; + clock_credit_ = 0; + for (auto& q : pending_txs_) { + q = std::queue(); + } + for (auto& q : read_rsps_) { + q.clear(); + } + for (auto& q : write_rsps_) { + q.clear(); + } + std::fill(inflight_.begin(), inflight_.end(), 0); + } + + void tick() { + ++cycle_; + clock_credit_ += config_.dram_clock; + this->handle_pending_responses(); + while (clock_credit_ >= config_.core_clock) { + this->handle_pending_requests(); + ramulator_memorysystem_->tick(); + clock_credit_ -= config_.core_clock; + } + } + + bool can_accept(uint32_t channel) const { + return (config_.window == 0) || (inflight_.at(channel) < config_.window); + } + + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { + // A block wider than a burst is several transactions. + uint32_t n = (config_.block_size > tx_size_) ? (config_.block_size / tx_size_) : 1; + uint64_t base = addr & ~uint64_t(config_.block_size - 1); + auto request = std::make_shared(request_t{response_cb, arg, n}); + auto& txs = pending_txs_.at(channel); + for (uint32_t i = 0; i < n; ++i) { + txs.push({dram_addr(channel, base + i * tx_size_), is_write, request}); + } + ++inflight_[channel]; + } }; /////////////////////////////////////////////////////////////////////////////// -DramSim::DramSim(uint32_t num_channels, uint32_t channel_size, float clock_ratio) - : impl_(new Impl(num_channels, channel_size, clock_ratio)) +#define DRAM_STR_(x) #x +#define DRAM_STR(x) DRAM_STR_(x) + +DramSim::Config DramSim::platform_config() { + // The enum's value macro, not its per-value companion: the simulators get + // raw CONFIGS, where -DVX_CFG_PLATFORM_MEMORY_TYPE= defines no + // companion. + const std::string type = DRAM_STR(VX_CFG_PLATFORM_MEMORY_TYPE); + Config config; + if (type == "HBM2") { + config.type = Type::HBM2; + } else if (type == "HBM2_A100") { + config.type = Type::HBM2_A100; + } else if (type == "GDDR6") { + config.type = Type::GDDR6; + } else { + throw std::invalid_argument("dram_sim: unknown VX_CFG_PLATFORM_MEMORY_TYPE " + type); + } + config.num_channels = VX_CFG_PLATFORM_MEMORY_NUM_BANKS; + config.block_size = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + config.core_clock = VX_CFG_PLATFORM_CLOCK_RATE; + config.dram_clock = VX_CFG_PLATFORM_MEMORY_CLOCK_RATE; + config.latency = VX_CFG_PLATFORM_MEMORY_LATENCY; + config.window = VX_CFG_PLATFORM_MEMORY_WINDOW; + return config; +} + +DramSim::DramSim(const Config& config) + : impl_(new Impl(config)) {} DramSim::~DramSim() { @@ -181,6 +402,10 @@ void DramSim::tick() { impl_->tick(); } -void DramSim::send_request(uint64_t addr, bool is_write, ResponseCallback callback, void* arg) { - impl_->send_request(addr, is_write, callback, arg); -} \ No newline at end of file +bool DramSim::can_accept(uint32_t channel) const { + return impl_->can_accept(channel); +} + +void DramSim::send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback callback, void* arg) { + impl_->send_request(channel, addr, is_write, callback, arg); +} diff --git a/sim/common/dram_sim.h b/sim/common/dram_sim.h index 7e8741791c..cc88ff5d78 100644 --- a/sim/common/dram_sim.h +++ b/sim/common/dram_sim.h @@ -11,27 +11,53 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#pragma once + +#include namespace vortex { +// Device memory behind the platform's memory banks: one DRAM channel (an +// independent controller) per bank, timed by Ramulator. class DramSim { public: typedef bool (*ResponseCallback)(void *arg); - DramSim(uint32_t num_channels, uint32_t channel_size, float clock_ratio); + enum class Type { + HBM2, // HBM2 at the 2 Gbps JEDEC preset, 64 B bursts + HBM2_A100, // HBM2 with the A100 timing of GPGPU-Sim's SM80 configuration + GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages + }; + + struct Config { + Type type; + uint32_t num_channels; // need not be a power of two + uint32_t block_size; // bytes per request + uint32_t core_clock; // MHz + uint32_t dram_clock; // MHz (GDDR6: the data clock, half the transfer rate) + uint32_t latency; // fixed read latency added in core cycles (PHY, controller, NoC) + uint32_t window; // requests a channel holds in flight; 0 = unbounded + }; + + // The memory configured in VX_config.toml. + static Config platform_config(); + + DramSim(const Config& config); ~DramSim(); void reset(); void tick(); - // addr: per-channel block address - void send_request(uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg); + // False while the channel holds its window of requests. + bool can_accept(uint32_t channel) const; + + // addr: byte address within the channel. + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg); private: - class Impl; - Impl* impl_; + class Impl; + Impl* impl_; }; -} \ No newline at end of file +} diff --git a/sim/opaesim/Makefile b/sim/opaesim/Makefile index 61fc9aa0d8..f4dbfb723a 100644 --- a/sim/opaesim/Makefile +++ b/sim/opaesim/Makefile @@ -21,7 +21,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state -pthread # debug trace enable flags DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE diff --git a/sim/opaesim/opae_sim.cpp b/sim/opaesim/opae_sim.cpp index 94e9adfbe5..7e37ee5dde 100644 --- a/sim/opaesim/opae_sim.cpp +++ b/sim/opaesim/opae_sim.cpp @@ -40,15 +40,12 @@ #include #include #include +#include #include #include #include #include -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif - #define CACHE_BLOCK_SIZE 64 #define CCI_LATENCY 8 @@ -100,7 +97,7 @@ class opae_sim::Impl { Impl() : device_(nullptr) , ram_(nullptr) - , dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) + , dram_sim_(DramSim::platform_config()) , stop_(false) , host_buffer_ids_(0) #ifdef VCD_OUTPUT @@ -297,9 +294,8 @@ class opae_sim::Impl { reqs.clear(); } - { - std::queue empty; - std::swap(dram_queue_, empty); + for (auto& queue : dram_queues_) { + queue = std::queue(); } device_->reset = 1; @@ -318,9 +314,15 @@ class opae_sim::Impl { this->cci_bus_eval(); this->avs_bus_eval(); - if (!dram_queue_.empty()) { - auto mem_req = dram_queue_.front(); - dram_sim_.send_request(mem_req->addr, mem_req->write, [](void* arg)->bool { + dram_sim_.tick(); + + for (uint32_t b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& queue = dram_queues_[b]; + if (queue.empty() || !dram_sim_.can_accept(b)) { + continue; + } + auto mem_req = queue.front(); + dram_sim_.send_request(b, uint64_t(mem_req->addr) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE, mem_req->write, [](void* arg)->bool { auto orig_req = reinterpret_cast(arg); if (orig_req->ready) { delete orig_req; @@ -329,11 +331,9 @@ class opae_sim::Impl { } return true; }, mem_req); - dram_queue_.pop(); + queue.pop(); } - dram_sim_.tick(); - device_->clk = 0; this->eval(); device_->clk = 1; @@ -499,7 +499,7 @@ class opae_sim::Impl { mem_req->write = true; mem_req->ready = true; - dram_queue_.push(mem_req); + dram_queues_[b].push(mem_req); } else if (device_->avs_read[b]) { // process read request @@ -512,7 +512,7 @@ class opae_sim::Impl { pending_mem_reqs_[b].emplace_back(mem_req); // send dram request - dram_queue_.push(mem_req); + dram_queues_[b].push(mem_req); } device_->avs_waitrequest[b] = false; @@ -580,7 +580,7 @@ class opae_sim::Impl { } }; - std::queue dram_queue_; + std::array, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> dram_queues_; #ifdef VCD_OUTPUT VerilatedVcdC *tfp_; diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index f6b8ca3cf8..f15c7da8a7 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -20,7 +20,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index a0e223f2e0..b383d556ff 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -48,9 +48,6 @@ #include #include -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif #ifndef TRACE_START_TIME #define TRACE_START_TIME 0ull @@ -101,7 +98,7 @@ bool sim_trace_enabled() { class Processor::Impl { public: - Impl() : dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) { + Impl() : dram_sim_(DramSim::platform_config()) { // restart sim time so a re-created model registers at time zero timestamp = 0; @@ -306,9 +303,9 @@ class Processor::Impl { dram_sim_.tick(); for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { - if (!dram_queue_[b].empty()) { + if (!dram_queue_[b].empty() && dram_sim_.can_accept(b)) { auto mem_req = dram_queue_[b].front(); - dram_sim_.send_request(mem_req->addr, mem_req->write, [](void* arg)->bool { + dram_sim_.send_request(b, mem_req->bank_addr, mem_req->write, [](void* arg)->bool { // mark completed request as ready auto orig_req = reinterpret_cast(arg); orig_req->ready = true; @@ -400,6 +397,7 @@ class Processor::Impl { auto mem_req = new mem_req_t(); mem_req->tag = device_->mem_req_tag[b]; mem_req->addr = byte_addr; + mem_req->bank_addr = uint64_t(device_->mem_req_addr[b]) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; mem_req->write = true; mem_req->ready = false; @@ -414,6 +412,7 @@ class Processor::Impl { auto mem_req = new mem_req_t(); mem_req->tag = device_->mem_req_tag[b]; mem_req->addr = byte_addr; + mem_req->bank_addr = uint64_t(device_->mem_req_addr[b]) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; mem_req->write = false; mem_req->ready = false; ram_->read(mem_req->data.data(), byte_addr, VX_CFG_PLATFORM_MEMORY_DATA_SIZE); @@ -445,6 +444,7 @@ class Processor::Impl { Vrtlsim_shim* device; std::array data; uint64_t addr; + uint64_t bank_addr; uint64_t tag; bool write; bool ready; diff --git a/sim/simx/Makefile b/sim/simx/Makefile index a9638e5ac7..746bd6f7f8 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -27,7 +27,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 1bdfce56a5..2ef860870a 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -20,10 +20,6 @@ #define RAM_PAGE_SIZE 4096 #endif -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif - #ifndef __MAX #define __MAX(a,b) (((a) > (b)) ? (a) : (b)) #endif diff --git a/sim/simx/mem/memory.cpp b/sim/simx/mem/memory.cpp index 74c3782edc..50db7006ae 100644 --- a/sim/simx/mem/memory.cpp +++ b/sim/simx/mem/memory.cpp @@ -55,13 +55,13 @@ class Memory::Impl { Impl(Memory* simobject, const Config& config) : simobject_(simobject) , config_(config) - , dram_sim_(config.num_banks, config.block_size, config.clock_ratio) + , dram_sim_(config.dram) , ram_(nullptr) { char sname[100]; snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); - mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config.num_ports, config.num_banks, - [lg2_block_size = log2ceil(config.block_size), num_banks = config.num_banks](const MemCrossBar::ReqType& req) { + mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config.num_ports, config.dram.num_channels, + [lg2_block_size = log2ceil(config.dram.block_size), num_banks = config.dram.num_channels](const MemCrossBar::ReqType& req) { // Custom logic to calculate the output index using bank interleaving return (uint32_t)((req.addr >> lg2_block_size) & (num_banks-1)); }); @@ -85,8 +85,10 @@ class Memory::Impl { void tick() { dram_sim_.tick(); - for (uint32_t i = 0; i < config_.num_banks; ++i) { - if (mem_xbar_->ReqOut.at(i).empty()) + uint32_t num_banks = config_.dram.num_channels; + uint32_t lg2_block_size = log2ceil(config_.dram.block_size); + for (uint32_t i = 0; i < num_banks; ++i) { + if (mem_xbar_->ReqOut.at(i).empty() || !dram_sim_.can_accept(i)) continue; auto& mem_req = mem_xbar_->ReqOut.at(i).peek(); @@ -126,8 +128,10 @@ class Memory::Impl { // enqueue the request to the memory system auto req_args = new DramCallbackArgs{this, mem_req, i, rsp_data}; + uint64_t bank_addr = ((mem_req.addr >> lg2_block_size) / num_banks) << lg2_block_size; dram_sim_.send_request( - mem_req.addr, + i, + bank_addr, mem_req.is_write(), [](void* arg)->bool { auto rsp_args = reinterpret_cast(arg); diff --git a/sim/simx/mem/memory.h b/sim/simx/mem/memory.h index ea060c0e05..dd4a95be39 100644 --- a/sim/simx/mem/memory.h +++ b/sim/simx/mem/memory.h @@ -15,6 +15,7 @@ #include "types.h" #include +#include #include namespace vortex { @@ -22,10 +23,8 @@ namespace vortex { class Memory : public SimObject{ public: struct Config { - uint32_t num_banks; uint32_t num_ports; - uint32_t block_size; - float clock_ratio; + DramSim::Config dram; // one DRAM channel per memory bank }; struct PerfStats { diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 0aa304656f..52ad76ca1a 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -51,12 +51,9 @@ ProcessorImpl::ProcessorImpl() kmu_ = Kmu::Create("kmu", total_cores); // create memory simulator - memsim_ = Memory::Create("dram", Memory::Config{ - VX_CFG_PLATFORM_MEMORY_NUM_BANKS, - VX_CFG_L3_MEM_PORTS, - VX_CFG_MEM_BLOCK_SIZE, - MEM_CLOCK_RATIO - }); + auto dram_config = DramSim::platform_config(); + dram_config.block_size = VX_CFG_MEM_BLOCK_SIZE; + memsim_ = Memory::Create("dram", Memory::Config{VX_CFG_L3_MEM_PORTS, dram_config}); char sname[100]; diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index 231d745806..4e51883d19 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -22,7 +22,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state -pthread # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE diff --git a/sim/xrtsim/xrt_sim.cpp b/sim/xrtsim/xrt_sim.cpp index fcab150cfe..af948d92dc 100644 --- a/sim/xrtsim/xrt_sim.cpp +++ b/sim/xrtsim/xrt_sim.cpp @@ -50,9 +50,6 @@ #include -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif #define CACHE_BLOCK_SIZE 64 @@ -144,7 +141,7 @@ class xrt_sim::Impl { , ram_{} , host_ram_(nullptr) , host_alloc_(nullptr) - , dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) + , dram_sim_(DramSim::platform_config()) , stop_(false) #ifdef VCD_OUTPUT , tfp_(nullptr) @@ -536,9 +533,9 @@ class xrt_sim::Impl { dram_sim_.tick(); for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { - if (!dram_queues_[b].empty()) { + if (!dram_queues_[b].empty() && dram_sim_.can_accept(b)) { auto mem_req = dram_queues_[b].front(); - dram_sim_.send_request(mem_req->addr, mem_req->write, [](void* arg)->bool { + dram_sim_.send_request(b, bank_local_addr(mem_req->addr), mem_req->write, [](void* arg)->bool { auto orig_req = reinterpret_cast(arg); if (orig_req->ready) { delete orig_req; @@ -556,6 +553,19 @@ class xrt_sim::Impl { #endif } + // Byte address within its bank (DRAM channel) of an AXI bank address: + // VX_mem_to_axi places the bank select above the data-size bits when the + // banks are interleaved. + static uint64_t bank_local_addr(uint64_t axi_addr) { + #if VX_CFG_PLATFORM_MEMORY_INTERLEAVE == 1 + constexpr uint32_t lg2_data_size = log2ceil(VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + constexpr uint32_t bank_sel_bits = log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + return (axi_addr >> (bank_sel_bits + lg2_data_size)) << lg2_data_size; + #else + return axi_addr; + #endif + } + void eval() { device_->eval(); #ifdef VCD_OUTPUT diff --git a/tests/unittest/Makefile b/tests/unittest/Makefile index 0c561d642e..33554509cd 100644 --- a/tests/unittest/Makefile +++ b/tests/unittest/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../..) include $(ROOT_DIR)/config.mk -PROJECTS := vx_malloc linked_list dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt +PROJECTS := vx_malloc linked_list dram_sim dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt all: $(foreach p,$(PROJECTS),$(MAKE) -C $(p) &&) true diff --git a/tests/unittest/dram_sim/Makefile b/tests/unittest/dram_sim/Makefile new file mode 100644 index 0000000000..7994c7a91d --- /dev/null +++ b/tests/unittest/dram_sim/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := dram_sim + +SRC_DIR := $(VORTEX_HOME)/tests/unittest/$(PROJECT) +RAMULATOR_DIR := $(THIRD_PARTY_DIR)/ramulator + +SRCS := $(SRC_DIR)/main.cpp $(SIM_COMMON_DIR)/dram_sim.cpp + +CXXFLAGS += -I$(SIM_COMMON_DIR) +CXXFLAGS += -isystem $(RAMULATOR_DIR)/ext/spdlog/include -isystem $(RAMULATOR_DIR)/ext/yaml-cpp/include -isystem $(RAMULATOR_DIR)/src +LDFLAGS += -Wl,-rpath,$(RAMULATOR_DIR) -L$(RAMULATOR_DIR) -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state + +include ../common.mk diff --git a/tests/unittest/dram_sim/main.cpp b/tests/unittest/dram_sim/main.cpp new file mode 100644 index 0000000000..81e127e145 --- /dev/null +++ b/tests/unittest/dram_sim/main.cpp @@ -0,0 +1,228 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +using namespace vortex; + +static uint64_t g_responses = 0; + +static bool on_response(void*) { + ++g_responses; + return true; +} + +// A100 HBM as GPGPU-Sim's SM80 configuration models it: 40 channels at +// 1,512 MHz behind a 1,410 MHz core, 190 cycles of fixed latency, a 257-request +// window per channel. +static DramSim::Config a100_config() { + DramSim::Config config; + config.type = DramSim::Type::HBM2_A100; + config.num_channels = 40; + config.block_size = 64; + config.core_clock = 1410; + config.dram_clock = 1512; + config.latency = 190; + config.window = 257; + return config; +} + +// Median core cycles for one read on an idle channel (the median, so the few +// reads that wait behind a refresh do not count); `addr_of(i)` picks the i-th +// address, and the first read (cold) is not counted. +template +static double unloaded_latency(const DramSim::Config& config, F addr_of) { + DramSim dram(config); + dram.reset(); + const int n = 33; + std::vector samples; + for (int i = 0; i < n; ++i) { + g_responses = 0; + uint64_t cycles = 0; + dram.send_request(0, addr_of(i), false, on_response, nullptr); + while (g_responses == 0) { + dram.tick(); + ++cycles; + } + if (i != 0) { + samples.push_back(cycles); + } + for (int k = 0; k < 256; ++k) { + dram.tick(); + } + } + std::sort(samples.begin(), samples.end()); + return double(samples[samples.size() / 2]); +} + +// Bytes per core cycle when every channel streams `blocks` sequential blocks; +// `peak_inflight` receives the most reads outstanding at once. +static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, uint64_t* peak_inflight = nullptr) { + DramSim dram(config); + dram.reset(); + g_responses = 0; + std::vector sent(config.num_channels, 0); + uint64_t total = uint64_t(blocks) * config.num_channels; + uint64_t issued = 0; + uint64_t cycles = 0; + uint64_t peak = 0; + while (g_responses < total) { + for (uint32_t ch = 0; ch < config.num_channels; ++ch) { + if (sent[ch] < blocks && dram.can_accept(ch)) { + dram.send_request(ch, uint64_t(sent[ch]) * config.block_size, false, on_response, nullptr); + ++sent[ch]; + ++issued; + } + } + peak = std::max(peak, issued - g_responses); + dram.tick(); + ++cycles; + } + if (peak_inflight) { + *peak_inflight = peak; + } + return double(total) * config.block_size / cycles; +} + +static int g_errors = 0; + +static void check(bool cond, const char* what, double got) { + printf("%s %s (%.1f)\n", cond ? "PASS" : "FAIL", what, got); + if (!cond) { + ++g_errors; + } +} + +int main() { + auto a100 = a100_config(); + + // Unloaded latency: open-row hits, then row conflicts (precharge + activate). + double open = unloaded_latency(a100, [](int i) { return uint64_t(i % 8) * 128; }); + double conflict = unloaded_latency(a100, [](int i) { return uint64_t(i) << 20; }); + check(open > 205 && open < 225, "A100 open-row read latency ~215 core cycles", open); + check(conflict > 245 && conflict < 265, "A100 row-conflict read latency ~255 core cycles", conflict); + + // The fixed latency is added exactly once per read (measured at a 1:1 clock + // ratio, so the DRAM clock phase cannot shift the comparison). + auto unit_clock = a100; + unit_clock.core_clock = unit_clock.dram_clock; + auto no_latency = unit_clock; + no_latency.latency = 0; + auto open_row = [](int i) { return uint64_t(i % 8) * 128; }; + double added = unloaded_latency(unit_clock, open_row) - unloaded_latency(no_latency, open_row); + check(added == 190, "fixed latency adds 190 cycles", added); + + // Every channel is an independent controller: 40 channels stream ~40x one. + auto one = a100; + one.num_channels = 1; + double bw1 = stream_bandwidth(one, 4096); + double bw40 = stream_bandwidth(a100, 4096); + check(bw40 / bw1 > 36 && bw40 / bw1 < 41, "bandwidth scales with the channel count", bw40 / bw1); + + // Saturated stream efficiency against the A100's 1,373 B/cycle peak. + check(bw40 > 0.6 * 1373 && bw40 <= 1373, "A100 stream reaches 60-100% of peak", bw40); + + // The window bounds the requests in flight. + auto narrow = a100; + narrow.num_channels = 1; + narrow.window = 8; + uint64_t peak = 0; + stream_bandwidth(narrow, 1024, &peak); + check(peak <= 8, "window of 8 bounds requests in flight", peak); + + // A block of two bursts returns one response. + auto wide = a100; + wide.num_channels = 1; + wide.block_size = 128; + { + DramSim dram(wide); + dram.reset(); + g_responses = 0; + dram.send_request(0, 0, false, on_response, nullptr); + for (int k = 0; k < 2000; ++k) { + dram.tick(); + } + check(g_responses == 1, "a 128 B block returns one response", double(g_responses)); + } + + // Plain HBM2 at the platform default: still independent channels. + auto hbm2 = a100; + hbm2.type = DramSim::Type::HBM2; + hbm2.num_channels = 2; + hbm2.latency = 0; + hbm2.window = 0; + double bw2 = stream_bandwidth(hbm2, 4096); + hbm2.num_channels = 1; + double bw1h = stream_bandwidth(hbm2, 4096); + check(bw2 / bw1h > 1.8 && bw2 / bw1h < 2.05, "HBM2 2 channels stream ~2x one", bw2 / bw1h); + + // Every channel refreshes: over many refresh intervals some open-row reads + // wait behind a refresh (nRFC = 160 cycles at a 1:1 clock). + { + auto refresh = unit_clock; + refresh.type = DramSim::Type::HBM2; + refresh.num_channels = 1; + refresh.latency = 0; + DramSim dram(refresh); + dram.reset(); + uint64_t min_lat = ~0ull, max_lat = 0; + for (int i = 0; i < 400; ++i) { + g_responses = 0; + uint64_t cycles = 0; + dram.send_request(0, uint64_t(i % 8) * 128, false, on_response, nullptr); + while (g_responses == 0) { + dram.tick(); + ++cycles; + } + if (i != 0) { + min_lat = std::min(min_lat, cycles); + max_lat = std::max(max_lat, cycles); + } + for (int k = 0; k < 97; ++k) { + dram.tick(); + } + } + check(max_lat - min_lat > 100, "refresh stalls reads by ~nRFC", double(max_lat - min_lat)); + } + + // GDDR6 at the rtx5080 profile's shape: 32 x16 channels at 15 GT/s (7,500 + // MHz data clock) behind a 2,295 MHz core, 32 B per 8-clock burst: 418 B/cycle + // peak, the card's 960 GB/s. + auto gddr6 = a100; + gddr6.type = DramSim::Type::GDDR6; + gddr6.num_channels = 32; + gddr6.core_clock = 2295; + gddr6.dram_clock = 7500; + gddr6.latency = 0; + gddr6.window = 0; + double gpeak = 32.0 * 32 / 8 * 7500 / 2295; + double gbw32 = stream_bandwidth(gddr6, 4096); + check(gbw32 > 0.6 * gpeak && gbw32 <= gpeak, "GDDR6 stream reaches 60-100% of peak", gbw32); + auto gddr6_one = gddr6; + gddr6_one.num_channels = 1; + double gbw1 = stream_bandwidth(gddr6_one, 4096); + check(gbw32 / gbw1 > 29 && gbw32 / gbw1 < 33, "GDDR6 bandwidth scales with the channel count", gbw32 / gbw1); + double gopen = unloaded_latency(gddr6_one, [](int i) { return uint64_t(i % 8) * 32; }); + double gconflict = unloaded_latency(gddr6_one, [](int i) { return uint64_t(i) << 22; }); + check(gopen < gconflict, "GDDR6 row conflicts cost more than row hits", gconflict - gopen); + + if (g_errors != 0) { + printf("FAILED: %d check(s)\n", g_errors); + return 1; + } + printf("PASSED!\n"); + return 0; +} diff --git a/third_party/Makefile b/third_party/Makefile index a5df42e081..b9e5d9f37f 100644 --- a/third_party/Makefile +++ b/third_party/Makefile @@ -7,7 +7,24 @@ softfloat: ramulator/libramulator.so: cd ramulator && mkdir -p build && cd build && cmake .. && make -j4 -ramulator: ramulator/libramulator.so + +# Vortex's Ramulator extensions: a per-channel refresh manager, and the GDDR6 +# device model Ramulator ships but does not build. Both register with +# Ramulator's factory when the library loads, built as Ramulator builds itself. +RAMULATOR_EXT_DIR := ramulator_ext +RAMULATOR_EXT_INCS := -isystem ramulator/src -isystem ramulator/ext/spdlog/include -isystem ramulator/ext/yaml-cpp/include +RAMULATOR_EXT_CXXFLAGS := -std=c++20 -O3 -DNDEBUG -fPIC $(RAMULATOR_EXT_INCS) + +$(RAMULATOR_EXT_DIR)/channel_refresh.o: $(RAMULATOR_EXT_DIR)/channel_refresh.cpp + $(CXX) $(RAMULATOR_EXT_CXXFLAGS) -Wall -Wextra -Werror -c $< -o $@ + +$(RAMULATOR_EXT_DIR)/gddr6.o: ramulator/src/dram/impl/GDDR6.cpp + $(CXX) $(RAMULATOR_EXT_CXXFLAGS) -c $< -o $@ + +$(RAMULATOR_EXT_DIR)/libramulator-vortex.so: $(RAMULATOR_EXT_DIR)/channel_refresh.o $(RAMULATOR_EXT_DIR)/gddr6.o ramulator/libramulator.so + $(CXX) -shared $(filter %.o,$^) -Lramulator -lramulator -o $@ + +ramulator: ramulator/libramulator.so $(RAMULATOR_EXT_DIR)/libramulator-vortex.so cocogfx/libcocogfx.a: $(MAKE) -C cocogfx @@ -16,6 +33,7 @@ cocogfx: cocogfx/libcocogfx.a clean: $(MAKE) -C softfloat/build/Linux-x86_64-GCC clean rm -rf ramulator/build ramulator/libramulator.so + rm -f $(RAMULATOR_EXT_DIR)/*.o $(RAMULATOR_EXT_DIR)/*.so $(MAKE) -C cocogfx clean .PHONY: all cvfpu softfloat ramulator cocogfx \ No newline at end of file diff --git a/third_party/ramulator_ext/channel_refresh.cpp b/third_party/ramulator_ext/channel_refresh.cpp new file mode 100644 index 0000000000..3888669fe9 --- /dev/null +++ b/third_party/ramulator_ext/channel_refresh.cpp @@ -0,0 +1,65 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "base/base.h" +#include "dram_controller/controller.h" +#include "dram_controller/refresh.h" + +namespace Ramulator { + +// All-bank refresh every nREFI, one REFab per channel, the channels staggered +// across the interval as independent controllers are. Ramulator's AllBank +// manager refreshes per rank, and HBM2 and GDDR6 have no rank level, so with it +// these devices never refresh. +class ChannelRefresh : public IRefreshManager, public Implementation { + RAMULATOR_REGISTER_IMPLEMENTATION(IRefreshManager, ChannelRefresh, "ChannelRefresh", "All-bank refresh of a rank-less channel.") +private: + Clk_t clk_ = 0; + IDRAMController* ctrl_ = nullptr; + int num_levels_ = -1; + int nrefi_ = -1; + int req_id_ = -1; + Clk_t next_refresh_ = -1; + +public: + void init() override { + ctrl_ = cast_parent(); + } + + void setup(IFrontEnd* /*frontend*/, IMemorySystem* /*memory_system*/) override { + auto dram = ctrl_->m_dram; + num_levels_ = int(dram->m_levels.size()); + nrefi_ = dram->m_timing_vals("nREFI"); + req_id_ = dram->m_requests("all-bank-refresh"); + int num_channels = dram->get_level_size("channel"); + next_refresh_ = nrefi_ + Clk_t(ctrl_->m_channel_id) * nrefi_ / num_channels; + } + + void tick() override { + if (++clk_ != next_refresh_) { + return; + } + next_refresh_ += nrefi_; + std::vector addr_vec(num_levels_, -1); + addr_vec[0] = ctrl_->m_channel_id; + Request req(addr_vec, req_id_); + if (!ctrl_->priority_send(req)) { + throw std::runtime_error("Failed to send refresh!"); + } + } +}; + +} From 0897cbd9af42b08e0371d13dae98adb60705033d Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:52:22 -0700 Subject: [PATCH 020/336] profiles: add a100, h100, v100, rtx1080 and tinygpu; document profiles Adds the named machines next to rtx5080 (renamed from rtx_5080), each with a source for every value and sizes rounded to the powers of two the caches accept: - tinygpu: 1 core, 2 warps x 2 threads, shared memory at its default size, default ISA (so it runs the shipped RV32IMAF/RV64IMAFD runtime); the FPGA bring-up and fast simulation machine; - rtx1080 (GTX 1080), v100, a100, h100 and rtx5080 as SimX-scale study machines, now including their DRAM: HBM2 for V100, A100 HBM timing, HBM3 bandwidth on the HBM2 model for H100, GDDR5X/GDDR7 bandwidth on the GDDR6 model for the GeForce parts. Documents profiles and the DRAM keys in docs/simulation.md, configure --profile in docs/install_vortex.md and the profile layer in the build configuration design. Verified: every profile builds and passes vecadd on simx (XLEN 32); tinygpu passes sgemm on rtlsim at XLEN 32 and 64 with the same instruction count as simx, and sgemm2 (-t2) and dotproduct, which go through shared memory, on simx and rtlsim at XLEN 32 and 64. Co-Authored-By: Claude Opus 5.5 --- docs/codebase.md | 2 ++ docs/designs/build_configuration_system.md | 20 ++++++++++++ docs/install_vortex.md | 6 ++-- docs/simulation.md | 36 ++++++++++++++++++++++ profiles/a100.json | 35 +++++++++++++++++++++ profiles/h100.json | 32 +++++++++++++++++++ profiles/rtx1080.json | 32 +++++++++++++++++++ profiles/{rtx_5080.json => rtx5080.json} | 32 ++++++++++++++----- profiles/tinygpu.json | 15 +++++++++ profiles/v100.json | 32 +++++++++++++++++++ 10 files changed, 233 insertions(+), 9 deletions(-) create mode 100644 profiles/a100.json create mode 100644 profiles/h100.json create mode 100644 profiles/rtx1080.json rename profiles/{rtx_5080.json => rtx5080.json} (62%) create mode 100644 profiles/tinygpu.json create mode 100644 profiles/v100.json diff --git a/docs/codebase.md b/docs/codebase.md index 257af03e13..52d8951279 100644 --- a/docs/codebase.md +++ b/docs/codebase.md @@ -5,6 +5,8 @@ The directory/file layout of the Vortex codebase is as follows: - `VX_config.toml` / `VX_types.toml`: hardware configuration system — the single source of truth for all build-time parameters (see the [build configuration system](designs/build_configuration_system.md)). +- `profiles`: hardware profiles — named machines (`a100`, `rtx5080`, `tinygpu`, ...) + layered over `VX_config.toml` (see [Hardware Profiles](simulation.md#hardware-profiles)) - `ci`: continuous integration scripts - `testcases`: declarative test catalog (YAML), executed by the pytest harness - `baselines/perf`: golden performance baselines for the perf-regression gate diff --git a/docs/designs/build_configuration_system.md b/docs/designs/build_configuration_system.md index c231a03bbe..ccc4a9a3bb 100644 --- a/docs/designs/build_configuration_system.md +++ b/docs/designs/build_configuration_system.md @@ -241,6 +241,26 @@ Two details are load-bearing: against a driver built with another disagrees about the hardware shape with no diagnostic; rebuild the application with the same `CONFIGS` first. +### 6.1 Hardware profiles + +A hardware profile (`profiles/.json`) is a named machine layered between +the TOML defaults and the caller's flags. `PROFILE` — set by +`configure --profile=` or `blackbox.sh --profile=` — makes +`config.mk` fold the profile into `CONFIGS` once, so every consumer in the +table above sees one `CONFIGS`: + +```make +override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=... \ + --profile=$(PROFILE) --merge-configs --cflags='$(CONFIGS)') +``` + +`--merge-configs` drops every profile entry for a knob the caller already sets +(recognising `X_DISABLE`/`X_ENABLE` and `X_`/`X=` as one knob), so +explicit flags win, then appends the caller's `CONFIGS` verbatim. The +generator validates a profile against the TOML: every key must be a public +knob, enum values must be declared, values are literals (expressions belong in +the TOML), and `sources` is required. + --- ## 7. The hardware/software boundary diff --git a/docs/install_vortex.md b/docs/install_vortex.md index d473aa34e3..d849384e75 100644 --- a/docs/install_vortex.md +++ b/docs/install_vortex.md @@ -40,8 +40,10 @@ sourcing is required and multiple Vortex trees can coexist on one machine without a global `~/.bashrc` clobber. - Useful configure options: `--xlen={32,64}` selects the ISA width and - `--prefix=` the install location. Two SimX build knobs are plain + Useful configure options: `--xlen={32,64}` selects the ISA width, + `--prefix=` the install location, and `--profile=` builds the + machine described by `profiles/.json` by default (see + [Hardware Profiles](simulation.md#hardware-profiles)). Two SimX build knobs are plain config defines, passed like any `VX_CFG_*` option through `CONFIGS`: `-DSIMX_MT=` runs the simulator on `T` host threads (default serial; cycle results are bit-identical for every thread count), and diff --git a/docs/simulation.md b/docs/simulation.md index 4b9d67fb8e..af69ae9a27 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -138,6 +138,42 @@ PERF: instrs=363180, cycles=53108, IPC=6.838518 > to compare configurations relative to one another, not as fixed targets to > reproduce. +### Hardware Profiles + +A hardware profile is a named machine in `profiles/.json`: a set of +`VX_CFG_*` values, grouped into `core`, `chip` and `platform` sections, with a +`sources` entry saying where each value comes from (optionally also a +`reference` block of measurements from the real part, and `extends` to build +on another profile). Every flow that includes `config.mk` builds the same +machine from it: SimX, rtlsim, the FPGA and ASIC flows, and the kernels. + +| Profile | Machine | +|---|---| +| `tinygpu` | 1 core, 2 warps x 2 threads, shared memory | +| `rtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 2 MB L2, GDDR5X as 16 GDDR6 channels | +| `v100` | Tesla V100 (Volta): 80 cores, 4 MB L2, 32 HBM2 channels at 877 MHz | +| `a100` | A100 (Ampere): 108 cores, 32 MB L2, GPGPU-Sim's A100 HBM timing | +| `h100` | H100 SXM5 (Hopper): 132 cores, 32 MB L2, HBM3 bandwidth on the HBM2 model | +| `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 MB L2, GDDR7 bandwidth on the GDDR6 model | + +Select one per build with `../configure --profile=`, or per run: + +```bash +$ ./ci/blackbox.sh --driver=simx --profile=rtx5080 --app=sgemm --args="-n256" +``` + +`CONFIGS` and blackbox's own `--clusters/--cores/--warps/--threads/--l2cache/--l3cache` +flags win over the profile: the profile only supplies the knobs they leave +unset, and everything else falls back to `VX_config.toml`. + +```bash +$ CONFIGS="-DVX_CFG_L2_SIZE=8388608" ./ci/blackbox.sh --driver=simx --profile=a100 --warps=32 --app=sgemm +``` + +The other profiles (20 to 132 cores) are SimX-scale machines for architecture +studies, not FPGA builds; `tinygpu` is the small one, sized for FPGA +bring-up and fast simulation. + ### Configuring the DRAM Model The simulators (simx, rtlsim, opaesim, xrtsim) all model the device memory diff --git a/profiles/a100.json b/profiles/a100.json new file mode 100644 index 0000000000..1449b1fbf1 --- /dev/null +++ b/profiles/a100.json @@ -0,0 +1,35 @@ +{ + "name": "a100", + "description": "NVIDIA A100 SXM4 (GA100, Ampere, sm_80) at its 1410 MHz boost clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 131072 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 108, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 33554432, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 1410, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 1555000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2_A100", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 1512, + "VX_CFG_PLATFORM_MEMORY_LATENCY": 190, + "VX_CFG_PLATFORM_MEMORY_WINDOW": 257 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "108 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA A100 Tensor Core GPU Architecture whitepaper, Table 3)", + "DCACHE_SIZE": "192 KB combined L1/shared per SM (whitepaper), rounded down to the power of two the cache accepts", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", + "PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "HBM2 at GPGPU-Sim SM80_A100's 1512 MHz DRAM clock; 32 channels x 32 B per DRAM cycle = 1548 GB/s, the card's 1555 GB/s", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_WINDOW": "HBM timing, fixed DRAM latency and per-channel request window of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)" + } +} diff --git a/profiles/h100.json b/profiles/h100.json new file mode 100644 index 0000000000..ce1a216401 --- /dev/null +++ b/profiles/h100.json @@ -0,0 +1,32 @@ +{ + "name": "h100", + "description": "NVIDIA H100 SXM5 (GH100, Hopper, sm_90) at its 1980 MHz boost clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 262144 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 132, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 33554432, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 1980, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 3350000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 3272 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "132 SMs (SXM5), 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", + "DCACHE_SIZE": "256 KB combined L1/shared per SM (whitepaper)", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways and 32 banks as in the a100 profile", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_CLOCK_RATE": "HBM3 modelled with the HBM2 timing: 32 channels x 32 B per DRAM cycle at 3272 MHz = 3.35 TB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)" + } +} diff --git a/profiles/rtx1080.json b/profiles/rtx1080.json new file mode 100644 index 0000000000..7728591970 --- /dev/null +++ b/profiles/rtx1080.json @@ -0,0 +1,32 @@ +{ + "name": "rtx1080", + "description": "NVIDIA GeForce GTX 1080 (GP104, Pascal, sm_61) at its 1607 MHz base clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 32768 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 20, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 2097152, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 16 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, + "VX_CFG_PLATFORM_CLOCK_RATE": 1607, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 320000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 5000 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "20 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA GeForce GTX 1080 whitepaper)", + "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at a 5000 MHz data clock: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)" + } +} diff --git a/profiles/rtx_5080.json b/profiles/rtx5080.json similarity index 62% rename from profiles/rtx_5080.json rename to profiles/rtx5080.json index 8dc5906364..6aa20dcd91 100644 --- a/profiles/rtx_5080.json +++ b/profiles/rtx5080.json @@ -1,5 +1,5 @@ { - "name": "rtx_5080", + "name": "rtx5080", "description": "NVIDIA GeForce RTX 5080 (GB203, Blackwell, sm_120) at its 2295 MHz base clock: one Vortex core per SM.", "core": { "VX_CFG_NUM_WARPS": 48, @@ -20,7 +20,10 @@ "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, "VX_CFG_PLATFORM_CLOCK_RATE": 2295, - "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000 + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 7500, + "VX_CFG_PLATFORM_MEMORY_LATENCY": 407 }, "reference": { "clock_mhz": 2295, @@ -28,18 +31,33 @@ "l2_hit_cycles": 354.4, "dram_cycles": 802, "dram_peak_bytes_per_cycle": 423, - "stream_efficiency": { "4MiB": 0.795, "16MiB": 0.894, "64MiB": 0.919 }, - "sgemm_cycles": { "128": 24193, "256": 45476, "512": 270396, "1024": 1845287 }, - "stream_cycles": { "4MiB": 25569, "16MiB": 90741, "64MiB": 427878 } + "stream_efficiency": { + "4MiB": 0.795, + "16MiB": 0.894, + "64MiB": 0.919 + }, + "sgemm_cycles": { + "128": 24193, + "256": 45476, + "512": 270396, + "1024": 1845287 + }, + "stream_cycles": { + "4MiB": 25569, + "16MiB": 90741, + "64MiB": 427878 + } }, "sources": { - "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "84 SMs, 48 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA RTX Blackwell GPU Architecture whitepaper, App. B; cudaGetDeviceProperties on the user's RTX 5080)", + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "84 SMs, 48 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA RTX Blackwell GPU Architecture whitepaper, App. B; cudaGetDeviceProperties on an RTX 5080)", "DCACHE_SIZE": "128 KB L1/shared per SM (whitepaper App. B)", "L2_SIZE, L2_NUM_WAYS": "64 MB L2 (whitepaper App. B); 16 ways as in GPGPU-Sim's Ampere/Hopper L2 configs", "L2_NUM_BANKS": "32 banks as the nearest power of two to the 48 L2 slices measured by arXiv 2507.10789", "DCACHE_MSHR_SIZE, L2_MSHR_SIZE": "384 L1 MSHR entries per SM (24 x 16 banks) and 192 per L2 slice, GPGPU-Sim SM86_RTX3070 (-gpgpu_cache:dl1 A:384:48, dl2 A:192:4)", "PLATFORM_MEMORY_NUM_BANKS": "8 x 32-bit GDDR7 devices = 32 independent channels (whitepaper App. B)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", - "reference": "Nsight Compute on the user's RTX 5080, clocks locked to base, cold L2, 2026-10-05 (docs/reports/simx_vs_rtx5080_mlp_report.md)" + "reference": "Nsight Compute on an RTX 5080 at its 2295 MHz base clock with a cold L2, 2026-10-05", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_CLOCK_RATE": "GDDR7 modelled as GDDR6 x16 channels: 32 channels x 2 B x 15 GT/s (7500 MHz data clock) = 960 GB/s, the card's bandwidth", + "PLATFORM_MEMORY_LATENCY": "407 core cycles of PHY/NoC latency, calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference' (recalibrate when the memory model changes)" } } diff --git a/profiles/tinygpu.json b/profiles/tinygpu.json new file mode 100644 index 0000000000..9c496a6237 --- /dev/null +++ b/profiles/tinygpu.json @@ -0,0 +1,15 @@ +{ + "name": "tinygpu", + "description": "The smallest Vortex: one core of 2 warps x 2 threads with shared memory.", + "core": { + "VX_CFG_NUM_WARPS": 2, + "VX_CFG_NUM_THREADS": 2 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 1 + }, + "sources": { + "all": "A minimal reference machine for small FPGAs and fast simulation (1 core, NW=NT=2, shared memory at its default size); the ISA keeps the defaults (RV32IMAF/RV64IMAFD) so it runs the shipped toolchain runtime" + } +} diff --git a/profiles/v100.json b/profiles/v100.json new file mode 100644 index 0000000000..c58be16476 --- /dev/null +++ b/profiles/v100.json @@ -0,0 +1,32 @@ +{ + "name": "v100", + "description": "NVIDIA Tesla V100 SXM2 (GV100, Volta, sm_70) at its 1530 MHz boost clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 131072 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 80, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 4194304, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 1530, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 900000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 877 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "80 SMs, 64 resident warps/SM, 4 processing blocks with one scheduler each (NVIDIA Tesla V100 GPU Architecture whitepaper, Table 1)", + "DCACHE_SIZE": "128 KB combined L1/shared per SM (whitepaper)", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "6 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways, 32 banks for the 32 memory channels", + "PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "4 HBM2 stacks x 8 channels = 32 channels at 877 MHz: 32 x 32 B x 877 MHz = 898 GB/s (whitepaper)", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)" + } +} From 21861d671c662f15b06c6669f6730cfde0167615 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:52:29 -0700 Subject: [PATCH 021/336] CHANGELOG: profiles, RV64 portability, zero warnings, DRAM model Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d7cb17ba7..3e15dd5077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to Vortex are documented here. The format is based on follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, `TOOLCHAIN_REV`, `GEM5_REV`). +## [Unreleased] + +### Added + +- **Hardware profiles.** `profiles/.json` names a machine (`tinygpu`, `rtx1080`, `v100`, `a100`, `h100`, `rtx5080`) that every flow + builds through `configure --profile=` or `blackbox.sh --profile=`; explicit `CONFIGS` win over the profile. + See [docs/simulation.md](docs/simulation.md#hardware-profiles). + +### Changed + +- **RV64 portability.** Device index math, DXA/OM addresses, pack-load strides, RTU scene/payload pointers and the ray log carry full + width on RV64 (`size_t`/`uintptr_t`, identical RV32 code); RTU slot reads into FP registers are NaN-boxed. +- **Zero warnings restored.** Every C/C++ build (simulators, runtime, kernel library, tests, OpenCL/HIP/Vulkan apps) is `-Werror` and + Verilator warnings are fatal again, with the RTL and host code fixed to build clean. +- **DRAM model redesign.** simx, rtlsim, opaesim and xrtsim model one independent Ramulator channel per memory bank (one burst per + transaction, per-channel responses, refresh); `VX_CFG_PLATFORM_MEMORY_{TYPE,CLOCK_RATE,LATENCY,WINDOW}` select HBM2, HBM2_A100 or GDDR6. + ## [3.0] — 2026-10-07 The 3.0 release introduces a fixed-function graphics stack (rasterizer, texture units, and output mergers), tensor core structured sparsity (2:4), warpgroup-level matrix multiplication (WGMMA), global-to-local data transfer acceleration (DXA), a new hardware kernel scheduler (KMU) and Command Processor (CP) architecture, a new asynchronous runtime API (`vortex2.h`), asynchronous barriers with arrive/wait/event semantics, compressed instruction set (RVC) support, hardware atomics, an MMU/SV32 virtual memory stack, a Mesa/lavapipe Vulkan backend (`vortexpipe`), HIP via chipStar, gem5 integration, a SimX v3 TLM architecture with fixed-size handshake channels, productized Synopsys and Yosys ASIC synthesis flows, and a refreshed toolchain (LLVM 20, POCL 7.0). Build and configuration infrastructure was reworked: TOML-driven HW configuration ([VX_config.toml](VX_config.toml) + [VX_types.toml](VX_types.toml)) decoupling SimX/runtime from the RTL source tree, a `VX_CFG_` macro namespace that resolves toolchain preprocessor collisions, retirement of the global `toolchain_env.sh` to enable parallel multi-version Vortex worktrees on the same shell, consolidation of `kernel/`/`runtime/` under a shared `sw/` root, a single-source [VERSION](VERSION) file driving CI toolchain pinning, Perfetto trace export ([ci/perfetto.py](ci/perfetto.py)), and new top-level [AGENTS.md](AGENTS.md) + [CONTRIBUTING.md](CONTRIBUTING.md) for AI-agent and contributor workflows. From 49613f892751d9ec388f4be38718d88e1b273811 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 11 Oct 2026 01:36:30 -0700 Subject: [PATCH 022/336] ci: default GEM5_HOME to the configured tool directory regression.sh fell back to $HOME/tools/gem5 when GEM5_HOME was unset, but ci/gem5_install.sh installs gem5 under $TOOLDIR/gem5, so the gem5 cell failed in any build configured with another --tooldir ("gem5.opt not found"). CI sets GEM5_HOME explicitly and is unaffected. The fallback now follows TOOLDIR, defaulting to the configure-time value. Verified: the gem5 cell passes at XLEN 64 in a build configured with a non-default --tooldir. Co-Authored-By: Claude Opus 5.5 --- ci/regression.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/regression.sh.in b/ci/regression.sh.in index 70165bfa08..774434c5d3 100755 --- a/ci/regression.sh.in +++ b/ci/regression.sh.in @@ -118,7 +118,7 @@ gem5() echo "begin gem5 tests..." if [ -z "$GEM5_HOME" ]; then - GEM5_HOME=$HOME/tools/gem5 + GEM5_HOME=${TOOLDIR:-@TOOLDIR@}/gem5 fi if [ ! -x "$GEM5_HOME/build/X86/gem5.opt" ]; then echo "error: $GEM5_HOME/build/X86/gem5.opt not found — run ci/gem5_install.sh first" From 1c6ef7efa7314dbb78a33ad906c46aa24b1ee7d8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 11 Oct 2026 02:22:39 -0700 Subject: [PATCH 023/336] ci: re-record the rtlsim perf-gate baselines Regenerated with `pytest ci -m perf_gate --update-baselines` (XLEN 32, rtlsim) on this series, with the toolchain the CI will download (llvm-vortex a81a5957f406). All 49 entries move, because the per-channel DRAM model changes every rtlsim cycle count: - dxa/wgmma cases: 13-20% fewer cycles (independent channels, no shared injection queue); - most core, graphics and tensor cases: within a few percent; - vecadd: +21% instructions and +24%/+32% cycles (nt4/nt16) from the global index guard the kernel now needs on a rounded-up grid (one argument load and a branch in a kernel of ~two dozen instructions); - nine entries change instruction counts, from the toolchain and the test kernel changes in this series. Verified: the 49 perf_gate cases pass on rtlsim while recording. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 72 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 44 +++++++++---------- ci/baselines/perf/graphics.json | 46 ++++++++++---------- ci/baselines/perf/raytracing.json | 12 +++--- ci/baselines/perf/tensor.json | 20 ++++----- ci/baselines/perf/tensor_mx.json | 4 +- ci/baselines/perf/tensor_sp.json | 4 +- ci/baselines/perf/tensor_wg.json | 12 +++--- 8 files changed, 107 insertions(+), 107 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 5e226eba16..b08f98ad35 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 568490, + "cycles": 562137, "instrs": 137486 }, "app": "raycast", "args": "", - "config_hash": "ee3a227211fa9942", + "config_hash": "314d6d30965b89c4", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 1495440, + "cycles": 1495556, "instrs": 259228 }, "app": "raycast", "args": "", - "config_hash": "c8b680eabeb18655", + "config_hash": "909d80cfbaadadc2", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 781696, + "cycles": 779835, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "58eb3cc06964207a", + "config_hash": "92050636b69caeb2", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2211340, + "cycles": 2228982, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "aa6f26c7306417a8", + "config_hash": "72b6506beadbe8a0", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1823521, + "cycles": 1909825, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "b390d957eec8220f", + "config_hash": "514f489ab80d25c0", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 6055079, + "cycles": 6189932, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "13d03df76d55105e", + "config_hash": "4af79268b2c7557f", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1297311, + "cycles": 1261549, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "68638b6685801699", + "config_hash": "d53587fae488c32b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4210203, + "cycles": 4181334, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "837240be3049d62e", + "config_hash": "572494ba2df6f58a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 633593, - "instrs": 79120 + "cycles": 583960, + "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "c6a3720118661bf7", + "config_hash": "587ac34faae9f341", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 1060219, - "instrs": 316432 + "cycles": 931381, + "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "db6bb5af90ecc1a5", + "config_hash": "93f8522bd055dda2", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1274656, + "cycles": 1273285, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "1c0e7742deb14093", + "config_hash": "9c8c364608242127", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4860631, + "cycles": 4861515, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "3797eecae3200315", + "config_hash": "b2c24fbda6abd30b", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2388738, + "cycles": 2395091, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "aa768c2a9c65b092", + "config_hash": "fec6d70d6e8e83c3", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4177542, + "cycles": 4283200, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "5400d882a6c96ac1", + "config_hash": "bb64a7573d2c2365", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 75714, - "instrs": 24592 + "cycles": 99638, + "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "98b8bdc5e546ea62", + "config_hash": "58a51468cf8f99e7", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 293078, - "instrs": 98320 + "cycles": 363323, + "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "1510b8438164b252", + "config_hash": "4ca729a11fbf88a3", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index d864f76b96..453b0775f5 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -1,122 +1,122 @@ { "dxa:perf_gate-copy-mcast:rtlsim": { "32": { - "cycles": 3411, + "cycles": 3313, "instrs": 961 }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "cbcd10be884ceab7", + "config_hash": "2df97b749a0febe3", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 2096, + "cycles": 1869, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "3a1ec1b81b5b3baf", + "config_hash": "59a6c715ed820634", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt4:rtlsim": { "32": { - "cycles": 6844, + "cycles": 6773, "instrs": 1792 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "3a888067fcc449d6", + "config_hash": "534d84e7f493e414", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14333410, + "cycles": 14056427, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "14050e4dcef65266", + "config_hash": "589585790379eee0", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14345910, + "cycles": 14108810, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "a55b900d67aa3657", + "config_hash": "2879ae8c33e35bb8", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3978026, + "cycles": 3895374, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "36f5886fdc97e040", + "config_hash": "f3759c20ec885c91", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14632454, + "cycles": 14130928, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "9ccf890d5ac45a62", + "config_hash": "aa3ad14268d9b8b5", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 98435, + "cycles": 79024, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "ee7678a7e7e9b79b", + "config_hash": "7080653c22396eb6", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 400679, + "cycles": 348356, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "c79015e4caec7b93", + "config_hash": "401d267a03887e99", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 39761, + "cycles": 34417, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "5a35710ef3aec3a6", + "config_hash": "3117dd957603b2e8", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 68208, + "cycles": 55358, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "d12fad3fd43e61c6", + "config_hash": "5bec3cc54a0c63f1", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 32c723239d..f4f9eb603d 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,100 +1,100 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 531740, - "instrs": 204102 + "cycles": 529292, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "c8c2af44277e2cb0", + "config_hash": "ae1062062a26ef92", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 145662, - "instrs": 52516 + "cycles": 142905, + "instrs": 51650 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "ee0410ff09e7ea58", + "config_hash": "e397892c0cb72ec2", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 531359, - "instrs": 204102 + "cycles": 528252, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "00c7939a6d958e0e", + "config_hash": "15664f7fbe13f120", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 191808, - "instrs": 69648 + "cycles": 188898, + "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "60103e7b6eb2c625", + "config_hash": "b2ccda0f4e3d0471", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 754454, - "instrs": 278544 + "cycles": 740590, + "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "ee88907a281d364a", + "config_hash": "e542501f5bf6269e", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14959, + "cycles": 14865, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "3ba867af954e5cbf", + "config_hash": "7272f5969f7ee30d", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55633, + "cycles": 55526, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "859b3c4bd2208b92", + "config_hash": "05e09632b1bbf2b7", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 34994, + "cycles": 35342, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "1c38a250b6b85273", + "config_hash": "9669b4d06963f72d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 136738, + "cycles": 138329, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "226df03058dfb436", + "config_hash": "aee6b6136e6a4575", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 86b562e3b1..bda30910c0 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,34 +1,34 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 84873, + "cycles": 84378, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "c71b720ea7aa7e72", + "config_hash": "937f99317c42a59f", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53310, + "cycles": 53283, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "3049a9d721044cde", + "config_hash": "dfd33a99196e04ad", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt4:rtlsim": { "32": { - "cycles": 159542, + "cycles": 159558, "instrs": 39330 }, "app": "rt_raycast", "args": "", - "config_hash": "14ae0ed831d1cd4a", + "config_hash": "c17be24af5bd78b9", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 3249cc1b16..fae07189e9 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 195794, + "cycles": 191642, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "6a095ed165469a30", + "config_hash": "d21f5479c2459776", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 44583, + "cycles": 42941, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "0fe2a8e2a75fcc10", + "config_hash": "23f4b1de5a361780", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221331, + "cycles": 221484, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "b2ef81f2fa1e6a43", + "config_hash": "c4461f9b2a2b71d7", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 251208, + "cycles": 253809, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "5534ddd0051a5c15", + "config_hash": "9c6dfe0c35a740a9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 592658, + "cycles": 594027, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "421a4ab52f0f072a", + "config_hash": "9c8bbf01434d9d8d", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index c9b137227e..1fbd356b34 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 77127, + "cycles": 76601, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "719fe3a7ce81115d", + "config_hash": "46d80398194a01e4", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index e8e15a7ab2..47e3a2926a 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 187290, + "cycles": 184985, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "acf4fa4675243f0f", + "config_hash": "cfe93fc4d3413031", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 6ea2c1d004..57e02b2b8f 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 157272, + "cycles": 158687, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "7e8b9759dc36dba0", + "config_hash": "f019a0cb823990c9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 841657, + "cycles": 836986, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "c5a71c297fc4dafe", + "config_hash": "7b6f63cb95491231", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 401322, + "cycles": 398222, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "ab1451598dfbc872", + "config_hash": "f620d8a9b0396d1e", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From 2970e9da33bfa90ba61a51f47c0b9be2f41c1df2 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 11 Oct 2026 02:40:46 -0700 Subject: [PATCH 024/336] tests: compute dogfood's fused multiply-add references fused The kernels write a*b +/- b, which clang contracts into fmadd/fmsub/fnmadd/ fnmsub (single rounding), but the host references computed the same expressions unfused (a*b rounded first). Near a = 1, a*b - b cancels, and the reference's rounding of a*b becomes hundreds of ulp of the small result, so fmsub failed whenever the configuration's element count put a sample near that point (4 cores x 8 warps x 8 threads with an L2: up to 512 ulp). The references now use fmaf, matching the instructions the kernels exercise; the kernels are unchanged. Verified: dogfood passes 22/22 on simx at that configuration (failing before) and at the default, and on rtlsim at that configuration; the kernel still emits fmadd.s x2, fmsub.s, fnmadd.s x2 and fnmsub.s. Co-Authored-By: Claude Opus 5.5 --- tests/regression/dogfood/testcases.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/regression/dogfood/testcases.h b/tests/regression/dogfood/testcases.h index f3562bb179..29e8e819c1 100644 --- a/tests/regression/dogfood/testcases.h +++ b/tests/regression/dogfood/testcases.h @@ -351,7 +351,7 @@ class Test_FMADD : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = a[i] * b[i] + b[i]; + auto ref = fmaf(a[i], b[i], b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -381,7 +381,7 @@ class Test_FMSUB : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = a[i] * b[i] - b[i]; + auto ref = fmaf(a[i], b[i], -b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -411,7 +411,7 @@ class Test_FNMADD : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = -a[i] * b[i] - b[i]; + auto ref = fmaf(-a[i], b[i], -b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -441,7 +441,7 @@ class Test_FNMSUB : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = -a[i] * b[i] + b[i]; + auto ref = fmaf(-a[i], b[i], b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -471,8 +471,8 @@ class Test_FNMADD_MADD : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto x = -a[i] * b[i] - b[i]; - auto y = a[i] * b[i] + b[i]; + auto x = fmaf(-a[i], b[i], -b[i]); + auto y = fmaf(a[i], b[i], b[i]); auto ref = x + y; if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; From 0a46eef3b56aaf77a273ac922bd759f92994ed94 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 11 Oct 2026 10:46:39 -0700 Subject: [PATCH 025/336] vm: share one VPN superpage mask function from the TLB package VX_tlb_cam and VX_mmu each defined an identical vpn_mask function. In the multi-core VM configurations (2-4 cores with an L2/L3) Verilator reported VARHIDDEN for it, which the zero-warnings policy makes fatal, so the vm:tlb-concurrent, vm:mc-l2-amo and vm:mc-l3 rtlsim builds failed. The function now lives once in VX_tlb_pkg as tlb_vpn_mask, next to tlb_perm_ok, and both modules call it. Verified: the vm cell on rtlsim passes 23/23 at XLEN 32 and 64. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/vm/VX_mmu.sv | 5 +---- hw/rtl/vm/VX_tlb_cam.sv | 9 ++------- hw/rtl/vm/VX_tlb_pkg.sv | 5 +++++ 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index 06b4ea7f48..9166421400 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -67,9 +67,6 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( localparam FIELDS_W = F_RW_LO + 1; localparam PAYLOAD_W = LANE_W + FIELDS_W; - function automatic [TLB_VPN_WIDTH-1:0] vpn_mask (input [TLB_LEVEL_WIDTH-1:0] level); - vpn_mask = {TLB_VPN_WIDTH{1'b1}} << (level * TLB_LEVEL_BITS); - endfunction function automatic tlb_access_e access_of (input logic rw); access_of = (EXEC_SIDE != 0) ? TLB_ACC_EX : (rw ? TLB_ACC_WR : TLB_ACC_RD); @@ -255,7 +252,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( if (ADDR_WIDTH > PAGE_OFFSET_W + TLB_VPN_WIDTH) begin : g_replay_va_hi `UNUSED_VAR (replay_raddr[ADDR_WIDTH-1 : PAGE_OFFSET_W + TLB_VPN_WIDTH]) end - wire [TLB_VPN_WIDTH-1:0] replay_low = replay_vpn & ~vpn_mask(replay_level); + wire [TLB_VPN_WIDTH-1:0] replay_low = replay_vpn & ~tlb_vpn_mask(replay_level); wire [TLB_PPN_WIDTH-1:0] replay_ppn_sp = replay_ppn | TLB_PPN_WIDTH'(replay_low); wire [ADDR_WIDTH-1:0] replay_taddr = {replay_ppn_sp, replay_raddr[PAGE_OFFSET_W-1:0]}; diff --git a/hw/rtl/vm/VX_tlb_cam.sv b/hw/rtl/vm/VX_tlb_cam.sv index ec63559c33..950d974626 100644 --- a/hw/rtl/vm/VX_tlb_cam.sv +++ b/hw/rtl/vm/VX_tlb_cam.sv @@ -56,11 +56,6 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( reg [TLB_SIZE-1:0] valid_r; reg [TLB_SIZE-1:0] mru_r; - // Mask that drops the low level*TLB_LEVEL_BITS VPN bits (superpages). - function automatic [TLB_VPN_WIDTH-1:0] vpn_mask (input [TLB_LEVEL_WIDTH-1:0] level); - vpn_mask = {TLB_VPN_WIDTH{1'b1}} << (level * TLB_LEVEL_BITS); - endfunction - // --------------------------------------------------------------------- // Per-lane lookup // --------------------------------------------------------------------- @@ -77,7 +72,7 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lookup wire [TLB_SIZE-1:0] hit_vec; for (genvar i = 0; i < TLB_SIZE; ++i) begin : g_match - wire [TLB_VPN_WIDTH-1:0] mask = vpn_mask(entries_r[i].level); + wire [TLB_VPN_WIDTH-1:0] mask = tlb_vpn_mask(entries_r[i].level); assign hit_vec[i] = valid_r[i] && ((entries_r[i].vpn & mask) == (lookup_vpn[l] & mask)); end @@ -115,7 +110,7 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( wire [TLB_FLAGS_WIDTH-1:0] sel_flags = sel[TLB_LEVEL_WIDTH +: TLB_FLAGS_WIDTH]; wire [TLB_LEVEL_WIDTH-1:0] sel_level = sel[0 +: TLB_LEVEL_WIDTH]; - wire [TLB_VPN_WIDTH-1:0] low = lookup_vpn[l] & ~vpn_mask(sel_level); + wire [TLB_VPN_WIDTH-1:0] low = lookup_vpn[l] & ~tlb_vpn_mask(sel_level); // Splice the superpage's intra-page index from the VPN into the PPN. assign lookup_ppn[l] = sel_ppn | TLB_PPN_WIDTH'(low); assign lookup_flags[l] = sel_flags; diff --git a/hw/rtl/vm/VX_tlb_pkg.sv b/hw/rtl/vm/VX_tlb_pkg.sv index 8392a7ffbe..bc9aa17da4 100644 --- a/hw/rtl/vm/VX_tlb_pkg.sv +++ b/hw/rtl/vm/VX_tlb_pkg.sv @@ -72,6 +72,11 @@ package VX_tlb_pkg; && ( want_w || want_x || flags[TLB_FLAG_R]); endfunction + // Mask that drops the low level*TLB_LEVEL_BITS VPN bits (superpages). + function automatic logic [TLB_VPN_WIDTH-1:0] tlb_vpn_mask (input logic [TLB_LEVEL_WIDTH-1:0] level); + tlb_vpn_mask = {TLB_VPN_WIDTH{1'b1}} << (level * TLB_LEVEL_BITS); + endfunction + endpackage `endif // VX_TLB_PKG_VH From f80af4c365a3d3c13398c62d60fccf4a6fed0c5e Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 12 Oct 2026 02:24:42 -0700 Subject: [PATCH 026/336] sim: configure the DRAM model from data-sheet values The DRAM keys added with the model redesign mixed device settings with wrapper internals: a DRAM clock, a request window, an A100-specific type, per-type timing tables in DRAM cycles, a forced 256-bit channel and custom address maps. Studying another GPU meant re-deriving all of them. A channel is now described the way a data sheet describes it: VX_CFG_PLATFORM_MEMORY_TYPE (HBM2, HBM3, GDDR6), _DATA_RATE (MT/s per pin, default twice the core clock), _NUM_BANKS (channels), _LATENCY (fixed PHY/controller/interconnect cycles) and _QUEUE_SIZE (requests in flight). - One Ramulator memory system per channel: independent controllers, any channel count. - JEDEC organization with 32 B bursts (HBM: BL4 on 64-bit pseudo-channels; GDDR6: x16), array timings in nanoseconds converted by Ramulator at the data rate, bus timings in clocks. HBM array timing is JEDEC HBM2 as GPGPU-Sim's QV100 configuration lists it (tRCD/tCL/tRP 14, tRAS 33, tRC 47 ns); GDDR6 is Ramulator's own x16 preset (the GPGPU-Sim RTX 3070 values used before were half of real GDDR6). - Address map: four bursts per row visit, then the next pseudo-channel, bank group and bank, with the bank permuted by the folded row, so streams a power of two apart do not share banks; Ramulator's MOP4CLXOR mapper XORs with column bits instead and sent such streams to the same banks. - No refresh: GPU controllers hide refresh with per-bank refresh, for which Ramulator has no manager (its HBM REFsb also lacks the tRFCsb busy time); all-bank refresh cost every channel ~9% of its time and added 8.5 cycles to sgemm's mean memory latency. ChannelRefresh is removed; ramulator_ext only builds GDDR6.cpp. The default machine clocks the DRAM with the core: 14/26 cycles idle (open row / row conflict; master 13/27), one 64 B block per cycle at peak. SimX sgemm -n128 mean memory latency at NT=NW=4/8/16: 17.9/18.4/18.5 cycles (master 16.4/17.6/19.0). Profiles: v100 HBM2 1752 MT/s, a100 HBM2 3038 and h100 HBM3 6545 (32 channels carrying the cards' 40-channel bandwidth), rtx1080 GDDR6 10000, rtx5080 GDDR6 15000 with LATENCY 339 (407 less the 68 cycles JEDEC GDDR6 adds to a row conflict; derived, not re-measured). Streams reach 93-98% of peak. The dram_sim unit test checks idle latency, channel scaling, peak efficiency, strided streams and the queue bound. Co-Authored-By: Claude Opus 5.5 --- .github/actions/setup-vortex/action.yml | 2 +- CHANGELOG.md | 6 +- VX_config.toml | 15 +- docs/codebase.md | 4 +- docs/simulation.md | 38 +- profiles/a100.json | 10 +- profiles/h100.json | 6 +- profiles/rtx1080.json | 4 +- profiles/rtx5080.json | 8 +- profiles/v100.json | 4 +- sim/common/dram_sim.cpp | 448 +++++++++--------- sim/common/dram_sim.h | 14 +- tests/unittest/dram_sim/main.cpp | 154 +++--- third_party/Makefile | 9 +- third_party/ramulator_ext/channel_refresh.cpp | 65 --- 15 files changed, 367 insertions(+), 420 deletions(-) delete mode 100644 third_party/ramulator_ext/channel_refresh.cpp diff --git a/.github/actions/setup-vortex/action.yml b/.github/actions/setup-vortex/action.yml index a73448a887..b27d3c1305 100644 --- a/.github/actions/setup-vortex/action.yml +++ b/.github/actions/setup-vortex/action.yml @@ -49,7 +49,7 @@ runs: uses: actions/cache@v4 with: path: third_party - key: ${{ runner.os }}-thirdparty-${{ hashFiles('third_party/Makefile', 'third_party/ramulator_ext/**', '.gitmodules') }} + key: ${{ runner.os }}-thirdparty-${{ hashFiles('third_party/Makefile', '.gitmodules') }} # Publish WHERE the toolchain cache above was restored to. The `tools` cache # path is relative to the workspace, but nothing downstream could know that: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e15dd5077..b384f6a82b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,10 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, width on RV64 (`size_t`/`uintptr_t`, identical RV32 code); RTU slot reads into FP registers are NaN-boxed. - **Zero warnings restored.** Every C/C++ build (simulators, runtime, kernel library, tests, OpenCL/HIP/Vulkan apps) is `-Werror` and Verilator warnings are fatal again, with the RTL and host code fixed to build clean. -- **DRAM model redesign.** simx, rtlsim, opaesim and xrtsim model one independent Ramulator channel per memory bank (one burst per - transaction, per-channel responses, refresh); `VX_CFG_PLATFORM_MEMORY_{TYPE,CLOCK_RATE,LATENCY,WINDOW}` select HBM2, HBM2_A100 or GDDR6. +- **DRAM model redesign.** simx, rtlsim, opaesim and xrtsim model one independent Ramulator controller and device per memory bank + (JEDEC HBM2/HBM3/GDDR6 organization and nanosecond timing, 32 B bursts, permuted open-page bank interleave, per-channel + responses), configured from data-sheet values: `VX_CFG_PLATFORM_MEMORY_{TYPE,DATA_RATE,LATENCY,QUEUE_SIZE}`. The default machine + keeps its idle read latency (14/26 cycles open-row/conflict, was 13/27) and one-block-per-cycle peak. ## [3.0] — 2026-10-07 diff --git a/VX_config.toml b/VX_config.toml index 8d331ff67b..f51186cdec 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -80,13 +80,16 @@ VX_CFG_PLATFORM_MEMORY_INTERLEAVE = 1 VX_CFG_PLATFORM_MEMORY_PEAK_BW = 460000 # 460 GB/s VX_CFG_PLATFORM_CLOCK_RATE = 400 # DRAM the simulators model behind each memory bank (one channel per bank), -# its clock (MHz), the fixed read latency added for the PHY, controller and -# interconnect (core cycles), and the requests a channel holds in flight -# (0 = unbounded). +# as a data sheet gives it: the JEDEC standard, its data rate (MT/s per pin), +# the fixed read latency added for the PHY, controller and interconnect (core +# cycles), and the requests a channel holds in flight (0 = unbounded). +# Simulation-only: the RTL never reads them. The default clocks the DRAM with +# the core: 14 cycles for an idle open-row read, 26 for a row conflict, one +# 64 B block per cycle at peak; profiles set their own. VX_CFG_PLATFORM_MEMORY_TYPE = "HBM2" -VX_CFG_PLATFORM_MEMORY_CLOCK_RATE = 1000 +VX_CFG_PLATFORM_MEMORY_DATA_RATE = "expr: 2 * $VX_CFG_PLATFORM_CLOCK_RATE" VX_CFG_PLATFORM_MEMORY_LATENCY = 0 -VX_CFG_PLATFORM_MEMORY_WINDOW = 0 +VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE = 0 [alu] VX_CFG_NUM_ALU_LANES = "expr: $VX_CFG_SIMD_WIDTH" @@ -464,7 +467,7 @@ VX_CFG_XLEN = [32, 64] VX_CFG_FLEN = [32, 64] VX_CFG_FPU_TYPE = ["DPI", "DSP", "FPNEW", "STD"] VX_CFG_TCU_TYPE = ["DPI", "DSP", "BHF", "TFR", "FPNEW"] -VX_CFG_PLATFORM_MEMORY_TYPE = ["HBM2", "HBM2_A100", "GDDR6"] +VX_CFG_PLATFORM_MEMORY_TYPE = ["HBM2", "HBM3", "GDDR6"] [[param]] VX_CFG_DCACHE_NUM_REQS = "int" diff --git a/docs/codebase.md b/docs/codebase.md index 52d8951279..9e68135e0f 100644 --- a/docs/codebase.md +++ b/docs/codebase.md @@ -76,7 +76,7 @@ The directory/file layout of the Vortex codebase is as follows: - `runtime`: driver API tests - `mpi`: multi-process tests - `third_party`: external library submodules (cvfpu, softfloat, hardfloat, - ramulator, cocogfx), plus `ramulator_ext` (Vortex's Ramulator extensions: - per-channel refresh and the GDDR6 device) + ramulator, cocogfx); `ramulator_ext` holds the build of the GDDR6 device + model Ramulator ships but does not build - `perf`: performance analysis resources - `miscs`: miscellaneous resources diff --git a/docs/simulation.md b/docs/simulation.md index af69ae9a27..8fbc55f94d 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -151,9 +151,9 @@ machine from it: SimX, rtlsim, the FPGA and ASIC flows, and the kernels. |---|---| | `tinygpu` | 1 core, 2 warps x 2 threads, shared memory | | `rtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 2 MB L2, GDDR5X as 16 GDDR6 channels | -| `v100` | Tesla V100 (Volta): 80 cores, 4 MB L2, 32 HBM2 channels at 877 MHz | -| `a100` | A100 (Ampere): 108 cores, 32 MB L2, GPGPU-Sim's A100 HBM timing | -| `h100` | H100 SXM5 (Hopper): 132 cores, 32 MB L2, HBM3 bandwidth on the HBM2 model | +| `v100` | Tesla V100 (Volta): 80 cores, 4 MB L2, 32 HBM2 channels at 1752 MT/s | +| `a100` | A100 (Ampere): 108 cores, 32 MB L2, HBM2 at the card's 1555 GB/s | +| `h100` | H100 SXM5 (Hopper): 132 cores, 32 MB L2, HBM3 at the card's 3.35 TB/s | | `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 MB L2, GDDR7 bandwidth on the GDDR6 model | Select one per build with `../configure --profile=`, or per run: @@ -177,21 +177,35 @@ bring-up and fast simulation. ### Configuring the DRAM Model The simulators (simx, rtlsim, opaesim, xrtsim) all model the device memory -the same way: one independent DRAM channel per platform memory bank, timed by -[Ramulator](https://github.com/CMU-SAFARI/ramulator2), with its refresh. These -`VX_config.toml` keys select it, overridable through `CONFIGS` or a profile: +the same way: one independent DRAM channel (controller and device) per +platform memory bank, timed by +[Ramulator](https://github.com/CMU-SAFARI/ramulator2) with its JEDEC device +models. Each channel maps addresses the way GPU memory controllers do: four +bursts per row visit, then the next bank, with the bank permuted by the row so +strided streams spread over the banks. Refresh is not modelled: GPU +controllers hide it with per-bank refresh, for which Ramulator has no manager. A channel is described the way a data sheet describes it; these +`VX_config.toml` keys select it, overridable through `CONFIGS` or a profile, +and the RTL never reads them: | Key | Default | Meaning | |---|---|---| -| `VX_CFG_PLATFORM_MEMORY_TYPE` | `HBM2` | `HBM2` (JEDEC 2 Gbps), `HBM2_A100` (GPGPU-Sim's A100 timing) or `GDDR6` (x16 channels, 32 B bursts) | -| `VX_CFG_PLATFORM_MEMORY_NUM_BANKS` | 2 | memory banks, and so DRAM channels | -| `VX_CFG_PLATFORM_MEMORY_CLOCK_RATE` | 1000 | DRAM clock in MHz (GDDR6: the data clock, half the transfer rate) | -| `VX_CFG_PLATFORM_CLOCK_RATE` | 400 | core clock in MHz; with the DRAM clock it sets how many DRAM cycles a core cycle spans | +| `VX_CFG_PLATFORM_MEMORY_TYPE` | `HBM2` | JEDEC standard: `HBM2` (also HBM2E), `HBM3` or `GDDR6` (x16 channels) | +| `VX_CFG_PLATFORM_MEMORY_NUM_BANKS` | 2 | memory banks, and so DRAM channels (HBM: 128-bit channels) | +| `VX_CFG_PLATFORM_MEMORY_DATA_RATE` | 2 x `PLATFORM_CLOCK_RATE` | transfer rate per pin in MT/s; the default clocks the DRAM with the core | +| `VX_CFG_PLATFORM_CLOCK_RATE` | 400 | core clock in MHz; with the data rate it sets how many DRAM cycles a core cycle spans | | `VX_CFG_PLATFORM_MEMORY_LATENCY` | 0 | fixed read latency added for the PHY, controller and interconnect, in core cycles | -| `VX_CFG_PLATFORM_MEMORY_WINDOW` | 0 | requests a channel holds in flight (0 = unbounded) | +| `VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE` | 0 | requests a channel holds in flight (0 = unbounded) | + +The model's peak bandwidth follows from the keys: channels x channel width x +data rate (HBM 16 B, GDDR6 2 B per transfer); `VX_CFG_PLATFORM_MEMORY_PEAK_BW` +separately states the platform's, which the runtime reports. The device's +array timings are held in nanoseconds, so a different data rate changes the +bandwidth but not the latency. The default machine clocks the DRAM with the +core: an idle read takes 14 cycles on an open row and 26 on a row conflict, +and the memory delivers up to one 64 B block per cycle. ```bash -$ CONFIGS="-DVX_CFG_PLATFORM_MEMORY_TYPE=GDDR6 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_CLOCK_RATE=8000" \ +$ CONFIGS="-DVX_CFG_PLATFORM_MEMORY_TYPE=GDDR6 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_DATA_RATE=14000" \ ./ci/blackbox.sh --driver=simx --app=vecadd ``` diff --git a/profiles/a100.json b/profiles/a100.json index 1449b1fbf1..7a6003fe0b 100644 --- a/profiles/a100.json +++ b/profiles/a100.json @@ -19,17 +19,17 @@ "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, "VX_CFG_PLATFORM_CLOCK_RATE": 1410, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 1555000, - "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2_A100", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 1512, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 3038, "VX_CFG_PLATFORM_MEMORY_LATENCY": 190, - "VX_CFG_PLATFORM_MEMORY_WINDOW": 257 + "VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE": 257 }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "108 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA A100 Tensor Core GPU Architecture whitepaper, Table 3)", "DCACHE_SIZE": "192 KB combined L1/shared per SM (whitepaper), rounded down to the power of two the cache accepts", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", - "PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "HBM2 at GPGPU-Sim SM80_A100's 1512 MHz DRAM clock; 32 channels x 32 B per DRAM cycle = 1548 GB/s, the card's 1555 GB/s", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_WINDOW": "HBM timing, fixed DRAM latency and per-channel request window of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM2 stacks x 8 128-bit channels = 40 channels at 2430 MT/s, modelled as 32 channels at 3038 MT/s for a power-of-two interleave: 32 x 16 B x 3038 MT/s = 1555 GB/s, the card's bandwidth; the array timing is in nanoseconds, so latency is unchanged", + "PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_QUEUE_SIZE": "Fixed DRAM latency and per-channel request queue of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)" } } diff --git a/profiles/h100.json b/profiles/h100.json index ce1a216401..d95d2fabda 100644 --- a/profiles/h100.json +++ b/profiles/h100.json @@ -19,14 +19,14 @@ "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, "VX_CFG_PLATFORM_CLOCK_RATE": 1980, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 3350000, - "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 3272 + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM3", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 6545 }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "132 SMs (SXM5), 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", "DCACHE_SIZE": "256 KB combined L1/shared per SM (whitepaper)", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways and 32 banks as in the a100 profile", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_CLOCK_RATE": "HBM3 modelled with the HBM2 timing: 32 channels x 32 B per DRAM cycle at 3272 MHz = 3.35 TB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM3 stacks x 16 64-bit channels at 5236 MT/s = 40 128-bit channel pairs, modelled as 32 at 6545 MT/s for a power-of-two interleave: 32 x 16 B x 6545 MT/s = 3.35 TB/s, the card's bandwidth; latency not calibrated", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)" } } diff --git a/profiles/rtx1080.json b/profiles/rtx1080.json index 7728591970..73908bb3d2 100644 --- a/profiles/rtx1080.json +++ b/profiles/rtx1080.json @@ -20,13 +20,13 @@ "VX_CFG_PLATFORM_CLOCK_RATE": 1607, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 320000, "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 5000 + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 10000 }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "20 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA GeForce GTX 1080 whitepaper)", "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at a 5000 MHz data clock: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at 10000 MT/s: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)" } } diff --git a/profiles/rtx5080.json b/profiles/rtx5080.json index 6aa20dcd91..d90b047d99 100644 --- a/profiles/rtx5080.json +++ b/profiles/rtx5080.json @@ -22,8 +22,8 @@ "VX_CFG_PLATFORM_CLOCK_RATE": 2295, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000, "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 7500, - "VX_CFG_PLATFORM_MEMORY_LATENCY": 407 + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 15000, + "VX_CFG_PLATFORM_MEMORY_LATENCY": 339 }, "reference": { "clock_mhz": 2295, @@ -57,7 +57,7 @@ "PLATFORM_MEMORY_NUM_BANKS": "8 x 32-bit GDDR7 devices = 32 independent channels (whitepaper App. B)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", "reference": "Nsight Compute on an RTX 5080 at its 2295 MHz base clock with a cold L2, 2026-10-05", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_CLOCK_RATE": "GDDR7 modelled as GDDR6 x16 channels: 32 channels x 2 B x 15 GT/s (7500 MHz data clock) = 960 GB/s, the card's bandwidth", - "PLATFORM_MEMORY_LATENCY": "407 core cycles of PHY/NoC latency, calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference' (recalibrate when the memory model changes)" + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_DATA_RATE": "30 Gbps GDDR7 x8 channels modelled as GDDR6 x16 channels at 15000 MT/s: the same channel count, 32 B burst time and 960 GB/s (32 x 2 B x 15 GT/s)", + "PLATFORM_MEMORY_LATENCY": "339 core cycles of PHY/NoC latency: the 407 calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference', less the 68 cycles JEDEC GDDR6 timing adds to the device's row-conflict latency (derived, not re-run; recalibrate with a pointer chase when the memory model changes)" } } diff --git a/profiles/v100.json b/profiles/v100.json index c58be16476..451551d910 100644 --- a/profiles/v100.json +++ b/profiles/v100.json @@ -20,13 +20,13 @@ "VX_CFG_PLATFORM_CLOCK_RATE": 1530, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 900000, "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 877 + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 1752 }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "80 SMs, 64 resident warps/SM, 4 processing blocks with one scheduler each (NVIDIA Tesla V100 GPU Architecture whitepaper, Table 1)", "DCACHE_SIZE": "128 KB combined L1/shared per SM (whitepaper)", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "6 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways, 32 banks for the 32 memory channels", - "PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "4 HBM2 stacks x 8 channels = 32 channels at 877 MHz: 32 x 32 B x 877 MHz = 898 GB/s (whitepaper)", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "4 HBM2 stacks x 8 128-bit channels = 32 channels at 1752 MT/s: 32 x 16 B x 1752 MT/s = 897 GB/s (whitepaper)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)" } } diff --git a/sim/common/dram_sim.cpp b/sim/common/dram_sim.cpp index e6f7f1a0c5..bc899b89fc 100644 --- a/sim/common/dram_sim.cpp +++ b/sim/common/dram_sim.cpp @@ -36,17 +36,64 @@ using namespace vortex; namespace { -// IPOLY(5) bank hash over 16 banks (Rau, "Pseudo-randomly interleaved memory", -// ISCA 1991), GPGPU-Sim's -dram_bnk_indexing_policy 2: the bank XORed with a -// function of the row, so streams whose bases differ by a power of two land in -// different banks instead of closing each other's rows. -uint32_t ipoly16(uint64_t row, uint32_t bank) { - auto bit = [row](int i) { return uint32_t(row >> i) & 1u; }; - uint32_t i0 = bit(11) ^ bit(10) ^ bit(9) ^ bit(8) ^ bit(6) ^ bit(4) ^ bit(3) ^ bit(0); - uint32_t i1 = bit(12) ^ bit(8) ^ bit(7) ^ bit(6) ^ bit(5) ^ bit(3) ^ bit(1) ^ bit(0); - uint32_t i2 = bit(9) ^ bit(8) ^ bit(7) ^ bit(6) ^ bit(4) ^ bit(2) ^ bit(1); - uint32_t i3 = bit(10) ^ bit(9) ^ bit(8) ^ bit(7) ^ bit(5) ^ bit(3) ^ bit(2); - return bank ^ (i0 | (i1 << 1) | (i2 << 2) | (i3 << 3)); +// Bus timings are DRAM clocks, fixed by the burst; array timings are +// nanoseconds, which Ramulator converts at the configured data rate. +struct clock_timing_t { + const char* name; + int clocks; +}; + +struct ns_timing_t { + const char* name; + float ns; +}; + +// A 32 B burst is BL4 on a 64-bit pseudo-channel: two clocks, and the same +// spacing between column commands to another bank group (four within one). +// nRTW only completes the set Ramulator requires. +const clock_timing_t hbm_bus_timing[] = { + {"nBL", 2}, {"nCCDS", 2}, {"nCCDL", 4}, {"nRTW", 3}, +}; + +// JEDEC HBM2 array timing as GPGPU-Sim's SM7_QV100 configuration lists it (its +// JEDEC line, at 1 GHz: RCD=RP=CL=14, RAS=33, RC=47, WR=12, RRD=4, RTPL=4); +// the rest from Ramulator's HBM2_2Gbps preset, whose 1 ns clock makes its +// cycles nanoseconds. The array does not speed up with the interface: HBM2E +// and HBM3 parts keep these timings (GPGPU-Sim's SM80_A100 and SM90_H100 HBM +// lines are the same 14/33/47 ns at their own clocks). Refresh timings complete +// the set Ramulator requires. +const ns_timing_t hbm_array_timing[] = { + {"tCL", 14}, {"tRCDRD", 14}, {"tRCDWR", 14}, {"tRP", 14}, {"tRAS", 33}, {"tRC", 47}, + {"tCWL", 2}, {"tWR", 12}, {"tRTPS", 2}, {"tRTPL", 4}, {"tRRDS", 4}, {"tRRDL", 4}, + {"tWTRS", 3}, {"tWTRL", 4}, {"tFAW", 15}, {"tRFCSB", 160}, {"tREFI", 3900}, {"tRREFD", 8}, +}; + +// Ramulator's GDDR6 clock is half the data rate: a 32 B burst, 16 beats on a +// x16 channel, is eight clocks, and column commands are two (three within a +// bank group) command clocks apart, a command clock being eight data beats. +const clock_timing_t gddr6_bus_timing[] = { + {"nBL", 8}, {"nCCDS", 8}, {"nCCDL", 12}, +}; + +// Ramulator's x16 GDDR6 timing at 2000 (its GDDR6_2000_1350mV_double preset +// and secondary tables) at its 570 ps command clock, in nanoseconds; refresh +// timings complete the set Ramulator requires. +const ns_timing_t gddr6_array_timing[] = { + {"tCL", 13.7f}, {"tRCDRD", 14.8f}, {"tRCDWD", 9.1f}, {"tRP", 14.8f}, {"tRAS", 30.2f}, {"tRC", 45.0f}, + {"tWR", 14.8f}, {"tRTP", 2.3f}, {"tCWL", 3.4f}, {"tRRDS", 2.9f}, {"tRRDL", 3.4f}, + {"tWTRS", 5.1f}, {"tWTRL", 6.3f}, {"tFAW", 16.0f}, + {"tRFC", 120.0f}, {"tRFCpb", 60.0f}, {"tREFI", 1900.0f}, {"tRREFD", 8.0f}, +}; + +template +void set_timing(YAML::Node timing, uint32_t data_rate, const clock_timing_t (&bus)[B], const ns_timing_t (&array)[A]) { + timing["rate"] = data_rate; + for (auto& t : bus) { + timing[t.name] = t.clocks; + } + for (auto& t : array) { + timing[t.name] = t.ns; + } } } @@ -71,139 +118,126 @@ class DramSim::Impl { uint64_t ready_cycle; }; - // GDDR6 holds a fixed 16 Gb per device, so its row count shrinks as the - // channel count grows (Ramulator only knows the refresh timing of 4/8/16 Gb - // devices); rows alias above that capacity, which only affects row hits. - static constexpr uint32_t gddr6_density_mb_ = 16384; + // An independent controller and its device. It accepts one transaction per + // DRAM cycle and returns its responses independently of the other channels; + // write acknowledgements do not wait behind read data. + struct channel_t { + Ramulator::IFrontEnd* frontend; + Ramulator::IMemorySystem* memory_system; + std::queue pending_txs; + std::deque read_rsps; + std::deque write_rsps; + uint32_t inflight; + }; + + // Each standard moves 32 B per column command (prefetch x channel width). + static constexpr uint32_t tx_size_ = 32; + + // Every device here holds 8 Gb per channel; addresses above alias, which + // only affects row hits. + static constexpr uint64_t channel_size_ = uint64_t(1) << 30; + + // Bursts per row visit (minimalist open page, 128 B). + static constexpr uint32_t row_visit_bits_ = 2; Config config_; - uint32_t tx_size_; - uint32_t row_mask_; - Ramulator::IFrontEnd* ramulator_frontend_; - Ramulator::IMemorySystem* ramulator_memorysystem_; - uint32_t ch_bits_; + uint32_t bank_bits_; // pseudo-channel, bank group and bank + uint32_t column_bits_; // bursts per row + std::vector channels_; uint64_t cycle_; - // DRAM ticks owed, in units of 1/core_clock: each core cycle adds - // dram_clock, each DRAM tick takes core_clock. + // DRAM ticks owed, in units of 1/(2 x core_clock): each core cycle adds the + // data rate, each DRAM tick (half the data rate) takes 2 x core_clock. uint64_t clock_credit_; - // A channel accepts one transaction per DRAM cycle and returns its - // responses independently of the others; write acknowledgements do not wait - // behind read data. - std::vector> pending_txs_; - std::vector> read_rsps_; - std::vector> write_rsps_; - std::vector inflight_; - - // Ramulator address of a transaction: the channel in the low bits, then the - // fields in the order the RoBaRaCoCh mapper slices them. - uint64_t dram_addr(uint32_t channel, uint64_t addr) const { - uint64_t t; - if (config_.type == Type::GDDR6) { - // GPGPU-Sim's RTX 3070 layout (-gpgpu_mem_addr_mapping): 32 B bursts, - // two column bits, one bank bit, four column bits, three bank bits, - // row; the low bank bit selects the bank group. - uint64_t col = ((addr >> 5) & 0x3) | (((addr >> 8) & 0xf) << 2); - uint64_t row = (addr >> 15) & row_mask_; - uint32_t bank = ipoly16(row, uint32_t(((addr >> 7) & 0x1) | (((addr >> 12) & 0x7) << 1))); - t = row; - t = (t << 2) | (bank >> 2); - t = (t << 2) | (bank & 0x3); - t = (t << 6) | col; - } else { - // The A100 HBM interleave: a stream alternates column, pseudo-channel - // and column again at 64 B granules, then changes bank group every 4 KB - // and bank every 16 KB, with 2 KB rows. HBM2_A100 uses 2 of the 4 banks - // of a group (16 per channel), as GPGPU-Sim models the A100; HBM2 all 4. - uint32_t bank_bits = (config_.type == Type::HBM2_A100) ? 1 : 2; - uint64_t col = (((addr >> 8) & 0xf) << 1) | ((addr >> 6) & 0x1); - uint64_t pc = (addr >> 7) & 0x1; - uint64_t bg = (addr >> 12) & 0x3; - uint64_t bank = (addr >> 14) & ((1u << bank_bits) - 1); - uint64_t row = (addr >> (14 + bank_bits)) & row_mask_; - t = row; - t = (t << 2) | bank; - t = (t << 2) | bg; - t = (t << 1) | pc; - t = (t << 5) | col; - } - t = (t << ch_bits_) | channel; - return t * tx_size_; - } - static void set_hbm2_device(YAML::Node& dram, const Config& config, uint32_t pow2_channels) { - dram["impl"] = "HBM2"; - dram["org"]["preset"] = "HBM2_8Gb"; + static void set_hbm_device(YAML::Node dram, const Config& config) { + // Ramulator's 8 Gb organization: 2 pseudo-channels x 4 bank groups x 4 + // banks, 1 KB rows; a 128-bit channel width with the 2n prefetch makes a + // transaction one 32 B pseudo-channel burst. + dram["impl"] = (config.type == Type::HBM3) ? "HBM3" : "HBM2"; + dram["org"]["preset"] = (config.type == Type::HBM3) ? "HBM3_8Gb" : "HBM2_8Gb"; dram["org"]["density"] = 8192; - dram["org"]["channel"] = pow2_channels; - // A transaction is one burst: 2 x 256-bit prefetch = 64 B. - dram["org"]["channel_width"] = 256; - dram["timing"]["preset"] = "HBM2_2Gbps"; - if (config.type == Type::HBM2_A100) { - // GPGPU-Sim SM80_A100 HBM timing in DRAM cycles. One 64 B transaction - // is two of its 32 B atoms, so the same-bank-group spacing is twice its - // tCCDL. - auto timing = dram["timing"]; - timing["nCL"] = 22; - timing["nRCDRD"] = 22; - timing["nRCDWR"] = 22; - timing["nRP"] = 22; - timing["nRAS"] = 50; - timing["nRC"] = 72; - timing["nWR"] = 19; - timing["nRTPS"] = 7; - timing["nRTPL"] = 7; - timing["nCWL"] = 4; - timing["nCCDS"] = 2; - timing["nCCDL"] = 8; - timing["nRRDS"] = 7; - timing["nRRDL"] = 7; - } + dram["org"]["channel"] = 1; + dram["org"]["channel_width"] = 128; + set_timing(dram["timing"], config.data_rate, hbm_bus_timing, hbm_array_timing); } - static void set_gddr6_device(YAML::Node& dram, const Config& config, uint32_t pow2_channels) { - // x16 channels with 2 KB pages: 512 columns of 16 bits, 4 bank groups of 4 - // banks; a 32 B transaction is a 16-beat burst (8 prefetched columns over a - // 32-bit internal channel width). + static void set_gddr6_device(YAML::Node dram, const Config& config) { + // One x16 channel of a 16 Gb device (8 Gb), 4 bank groups of 4 banks, + // 2 KB pages. A 32 B burst is 16n prefetch on 16 pins; Ramulator's GDDR6 + // prefetches 8 columns, so its columns are 32 bits. dram["impl"] = "GDDR6"; - dram["org"]["dq"] = 16; - dram["org"]["channel"] = pow2_channels; + dram["org"]["dq"] = 32; + dram["org"]["channel"] = 1; dram["org"]["bankgroup"] = 4; dram["org"]["bank"] = 4; + dram["org"]["row"] = 1 << 15; dram["org"]["column"] = 512; - dram["org"]["row"] = uint32_t((uint64_t(gddr6_density_mb_) << 20) / (uint64_t(pow2_channels) * 4 * 4 * 512 * 16)); - dram["org"]["density"] = gddr6_density_mb_; + dram["org"]["density"] = 8192; dram["org"]["channel_width"] = 32; - // The transfer rate is twice the data clock. Bus timings are in clocks of - // the 32 B burst; array timings are GPGPU-Sim's RTX 3070 values - // (-gpgpu_dram_timing_opt) held in nanoseconds; refresh is JEDEC GDDR6. - auto timing = dram["timing"]; - timing["rate"] = 2 * config.dram_clock; - timing["nBL"] = 8; - timing["nCCDS"] = 8; - timing["nCCDL"] = 12; - timing["tCL"] = 6.8f; - timing["tRCDRD"] = 6.8f; - timing["tRCDWD"] = 6.8f; - timing["tRP"] = 6.8f; - timing["tRAS"] = 15.7f; - timing["tRC"] = 22.3f; - timing["tWR"] = 6.8f; - timing["tRTP"] = 1.2f; - timing["tCWL"] = 2.3f; - timing["tRRDS"] = 3.5f; - timing["tRRDL"] = 3.5f; - timing["tWTRS"] = 2.8f; - timing["tWTRL"] = 2.8f; - timing["tFAW"] = 16.0f; - timing["tRFC"] = 120.0f; - timing["tRFCpb"] = 60.0f; - timing["tREFI"] = 1900.0f; - timing["tRREFD"] = 8.0f; + set_timing(dram["timing"], config.data_rate, gddr6_bus_timing, gddr6_array_timing); + } + + YAML::Node channel_config(uint32_t ch) const { + YAML::Node config; + config["Frontend"]["impl"] = "GEM5"; + auto memory_system = config["MemorySystem"]; + memory_system["impl"] = "GenericDRAM"; + memory_system["clock_ratio"] = 1; + if (config_.type == Type::GDDR6) { + set_gddr6_device(memory_system["DRAM"], config_); + } else { + set_hbm_device(memory_system["DRAM"], config_); + } + auto controller = memory_system["Controller"]; + controller["impl"] = "Generic"; + controller["Scheduler"]["impl"] = "FRFCFS"; + // No refresh is modelled: Ramulator's refresh manager refreshes per rank + // and these devices have no rank level. GPU controllers hide refresh with + // per-bank refresh, for which Ramulator has no manager (and its HBM REFsb + // lacks the bank's tRFCsb busy time); all-bank refresh would cost every + // channel ~9% of its time instead. + controller["RefreshManager"]["impl"] = "AllBank"; + controller["RowPolicy"]["impl"] = "OpenRowPolicy"; + // Per-command DRAM trace recording is a debugging aid only: it writes an + // entry for every DRAM command and dominates simulation time (100s of MB + // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env + // var so default simx/rtlsim runs stay fast. + if (const char* trace_path = getenv("DRAM_TRACE")) { + std::string path = trace_path[0] ? trace_path : "./trace/ramulator.log"; + YAML::Node trace_plugin; + trace_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; + trace_plugin["ControllerPlugin"]["path"] = path + "." + std::to_string(ch); + controller["plugins"].push_back(trace_plugin); + } + // Fields in the order dram_addr lays them out. + memory_system["AddrMapper"]["impl"] = "RoBaRaCoCh"; + return config; + } + + // Ramulator address of a transaction. A row visit is four bursts, then the + // next pseudo-channel, bank group and bank (minimalist open page), and the + // bank is XORed with the row folded to the bank's width (permutation-based + // interleaving), so streams a power of two apart open rows in different + // banks. + uint64_t dram_addr(uint64_t addr) const { + uint64_t tx = (addr & (channel_size_ - 1)) / tx_size_; + uint64_t visit = tx & ((1u << row_visit_bits_) - 1); + tx >>= row_visit_bits_; + uint64_t bank = tx & ((1u << bank_bits_) - 1); + tx >>= bank_bits_; + uint64_t column = tx & ((1u << (column_bits_ - row_visit_bits_)) - 1); + uint64_t row = tx >> (column_bits_ - row_visit_bits_); + for (uint64_t r = row; r != 0; r >>= bank_bits_) { + bank ^= r & ((1u << bank_bits_) - 1); + } + uint64_t t = (row << bank_bits_) | bank; + t = (t << column_bits_) | (column << row_visit_bits_) | visit; + return t * tx_size_; } // Retires the ready head responses of `rsps`; a request completes with its // last transaction. - void drain(std::deque& rsps, uint32_t ch) { + void drain(std::deque& rsps, channel_t& channel) { while (!rsps.empty()) { auto& rsp = rsps.front(); if (rsp.ready_cycle > cycle_) { @@ -214,94 +248,49 @@ class DramSim::Impl { break; } if (--request.pending == 0) { - --inflight_[ch]; + --channel.inflight; } rsps.pop_front(); } } - void handle_pending_responses() { - for (uint32_t ch = 0; ch < config_.num_channels; ++ch) { - this->drain(read_rsps_[ch], ch); - this->drain(write_rsps_[ch], ch); + void issue(channel_t& channel) { + if (channel.pending_txs.empty()) { + return; } - } - - void handle_pending_requests() { - for (uint32_t ch = 0; ch < config_.num_channels; ++ch) { - auto& txs = pending_txs_[ch]; - if (txs.empty()) { - continue; - } - auto& tx = txs.front(); - auto req_type = tx.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; - std::function callback = nullptr; - if (!tx.is_write) { - callback = [this, ch, request = tx.request](Ramulator::Request& /*dram_req*/) { - this->read_rsps_[ch].push_back({request, this->cycle_ + 1 + this->config_.latency}); - }; - } - if (ramulator_frontend_->receive_external_requests(req_type, tx.addr, 0, callback)) { - // Ramulator does not handle write responses, so we handle the callback ourselves. - if (tx.is_write) { - write_rsps_[ch].push_back({tx.request, cycle_}); - } - txs.pop(); + auto& tx = channel.pending_txs.front(); + auto req_type = tx.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; + std::function callback = nullptr; + if (!tx.is_write) { + callback = [this, &channel, request = tx.request](Ramulator::Request& /*dram_req*/) { + channel.read_rsps.push_back({request, this->cycle_ + 1 + this->config_.latency}); + }; + } + if (channel.frontend->receive_external_requests(req_type, tx.addr, 0, callback)) { + // Ramulator does not handle write responses, so we handle the callback ourselves. + if (tx.is_write) { + channel.write_rsps.push_back({tx.request, cycle_}); } + channel.pending_txs.pop(); } } public: - Impl(const Config& config) : config_(config) { - uint32_t pow2_channels = 1; - ch_bits_ = 0; - while (pow2_channels < config.num_channels) { - pow2_channels <<= 1; - ++ch_bits_; + Impl(const Config& config) + : config_(config) + // HBM: 2 pseudo-channels x 4 bank groups x 4 banks, 32 bursts per 1 KB + // row; GDDR6: 4 x 4 banks, 64 bursts per 2 KB row. + , bank_bits_((config.type == Type::GDDR6) ? 4 : 5) + , column_bits_((config.type == Type::GDDR6) ? 6 : 5) + , channels_(config.num_channels) { + for (uint32_t ch = 0; ch < config.num_channels; ++ch) { + auto ramulator_config = this->channel_config(ch); + auto& channel = channels_[ch]; + channel.frontend = Ramulator::Factory::create_frontend(ramulator_config); + channel.memory_system = Ramulator::Factory::create_memory_system(ramulator_config); + channel.frontend->connect_memory_system(channel.memory_system); + channel.memory_system->connect_frontend(channel.frontend); } - - YAML::Node dram; - if (config.type == Type::GDDR6) { - set_gddr6_device(dram, config, pow2_channels); - tx_size_ = 32; - row_mask_ = dram["org"]["row"].as() - 1; - } else { - set_hbm2_device(dram, config, pow2_channels); - tx_size_ = 64; - row_mask_ = 0x7fff; - } - - YAML::Node dram_config; - dram_config["Frontend"]["impl"] = "GEM5"; - dram_config["MemorySystem"]["impl"] = "GenericDRAM"; - dram_config["MemorySystem"]["clock_ratio"] = 1; - dram_config["MemorySystem"]["DRAM"] = dram; - dram_config["MemorySystem"]["Controller"]["impl"] = "Generic"; - dram_config["MemorySystem"]["Controller"]["Scheduler"]["impl"] = "FRFCFS"; - // libramulator-vortex: Ramulator's AllBank refresh never fires on rank-less HBM2/GDDR6. - dram_config["MemorySystem"]["Controller"]["RefreshManager"]["impl"] = "ChannelRefresh"; - dram_config["MemorySystem"]["Controller"]["RowPolicy"]["impl"] = "OpenRowPolicy"; - // Per-command DRAM trace recording is a debugging aid only: it writes an - // entry for every DRAM command and dominates simulation time (100s of MB - // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env - // var so default simx/rtlsim runs stay fast. - if (const char* trace_path = getenv("DRAM_TRACE")) { - YAML::Node draw_plugin; - draw_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; - draw_plugin["ControllerPlugin"]["path"] = trace_path[0] ? trace_path : "./trace/ramulator.log"; - dram_config["MemorySystem"]["Controller"]["plugins"].push_back(draw_plugin); - } - dram_config["MemorySystem"]["AddrMapper"]["impl"] = "RoBaRaCoCh"; - - ramulator_frontend_ = Ramulator::Factory::create_frontend(dram_config); - ramulator_memorysystem_ = Ramulator::Factory::create_memory_system(dram_config); - ramulator_frontend_->connect_memory_system(ramulator_memorysystem_); - ramulator_memorysystem_->connect_frontend(ramulator_frontend_); - - pending_txs_.resize(config.num_channels); - read_rsps_.resize(config.num_channels); - write_rsps_.resize(config.num_channels); - inflight_.resize(config.num_channels, 0); this->reset(); } @@ -309,39 +298,42 @@ class DramSim::Impl { std::ofstream nullstream("ramulator.stats.log"); auto original_buf = std::cout.rdbuf(); std::cout.rdbuf(nullstream.rdbuf()); - ramulator_frontend_->finalize(); - ramulator_memorysystem_->finalize(); + for (auto& channel : channels_) { + channel.frontend->finalize(); + channel.memory_system->finalize(); + } std::cout.rdbuf(original_buf); } void reset() { cycle_ = 0; clock_credit_ = 0; - for (auto& q : pending_txs_) { - q = std::queue(); - } - for (auto& q : read_rsps_) { - q.clear(); + for (auto& channel : channels_) { + channel.pending_txs = std::queue(); + channel.read_rsps.clear(); + channel.write_rsps.clear(); + channel.inflight = 0; } - for (auto& q : write_rsps_) { - q.clear(); - } - std::fill(inflight_.begin(), inflight_.end(), 0); } void tick() { ++cycle_; - clock_credit_ += config_.dram_clock; - this->handle_pending_responses(); - while (clock_credit_ >= config_.core_clock) { - this->handle_pending_requests(); - ramulator_memorysystem_->tick(); - clock_credit_ -= config_.core_clock; + clock_credit_ += config_.data_rate; + for (auto& channel : channels_) { + this->drain(channel.read_rsps, channel); + this->drain(channel.write_rsps, channel); + } + while (clock_credit_ >= 2 * config_.core_clock) { + for (auto& channel : channels_) { + this->issue(channel); + channel.memory_system->tick(); + } + clock_credit_ -= 2 * config_.core_clock; } } bool can_accept(uint32_t channel) const { - return (config_.window == 0) || (inflight_.at(channel) < config_.window); + return (config_.queue_size == 0) || (channels_.at(channel).inflight < config_.queue_size); } void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { @@ -349,11 +341,11 @@ class DramSim::Impl { uint32_t n = (config_.block_size > tx_size_) ? (config_.block_size / tx_size_) : 1; uint64_t base = addr & ~uint64_t(config_.block_size - 1); auto request = std::make_shared(request_t{response_cb, arg, n}); - auto& txs = pending_txs_.at(channel); + auto& ch = channels_.at(channel); for (uint32_t i = 0; i < n; ++i) { - txs.push({dram_addr(channel, base + i * tx_size_), is_write, request}); + ch.pending_txs.push({dram_addr(base + i * tx_size_), is_write, request}); } - ++inflight_[channel]; + ++ch.inflight; } }; @@ -370,8 +362,8 @@ DramSim::Config DramSim::platform_config() { Config config; if (type == "HBM2") { config.type = Type::HBM2; - } else if (type == "HBM2_A100") { - config.type = Type::HBM2_A100; + } else if (type == "HBM3") { + config.type = Type::HBM3; } else if (type == "GDDR6") { config.type = Type::GDDR6; } else { @@ -380,9 +372,9 @@ DramSim::Config DramSim::platform_config() { config.num_channels = VX_CFG_PLATFORM_MEMORY_NUM_BANKS; config.block_size = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; config.core_clock = VX_CFG_PLATFORM_CLOCK_RATE; - config.dram_clock = VX_CFG_PLATFORM_MEMORY_CLOCK_RATE; + config.data_rate = VX_CFG_PLATFORM_MEMORY_DATA_RATE; config.latency = VX_CFG_PLATFORM_MEMORY_LATENCY; - config.window = VX_CFG_PLATFORM_MEMORY_WINDOW; + config.queue_size = VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE; return config; } diff --git a/sim/common/dram_sim.h b/sim/common/dram_sim.h index cc88ff5d78..565dede377 100644 --- a/sim/common/dram_sim.h +++ b/sim/common/dram_sim.h @@ -23,10 +23,12 @@ class DramSim { public: typedef bool (*ResponseCallback)(void *arg); + // The JEDEC standards Ramulator models, each at its own organization and + // timing; the data rate picks the speed. enum class Type { - HBM2, // HBM2 at the 2 Gbps JEDEC preset, 64 B bursts - HBM2_A100, // HBM2 with the A100 timing of GPGPU-Sim's SM80 configuration - GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages + HBM2, // HBM2/HBM2E: 128-bit channels of two 64-bit pseudo-channels, 32 B bursts + HBM3, // HBM3, a 128-bit channel per pair of 64-bit channels, 32 B bursts + GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages }; struct Config { @@ -34,9 +36,9 @@ class DramSim { uint32_t num_channels; // need not be a power of two uint32_t block_size; // bytes per request uint32_t core_clock; // MHz - uint32_t dram_clock; // MHz (GDDR6: the data clock, half the transfer rate) + uint32_t data_rate; // MT/s per pin uint32_t latency; // fixed read latency added in core cycles (PHY, controller, NoC) - uint32_t window; // requests a channel holds in flight; 0 = unbounded + uint32_t queue_size; // requests a channel holds in flight; 0 = unbounded }; // The memory configured in VX_config.toml. @@ -49,7 +51,7 @@ class DramSim { void tick(); - // False while the channel holds its window of requests. + // False while the channel holds queue_size requests. bool can_accept(uint32_t channel) const; // addr: byte address within the channel. diff --git a/tests/unittest/dram_sim/main.cpp b/tests/unittest/dram_sim/main.cpp index 81e127e145..4ec135770c 100644 --- a/tests/unittest/dram_sim/main.cpp +++ b/tests/unittest/dram_sim/main.cpp @@ -25,24 +25,30 @@ static bool on_response(void*) { return true; } -// A100 HBM as GPGPU-Sim's SM80 configuration models it: 40 channels at -// 1,512 MHz behind a 1,410 MHz core, 190 cycles of fixed latency, a 257-request -// window per channel. +// The a100 profile: 32 HBM2 channels at 3,038 MT/s (the card's 40 at 2,430) +// behind a 1,410 MHz core, with the fixed latency (190 cycles) and request +// queue (257) of GPGPU-Sim's SM80 configuration. static DramSim::Config a100_config() { DramSim::Config config; - config.type = DramSim::Type::HBM2_A100; - config.num_channels = 40; + config.type = DramSim::Type::HBM2; + config.num_channels = 32; config.block_size = 64; config.core_clock = 1410; - config.dram_clock = 1512; + config.data_rate = 3038; config.latency = 190; - config.window = 257; + config.queue_size = 257; return config; } -// Median core cycles for one read on an idle channel (the median, so the few -// reads that wait behind a refresh do not count); `addr_of(i)` picks the i-th -// address, and the first read (cold) is not counted. +// Peak bytes per core cycle: a 32 B burst every two DRAM clocks per +// pseudo-channel (two per channel), or every eight clocks per GDDR6 channel. +static double peak_bandwidth(const DramSim::Config& config) { + double bytes_per_clock = (config.type == DramSim::Type::GDDR6) ? 4.0 : 32.0; + return config.num_channels * bytes_per_clock * (config.data_rate / 2.0) / config.core_clock; +} + +// Median core cycles for one read on an idle channel; `addr_of(i)` picks the +// i-th address, and the first read (cold) is not counted. template static double unloaded_latency(const DramSim::Config& config, F addr_of) { DramSim dram(config); @@ -68,9 +74,11 @@ static double unloaded_latency(const DramSim::Config& config, F addr_of) { return double(samples[samples.size() / 2]); } -// Bytes per core cycle when every channel streams `blocks` sequential blocks; +// Bytes per core cycle when every channel streams `blocks` blocks, taken in +// turn from `streams` sequential streams `stride` bytes apart; // `peak_inflight` receives the most reads outstanding at once. -static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, uint64_t* peak_inflight = nullptr) { +static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, uint64_t* peak_inflight = nullptr, + uint32_t streams = 1, uint64_t stride = 0) { DramSim dram(config); dram.reset(); g_responses = 0; @@ -82,7 +90,8 @@ static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, u while (g_responses < total) { for (uint32_t ch = 0; ch < config.num_channels; ++ch) { if (sent[ch] < blocks && dram.can_accept(ch)) { - dram.send_request(ch, uint64_t(sent[ch]) * config.block_size, false, on_response, nullptr); + uint64_t addr = uint64_t(sent[ch] % streams) * stride + uint64_t(sent[ch] / streams) * config.block_size; + dram.send_request(ch, addr, false, on_response, nullptr); ++sent[ch]; ++issued; } @@ -97,6 +106,12 @@ static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, u return double(total) * config.block_size / cycles; } +// Alternates between two rows of one bank: the rows from address bit 15 on +// fold to the same bank permutation (HBM 33 = 1|1 in 5-bit chunks, GDDR6 +// 17 = 1|1 in 4-bit chunks). +static uint64_t hbm_conflict(int i) { return uint64_t(i % 2) * 33 << 15; } +static uint64_t gddr6_conflict(int i) { return uint64_t(i % 2) * 17 << 15; } + static int g_errors = 0; static void check(bool cond, const char* what, double got) { @@ -111,37 +126,44 @@ int main() { // Unloaded latency: open-row hits, then row conflicts (precharge + activate). double open = unloaded_latency(a100, [](int i) { return uint64_t(i % 8) * 128; }); - double conflict = unloaded_latency(a100, [](int i) { return uint64_t(i) << 20; }); + double conflict = unloaded_latency(a100, hbm_conflict); check(open > 205 && open < 225, "A100 open-row read latency ~215 core cycles", open); check(conflict > 245 && conflict < 265, "A100 row-conflict read latency ~255 core cycles", conflict); // The fixed latency is added exactly once per read (measured at a 1:1 clock // ratio, so the DRAM clock phase cannot shift the comparison). auto unit_clock = a100; - unit_clock.core_clock = unit_clock.dram_clock; + unit_clock.core_clock = unit_clock.data_rate / 2; auto no_latency = unit_clock; no_latency.latency = 0; auto open_row = [](int i) { return uint64_t(i % 8) * 128; }; double added = unloaded_latency(unit_clock, open_row) - unloaded_latency(no_latency, open_row); check(added == 190, "fixed latency adds 190 cycles", added); - // Every channel is an independent controller: 40 channels stream ~40x one. + // Every channel is an independent controller: 32 channels stream ~32x one, + // and the count need not be a power of two. auto one = a100; one.num_channels = 1; double bw1 = stream_bandwidth(one, 4096); - double bw40 = stream_bandwidth(a100, 4096); - check(bw40 / bw1 > 36 && bw40 / bw1 < 41, "bandwidth scales with the channel count", bw40 / bw1); - - // Saturated stream efficiency against the A100's 1,373 B/cycle peak. - check(bw40 > 0.6 * 1373 && bw40 <= 1373, "A100 stream reaches 60-100% of peak", bw40); - - // The window bounds the requests in flight. + double bw32 = stream_bandwidth(a100, 4096); + check(bw32 / bw1 > 29 && bw32 / bw1 < 33, "bandwidth scales with the channel count", bw32 / bw1); + auto five = a100; + five.num_channels = 5; + double bw5 = stream_bandwidth(five, 4096); + check(bw5 / bw1 > 4.5 && bw5 / bw1 < 5.1, "5 channels stream ~5x one", bw5 / bw1); + + // Saturated stream efficiency against the computed peak (1,103 B/cycle, the + // card's 1,555 GB/s). + double a100_peak = peak_bandwidth(a100); + check(bw32 > 0.75 * a100_peak && bw32 <= a100_peak, "A100 stream reaches 75-100% of peak", bw32); + + // The controller queue bounds the requests in flight. auto narrow = a100; narrow.num_channels = 1; - narrow.window = 8; + narrow.queue_size = 8; uint64_t peak = 0; stream_bandwidth(narrow, 1024, &peak); - check(peak <= 8, "window of 8 bounds requests in flight", peak); + check(peak <= 8, "queue of 8 bounds requests in flight", peak); // A block of two bursts returns one response. auto wide = a100; @@ -158,65 +180,45 @@ int main() { check(g_responses == 1, "a 128 B block returns one response", double(g_responses)); } - // Plain HBM2 at the platform default: still independent channels. - auto hbm2 = a100; - hbm2.type = DramSim::Type::HBM2; - hbm2.num_channels = 2; - hbm2.latency = 0; - hbm2.window = 0; - double bw2 = stream_bandwidth(hbm2, 4096); - hbm2.num_channels = 1; - double bw1h = stream_bandwidth(hbm2, 4096); + // The platform default: 2 HBM2 channels clocked with a 400 MHz core. + DramSim::Config base{DramSim::Type::HBM2, 2, 64, 400, 800, 0, 0}; + double base_open = unloaded_latency(base, [](int i) { return uint64_t(i % 8) * 64; }); + double base_conflict = unloaded_latency(base, hbm_conflict); + check(base_open == 14, "default open-row read latency 14 cycles", base_open); + check(base_conflict == 26, "default row-conflict read latency 26 cycles", base_conflict); + + // Streams a power of two apart open their rows in different banks: four + // streams 1 MB apart on one channel keep over half a single stream's rate + // (an unpermuted map keeps a third). They still share banks over time, so + // each row visit pays an activation. + auto base_one = base; + base_one.num_channels = 1; + double seq = stream_bandwidth(base_one, 4096); + double strided = stream_bandwidth(base_one, 4096, nullptr, 4, 1 << 20); + check(strided > 0.55 * seq, "four streams 1 MB apart keep over half one stream's rate", strided / seq); + double bw2 = stream_bandwidth(base, 4096); + double bw1h = stream_bandwidth(base_one, 4096); check(bw2 / bw1h > 1.8 && bw2 / bw1h < 2.05, "HBM2 2 channels stream ~2x one", bw2 / bw1h); - // Every channel refreshes: over many refresh intervals some open-row reads - // wait behind a refresh (nRFC = 160 cycles at a 1:1 clock). - { - auto refresh = unit_clock; - refresh.type = DramSim::Type::HBM2; - refresh.num_channels = 1; - refresh.latency = 0; - DramSim dram(refresh); - dram.reset(); - uint64_t min_lat = ~0ull, max_lat = 0; - for (int i = 0; i < 400; ++i) { - g_responses = 0; - uint64_t cycles = 0; - dram.send_request(0, uint64_t(i % 8) * 128, false, on_response, nullptr); - while (g_responses == 0) { - dram.tick(); - ++cycles; - } - if (i != 0) { - min_lat = std::min(min_lat, cycles); - max_lat = std::max(max_lat, cycles); - } - for (int k = 0; k < 97; ++k) { - dram.tick(); - } - } - check(max_lat - min_lat > 100, "refresh stalls reads by ~nRFC", double(max_lat - min_lat)); - } - - // GDDR6 at the rtx5080 profile's shape: 32 x16 channels at 15 GT/s (7,500 - // MHz data clock) behind a 2,295 MHz core, 32 B per 8-clock burst: 418 B/cycle - // peak, the card's 960 GB/s. - auto gddr6 = a100; - gddr6.type = DramSim::Type::GDDR6; - gddr6.num_channels = 32; - gddr6.core_clock = 2295; - gddr6.dram_clock = 7500; - gddr6.latency = 0; - gddr6.window = 0; - double gpeak = 32.0 * 32 / 8 * 7500 / 2295; + // HBM3 at the h100 profile's shape: 32 channels at 6,545 MT/s behind a + // 1,980 MHz core, the card's 3.35 TB/s. + DramSim::Config h100{DramSim::Type::HBM3, 32, 64, 1980, 6545, 0, 0}; + double h100_peak = peak_bandwidth(h100); + double hbw = stream_bandwidth(h100, 4096); + check(hbw > 0.75 * h100_peak && hbw <= h100_peak, "H100 stream reaches 75-100% of peak", hbw); + + // GDDR6 at the rtx5080 profile's shape: 32 x16 channels at 15 GT/s behind a + // 2,295 MHz core: 418 B/cycle peak, the card's 960 GB/s. + DramSim::Config gddr6{DramSim::Type::GDDR6, 32, 64, 2295, 15000, 0, 0}; + double gpeak = peak_bandwidth(gddr6); double gbw32 = stream_bandwidth(gddr6, 4096); - check(gbw32 > 0.6 * gpeak && gbw32 <= gpeak, "GDDR6 stream reaches 60-100% of peak", gbw32); + check(gbw32 > 0.7 * gpeak && gbw32 <= gpeak, "GDDR6 stream reaches 70-100% of peak", gbw32); auto gddr6_one = gddr6; gddr6_one.num_channels = 1; double gbw1 = stream_bandwidth(gddr6_one, 4096); check(gbw32 / gbw1 > 29 && gbw32 / gbw1 < 33, "GDDR6 bandwidth scales with the channel count", gbw32 / gbw1); double gopen = unloaded_latency(gddr6_one, [](int i) { return uint64_t(i % 8) * 32; }); - double gconflict = unloaded_latency(gddr6_one, [](int i) { return uint64_t(i) << 22; }); + double gconflict = unloaded_latency(gddr6_one, gddr6_conflict); check(gopen < gconflict, "GDDR6 row conflicts cost more than row hits", gconflict - gopen); if (g_errors != 0) { diff --git a/third_party/Makefile b/third_party/Makefile index b9e5d9f37f..a32ff0ff30 100644 --- a/third_party/Makefile +++ b/third_party/Makefile @@ -8,20 +8,17 @@ softfloat: ramulator/libramulator.so: cd ramulator && mkdir -p build && cd build && cmake .. && make -j4 -# Vortex's Ramulator extensions: a per-channel refresh manager, and the GDDR6 -# device model Ramulator ships but does not build. Both register with +# The GDDR6 device model Ramulator ships but does not build. It registers with # Ramulator's factory when the library loads, built as Ramulator builds itself. RAMULATOR_EXT_DIR := ramulator_ext RAMULATOR_EXT_INCS := -isystem ramulator/src -isystem ramulator/ext/spdlog/include -isystem ramulator/ext/yaml-cpp/include RAMULATOR_EXT_CXXFLAGS := -std=c++20 -O3 -DNDEBUG -fPIC $(RAMULATOR_EXT_INCS) -$(RAMULATOR_EXT_DIR)/channel_refresh.o: $(RAMULATOR_EXT_DIR)/channel_refresh.cpp - $(CXX) $(RAMULATOR_EXT_CXXFLAGS) -Wall -Wextra -Werror -c $< -o $@ - $(RAMULATOR_EXT_DIR)/gddr6.o: ramulator/src/dram/impl/GDDR6.cpp + mkdir -p $(RAMULATOR_EXT_DIR) $(CXX) $(RAMULATOR_EXT_CXXFLAGS) -c $< -o $@ -$(RAMULATOR_EXT_DIR)/libramulator-vortex.so: $(RAMULATOR_EXT_DIR)/channel_refresh.o $(RAMULATOR_EXT_DIR)/gddr6.o ramulator/libramulator.so +$(RAMULATOR_EXT_DIR)/libramulator-vortex.so: $(RAMULATOR_EXT_DIR)/gddr6.o ramulator/libramulator.so $(CXX) -shared $(filter %.o,$^) -Lramulator -lramulator -o $@ ramulator: ramulator/libramulator.so $(RAMULATOR_EXT_DIR)/libramulator-vortex.so diff --git a/third_party/ramulator_ext/channel_refresh.cpp b/third_party/ramulator_ext/channel_refresh.cpp deleted file mode 100644 index 3888669fe9..0000000000 --- a/third_party/ramulator_ext/channel_refresh.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "base/base.h" -#include "dram_controller/controller.h" -#include "dram_controller/refresh.h" - -namespace Ramulator { - -// All-bank refresh every nREFI, one REFab per channel, the channels staggered -// across the interval as independent controllers are. Ramulator's AllBank -// manager refreshes per rank, and HBM2 and GDDR6 have no rank level, so with it -// these devices never refresh. -class ChannelRefresh : public IRefreshManager, public Implementation { - RAMULATOR_REGISTER_IMPLEMENTATION(IRefreshManager, ChannelRefresh, "ChannelRefresh", "All-bank refresh of a rank-less channel.") -private: - Clk_t clk_ = 0; - IDRAMController* ctrl_ = nullptr; - int num_levels_ = -1; - int nrefi_ = -1; - int req_id_ = -1; - Clk_t next_refresh_ = -1; - -public: - void init() override { - ctrl_ = cast_parent(); - } - - void setup(IFrontEnd* /*frontend*/, IMemorySystem* /*memory_system*/) override { - auto dram = ctrl_->m_dram; - num_levels_ = int(dram->m_levels.size()); - nrefi_ = dram->m_timing_vals("nREFI"); - req_id_ = dram->m_requests("all-bank-refresh"); - int num_channels = dram->get_level_size("channel"); - next_refresh_ = nrefi_ + Clk_t(ctrl_->m_channel_id) * nrefi_ / num_channels; - } - - void tick() override { - if (++clk_ != next_refresh_) { - return; - } - next_refresh_ += nrefi_; - std::vector addr_vec(num_levels_, -1); - addr_vec[0] = ctrl_->m_channel_id; - Request req(addr_vec, req_id_); - if (!ctrl_->priority_send(req)) { - throw std::runtime_error("Failed to send refresh!"); - } - } -}; - -} From 03efecbaf3017cb53c95dbd74897dfa666415f15 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 12 Oct 2026 02:24:49 -0700 Subject: [PATCH 027/336] simx: issue warps with the RTL's greedy round-robin arbiter SimX picked the next warp greedy-then-oldest, with ages reset whenever a warp stopped requesting and ties going to the lowest warp id; the RTL's scoreboard arbiter (VX_scoreboard out_arb, round-robin, STICKY) keeps the current warp while it is ready and then passes to the next one in index order. On sgemm_tcu fp16 (NT=8) RTL warps run in lockstep, loading and waiting on memory together, while SimX warps drift apart (17/17/16/14 blocks per warp against 16 each) and overlap their memory waits. The gap grew with memory latency and reached 6.9% (tensor:model_parity-fp16, 5% tolerance) once the DRAM clock followed the core clock. SimX now uses a greedy round-robin arbiter matching VX_rr_arbiter with STICKY=1. model_parity-fp16: simx 39747 vs rtlsim 39221 cycles (+1.3%, was -6.9%); at 2000 MT/s +1.75% (was -3.8%); sgemm parity unchanged within 0.3%. Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 2 +- sim/simx/types.h | 48 ++++++++++++++--------------------------------- 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index f67dba6e23..ec729d83e3 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -69,7 +69,7 @@ class Core::Impl { , fetch_latch_(ctx, "fetch_latch", 2, 2) , decode_latch_(ctx, "decode_latch", 1, 2) , pending_icache_(VX_CFG_NUM_WARPS) - , ibuffer_arbs_(VX_CFG_ISSUE_WIDTH, {ArbiterType::GTO, PER_ISSUE_WARPS}) + , ibuffer_arbs_(VX_CFG_ISSUE_WIDTH, {ArbiterType::GreedyRR, PER_ISSUE_WARPS}) , issue_lock_owner_(VX_CFG_ISSUE_WIDTH, kLockOpen) , fu_credits_(VX_CFG_ISSUE_WIDTH, std::vector((uint32_t)FUType::Count, 0)) , fu_full_(VX_CFG_ISSUE_WIDTH, BitVector<>((uint32_t)FUType::Count, 0)) diff --git a/sim/simx/types.h b/sim/simx/types.h index 6423d1a61a..1658771c6e 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -895,7 +895,7 @@ enum class ArbiterType { Priority, RoundRobin, Matrix, - GTO + GreedyRR }; inline std::ostream &operator<<(std::ostream &os, const ArbiterType& type) { @@ -903,7 +903,7 @@ inline std::ostream &operator<<(std::ostream &os, const ArbiterType& type) { case ArbiterType::Priority: os << "Priority"; break; case ArbiterType::RoundRobin: os << "RoundRobin"; break; case ArbiterType::Matrix: os << "Matrix"; break; - case ArbiterType::GTO: os << "GTO"; break; + case ArbiterType::GreedyRR: os << "GreedyRR"; break; default: assert(false); } return os; @@ -1017,56 +1017,36 @@ class MatrixArbiter : public IArbiterImpl { std::vector> priority_matrix_; }; -class GTOArbiter : public IArbiterImpl { +// Greedy round-robin: keeps granting the current owner while it requests, +// then hands over to the next requester after it in index order. +class GreedyRRArbiter : public IArbiterImpl { public: - GTOArbiter(uint32_t size) - : size_(size) - , age_(size, 0) - , last_grant_(-1u) { + GreedyRRArbiter(uint32_t size) : size_(size) { this->reset(); } uint32_t grant(const BitVector<>& requests) override { assert(requests.size() == size_); - // greedy: keep granting same requester if still active if (last_grant_ < size_ && requests.test(last_grant_)) { - this->update_ages(requests, last_grant_); return last_grant_; } - // Then-Oldest: find the requester with the highest age - uint32_t best = -1u; - uint32_t best_age = 0; + uint32_t start = (last_grant_ < size_) ? (last_grant_ + 1) % size_ : 0; for (uint32_t i = 0; i < size_; ++i) { - if (requests.test(i) && (best == -1u || age_[i] > best_age)) { - best = i; - best_age = age_[i]; + uint32_t idx = (start + i) % size_; + if (requests.test(idx)) { + last_grant_ = idx; + return idx; } } - if (best != -1u) { - last_grant_ = best; - this->update_ages(requests, best); - } - return best; + return -1u; } void reset() override { - std::fill(age_.begin(), age_.end(), 0); last_grant_ = -1u; } private: - void update_ages(const BitVector<>& requests, uint32_t granted) { - for (uint32_t i = 0; i < size_; ++i) { - if (i == granted || !requests.test(i)) { - age_[i] = 0; - } else { - ++age_[i]; - } - } - } - uint32_t size_; - std::vector age_; uint32_t last_grant_; }; @@ -1083,8 +1063,8 @@ class Arbiter { case ArbiterType::Matrix: impl_ = std::make_shared(size); break; - case ArbiterType::GTO: - impl_ = std::make_shared(size); + case ArbiterType::GreedyRR: + impl_ = std::make_shared(size); break; default: assert(false); From 72b574b0af7246823e9b679397361e4fa6555f50 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 12 Oct 2026 02:24:50 -0700 Subject: [PATCH 028/336] simx: let a cache bank pipeline hold one request per stage The bank pipeline is a timed FIFO whose delay is the configured latency, created with the default capacity of two, so a bank accepted only two requests per LATENCY cycles: 0.5 per cycle at the stock L2 latency of 4, far less with large caches. The RTL bank (PIPELINE_STAGES = LATENCY) accepts one request per cycle at any depth. Size the FIFO to the latency (at least two). Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/cache.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 41ed3b411a..8c4dcfc0a8 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -18,6 +18,7 @@ #if VX_CFG_EXT_A_ENABLED #include "amo_unit.h" #endif +#include #include #include #include @@ -649,7 +650,7 @@ class CacheBank : public SimObject { const Cache::Config &config, const params_t ¶ms, uint32_t bank_id) - : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency)), rand_ctr_(0), adm_seq_ctr_(0) + : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), rand_ctr_(0), adm_seq_ctr_(0) #if VX_CFG_EXT_A_ENABLED , amo_unit_(__MAX(2u, (uint32_t)VX_CFG_AMO_RS_SIZE)) #endif From c068cea4db39199be4791bcdee63bbb607854fcd Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 12 Oct 2026 02:24:51 -0700 Subject: [PATCH 029/336] ci: re-record the rtlsim perf-gate baselines for the DRAM model Regenerated with pytest ci -m perf_gate --update-baselines (49/49) after the DRAM model's data-sheet configuration: the default DRAM is now clocked with the core instead of at 1 GHz, with JEDEC timing, 32 B bursts and permuted bank interleave, so memory-bound cases moved. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 64 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 44 ++++++++++----------- ci/baselines/perf/graphics.json | 38 +++++++++--------- ci/baselines/perf/raytracing.json | 12 +++--- ci/baselines/perf/tensor.json | 20 +++++----- ci/baselines/perf/tensor_mx.json | 4 +- ci/baselines/perf/tensor_sp.json | 4 +- ci/baselines/perf/tensor_wg.json | 12 +++--- 8 files changed, 99 insertions(+), 99 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index b08f98ad35..77e8c4785a 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 562137, + "cycles": 563524, "instrs": 137486 }, "app": "raycast", "args": "", - "config_hash": "314d6d30965b89c4", + "config_hash": "396cee9df6025cdb", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 1495556, + "cycles": 1498642, "instrs": 259228 }, "app": "raycast", "args": "", - "config_hash": "909d80cfbaadadc2", + "config_hash": "262f24032ef14b58", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 779835, + "cycles": 787850, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "92050636b69caeb2", + "config_hash": "ad2b37492d761e8f", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2228982, + "cycles": 2239426, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "72b6506beadbe8a0", + "config_hash": "606d011dc6f612d3", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1909825, + "cycles": 1838654, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "514f489ab80d25c0", + "config_hash": "9390754d3719fca3", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 6189932, + "cycles": 5939731, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "4af79268b2c7557f", + "config_hash": "740fd84fa2938103", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1261549, + "cycles": 1290130, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "d53587fae488c32b", + "config_hash": "2e1c44e05f9ac34d", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4181334, + "cycles": 4219466, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "572494ba2df6f58a", + "config_hash": "024adb49ed96bdd3", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 583960, + "cycles": 685107, "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "587ac34faae9f341", + "config_hash": "a3d9df9695acbb95", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 931381, + "cycles": 872926, "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "93f8522bd055dda2", + "config_hash": "0e8990b25e64ad7f", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1273285, + "cycles": 1272929, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "9c8c364608242127", + "config_hash": "9e0c1c9767c760ea", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4861515, + "cycles": 4860309, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "b2c24fbda6abd30b", + "config_hash": "eb5fed4bfce19d76", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2395091, + "cycles": 2306960, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "fec6d70d6e8e83c3", + "config_hash": "6925a5138be930ea", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4283200, + "cycles": 4180969, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "bb64a7573d2c2365", + "config_hash": "b83a1e3735f79d39", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 99638, + "cycles": 100640, "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "58a51468cf8f99e7", + "config_hash": "ef0c2548e7c5bf86", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 363323, + "cycles": 367765, "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "4ca729a11fbf88a3", + "config_hash": "ce1eb79984a98552", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index 453b0775f5..bf1e1a010c 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -1,122 +1,122 @@ { "dxa:perf_gate-copy-mcast:rtlsim": { "32": { - "cycles": 3313, + "cycles": 3381, "instrs": 961 }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "2df97b749a0febe3", + "config_hash": "6a815f855f6e0d05", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 1869, + "cycles": 1994, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "59a6c715ed820634", + "config_hash": "626c5f6244584b1f", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt4:rtlsim": { "32": { - "cycles": 6773, + "cycles": 6827, "instrs": 1792 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "534d84e7f493e414", + "config_hash": "7c32a11d123bfb13", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14056427, + "cycles": 14022240, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "589585790379eee0", + "config_hash": "822d0ca194266a27", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14108810, + "cycles": 14063502, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "2879ae8c33e35bb8", + "config_hash": "87ea544e77719254", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3895374, + "cycles": 3941267, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "f3759c20ec885c91", + "config_hash": "e55b61598f5f10ea", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14130928, + "cycles": 14352313, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "aa3ad14268d9b8b5", + "config_hash": "21da2f2737bf03dd", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 79024, + "cycles": 82160, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "7080653c22396eb6", + "config_hash": "20304652a1bfb94c", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 348356, + "cycles": 360010, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "401d267a03887e99", + "config_hash": "bf4520d008c07dc9", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 34417, + "cycles": 36021, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "3117dd957603b2e8", + "config_hash": "2a4e017d98740636", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 55358, + "cycles": 59789, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "5bec3cc54a0c63f1", + "config_hash": "f05069d7cf0ee3ff", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index f4f9eb603d..b5ee86bfc2 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,100 +1,100 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 529292, + "cycles": 530195, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "ae1062062a26ef92", + "config_hash": "1e2937b20579ad3f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 142905, - "instrs": 51650 + "cycles": 143620, + "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "e397892c0cb72ec2", + "config_hash": "edc924bc4f246971", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 528252, + "cycles": 529393, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "15664f7fbe13f120", + "config_hash": "d682cc8c6f99c06b", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 188898, + "cycles": 189231, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "b2ccda0f4e3d0471", + "config_hash": "46a66220c1d9956b", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 740590, + "cycles": 740701, "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "e542501f5bf6269e", + "config_hash": "a7554f6fa88474dc", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14865, + "cycles": 14785, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "7272f5969f7ee30d", + "config_hash": "e2e1879d7d6f9aaf", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55526, + "cycles": 55565, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "05e09632b1bbf2b7", + "config_hash": "6e48f6b249533cc5", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 35342, + "cycles": 35441, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "9669b4d06963f72d", + "config_hash": "c8c6b30537e93e2f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 138329, + "cycles": 136487, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "aee6b6136e6a4575", + "config_hash": "a4fc4d97e6e80586", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index bda30910c0..1fb0ab7f7b 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,34 +1,34 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 84378, + "cycles": 85045, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "937f99317c42a59f", + "config_hash": "9a4f28d2555289b5", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53283, + "cycles": 53434, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "dfd33a99196e04ad", + "config_hash": "aef578263250d585", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt4:rtlsim": { "32": { - "cycles": 159558, + "cycles": 159655, "instrs": 39330 }, "app": "rt_raycast", "args": "", - "config_hash": "c17be24af5bd78b9", + "config_hash": "c7b45fe5c1828bc5", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index fae07189e9..3552c4620b 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 191642, + "cycles": 197465, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "d21f5479c2459776", + "config_hash": "da0a50e46103b7dc", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 42941, + "cycles": 42404, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "23f4b1de5a361780", + "config_hash": "2d12f1a0058fcc3b", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221484, + "cycles": 221592, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "c4461f9b2a2b71d7", + "config_hash": "fce886092e60e0eb", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 253809, + "cycles": 256743, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "9c6dfe0c35a740a9", + "config_hash": "f0558a3b3dfde51f", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 594027, + "cycles": 590798, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "9c8bbf01434d9d8d", + "config_hash": "24fd7a811fabb8c4", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 1fbd356b34..1dc1b78f41 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 76601, + "cycles": 76393, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "46d80398194a01e4", + "config_hash": "3cfc301023342a7d", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 47e3a2926a..2c4f92004c 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 184985, + "cycles": 183094, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "cfe93fc4d3413031", + "config_hash": "90defe3d20b6f739", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 57e02b2b8f..c7f771c72f 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 158687, + "cycles": 158181, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "f019a0cb823990c9", + "config_hash": "ba81956fe68e94b6", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 836986, + "cycles": 834873, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "7b6f63cb95491231", + "config_hash": "6ee14f25ec573117", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 398222, + "cycles": 400023, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "f620d8a9b0396d1e", + "config_hash": "a9ea3d8bd83a7eed", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From 046ff33b0fe44ff67d0901ed1089bfbcde84faea Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 13 Oct 2026 16:46:30 -0700 Subject: [PATCH 030/336] Flush caches with one broadcast write and per-port flush tokens The kernel-boundary flush had three costs and one race: - the command processor flushed the cores one after another, each with a separate read that started and awaited that core's flush; - every cache forwarded its flush through the shared levels, including the read-only icache, so each L1 walked L2/L3 again; - a cache forwarded a single token on memory port 0, so writes queued on another port could reach the next level after its walk and stay dirty (reproduced with random input backpressure; the software command processor's long turnaround hides it in plain rtlsim). Now: - The CP starts every core with one broadcast DCR write to CACHE_FLUSH, then reads the cores in turn. VX_dcr_data tracks the flush with pending/complete/read state, so the flushes overlap and the responses, which have no back-pressure, never collide. A flush read with no write before it is no longer a way to start one. - VX_cache_init answers flush tokens locally: walk all banks, then (with FLUSH_FORWARD) send one token per memory port once the request crossbar has drained, wait for every port's answer, and answer the requesters. Token responses are recognized by phase; answers take bank 0's lane into the response crossbar. Boundary nets stay flop-driven. - Read-only caches (icache, tcache, rtcache, rcache) set FLUSH_FORWARD=0: they hold no dirty data, and the dcache's flush walks the shared levels. - Every independently buffered write path carries its own token: every dcache lane unless the core has a private dcache (store_drained covers that case), and every OM input of the ocache. - OM aperture stores are invisible to busy between the dcache and the OM ingress, so VX_om_steer now holds a flush token bound for L2 until a VX_graphics controller has drained the OM and flushed the ocache. A half-received colour+depth record is not waited for, so a fence in a fragment shader cannot deadlock on a beat queued behind the held token. SimX already flushes the whole hierarchy functionally and does not time the flush; this brings the RTL's kernel-boundary cost toward that model (model_parity and perf_gate unchanged). Tests: new tests/regression/flush_coherence (rotating writes plus a tail store burst, checked every round). CI adds flush_coherence-{1,2,3} and DBG_FLUSH_STALL stress cases (flush_coherence-stress-{1,2,3}, graphics:draw3d-flush-stress-{1,2}); DBG_FLUSH_STALL adds random input backpressure in VX_cache_init. Under stress, the single-token design, port-0-only tokens and lane-0-only tokens all fail. cp_dcr_proxy unittest covers the broadcast flush over 3 and 0 cores. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/graphics.yaml | 23 ++- ci/testcases/regression.yaml | 72 +++++++ docs/assets/img/cache_flush_sequence.svg | 175 ++++++++-------- docs/assets/img/cache_flush_tree.svg | 187 ++++++++---------- docs/designs/cache_subsystem.md | 158 +++++++++------ docs/designs/command_processor.md | 15 +- docs/designs/lsu_pipeline_design.md | 2 +- docs/designs/output_merger_architecture.md | 18 +- hw/rtl/VX_cluster.sv | 4 +- hw/rtl/VX_gpu_pkg.sv | 5 +- hw/rtl/VX_graphics.sv | 143 +++++++++++--- hw/rtl/VX_socket.sv | 6 + hw/rtl/cache/VX_cache.sv | 154 +++++++++++++-- hw/rtl/cache/VX_cache_bank.sv | 1 - hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_flush.sv | 8 +- hw/rtl/cache/VX_cache_init.sv | 209 ++++++++++++++------ hw/rtl/cache/VX_cache_wrap.sv | 2 + hw/rtl/core/VX_core.sv | 9 +- hw/rtl/core/VX_dcr_data.sv | 50 +++-- hw/rtl/core/VX_dcr_flush.sv | 2 +- hw/rtl/core/VX_mem_unit.sv | 35 +++- hw/rtl/cp/VX_cp_dcr_proxy.sv | 33 ++-- hw/rtl/om/VX_om_ingress.sv | 7 +- hw/rtl/om/VX_om_steer.sv | 46 ++++- hw/unittest/cp_dcr_proxy/main.cpp | 65 +++++- hw/unittest/gfx/VX_gfx_top.sv | 2 + hw/unittest/vm/VX_vm_top.sv | 1 + sim/common/cmd_processor.cpp | 33 ++-- sim/common/cmd_processor.h | 2 + sim/rtlsim/main.cpp | 1 + sim/rtlsim/processor.cpp | 4 +- sim/simx/main.cpp | 1 + sw/runtime/rtlsim/vortex.cpp | 5 + sw/runtime/simx/vortex.cpp | 4 + tests/regression/Makefile | 1 + tests/regression/flush_coherence/Makefile | 16 ++ tests/regression/flush_coherence/common.h | 21 ++ tests/regression/flush_coherence/kernel.cpp | 28 +++ tests/regression/flush_coherence/main.cpp | 190 ++++++++++++++++++ 40 files changed, 1301 insertions(+), 439 deletions(-) create mode 100644 tests/regression/flush_coherence/Makefile create mode 100644 tests/regression/flush_coherence/common.h create mode 100644 tests/regression/flush_coherence/kernel.cpp create mode 100644 tests/regression/flush_coherence/main.cpp diff --git a/ci/testcases/graphics.yaml b/ci/testcases/graphics.yaml index c337b03306..252736abaf 100644 --- a/ci/testcases/graphics.yaml +++ b/ci/testcases/graphics.yaml @@ -693,4 +693,25 @@ tests: app: gfx_draw3d args: -w 128 -h 128 -t box.cgltrace -r box_ref_128.png configs: -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE - shape: {cores: 2, l2cache: true, threads: 4} \ No newline at end of file + shape: {cores: 2, l2cache: true, threads: 4} +# Flush ordering of OM aperture stores under random input backpressure +# (DBG_FLUSH_STALL): the last fragments' stores are often still in transit to +# the OM ingress when the kernel-boundary flush starts, so the flush must wait +# for them behind the dcache's flush tokens held at the OM steers. +- id: draw3d-flush-stress-1 + via: blackbox + drivers: + - rtlsim + app: gfx_draw3d + args: -w 128 -h 128 -t box.cgltrace -r box_ref_128.png + configs: -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DDBG_FLUSH_STALL + shape: {threads: 16} +- id: draw3d-flush-stress-2 + via: blackbox + drivers: + - rtlsim + app: gfx_draw3d + args: -w 128 -h 128 -t box.cgltrace -r box_ref_128.png + configs: -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DDBG_FLUSH_STALL + shape: {cores: 2, l2cache: true, threads: 4} + tier: full diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 50e45420c0..b180c479b4 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -155,3 +155,75 @@ tests: drivers: - simx app: module_reload +# Kernel-boundary flush coherence: back-to-back launches with host rewrites, +# rotating line ownership across cores and cross-launch read-modify-writes. +# The shapes cover a write-back L1, a write-back L2 shared by 4 cores, and a +# write-back L3 shared by two clusters, where every core flushes at once. +- id: flush_coherence-1 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence +# The xrt leg runs the RTL command processor's flush (broadcast write, then +# per-core reads). +- id: flush_coherence-2 + via: blackbox + drivers: + - simx + - xrt + app: flush_coherence + shape: + cores: 4 + l2cache: true +- id: flush_coherence-3 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +# Flush ordering under random input backpressure (DBG_FLUSH_STALL): each round +# ends on a store burst, so a core's last writes are often still queued on one +# memory port or dcache lane while its token on another reaches the next level. +# Without a token per path these shapes lose those writes. +- id: flush_coherence-stress-1 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r8 -b16 + configs: -DDBG_FLUSH_STALL + shape: + cores: 4 + l2cache: true +- id: flush_coherence-stress-2 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r8 -b16 + configs: -DDBG_FLUSH_STALL + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true + tier: full +# Cores sharing a dcache: a lost lane store is only visible when it belongs to +# the core answered last, so this shape needs more rounds to expose it. +- id: flush_coherence-stress-3 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r24 -b16 + configs: -DDBG_FLUSH_STALL -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 4 + l2cache: true + tier: full diff --git a/docs/assets/img/cache_flush_sequence.svg b/docs/assets/img/cache_flush_sequence.svg index 780747378b..843f55ec28 100644 --- a/docs/assets/img/cache_flush_sequence.svg +++ b/docs/assets/img/cache_flush_sequence.svg @@ -1,4 +1,4 @@ - + @@ -22,6 +22,7 @@ .cm { font-family:"DejaVu Sans Mono", monospace; font-size:10px; fill:#1b2733; } .act { fill:#e8f0f7; stroke:#2e5e82; stroke-width:1.3; } .actg { fill:#f4faf4; stroke:#3a7d44; stroke-width:1.3; } + .acty { fill:#fdf6e3; stroke:#b58900; stroke-width:1.3; } .note { fill:#fbfcfd; stroke:#5a7184; stroke-width:1.2; } .loop { fill:none; stroke:#a03020; stroke-width:1.5; stroke-dasharray:7 4; } .loopl{ fill:#fbeeea; stroke:#a03020; stroke-width:1.3; } @@ -37,52 +38,51 @@ .lbr { fill:#a03020; font-size:11px; font-weight:bold; } - - Cache flush — DCR read to last writeback drained - One flush primitive exists: whole-cache invalidate-and-writeback. It is triggered by a DCR READ, carried as an ordinary MemReq with attr.is_flush, and it freezes the cache for its whole sweep. + + Cache flush — broadcast write to the last core's completion read + One flush primitive exists: whole-cache invalidate-and-writeback, requested by a flush token (a MemReq with attr.is_flush). Every ordered path carries its own token, and a cache answers a token only after flushing the next level. host / CP VX_dcr_data - VX_dcr_flush + VX_dcr_flush (per lane) VX_cache_init VX_cache_flush (per bank) next level - - - - - - + + + + + + - dcr_read(VX_DCR_BASE_CACHE_FLUSH, core_id) + dcr_write(VX_DCR_BASE_CACHE_FLUSH) - latch flush_pending_r - a DCR READ, not a write — the read's - response is what acknowledges the flush + every core: flush_pending_r = 1 + a write reaches all cores at once and has no + response, so all cores flush concurrently - dcr_flush_if.req = 1 (level-held) + dcr_flush_if.req = 1 (level-held, once stores drain) - - synthesize a degenerate MemReq: + a flush token per buffered dcache lane: rw=0, addr=0, data=0, byteen=0 attr = 1 << MEM_ATTR_FLUSH_OFFS - the flush rides port 0 as ordinary cache traffic + each token behind its own lane's stores - LOCK EVERY INPUT PORT + BLOCK EVERY INPUT input_enable = ~flush_req_enable - || lock_released[i] - valid=0 downstream, ready=0 upstream + while any token is visible at any input: + valid=0 downstream, ready=0 upstream STATE_WAIT1: drain the @@ -115,74 +115,89 @@ write-through: valid cleared, zero writebacks ever - writeback - - one dirty sector - per beat + writeback (on its bank's port) flush_ready gates the counter, so the walk pauses whenever the memory-request queue is near full. A write-through bank never gates on it — it emits no writebacks at all. - - STATE_WAIT2 — every bank EXCEPT bank 0 - Banks 1..N−1 wait for bank_empty. Bank 0 skips to - DONE, so its request — the one that propagates the - flush downstream — is the last to be issued. + + STATE_WAIT2 — every bank + waits for bank_empty: its write-backs must leave + the bank before the tokens go out behind them + + + flush_end — one pulse per bank + + + STATE_WAIT2 done: the tokens visible NOW are + this walk's; a later token starts the next walk + + + + STATE_FORWARD (FLUSH_FORWARD=1) + wait mem_req_empty: the crossbar has drained + + + one flush token on EVERY memory port, behind that port's write-backs - - flush_end — one pulse per bank + + same sequence one level down: one walk + for every core's tokens, then answers + each port's token - - STATE_WAIT2: accumulate until - flush_done == {NUM_BANKS{1'b1}} + + one response per port — recognized by phase (cache quiescent) - - release ONLY the originating port; its MemReq acks + + + STATE_RESPOND: each token answered locally - - dcr_flush_if.done — LEVEL-HELD until req drops + + done — all lanes answered; LEVEL-HELD - - DCR response — the flush read finally returns + + dcr_read(CACHE_FLUSH, cid), one core at a time + + answered when that core is done (at once if already) - - - - Core traffic has arbitration PRIORITY - The 2:1 arb puts core_bus_if on input 0, not the - flush. The flush waits until the LSU goes quiet — - which it does, because the host only issues a flush - after busy drops. STICKY hardens the 1-cycle gaps. - - - Why done is level-held rather than a pulse - One core's flush fans out to the dcache, the icache, and — with graphics enabled — the - socket and cluster gfx caches. Each leg finishes at a different time. Holding every - done high until req drops lets the parent simply AND them together, race-free. - A pulse would need a latch at every junction and would drop legs that finished early. - - - What the lock buys - Once locked, no core request can reach any bank pipeline. That single fact is what lets - STATE_FLUSH assume the only in-flight entries are its own walk, and it is why WAIT2's - bank_empty check need not re-test mshr_empty — nothing can refill the MSHR. - The cost: the cache is effectively OFF for the whole walk. Every other warp's loads and - stores stall. Flush is linear in cache size and is meant to be infrequent. - - - Walk length - write-through: LINES_PER_BANK - write-back: LINES_PER_BANK x NUM_WAYS - Write-through clears all ways in parallel (no way_en gate), so one pass over the lines - suffices. Write-back must visit each (way, line) separately because each dirty line - emits its own memory request. All banks walk concurrently. - - - Reset reuses the same machinery - The FSM's reset state is STATE_INIT, not STATE_IDLE: it walks every line clearing valid - bits, so the tag SRAM needs no async reset — which is what makes it map cleanly to BRAM. - A flush arriving during init is latched and answered immediately: init already invalidated - everything, so the init walk stands in for the flush walk. + + Why one token per memory port + Each port is a separately buffered path to the next level, and nothing orders one port's + writes against another port's token. A single token could reach the next level while another + port's last writes are still queued in front of it: that level's walk would release the token + and the writes would land after the walk, still dirty. A token behind each port's own data + is answered only once that data is in. The same rule gives one token per dcache lane. + The no-ack equivalent of waiting for store acknowledgements on every path. + + + Why done is level-held rather than a pulse + One core's flush fans out to every dcache lane, the icache, and — with graphics enabled — the + socket and cluster gfx caches. Each leg finishes at a different time. Holding every + done high until req drops lets the parent simply AND them together, race-free. + A pulse would need a latch at every junction and would drop legs that finished early. + + + What the block buys + While any token is visible, no core request can reach any bank pipeline. That is what lets + STATE_FLUSH assume the only in-flight entries are its own walk, makes every memory response + in STATE_FORWARD a token response, and makes a late token safe: it can only become visible + once all of its input's earlier requests were accepted, before the walk. + The cost: the cache is effectively OFF for the walk. Flush is linear in cache size. + + + Walk length + write-through: LINES_PER_BANK + write-back: LINES_PER_BANK x NUM_WAYS + Write-through clears all ways in parallel (no way_en gate), so one pass over the lines + suffices. Write-back must visit each (way, line) separately because each dirty line + emits its own memory request. All banks walk concurrently. + + + Reset reuses the same machinery + The FSM's reset state is STATE_INIT, not STATE_IDLE: it walks every line clearing valid + bits, so the tag SRAM needs no async reset — which is what makes it map cleanly to BRAM. + A flush arriving during init is latched and answered immediately: init already invalidated + everything, so the init walk stands in for the flush walk. diff --git a/docs/assets/img/cache_flush_tree.svg b/docs/assets/img/cache_flush_tree.svg index 8da61c02cd..aac49bd251 100644 --- a/docs/assets/img/cache_flush_tree.svg +++ b/docs/assets/img/cache_flush_tree.svg @@ -1,4 +1,4 @@ - + @@ -6,9 +6,6 @@ - - - - The flush fan-out tree — one DCR read, every cache + The flush fan-out tree — one broadcast write, every cache Solid blue = req fans DOWN. Dashed green = done ANDs back UP. Every done is level-held until req drops, which is what makes the AND race-free. - - VX_dcr_data - dcr_read(CACHE_FLUSH, cid) - holds req until done comes back + + VX_dcr_data (per core) + dcr_write(CACHE_FLUSH): all cores start + dcr_read(CACHE_FLUSH, cid): report one + holds req until done comes back - + - + VX_core — the fan-out point dcr_flush_if - - - req + + + req && store_drained - - VX_dcr_flush (dcache) - injects the synthetic MemReq onto - dcache port 0 only - VX_mem_unit.sv — i==0 && j==0 + + VX_dcr_flush per buffered lane + lane 0 of a private dcache, else every + lane. Lane dones AND together + VX_mem_unit.sv - - dcache - the lock freezes ALL its ports - - - - ORDERING CONSTRAINT - icache.req = dcr_flush_if.req - && dcache.done - The icache flush is SEQUENCED - BEHIND the dcache flush. - - - - - - - VX_dcr_flush (icache) - VX_core.sv — between fetch and iMMU - - - - icache - no dirty data — retires fast - - - Why the icache waits for the dcache - Both L1s forward their flush to the shared next level, and a cache that is - flushing locks out incoming core requests for its whole sweep. - The icache carries no dirty data, so it retires almost immediately. If it ran - concurrently, its forward would arrive at the next level while the dcache is - still evicting — and that level's lock would strand the dcache's writebacks - upstream of memory. - - - Why the icache is flushed at all - A kernel re-loaded to the same virtual address after a CACHE_FLUSH would - otherwise execute stale lines from the previous launch. The host issues - CMD_CACHE_FLUSH after every kernel, so this is the instruction-side half - of that contract. + + dcache + walks, then one token per + memory port to the next level + + + + req (concurrent with the dcache) + + + VX_dcr_flush (icache) + VX_core.sv — between fetch and iMMU + + + + icache (FLUSH_FORWARD=0) + invalidates locally; no forward + + + + Why a token per write path + A token only orders the requests ahead of it on its own path. Separate paths + into a cache can be buffered independently (a shared dcache's arbiter buffers + each lane), so a store on one lane could still be queued when another lane's + token arrives. The same rule gives one token per memory port between levels. + + + Why the icache flushes, and why only locally + A kernel re-loaded to the same virtual address would otherwise execute stale + lines from the previous launch. The icache holds no dirty data, and the dcache's + flush already walks and invalidates every shared level the icache reads through, + so the icache's flush ends at the icache. - - EXT_GFX_ANY_ENABLE only + + EXT_GFX_ANY_ENABLE only VX_socket — OR of core reqs socket_flush_req = |per_core_req - + tcache (TEX) - + rtcache (RTU) - + VX_cluster — OR of socket reqs cluster_flush_if.req - - rcache (raster) - - ocache (output merger) - - Sockets fire their reqs - sequentially; the shared - resource flushes once per - req. Later reqs see an - already-empty cache, so the - redundant sweeps are cheap, - not incorrect. - - rcache/ocache are cluster- - level, inside VX_graphics; - tcache/rtcache are socket- - level. + + rcache (raster, 1 token) + + ocache (1 token / OM input) + + tcache, rtcache and rcache + flush once for all cores; + the OR drops when the + last core finishes. + The ocache flushes once the + OM has drained, for the + cluster flush and for each + dcache token a steer holds + on its way to the L2. + tcache/rtcache: socket; + rcache/ocache: cluster. - - The completion AND tree + + The completion AND tree - VX_cluster: per_socket.done = cluster_flush_if.done - VX_socket: per_core.done = cluster_flush_if.done - & tcache.done - & rtcache.done - VX_core: dcr_flush_if.done = dcache.done - & icache.done - & cluster_flush_if.done + VX_cluster: per_socket.done = cluster_flush_if.done + VX_socket: per_core.done = cluster_flush_if.done + & tcache.done + & rtcache.done + VX_core: dcr_flush_if.done = dcache.done + & icache.done + & cluster_flush_if.done + VX_mem_unit: dcache.done = &lane_flush_done - Only when every leg reports done does VX_dcr_data return the DCR read - and drop req — which re-arms every VX_dcr_flush for the next flush. + When every leg reports done, VX_dcr_data drops req (re-arming every + VX_dcr_flush) and answers this core's flush read, at once if it came first. - - - done + + + done - - + + diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 0ed3e50e98..73f7e308f1 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -511,11 +511,22 @@ The same machinery is reused for reset-time tag initialization. ![Flush fan-out tree](../assets/img/cache_flush_tree.svg) -The flush is triggered by a DCR **read** of `VX_DCR_BASE_CACHE_FLUSH` — a read, -not a write, because the read's *response* is what acknowledges completion. -[VX_dcr_data.sv](../../hw/rtl/core/VX_dcr_data.sv) latches `flush_pending_r`, -holds `dcr_flush_if.req` high, and returns the DCR response only once `done` -comes back. +The command processor flushes in two phases, because the DCR response bus has +no back-pressure and two cores answering in the same cycle would lose one: + +1. one DCR **write** to `VX_DCR_BASE_CACHE_FLUSH`. Writes reach every core, so + all cores start flushing together; a write has no response; +2. a DCR **read** of the same address per core, one at a time. The read's + *response* acknowledges that core's completion, at once if it has already + finished. + +[VX_dcr_data.sv](../../hw/rtl/core/VX_dcr_data.sv) keeps three bits per core: +`flush_pending_r` (running; holds `dcr_flush_if.req` high), `flush_complete_r` +(finished, not yet reported) and `flush_read_r` (a read waiting). A write opens +a new flush and clears `flush_complete_r`, so an older completion cannot answer +it. A read with no write before it starts the flush itself, so a requester that +only reads still works. All cores' tokens reach the shared levels together, +where one walk serves them all. From `VX_core` the request fans out to six possible destinations through six `VX_dcr_flush` instances — dcache, icache, tcache, rtcache, rcache, ocache — @@ -532,18 +543,17 @@ holding every `done` high until `req` drops is what lets each parent simply AND them together race-free. A pulse would need a latch at every junction and would drop legs that finished early. -**The icache flush is sequenced behind the dcache flush:** +**The icache flush stays local** (`FLUSH_FORWARD=0`): it holds no dirty data, +and the dcache's flush already walks and invalidates every shared level the +icache reads through. So the two L1s flush concurrently, and only the dcache's +tokens reach the next level. + +The dcache flush waits for the core's stores to leave the LSU: ```systemverilog -assign dcr_flush_icache_if.req = dcr_flush_if.req && dcr_flush_dcache_if.done; +assign dcr_flush_dcache_if.req = dcr_flush_if.req && store_drained; ``` -Both L1s forward their flush to the shared next level, and a cache that is -flushing locks out incoming core requests for its whole sweep. The icache carries -no dirty data and so retires almost immediately; if it ran concurrently, its -forward would arrive at the next level while the dcache is still evicting, and -that level's lock would strand the dcache's writebacks upstream of memory. - The icache is flushed at all because a kernel re-loaded to the same virtual address after a `CACHE_FLUSH` would otherwise execute stale lines from the previous launch. The host issues `CMD_CACHE_FLUSH` after every kernel launch — see @@ -552,8 +562,8 @@ previous launch. The host issues `CMD_CACHE_FLUSH` after every kernel launch — ### 6.2 VX_dcr_flush — injecting the request `VX_dcr_flush` synthesizes a degenerate `MemReq` (`rw=0, addr=0, data=0, -byteen=0, attr = 1 << MEM_ATTR_FLUSH_OFFS`) and merges it into the cache's port 0 -stream through a 2:1 `VX_mem_bus_arb`. +byteen=0, attr = 1 << MEM_ATTR_FLUSH_OFFS`), a **flush token**, and merges it +into one cache input stream through a 2:1 `VX_mem_bus_arb`. **Core traffic sits on the priority input, not the flush.** The flush waits until upstream is quiescent — which it reliably is, because the host only issues @@ -567,41 +577,81 @@ the first completion until the initiator drops `req`. Without that latch, a shar `req` across several instances would make the fast ones re-flush forever while waiting on the slow one. -Only **port 0** of the dcache is routed through `VX_dcr_flush` -([VX_mem_unit.sv:381](../../hw/rtl/core/VX_mem_unit.sv#L381), `i==0 && j==0`). The -input lock inside `VX_cache_init` is what propagates the freeze to every other -port. +**Every buffered write path gets its own token.** A token only orders the +requests ahead of it on its own path. A private dcache takes the lanes straight +from the LSU adapter, whose drain `store_drained` already waits for, so one +token on lane 0 suffices. Otherwise the lanes are buffered independently (a +shared dcache's socket arbiter, or a disabled dcache's passthrough), so each +lane gets its own `VX_dcr_flush` ([VX_mem_unit.sv](../../hw/rtl/core/VX_mem_unit.sv)) +and their level-held dones AND together. Each OM input of the ocache likewise has its own token +([VX_graphics.sv](../../hw/rtl/VX_graphics.sv)). The read-only rcache keeps a +single token. ### 6.3 End-to-end sequence ![Flush sequence](../assets/img/cache_flush_sequence.svg) -### 6.4 VX_cache_init — the input lock +### 6.4 VX_cache_init — the flush sequencer -`VX_cache_init` is a 5-state FSM (`IDLE → WAIT1 → FLUSH → WAIT2 → DONE`) sitting -between the cache's input ports and the request crossbar. +`VX_cache_init` sits between the cache's input ports and the request crossbar. +Its FSM is `IDLE → WAIT1 → FLUSH → WAIT2 → FORWARD → RESPOND`. ```systemverilog -wire input_enable = ~flush_req_enable || lock_released[i]; +wire input_enable = ~flush_req_enable; assign core_bus_out_if[i].req_valid = core_bus_in_if[i].req_valid && input_enable; -assign core_bus_in_if[i].req_ready = core_bus_out_if[i].req_ready && input_enable; +assign core_bus_in_if[i].req_ready = take_token || (core_bus_out_if[i].req_ready && input_enable); ``` -While a flush is in flight every input port presents `valid=0` downstream and -`ready=0` upstream, so upstream stalls. Only the port that originated the flush is -unlocked (`lock_released_n = flush_req_mask`), so its `MemReq` enters the cache and -generates the acknowledging response; the rest unlock on the return to IDLE. +While any token is visible at any input, every input presents `valid=0` +downstream and `ready=0` upstream, so upstream stalls. - `STATE_WAIT1` waits for `BANK_SEL_LATENCY × NUM_BANKS` outstanding crossbar requests to drain. When there is no crossbar latency this state is bypassed and IDLE goes straight to FLUSH. - `STATE_FLUSH` is one cycle: it pulses `flush_begin` to **all** banks at once. - `STATE_WAIT2` accumulates `flush_done |= flush_end` until every bank has - reported. - -That lock is the load-bearing piece for correctness. Once engaged, no core request -can reach any bank pipeline — which is what lets the per-bank walk assume the only -in-flight entries are its own. + reported. The tokens visible at that point are the ones this walk answers; + a token that arrives later starts the next walk. +- `STATE_FORWARD` (only with `FLUSH_FORWARD=1`) flushes the next level: once + the memory-request crossbar has drained (`mem_req_empty`), the cache sends + **one token on every memory port** and waits for every port's response. The + token for port *i* takes bank *i*'s lane into the request crossbar (bank *i* + maps to port *i*), so it leaves through the same registered outputs as the + write-backs ahead of it. +- `STATE_RESPOND` answers the tokens one per cycle. Each answer carries the + token's own tag, so upstream arbiters route it back, and takes bank 0's lane + into the response crossbar whenever bank 0 has no response of its own; the + token is consumed at its input as its answer is sent. Then the inputs reopen. + +**Why a token per memory port.** Each port is a separate, separately buffered +path to the next level, and nothing orders one port's data against another +port's token. A token sent on one port only could reach the next level while +another port's last writes are still queued in front of it; that level's walk +would release the token, and the writes would enter after the walk and stay +dirty. With one token per port, each token sits behind its own port's data in +that port's FIFO path, so the next level answers it only once that data is in. + +**Why a late token is safe.** A token becomes visible only at the head of its +input. Once any token is visible, every input is blocked, so a token that +appears during the walk had all of its input's earlier requests accepted before +the walk started. + +**Recognizing the token responses.** During `STATE_FORWARD` the cache is +quiescent: inputs blocked, MSHRs empty, no misses outstanding, and writes get no +responses. So every response on a memory port is a token response; they are +recognized by phase, with no extra tag bit. Non-cacheable requests are steered +around the cache by `VX_cache_bypass` and never reach this interface. + +**IO ordering.** `VX_cache_bypass` merges the cache's memory requests with +non-cacheable (IO) requests in a round-robin per-port arbiter, and the IO path +is unbuffered up to it. An IO request waiting there holds its own input, so the +token behind it on that input is not yet visible: a later walk answers it, and +that walk's tokens follow the IO request. A private dcache's lanes 1+ carry no +token, and `store_drained` waits for them before the core injects its token. + +**Passthrough levels** (a disabled cache) need nothing: `VX_mem_bus_arb` maps +input *i* to output *i* mod `MEM_PORTS`, so each port's stream, token included, +stays in order on one output. ### 6.5 VX_cache_flush — the per-bank walk @@ -613,7 +663,7 @@ A 6-state FSM per bank, about 14 flip-flops: | `STATE_IDLE` | wait for `flush_begin` | | `STATE_WAIT1` | wait for `mshr_empty` **and** `bank_empty` | | `STATE_FLUSH` | counter walk; drives `flush_valid`, `flush_line`, `flush_way`; increments on `flush_ready` | -| `STATE_WAIT2` | **every bank except bank 0** waits for `bank_empty` | +| `STATE_WAIT2` | wait for `bank_empty`: the write-backs must leave the bank before the tokens go out | | `STATE_DONE` | one-cycle `flush_end` pulse | **The counter width depends on the write policy:** @@ -627,18 +677,6 @@ since `do_flush = flush && (!WRITEBACK || way_en)` drops the way gate. Write-bac walks each `(way, line)` separately because each dirty line emits its own memory request. -**`STATE_WAIT2` is every bank *except* bank 0.** Bank 0 goes straight to DONE: - -```systemverilog -state_n = (BANK_ID == 0) ? STATE_DONE : STATE_WAIT2; -``` - -Bank 0 is the canonical egress that propagates the flush downstream to lower -levels, and that request must be issued **last**. The other banks therefore hold -their `flush_end` until their own writebacks have drained; since `VX_cache_init` -waits for the AND of all `flush_end`, bank 0's downstream forward cannot be -overtaken. - `flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall` gates the counter, so under write-back with a near-full egress queue the walk pauses until it drains. @@ -676,8 +714,9 @@ tag/data write ports and the existing memory-request queue. | Component | Storage | |---|---| | `VX_cache_flush` | 3 bits state, 1 bit pending, ~10 bits counter — ~14 FF/bank | -| `VX_cache_init` | 3 bits state, `NUM_BANKS`-bit `flush_done`, `NUM_REQS`-bit `lock_released`, UUID register | -| `VX_dcr_flush` | 2 bits (`flush_inflight_r`, `flush_done_r`) + a 2:1 arbiter | +| `VX_cache_init` | 3 bits state, `NUM_BANKS`-bit `flush_done`, 2 × `MEM_PORTS`-bit token tracking, `NUM_REQS`-bit answer tracking, UUID register | +| `VX_dcr_flush` | 2 bits (`flush_inflight_r`, `flush_done_r`) + a 2:1 arbiter, per write path | +| `VX_cache` | a request-crossbar in-flight counter (forwarding caches only) and 2:1 muxes on bank 0's response lane and on the first `MEM_PORTS` request lanes | | tag / data RAM | **no extra storage** | | mreq queue | **no extra entries** | @@ -704,18 +743,20 @@ primitive. ### 6.8 Correctness invariants -1. **Atomicity against normal traffic.** Once a flush is in flight no core request - reaches a bank pipeline — enforced by the `VX_cache_init` input lock. +1. **Atomicity against normal traffic.** While any token is visible no core + request reaches a bank pipeline — enforced by the `VX_cache_init` input block. 2. **The bank is fully quiesced before the walk starts.** `STATE_WAIT1` waits for `mshr_empty && bank_empty` — both, not just the MSHR. Otherwise a fill could install a fresh line behind the walk pointer and survive the flush. 3. **Reset-time valid bits are zero** without async reset on the tag SRAM, via the `STATE_INIT` walk. This is what lets the tag array map cleanly to BRAM. The bank's highest-priority `init_valid` masks every other source while it runs. -4. **All banks finish before the cache acks** — `VX_cache_init` waits for the AND - of `flush_end` across all banks before unlocking the originating input. -5. **Bank 0's downstream forward goes out last** — banks 1..N−1 hold `flush_end` - until `bank_empty`. +4. **All banks finish before the next level is flushed** — `VX_cache_init` + waits for the AND of `flush_end` across all banks, and every bank holds + `flush_end` until `bank_empty`. +5. **A token is answered only after the next level has each port's data** — + with `FLUSH_FORWARD`, one token per memory port goes out once the request + crossbar has drained, and every port must answer before `STATE_RESPOND`. 6. **The init walk is one-shot per reset.** `flush_pending_r` only records a `flush_begin` arriving *during* init; it never re-enters `STATE_INIT`. @@ -724,14 +765,15 @@ Edge cases: - **Flush during init** → `flush_pending_r` records it, and `STATE_DONE` fires as soon as init completes. The init walk stands in for the flush walk, which is correct because init invalidated everything anyway. -- **Multiple ports racing the flush flag** → the loop in `STATE_IDLE` counts down, - so the last write wins and it latches the lowest-indexed one's UUID; all flagged - ports unlock together at the end. +- **Tokens on several inputs** → the loop in `STATE_IDLE` counts down, so it + latches the lowest-indexed one's UUID. Every token visible at the end of the + walk is answered, one per cycle through bank 0's response lane (behind any + bank response); a token that arrives later starts the next walk. - **Flush during a fill-forward drain** is asserted impossible (`RUNTIME_ASSERT(~(flush_fire && fwd_pending))`) — `STATE_WAIT1`'s `mshr_empty` precondition already excludes it. - **`STATE_WAIT2`'s `bank_empty` does not re-check `mshr_empty`.** It does not need - to: WAIT1 established it and the input lock guarantees nothing can refill. + to: WAIT1 established it and the input block guarantees nothing can refill. ### 6.9 Why flush is not the substrate for line-granular invalidation diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index 118c5dbdcb..0ad30a8011 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -129,7 +129,7 @@ come from `cmd_size_bytes` | `CMD_FENCE` | 0x07 | 8 | — | yes | yes | Ordering barrier (retires as a NOP today, §10) | | `CMD_EVENT_SIGNAL` | 0x08 | 20 | EVENT | yes | yes | Write a counter slot | | `CMD_EVENT_WAIT` | 0x09 | 28 | EVENT | yes | yes | Spin until a counter satisfies a compare | -| `CMD_CACHE_FLUSH` | 0x0A | 12 | DCR | yes | yes | Per-core cache flush sweep | +| `CMD_CACHE_FLUSH` | 0x0A | 12 | DCR | yes | yes | Cache flush: broadcast start, per-core completion | | `CMD_LAUNCH_QMD` | 0x0B | 12 | KMU | **no** | yes | Atomic launch: KMU descriptor read from memory (§2.3) | | `CMD_DRAW` | 0x0C | 12 | KMU | **no** | yes | Device-orchestrated graphics draw (§8.1) | @@ -470,10 +470,15 @@ unconditionally on every backend. barrier** — the property §8.1 depends on. - **DCR proxy** ([`VX_cp_dcr_proxy.sv`](../../hw/rtl/cp/VX_cp_dcr_proxy.sv)) — `IDLE→REQ→(WAIT_RSP)→DONE`. Writes complete at `REQ`; reads wait for - `dcr_rsp_valid`. `CMD_CACHE_FLUSH` re-enters `REQ` once per core, - sweeping a DCR *read* of `VX_DCR_BASE_CACHE_FLUSH` with the target core - index in `dcr_req_data[15:0]`, and retires only once the last core's - flush completes ([`:105-134`](../../hw/rtl/cp/VX_cp_dcr_proxy.sv#L105)). + `dcr_rsp_valid`. `CMD_CACHE_FLUSH` first passes through `START`: one DCR + *write* of `VX_DCR_BASE_CACHE_FLUSH`, which reaches every core and starts + all their flushes at once. It then re-enters `REQ` once per core, sweeping + a DCR *read* of the same address with the target core index in + `dcr_req_data[15:0]`; each read is answered when that core's flush has + completed, and the command retires after the last one. The DCR response + bus has no back-pressure, so the reads stay one at a time; the flushes + themselves overlap and share one walk of the shared caches + ([cache_subsystem.md §6](cache_subsystem.md#6-flush-architecture)). - **DMA** ([`VX_cp_dma.sv`](../../hw/rtl/cp/VX_cp_dma.sv)) — dual-port multi-beat bursts in ≤4 KB INCR chunks (`MAX_BURST` = 64 × 64 B beats), each chunk clipped so no burst crosses a 4 KB boundary on either diff --git a/docs/designs/lsu_pipeline_design.md b/docs/designs/lsu_pipeline_design.md index c1d19bbeea..7ba07a2124 100644 --- a/docs/designs/lsu_pipeline_design.md +++ b/docs/designs/lsu_pipeline_design.md @@ -318,7 +318,7 @@ the DXA writes and the tensor unit's reads. |---|---| | `VX_mem_coalescer` | merges word-sized lanes onto data-cache-word channels; lanes addressing the same cache word become one request | | `VX_lsu_adapter` | turns the lane bundle into one memory bus per channel; registers the request boundary | -| `VX_dcr_flush` | injects the host's cache-flush request, on channel 0 of block 0 | +| `VX_dcr_flush` | injects the host's cache-flush token on lane 0 of a private dcache, and on every lane otherwise (a shared dcache's socket arbiter, or a disabled dcache's passthrough, buffers each lane on its own) | The coalescer exists when there is more than one lane and the data-cache word is wider than an LSU word; otherwise the path is a pass-through that diff --git a/docs/designs/output_merger_architecture.md b/docs/designs/output_merger_architecture.md index 5c4d7adecf..69a4ecff43 100644 --- a/docs/designs/output_merger_architecture.md +++ b/docs/designs/output_merger_architecture.md @@ -183,9 +183,21 @@ bit arrives at the cluster untouched. One steer per socket→L2 trunk input (`L2_SOCKET_REQS = NUM_SOCKETS × L1_MEM_PORTS` of them, in [`VX_graphics.sv`](../../hw/rtl/VX_graphics.sv)). The steer demuxes on the single `is_addr_om` bit: aperture writes go to the -OM leg (registered), everything else passes **combinationally** to L2 (the -socket already registers its outgoing bus, and a skid here would sit on the -critical L1→L2 path). +OM leg, everything else to L2; each leg presents a registered master +interface (`OUT_BUF`). + +**Flush ordering: the steer is a fork.** A flush token from the socket (a +dcache's kernel-boundary flush or a fence) bound for L2 is held at the steer +until the OM side has been flushed. When the token reaches the steer, every +aperture store its trunk carried before it is already on the OM leg. A +controller in `VX_graphics` serves the requests present when a round starts +(the held tokens and the cluster flush), waits until no complete fragment is +left in the OM (steer OM legs, ingress record stages and OM cores idle), +flushes the ocache, and then releases those tokens to L2. A half-received +two-word record is not waited for: its depth beat can be queued behind a held +token, and the store it belongs to is not one the token orders. So a flush covers the OM writes of the aperture stores +issued before it, whatever `busy` reports while those stores are in transit +between the dcache and the ingress (a span no busy signal counts). **Placement is the deadlock argument.** The join is the last point where a request has left the socket but has not touched anything L2-owned. A full OM diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 86fc979410..92c3cb4e23 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -292,8 +292,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef EXT_GFX_ANY_ENABLE // OR all socket flush reqs together; broadcast .done to every socket. - // Sockets fire their reqs sequentially; the shared resource flushes - // once per req (subsequent reqs see an already-empty cache). + // The cores start their flushes together, so the read-only graphics caches + // flush once for all of them; the OR drops when the last core has finished. VX_dcr_flush_if cluster_flush_if(); VX_dcr_flush_if per_socket_cluster_flush_if [NUM_SOCKETS](); wire [NUM_SOCKETS-1:0] per_socket_cluster_flush_req; diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index d836c5a760..6a09fcfbea 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1540,9 +1540,8 @@ package VX_gpu_pkg; localparam TCACHE_TAG_ID_BITS = (`CLOG2(`VX_CFG_TEX_MEM_QUEUE_SIZE) + TCACHE_BATCH_SEL_BITS); // Core-side (tex_core's cache_bus_if) tag width. localparam TCACHE_TAG_WIDTH = (UUID_WIDTH + TCACHE_TAG_ID_BITS); - // Cache-side tag width = core-side + 1 bit reserved by VX_dcr_flush so a - // host-issued CACHE_FLUSH DCR-read can inject a synthetic flush request - // onto port 0 without aliasing real requests. + // Cache-side tag width = core-side + 1 bit reserved by VX_dcr_flush, so + // a flush token can be injected without aliasing real requests. localparam TCACHE_BUS_TAG_WIDTH = (TCACHE_TAG_WIDTH + 1); localparam TCACHE_MEM_DATA_WIDTH = (TCACHE_LINE_SIZE * 8); localparam TCACHE_MEM_PORTS = 1; diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index 64510dba1b..a2a27fa7ed 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -263,6 +263,8 @@ module VX_graphics import VX_gpu_pkg::* .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) rcache ( @@ -297,6 +299,7 @@ module VX_graphics import VX_gpu_pkg::* om_dcrs_t om_core_dcrs [`VX_CFG_NUM_OM_CORES]; wire [L2_SOCKET_REQS-1:0] om_ingress_busy; + wire [L2_SOCKET_REQS-1:0] om_ingress_pending; // The DCRs are broadcast, so every OM core decodes identical values; the // ingresses read the aperture fields from core 0. om_dcrs_t om_dcrs; @@ -321,6 +324,11 @@ module VX_graphics import VX_gpu_pkg::* .TAG_WIDTH (L2_TAG_WIDTH) ) om_aperture_bus_if [L2_SOCKET_REQS](); + // Flush tokens are held at the steers until the OM side is flushed (below). + wire [L2_SOCKET_REQS-1:0] steer_flush_req; + reg [L2_SOCKET_REQS-1:0] steer_flush_ack_r; + wire [L2_SOCKET_REQS-1:0] steer_om_pending; + for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_steer VX_om_steer #( .INSTANCE_ID (`SFORMATF(("cluster%0d-om-steer%0d", CLUSTER_ID, i))), @@ -328,11 +336,14 @@ module VX_graphics import VX_gpu_pkg::* .TAG_WIDTH (L2_TAG_WIDTH), .OUT_BUF (3) ) om_steer ( - .clk (clk), - .reset (reset), - .bus_in_if (socket_mem_bus_if[i]), - .l2_out_if (l2_out_bus_if[i]), - .om_out_if (om_aperture_bus_if[i]) + .clk (clk), + .reset (reset), + .bus_in_if (socket_mem_bus_if[i]), + .l2_out_if (l2_out_bus_if[i]), + .om_out_if (om_aperture_bus_if[i]), + .flush_req (steer_flush_req[i]), + .flush_ack (steer_flush_ack_r[i]), + .om_pending (steer_om_pending[i]) ); end @@ -344,12 +355,13 @@ module VX_graphics import VX_gpu_pkg::* .NUM_LANES (OM_CORE_LANES), .OUT_BUF (3) ) om_ingress ( - .clk (clk), - .reset (reset), - .dcrs (om_dcrs), - .mem_bus_if (om_aperture_bus_if[i]), - .om_bus_if (om_arb_in_if[i]), - .busy (om_ingress_busy[i]) + .clk (clk), + .reset (reset), + .dcrs (om_dcrs), + .mem_bus_if (om_aperture_bus_if[i]), + .om_bus_if (om_arb_in_if[i]), + .busy (om_ingress_busy[i]), + .frag_pending (om_ingress_pending[i]) ); end @@ -424,37 +436,106 @@ module VX_graphics import VX_gpu_pkg::* .TAG_WIDTH (OCACHE_BUS_TAG_WIDTH) ) ocache_flushable_bus_if [OCACHE_NUM_INPUTS * OCACHE_NUM_REQS] (); + // OM-side flush. Each round serves the requests present when it starts: the + // steers holding a flush token (every aperture store their trunk carried + // before it is past the steer) and the cluster flush. It waits until no + // complete fragment is left in the OM, so the ocache's flush tokens queue + // behind its writes, then flushes the ocache and releases those requesters. + // A half-received two-word record is not waited for: its depth beat can sit + // behind a held token, and it belongs to a store the token does not order. + localparam OMF_IDLE = 0; + localparam OMF_DRAIN = 1; + localparam OMF_FLUSH = 2; + localparam OMF_REARM = 3; + VX_dcr_flush_if ocache_flush_if(); - assign ocache_flush_if.req = cluster_flush_if.req; + + reg [1:0] omf_state_r; + reg [L2_SOCKET_REQS-1:0] omf_steers_r; // steers this round answers + reg omf_cluster_r; // this round answers the cluster flush + reg omf_cluster_done_r; + wire omf_cluster_req = cluster_flush_if.req && ~omf_cluster_done_r; + wire omf_drained = ~(| om_busy_w) && ~(| om_ingress_pending) && ~(| steer_om_pending); + + always @(posedge clk) begin + if (reset) begin + omf_state_r <= OMF_IDLE; + omf_steers_r <= '0; + omf_cluster_r <= 1'b0; + omf_cluster_done_r <= 1'b0; + steer_flush_ack_r <= '0; + end else begin + steer_flush_ack_r <= '0; + if (~cluster_flush_if.req) begin + omf_cluster_done_r <= 1'b0; + end + case (omf_state_r) + OMF_IDLE: begin + if ((| steer_flush_req) || omf_cluster_req) begin + omf_steers_r <= steer_flush_req; + omf_cluster_r <= omf_cluster_req; + omf_state_r <= OMF_DRAIN; + end + end + OMF_DRAIN: begin + if (omf_drained) begin + omf_state_r <= OMF_FLUSH; + end + end + OMF_FLUSH: begin + if (ocache_flush_if.done) begin + steer_flush_ack_r <= omf_steers_r; + if (omf_cluster_r) begin + omf_cluster_done_r <= 1'b1; + end + omf_state_r <= OMF_REARM; + end + end + default: begin // OMF_REARM: the injectors re-arm once req drops + if (~ocache_flush_if.done) begin + omf_state_r <= OMF_IDLE; + end + end + endcase + end + end + assign ocache_flush_if.req = (omf_state_r == OMF_FLUSH); // OM-core flush chain carries the OM tag + 1 flush bit. When early-Z widens // the shared bus tag (a wider early-Z requester), the flushed OM bus is - // zero-extended up to OCACHE_BUS_TAG_WIDTH below. + // zero-extended up to OCACHE_BUS_TAG_WIDTH below. Each OM input is a + // separately ordered write path into the ocache, so each carries its own + // flush token behind its writes. + localparam OCACHE_OM_INPUTS = `VX_CFG_NUM_OM_CORES * OCACHE_NUM_REQS; localparam OCACHE_OM_FLUSH_TAG_WIDTH = OCACHE_TAG_WIDTH + 1; VX_mem_bus_if #( .DATA_SIZE (OCACHE_WORD_SIZE), .TAG_WIDTH (OCACHE_OM_FLUSH_TAG_WIDTH) - ) ocache_om_flush_bus_if [1] (); + ) ocache_om_flush_bus_if [OCACHE_OM_INPUTS] (); - VX_dcr_flush #( - .WORD_SIZE (OCACHE_WORD_SIZE), - .TAG_WIDTH (OCACHE_TAG_WIDTH), - .REQ_OUT_BUF (3) // register cache-request master boundary; rsp registered by cache CORE_OUT_BUF - ) ocache_dcr_flush ( - .clk (clk), - .reset (reset), - .dcr_flush_if (ocache_flush_if), - .core_bus_if (ocache_bus_if[0]), - .cache_bus_if (ocache_om_flush_bus_if[0]) - ); + VX_dcr_flush_if ocache_om_flush_if [OCACHE_OM_INPUTS] (); + wire [OCACHE_OM_INPUTS-1:0] ocache_om_flush_done; + + for (genvar i = 0; i < OCACHE_OM_INPUTS; ++i) begin : g_ocache_om_flush + assign ocache_om_flush_if[i].req = ocache_flush_if.req; + assign ocache_om_flush_done[i] = ocache_om_flush_if[i].done; - `ASSIGN_VX_MEM_BUS_IF_EX (ocache_flushable_bus_if[0], ocache_om_flush_bus_if[0], - OCACHE_BUS_TAG_WIDTH, OCACHE_OM_FLUSH_TAG_WIDTH, 0); + VX_dcr_flush #( + .WORD_SIZE (OCACHE_WORD_SIZE), + .TAG_WIDTH (OCACHE_TAG_WIDTH), + .REQ_OUT_BUF (3) // register cache-request master boundary; rsp registered by cache CORE_OUT_BUF + ) ocache_dcr_flush ( + .clk (clk), + .reset (reset), + .dcr_flush_if (ocache_om_flush_if[i]), + .core_bus_if (ocache_bus_if[i]), + .cache_bus_if (ocache_om_flush_bus_if[i]) + ); - for (genvar i = 1; i < `VX_CFG_NUM_OM_CORES * OCACHE_NUM_REQS; ++i) begin : g_ocache_passthru - `ASSIGN_VX_MEM_BUS_IF_EX (ocache_flushable_bus_if[i], ocache_bus_if[i], - OCACHE_BUS_TAG_WIDTH, OCACHE_TAG_WIDTH, 0); + `ASSIGN_VX_MEM_BUS_IF_EX (ocache_flushable_bus_if[i], ocache_om_flush_bus_if[i], + OCACHE_BUS_TAG_WIDTH, OCACHE_OM_FLUSH_TAG_WIDTH, 0); end + assign ocache_flush_if.done = (& ocache_om_flush_done); `ifdef VX_CFG_RASTER_EARLYZ_ENABLE // Attach the early-Z depth readers as the trailing ocache inputs. Each reader @@ -516,7 +597,7 @@ module VX_graphics import VX_gpu_pkg::* assign rcache_flush_done = 1'b1; `endif `ifdef VX_CFG_EXT_OM_ENABLE - assign ocache_flush_done = ocache_flush_if.done; + assign ocache_flush_done = omf_cluster_done_r; `else assign ocache_flush_done = 1'b1; `endif diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index c14be8a0c4..652a4509db 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -160,6 +160,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITE_ENABLE (0), .REPL_POLICY (`VX_CFG_ICACHE_REPL_POLICY), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) icache ( @@ -515,6 +517,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) tcache ( @@ -641,6 +645,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) rtcache ( diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 7982a79cd4..46659a46b4 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -49,6 +49,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // last-level cache: banks own AMO commit + RVA reservation tracking + parameter FLUSH_FORWARD = 1, // a flush also flushes the next level + parameter AMO_ENABLE = 0 // synthesize atomic-op logic in the banks ) ( `ifdef PERF_ENABLE @@ -98,20 +100,41 @@ module VX_cache import VX_gpu_pkg::*; #( wire [`UP(UUID_WIDTH)-1:0] flush_uuid; wire [NUM_BANKS-1:0] per_bank_flush_end; wire [NUM_BANKS-1:0] per_bank_core_req_fire; + wire mem_req_empty; + wire [MEM_PORTS-1:0] mem_flush_valid; + wire [MEM_PORTS-1:0] mem_flush_ready; + wire mem_flush_rsp_phase; + wire [MEM_PORTS-1:0] mem_flush_rsp_valid; + wire flush_rsp_valid; + wire [REQ_SEL_WIDTH-1:0] flush_rsp_idx; + wire [TAG_WIDTH-1:0] flush_rsp_tag; + wire flush_rsp_ready; VX_cache_init #( - .NUM_REQS (NUM_REQS), - .NUM_BANKS (NUM_BANKS), - .TAG_WIDTH (TAG_WIDTH), - .BANK_SEL_LATENCY (`TO_OUT_BUF_REG(REQ_XBAR_BUF)) // request xbar latency + .NUM_REQS (NUM_REQS), + .NUM_BANKS (NUM_BANKS), + .MEM_PORTS (MEM_PORTS), + .FLUSH_FORWARD (FLUSH_FORWARD), + .TAG_WIDTH (TAG_WIDTH), + .BANK_SEL_LATENCY (`TO_OUT_BUF_REG(REQ_XBAR_BUF)), // request xbar latency + .REQ_SEL_WIDTH (REQ_SEL_WIDTH) ) cache_init ( - .clk (clk), - .reset (reset), - .core_bus_in_if (core_bus_if), - .core_bus_out_if (core_bus2_if), - .bank_req_fire (per_bank_core_req_fire), - .flush_begin (per_bank_flush_begin), - .flush_uuid (flush_uuid), - .flush_end (per_bank_flush_end) + .clk (clk), + .reset (reset), + .core_bus_in_if (core_bus_if), + .core_bus_out_if (core_bus2_if), + .bank_req_fire (per_bank_core_req_fire), + .flush_begin (per_bank_flush_begin), + .flush_uuid (flush_uuid), + .flush_end (per_bank_flush_end), + .mem_req_empty (mem_req_empty), + .mem_flush_valid (mem_flush_valid), + .mem_flush_ready (mem_flush_ready), + .mem_flush_rsp_phase (mem_flush_rsp_phase), + .mem_flush_rsp_valid (mem_flush_rsp_valid), + .flush_rsp_valid (flush_rsp_valid), + .flush_rsp_idx (flush_rsp_idx), + .flush_rsp_tag (flush_rsp_tag), + .flush_rsp_ready (flush_rsp_ready) ); VX_mem_bus_if #( .DATA_SIZE (SECTOR_SIZE), @@ -161,6 +184,15 @@ module VX_cache import VX_gpu_pkg::*; #( assign mem_rsp_queue_sel[i] = 0; end end + // While the flush tokens are out the cache is quiescent (inputs blocked, + // MSHRs empty), so every memory response is a token's. + wire [MEM_PORTS-1:0] mem_rsp_xbar_valid; + wire [MEM_PORTS-1:0] mem_rsp_xbar_ready; + for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_flush_rsp + assign mem_flush_rsp_valid[i] = mem_flush_rsp_phase && mem_rsp_queue_valid[i]; + assign mem_rsp_xbar_valid[i] = mem_rsp_queue_valid[i] && ~mem_flush_rsp_phase; + assign mem_rsp_queue_ready[i] = mem_flush_rsp_phase || mem_rsp_xbar_ready[i]; + end wire [NUM_BANKS-1:0] per_bank_mem_rsp_valid; wire [NUM_BANKS-1:0][MEM_RSP_DATAW-MEM_ARB_SEL_BITS-1:0] per_bank_mem_rsp_pdata; wire [NUM_BANKS-1:0] per_bank_mem_rsp_ready; @@ -175,10 +207,10 @@ module VX_cache import VX_gpu_pkg::*; #( ) mem_rsp_xbar ( .clk (clk), .reset (reset), - .valid_in (mem_rsp_queue_valid), + .valid_in (mem_rsp_xbar_valid), .data_in (mem_rsp_queue_data_s), .sel_in (mem_rsp_queue_sel), - .ready_in (mem_rsp_queue_ready), + .ready_in (mem_rsp_xbar_ready), .valid_out (per_bank_mem_rsp_valid), .data_out (per_bank_mem_rsp_pdata), `UNUSED_PIN (sel_out), @@ -379,6 +411,28 @@ module VX_cache import VX_gpu_pkg::*; #( for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_core_rsp_pdata assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i], per_bank_core_rsp_tag[i]}; end + // A flush token's answer takes bank 0's lane into the response crossbar + // when the bank has no response of its own, so it leaves through the same + // registered outputs as every other response. + wire [NUM_BANKS-1:0] core_rsp_xbar_valid; + wire [NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] core_rsp_xbar_pdata; + wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] core_rsp_xbar_idx; + wire [NUM_BANKS-1:0] core_rsp_xbar_ready; + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_core_rsp_xbar_in + if (i == 0) begin : g_flush_rsp + wire flush_rsp_sel = flush_rsp_valid && ~per_bank_core_rsp_valid[0]; + assign core_rsp_xbar_valid[0] = per_bank_core_rsp_valid[0] || flush_rsp_valid; + assign core_rsp_xbar_pdata[0] = flush_rsp_sel ? {`CS_WORD_WIDTH'(0), flush_rsp_tag} : per_bank_core_rsp_pdata[0]; + assign core_rsp_xbar_idx[0] = flush_rsp_sel ? flush_rsp_idx : per_bank_core_rsp_idx[0]; + assign per_bank_core_rsp_ready[0] = core_rsp_xbar_ready[0]; + assign flush_rsp_ready = core_rsp_xbar_ready[0] && ~per_bank_core_rsp_valid[0]; + end else begin : g_bank_rsp + assign core_rsp_xbar_valid[i] = per_bank_core_rsp_valid[i]; + assign core_rsp_xbar_pdata[i] = per_bank_core_rsp_pdata[i]; + assign core_rsp_xbar_idx[i] = per_bank_core_rsp_idx[i]; + assign per_bank_core_rsp_ready[i] = core_rsp_xbar_ready[i]; + end + end VX_stream_omega #( .NUM_INPUTS (NUM_BANKS), .NUM_OUTPUTS (NUM_REQS), @@ -388,10 +442,10 @@ module VX_cache import VX_gpu_pkg::*; #( ) core_rsp_xbar ( .clk (clk), .reset (reset), - .valid_in (per_bank_core_rsp_valid), - .data_in (per_bank_core_rsp_pdata), - .sel_in (per_bank_core_rsp_idx), - .ready_in (per_bank_core_rsp_ready), + .valid_in (core_rsp_xbar_valid), + .data_in (core_rsp_xbar_pdata), + .sel_in (core_rsp_xbar_idx), + .ready_in (core_rsp_xbar_ready), .valid_out (core_rsp_valid), .data_out (core_rsp_pdata), `UNUSED_PIN (sel_out), @@ -426,6 +480,64 @@ module VX_cache import VX_gpu_pkg::*; #( wire [MEM_PORTS-1:0] mem_req_tmp_valid; wire [MEM_PORTS-1:0][`UP(`LOG2UP(NUM_BANKS))-1:0] mem_req_tmp_idx; wire [MEM_PORTS-1:0] mem_req_tmp_ready; + // The flush token for port i is a read of the port's first line with the + // flush attribute. It takes bank i's lane into the request crossbar (bank i + // maps to port i), so the crossbar forms its address and it leaves through + // the same registered outputs as the write-backs ahead of it. + wire [NUM_BANKS-1:0] mem_req_xbar_valid; + wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] mem_req_xbar_pdata; + wire [NUM_BANKS-1:0] mem_req_xbar_ready; + wire [MEM_REQ_DATAW-1:0] mem_flush_pdata = { + `CS_LINE_SECTOR_ADDR_WIDTH'(0), + 1'b0, + SECTOR_SIZE'(0), + `CS_SECTOR_WIDTH'(0), + BANK_MEM_TAG_WIDTH'(0), + `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) + }; + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_mem_req_xbar_in + if (i < MEM_PORTS) begin : g_flush_req + assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; + assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; + assign mem_flush_ready[i] = mem_req_xbar_ready[i] && ~per_bank_mem_req_valid[i]; + `RUNTIME_ASSERT (~(mem_flush_valid[i] && per_bank_mem_req_valid[i]), ("%t: %s flush token collides with a bank %0d request", $time, INSTANCE_ID, i)) + end else begin : g_bank_req + assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i]; + assign mem_req_xbar_pdata[i] = per_bank_mem_req_pdata[i]; + end + assign per_bank_mem_req_ready[i] = mem_req_xbar_ready[i]; + end + // Write-backs from different banks are not ordered with each other inside + // the crossbar, so the tokens wait until it has drained. + if (FLUSH_FORWARD != 0) begin : g_mem_req_xbar_pending + localparam XBAR_IN_W = `CLOG2(NUM_BANKS+1); + localparam XBAR_OUT_W = `CLOG2(MEM_PORTS+1); + // generous bound on what the crossbar can hold: buffered stages plus outputs + localparam XBAR_CAPACITY = 4 * NUM_BANKS * (`CLOG2(NUM_BANKS) + 1); + wire [NUM_BANKS-1:0] xbar_in_fire = mem_req_xbar_valid & mem_req_xbar_ready; + wire [MEM_PORTS-1:0] xbar_out_fire = mem_req_tmp_valid & mem_req_tmp_ready; + wire [XBAR_IN_W-1:0] xbar_in_cnt; + wire [XBAR_OUT_W-1:0] xbar_out_cnt; + `POP_COUNT(xbar_in_cnt, xbar_in_fire); + `POP_COUNT(xbar_out_cnt, xbar_out_fire); + VX_pending_size #( + .SIZE (XBAR_CAPACITY), + .INCRW (XBAR_IN_W), + .DECRW (XBAR_OUT_W) + ) xbar_pending ( + .clk (clk), + .reset (reset), + .incr (xbar_in_cnt), + .decr (xbar_out_cnt), + .empty (mem_req_empty), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + end else begin : g_no_mem_req_xbar_pending + assign mem_req_empty = 1'b1; + end VX_stream_omega #( .NUM_INPUTS (NUM_BANKS), .NUM_OUTPUTS (MEM_PORTS), @@ -435,10 +547,10 @@ module VX_cache import VX_gpu_pkg::*; #( ) mem_req_xbar ( .clk (clk), .reset (reset), - .valid_in (per_bank_mem_req_valid), - .data_in (per_bank_mem_req_pdata), + .valid_in (mem_req_xbar_valid), + .data_in (mem_req_xbar_pdata), .sel_in (per_bank_mem_req_bid), - .ready_in (per_bank_mem_req_ready), + .ready_in (mem_req_xbar_ready), .valid_out (mem_req_tmp_valid), .data_out (mem_req_pdata), .sel_out (mem_req_tmp_idx), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 28576fcd39..ee64fe7937 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -286,7 +286,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire no_pending_req = ~pipe_inflight && mreq_queue_empty && ~amo_commit_busy; VX_cache_flush #( - .BANK_ID (BANK_ID), .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), .NUM_BANKS (NUM_BANKS), diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 886f898d62..2efb8ab859 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -55,6 +55,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Indicates this cache cluster is the LLC (handles AMO completion). + parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter AMO_ENABLE = 0 // Enable atomic memory operations. ) ( @@ -145,6 +146,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .NC_ENABLE (NC_ENABLE), .PASSTHRU (PASSTHRU), .IS_LLC (IS_LLC), + .FLUSH_FORWARD (FLUSH_FORWARD), .AMO_ENABLE (AMO_ENABLE) ) cache_wrap ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache_flush.sv b/hw/rtl/cache/VX_cache_flush.sv index 79e5aad266..c51a5aa2cc 100644 --- a/hw/rtl/cache/VX_cache_flush.sv +++ b/hw/rtl/cache/VX_cache_flush.sv @@ -14,7 +14,6 @@ `include "VX_cache_define.vh" module VX_cache_flush import VX_gpu_pkg::*; #( - parameter BANK_ID = 0, parameter CACHE_SIZE = 1024, // Size of cache in bytes parameter LINE_SIZE = 64, // Size of line inside a bank in bytes parameter NUM_BANKS = 1, // Number of banks @@ -80,13 +79,12 @@ module VX_cache_flush import VX_gpu_pkg::*; #( end STATE_FLUSH: begin if (counter == ((2 ** CTR_WIDTH)-1) && flush_ready) begin - state_n = (BANK_ID == 0) ? STATE_DONE : STATE_WAIT2; + state_n = STATE_WAIT2; end end STATE_WAIT2: begin - // ensure the bank is empty before notifying the cache flush unit, - // because the flush request to lower caches only goes through bank0 - // and it is important that request gets send out last. + // the write-backs must leave the bank before the cache sends + // its flush tokens behind them if (bank_empty) begin state_n = STATE_DONE; end diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index a27ba56dc6..8159a6ba32 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -13,28 +13,51 @@ `include "VX_cache_define.vh" +// Flush sequencer. Flush tokens (requests with the flush attribute) at any +// input start one walk: every input is blocked, in-flight requests drain and +// every bank writes back and invalidates its lines. With FLUSH_FORWARD, the +// cache then sends one token down each memory port, behind that port's +// write-backs, and waits for every port to answer, so the next level has +// each port's data before it releases that port's token. Finally each token +// visible at the end of the walk is answered locally, one at a time through +// the cache's own response path, and the inputs reopen. +// A token that arrives during the walk is visible only after all of its +// input's earlier requests were accepted, so the walk covered them. module VX_cache_init import VX_gpu_pkg::*; #( - parameter NUM_REQS = 4, // Number of Word requests per cycle - parameter NUM_BANKS = 1, // Number of banks - parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size - parameter BANK_SEL_LATENCY = 1 // Bank select latency + parameter NUM_REQS = 4, // Number of Word requests per cycle + parameter NUM_BANKS = 1, // Number of banks + parameter MEM_PORTS = 1, // Number of memory ports + parameter FLUSH_FORWARD = 1, // Forward the flush to the next level + parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size + parameter BANK_SEL_LATENCY = 1, // Bank select latency + parameter REQ_SEL_WIDTH = `UP(`CLOG2(NUM_REQS)) ) ( - input wire clk, - input wire reset, - VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], - VX_mem_bus_if.master core_bus_out_if [NUM_REQS], - input wire [NUM_BANKS-1:0] bank_req_fire, - output wire [NUM_BANKS-1:0] flush_begin, + input wire clk, + input wire reset, + VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], + VX_mem_bus_if.master core_bus_out_if [NUM_REQS], + input wire [NUM_BANKS-1:0] bank_req_fire, + output wire [NUM_BANKS-1:0] flush_begin, output wire [`UP(UUID_WIDTH)-1:0] flush_uuid, - input wire [NUM_BANKS-1:0] flush_end + input wire [NUM_BANKS-1:0] flush_end, + // flush tokens to the next level + input wire mem_req_empty, // no write-back left between the banks and the ports + output wire [MEM_PORTS-1:0] mem_flush_valid, + input wire [MEM_PORTS-1:0] mem_flush_ready, + output wire mem_flush_rsp_phase, + input wire [MEM_PORTS-1:0] mem_flush_rsp_valid, + // answers to the requesters' tokens + output wire flush_rsp_valid, + output wire [REQ_SEL_WIDTH-1:0] flush_rsp_idx, + output wire [TAG_WIDTH-1:0] flush_rsp_tag, + input wire flush_rsp_ready ); - `UNUSED_PARAM (TAG_WIDTH) - - localparam STATE_IDLE = 0; - localparam STATE_WAIT1 = 1; - localparam STATE_FLUSH = 2; - localparam STATE_WAIT2 = 3; - localparam STATE_DONE = 4; + localparam STATE_IDLE = 0; + localparam STATE_WAIT1 = 1; + localparam STATE_FLUSH = 2; + localparam STATE_WAIT2 = 3; + localparam STATE_FORWARD = 4; + localparam STATE_RESPOND = 5; reg [2:0] state, state_n; @@ -80,52 +103,106 @@ module VX_cache_init import VX_gpu_pkg::*; #( `UNUSED_VAR (bank_req_fire) end - reg [NUM_BANKS-1:0] flush_done, flush_done_n; - wire [NUM_REQS-1:0] flush_req_mask; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_req_mask assign flush_req_mask[i] = core_bus_in_if[i].req_valid && core_bus_in_if[i].req_data.attr[MEM_ATTR_FLUSH_OFFS]; end wire flush_req_enable = (| flush_req_mask); - reg [NUM_REQS-1:0] lock_released, lock_released_n; + reg [NUM_BANKS-1:0] flush_done, flush_done_n; + reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now + reg [MEM_PORTS-1:0] fwd_sent, fwd_sent_n; + reg [MEM_PORTS-1:0] fwd_done, fwd_done_n; + reg [NUM_REQS-1:0] rsp_pending, rsp_pending_n; // token still to be answered reg [`UP(UUID_WIDTH)-1:0] flush_uuid_r, flush_uuid_n; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_req - // block core requests if a flush is active, - // unless the lock for this specific request has been released - wire input_enable = ~flush_req_enable || lock_released[i]; + // answer the lowest pending input; its token stays at the input until then + wire [NUM_REQS-1:0] rsp_sel_onehot; + wire [REQ_SEL_WIDTH-1:0] rsp_sel_idx; + wire rsp_sel_valid; + VX_priority_encoder #( + .N (NUM_REQS) + ) rsp_sel ( + .data_in (rsp_pending), + .onehot_out (rsp_sel_onehot), + .index_out (rsp_sel_idx), + .valid_out (rsp_sel_valid) + ); + assign flush_rsp_valid = (state == STATE_RESPOND) && rsp_sel_valid; + assign flush_rsp_idx = rsp_sel_idx; + wire flush_rsp_fire = flush_rsp_valid && flush_rsp_ready; + + wire [NUM_REQS-1:0][TAG_WIDTH-1:0] core_bus_in_tag; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_in_tag + assign core_bus_in_tag[i] = core_bus_in_if[i].req_data.tag; + end + VX_onehot_mux #( + .DATAW (TAG_WIDTH), + .N (NUM_REQS) + ) rsp_tag_mux ( + .data_in (core_bus_in_tag), + .sel_in (rsp_sel_onehot), + .data_out (flush_rsp_tag) + ); + + wire [NUM_REQS-1:0] dbg_stall; +`ifdef DBG_FLUSH_STALL + // Random input backpressure, a new mask every 128 cycles (each input + // stalled half the time), to widen ordering windows under test. + reg [31:0] dbg_lfsr_r; + reg [6:0] dbg_cnt_r; + reg [NUM_REQS-1:0] dbg_stall_r; + always @(posedge clk) begin + if (reset) begin + dbg_lfsr_r <= 32'hACE1_2468; + dbg_cnt_r <= '0; + dbg_stall_r <= '0; + end else begin + dbg_lfsr_r <= {dbg_lfsr_r[30:0], dbg_lfsr_r[31] ^ dbg_lfsr_r[21] ^ dbg_lfsr_r[1] ^ dbg_lfsr_r[0]}; + dbg_cnt_r <= dbg_cnt_r + 7'd1; + if (dbg_cnt_r == '0) begin + dbg_stall_r <= (NUM_REQS > 1) ? NUM_REQS'(dbg_lfsr_r) : '0; + end + end + end + assign dbg_stall = dbg_stall_r; +`else + assign dbg_stall = '0; +`endif + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_req + // Normal requests stay blocked while any token is visible. A token is + // consumed here when its answer is sent and never reaches the banks. + wire take_token = flush_rsp_fire && rsp_sel_onehot[i]; + wire input_enable = ~flush_req_enable && ~dbg_stall[i]; assign core_bus_out_if[i].req_valid = core_bus_in_if[i].req_valid && input_enable; assign core_bus_out_if[i].req_data = core_bus_in_if[i].req_data; - assign core_bus_in_if[i].req_ready = core_bus_out_if[i].req_ready && input_enable; + assign core_bus_in_if[i].req_ready = take_token || (core_bus_out_if[i].req_ready && input_enable); end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_in_rsp - assign core_bus_in_if[i].rsp_valid = core_bus_out_if[i].rsp_valid; - assign core_bus_in_if[i].rsp_data = core_bus_out_if[i].rsp_data; + assign core_bus_in_if[i].rsp_valid = core_bus_out_if[i].rsp_valid; + assign core_bus_in_if[i].rsp_data = core_bus_out_if[i].rsp_data; assign core_bus_out_if[i].rsp_ready = core_bus_in_if[i].rsp_ready; end - reg [NUM_REQS-1:0][`UP(UUID_WIDTH)-1:0] core_bus_out_uuid; - wire [NUM_REQS-1:0] core_bus_out_ready; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_uuid + reg [NUM_REQS-1:0][`UP(UUID_WIDTH)-1:0] core_bus_in_uuid; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_in_uuid if (UUID_WIDTH != 0) begin : g_uuid - assign core_bus_out_uuid[i] = core_bus_in_if[i].req_data.tag.uuid; + assign core_bus_in_uuid[i] = core_bus_in_if[i].req_data.tag.uuid; end else begin : g_no_uuid - assign core_bus_out_uuid[i] = 0; + assign core_bus_in_uuid[i] = 0; end end - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_ready - assign core_bus_out_ready[i] = core_bus_out_if[i].req_ready; - end - always @(*) begin - state_n = state; - flush_done_n = flush_done; - lock_released_n = lock_released; - flush_uuid_n = flush_uuid_r; + state_n = state; + flush_done_n = flush_done; + fwd_go_n = fwd_go; + fwd_sent_n = fwd_sent; + fwd_done_n = fwd_done; + rsp_pending_n = rsp_pending & ~(rsp_sel_onehot & {NUM_REQS{flush_rsp_fire}}); + flush_uuid_n = flush_uuid_r; case (state) // STATE_IDLE: default: begin @@ -133,7 +210,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( state_n = (BANK_SEL_LATENCY != 0) ? STATE_WAIT1 : STATE_FLUSH; for (integer i = NUM_REQS-1; i >= 0; --i) begin if (flush_req_mask[i]) begin - flush_uuid_n = core_bus_out_uuid[i]; + flush_uuid_n = core_bus_in_uuid[i]; end end end @@ -151,18 +228,25 @@ module VX_cache_init import VX_gpu_pkg::*; #( // wait for all banks to finish flushing flush_done_n = flush_done | flush_end; if (flush_done_n == {NUM_BANKS{1'b1}}) begin - state_n = STATE_DONE; flush_done_n = '0; - // only release current flush requests - // and keep normal requests locked - lock_released_n = flush_req_mask; + // answer the tokens visible now; later ones start a new walk + rsp_pending_n = flush_req_mask; + state_n = FLUSH_FORWARD ? STATE_FORWARD : STATE_RESPOND; end end - STATE_DONE: begin - // wait until released flush requests are issued - // when returning to IDLE state other requests will unlock - lock_released_n = lock_released & ~core_bus_out_ready; - if (lock_released_n == 0) begin + STATE_FORWARD: begin + fwd_go_n = fwd_go | mem_req_empty; + fwd_sent_n = fwd_sent | (mem_flush_valid & mem_flush_ready); + fwd_done_n = fwd_done | mem_flush_rsp_valid; + if (fwd_done_n == {MEM_PORTS{1'b1}}) begin + fwd_go_n = 1'b0; + fwd_sent_n = '0; + fwd_done_n = '0; + state_n = STATE_RESPOND; + end + end + STATE_RESPOND: begin + if (rsp_pending_n == '0) begin state_n = STATE_IDLE; end end @@ -171,13 +255,19 @@ module VX_cache_init import VX_gpu_pkg::*; #( always @(posedge clk) begin if (reset) begin - state <= STATE_IDLE; - flush_done <= '0; - lock_released <= '0; + state <= STATE_IDLE; + flush_done <= '0; + fwd_go <= 1'b0; + fwd_sent <= '0; + fwd_done <= '0; + rsp_pending <= '0; end else begin - state <= state_n; - flush_done <= flush_done_n; - lock_released <= lock_released_n; + state <= state_n; + flush_done <= flush_done_n; + fwd_go <= fwd_go_n; + fwd_sent <= fwd_sent_n; + fwd_done <= fwd_done_n; + rsp_pending <= rsp_pending_n; end flush_uuid_r <= flush_uuid_n; end @@ -185,4 +275,9 @@ module VX_cache_init import VX_gpu_pkg::*; #( assign flush_begin = {NUM_BANKS{state == STATE_FLUSH}}; assign flush_uuid = flush_uuid_r; + // The drain counter counts the tokens too, so the first one sent must not + // take the others' valid away. + assign mem_flush_valid = {MEM_PORTS{(state == STATE_FORWARD) && (fwd_go || mem_req_empty)}} & ~fwd_sent; + assign mem_flush_rsp_phase = (state == STATE_FORWARD); + endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index caff8c34d3..6aa1266e7a 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -55,6 +55,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Set when this cache is the LLC (enables AMO completion at this level). + parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter AMO_ENABLE = 0 // Enable atomic operations in cache banks. ) ( @@ -147,6 +148,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .CORE_OUT_BUF (BYPASS_ENABLE ? 1 : CORE_OUT_BUF), .MEM_OUT_BUF (BYPASS_ENABLE ? 1 : MEM_OUT_BUF), .IS_LLC (IS_LLC), + .FLUSH_FORWARD (FLUSH_FORWARD), .AMO_ENABLE (AMO_ENABLE) ) cache ( .clk (clk), diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 0de66a70f5..46b3918de4 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -185,12 +185,9 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // flush could inject ahead of a store still in the LSU/coalescer and lose it. wire store_drained = (&lsu_sched_empty) && mem_unit_empty; assign dcr_flush_dcache_if.req = dcr_flush_if.req && store_drained; - // Both L1s forward their flush to the shared next level, and a cache that - // is flushing locks out incoming core requests for its whole sweep. The - // icache carries no dirty data and so retires almost immediately; gate it - // behind the dcache to keep that forward from arriving while the dcache is - // still evicting, which would strand the evictions upstream of memory. - assign dcr_flush_icache_if.req = dcr_flush_if.req && dcr_flush_dcache_if.done; + // Only the dcache flush walks the shared levels (the icache does not + // forward its flush), so the icache flushes alongside it. + assign dcr_flush_icache_if.req = dcr_flush_if.req; // Each VX_dcr_flush holds .done level-high until its req drops, so a // straight AND of the two dones reports the combined completion to // VX_dcr_data — which then drops req, re-arming both for the next flush. diff --git a/hw/rtl/core/VX_dcr_data.sv b/hw/rtl/core/VX_dcr_data.sv index 9c10d06f9b..746848286e 100644 --- a/hw/rtl/core/VX_dcr_data.sv +++ b/hw/rtl/core/VX_dcr_data.sv @@ -74,28 +74,40 @@ module VX_dcr_data import VX_gpu_pkg::*; #( // Cache flush via DCR interface ///////////////////////////////////////// - reg flush_pending_r; - - wire is_flush_read = dcr_bus_if.req_valid - && ~dcr_bus_if.req_data.rw - && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH) - && (mpm_target_cid == 16'(CORE_ID)) - && ~flush_pending_r; + // A flush starts on a DCR write to CACHE_FLUSH, which reaches every core, + // and this core's flush read is answered once its flush has completed. The + // DCR response bus has no back-pressure, so the command processor starts + // all cores with one write and then reads them one at a time: the flushes + // overlap and the responses never collide. + reg flush_pending_r; // flush running + reg flush_complete_r; // flush finished, not yet reported + reg flush_read_r; // flush read waiting for completion + + wire is_flush_req = dcr_bus_if.req_valid + && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); + wire is_flush_write = is_flush_req && dcr_bus_if.req_data.rw; + wire is_flush_read = is_flush_req && ~dcr_bus_if.req_data.rw + && (mpm_target_cid == 16'(CORE_ID)); + + wire flush_done = flush_pending_r && dcr_flush_if.done; + wire flush_start = ~flush_pending_r && is_flush_write; + wire flush_rsp = flush_read_r && (flush_complete_r || flush_done); always @(posedge clk) begin if (reset) begin - flush_pending_r <= 1'b0; - end else if (is_flush_read) begin - flush_pending_r <= 1'b1; - end else if (dcr_flush_if.done) begin - flush_pending_r <= 1'b0; + flush_pending_r <= 1'b0; + flush_complete_r <= 1'b0; + flush_read_r <= 1'b0; + end else begin + flush_pending_r <= flush_start || (flush_pending_r && ~flush_done); + // a write opens a new flush, so an older completion cannot answer it + flush_complete_r <= ~is_flush_write && ~flush_rsp && (flush_complete_r || flush_done); + flush_read_r <= ~flush_rsp && (flush_read_r || is_flush_read); end end assign dcr_flush_if.req = flush_pending_r; - wire flush_done = flush_pending_r && dcr_flush_if.done; - // Return CSR data or flush-done on DCR response bus // Use intermediate registers — Vivado (Synth 8-453) forbids procedural // assignment to interface net signals; drive them with continuous assigns. @@ -107,10 +119,10 @@ module VX_dcr_data import VX_gpu_pkg::*; #( rsp_valid_r <= 1'b0; rsp_data_r <= '0; end else begin - rsp_valid_r <= dcr_csr_if_fire || flush_done; + rsp_valid_r <= dcr_csr_if_fire || flush_rsp; if (dcr_csr_if_fire) begin rsp_data_r <= dcr_csr_if.value; - end else if (flush_done) begin + end else if (flush_rsp) begin rsp_data_r <= '0; end end @@ -119,6 +131,12 @@ module VX_dcr_data import VX_gpu_pkg::*; #( assign dcr_bus_if.rsp_valid = rsp_valid_r; assign dcr_bus_if.rsp_data = rsp_data_r; + // the requester issues one read at a time, so the two answers never meet + `RUNTIME_ASSERT (~(dcr_csr_if_fire && flush_rsp), ("%t: %s CSR read and flush answered in the same cycle", $time, INSTANCE_ID)) + // a new flush starts only after every core reported the previous one + `RUNTIME_ASSERT (~(is_flush_write && flush_pending_r), ("%t: %s flush started while one is running", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~(is_flush_read && ~flush_pending_r && ~flush_complete_r), ("%t: %s flush read with no flush started", $time, INSTANCE_ID)) + `ifdef DBG_TRACE_PIPELINE always @(posedge clk) begin if (dcr_bus_if.req_valid && dcr_bus_if.req_data.rw) begin diff --git a/hw/rtl/core/VX_dcr_flush.sv b/hw/rtl/core/VX_dcr_flush.sv index 7617f23b46..051eaf85d2 100644 --- a/hw/rtl/core/VX_dcr_flush.sv +++ b/hw/rtl/core/VX_dcr_flush.sv @@ -87,7 +87,7 @@ module VX_dcr_flush import VX_gpu_pkg::*; #( assign dcr_flush_if.done = flush_done_r; assign flush_bus_if.rsp_ready = 1'b1; - // 2-to-1 arb: input 0 = flush, input 1 = real dcache port 0 + // 2-to-1 arb: input 0 = cache traffic, input 1 = the flush token VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (TAG_WIDTH) diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 28e7b51377..bcd4083306 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -366,6 +366,22 @@ module VX_mem_unit import VX_gpu_pkg::*; #( end + // A private dcache takes the lanes straight from the adapter, whose drain + // store_drained already waits for, so one token suffices. Otherwise (a + // shared dcache, or none) the lanes are buffered independently before the + // next flush point, so a store on one lane can still be queued when another + // lane's token arrives: every lane then carries its own token behind its + // stores, and the flush is done when every lane's is. + localparam DCACHE_PRIVATE = (`VX_CFG_SOCKET_SIZE == `VX_CFG_NUM_DCACHES); + localparam FLUSH_LANES = DCACHE_PRIVATE ? 1 : DCACHE_NUM_REQS; + VX_dcr_flush_if lane_flush_if [FLUSH_LANES] (); + wire [FLUSH_LANES-1:0] lane_flush_done; + for (genvar k = 0; k < FLUSH_LANES; ++k) begin : g_lane_flush + assign lane_flush_if[k].req = dcr_flush_if.req; + assign lane_flush_done[k] = lane_flush_if[k].done; + end + assign dcr_flush_if.done = (& lane_flush_done); + for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_dcache_adapters VX_mem_bus_if #( @@ -398,8 +414,8 @@ module VX_mem_unit import VX_gpu_pkg::*; #( assign per_block_adapter_empty[i] = ~(| adapter_req_valid); for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if - if (i == 0 && j == 0) begin : g_flush_port - // Port 0: route through VX_dcr_flush to inject flush requests + localparam LANE = i * DCACHE_CHANNELS + j; + if (LANE < FLUSH_LANES) begin : g_flush_lane VX_dcr_flush #( .WORD_SIZE (DCACHE_WORD_SIZE), .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), @@ -407,15 +423,14 @@ module VX_mem_unit import VX_gpu_pkg::*; #( ) dcr_flush ( .clk (clk), .reset (reset), - .dcr_flush_if (dcr_flush_if), - .core_bus_if (dcache_bus_tmp_if[0]), - .cache_bus_if (dcache_bus_if[0]) + .dcr_flush_if (lane_flush_if[LANE]), + .core_bus_if (dcache_bus_tmp_if[j]), + .cache_bus_if (dcache_bus_if[LANE]) ); - end else begin : g_passthru_port - // Ports 1+: pass through; tag is zero-extended from - // DCACHE_CORE_TAG_WIDTH to DCACHE_TAG_WIDTH on request, - // and MSB-stripped on response. - `ASSIGN_VX_MEM_BUS_IF_EX (dcache_bus_if[i * DCACHE_CHANNELS + j], dcache_bus_tmp_if[j], DCACHE_TAG_WIDTH_BASE, DCACHE_CORE_TAG_WIDTH, 0); + end else begin : g_passthru_lane + // tag is zero-extended from DCACHE_CORE_TAG_WIDTH to + // DCACHE_TAG_WIDTH on request, and MSB-stripped on response + `ASSIGN_VX_MEM_BUS_IF_EX (dcache_bus_if[LANE], dcache_bus_tmp_if[j], DCACHE_TAG_WIDTH_BASE, DCACHE_CORE_TAG_WIDTH, 0); end end diff --git a/hw/rtl/cp/VX_cp_dcr_proxy.sv b/hw/rtl/cp/VX_cp_dcr_proxy.sv index 94428c07ed..05a90fd6de 100644 --- a/hw/rtl/cp/VX_cp_dcr_proxy.sv +++ b/hw/rtl/cp/VX_cp_dcr_proxy.sv @@ -15,13 +15,16 @@ // when valid) → DONE → IDLE. // // For CMD_CACHE_FLUSH (cmd.arg0 = number of cores): -// A cache flush is a per-core DCR-read to VX_DCR_BASE_CACHE_FLUSH whose -// response is the per-core flush-complete signal (see VX_dcr_data). One -// CMD_CACHE_FLUSH sweeps that read across every core — REQ/WAIT_RSP per -// core — and retires only once the last core's flush completes. This is -// the ACQUIRE_MEM model: a single command in the ring, executed by -// the CP across all cores, that the host posts after CMD_LAUNCH so it -// sees coherent results. The host fills cmd.arg0 from VX_CAPS_NUM_CORES. +// IDLE → START (DCR write to VX_DCR_BASE_CACHE_FLUSH, which every core +// receives and starts flushing on) → then REQ/WAIT_RSP once per core: a +// DCR read of VX_DCR_BASE_CACHE_FLUSH answered when that core's flush has +// completed (see VX_dcr_data). The cores flush concurrently, so their dcache +// flushes reach the shared caches together and share one walk; the reads +// are serial because the DCR response bus cannot carry two at once. The +// command retires once the last core reports. This is the ACQUIRE_MEM +// model: a single command in the ring, executed by the CP across all cores, +// that the host posts after CMD_LAUNCH so it sees coherent results. The +// host fills cmd.arg0 from VX_CAPS_NUM_CORES. // // The most-recent read response is published on `last_rsp_data` and is // also exposed on the AXI-Lite regfile so the host can poll it after @@ -57,8 +60,9 @@ module VX_cp_dcr_proxy input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data ); - typedef enum logic [1:0] { + typedef enum logic [2:0] { S_IDLE, + S_START, // CMD_CACHE_FLUSH: broadcast write starting every core S_REQ, // hold dcr_req_valid until consumed (single cycle here) S_WAIT_RSP, // read commands only S_DONE @@ -106,12 +110,15 @@ module VX_cp_dcr_proxy flush_total <= cmd_ncores; flush_cid <= '0; // a 0-core flush (degenerate) retires immediately - state <= (cmd_ncores == '0) ? S_DONE : S_REQ; + state <= (cmd_ncores == '0) ? S_DONE : S_START; end else begin state <= S_REQ; end end end + S_START: begin + state <= S_REQ; + end S_REQ: begin // The Vortex DCR bus consumes the request in a single cycle // (req_valid handshakes combinationally; no req_ready backpressure). @@ -141,11 +148,11 @@ module VX_cp_dcr_proxy end always_comb begin - dcr_req_valid = (state == S_REQ); + dcr_req_valid = (state == S_START) || (state == S_REQ); if (pending_is_flush) begin - // a cache flush is a per-core DCR read to VX_DCR_BASE_CACHE_FLUSH; - // the target core index rides in the low 16 bits of dcr_req_data. - dcr_req_rw = 1'b0; + // the START write reaches every core; each per-core read carries the + // target core index in the low 16 bits of dcr_req_data. + dcr_req_rw = (state == S_START); dcr_req_addr = `VX_DCR_ADDR_BITS'(`VX_DCR_BASE_CACHE_FLUSH); dcr_req_data = `VX_DCR_DATA_BITS'(flush_cid); end else begin diff --git a/hw/rtl/om/VX_om_ingress.sv b/hw/rtl/om/VX_om_ingress.sv index c1344b4227..5aba9c0991 100644 --- a/hw/rtl/om/VX_om_ingress.sv +++ b/hw/rtl/om/VX_om_ingress.sv @@ -52,7 +52,8 @@ module VX_om_ingress import VX_gpu_pkg::*, VX_om_pkg::*; #( VX_mem_bus_if.slave mem_bus_if, // aperture writes, peeled by VX_om_steer VX_om_bus_if.master om_bus_if, // fragments -> VX_om_bus_arb -> VX_om_core - output wire busy + output wire busy, + output wire frag_pending ); `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_PARAM (TAG_WIDTH) @@ -307,6 +308,10 @@ module VX_om_ingress import VX_gpu_pkg::*, VX_om_pkg::*; #( assign busy = (open_valid != '0) || rec_valid || om_bus_if.req_valid; + // Work that completes without another beat. An open record waits for its + // depth beat, which may be queued behind a flush token held at the steer. + assign frag_pending = rec_valid || om_bus_if.req_valid; + `RUNTIME_ASSERT(~(s2_fire && alloc && ~has_free), ("%t: *** %s: OM ingress hold table overflow — MAX_OPEN bound is wrong", $time, INSTANCE_ID)) `RUNTIME_ASSERT(~(s2_fire && s2_is_pair && s2_is_depth && ~hit), diff --git a/hw/rtl/om/VX_om_steer.sv b/hw/rtl/om/VX_om_steer.sv index cb94dce869..7aa803e5ee 100644 --- a/hw/rtl/om/VX_om_steer.sv +++ b/hw/rtl/om/VX_om_steer.sv @@ -25,10 +25,12 @@ // aperture store is presented to the caches as an ordinary IO store (they carry // attr through verbatim), so this bit arrives untouched. See VX_lsu_slice. // -// Buffering: only the OM leg is registered. The L2 pass-through stays -// combinational -- it is not a new driver (the socket already registers its -// outgoing bus), and a skid here would sit on the critical L1->L2 path and -// double the switch's cost for nothing. +// Flush ordering: the steer is a fork, so a flush token bound for L2 is held +// here until the OM side has been flushed. Every aperture store this trunk +// carried before the token is past the steer by then, so the OM flush, which +// waits for the OM to drain, covers them; then the token goes on to L2. +// +// Buffering: each leg presents a registered master interface (OUT_BUF). `include "VX_define.vh" @@ -43,7 +45,11 @@ module VX_om_steer import VX_gpu_pkg::*; #( VX_mem_bus_if.slave bus_in_if, // from the socket VX_mem_bus_if.master l2_out_if, // everything else -> L2 - VX_mem_bus_if.master om_out_if // aperture writes -> OM ingress (posted) + VX_mem_bus_if.master om_out_if, // aperture writes -> OM ingress (posted) + + output wire flush_req, // a flush token is held until the OM side is flushed + input wire flush_ack, // pulse: the OM side was flushed after flush_req rose + output wire om_pending // an aperture write is still buffered on the OM leg ); `UNUSED_SPARAM (INSTANCE_ID) @@ -53,11 +59,37 @@ module VX_om_steer import VX_gpu_pkg::*; #( wire is_om = bus_in_if.req_data.attr[MEM_ATTR_OM_OFFS]; + // ── flush tokens: held until the OM side is flushed ──────────────────── + wire is_flush = bus_in_if.req_valid && ~is_om && bus_in_if.req_data.attr[MEM_ATTR_FLUSH_OFFS]; + reg flush_req_r; // token held, waiting for the OM flush + reg flush_pass_r; // OM flushed: the token may go to L2 + wire flush_block = is_flush && ~flush_pass_r; + // ── request: demux by is_addr_om ─────────────────────────────────────── wire l2_ready; wire om_ready; - assign bus_in_if.req_ready = is_om ? om_ready : l2_ready; + assign bus_in_if.req_ready = is_om ? om_ready : (l2_ready && ~flush_block); + + wire flush_fire = is_flush && flush_pass_r && l2_ready; + always @(posedge clk) begin + if (reset) begin + flush_req_r <= 1'b0; + flush_pass_r <= 1'b0; + end else begin + if (flush_req_r && flush_ack) begin + flush_req_r <= 1'b0; + flush_pass_r <= 1'b1; + end else if (flush_block) begin + flush_req_r <= 1'b1; + end + if (flush_fire) begin + flush_pass_r <= 1'b0; + end + end + end + assign flush_req = flush_req_r; + assign om_pending = om_out_if.req_valid; // Both legs go through an OUT_BUF elastic buffer so the socket->L2 trunk and // the OM aperture path each present a fully-registered master interface. @@ -71,7 +103,7 @@ module VX_om_steer import VX_gpu_pkg::*; #( ) l2_req_buf ( .clk (clk), .reset (reset), - .valid_in (bus_in_if.req_valid && ~is_om), + .valid_in (bus_in_if.req_valid && ~is_om && ~flush_block), .ready_in (l2_ready), .data_in (l2_req_data_in), .data_out (l2_req_data_out), diff --git a/hw/unittest/cp_dcr_proxy/main.cpp b/hw/unittest/cp_dcr_proxy/main.cpp index 56f3e18cf3..861c0c2932 100644 --- a/hw/unittest/cp_dcr_proxy/main.cpp +++ b/hw/unittest/cp_dcr_proxy/main.cpp @@ -9,6 +9,8 @@ // S_REQ → write: S_DONE; read: S_WAIT_RSP // S_WAIT_RSP → dcr_rsp_valid ⇒ latch rsp_data_r, S_DONE // S_DONE → IDLE +// CMD_CACHE_FLUSH: IDLE → S_START (broadcast write) → per core +// S_REQ/S_WAIT_RSP (read with the core index) → S_DONE after the last. // // Coverage: // 1. Reset: no transitions, dcr_req_valid stays 0, done stays 0. @@ -19,11 +21,16 @@ // S_WAIT_RSP until dcr_rsp_valid arrives; rsp_data is latched // into last_rsp_data and visible while done pulses. // 4. Back-to-back write→read: FSM re-arms cleanly. -// 5. WAIT_RSP hangs if rsp_valid never arrives (no spurious done). +// 5. CMD_CACHE_FLUSH over 3 cores: one broadcast write to +// VX_DCR_BASE_CACHE_FLUSH, then one read per core in order with the +// core index as data, each waiting for its response; done only +// after the last core answers. +// 6. CMD_CACHE_FLUSH over 0 cores retires without any DCR request. // ============================================================================ #include "vl_simulator.h" #include "VVX_cp_dcr_proxy_top.h" +#include #include #include #include @@ -51,6 +58,7 @@ void sim_trace_enable(bool e) { trace_en = e; } enum CmdOp : uint8_t { OP_DCR_WRITE = 0x04, OP_DCR_READ = 0x05, + OP_CACHE_FLUSH = 0x0A, }; // Same packed-cmd layout as the cp_engine TB: hdr in the MSB word @@ -194,6 +202,61 @@ int main(int argc, char** argv) { EXPECT(sim->done == 1, "re-arm: done not asserted"); cycle(sim, tick); + // ----- Test 5: CMD_CACHE_FLUSH over 3 cores. arg0 = core count. ----- + constexpr uint32_t NCORES = 3; + set_cmd(sim.operator->(), OP_CACHE_FLUSH, NCORES); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_START + sim->grant = 0; + + // S_START: one write that every core receives. + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSH: start write not issued"); + EXPECT(sim->dcr_req_rw == 1, "FLUSH: start must be a write"); + EXPECT(sim->dcr_req_addr == VX_DCR_BASE_CACHE_FLUSH, "FLUSH: start addr mismatch"); + EXPECT(sim->done == 0, "FLUSH: done premature in S_START"); + cycle(sim, tick); // S_START → S_REQ + + for (uint32_t cid = 0; cid < NCORES; ++cid) { + // S_REQ: this core's completion read. + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSH: per-core read not issued"); + EXPECT(sim->dcr_req_rw == 0, "FLUSH: per-core request must be a read"); + EXPECT(sim->dcr_req_addr == VX_DCR_BASE_CACHE_FLUSH, "FLUSH: read addr mismatch"); + EXPECT((sim->dcr_req_data & 0xffff) == cid, "FLUSH: read must target the cores in order"); + EXPECT(sim->done == 0, "FLUSH: done before every core answered"); + cycle(sim, tick); // S_REQ → S_WAIT_RSP + + // Nothing moves until this core answers. + for (int i = 0; i < 3; ++i) { + sim->eval(); + EXPECT(sim->dcr_req_valid == 0, "FLUSH: request while waiting for a core"); + EXPECT(sim->done == 0, "FLUSH: spurious done while waiting for a core"); + cycle(sim, tick); + } + sim->dcr_rsp_valid = 1; + sim->dcr_rsp_data = 0; + cycle(sim, tick); // → next core's S_REQ, or S_DONE + sim->dcr_rsp_valid = 0; + } + + sim->eval(); + EXPECT(sim->done == 1, "FLUSH: done not asserted after the last core"); + EXPECT(sim->dcr_req_valid == 0, "FLUSH: request after the last core"); + cycle(sim, tick); // S_DONE → IDLE + sim->eval(); + EXPECT(sim->done == 0, "FLUSH: done should pulse only one cycle"); + + // ----- Test 6: CMD_CACHE_FLUSH over 0 cores retires at once. ----- + set_cmd(sim.operator->(), OP_CACHE_FLUSH, 0); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_DONE + sim->grant = 0; + sim->eval(); + EXPECT(sim->dcr_req_valid == 0, "FLUSH0: no request expected"); + EXPECT(sim->done == 1, "FLUSH0: done not asserted"); + cycle(sim, tick); + std::printf("PASSED\n"); return 0; } diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index 51aed0b6d5..c16aae9aaf 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -322,6 +322,7 @@ module VX_gfx_top import VX_gpu_pkg::* .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) tcache ( @@ -473,6 +474,7 @@ module VX_gfx_top import VX_gpu_pkg::* .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) rtcache ( diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 7ac7aec7f0..6e2f7a32e0 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -410,6 +410,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .WRITE_ENABLE (0), .REPL_POLICY (`VX_CFG_ICACHE_REPL_POLICY), .NC_ENABLE (0), + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) icache ( diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 0972440954..c6e97fb46f 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -328,6 +328,22 @@ void CommandProcessor::apply_qmd_(uint64_t qmd_addr) { } } +// One DCR write to VX_DCR_BASE_CACHE_FLUSH reaches every core and starts its +// flush, so the cores flush concurrently and share one walk of the shared +// caches; the per-core reads, answered once that core's flush completes, are +// serial because the DCR response bus cannot carry two responses at once. +void CommandProcessor::flush_cores_(uint32_t num_cores) { + if (num_cores == 0 || !hooks_.vortex_dcr_read) { + return; + } + if (hooks_.vortex_dcr_write) { + hooks_.vortex_dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); + } + for (uint32_t cid = 0; cid < num_cores; ++cid) { + (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); + } +} + // Execute one draw-bundle step. Inline ops (DCR_WRITE/DCR_READ/CACHE_FLUSH) // complete immediately and return false; a launch step (LAUNCH/LAUNCH_QMD) // kicks the launch sub-FSM and returns true so the caller waits for the drain @@ -351,11 +367,7 @@ bool CommandProcessor::exec_inline_cmd_(const Cmd& c) { uint32_t(c.arg0 & 0xFFF), uint32_t(c.arg1 & 0xFFFFFFFF)); return false; case OP_CACHE_FLUSH: - if (hooks_.vortex_dcr_read) { - uint32_t n = uint32_t(c.arg0 & 0xFFFFFFFF); - for (uint32_t cid = 0; cid < n; ++cid) - (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); - } + this->flush_cores_(uint32_t(c.arg0 & 0xFFFFFFFF)); return false; default: // NOP / FENCE / unknown step — no-op (draws don't use MEM_*/EVENT_*). @@ -484,15 +496,8 @@ void CommandProcessor::tick_engine() { } eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_CACHE_FLUSH) { - // Sweep a per-core DCR-read of VX_DCR_BASE_CACHE_FLUSH across - // [0, num_cores). arg0 carries num_cores; dcr_read routes each - // call to flush_caches() (drains dcache/L2/L3 to memsim). - if (hooks_.vortex_dcr_read) { - uint32_t n = uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF); - for (uint32_t cid = 0; cid < n; ++cid) { - (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); - } - } + // arg0 carries num_cores. + this->flush_cores_(uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF)); eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_EVENT_SIG) { // CMD_EVENT_SIGNAL: write arg1 (8-byte value) to arg0 diff --git a/sim/common/cmd_processor.h b/sim/common/cmd_processor.h index 0ccd6fc379..38e0f80f4c 100644 --- a/sim/common/cmd_processor.h +++ b/sim/common/cmd_processor.h @@ -242,6 +242,8 @@ class CommandProcessor { // CMD_LAUNCH_QMD: read the in-memory KMU descriptor at `qmd_addr` and // replay its {dcr_addr,value} pairs through the DCR-write hook. void apply_qmd_(uint64_t qmd_addr); + // CMD_CACHE_FLUSH across cores [0, num_cores). + void flush_cores_(uint32_t num_cores); // Advance the launch FSM one step using cur_cmd_. void tick_launch(); // Advance the engine FSM one step. diff --git a/sim/rtlsim/main.cpp b/sim/rtlsim/main.cpp index ba92ba1a03..e6287b8488 100644 --- a/sim/rtlsim/main.cpp +++ b/sim/rtlsim/main.cpp @@ -139,6 +139,7 @@ int main(int argc, char **argv) { // flush GPU caches before reading back results { uint32_t dummy; + processor.dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); for (uint32_t cid = 0; cid < VX_CFG_NUM_CORES * VX_CFG_NUM_CLUSTERS; ++cid) { processor.dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid, &dummy); } diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index b383d556ff..416c60208b 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -237,8 +237,8 @@ class Processor::Impl { device_->dcr_req_data = tag; this->tick(); device_->dcr_req_valid = 0; - this->tick(); - // READ response is returned when dcr_rsp_valid is high + // A core whose flush already finished answers on the next cycle, and the + // response is a one-cycle pulse, so start polling right away. while (!device_->dcr_rsp_valid) { this->tick(); } diff --git a/sim/simx/main.cpp b/sim/simx/main.cpp index ff1d406692..41f8bd473c 100644 --- a/sim/simx/main.cpp +++ b/sim/simx/main.cpp @@ -233,6 +233,7 @@ int main(int argc, char **argv) { // flush GPU caches before reading back results { uint32_t dummy; + processor.dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); for (uint32_t cid = 0; cid < VX_CFG_NUM_CORES * VX_CFG_NUM_CLUSTERS; ++cid) { processor.dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid, &dummy); } diff --git a/sw/runtime/rtlsim/vortex.cpp b/sw/runtime/rtlsim/vortex.cpp index e0677b66c8..e5aa68d64f 100644 --- a/sw/runtime/rtlsim/vortex.cpp +++ b/sw/runtime/rtlsim/vortex.cpp @@ -153,6 +153,11 @@ class vx_device { ram_.write(static_cast(src), addr, bytes); }; h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { + // Wait for any background processor_.run() to finish so dcr_write + // does not race the Verilator state. + if (future_.valid()) { + future_.wait(); + } processor_.dcr_write(addr, value); }; h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { diff --git a/sw/runtime/simx/vortex.cpp b/sw/runtime/simx/vortex.cpp index 3d3088850f..d72ba8d60d 100644 --- a/sw/runtime/simx/vortex.cpp +++ b/sw/runtime/simx/vortex.cpp @@ -160,6 +160,10 @@ class vx_device { ram_.write(static_cast(src), addr, bytes); }; h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { + // Wait for any background processor_.run() to finish before writing DCR. + if (future_.valid()) { + future_.wait(); + } processor_.dcr_write(addr, value); }; h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { diff --git a/tests/regression/Makefile b/tests/regression/Makefile index e50ebfff69..3fd4c75708 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -15,6 +15,7 @@ TESTS := \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ async_barrier async_gbarrier gbar_phase bar_slot_phase packld wgather wsync quad_lod \ occupancy multikernel reopen module_reload single_cta \ + flush_coherence \ vm_test vm_fault vm_stress # --- common exclude list --------------------------------------------- diff --git a/tests/regression/flush_coherence/Makefile b/tests/regression/flush_coherence/Makefile new file mode 100644 index 0000000000..5ceced321a --- /dev/null +++ b/tests/regression/flush_coherence/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := flush_coherence + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n2048 -r16 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/flush_coherence/common.h b/tests/regression/flush_coherence/common.h new file mode 100644 index 0000000000..bbdabc90a1 --- /dev/null +++ b/tests/regression/flush_coherence/common.h @@ -0,0 +1,21 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +typedef struct { + uint32_t count; + uint32_t round; + uint32_t rotate; + uint32_t burst; + uint64_t src_addr; + uint64_t dst_addr; + uint64_t acc_addr; + uint64_t tail_addr; +} kernel_arg_t; + +inline uint32_t tail_value(uint32_t i, uint32_t k, uint32_t round) { + return (i * 0x01000193u) ^ (k << 20) ^ (round * 0x2545F491u); +} + +#endif diff --git a/tests/regression/flush_coherence/kernel.cpp b/tests/regression/flush_coherence/kernel.cpp new file mode 100644 index 0000000000..de656daa29 --- /dev/null +++ b/tests/regression/flush_coherence/kernel.cpp @@ -0,0 +1,28 @@ +#include +#include "common.h" + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= arg->count) { + return; + } + // The rotation moves each element, and so each line, to another thread + // and core from one launch to the next. + uint32_t i = tid + arg->rotate; + if (i >= arg->count) { + i -= arg->count; + } + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + auto acc = reinterpret_cast(arg->acc_addr); + uint32_t v = src[i]; + acc[i] += v ^ arg->round; + // A store burst saturates the shared levels, so the final stores are + // still queued in front of them when the flushes start. + auto tail = reinterpret_cast(arg->tail_addr); + for (uint32_t k = 0; k < arg->burst; ++k) { + tail[k * arg->count + i] = tail_value(i, k, arg->round); + } + // Ending on a store leaves the kernel's last writes in flight at exit. + dst[i] = v * 3u + arg->round; +} diff --git a/tests/regression/flush_coherence/main.cpp b/tests/regression/flush_coherence/main.cpp new file mode 100644 index 0000000000..573dbe66ad --- /dev/null +++ b/tests/regression/flush_coherence/main.cpp @@ -0,0 +1,190 @@ +// Kernel-boundary flush coherence under back-to-back launches. +// +// Every round the host rewrites src, then launches a kernel that reads src, +// accumulates into acc and writes dst. The element-to-thread mapping is +// rotated on three rounds out of four, so a line usually moves to another +// thread and often to another core. The post-launch cache flush must: +// - publish each launch's writes before the host reads dst or acc; +// - invalidate every core's (and every shared level's) copy of src before +// the next launch, or a core re-reading a line it cached would see the +// previous round's value; +// - make one launch's acc writes visible to the next launch's reads on +// other cores, also when no host transfer separates them (odd rounds do +// not read dst back). +// With -b, each thread ends on a burst of stores to a tail buffer, so the +// last stores may still be queued in front of the shared levels when the +// flushes start, and every round is checked. + +#include +#include "common.h" + +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ + } while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t count = 2048; +uint32_t rounds = 16; +uint32_t burst = 0; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:r:b:k:h")) != -1) { + switch (c) { + case 'n': count = (uint32_t)std::atoi(optarg); break; + case 'r': rounds = (uint32_t)std::atoi(optarg); break; + case 'b': burst = (uint32_t)std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::printf("Usage: [-k kernel] [-n words] [-r rounds] [-b burst] [-h]\n"); + std::exit(c == 'h' ? 0 : -1); + } + } +} + +uint32_t src_value(uint32_t i, uint32_t round) { + uint32_t x = i * 0x9E3779B1u + round * 0x85EBCA77u + 1; + x ^= x >> 15; + return x * 0x2C1B3C6Du; +} + +uint32_t rotation(uint32_t round) { + return (round % 4 == 0) ? 0 : (round * 0x9E3779B1u) % count; +} + +int check(const char* what, uint32_t round, const std::vector& got, + const std::vector& ref) { + int errors = 0; + for (size_t i = 0; i < ref.size(); ++i) { + if (got[i] != ref[i]) { + if (errors < 8) { + std::printf("*** round %u %s[%zu]: expected=0x%08x actual=0x%08x\n", + round, what, i, ref[i], got[i]); + } + ++errors; + } + } + return errors; +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + const uint64_t bytes = (uint64_t)count * sizeof(uint32_t); + const uint64_t tail_bytes = bytes * burst; + std::printf("flush_coherence: n=%u rounds=%u burst=%u\n", count, rounds, burst); + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + vx_buffer_h src_buf = nullptr, dst_buf = nullptr, acc_buf = nullptr, tail_buf = nullptr; + CHECK(vx_buffer_create(dev, bytes, VX_MEM_READ, &src_buf)); + CHECK(vx_buffer_create(dev, bytes, VX_MEM_WRITE, &dst_buf)); + CHECK(vx_buffer_create(dev, bytes, VX_MEM_READ_WRITE, &acc_buf)); + if (burst) { + CHECK(vx_buffer_create(dev, tail_bytes, VX_MEM_WRITE, &tail_buf)); + } + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t arg{}; + arg.count = count; + CHECK(vx_buffer_address(src_buf, &arg.src_addr)); + CHECK(vx_buffer_address(dst_buf, &arg.dst_addr)); + CHECK(vx_buffer_address(acc_buf, &arg.acc_addr)); + if (burst) { + CHECK(vx_buffer_address(tail_buf, &arg.tail_addr)); + } + arg.burst = burst; + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &count, grid, block)); + + // Enqueued writes and launches read their host data asynchronously, so + // each round gets its own source buffer and argument block. + std::vector> h_src(rounds, std::vector(count)); + std::vector args(rounds, arg); + std::vector h_dst(count), h_acc(count, 0); + std::vector ref_dst(count), ref_acc(count, 0); + std::vector h_tail(count * burst), ref_tail(count * burst); + CHECK(vx_enqueue_write(q, acc_buf, 0, h_acc.data(), bytes, 0, nullptr, nullptr)); + + int errors = 0; + for (uint32_t r = 0; r < rounds; ++r) { + auto& src = h_src[r]; + for (uint32_t i = 0; i < count; ++i) { + src[i] = src_value(i, r); + ref_dst[i] = src[i] * 3u + r; + ref_acc[i] += src[i] ^ r; + for (uint32_t k = 0; k < burst; ++k) { + ref_tail[k * count + i] = tail_value(i, k, r); + } + } + CHECK(vx_enqueue_write(q, src_buf, 0, src.data(), bytes, 0, nullptr, nullptr)); + + args[r].round = r; + args[r].rotate = rotation(r); + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &args[r]; + li.args_size = sizeof(args[r]); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0] = block[0]; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, nullptr)); + + if (burst || r % 2 == 0 || r + 1 == rounds) { + CHECK(vx_enqueue_read(q, h_dst.data(), dst_buf, 0, bytes, 0, nullptr, nullptr)); + if (burst) { + CHECK(vx_enqueue_read(q, h_tail.data(), tail_buf, 0, tail_bytes, 0, nullptr, nullptr)); + } + CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); + errors += check("dst", r, h_dst, ref_dst); + if (burst) { + errors += check("tail", r, h_tail, ref_tail); + } + } + } + + CHECK(vx_enqueue_read(q, h_acc.data(), acc_buf, 0, bytes, 0, nullptr, nullptr)); + CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); + errors += check("acc", rounds, h_acc, ref_acc); + + if (tail_buf) { + vx_buffer_release(tail_buf); + } + vx_buffer_release(acc_buf); + vx_buffer_release(dst_buf); + vx_buffer_release(src_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_release(dev); + + if (errors) { + std::printf("Found %d errors!\nFAILED!\n", errors); + return 1; + } + std::printf("PASSED!\n"); + return 0; +} From ef75f8e624643b92550af0d42d92fdc920830fce Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 14 Oct 2026 19:05:09 -0700 Subject: [PATCH 031/336] Hash the cache bank select so power-of-two strides spread over the banks With the bank taken straight from the address bits above the word offset, any stride that is a multiple of NUM_BANKS x LINE_SIZE lands every access of a warp in one or two banks: a 1 KiB row stride on a 16-bank cache uses two banks while the rest idle. Matrix columns, transposed tiles and per-row reductions all produce such strides. GPUs select banks and L2 slices with an XOR hash of the higher address bits instead. The bank is now the raw bank bits XORed with a function of the line address above them: IPOLY(5) for 16 banks and IPOLY(37) for 32 (Rau, ISCA 1991), an XOR fold otherwise. The same XOR restores the raw bits, so it is applied once on the request side and once where the memory address is rebuilt; each bank forms its raw bits before the memory crossbar, so the outgoing address still leaves from the crossbar's output register. Banks keep storing the line address without the bank field, so tags, MSHR, flush tokens and sectoring are unchanged. CS_BANK_TO_FULL_ADDR applies the hash so trace and assert addresses are real. SimX implements the identical function. Defaults: VX_CFG_L2_BANK_HASH = 1, and VX_CFG_DCACHE_BANK_HASH on from 4 banks. With two banks the hash can only swap lines on parity; requests that no longer serialize at a hot bank reach DRAM together and the queuing outweighs the spreading (sgemv-nt4 +8.8%, mxfp8 +7.4% when forced on). A 2-bank L2 measured neutral. perf_gate (rtlsim, re-recorded with --update-baselines; retired instructions unchanged everywhere, cases without a hashed cache unchanged): sgemv-nt16 -32.8%, fp16-mc-nt16 -13.8%, wgmma-dxa -13.5%, sgemm-mc-nt16 -8.2%, wgmma-sp-dxa -7.2%, sgemmx-nt16 -4.9%, wgmma-dxa-mcast-nt16 -3.7%, sgemm-nt16 -3.1%; worst +0.09%. fpga_gate (U55C, 300 MHz target): cache DUT 281.4 MHz / 27.4K LUT with the hash on (278.8 MHz before); core DUT 279.4 MHz, identical to the parent commit, so the core's distance from its 298 MHz baseline predates this change (its worst paths are in the LSU and operand collectors). Tests: cache:bank_hash-1..4 (hash off; 16-thread IPOLY dcache; 4-core write-back L2; write-back dirty-byte dcache) on SimX and rtlsim. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 7 +++++ ci/baselines/perf/core.json | 50 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 34 ++++++++++----------- ci/baselines/perf/graphics.json | 30 +++++++++---------- ci/baselines/perf/raytracing.json | 10 +++---- ci/baselines/perf/tensor.json | 14 ++++----- ci/baselines/perf/tensor_mx.json | 2 +- ci/baselines/perf/tensor_sp.json | 2 +- ci/baselines/perf/tensor_wg.json | 6 ++-- ci/testcases/cache.yaml | 40 +++++++++++++++++++++++++ docs/designs/cache_subsystem.md | 20 +++++++++++-- hw/rtl/VX_cluster.sv | 1 + hw/rtl/VX_gpu_pkg.sv | 28 +++++++++++++++++ hw/rtl/VX_socket.sv | 1 + hw/rtl/cache/VX_cache.sv | 49 +++++++++++++++++++++--------- hw/rtl/cache/VX_cache_bank.sv | 6 +++- hw/rtl/cache/VX_cache_cluster.sv | 2 ++ hw/rtl/cache/VX_cache_define.vh | 3 +- hw/rtl/cache/VX_cache_mshr.sv | 2 ++ hw/rtl/cache/VX_cache_wrap.sv | 2 ++ hw/unittest/cache/VX_cache_top.sv | 4 +++ sim/simx/cluster.cpp | 1 + sim/simx/mem/addr_hash.h | 46 ++++++++++++++++++++++++++++ sim/simx/mem/cache.cpp | 37 ++++++++++++++++++++--- sim/simx/mem/cache.h | 1 + sim/simx/socket.cpp | 1 + 26 files changed, 303 insertions(+), 96 deletions(-) create mode 100644 sim/simx/mem/addr_hash.h diff --git a/VX_config.toml b/VX_config.toml index f51186cdec..b3b89c6833 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -235,6 +235,10 @@ VX_CFG_NUM_ICACHES = "expr: up($VX_CFG_SOCKET_SIZE / 4) if $VX_CFG_ICACHE_ENABLE VX_CFG_NUM_DCACHES = "expr: up($VX_CFG_SOCKET_SIZE / 4) if $VX_CFG_DCACHE_ENABLE else 0" VX_CFG_DCACHE_NUM_BANKS = "expr: pow(2, clog2(min($VX_CFG_DCACHE_NUM_REQS, 16)))" +# XOR/IPOLY bank-select hash, so power-of-two strides spread over the banks. +# With two banks it can only swap a line on parity, and the extra parallel +# misses cost more in memory queuing than the spreading saves. +VX_CFG_DCACHE_BANK_HASH = "expr: int($VX_CFG_DCACHE_NUM_BANKS >= 4)" VX_CFG_L1_MEM_PORTS = "expr: min($VX_CFG_DCACHE_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS) if ($VX_CFG_ICACHE_ENABLE or $VX_CFG_DCACHE_ENABLE) else min($VX_CFG_DCACHE_NUM_REQS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS)" @@ -251,6 +255,9 @@ VX_CFG_L2_MREQ_SIZE = 0 VX_CFG_L2_CRSQ_SIZE = 0 VX_CFG_L2_NUM_BANKS = "expr: pow(2, clog2(min($VX_CFG_L2_NUM_REQS, 16)))" +# On at every bank count: unlike the dcache, a 2-bank L2 measured neutral +# (sgemm-mc-nt4 -0.6%, fp16-mc-nt4 +0.1%), and it gains from 4 banks up. +VX_CFG_L2_BANK_HASH = 1 VX_CFG_L2_MEM_PORTS = "expr: min($VX_CFG_L2_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS) if $VX_CFG_L2_ENABLE else min($VX_CFG_L2_NUM_REQS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS)" [l3cache] diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 77e8c4785a..a02138c6d0 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,12 +1,12 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 563524, + "cycles": 564053, "instrs": 137486 }, "app": "raycast", "args": "", - "config_hash": "396cee9df6025cdb", + "config_hash": "7a2c92e634460e90", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -17,40 +17,40 @@ }, "app": "raycast", "args": "", - "config_hash": "262f24032ef14b58", + "config_hash": "2532cbe6c87ca233", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 787850, + "cycles": 723234, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "ad2b37492d761e8f", + "config_hash": "6106872241e50c60", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2239426, + "cycles": 2227069, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "606d011dc6f612d3", + "config_hash": "ecbd8e8a804188dd", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1838654, + "cycles": 1782470, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "9390754d3719fca3", + "config_hash": "6dcefd7e66ed0cc0", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -61,18 +61,18 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "740fd84fa2938103", + "config_hash": "74a554f940a83c4a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1290130, + "cycles": 1226398, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "2e1c44e05f9ac34d", + "config_hash": "efc73b5e23a33b8f", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -83,18 +83,18 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "024adb49ed96bdd3", + "config_hash": "0f2e76cd4d21f998", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 685107, + "cycles": 460344, "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "a3d9df9695acbb95", + "config_hash": "bc1d0a1801524d27", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -105,18 +105,18 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "0e8990b25e64ad7f", + "config_hash": "7fbf266181d2ec8b", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1272929, + "cycles": 1271056, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "9e0c1c9767c760ea", + "config_hash": "c53ddc9ba2cd96bd", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -127,18 +127,18 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "eb5fed4bfce19d76", + "config_hash": "6dea102e1b20464c", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2306960, + "cycles": 2272549, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "6925a5138be930ea", + "config_hash": "542851d41daf9431", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -149,18 +149,18 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "b83a1e3735f79d39", + "config_hash": "fa607fa9e1280813", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 100640, + "cycles": 99953, "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "ef0c2548e7c5bf86", + "config_hash": "0f4e3087ce474e4b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -171,7 +171,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "ce1eb79984a98552", + "config_hash": "0169f39a14561223", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index bf1e1a010c..a331465e29 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -6,18 +6,18 @@ }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "6a815f855f6e0d05", + "config_hash": "2be8c5960886f23f", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 1994, + "cycles": 1992, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "626c5f6244584b1f", + "config_hash": "37f76aae77fd1afd", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,18 +28,18 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "7c32a11d123bfb13", + "config_hash": "c3b3447f163ec6c3", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14022240, + "cycles": 14023305, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "822d0ca194266a27", + "config_hash": "62067f11f3ef5af4", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,18 +50,18 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "87ea544e77719254", + "config_hash": "98c58db8047bceb9", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3941267, + "cycles": 3921184, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "e55b61598f5f10ea", + "config_hash": "845bebf9fbc108b1", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,18 +72,18 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "21da2f2737bf03dd", + "config_hash": "8cf4b0e4e0c7d144", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 82160, + "cycles": 79131, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "20304652a1bfb94c", + "config_hash": "d3c9c80b16cdf2c2", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,29 +94,29 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "bf4520d008c07dc9", + "config_hash": "38993044a82f32e0", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 36021, + "cycles": 31160, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "2a4e017d98740636", + "config_hash": "699f26ecbecc6bb2", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 59789, + "cycles": 55512, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "f05069d7cf0ee3ff", + "config_hash": "8e607400e57e51f5", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index b5ee86bfc2..b402bd0b6f 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,45 +1,45 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 530195, + "cycles": 529875, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "1e2937b20579ad3f", + "config_hash": "0777067f98f2730f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 143620, + "cycles": 143542, "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "edc924bc4f246971", + "config_hash": "550e1e3d5ffe23d0", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 529393, + "cycles": 529389, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "d682cc8c6f99c06b", + "config_hash": "5cc3a870f65d86fa", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 189231, + "cycles": 189356, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "46a66220c1d9956b", + "config_hash": "f74350c24c97b1f5", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,18 +50,18 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "a7554f6fa88474dc", + "config_hash": "8a9749c1c0d8264d", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14785, + "cycles": 14626, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "e2e1879d7d6f9aaf", + "config_hash": "967474845abde936", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,18 +72,18 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "6e48f6b249533cc5", + "config_hash": "40f6d7ca7755b6ca", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 35441, + "cycles": 35297, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "c8c6b30537e93e2f", + "config_hash": "e44189b53e136920", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "a4fc4d97e6e80586", + "config_hash": "f0cdf08354c9ae88", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 1fb0ab7f7b..e0ee3a7da4 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,23 +1,23 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 85045, + "cycles": 85040, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "9a4f28d2555289b5", + "config_hash": "4f78e65dfd2497cb", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53434, + "cycles": 53443, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "aef578263250d585", + "config_hash": "c0af18c64430f778", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "c7b45fe5c1828bc5", + "config_hash": "e394fceca77fe6f1", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 3552c4620b..112a272e80 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -6,29 +6,29 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "da0a50e46103b7dc", + "config_hash": "9cdc8bb95c856a38", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 42404, + "cycles": 36534, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "2d12f1a0058fcc3b", + "config_hash": "153ac3d8001d2be6", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221592, + "cycles": 221752, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "fce886092e60e0eb", + "config_hash": "21c6d64b10dab7cb", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "f0558a3b3dfde51f", + "config_hash": "04953ed130a8c1a9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "24fd7a811fabb8c4", + "config_hash": "26d22a59f411cfbe", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 1dc1b78f41..a49403e311 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "3cfc301023342a7d", + "config_hash": "7c8765889acb4f32", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 2c4f92004c..3ee3f24144 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "90defe3d20b6f739", + "config_hash": "6e523b9d0f6fe4ee", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index c7f771c72f..b2af51b8f2 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "ba81956fe68e94b6", + "config_hash": "c65e3b69950519cb", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "6ee14f25ec573117", + "config_hash": "0d41f1f51f46d61d", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "a9ea3d8bd83a7eed", + "config_hash": "b4944cbd79928ac8", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index de3e6c289d..3a2c5b7088 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -264,3 +264,43 @@ tests: configs: -DVX_CFG_SOCKET_SIZE=3 -DVX_CFG_NUM_ICACHES=2 shape: cores: 3 +# Hashed bank select, on by default for the dcache and the L2. These cover the +# hash off at both levels, 16 threads, and a write-back L2, where every fill and +# write-back rebuilds the address. +- id: bank_hash-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_DCACHE_BANK_HASH=0 -DVX_CFG_L2_BANK_HASH=0 + shape: + cores: 2 + l2cache: true +- id: bank_hash-2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + shape: + threads: 16 +- id: bank_hash-3 + via: blackbox + drivers: + - simx + - rtlsim + app: mstress + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 + shape: + cores: 4 + l2cache: true +- id: bank_hash-4 + via: blackbox + drivers: + - simx + - rtlsim + app: mstress + configs: -DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_DIRTYBYTES=1 + shape: + threads: 16 diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 73f7e308f1..8fce9737ef 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -224,6 +224,22 @@ rather than being a field of its own. Two consequences are worth internalizing: pipeline carrying only `word_idx` and still tell the tag and data stages which sector to mark and write. +**Hashed bank select (`BANK_HASH`).** With the bank taken straight from the +address bits, any power-of-two stride that is a multiple of `NUM_BANKS × LINE` +lands every access in one or two banks. With `BANK_HASH=1` the bank is the raw +bank bits XORed with `cache_bank_hash` of the line address above them +([VX_gpu_pkg.sv](../../hw/rtl/VX_gpu_pkg.sv)): IPOLY for 16 and 32 banks, which +spreads every power-of-two stride evenly, and an XOR fold of the higher bits for +other bank counts. The same mask XORed again restores the raw bits, so it is +applied once on the request side (`core_req_bid`) and once where the memory +address is rebuilt from `{line, bank}`. The banks keep storing the line address +without the bank field, so tags, MSHR, the flush walk and sectoring are +unchanged. `VX_CFG_DCACHE_BANK_HASH` and `VX_CFG_L2_BANK_HASH` set it for the +dcache and the L2; SimX uses the same functions. The L2 default is on; the dcache +default is on from 4 banks, because with two banks the hash can only swap lines on +parity and the extra parallel misses cost more in memory queuing than the +spreading saves. + The memory side transacts in sectors, so the memory address carries `CS_SECTOR_SEL_BITS` below the line address (`CS_MEM_SECTOR_ADDR_WIDTH`, `CS_LINE_SECTOR_ADDR_WIDTH`). With one sector per line these collapse back to the @@ -1001,8 +1017,8 @@ identically at all three levels — the dcache is not special. - **Miss latency hiding** insufficient (banks idle while misses pend) → raise `*_MSHR_SIZE` and check the LSU pending pool upstream; verify `MEM_PORTS` is not serializing the drain. -- **Bank conflicts** (hit-path stalls) → more banks via a smaller - `DCACHE_WORD_SIZE`; check address strides against the line-interleaved mapping. +- **Bank conflicts** (hit-path stalls) → check that the bank hash is on for that + level, then more banks via a smaller `DCACHE_WORD_SIZE`. - **Store bandwidth** bound → enable a shared level so `WRITEBACK` derives to 1 where legal; add `*_DIRTYBYTES` for partial-line store patterns. - **BRAM / timing** pressure → cut `*_NUM_WAYS` or `*_SIZE` before cutting banks; diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 92c3cb4e23..f02c30047b 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -232,6 +232,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .NC_ENABLE (1), .PASSTHRU (!`VX_CFG_L2_ENABLED), .IS_LLC (L2_IS_LLC), + .BANK_HASH (`VX_CFG_L2_BANK_HASH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) l2cache ( .clk (clk), diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 6a09fcfbea..9ae6a9a2a9 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -224,6 +224,34 @@ package VX_gpu_pkg; // wire/reg sizing for any per-cache plumbing. localparam MEM_ATTR_WIDTH = $bits(mem_bus_attr_t); + // Cache bank-select hash: XORed into the raw bank bits, a function of the + // line address above them, so XORing it again restores the raw bits. IPOLY + // for 16 and 32 banks (Rau, ISCA 1991: conflict-free for power-of-two + // strides); an XOR fold of the bits above the bank field otherwise. + function automatic logic [7:0] cache_bank_hash(input logic [31:0] hi, input integer bits); + logic [7:0] f; + f = '0; + if (bits == 4) begin + f[0] = hi[11] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[6] ^ hi[4] ^ hi[3] ^ hi[0]; + f[1] = hi[12] ^ hi[8] ^ hi[7] ^ hi[6] ^ hi[5] ^ hi[3] ^ hi[1] ^ hi[0]; + f[2] = hi[9] ^ hi[8] ^ hi[7] ^ hi[6] ^ hi[4] ^ hi[2] ^ hi[1]; + f[3] = hi[10] ^ hi[9] ^ hi[8] ^ hi[7] ^ hi[5] ^ hi[3] ^ hi[2]; + end else if (bits == 5) begin + f[0] = hi[13] ^ hi[12] ^ hi[11] ^ hi[10] ^ hi[9] ^ hi[6] ^ hi[5] ^ hi[3] ^ hi[0]; + f[1] = hi[14] ^ hi[13] ^ hi[12] ^ hi[11] ^ hi[10] ^ hi[7] ^ hi[6] ^ hi[4] ^ hi[1]; + f[2] = hi[14] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[7] ^ hi[6] ^ hi[3] ^ hi[2] ^ hi[0]; + f[3] = hi[11] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[7] ^ hi[4] ^ hi[3] ^ hi[1]; + f[4] = hi[12] ^ hi[11] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[5] ^ hi[4] ^ hi[2]; + end else if (bits > 0 && bits <= 8) begin + for (integer i = 0; i < 15; ++i) begin + if ((i % bits) == 0) begin + f = f ^ (8'(hi >> i) & 8'((1 << bits) - 1)); + end + end + end + return f; + endfunction + localparam VX_DCR_ADDR_WIDTH = `VX_DCR_ADDR_BITS; localparam VX_DCR_DATA_WIDTH = `VX_DCR_DATA_BITS; diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 652a4509db..315ccf713a 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -220,6 +220,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), .IS_LLC (DCACHE_IS_LLC), + .BANK_HASH (`VX_CFG_DCACHE_BANK_HASH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) dcache ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 46659a46b4..de35722db6 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -51,6 +51,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter FLUSH_FORWARD = 1, // a flush also flushes the next level + parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) + parameter AMO_ENABLE = 0 // synthesize atomic-op logic in the banks ) ( `ifdef PERF_ENABLE @@ -65,6 +67,7 @@ module VX_cache import VX_gpu_pkg::*; #( `STATIC_ASSERT(WRITE_ENABLE || !WRITEBACK, ("invalid parameter: writeback requires write enable")) `STATIC_ASSERT(WRITEBACK || !DIRTY_BYTES, ("invalid parameter: dirty bytes require writeback")) `STATIC_ASSERT(NUM_BANKS >= MEM_PORTS, ("invalid parameter: number of banks must be greater or equal to number of memory ports")) + `STATIC_ASSERT(!BANK_HASH || NUM_BANKS <= 256, ("invalid parameter: bank hash supports up to 256 banks")) localparam REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS); localparam WORD_SEL_WIDTH = `UP(`CS_WORD_SEL_BITS); localparam MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE); @@ -78,7 +81,7 @@ module VX_cache import VX_gpu_pkg::*; #( localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + TAG_WIDTH + `UP(MEM_ATTR_WIDTH); localparam CORE_RSP_DATAW = WORD_WIDTH + TAG_WIDTH; localparam BANK_MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH; - localparam MEM_REQ_DATAW = (`CS_LINE_SECTOR_ADDR_WIDTH + 1 + SECTOR_SIZE + `CS_SECTOR_WIDTH + BANK_MEM_TAG_WIDTH + `UP(MEM_ATTR_WIDTH)); + localparam MEM_REQ_DATAW = (`UP(BANK_SEL_BITS) + `CS_LINE_SECTOR_ADDR_WIDTH + 1 + SECTOR_SIZE + `CS_SECTOR_WIDTH + BANK_MEM_TAG_WIDTH + `UP(MEM_ATTR_WIDTH)); localparam MEM_RSP_DATAW = `CS_SECTOR_WIDTH + MEM_TAG_WIDTH; localparam MEM_PORTS_SEL_BITS = `CLOG2(MEM_PORTS); localparam MEM_PORTS_SEL_WIDTH = `UP(MEM_PORTS_SEL_BITS); @@ -283,7 +286,12 @@ module VX_cache import VX_gpu_pkg::*; #( end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_req_bid if (NUM_BANKS > 1) begin : g_multibanks - assign core_req_bid[i] = core_req_addr[i][WORD_SEL_BITS +: BANK_SEL_BITS]; + wire [BANK_SEL_BITS-1:0] raw_bid = core_req_addr[i][WORD_SEL_BITS +: BANK_SEL_BITS]; + if (BANK_HASH != 0) begin : g_hash + assign core_req_bid[i] = raw_bid ^ BANK_SEL_BITS'(cache_bank_hash(32'(core_req_line_addr[i]), BANK_SEL_BITS)); + end else begin : g_raw + assign core_req_bid[i] = raw_bid; + end end else begin : g_singlebank assign core_req_bid[i] = '0; end @@ -341,6 +349,7 @@ module VX_cache import VX_gpu_pkg::*; #( for (genvar bank_id = 0; bank_id < NUM_BANKS; ++bank_id) begin : g_banks VX_cache_bank #( .BANK_ID (bank_id), + .BANK_HASH (BANK_HASH), .INSTANCE_ID (`SFORMATF(("%s-bank%0d", INSTANCE_ID, bank_id))), .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -459,7 +468,16 @@ module VX_cache import VX_gpu_pkg::*; #( end wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] per_bank_mem_req_pdata; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_mem_req_pdata + // the raw bank bits of the memory address, formed while the bank id is a constant + wire [`UP(BANK_SEL_BITS)-1:0] raw_bid; + if (NUM_BANKS > 1 && BANK_HASH != 0) begin : g_hash + wire [`CS_LINE_ADDR_WIDTH-1:0] line_addr = per_bank_mem_req_addr[i][`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]; + assign raw_bid = BANK_SEL_BITS'(i) ^ BANK_SEL_BITS'(cache_bank_hash(32'(line_addr), BANK_SEL_BITS)); + end else begin : g_raw + assign raw_bid = `UP(BANK_SEL_BITS)'(i); + end assign per_bank_mem_req_pdata[i] = { + raw_bid, per_bank_mem_req_addr[i], per_bank_mem_req_rw[i], per_bank_mem_req_byteen[i], @@ -487,16 +505,18 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0] mem_req_xbar_valid; wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] mem_req_xbar_pdata; wire [NUM_BANKS-1:0] mem_req_xbar_ready; - wire [MEM_REQ_DATAW-1:0] mem_flush_pdata = { - `CS_LINE_SECTOR_ADDR_WIDTH'(0), - 1'b0, - SECTOR_SIZE'(0), - `CS_SECTOR_WIDTH'(0), - BANK_MEM_TAG_WIDTH'(0), - `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) - }; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_mem_req_xbar_in if (i < MEM_PORTS) begin : g_flush_req + // line 0 hashes to 0, so the raw bank bits are the lane's bank id + wire [MEM_REQ_DATAW-1:0] mem_flush_pdata = { + `UP(BANK_SEL_BITS)'(i), + `CS_LINE_SECTOR_ADDR_WIDTH'(0), + 1'b0, + SECTOR_SIZE'(0), + `CS_SECTOR_WIDTH'(0), + BANK_MEM_TAG_WIDTH'(0), + `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) + }; assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; assign mem_flush_ready[i] = mem_req_xbar_ready[i] && ~per_bank_mem_req_valid[i]; @@ -558,6 +578,7 @@ module VX_cache import VX_gpu_pkg::*; #( `UNUSED_PIN (collisions) ); for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_tmp_if + wire [`UP(BANK_SEL_BITS)-1:0] mem_req_raw_bid_w; wire mem_req_rw_w; wire [`CS_LINE_SECTOR_ADDR_WIDTH-1:0] mem_req_addr_w; wire [SECTOR_SIZE-1:0] mem_req_byteen_w; @@ -565,6 +586,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [BANK_MEM_TAG_WIDTH-1:0] mem_req_tag_w; wire [`UP(MEM_ATTR_WIDTH)-1:0] mem_req_attr_w; assign { + mem_req_raw_bid_w, mem_req_addr_w, mem_req_rw_w, mem_req_byteen_w, @@ -588,17 +610,16 @@ module VX_cache import VX_gpu_pkg::*; #( if (NUM_BANKS != MEM_PORTS) begin : g_arb_sel wire [MEM_ARB_SEL_BITS-1:0] mem_req_arb_sel; assign mem_req_arb_sel = mem_req_tmp_idx[i][`LOG2UP(NUM_BANKS)-1 -: MEM_ARB_SEL_BITS]; - assign mem_bus_tmp_if[i].req_data.addr = (`CS_MEM_SECTOR_ADDR_WIDTH'({mem_req_line_w, mem_req_tmp_idx[i]}) << `CS_SECTOR_SEL_BITS) - | `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_sec_w); assign mem_bus_tmp_if[i].req_data.tag = {mem_req_tag_w, mem_req_arb_sel}; end else begin : g_no_arb_sel `UNUSED_VAR (mem_req_tmp_idx) - assign mem_bus_tmp_if[i].req_data.addr = (`CS_MEM_SECTOR_ADDR_WIDTH'({mem_req_line_w, MEM_PORTS_SEL_WIDTH'(i)}) << `CS_SECTOR_SEL_BITS) - | `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_sec_w); assign mem_bus_tmp_if[i].req_data.tag = MEM_TAG_WIDTH'(mem_req_tag_w); end + assign mem_bus_tmp_if[i].req_data.addr = (`CS_MEM_SECTOR_ADDR_WIDTH'({mem_req_line_w, mem_req_raw_bid_w}) << `CS_SECTOR_SEL_BITS) + | `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_sec_w); end else begin : g_singlebank `UNUSED_VAR (mem_req_tmp_idx) + `UNUSED_VAR (mem_req_raw_bid_w) `UNUSED_VAR (mem_req_line_w) `UNUSED_VAR (mem_req_sec_w) assign mem_bus_tmp_if[i].req_data.addr = `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_addr_w); diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index ee64fe7937..410a61dcc6 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -16,6 +16,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID= "", parameter BANK_ID = 0, + parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) parameter NUM_REQS = 1, parameter CACHE_SIZE = 1024, // cache size in bytes parameter LINE_SIZE = 16, // line size in bytes @@ -834,6 +835,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( VX_cache_mshr #( .INSTANCE_ID (`SFORMATF(("%s-mshr", INSTANCE_ID))), .BANK_ID (BANK_ID), + .BANK_HASH (BANK_HASH), .LINE_SIZE (LINE_SIZE), .SECTOR_SIZE (SECTOR_SIZE), .NUM_BANKS (NUM_BANKS), @@ -1316,7 +1318,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [`VX_CFG_XLEN-1:0] core_req_full_addr = `CS_BANK_TO_FULL_ADDR(core_req_addr, BANK_ID); wire [`VX_CFG_XLEN-1:0] full_addr_st0 = `CS_BANK_TO_FULL_ADDR(st0.req.addr, BANK_ID); wire [`VX_CFG_XLEN-1:0] full_addr_st1 = `CS_BANK_TO_FULL_ADDR(st1.req.addr, BANK_ID); - wire [`VX_CFG_XLEN-1:0] mreq_queue_full_addr = `CS_BANK_TO_FULL_ADDR(mreq_queue_addr, BANK_ID); + wire [`CS_LINE_ADDR_WIDTH-1:0] mreq_queue_line = mreq_queue_addr[`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]; + wire [`VX_CFG_XLEN-1:0] mreq_queue_full_addr = `CS_BANK_TO_FULL_ADDR(mreq_queue_line, BANK_ID) + | ((`VX_CFG_XLEN'(mreq_queue_addr) & `VX_CFG_XLEN'((1 << `CS_SECTOR_SEL_BITS) - 1)) << `CLOG2(SECTOR_SIZE)); always @(posedge clk) begin if (input_stall || pipe_stall) begin diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 2efb8ab859..31868bcd1b 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -56,6 +56,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // Indicates this cache cluster is the LLC (handles AMO completion). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. + parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter AMO_ENABLE = 0 // Enable atomic memory operations. ) ( @@ -147,6 +148,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .PASSTHRU (PASSTHRU), .IS_LLC (IS_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), + .BANK_HASH (BANK_HASH), .AMO_ENABLE (AMO_ENABLE) ) cache_wrap ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache_define.vh b/hw/rtl/cache/VX_cache_define.vh index 3f904816fc..c3ccbb49b0 100644 --- a/hw/rtl/cache/VX_cache_define.vh +++ b/hw/rtl/cache/VX_cache_define.vh @@ -74,7 +74,8 @@ /////////////////////////////////////////////////////////////////////////////// -`define CS_BANK_TO_FULL_ADDR(x, b) {x, (`VX_CFG_XLEN-$bits(x))'(b << (`VX_CFG_XLEN-$bits(x)-`CS_BANK_SEL_BITS))} +// b is the bank id; with BANK_HASH the raw bank bits are b XOR the line's hash +`define CS_BANK_TO_FULL_ADDR(x, b) {x, (`VX_CFG_XLEN-$bits(x))'(((b) ^ (BANK_HASH ? int'(cache_bank_hash(32'(x), `CS_BANK_SEL_BITS)) : 0)) << (`VX_CFG_XLEN-$bits(x)-`CS_BANK_SEL_BITS))} `define CS_MEM_TO_FULL_ADDR(x) {x, (`VX_CFG_XLEN-$bits(x))'(0)} /////////////////////////////////////////////////////////////////////////////// diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index 22605fd148..e088fe95fc 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -45,6 +45,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID= "", parameter BANK_ID = 0, + parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) parameter LINE_SIZE = 16, // Size of line inside a bank in bytes parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (coalescing/fill granule); = LINE_SIZE => 1 sector parameter NUM_BANKS = 1, // Number of banks @@ -128,6 +129,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire empty ); `UNUSED_PARAM (BANK_ID) + `UNUSED_PARAM (BANK_HASH) reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; // Per-entry sector. Coalescing matches on {line, sector} so same-line diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 6aa1266e7a..24ca0f7a62 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -56,6 +56,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // Set when this cache is the LLC (enables AMO completion at this level). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. + parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter AMO_ENABLE = 0 // Enable atomic operations in cache banks. ) ( @@ -149,6 +150,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .MEM_OUT_BUF (BYPASS_ENABLE ? 1 : MEM_OUT_BUF), .IS_LLC (IS_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), + .BANK_HASH (BANK_HASH), .AMO_ENABLE (AMO_ENABLE) ) cache ( .clk (clk), diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index b58bd48dd5..0e25d21b96 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -71,6 +71,9 @@ module VX_cache_top import VX_gpu_pkg::*; #( // Memory request output buffer parameter MEM_OUT_BUF = 3, + // Hashed bank select + parameter BANK_HASH = (NUM_BANKS >= 4), + parameter MEM_TAG_WIDTH = `CACHE_MEM_TAG_WIDTH(MSHR_SIZE, NUM_BANKS, MEM_PORTS, UUID_WIDTH) ) ( input wire clk, @@ -182,6 +185,7 @@ module VX_cache_top import VX_gpu_pkg::*; #( .DIRTY_BYTES (DIRTY_BYTES), .AMO_ENABLE (AMO_ENABLE), .IS_LLC (IS_LLC), + .BANK_HASH (BANK_HASH), .CORE_OUT_BUF (CORE_OUT_BUF), .MEM_OUT_BUF (MEM_OUT_BUF) ) cache ( diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 937c2ae131..d8451f2618 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -84,6 +84,7 @@ class Cluster::Impl { VX_CFG_L2_LATENCY, // pipeline latency VX_CFG_L2_REPL_POLICY, // replacement policy (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash }); diff --git a/sim/simx/mem/addr_hash.h b/sim/simx/mem/addr_hash.h new file mode 100644 index 0000000000..e9117edd33 --- /dev/null +++ b/sim/simx/mem/addr_hash.h @@ -0,0 +1,46 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace vortex { + +// IPOLY index hashing (Rau, "Pseudo-randomly interleaved memory", ISCA 1991), +// with the GF(2) equations GPGPU-Sim uses for its memory partitions and DRAM +// banks. Each returns the mask f(hi) that is XORed into the index; f depends +// only on the higher bits, so the same XOR restores the original index. + +inline uint32_t ipoly_bit(uint64_t v, int i) { return uint32_t(v >> i) & 1u; } + +// IPOLY(37), 32 sets. +inline uint32_t ipoly32(uint64_t a) { + uint32_t i0 = ipoly_bit(a,13)^ipoly_bit(a,12)^ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,6)^ipoly_bit(a,5)^ipoly_bit(a,3)^ipoly_bit(a,0); + uint32_t i1 = ipoly_bit(a,14)^ipoly_bit(a,13)^ipoly_bit(a,12)^ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,4)^ipoly_bit(a,1); + uint32_t i2 = ipoly_bit(a,14)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,3)^ipoly_bit(a,2)^ipoly_bit(a,0); + uint32_t i3 = ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,4)^ipoly_bit(a,3)^ipoly_bit(a,1); + uint32_t i4 = ipoly_bit(a,12)^ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,5)^ipoly_bit(a,4)^ipoly_bit(a,2); + return i0 | (i1 << 1) | (i2 << 2) | (i3 << 3) | (i4 << 4); +} + +// IPOLY(5), 16 sets. +inline uint32_t ipoly16(uint64_t a) { + uint32_t i0 = ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,6)^ipoly_bit(a,4)^ipoly_bit(a,3)^ipoly_bit(a,0); + uint32_t i1 = ipoly_bit(a,12)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,5)^ipoly_bit(a,3)^ipoly_bit(a,1)^ipoly_bit(a,0); + uint32_t i2 = ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,4)^ipoly_bit(a,2)^ipoly_bit(a,1); + uint32_t i3 = ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,5)^ipoly_bit(a,3)^ipoly_bit(a,2); + return i0 | (i1 << 1) | (i2 << 2) | (i3 << 3); +} + +} diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 8c4dcfc0a8..81b10a3692 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -15,6 +15,7 @@ #include "mem_block_pool.h" #include "debug.h" #include "types.h" +#include "addr_hash.h" #if VX_CFG_EXT_A_ENABLED #include "amo_unit.h" #endif @@ -50,7 +51,9 @@ struct params_t { int32_t tag_select_addr_start; int32_t tag_select_addr_end; - params_t(const Cache::Config &config) { + bool bank_hash; + + params_t(const Cache::Config &config) : bank_hash(config.bank_hash) { int32_t offset_bits = config.L - config.W; int32_t index_bits = config.C - (config.L + config.A + config.B); assert(offset_bits >= 0); @@ -85,13 +88,35 @@ struct params_t { // Tag select this->tag_select_addr_start = (1 + this->set_select_addr_end); this->tag_select_addr_end = (config.addr_width - 1); + assert(!bank_hash || (config.B != 0 && config.B <= 8)); + } + + // The bank is the raw bank bits XORed with a function of the bits above + // them, so power-of-two strides spread over the banks; applying the same + // XOR again restores the raw bits. IPOLY for 16/32 banks; other bank counts + // XOR-fold the higher bits (Frailong, Jalby, Lenfant, "XOR-schemes", ICPP 1985). + uint32_t bank_xor(uint64_t addr, uint32_t raw) const { + uint32_t bits = bank_select_addr_end - bank_select_addr_start + 1; + uint64_t hi = addr >> (bank_select_addr_end + 1); + if (bits == 4) { + return raw ^ ipoly16(hi); + } + if (bits == 5) { + return raw ^ ipoly32(hi); + } + uint32_t mask = (1u << bits) - 1, f = 0; + for (uint32_t i = 0; i < 15; i += bits) { + f ^= uint32_t(hi >> i) & mask; + } + return raw ^ f; } uint32_t addr_bank_id(uint64_t addr) const { - if (bank_select_addr_end >= bank_select_addr_start) - return (uint32_t)bit_getw(addr, bank_select_addr_start, bank_select_addr_end); - else + if (bank_select_addr_end < bank_select_addr_start) { return 0; + } + uint32_t raw = (uint32_t)bit_getw(addr, bank_select_addr_start, bank_select_addr_end); + return bank_hash ? bank_xor(addr, raw) : raw; } uint32_t addr_set_id(uint64_t addr) const { @@ -123,6 +148,10 @@ struct params_t { addr = bit_setw(addr, set_select_addr_start, set_select_addr_end, set_id); if (tag_select_addr_end >= tag_select_addr_start) addr = bit_setw(addr, tag_select_addr_start, tag_select_addr_end, tag); + if (bank_hash && bank_select_addr_end >= bank_select_addr_start) { + uint32_t raw = bank_xor(addr, bank_id); + addr = bit_setw(addr, bank_select_addr_start, bank_select_addr_end, raw); + } return addr; } diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 70ad9e6e3f..8729a52e88 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -43,6 +43,7 @@ class Cache : public SimObject { uint16_t latency; // pipeline latency uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache + bool bank_hash = false; // hashed bank select }; struct PerfStats { diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 73f8bd0625..38ea84f880 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -91,6 +91,7 @@ class Socket::Impl { VX_CFG_DCACHE_LATENCY, // pipeline latency (capacity-scaled, matches hardware) VX_CFG_DCACHE_REPL_POLICY, // replacement policy (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash }); #ifdef VX_CFG_EXT_TEX_ENABLE From bdf39536d2c146452225ff1b4b084bd3cca64b23 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 14 Oct 2026 19:05:33 -0700 Subject: [PATCH 032/336] Keep the other sectors' dirty bytes when a sector refills a line With DIRTY_BYTES=1 and more than one sector per line, every fill cleared the whole line's byte mask, but a refill (a sector installed into a line that is already resident) leaves the other sectors resident and the tags keep their dirty bits. A dirty sibling sector then wrote back with byteen 0 and DRAM dropped its data: mstress on 4 cores with a write-back, sectored L2 and L2_DIRTYBYTES=1 read back never-written memory in 2048 results, every line losing its first sector. A refill now clears only the installed sector's mask; a fresh install still clears the whole line. This matches the tags and SimX, which already reset one sector on a refill. One-sector caches are unchanged, since a refill cannot happen there. The commit-stage check compared line-wide ORs, which a dirty sibling satisfies; a per-sector check now asserts that every dirty sector carries dirty bytes. With the fix reverted it fires on the first lost beat. Tests: cache:mstress-6 (the failing config) on SimX and rtlsim. Also run on rtlsim: mstress-1/2/4, bank_hash-4, L2 dirty bytes off, and L2+L3 write-back with dirty bytes. Cycle counts are unchanged. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 12 ++++++++++++ hw/rtl/cache/VX_cache_bank.sv | 9 +++++++++ hw/rtl/cache/VX_cache_data.sv | 13 ++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 3a2c5b7088..d78b7f904d 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -196,6 +196,18 @@ tests: clusters: 2 l2cache: true l3cache: true +# Dirty-byte write-back on a sectored L2: a sector refill keeps the other +# sectors' dirty bytes. +- id: mstress-6 + via: blackbox + drivers: + - simx + - rtlsim + app: mstress + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_DIRTYBYTES=1 + shape: + cores: 4 + l2cache: true - id: sgemm-20 via: blackbox drivers: diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 410a61dcc6..cb3f748a3b 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -144,6 +144,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( req_t req; logic [`CS_WORD_WIDTH-1:0] wdata; logic [NUM_WAYS-1:0] tag_matches; + logic is_refill; // fill into an already-resident line } data_t; typedef struct packed { // response + memory request (S0 -> S1 -> stC) @@ -664,6 +665,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( dat_in = st0; dat_in.req.way_idx = evict_way_st0; dat_in.tag_matches = tag_matches_st0 & {NUM_WAYS{~creq_hit_order_hazard_st0}}; + dat_in.is_refill = lk_st0.is_refill; end // commit path: the request (with the resolved hit/victim way and MSHR id) @@ -752,6 +754,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .reset (reset), .init (do_init_std), .fill (do_fill_std && ~stD.req.is_passthru_fill && ~pipe_stall), + .refill (stD.is_refill), .flush (do_flush_std && ~pipe_stall), .read (do_read_std && ~pipe_stall), .write (do_write_std && ~pipe_stall), @@ -1226,6 +1229,12 @@ module VX_cache_bank import VX_gpu_pkg::*; #( if (DIRTY_BYTES) begin : g_dirty_bytes wire has_dirty_bytes = (| evict_byteen_stc); `RUNTIME_ASSERT (~do_fill_or_flush_stc || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + // per-sector: a dirty sector must carry dirty bytes, or its writeback beat is dropped. + wire [SEC-1:0] sec_has_dirty_bytes; + for (genvar s = 0; s < SEC; ++s) begin : g_sec_has_dirty_bytes + assign sec_has_dirty_bytes[s] = (| evict_byteen_stc[s*SECTOR_SIZE +: SECTOR_SIZE]); + end + `RUNTIME_ASSERT (~do_fill_or_flush_stc || ((stC.lk.evict_dirty_mask & ~sec_has_dirty_bytes) == '0), ("dirty sector without dirty bytes: dirty_mask=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.evict_dirty_mask, sec_has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) end // fill request on a read/write miss (one sector); multi-beat writeback // on a dirty eviction. The two are mutually exclusive (a writeback is a diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index abc3472ba3..d30d289f13 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -46,6 +46,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( // inputs input wire init, input wire fill, + input wire refill, // fill into an already-resident line input wire flush, input wire read, input wire write, @@ -108,10 +109,19 @@ module VX_cache_data import VX_gpu_pkg::*; #( assign write_mask[i] = write_byteen & {WORD_SIZE{word_en}}; end + // A sector refill must clear only the installed sector's mask: the other + // resident sectors may hold dirty bytes not yet written back. + wire [`CS_SECTORS_PER_LINE-1:0][SECTOR_SIZE-1:0] fill_clr_mask; + for (genvar i = 0; i < `CS_SECTORS_PER_LINE; ++i) begin : g_fill_clr_mask + wire sec_en = ~refill || (`CS_SECTORS_PER_LINE == 1) + || (sector_idx == `UP(`CS_SECTOR_SEL_BITS)'(i)); + assign fill_clr_mask[i] = {SECTOR_SIZE{fill && sec_en}}; + end + wire [NUM_WAYS-1:0][LINE_SIZE-1:0] byteen_rdata; for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_byteen_store wire [LINE_SIZE-1:0] byteen_wdata = {LINE_SIZE{write}}; // only asserted on writes - wire [LINE_SIZE-1:0] byteen_wren = {LINE_SIZE{init || fill || flush}} | write_mask; + wire [LINE_SIZE-1:0] byteen_wren = {LINE_SIZE{init || flush}} | fill_clr_mask | write_mask; wire byteen_write = ((fill || flush) && ((NUM_WAYS == 1) || (evict_way == i))) || (write && tag_matches[i]) || init; @@ -141,6 +151,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( assign evict_byteen = byteen_rdata[way_idx_r]; end else begin : g_no_dirty_bytes `UNUSED_VAR (init) + `UNUSED_VAR (refill) `UNUSED_VAR (flush) `UNUSED_VAR (way_idx_r) assign evict_byteen = '1; // update whole line From f9dfa6a13be344f3694d03a3105906cb0a86b31a Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 14 Oct 2026 23:37:18 -0700 Subject: [PATCH 033/336] Let fills follow each other into a cache bank pipe back to back A bank with more than two pipeline stages (PIPE_EX > 0: the L2, the L3, and L1s above 16 KB) accepted a memory fill only when no other fill was in its pipe. The fill sector waited in one staging register until its data-array write at stD, so a second fill would have overwritten it. That capped a bank at one fill every 1 + max(chain, PIPE_EX) cycles. GPU caches take a fill every cycle while earlier fills are still writing (line-fill buffers). The staged sector now rides a resetless delay line beside the request to its array write, clock-enabled with the pipeline, so the next fill can restage the buffer right behind it. Each stage has one source, so it costs flops but no LUTs, and deep lines map onto shift registers. A simulation-only twin of the line asserts that the staged sector's address matches the fill writing the array. SimX woke a fill's MSHR chain at install while the RTL wakes it at accept, and gated fills on an empty pipe at every latency. It now wakes the chain and arms the forward window at accept, forwarding from the staged sector like the RTL. Because SimX links a core request to its chain at pipe exit rather than at S0, a fill waits while a same-line core request is still in the pipe, and further requests to that line wait with it, so a forwarded load can never overtake an older store. SimX now matches rtlsim on 4-core NT=16 sgemv to 0.03% (it was 3.5% optimistic). fpga_gate cache DUT at DCACHE_LATENCY=4 (PIPE_EX=2, 4 banks): 300.4 -> 309.6 MHz, LUT 27,534 -> 27,403, FF 26,589 -> 30,708 (the 4 x 2 x 512-bit delay line). perf_gate (rtlsim, re-recorded with --update-baselines; retired instructions unchanged): only the multi-core L2 cases move, sgemm-mc-nt16 -0.40%, fp16-mc-nt4 -0.08%, draw3d-mc 0.00%, fp16-mc-nt16 +0.52%. The stock L2 is shallow; on the rtx5080 profile (L2 latency 10) a 4 MiB stream gains 6.2%. A trace diff of fp16-mc-nt16 shows no L2 service time getting worse (hit latency 4.08 -> 4.02, the in-bank miss path identical, same DRAM traffic): a fill accepted one cycle earlier reorders the two cores' L2 traffic and their warps line up differently. The same case moves by up to 1.8% from a one-cycle change of the simulated DRAM latency alone, and over 11 problem sizes the change has a geomean of 0.985. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 2 +- ci/baselines/perf/graphics.json | 2 +- ci/baselines/perf/tensor.json | 4 +- hw/rtl/cache/VX_cache_bank.sv | 71 ++++++++++++--------- sim/simx/mem/cache.cpp | 109 +++++++++++++++++++------------- 5 files changed, 109 insertions(+), 79 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index a02138c6d0..01cd65540c 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -23,7 +23,7 @@ }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 723234, + "cycles": 720350, "instrs": 623680 }, "app": "sgemm", diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index b402bd0b6f..489e16a09b 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,7 +1,7 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 529875, + "cycles": 529868, "instrs": 203136 }, "app": "gfx_draw3d", diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 112a272e80..975aa8d248 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -12,7 +12,7 @@ }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 36534, + "cycles": 36723, "instrs": 47296 }, "app": "sgemm_tcu", @@ -23,7 +23,7 @@ }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221752, + "cycles": 221581, "instrs": 336448 }, "app": "sgemm_tcu", diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index cb3f748a3b..4b95432384 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -160,7 +160,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Shared signals // ------------------------------------------------------------------------ wire crsp_queue_stall, mshr_alm_full; - wire mshr_allocate_st0, fill_inflight; + wire mshr_allocate_st0; (* max_fanout = 64 *) wire pipe_stall; // Authoritative "no entry held", straight off the MSHR's valid mask. The // pending-size counter below is a proxy: it is fed by separate increment and @@ -217,8 +217,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire fwd_head, fwd_fire, fwd_pending; wire [`CS_WORD_WIDTH-1:0] fwd_word; - // Fill buffer: the sector staged at fill accept owns all in-flight fill - // data; it feeds the data-array fill port and the forward-response word. + // Fill buffer: the sector staged at fill accept; it feeds the forward- + // response word and enters the fill-data delay line to the array write. reg [`CS_SECTOR_WIDTH-1:0] fbuf_data_r; reg [`CS_LINE_ADDR_WIDTH-1:0] fbuf_addr_r; @@ -356,7 +356,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // request enters behind a pending AMO), so a fill waits until the AMO // writeback queue has room for it; the queue drains meanwhile. wire replay_mux = replay_valid && ~fwd_head && ~replay_link_hold; - wire fill_mux = mem_rsp_valid && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full; + wire fill_mux = mem_rsp_valid && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full; wire replay_grant = ~init_valid; wire replay_enable = replay_grant && replay_mux; @@ -374,7 +374,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire creq_enable = creq_grant && (amo_creq_path || amo_wb_path); assign replay_ready = replay_grant && ~fwd_head && ~replay_link_hold && ~(!WRITEBACK && replay_rw && mreq_queue_alm_full) && ~pipe_stall; - assign mem_rsp_ready = fill_grant && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; + assign mem_rsp_ready = fill_grant && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign core_req_ready = creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; @@ -391,25 +391,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [MSHR_ADDR_WIDTH-1:0] mem_rsp_id = mem_rsp_tag[MSHR_ADDR_WIDTH-1:0]; - // A fill's sector lives only in the staging buffer until its data-array - // write at stD, so with deferred stages (PIPE_EX>0) a back-to-back fill - // must be held off while one is in flight. At PIPE_EX=0 the array write - // samples the buffer on the same edge a new fill re-stages it (old value - // read), so no interlock is needed. - if (PIPE_EX > 0) begin : g_fill_inflight - reg [PIPE_EX-1:0] fill_busy; - always @(posedge clk) begin - if (reset) begin - fill_busy <= '0; - end else if (~pipe_stall) begin - fill_busy <= PIPE_EX'({fill_busy, (mem_rsp_fire && ~is_passthru_fill_sel)}); - end - end - assign fill_inflight = (| fill_busy); - end else begin : g_no_fill_inflight - assign fill_inflight = 1'b0; - end - // Fill word_idx: place the installed sector in the high (sector) bits of the // in-line word offset; low (word-in-sector) bits are don't-care for a fill. wire [WORD_SEL_WIDTH-1:0] fill_word_idx; @@ -442,8 +423,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign flush_tag = '0; end - // Word payload mux (store / AMO-writeback data). Fill data does not ride - // the pipeline: the staged fill buffer feeds the data array directly. + // Word payload mux (store / AMO-writeback data). Fill data takes its own + // delay line from the staged fill buffer to the data array. wire [`CS_WORD_WIDTH-1:0] wdata_sel; if (WRITE_ENABLE) begin : g_wdata_sel assign wdata_sel = replay_mux ? replay_data @@ -734,10 +715,40 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign sector_idx_std = '0; end - // The staged fill sector, replicated across the line so each sector's - // slices see their words; the data array writes only the installed sector - // (per-slice gate). One copy == the full line when 1 sector/line. - wire [`CS_LINE_WIDTH-1:0] fbuf_line = {`CS_SECTORS_PER_LINE{fbuf_data_r}}; + // A fill's sector is staged when the fill enters S0 and then rides beside + // it to its data-array write at stD, so the next fill may restage the + // buffer right behind it. Each stage has one source and holds on the + // pipeline enable; without a reset, deep stages map onto shift registers. + wire [`CS_SECTOR_WIDTH-1:0] fill_data_std; + VX_pipe_register #( + .DATAW (`CS_SECTOR_WIDTH), + .DEPTH (PIPE_EX) + ) reg_fill_data ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (fbuf_data_r), + .data_out (fill_data_std) + ); +`ifndef SYNTHESIS + wire [`CS_LINE_ADDR_WIDTH-1:0] fill_addr_std; + VX_pipe_register #( + .DATAW (`CS_LINE_ADDR_WIDTH), + .DEPTH (PIPE_EX) + ) reg_fill_addr ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (fbuf_addr_r), + .data_out (fill_addr_std) + ); + `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) +`endif + + // The fill sector, replicated across the line so each sector's slices see + // their words; the data array writes only the installed sector (per-slice + // gate). One copy == the full line when 1 sector/line. + wire [`CS_LINE_WIDTH-1:0] fbuf_line = {`CS_SECTORS_PER_LINE{fill_data_std}}; VX_cache_data #( .CACHE_SIZE (CACHE_SIZE), diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 81b10a3692..0bcd46007d 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -720,7 +720,6 @@ class CacheBank : public SimObject { perf_stats_ = Cache::PerfStats(); pending_mshr_size_ = 0; pending_amo_probes_ = 0; - pipe_fill_count_ = 0; pipe_core_lines_.clear(); pending_read_reqs_ = 0; pending_write_reqs_ = 0; @@ -735,6 +734,7 @@ class CacheBank : public SimObject { fwd_set_ = 0; fwd_tag_ = 0; fwd_sector_ = 0; + fwd_data_.reset(); crsp_sent_ = false; flushing_ = false; flush_set_idx_ = 0; @@ -816,11 +816,16 @@ class CacheBank : public SimObject { } // 2) fill only when no replay is pending and no forward drain is in - // progress (a new fill would re-arm the window mid-chain). A fill still - // riding the pipe also blocks the next one: its chain is only marked for - // replay at install, and the replay priority above must get a chance to - // drain it before another fill is accepted. - if (!this->mem_rsp_in.empty() && !fwd_active_ && pipe_fill_count_ == 0) { + // progress (a new fill would re-arm the window mid-chain). Each fill + // carries its own sector down the pipe, so fills may follow each other + // back to back. The fill wakes its chain on accept, so a same-line core + // request still in the pipe must reach its chain first (the hardware + // links it at S0): the fill waits, and so does any further request to + // that line, which keeps a stream of them from starving the fill. + bool fill_held = false; + uint32_t held_set_id = 0; + uint64_t held_addr_tag = 0; + if (!this->mem_rsp_in.empty() && !fwd_active_) { auto &mem_rsp = this->mem_rsp_in.peek(); #if VX_CFG_EXT_A_ENABLED // Non-LLC AMO passthrough response: forward straight to core @@ -848,25 +853,47 @@ class CacheBank : public SimObject { #endif uint32_t mshr_id = mem_rsp.tag; const auto &root_peek = mshr_.peek(mshr_id); - bank_req_t bank_req; - bank_req.reset(); - bank_req.type = bank_req_t::Fill; - bank_req.addr = params_.mem_addr_sector(bank_id_, root_peek.set_id, root_peek.addr_tag, root_peek.sector_id); - bank_req.hart_id = root_peek.bank_req.hart_id; - bank_req.uuid = root_peek.bank_req.uuid; - bank_req.mshr_id = mshr_id; - bank_req.data = mem_rsp.data; - pipe_req_->push(bank_req); - ++pipe_fill_count_; - DT(3, this->name() << " fill-rsp: " << mem_rsp); - this->mem_rsp_in.pop(); - --pending_fill_reqs_; - return; + const uint32_t set_id = root_peek.set_id; + const uint64_t addr_tag = root_peek.addr_tag; + const uint32_t sector_id = root_peek.sector_id; + if (this->pipeLineBusy(set_id, addr_tag)) { + fill_held = true; + held_set_id = set_id; + held_addr_tag = addr_tag; + } else { + bank_req_t bank_req; + bank_req.reset(); + bank_req.type = bank_req_t::Fill; + bank_req.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); + bank_req.hart_id = root_peek.bank_req.hart_id; + bank_req.uuid = root_peek.bank_req.uuid; + bank_req.mshr_id = mshr_id; + bank_req.data = mem_rsp.data; + pipe_req_->push(bank_req); + // The chain wakes at accept: its replays enter the pipe behind the fill, + // so they reach the array after the install. The forward window answers + // the chain's leading reads from the staged sector meanwhile. + mshr_.replay(mshr_id); + fwd_active_ = true; + fwd_set_ = set_id; + fwd_tag_ = addr_tag; + fwd_sector_ = sector_id; + fwd_data_ = mem_rsp.data; + DT(3, this->name() << " fill-rsp: " << mem_rsp); + this->mem_rsp_in.pop(); + --pending_fill_reqs_; + return; + } } // 3) core request if (!this->core_req_in.empty()) { auto &core_req = this->core_req_in.peek(); + if (fill_held + && params_.addr_set_id(core_req.addr) == held_set_id + && params_.addr_tag(core_req.addr) == held_addr_tag) { + return; + } // Conservative MSHR occupancy check: any request that may miss must // reserve a slot. Counts both currently-allocated entries and // in-flight pipe requests that haven't reached MSHR allocation yet. @@ -904,11 +931,9 @@ class CacheBank : public SimObject { // MSHR entry only at pipe exit; defer on those too, or the AMO // overtakes it and a younger same-address load can chain onto the // pre-AMO fill (and the fill installs a pre-AMO line). - for (const auto &kv : pipe_core_lines_) { - if (kv.first == amo_set_id && kv.second == amo_addr_tag) { - ++perf_stats_.mshr_stalls; - return; - } + if (this->pipeLineBusy(amo_set_id, amo_addr_tag)) { + ++perf_stats_.mshr_stalls; + return; } } #else @@ -1107,12 +1132,6 @@ class CacheBank : public SimObject { sec.dirty = false; sec.dirty_mask = 0; sec.data = bank_req.data; - mshr_.replay(bank_req.mshr_id); - // arm the fill-forward window for this chain - fwd_active_ = true; - fwd_set_ = set_id; - fwd_tag_ = addr_tag; - fwd_sector_ = sector_id; return true; } @@ -1127,6 +1146,7 @@ class CacheBank : public SimObject { bool processForward() { if (!mshr_.has_ready_reqs()) { fwd_active_ = false; // chain drained + fwd_data_.reset(); return false; } const auto *head = mshr_.peek_next_replay(); @@ -1139,15 +1159,7 @@ class CacheBank : public SimObject { || head->bank_req.write || memop_is_atomic(head->bank_req.op)) { fwd_active_ = false; - return false; - } - // The armed sector is normally resident (the fill just installed it); a - // concurrent invalidate falls back to the replay path, which re-fetches. - auto &set = sets_.at(head->set_id); - int present_id = set.find_resident(head->addr_tag); - uint32_t sector_id = head->sector_id; - if (present_id == -1 || !set.lines.at(present_id).sectors.at(sector_id).valid) { - fwd_active_ = false; + fwd_data_.reset(); return false; } if (crsp_sent_ || this->core_rsp_out.full()) { @@ -1156,7 +1168,7 @@ class CacheBank : public SimObject { bank_req_t req; mshr_.dequeue(&req); MemRsp rsp{req.req_tag, req.hart_id, req.uuid}; - rsp.data = set.lines.at(present_id).sectors.at(sector_id).data; + rsp.data = fwd_data_; this->core_rsp_out.send(rsp); DT(3, this->name() << " fwd-rsp: " << rsp); return true; @@ -1397,8 +1409,6 @@ class CacheBank : public SimObject { if (!this->processFill(bank_req)) { return; // stall } - assert(pipe_fill_count_ != 0); - --pipe_fill_count_; pipe_req_->pop(); } break; @@ -1736,12 +1746,20 @@ class CacheBank : public SimObject { MSHR mshr_; uint32_t pending_mshr_size_; uint32_t pending_amo_probes_; // AmoProbe requests in the pipe, each holding a table reservation - uint32_t pipe_fill_count_; // fills riding the pipe; gates the next fill's admission // (set, tag) of Core requests inside the pipe: their MSHR entries exist // only from pipe exit, so ordering checks against in-flight lines (the // AMO admission defer) must consult this mirror as well. std::vector> pipe_core_lines_; + bool pipeLineBusy(uint32_t set_id, uint64_t addr_tag) const { + for (const auto &kv : pipe_core_lines_) { + if (kv.first == set_id && kv.second == addr_tag) { + return true; + } + } + return false; + } + void pipeLineErase(uint32_t set_id, uint64_t addr_tag) { for (auto it = pipe_core_lines_.begin(); it != pipe_core_lines_.end(); ++it) { if (it->first == set_id && it->second == addr_tag) { @@ -1761,11 +1779,12 @@ class CacheBank : public SimObject { uint32_t rand_ctr_; uint64_t adm_seq_ctr_; // bank admission-order stamp (see bank_req_t::adm_seq) - // Fill-forward window: chain key armed by the last installed fill. + // Fill-forward window: chain key and sector staged by the last accepted fill. bool fwd_active_; uint32_t fwd_set_; uint64_t fwd_tag_; uint32_t fwd_sector_; + std::shared_ptr fwd_data_; bool crsp_sent_; // core-response port used this tick (pipeline priority) // Flush walk state. From 7ce8a7483bfb50c3c208502c058a03c91cec9f50 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 17 Oct 2026 13:05:22 -0700 Subject: [PATCH 034/336] Flush only the L1 caches between launches when the LLC is the coherence point A launch boundary used to walk every cache level. When a single last-level cache serves every memory client, it is already the point of coherence, so back-to-back launches only need the L1s cleaned. CMD_CACHE_FLUSH gains an L1-only flag that reaches the caches as a flush-token kind; a shared LLC answers such a token without walking, while replicated LLCs and caches with peer memory clients beside them (TEX, RTU, RASTER, OM, DXA) still walk and invalidate. Whether an LLC is shared is derived from topology, and static asserts reject a write-back LLC with more than one instance. The runtime posts flushes lazily and issues a full flush before any device-memory transfer. Skipping the per-launch full flush exposed that hardware busy dropped while uncached IO stores were still inside the hierarchy, losing console output. busy now stays high until every write issued by the cores is accepted at the next level, through the memory ports, and until the platform makes it visible (bank acceptance in rtlsim, the AXI B response in Vortex_axi). The command processor retires a full flush only after busy drops, so its next DMA cannot overtake LLC write-backs on a shared memory bank. SimX keeps a shared LLC's contents across an L1-only flush, matching the RTL. Flush cycles per launch: 1 core 422 -> 152, 4 cores + L2 2,750 -> 180 for L1-only launches; full flushes and sgemm/raycast cycles unchanged. Tests: flush_coherence chain-1..6 (incl. replicated-LLC and DXA-peer cases that fail without the fix), chain-stress-1, model_parity-flush-chain. Co-Authored-By: Claude Opus 5.5 --- VX_types.toml | 5 + ci/testcases/regression.yaml | 87 +++++++++++++ docs/designs/cache_subsystem.md | 68 +++++++++- docs/designs/command_processor.md | 40 ++++-- hw/rtl/VX_cluster.sv | 9 +- hw/rtl/VX_gpu_pkg.sv | 10 ++ hw/rtl/VX_graphics.sv | 54 +++++++- hw/rtl/VX_socket.sv | 49 +++++++- hw/rtl/Vortex.sv | 15 ++- hw/rtl/Vortex_axi.sv | 29 ++++- hw/rtl/cache/VX_cache.sv | 13 +- hw/rtl/cache/VX_cache_bypass.sv | 34 ++++- hw/rtl/cache/VX_cache_cluster.sv | 54 +++++++- hw/rtl/cache/VX_cache_init.sv | 38 +++++- hw/rtl/cache/VX_cache_wrap.sv | 11 +- hw/rtl/core/VX_core.sv | 14 ++- hw/rtl/core/VX_dcr_data.sv | 9 +- hw/rtl/core/VX_dcr_flush.sv | 8 +- hw/rtl/core/VX_lsu_slice.sv | 3 +- hw/rtl/core/VX_mem_unit.sv | 2 + hw/rtl/cp/VX_cp_core.sv | 3 +- hw/rtl/cp/VX_cp_dcr_proxy.sv | 30 +++-- hw/rtl/cp/VX_cp_pkg.sv | 2 + hw/rtl/mem/VX_mem_req_pending.sv | 53 ++++++++ hw/rtl/om/VX_om_steer.sv | 2 + hw/unittest/cache/VX_cache_top.sv | 3 +- .../cp_dcr_proxy/VX_cp_dcr_proxy_top.sv | 6 +- hw/unittest/cp_dcr_proxy/main.cpp | 70 ++++++++++- hw/unittest/gfx/VX_gfx_top.sv | 11 +- hw/unittest/mem_unit/VX_mem_unit_top.sv | 1 + hw/unittest/tensor/VX_tensor_top.sv | 7 +- hw/unittest/vm/VX_vm_top.sv | 9 +- sim/common/cmd_processor.cpp | 14 ++- sim/common/cmd_processor.h | 14 ++- sim/rtlsim/processor.cpp | 10 ++ sim/rtlsim/processor.h | 4 + sim/rtlsim/rtlsim_shim.sv | 28 ++++- sim/simx/cluster.cpp | 17 +-- sim/simx/cluster.h | 4 +- sim/simx/constants.h | 21 ++++ sim/simx/mem/cache.cpp | 28 +++-- sim/simx/mem/cache.h | 7 +- sim/simx/mem/cache_cluster.cpp | 4 +- sim/simx/mem/cache_cluster.h | 2 +- sim/simx/processor.cpp | 23 ++-- sim/simx/processor_impl.h | 3 +- sim/simx/socket.cpp | 7 +- sim/simx/socket.h | 2 +- sw/runtime/common/device.cpp | 107 +++++++++++----- sw/runtime/common/queue.cpp | 13 +- sw/runtime/common/vortex2_internal.h | 28 +++-- sw/runtime/firesim/vortex.cpp | 4 + sw/runtime/rtlsim/vortex.cpp | 4 + tests/regression/flush_coherence/common.h | 1 + tests/regression/flush_coherence/kernel.cpp | 24 +++- tests/regression/flush_coherence/main.cpp | 118 +++++++++++++++++- 56 files changed, 1077 insertions(+), 159 deletions(-) create mode 100644 hw/rtl/mem/VX_mem_req_pending.sv diff --git a/VX_types.toml b/VX_types.toml index 7143523976..2e00ed2573 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -73,6 +73,11 @@ VX_DCR_BASE_CACHE_FLUSH = 0x000 VX_DCR_BASE_MPM_VALUE = 0x001 VX_DCR_BASE_STATE_END = 0x002 +[cache_flush] +# VX_DCR_BASE_CACHE_FLUSH write value: flush only the levels above the +# last-level cache, which stays the point of coherence for the next kernel. +VX_CACHE_FLUSH_L1_ONLY = 0x10000 + [dcr_mmu] VX_DCR_MMU_STATE_BEGIN = 0x004 VX_DCR_MMU_SATP_LO = 0x004 diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index b180c479b4..0d4ad00450 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -187,6 +187,83 @@ tests: cores: 2 l2cache: true l3cache: true +# Kernel to kernel with no host transfer between (one command batch): the +# launch flushes are L1-only, so the last-level cache alone carries each +# round's writes to the next round, and the full flush runs before readback. +- id: flush_coherence-chain-1 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c +- id: flush_coherence-chain-2 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + configs: -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true +- id: flush_coherence-chain-3 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +# Replicated last-level caches (per-core dcaches without an L2, per-cluster L2s +# without an L3) are not a point of coherence, so an L1-only flush must still +# invalidate them. +- id: flush_coherence-chain-4 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + shape: + cores: 2 +- id: flush_coherence-chain-5 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + shape: + clusters: 2 + cores: 2 + l2cache: true +# Each round loads its input with a DXA tile copy, which reaches memory beside +# the single dcache, so that dcache is not a point of coherence and must still +# be written back on an L1-only flush. +- id: flush_coherence-chain-6 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c -x + configs: -DVX_CFG_EXT_DXA_ENABLE +# The shared L2 keeps its lines across the L1-only launch flushes. +- id: model_parity-flush-chain + check: model_parity + via: blackbox + app: flush_coherence + args: -n2048 -r16 -c + configs: -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true # Flush ordering under random input backpressure (DBG_FLUSH_STALL): each round # ends on a store burst, so a core's last writes are often still queued on one # memory port or dcache lane while its token on another reaches the next level. @@ -227,3 +304,13 @@ tests: cores: 4 l2cache: true tier: full +- id: flush_coherence-chain-stress-1 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r8 -b16 -c + configs: -DDBG_FLUSH_STALL -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 8fce9737ef..7f7c26e390 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -465,6 +465,35 @@ Replay is the mirror image: it gates only when **not** `WRITEBACK` and the replayed request is a write (`replay_rw`), because a write-through bank forwards every write downstream and so does need a slot for it. +### 4.9 Requests in flight and device `busy` + +Stores are ack-less, so a core goes idle while its last requests are still +on their way out, and the console rings (`VX_MEM_IO_COUT`) are read right +after a launch retires, with no flush in between. Device `busy` therefore +covers every request the cores issued until memory has completed it, the way +a GPU's outstanding-write counters do: + +- the core stays busy while a request waits at its dcache port, past the flush + injector's buffer; +- every place outside the banks where a request can wait reports it through + `req_empty`: `VX_cache_bypass` counts from its uncached inputs and the + cache's memory side to its memory ports, `VX_cache_cluster` adds its + core-side and memory-side arbiters, and the socket memory arbiter and the OM + steers do the same. Each counter is a `VX_mem_req_pending` over the + section's input and output handshakes, which holds because these sections + neither create nor drop requests; the socket, cluster and device fold them + into `busy`; +- the platform wrapper extends `busy` past the device's memory ports to the + point where a write is visible to the host's DMA: acceptance by the bank in + `rtlsim_shim`, the AXI B response in `Vortex_axi`. The OPAE AFU needs + neither, since its DMA queues behind Vortex's writes on the same in-order + Avalon path. + +Cacheable requests left in the banks are covered by the next flush instead +(§6). A request's next holding point reaches `busy` through no more registers +than the one it leaves, so the coverage handed from one counter to the next has +no gap. + --- ## 5. Parallelism and scaling @@ -578,8 +607,16 @@ previous launch. The host issues `CMD_CACHE_FLUSH` after every kernel launch — ### 6.2 VX_dcr_flush — injecting the request `VX_dcr_flush` synthesizes a degenerate `MemReq` (`rw=0, addr=0, data=0, -byteen=0, attr = 1 << MEM_ATTR_FLUSH_OFFS`), a **flush token**, and merges it -into one cache input stream through a 2:1 `VX_mem_bus_arb`. +byteen=l1_only, attr = 1 << MEM_ATTR_FLUSH_OFFS`), a **flush token**, and merges it +into one cache input stream through a 2:1 `VX_mem_bus_arb`. Byteen bit 0 +marks an **L1-only** token (§6.4; a receiving cache tests for any byteen bit, +since a level with wider words sees the bit at the token's line offset); `VX_dcr_data` latches it from the START +write's `VX_CACHE_FLUSH_L1_ONLY` bit, and the dcache and icache injectors carry +it. The ocache, whose tokens reach the L2, sends an L1-only token only when +every requester of its flush round is L1-only: the cluster flush (kind latched +from the same broadcast write) and the dcache tokens held at the OM steers. +The other graphics caches do not forward and send full tokens. A fence +carries no byte enables, so a fence token is always a full one. **Core traffic sits on the priority input, not the flush.** The flush waits until upstream is quiescent — which it reliably is, because the host only issues @@ -639,6 +676,33 @@ downstream and `ready=0` upstream, so upstream stalls. into the response crossbar whenever bank 0 has no response of its own; the token is consumed at its input as its answer is sent. Then the inputs reopen. +**L1-only tokens.** Between kernels nothing outside the GPU reads memory, so +the last-level cache, the point of coherence, need not be written back. A +token with a byteen bit set asks only the levels above the LLC to flush: + +- a non-LLC level walks as usual (write-back levels write back, write-through + levels invalidate, so the next kernel sees other cores' and clusters' + updates) and forwards the token on every port **with the same kind** + (`mem_flush_l1_only`, byteen bit 0 of the forwarded token). Forwarding is what + lands a write-through level's last stores in the next level before the + token is answered; +- a last-level cache that alone serves every memory client (`SHARED_LLC`: + the L3; the L2 of a single cluster; the dcache of a single core with no + texture, ray-tracing, DXA, raster or OM unit, since those reach memory below + or beside it) goes from IDLE (or WAIT1) straight to RESPOND, with no walk and + no forward: once the token is visible, its input's earlier requests are + inside the cache, ahead of anything the next kernel sends. Any other LLC + (per-core dcaches without an L2, per-cluster L2s without an L3, a dcache + beside fixed-function or DXA ports) walks like any other level: its lines + may be stale copies of another instance's writes, or dirty data a client + beside it must read from memory. `Vortex.sv` asserts that a write-back LLC + has a single instance. + +The kind of a walk is L1-only only if every token it answers is, so a full +token is never answered by an L1-only walk. A token without the bit behaves +exactly as before. The runtime issues a full flush before any transfer to or +from device memory, because the host DMA path bypasses the LLC. + **Why a token per memory port.** Each port is a separate, separately buffered path to the next level, and nothing orders one port's data against another port's token. A token sent on one port only could reach the next level while diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index 0ad30a8011..356ddb000b 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -168,7 +168,13 @@ come from `cmd_size_bytes` (`WAIT_OP_EQ/GE/GT/NE`, [`VX_cp_pkg.sv:109-114`](../../hw/rtl/cp/VX_cp_pkg.sv#L109)). - **`CMD_CACHE_FLUSH`** — `arg0` = number of cores. Provides AMD `ACQUIRE_MEM`-style cache maintenance; the host fills `arg0` from - `VX_CAPS_NUM_CORES`. + `VX_CAPS_NUM_CORES`. Header flag bit 2 (`F_FLUSH_L1_ONLY`) asks for an + L1-only flush: the START write carries `VX_CACHE_FLUSH_L1_ONLY`, and the + last-level cache answers without its walk + ([cache_subsystem.md §6.4](cache_subsystem.md#64-vx_cache_init--the-flush-sequencer)). + The runtime flushes lazily: an L1-only flush ahead of the next launch, a + full flush ahead of the next device-memory transfer (see the launch + encoding in §9). --- @@ -317,10 +323,10 @@ is a single-engine CP. Three things gate raising it: 3. **The host serializes anyway.** All host queues funnel into one ring behind one mutex (§9). -The **host-side** scaling limit is the launch path, not the ring: 22 +The **host-side** scaling limit is the launch path, not the ring: 21 commands per launch, unbatched, each with its own doorbell and its own busy-poll loop (§9). `CMD_LAUNCH_QMD` collapses the 20 DCR writes to one -command; batching collapses 22 doorbells to one. Both are implemented in +command; batching collapses 21 doorbells to one. Both are implemented in the Emulation CP and the runtime; the RTL CP needs the QMD mirror. ### 4.5 The submit handshake end to end @@ -475,7 +481,10 @@ unconditionally on every backend. all their flushes at once. It then re-enters `REQ` once per core, sweeping a DCR *read* of the same address with the target core index in `dcr_req_data[15:0]`; each read is answered when that core's flush has - completed, and the command retires after the last one. The DCR response + completed, and the command retires after the last one; a full flush then + holds `DONE` until Vortex `busy` drops, which covers every write until its + AXI B response, so a DMA after the flush cannot overtake the last-level + cache's write-backs on bank 0. The DCR response bus has no back-pressure, so the reads stay one at a time; the flushes themselves overlap and share one walk of the shared caches ([cache_subsystem.md §6](cache_subsystem.md#6-flush-architecture)). @@ -649,13 +658,24 @@ whole run costs one doorbell and one poll. **Launch encoding.** A kernel launch emits **20** `CMD_DCR_WRITE`s ([`queue.cpp:401-427`](../../sw/runtime/common/queue.cpp#L401)) — 4 for the startup/entry PCs, 2 for the args pointer, and 14 for block/grid/cluster -dims, lmem size, block size and warp steps — then `CMD_LAUNCH`, then -`CMD_CACHE_FLUSH` with `arg0 = num_cores`, then a COUT drain +dims, lmem size, block size and warp steps — then `CMD_LAUNCH`, then a COUT drain ([`device.cpp:502-520`](../../sw/runtime/common/device.cpp#L502)). Each group is independently skippable through the legacy escape hatches (no kernel, empty args, `ndim == 0`), so the real count ranges from 20 down to -0. This path does **not** batch: all 22 lines take the non-batch -`cp_submit_cl_`, so one launch costs 22 doorbells and 22 poll loops. Only +0. The kernel-boundary `CMD_CACHE_FLUSH` (`arg0 = num_cores`) is posted lazily +by `cp_ring_append_`, under the same ring lock, from two flags: `l1_dirty_` (a +kernel ran since the last flush) puts an L1-only flush ahead of the next +launch; `l1_dirty_` or `llc_dirty_` (a kernel ran since the last full flush) +puts a full flush ahead of the next `CMD_MEM_*` that touches device memory +outside the IO aperture, because the CP DMA reaches device memory beneath +the last-level cache. A launch followed by a transfer therefore costs one +full flush, and back-to-back launches one L1-only flush each. The COUT drain +needs no flush: the console rings are uncached, and the device `busy` the +launch retires on stays high until memory has completed every write the cores +issued ([cache_subsystem.md §4.9](cache_subsystem.md#49-requests-in-flight-and-device-busy)). The drain's +own transfers stay in the IO aperture and never trigger a flush. +This path does **not** batch: all 21 lines take the non-batch +`cp_submit_cl_`, so one launch costs 21 doorbells and 21 poll loops. Only `enqueue_commands` and the `enqueue_draw` fallback batch. **Queues and serialization.** `vx_queue_create` makes an unbounded number @@ -752,8 +772,8 @@ The following gaps remain **open**. defaults to 1; the Emulation CP models only `q0_`; the runtime programs only queue 0 and funnels every host queue through one ring lock (§9). Real concurrency depends on items 5 and 6. -9. **Batch the launch path.** `enqueue_launch` submits its 22 commands - unbatched — 22 doorbells, 22 busy-polls — while `cp_batch_begin/end` +9. **Batch the launch path.** `enqueue_launch` submits its 21 commands + unbatched — 21 doorbells, 21 busy-polls — while `cp_batch_begin/end` already exists and is used by `enqueue_commands`. Cheap win independent of item 6. 10. **EVENT_WAIT fairness + backoff.** `EVENT_WAIT` holds its arbiter diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index f02c30047b..9b817df196 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -206,6 +206,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .DATA_SIZE (L2_SECTOR_SIZE), .TAG_WIDTH (L2_MEM_TAG_WIDTH) ) l2_mem_bus_if[L2_MEM_PORTS](); + wire l2_req_empty; VX_cache_wrap #( .INSTANCE_ID (`SFORMATF(("%s-l2cache", INSTANCE_ID))), @@ -232,6 +233,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .NC_ENABLE (1), .PASSTHRU (!`VX_CFG_L2_ENABLED), .IS_LLC (L2_IS_LLC), + .SHARED_LLC (L2_SHARED_LLC), .BANK_HASH (`VX_CFG_L2_BANK_HASH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) l2cache ( @@ -241,7 +243,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .cache_perf (l2_perf), `endif .core_bus_if (per_socket_mem_bus_if), - .mem_bus_if (l2_mem_bus_if) + .mem_bus_if (l2_mem_bus_if), + .req_empty (l2_req_empty) ); // Cluster DCR distribution — declared before its consumers (sockets/gfx). @@ -485,9 +488,9 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; wire busy_r; `ifdef EXT_GFX_ANY_ENABLE - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | gfx_busy | mmu_busy, 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | gfx_busy | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); `else - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | mmu_busy, 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); `endif assign busy = busy_r | dcr_bus_if.req_valid | launch_link_busy; diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 9ae6a9a2a9..39e4140148 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1544,6 +1544,13 @@ package VX_gpu_pkg; // L1 dcache is the LLC iff neither L2 nor L3 is enabled. localparam DCACHE_IS_LLC = !`VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; + // It is the point of coherence between kernels only as the one dcache with + // no other memory client beside it (the texture, ray-tracing and DXA ports + // join below it, the raster and OM caches beside it). + localparam DCACHE_SHARED_LLC = DCACHE_IS_LLC && (`VX_CFG_NUM_CORES == 1) && (`VX_CFG_NUM_CLUSTERS == 1) + && !`VX_CFG_EXT_TEX_ENABLED && !`VX_CFG_EXT_RTU_ENABLED + && !`VX_CFG_EXT_RASTER_ENABLED && !`VX_CFG_EXT_OM_ENABLED + && !`VX_CFG_EXT_DXA_ENABLED; ////////////////////////// Tex / Tcache Parameters //////////////////////// `ifdef VX_CFG_EXT_TEX_ENABLE @@ -1773,6 +1780,8 @@ package VX_gpu_pkg; // L2 is the LLC iff L2 is enabled and L3 is not. localparam L2_IS_LLC = `VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; + // Every memory client of a cluster reaches memory through its L2. + localparam L2_SHARED_LLC = L2_IS_LLC && (`VX_CFG_NUM_CLUSTERS == 1); /////////////////////////////// L3 Parameters ///////////////////////////// @@ -1806,6 +1815,7 @@ package VX_gpu_pkg; `endif // L3 is the LLC whenever it is enabled. localparam L3_IS_LLC = `VX_CFG_L3_ENABLED; + localparam L3_SHARED_LLC = L3_IS_LLC; /////////////////////////////////////////////////////////////////////////// diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index a2a27fa7ed..a2ef22cb37 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -67,6 +67,9 @@ module VX_graphics import VX_gpu_pkg::* ); `UNUSED_PARAM (CLUSTER_ID) + wire rcache_req_empty; + wire ocache_req_empty; + // Fan one DCR slave input out to one master per consumer unit. Each // unit's internal case-statement filters by DCR address range. The // VX_dcr_arb owns the rsp_valid/rsp_data signaling on dcr_bus_if so @@ -233,6 +236,7 @@ module VX_graphics import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (rcache_flush_if), + .l1_only (1'b0), .core_bus_if (rcache_bus_if[0]), .cache_bus_if (rcache_flushable_bus_if[0]) ); @@ -274,7 +278,8 @@ module VX_graphics import VX_gpu_pkg::* .cache_perf (rcache_perf), `endif .core_bus_if (rcache_flushable_bus_if), - .mem_bus_if (rcache_mem_bus_tmp_if) + .mem_bus_if (rcache_mem_bus_tmp_if), + .req_empty (rcache_req_empty) ); `ASSIGN_VX_MEM_BUS_IF_EX (rcache_mem_bus_if, rcache_mem_bus_tmp_if[0], @@ -326,8 +331,10 @@ module VX_graphics import VX_gpu_pkg::* // Flush tokens are held at the steers until the OM side is flushed (below). wire [L2_SOCKET_REQS-1:0] steer_flush_req; + wire [L2_SOCKET_REQS-1:0] steer_flush_l1_only; reg [L2_SOCKET_REQS-1:0] steer_flush_ack_r; wire [L2_SOCKET_REQS-1:0] steer_om_pending; + wire [L2_SOCKET_REQS-1:0] steer_req_empty; for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_steer VX_om_steer #( @@ -342,9 +349,24 @@ module VX_graphics import VX_gpu_pkg::* .l2_out_if (l2_out_bus_if[i]), .om_out_if (om_aperture_bus_if[i]), .flush_req (steer_flush_req[i]), + .flush_l1_only (steer_flush_l1_only[i]), .flush_ack (steer_flush_ack_r[i]), .om_pending (steer_om_pending[i]) ); + wire [1:0] steer_out_fire = { + om_aperture_bus_if[i].req_valid && om_aperture_bus_if[i].req_ready, + l2_out_bus_if[i].req_valid && l2_out_bus_if[i].req_ready + }; + VX_mem_req_pending #( + .NUM_INPUTS (1), + .NUM_OUTPUTS (2) + ) steer_pending ( + .clk (clk), + .reset (reset), + .in_fire (socket_mem_bus_if[i].req_valid && socket_mem_bus_if[i].req_ready), + .out_fire (steer_out_fire), + .empty (steer_req_empty[i]) + ); end for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_ingress @@ -454,7 +476,21 @@ module VX_graphics import VX_gpu_pkg::* reg [L2_SOCKET_REQS-1:0] omf_steers_r; // steers this round answers reg omf_cluster_r; // this round answers the cluster flush reg omf_cluster_done_r; + reg omf_l1_only_r; // every requester of this round is L1-only wire omf_cluster_req = cluster_flush_if.req && ~omf_cluster_done_r; + + // The cluster flush takes its kind from the broadcast write that starts it, + // which every core receives too. The cores raise the cluster flush request + // only after taking that write, so the latch is current by then. + reg cluster_flush_l1_only_r; + always @(posedge clk) begin + if (reset) begin + cluster_flush_l1_only_r <= 1'b0; + end else if (dcr_bus_if.req_valid && dcr_bus_if.req_data.rw + && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH)) begin + cluster_flush_l1_only_r <= (dcr_bus_if.req_data.data & VX_DCR_DATA_WIDTH'(`VX_CACHE_FLUSH_L1_ONLY)) != 0; + end + end wire omf_drained = ~(| om_busy_w) && ~(| om_ingress_pending) && ~(| steer_om_pending); always @(posedge clk) begin @@ -463,6 +499,7 @@ module VX_graphics import VX_gpu_pkg::* omf_steers_r <= '0; omf_cluster_r <= 1'b0; omf_cluster_done_r <= 1'b0; + omf_l1_only_r <= 1'b0; steer_flush_ack_r <= '0; end else begin steer_flush_ack_r <= '0; @@ -474,6 +511,8 @@ module VX_graphics import VX_gpu_pkg::* if ((| steer_flush_req) || omf_cluster_req) begin omf_steers_r <= steer_flush_req; omf_cluster_r <= omf_cluster_req; + omf_l1_only_r <= (& (~steer_flush_req | steer_flush_l1_only)) + && (~omf_cluster_req || cluster_flush_l1_only_r); omf_state_r <= OMF_DRAIN; end end @@ -528,6 +567,7 @@ module VX_graphics import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (ocache_om_flush_if[i]), + .l1_only (omf_l1_only_r), .core_bus_if (ocache_bus_if[i]), .cache_bus_if (ocache_om_flush_bus_if[i]) ); @@ -577,7 +617,8 @@ module VX_graphics import VX_gpu_pkg::* .cache_perf (ocache_perf), `endif .core_bus_if (ocache_flushable_bus_if), - .mem_bus_if (ocache_mem_bus_tmp_if) + .mem_bus_if (ocache_mem_bus_tmp_if), + .req_empty (ocache_req_empty) ); `ASSIGN_VX_MEM_BUS_IF_EX (ocache_mem_bus_if, ocache_mem_bus_tmp_if[0], @@ -615,9 +656,16 @@ module VX_graphics import VX_gpu_pkg::* `endif `ifdef VX_CFG_EXT_OM_ENABLE wire om_busy_any = (| om_busy_w) || (| om_ingress_busy); + // requests held in a steer's buffers on the way to L2 or the OM + wire om_req_busy = ~(& steer_req_empty); `else wire om_busy_any = 1'b0; + wire om_req_busy = 1'b0; + assign ocache_req_empty = 1'b1; +`endif +`ifndef VX_CFG_EXT_RASTER_ENABLE + assign rcache_req_empty = 1'b1; `endif - assign busy = raster_busy_any || om_busy_any; + assign busy = raster_busy_any || om_busy_any || om_req_busy || ~rcache_req_empty || ~ocache_req_empty; endmodule diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 315ccf713a..3b04931533 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -138,6 +138,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .DATA_SIZE (ICACHE_LINE_SIZE), .TAG_WIDTH (ICACHE_MEM_TAG_WIDTH) ) icache_mem_bus_if[1](); + wire icache_req_empty; + wire tcache_req_empty; + wire rtcache_req_empty; VX_cache_cluster #( .INSTANCE_ID (`SFORMATF(("%s-icache", INSTANCE_ID))), @@ -171,7 +174,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .core_bus_if (icache_phys_bus_if), - .mem_bus_if (icache_mem_bus_if) + .mem_bus_if (icache_mem_bus_if), + .req_empty (icache_req_empty) ); /////////////////////////////////////////////////////////////////////////// @@ -192,6 +196,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .DATA_SIZE (DCACHE_SECTOR_SIZE), .TAG_WIDTH (DCACHE_MEM_TAG_WIDTH) ) dcache_mem_bus_if[L1_MEM_PORTS](); + wire dcache_req_empty; VX_cache_cluster #( .INSTANCE_ID (`SFORMATF(("%s-dcache", INSTANCE_ID))), @@ -220,6 +225,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), .IS_LLC (DCACHE_IS_LLC), + .SHARED_LLC (DCACHE_SHARED_LLC), .BANK_HASH (`VX_CFG_DCACHE_BANK_HASH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) dcache ( @@ -229,7 +235,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .core_bus_if (dcache_phys_bus_if), - .mem_bus_if (dcache_mem_bus_if) + .mem_bus_if (dcache_mem_bus_if), + .req_empty (dcache_req_empty) ); /////////////////////////////////////////////////////////////////////////// @@ -488,6 +495,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .dcr_flush_if (tcache_flush_if), + .l1_only (1'b0), .core_bus_if (tcache_bus_if[0]), .cache_bus_if (tcache_flushable_bus_if[0]) ); @@ -529,7 +537,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .cache_perf (socket_tcache_perf), `endif .core_bus_if (tcache_flushable_bus_if), - .mem_bus_if (tcache_mem_bus_tmp_if) + .mem_bus_if (tcache_mem_bus_tmp_if), + .req_empty (tcache_req_empty) ); `endif // VX_CFG_EXT_TEX_ENABLE @@ -616,6 +625,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .dcr_flush_if (rtcache_flush_if), + .l1_only (1'b0), .core_bus_if (rtcache_bus_if[0]), .cache_bus_if (rtcache_flushable_bus_if[0]) ); @@ -657,7 +667,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `UNUSED_PIN (cache_perf), `endif .core_bus_if (rtcache_flushable_bus_if), - .mem_bus_if (rtcache_mem_bus_tmp_if) + .mem_bus_if (rtcache_mem_bus_tmp_if), + .req_empty (rtcache_req_empty) ); `endif // VX_CFG_EXT_RTU_ENABLE @@ -744,6 +755,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; /////////////////////////////////////////////////////////////////////////// + wire mem_arb_empty; + // L2-facing memory ports. Port 0 arbitrates icache and dcache with the // socket-resident units' memory ports (tcache, rtcache, DXA gmem) as // peers; priority order keeps icache first and DXA bulk traffic last so @@ -789,6 +802,21 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .bus_out_if (socket_arb_out_if) ); + wire [SOCKET_MEM_ARB_REQS-1:0] arb_in_fire; + for (genvar j = 0; j < SOCKET_MEM_ARB_REQS; ++j) begin : g_arb_in_fire + assign arb_in_fire[j] = socket_arb_in_if[j].req_valid && socket_arb_in_if[j].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (SOCKET_MEM_ARB_REQS), + .NUM_OUTPUTS (1) + ) mem_arb_pending ( + .clk (clk), + .reset (reset), + .in_fire (arb_in_fire), + .out_fire (socket_arb_out_if[0].req_valid && socket_arb_out_if[0].req_ready), + .empty (mem_arb_empty) + ); + `ASSIGN_VX_MEM_BUS_IF (mem_bus_if[0], socket_arb_out_if[0]); end else begin : g_i VX_mem_bus_if #( @@ -910,8 +938,19 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE; ++i) begin : g_kmu_link_valid assign per_core_kmu_valid[i] = per_core_kmu_bus_if[i].valid; end + // Stores are ack-less, so the cores go idle with their last requests still + // on the way out; the socket stays busy until none is held in its caches' + // uncached paths or arbiters. +`ifndef VX_CFG_EXT_TEX_ENABLE + assign tcache_req_empty = 1'b1; +`endif +`ifndef VX_CFG_EXT_RTU_ENABLE + assign rtcache_req_empty = 1'b1; +`endif + wire mem_req_empty = icache_req_empty && dcache_req_empty && tcache_req_empty + && rtcache_req_empty && mem_arb_empty; wire busy_r; - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy), 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy) | ~mem_req_empty, 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); assign busy = busy_r | dcr_bus_if.req_valid | kmu_bus_if[0].valid | (|per_core_kmu_valid); endmodule diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 2b4b49df89..98b5fb6c8e 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -54,6 +54,13 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_CORES), ("NUM_CORES must be a power of 2")); `STATIC_ASSERT(`IS_POW2(`VX_CFG_SOCKET_SIZE), ("SOCKET_SIZE must be a power of 2")); + // A write-back LLC must be the only instance: replicated ones never make + // their dirty lines visible to each other. + `STATIC_ASSERT(!(DCACHE_IS_LLC && `VX_CFG_DCACHE_WRITEBACK) || ((`VX_CFG_NUM_CORES == 1) && (`VX_CFG_NUM_CLUSTERS == 1)), + ("a write-back dcache that is the last-level cache must be the only dcache")); + `STATIC_ASSERT(!(L2_IS_LLC && `VX_CFG_L2_WRITEBACK) || (`VX_CFG_NUM_CLUSTERS == 1), + ("a write-back L2 that is the last-level cache must be the only L2")); + // Every cache strictly above the LLC must be write-through. // A WB intermediate could absorb a hart-B store without the LLC // seeing it; a later SC from hart-A on the same line would @@ -123,6 +130,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .DATA_SIZE (L3_SECTOR_SIZE), .TAG_WIDTH (L3_MEM_TAG_WIDTH) ) mem_bus_if[L3_MEM_PORTS](); + // no request held in the L3 bypass path short of the memory ports + wire l3_req_empty; VX_cache_wrap #( .INSTANCE_ID ("l3cache"), @@ -149,6 +158,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .NC_ENABLE (1), .PASSTHRU (!`VX_CFG_L3_ENABLED), .IS_LLC (L3_IS_LLC), + .SHARED_LLC (L3_SHARED_LLC), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) l3cache ( .clk (clk), @@ -159,7 +169,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `endif .core_bus_if (per_cluster_mem_bus_if), - .mem_bus_if (mem_bus_if) + .mem_bus_if (mem_bus_if), + .req_empty (l3_req_empty) ); for (genvar i = 0; i < L3_MEM_PORTS; ++i) begin : g_mem_bus_if @@ -364,7 +375,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( assign per_cluster_kmu_valid[c] = per_cluster_kmu_bus_if[c].valid; end wire busy_r; - `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy) | dev_mmu_busy, 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); + `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy) | dev_mmu_busy | ~l3_req_empty, 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid | (|per_cluster_kmu_valid); `ifdef PERF_ENABLE diff --git a/hw/rtl/Vortex_axi.sv b/hw/rtl/Vortex_axi.sv index 4566016c42..5a159c9d2a 100644 --- a/hw/rtl/Vortex_axi.sv +++ b/hw/rtl/Vortex_axi.sv @@ -107,6 +107,8 @@ module Vortex_axi import VX_gpu_pkg::*; #( wire [VX_MEM_TAG_WIDTH-1:0] mem_rsp_tag [VX_MEM_PORTS]; wire mem_rsp_ready [VX_MEM_PORTS]; + wire vx_busy; + `SCOPE_IO_SWITCH (1); Vortex vortex ( @@ -137,7 +139,7 @@ module Vortex_axi import VX_gpu_pkg::*; #( .dcr_rsp_data (dcr_rsp_data), .start (start), - .busy (busy) + .busy (vx_busy) ); wire mem_req_valid_a [VX_MEM_PORTS]; @@ -269,4 +271,29 @@ module Vortex_axi import VX_gpu_pkg::*; #( .m_axi_rresp (m_axi_rresp) ); + // AXI orders a write against another master's access only once its B + // response is back, so the device stays busy until every write is acked. + wire [VX_MEM_PORTS-1:0] vx_wr_fire; + wire [AXI_NUM_BANKS-1:0] axi_b_fire; + wire wr_empty; + for (genvar i = 0; i < VX_MEM_PORTS; ++i) begin : g_vx_wr_fire + assign vx_wr_fire[i] = mem_req_valid_a[i] && mem_req_ready_a[i] && mem_req_rw_a[i]; + end + for (genvar i = 0; i < AXI_NUM_BANKS; ++i) begin : g_axi_b_fire + assign axi_b_fire[i] = m_axi_bvalid[i] && m_axi_bready[i]; + end + VX_mem_req_pending #( + .NUM_INPUTS (VX_MEM_PORTS), + .NUM_OUTPUTS (AXI_NUM_BANKS), + .SIZE (32 * AXI_NUM_BANKS) + ) wr_pending ( + .clk (clk), + .reset (reset), + .in_fire (vx_wr_fire), + .out_fire (axi_b_fire), + .empty (wr_empty) + ); + + assign busy = vx_busy || ~wr_empty; + endmodule diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index de35722db6..a682b600b1 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -48,6 +48,7 @@ module VX_cache import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, parameter IS_LLC = 0, // last-level cache: banks own AMO commit + RVA reservation tracking + parameter SHARED_LLC = 0, // the LLC serves every memory client alone (point of coherence) parameter FLUSH_FORWARD = 1, // a flush also flushes the next level @@ -105,6 +106,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0] per_bank_core_req_fire; wire mem_req_empty; wire [MEM_PORTS-1:0] mem_flush_valid; + wire mem_flush_l1_only; wire [MEM_PORTS-1:0] mem_flush_ready; wire mem_flush_rsp_phase; wire [MEM_PORTS-1:0] mem_flush_rsp_valid; @@ -117,6 +119,7 @@ module VX_cache import VX_gpu_pkg::*; #( .NUM_BANKS (NUM_BANKS), .MEM_PORTS (MEM_PORTS), .FLUSH_FORWARD (FLUSH_FORWARD), + .SHARED_LLC (SHARED_LLC), .TAG_WIDTH (TAG_WIDTH), .BANK_SEL_LATENCY (`TO_OUT_BUF_REG(REQ_XBAR_BUF)), // request xbar latency .REQ_SEL_WIDTH (REQ_SEL_WIDTH) @@ -131,6 +134,7 @@ module VX_cache import VX_gpu_pkg::*; #( .flush_end (per_bank_flush_end), .mem_req_empty (mem_req_empty), .mem_flush_valid (mem_flush_valid), + .mem_flush_l1_only (mem_flush_l1_only), .mem_flush_ready (mem_flush_ready), .mem_flush_rsp_phase (mem_flush_rsp_phase), .mem_flush_rsp_valid (mem_flush_rsp_valid), @@ -499,9 +503,10 @@ module VX_cache import VX_gpu_pkg::*; #( wire [MEM_PORTS-1:0][`UP(`LOG2UP(NUM_BANKS))-1:0] mem_req_tmp_idx; wire [MEM_PORTS-1:0] mem_req_tmp_ready; // The flush token for port i is a read of the port's first line with the - // flush attribute. It takes bank i's lane into the request crossbar (bank i - // maps to port i), so the crossbar forms its address and it leaves through - // the same registered outputs as the write-backs ahead of it. + // flush attribute; byteen bit 0 carries the L1-only kind. It takes bank + // i's lane into the request crossbar (bank i maps to port i), so the + // crossbar forms its address and it leaves through the same registered + // outputs as the write-backs ahead of it. wire [NUM_BANKS-1:0] mem_req_xbar_valid; wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] mem_req_xbar_pdata; wire [NUM_BANKS-1:0] mem_req_xbar_ready; @@ -512,7 +517,7 @@ module VX_cache import VX_gpu_pkg::*; #( `UP(BANK_SEL_BITS)'(i), `CS_LINE_SECTOR_ADDR_WIDTH'(0), 1'b0, - SECTOR_SIZE'(0), + SECTOR_SIZE'(mem_flush_l1_only), `CS_SECTOR_WIDTH'(0), BANK_MEM_TAG_WIDTH'(0), `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) diff --git a/hw/rtl/cache/VX_cache_bypass.sv b/hw/rtl/cache/VX_cache_bypass.sv index 931cf0ebab..29232bc2fa 100644 --- a/hw/rtl/cache/VX_cache_bypass.sv +++ b/hw/rtl/cache/VX_cache_bypass.sv @@ -39,7 +39,10 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], VX_mem_bus_if.master core_bus_out_if [NUM_REQS], VX_mem_bus_if.slave mem_bus_in_if [MEM_PORTS], - VX_mem_bus_if.master mem_bus_out_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_out_if [MEM_PORTS], + + // no uncached or outgoing request held short of the memory side + output wire req_empty ); localparam DIRECT_PASSTHRU = !CACHE_ENABLE && (`CS_WORD_SEL_BITS == 0) && (NUM_REQS == MEM_PORTS); localparam CORE_DATA_WIDTH = WORD_SIZE * 8; @@ -224,4 +227,33 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( .bus_in_if (mem_bus_out_src_if), .bus_out_if (mem_bus_out_if) ); + + if (!DIRECT_PASSTHRU) begin : g_req_pending + localparam NUM_SRCS = NUM_REQS + (CACHE_ENABLE ? MEM_PORTS : 0); + wire [NUM_SRCS-1:0] src_fire; + wire [MEM_PORTS-1:0] dst_fire; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_nc_fire + assign src_fire[i] = core_bus_in_nc_if[i].req_valid && core_bus_in_nc_if[i].req_ready; + end + if (CACHE_ENABLE) begin : g_cache_fire + for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_in_fire + assign src_fire[NUM_REQS + i] = mem_bus_in_if[i].req_valid && mem_bus_in_if[i].req_ready; + end + end + for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_dst_fire + assign dst_fire[i] = mem_bus_out_if[i].req_valid && mem_bus_out_if[i].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (NUM_SRCS), + .NUM_OUTPUTS (MEM_PORTS) + ) req_pending ( + .clk (clk), + .reset (reset), + .in_fire (src_fire), + .out_fire (dst_fire), + .empty (req_empty) + ); + end else begin : g_no_req_pending + assign req_empty = 1'b1; + end endmodule diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 31868bcd1b..0dfc14eca1 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -55,6 +55,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Indicates this cache cluster is the LLC (handles AMO completion). + parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). @@ -67,7 +68,10 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( output cache_perf_t cache_perf, `endif VX_mem_bus_if.slave core_bus_if [NUM_INPUTS * NUM_REQS], - VX_mem_bus_if.master mem_bus_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_if [MEM_PORTS], + + // no uncached or outgoing request waiting in the arbiters or bypass paths + output wire req_empty ); localparam NUM_CACHES = `UP(NUM_UNITS); localparam PASSTHRU = (NUM_UNITS == 0); @@ -89,6 +93,10 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (ARB_TAG_WIDTH) ) arb_core_bus_if[NUM_CACHES * NUM_REQS](); + wire [NUM_REQS-1:0] core_arb_empty; + wire [NUM_CACHES-1:0] cache_req_empty; + wire [MEM_PORTS-1:0] mem_arb_empty; + assign req_empty = (& core_arb_empty) && (& cache_req_empty) && (& mem_arb_empty); for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_arb VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), @@ -116,6 +124,28 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .bus_in_if (core_bus_tmp_if), .bus_out_if (arb_core_bus_tmp_if) ); + if (NUM_INPUTS != NUM_CACHES) begin : g_core_arb_pending + wire [NUM_INPUTS-1:0] in_fire; + wire [NUM_CACHES-1:0] out_fire; + for (genvar j = 0; j < NUM_INPUTS; ++j) begin : g_in_fire + assign in_fire[j] = core_bus_tmp_if[j].req_valid && core_bus_tmp_if[j].req_ready; + end + for (genvar k = 0; k < NUM_CACHES; ++k) begin : g_out_fire + assign out_fire[k] = arb_core_bus_tmp_if[k].req_valid && arb_core_bus_tmp_if[k].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (NUM_INPUTS), + .NUM_OUTPUTS (NUM_CACHES) + ) core_arb_pending ( + .clk (clk), + .reset (reset), + .in_fire (in_fire), + .out_fire (out_fire), + .empty (core_arb_empty[i]) + ); + end else begin : g_no_core_arb_pending + assign core_arb_empty[i] = 1'b1; + end for (genvar k = 0; k < NUM_CACHES; ++k) begin : g_arb_core_bus_if `ASSIGN_VX_MEM_BUS_IF (arb_core_bus_if[k * NUM_REQS + i], arb_core_bus_tmp_if[k]); end @@ -147,6 +177,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .NC_ENABLE (NC_ENABLE), .PASSTHRU (PASSTHRU), .IS_LLC (IS_LLC), + .SHARED_LLC (SHARED_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .AMO_ENABLE (AMO_ENABLE) @@ -157,7 +188,8 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (arb_core_bus_if[i * NUM_REQS +: NUM_REQS]), - .mem_bus_if (cache_mem_bus_if[i * MEM_PORTS +: MEM_PORTS]) + .mem_bus_if (cache_mem_bus_if[i * MEM_PORTS +: MEM_PORTS]), + .req_empty (cache_req_empty[i]) ); end for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_if @@ -187,6 +219,24 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .bus_in_if (arb_core_bus_tmp_if), .bus_out_if (mem_bus_tmp_if) ); + if (NUM_CACHES > 1) begin : g_mem_arb_pending + wire [NUM_CACHES-1:0] in_fire; + for (genvar j = 0; j < NUM_CACHES; ++j) begin : g_in_fire + assign in_fire[j] = arb_core_bus_tmp_if[j].req_valid && arb_core_bus_tmp_if[j].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (NUM_CACHES), + .NUM_OUTPUTS (1) + ) mem_arb_pending ( + .clk (clk), + .reset (reset), + .in_fire (in_fire), + .out_fire (mem_bus_tmp_if[0].req_valid && mem_bus_tmp_if[0].req_ready), + .empty (mem_arb_empty[i]) + ); + end else begin : g_no_mem_arb_pending + assign mem_arb_empty[i] = 1'b1; + end if (WRITE_ENABLE) begin : g_we `ASSIGN_VX_MEM_BUS_IF (mem_bus_if[i], mem_bus_tmp_if[0]); end else begin : g_ro diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index 8159a6ba32..9dc988ff3a 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -23,11 +23,20 @@ // the cache's own response path, and the inputs reopen. // A token that arrives during the walk is visible only after all of its // input's earlier requests were accepted, so the walk covered them. +// An L1-only token (any byteen bit set: a next level with wider words sees +// the bit at the token's line offset) is forwarded with its kind. A +// last-level cache that alone serves every memory client is the point of +// coherence for it: once the token is visible, its input's earlier requests +// are inside the cache, ahead of anything the next kernel sends, so it answers +// the token without a walk. Any other last-level cache still walks, since a +// replicated instance or a client reaching memory beside it may hold or need +// lines it has not written back. module VX_cache_init import VX_gpu_pkg::*; #( parameter NUM_REQS = 4, // Number of Word requests per cycle parameter NUM_BANKS = 1, // Number of banks parameter MEM_PORTS = 1, // Number of memory ports parameter FLUSH_FORWARD = 1, // Forward the flush to the next level + parameter SHARED_LLC = 0, // The LLC serving every memory client alone: answers L1-only tokens without a walk parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size parameter BANK_SEL_LATENCY = 1, // Bank select latency parameter REQ_SEL_WIDTH = `UP(`CLOG2(NUM_REQS)) @@ -43,6 +52,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( // flush tokens to the next level input wire mem_req_empty, // no write-back left between the banks and the ports output wire [MEM_PORTS-1:0] mem_flush_valid, + output wire mem_flush_l1_only, input wire [MEM_PORTS-1:0] mem_flush_ready, output wire mem_flush_rsp_phase, input wire [MEM_PORTS-1:0] mem_flush_rsp_valid, @@ -109,12 +119,20 @@ module VX_cache_init import VX_gpu_pkg::*; #( end wire flush_req_enable = (| flush_req_mask); + wire [NUM_REQS-1:0] flush_l1_mask; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_l1_mask + assign flush_l1_mask[i] = flush_req_mask[i] && (| core_bus_in_if[i].req_data.byteen); + end + // every visible token is L1-only + wire flush_l1_only = (flush_l1_mask == flush_req_mask); + reg [NUM_BANKS-1:0] flush_done, flush_done_n; reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now reg [MEM_PORTS-1:0] fwd_sent, fwd_sent_n; reg [MEM_PORTS-1:0] fwd_done, fwd_done_n; reg [NUM_REQS-1:0] rsp_pending, rsp_pending_n; // token still to be answered reg [`UP(UUID_WIDTH)-1:0] flush_uuid_r, flush_uuid_n; + reg l1_only_r, l1_only_n; // answer the lowest pending input; its token stays at the input until then wire [NUM_REQS-1:0] rsp_sel_onehot; @@ -203,11 +221,19 @@ module VX_cache_init import VX_gpu_pkg::*; #( fwd_done_n = fwd_done; rsp_pending_n = rsp_pending & ~(rsp_sel_onehot & {NUM_REQS{flush_rsp_fire}}); flush_uuid_n = flush_uuid_r; + l1_only_n = l1_only_r; case (state) // STATE_IDLE: default: begin if (flush_req_enable) begin - state_n = (BANK_SEL_LATENCY != 0) ? STATE_WAIT1 : STATE_FLUSH; + if (BANK_SEL_LATENCY != 0) begin + state_n = STATE_WAIT1; + end else if (SHARED_LLC && flush_l1_only) begin + rsp_pending_n = flush_req_mask; + state_n = STATE_RESPOND; + end else begin + state_n = STATE_FLUSH; + end for (integer i = NUM_REQS-1; i >= 0; --i) begin if (flush_req_mask[i]) begin flush_uuid_n = core_bus_in_uuid[i]; @@ -217,7 +243,12 @@ module VX_cache_init import VX_gpu_pkg::*; #( end STATE_WAIT1: begin if (no_inflight_reqs) begin - state_n = STATE_FLUSH; + if (SHARED_LLC && flush_l1_only) begin + rsp_pending_n = flush_req_mask; + state_n = STATE_RESPOND; + end else begin + state_n = STATE_FLUSH; + end end end STATE_FLUSH: begin @@ -231,6 +262,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( flush_done_n = '0; // answer the tokens visible now; later ones start a new walk rsp_pending_n = flush_req_mask; + l1_only_n = flush_l1_only; state_n = FLUSH_FORWARD ? STATE_FORWARD : STATE_RESPOND; end end @@ -270,6 +302,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( rsp_pending <= rsp_pending_n; end flush_uuid_r <= flush_uuid_n; + l1_only_r <= l1_only_n; end assign flush_begin = {NUM_BANKS{state == STATE_FLUSH}}; @@ -279,5 +312,6 @@ module VX_cache_init import VX_gpu_pkg::*; #( // take the others' valid away. assign mem_flush_valid = {MEM_PORTS{(state == STATE_FORWARD) && (fwd_go || mem_req_empty)}} & ~fwd_sent; assign mem_flush_rsp_phase = (state == STATE_FORWARD); + assign mem_flush_l1_only = l1_only_r; endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 24ca0f7a62..e9f34d17bc 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -55,6 +55,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Set when this cache is the LLC (enables AMO completion at this level). + parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). @@ -68,7 +69,10 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( output cache_perf_t cache_perf, `endif VX_mem_bus_if.slave core_bus_if [NUM_REQS], - VX_mem_bus_if.master mem_bus_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_if [MEM_PORTS], + + // no uncached or outgoing request waiting in the bypass path + output wire req_empty ); `STATIC_ASSERT(NUM_BANKS == (1 << `CLOG2(NUM_BANKS)), ("invalid parameter")) localparam CACHE_MEM_TAG_WIDTH = `CACHE_MEM_TAG_WIDTH(MSHR_SIZE, NUM_BANKS, MEM_PORTS, UUID_WIDTH); @@ -108,9 +112,11 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .core_bus_in_if (core_bus_if), .core_bus_out_if(core_bus_cache_if), .mem_bus_in_if (mem_bus_cache_if), - .mem_bus_out_if (mem_bus_tmp_if) + .mem_bus_out_if (mem_bus_tmp_if), + .req_empty (req_empty) ); end else begin : g_no_bypass + assign req_empty = 1'b1; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_cache_if `ASSIGN_VX_MEM_BUS_IF (core_bus_cache_if[i], core_bus_if[i]); end @@ -149,6 +155,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .CORE_OUT_BUF (BYPASS_ENABLE ? 1 : CORE_OUT_BUF), .MEM_OUT_BUF (BYPASS_ENABLE ? 1 : MEM_OUT_BUF), .IS_LLC (IS_LLC), + .SHARED_LLC (SHARED_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .AMO_ENABLE (AMO_ENABLE) diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 46b3918de4..618fabade9 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -174,6 +174,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_dcr_flush_if dcr_flush_if(); VX_dcr_flush_if dcr_flush_dcache_if(); VX_dcr_flush_if dcr_flush_icache_if(); + wire dcr_flush_l1_only; // High when the in-core global-store path is empty (no store still in // flight toward the dcache). Gates `busy` so the flush waits for the tail. @@ -209,7 +210,8 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .reset (reset), .dcr_bus_if (dcr_bus_if), .dcr_csr_if (dcr_csr_if), - .dcr_flush_if(dcr_flush_if) + .dcr_flush_if(dcr_flush_if), + .flush_l1_only(dcr_flush_l1_only) ); `SCOPE_IO_SWITCH (3); @@ -271,6 +273,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .dcr_flush_if (dcr_flush_icache_if), + .l1_only (dcr_flush_l1_only), .core_bus_if (fetch_icache_if[0]), .cache_bus_if (mmu_icache_if[0]) ); @@ -445,6 +448,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .empty (mem_unit_empty), .lsu_mem_if (lsu_mem_if), .dcr_flush_if (dcr_flush_dcache_if), + .flush_l1_only (dcr_flush_l1_only), .dcache_bus_if (mmu_dcache_if) ); @@ -477,7 +481,13 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // busy tracks only kernel execution: warps resident or stores not yet at L1. // ~mmu_empty also keeps the core busy while a translation or parked store is // still in flight, so kernel completion cannot race un-drained accesses. - assign busy = sched_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || ~mmu_empty; + // A request waiting at the dcache port sits past the flush injector's + // buffer, which mem_unit_empty does not cover. + wire [DCACHE_NUM_REQS-1:0] dcache_req_valid; + for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_req_valid + assign dcache_req_valid[i] = mmu_dcache_if[i].req_valid; + end + assign busy = sched_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || (| dcache_req_valid) || ~mmu_empty; // BAR (vx_barrier / vx_barrier_arrive) drains LSU + MMU before suspending or registering arrival. assign warp_ctl_if.lsu_sched_drained = (&lsu_sched_empty) && mmu_empty; diff --git a/hw/rtl/core/VX_dcr_data.sv b/hw/rtl/core/VX_dcr_data.sv index 746848286e..44f5624b82 100644 --- a/hw/rtl/core/VX_dcr_data.sv +++ b/hw/rtl/core/VX_dcr_data.sv @@ -27,7 +27,8 @@ module VX_dcr_data import VX_gpu_pkg::*; #( VX_dcr_csr_if.master dcr_csr_if, // DCR-triggered cache flush - VX_dcr_flush_if.master dcr_flush_if + VX_dcr_flush_if.master dcr_flush_if, + output wire flush_l1_only ); `UNUSED_SPARAM (INSTANCE_ID) @@ -82,6 +83,7 @@ module VX_dcr_data import VX_gpu_pkg::*; #( reg flush_pending_r; // flush running reg flush_complete_r; // flush finished, not yet reported reg flush_read_r; // flush read waiting for completion + reg flush_l1_only_r; // the last-level cache keeps its lines wire is_flush_req = dcr_bus_if.req_valid && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); @@ -98,15 +100,20 @@ module VX_dcr_data import VX_gpu_pkg::*; #( flush_pending_r <= 1'b0; flush_complete_r <= 1'b0; flush_read_r <= 1'b0; + flush_l1_only_r <= 1'b0; end else begin flush_pending_r <= flush_start || (flush_pending_r && ~flush_done); // a write opens a new flush, so an older completion cannot answer it flush_complete_r <= ~is_flush_write && ~flush_rsp && (flush_complete_r || flush_done); flush_read_r <= ~flush_rsp && (flush_read_r || is_flush_read); + if (flush_start) begin + flush_l1_only_r <= (dcr_bus_if.req_data.data & VX_DCR_DATA_WIDTH'(`VX_CACHE_FLUSH_L1_ONLY)) != 0; + end end end assign dcr_flush_if.req = flush_pending_r; + assign flush_l1_only = flush_l1_only_r; // Return CSR data or flush-done on DCR response bus // Use intermediate registers — Vivado (Synth 8-453) forbids procedural diff --git a/hw/rtl/core/VX_dcr_flush.sv b/hw/rtl/core/VX_dcr_flush.sv index 051eaf85d2..8bf7ec4883 100644 --- a/hw/rtl/core/VX_dcr_flush.sv +++ b/hw/rtl/core/VX_dcr_flush.sv @@ -33,6 +33,7 @@ module VX_dcr_flush import VX_gpu_pkg::*; #( input wire reset, VX_dcr_flush_if.slave dcr_flush_if, + input wire l1_only, // the last-level cache need not walk VX_mem_bus_if.slave core_bus_if, // TAG_WIDTH VX_mem_bus_if.master cache_bus_if // TAG_WIDTH+1 ); @@ -72,12 +73,13 @@ module VX_dcr_flush import VX_gpu_pkg::*; #( rw: 1'b0, addr: '0, data: '0, - byteen: '0, + byteen: WORD_SIZE'(l1_only), attr: MEM_ATTR_WIDTH'(1 << MEM_ATTR_FLUSH_OFFS), tag: '0 }; - // The flush request only consumes MEM_ATTR_FLUSH_OFFS downstream; the - // remaining req_data fields are tied off and intentionally unused. + // The flush request only consumes MEM_ATTR_FLUSH_OFFS and byteen bit 0 + // (the L1-only kind) downstream; the remaining req_data fields are tied + // off and intentionally unused. `UNUSED_VAR (flush_bus_if.req_data.rw) `UNUSED_VAR (flush_bus_if.req_data.addr) `UNUSED_VAR (flush_bus_if.req_data.data) diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_slice.sv index affb932298..066df5073e 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_slice.sv @@ -261,7 +261,8 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( default : mem_req_byteen_w = {LSU_WORD_SIZE{1'b1}}; endcase end - assign mem_req_byteen[i] = mem_req_byteen_w; + // a fence is a flush token: it carries no data, so no byte enables + assign mem_req_byteen[i] = req_is_fence ? '0 : mem_req_byteen_w; end // memory misalignment not supported! diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index bcd4083306..a766858777 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -38,6 +38,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( VX_lsu_mem_if.slave lsu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], VX_dcr_flush_if.slave dcr_flush_if, + input wire flush_l1_only, VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] ); // Per-block empty: high when the global-store path holds no in-flight @@ -424,6 +425,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .dcr_flush_if (lane_flush_if[LANE]), + .l1_only (flush_l1_only), .core_bus_if (dcache_bus_tmp_if[j]), .cache_bus_if (dcache_bus_if[LANE]) ); diff --git a/hw/rtl/cp/VX_cp_core.sv b/hw/rtl/cp/VX_cp_core.sv index 9ff16e575a..ee0849b4e7 100644 --- a/hw/rtl/cp/VX_cp_core.sv +++ b/hw/rtl/cp/VX_cp_core.sv @@ -358,7 +358,8 @@ module VX_cp_core .dcr_req_addr (gpu_if_int.dcr_req_addr), .dcr_req_data (gpu_if_int.dcr_req_data), .dcr_rsp_valid (gpu_if_int.dcr_rsp_valid), - .dcr_rsp_data (gpu_if_int.dcr_rsp_data) + .dcr_rsp_data (gpu_if_int.dcr_rsp_data), + .gpu_busy (gpu_if_int.busy) ); `UNUSED_VAR (gpu_if_int.dcr_req_ready) diff --git a/hw/rtl/cp/VX_cp_dcr_proxy.sv b/hw/rtl/cp/VX_cp_dcr_proxy.sv index 05a90fd6de..3ae9a3d583 100644 --- a/hw/rtl/cp/VX_cp_dcr_proxy.sv +++ b/hw/rtl/cp/VX_cp_dcr_proxy.sv @@ -16,12 +16,15 @@ // // For CMD_CACHE_FLUSH (cmd.arg0 = number of cores): // IDLE → START (DCR write to VX_DCR_BASE_CACHE_FLUSH, which every core -// receives and starts flushing on) → then REQ/WAIT_RSP once per core: a +// receives and starts flushing on; F_FLUSH_L1_ONLY sets +// VX_CACHE_FLUSH_L1_ONLY in its value) → then REQ/WAIT_RSP once per core: a // DCR read of VX_DCR_BASE_CACHE_FLUSH answered when that core's flush has // completed (see VX_dcr_data). The cores flush concurrently, so their dcache // flushes reach the shared caches together and share one walk; the reads // are serial because the DCR response bus cannot carry two at once. The -// command retires once the last core reports. This is the ACQUIRE_MEM +// command retires once the last core reports and, for a full flush, once +// Vortex busy drops: busy covers every write until memory acknowledges it, +// so a DMA after the flush cannot overtake the write-backs. This is the ACQUIRE_MEM // model: a single command in the ring, executed by the CP across all cores, // that the host posts after CMD_LAUNCH so it sees coherent results. The // host fills cmd.arg0 from VX_CAPS_NUM_CORES. @@ -39,7 +42,7 @@ module VX_cp_dcr_proxy input wire grant, // verilator lint_off UNUSED - // Only cmd.hdr.opcode, cmd.arg0, and cmd.arg1 are read here. arg2 and + // Only cmd.hdr.opcode/flags, cmd.arg0, and cmd.arg1 are read here. arg2 and // profile_slot pass through untouched on the way to the engine; the // top-level instantiation hands us the full struct. input cmd_t cmd, @@ -57,7 +60,9 @@ module VX_cp_dcr_proxy output logic [`VX_DCR_ADDR_BITS-1:0] dcr_req_addr, output logic [`VX_DCR_DATA_BITS-1:0] dcr_req_data, input wire dcr_rsp_valid, - input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data + input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data, + + input wire gpu_busy ); typedef enum logic [2:0] { @@ -75,6 +80,7 @@ module VX_cp_dcr_proxy state_e state; logic pending_is_read; logic pending_is_flush; + logic pending_l1_only; // cmd fields are only valid for one cycle (grant pulse); capture on IDLE → REQ. logic [`VX_DCR_ADDR_BITS-1:0] pending_addr; logic [`VX_DCR_DATA_BITS-1:0] pending_data; @@ -88,11 +94,17 @@ module VX_cp_dcr_proxy wire [`VX_DCR_DATA_BITS-1:0] cmd_data = cmd.arg1[`VX_DCR_DATA_BITS-1:0]; wire [CIDW-1:0] cmd_ncores = cmd.arg0[CIDW-1:0]; + // The last-level cache forwards a full token behind its write-backs, so they + // are already counted in busy when the last core reports. An L1-only flush + // writes nothing back past the last-level cache. + wire flush_drain = pending_is_flush && !pending_l1_only && gpu_busy; + always_ff @(posedge clk) begin if (reset) begin state <= S_IDLE; pending_is_read <= 1'b0; pending_is_flush <= 1'b0; + pending_l1_only <= 1'b0; pending_addr <= '0; pending_data <= '0; rsp_data_r <= '0; @@ -104,6 +116,7 @@ module VX_cp_dcr_proxy if (grant) begin pending_is_read <= is_read; pending_is_flush <= is_flush; + pending_l1_only <= cmd.hdr.flags[F_FLUSH_L1_ONLY]; pending_addr <= cmd_addr; pending_data <= cmd_data; if (is_flush) begin @@ -140,7 +153,9 @@ module VX_cp_dcr_proxy end end S_DONE: begin - state <= S_IDLE; + if (!flush_drain) begin + state <= S_IDLE; + end end default: state <= S_IDLE; endcase @@ -154,13 +169,14 @@ module VX_cp_dcr_proxy // target core index in the low 16 bits of dcr_req_data. dcr_req_rw = (state == S_START); dcr_req_addr = `VX_DCR_ADDR_BITS'(`VX_DCR_BASE_CACHE_FLUSH); - dcr_req_data = `VX_DCR_DATA_BITS'(flush_cid); + dcr_req_data = (state == S_START) ? (pending_l1_only ? `VX_DCR_DATA_BITS'(`VX_CACHE_FLUSH_L1_ONLY) : '0) + : `VX_DCR_DATA_BITS'(flush_cid); end else begin dcr_req_rw = !pending_is_read; dcr_req_addr = pending_addr; dcr_req_data = pending_data; end - done = (state == S_DONE); + done = (state == S_DONE) && !flush_drain; last_rsp_data = rsp_data_r; end diff --git a/hw/rtl/cp/VX_cp_pkg.sv b/hw/rtl/cp/VX_cp_pkg.sv index a7f1d1ae70..8a8f1d9452 100644 --- a/hw/rtl/cp/VX_cp_pkg.sv +++ b/hw/rtl/cp/VX_cp_pkg.sv @@ -80,6 +80,8 @@ package VX_cp_pkg; localparam int F_PROFILE = 0; localparam int F_FENCE_PRE = 1; + // CMD_CACHE_FLUSH: flush only the levels above the last-level cache + localparam int F_FLUSH_L1_ONLY = 2; typedef struct packed { logic [15:0] reserved; diff --git a/hw/rtl/mem/VX_mem_req_pending.sv b/hw/rtl/mem/VX_mem_req_pending.sv new file mode 100644 index 0000000000..b92e287920 --- /dev/null +++ b/hw/rtl/mem/VX_mem_req_pending.sv @@ -0,0 +1,53 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Tracks the requests held between the input and output handshakes of a +// section that neither creates nor drops requests (arbiters, buffers). +module VX_mem_req_pending #( + parameter NUM_INPUTS = 1, + parameter NUM_OUTPUTS = 1, + parameter SIZE = 8 * NUM_OUTPUTS + 2 * NUM_INPUTS // bound on the requests the section can hold +) ( + input wire clk, + input wire reset, + input wire [NUM_INPUTS-1:0] in_fire, + input wire [NUM_OUTPUTS-1:0] out_fire, + output wire empty +); + localparam INCRW = `CLOG2(NUM_INPUTS+1); + localparam DECRW = `CLOG2(NUM_OUTPUTS+1); + + wire [INCRW-1:0] in_cnt; + wire [DECRW-1:0] out_cnt; + `POP_COUNT(in_cnt, in_fire); + `POP_COUNT(out_cnt, out_fire); + + VX_pending_size #( + .SIZE (SIZE), + .INCRW (INCRW), + .DECRW (DECRW) + ) pending ( + .clk (clk), + .reset (reset), + .incr (in_cnt), + .decr (out_cnt), + .empty (empty), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + +endmodule diff --git a/hw/rtl/om/VX_om_steer.sv b/hw/rtl/om/VX_om_steer.sv index 7aa803e5ee..0fd8ce9c8e 100644 --- a/hw/rtl/om/VX_om_steer.sv +++ b/hw/rtl/om/VX_om_steer.sv @@ -48,6 +48,7 @@ module VX_om_steer import VX_gpu_pkg::*; #( VX_mem_bus_if.master om_out_if, // aperture writes -> OM ingress (posted) output wire flush_req, // a flush token is held until the OM side is flushed + output wire flush_l1_only, // the held token is an L1-only one input wire flush_ack, // pulse: the OM side was flushed after flush_req rose output wire om_pending // an aperture write is still buffered on the OM leg ); @@ -89,6 +90,7 @@ module VX_om_steer import VX_gpu_pkg::*; #( end end assign flush_req = flush_req_r; + assign flush_l1_only = (| bus_in_if.req_data.byteen); assign om_pending = om_out_if.req_valid; // Both legs go through an OUT_BUF elastic buffer so the socket->L2 trunk and diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index 0e25d21b96..b75215a485 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -195,7 +195,8 @@ module VX_cache_top import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (core_bus_if), - .mem_bus_if (mem_bus_if) + .mem_bus_if (mem_bus_if), + `UNUSED_PIN (req_empty) ); endmodule diff --git a/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv b/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv index 060b56a28d..1de85c8b61 100644 --- a/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv +++ b/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv @@ -28,7 +28,8 @@ module VX_cp_dcr_proxy_top output wire [`VX_DCR_ADDR_BITS-1:0] dcr_req_addr, output wire [`VX_DCR_DATA_BITS-1:0] dcr_req_data, input wire dcr_rsp_valid, - input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data + input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data, + input wire gpu_busy ); cmd_t cmd_typed; @@ -46,7 +47,8 @@ module VX_cp_dcr_proxy_top .dcr_req_addr (dcr_req_addr), .dcr_req_data (dcr_req_data), .dcr_rsp_valid (dcr_rsp_valid), - .dcr_rsp_data (dcr_rsp_data) + .dcr_rsp_data (dcr_rsp_data), + .gpu_busy (gpu_busy) ); endmodule : VX_cp_dcr_proxy_top diff --git a/hw/unittest/cp_dcr_proxy/main.cpp b/hw/unittest/cp_dcr_proxy/main.cpp index 861c0c2932..171acef369 100644 --- a/hw/unittest/cp_dcr_proxy/main.cpp +++ b/hw/unittest/cp_dcr_proxy/main.cpp @@ -26,6 +26,11 @@ // core index as data, each waiting for its response; done only // after the last core answers. // 6. CMD_CACHE_FLUSH over 0 cores retires without any DCR request. +// 7. CMD_CACHE_FLUSH with the L1-only flag: the broadcast write carries +// VX_CACHE_FLUSH_L1_ONLY (test 5 checks a full flush writes 0), the +// per-core read still carries the core index. +// 8. A full CMD_CACHE_FLUSH holds done while Vortex is busy; an L1-only +// one retires regardless. // ============================================================================ #include "vl_simulator.h" @@ -80,11 +85,14 @@ static void pack_cmd(uint32_t out_words[9], (static_cast(flags) << 8); } +// CMD_CACHE_FLUSH header flag bit 2 (F_FLUSH_L1_ONLY). +constexpr uint8_t FLUSH_FLAG_L1_ONLY = 0x04; + template static void set_cmd(T* top, uint8_t opcode, - uint64_t arg0 = 0, uint64_t arg1 = 0) { + uint64_t arg0 = 0, uint64_t arg1 = 0, uint8_t flags = 0) { uint32_t words[9]; - pack_cmd(words, opcode, 0, arg0, arg1, /*arg2=*/0, /*profile_slot=*/0); + pack_cmd(words, opcode, flags, arg0, arg1, /*arg2=*/0, /*profile_slot=*/0); for (int i = 0; i < 9; ++i) top->cmd_packed[i] = words[i]; } @@ -105,6 +113,7 @@ int main(int argc, char** argv) { sim->grant = 0; sim->dcr_rsp_valid = 0; sim->dcr_rsp_data = 0; + sim->gpu_busy = 0; set_cmd(sim.operator->(), 0); tick = sim.reset(tick); @@ -214,6 +223,7 @@ int main(int argc, char** argv) { EXPECT(sim->dcr_req_valid == 1, "FLUSH: start write not issued"); EXPECT(sim->dcr_req_rw == 1, "FLUSH: start must be a write"); EXPECT(sim->dcr_req_addr == VX_DCR_BASE_CACHE_FLUSH, "FLUSH: start addr mismatch"); + EXPECT(sim->dcr_req_data == 0, "FLUSH: a full flush starts with value 0"); EXPECT(sim->done == 0, "FLUSH: done premature in S_START"); cycle(sim, tick); // S_START → S_REQ @@ -257,6 +267,62 @@ int main(int argc, char** argv) { EXPECT(sim->done == 1, "FLUSH0: done not asserted"); cycle(sim, tick); + // ----- Test 7: L1-only CMD_CACHE_FLUSH over 1 core. ----- + set_cmd(sim.operator->(), OP_CACHE_FLUSH, 1, 0, FLUSH_FLAG_L1_ONLY); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_START + sim->grant = 0; + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSHL1: start write not issued"); + EXPECT(sim->dcr_req_rw == 1, "FLUSHL1: start must be a write"); + EXPECT(sim->dcr_req_data == VX_CACHE_FLUSH_L1_ONLY, "FLUSHL1: start value must carry the L1-only flag"); + cycle(sim, tick); // S_START → S_REQ + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSHL1: per-core read not issued"); + EXPECT(sim->dcr_req_rw == 0, "FLUSHL1: per-core request must be a read"); + EXPECT(sim->dcr_req_data == 0, "FLUSHL1: read must carry the core index only"); + cycle(sim, tick); // S_REQ → S_WAIT_RSP + sim->dcr_rsp_valid = 1; + sim->dcr_rsp_data = 0; + cycle(sim, tick); // → S_DONE + sim->dcr_rsp_valid = 0; + sim->eval(); + EXPECT(sim->done == 1, "FLUSHL1: done not asserted"); + cycle(sim, tick); + + // ----- Test 8: a full flush waits for busy; an L1-only one does not. ----- + for (int l1_only = 0; l1_only < 2; ++l1_only) { + set_cmd(sim.operator->(), OP_CACHE_FLUSH, 1, 0, l1_only ? FLUSH_FLAG_L1_ONLY : 0); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_START + sim->grant = 0; + cycle(sim, tick); // S_START → S_REQ + cycle(sim, tick); // S_REQ → S_WAIT_RSP + sim->gpu_busy = 1; + sim->dcr_rsp_valid = 1; + cycle(sim, tick); // → S_DONE + sim->dcr_rsp_valid = 0; + if (l1_only) { + sim->eval(); + EXPECT(sim->done == 1, "DRAIN: an L1-only flush must not wait for busy"); + cycle(sim, tick); + sim->gpu_busy = 0; + } else { + for (int i = 0; i < 4; ++i) { + sim->eval(); + EXPECT(sim->done == 0, "DRAIN: a full flush retired while busy"); + EXPECT(sim->dcr_req_valid == 0, "DRAIN: request while draining"); + cycle(sim, tick); + } + sim->gpu_busy = 0; + sim->eval(); + EXPECT(sim->done == 1, "DRAIN: done not asserted once idle"); + cycle(sim, tick); + sim->eval(); + EXPECT(sim->done == 0, "DRAIN: done should pulse only one cycle"); + } + } + std::printf("PASSED\n"); return 0; } diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index c16aae9aaf..aae201f6e1 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -292,6 +292,7 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (tcache_flush_if), + .l1_only (1'b0), .core_bus_if (tcache_bus_if[0]), .cache_bus_if (tcache_flushable_bus_if[0]) ); @@ -329,7 +330,8 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .core_bus_if (tcache_flushable_bus_if), - .mem_bus_if (tcache_mem_bus_tmp_if) + .mem_bus_if (tcache_mem_bus_tmp_if), + `UNUSED_PIN (req_empty) ); `else assign flush_done_w[0] = 1'b1; @@ -444,6 +446,7 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (rtcache_flush_if), + .l1_only (1'b0), .core_bus_if (rtcache_bus_if[0]), .cache_bus_if (rtcache_flushable_bus_if[0]) ); @@ -481,7 +484,8 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .core_bus_if (rtcache_flushable_bus_if), - .mem_bus_if (rtcache_mem_bus_tmp_if) + .mem_bus_if (rtcache_mem_bus_tmp_if), + `UNUSED_PIN (req_empty) ); `else assign flush_done_w[1] = 1'b1; @@ -765,7 +769,8 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .core_bus_if (l2_core_if), - .mem_bus_if (l2_mem_if) + .mem_bus_if (l2_mem_if), + `UNUSED_PIN (req_empty) ); endmodule diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index e88c3d87b9..ad2c75034e 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -130,6 +130,7 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( .empty (mem_unit_empty), .lsu_mem_if (lsu_mem_if), .dcr_flush_if (dcr_flush_if), + .flush_l1_only (1'b0), .dcache_bus_if (mem_bus_if) ); diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 70244c6cc7..629ab576c4 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -278,6 +278,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .empty (mem_unit_empty[c]), .lsu_mem_if (lsu_mem_if), .dcr_flush_if (dcr_flush_dcache_if), + .flush_l1_only (1'b0), .dcache_bus_if (dcache_bus_if) ); @@ -394,7 +395,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (all_dc_core_if), - .mem_bus_if (dcache_mem_bus_if) + .mem_bus_if (dcache_mem_bus_if), + `UNUSED_PIN (req_empty) ); // ======================================================================== @@ -513,7 +515,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (l2_core_if), - .mem_bus_if (l2_mem_if) + .mem_bus_if (l2_mem_if), + `UNUSED_PIN (req_empty) ); assign empty = (& mem_unit_empty) && (& lsu_sched_empty) && !dxa_busy; diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 6e2f7a32e0..54e4c29b15 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -315,7 +315,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (dmmu_phys_if), - .mem_bus_if (all_dc_mem_if[s * L1_MEM_PORTS +: L1_MEM_PORTS]) + .mem_bus_if (all_dc_mem_if[s * L1_MEM_PORTS +: L1_MEM_PORTS]), + `UNUSED_PIN (req_empty) ); if (INCLUDE_IMMU != 0) begin : g_immu @@ -417,7 +418,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (immu_phys_if), - .mem_bus_if (ic_mem_if) + .mem_bus_if (ic_mem_if), + `UNUSED_PIN (req_empty) ); end else begin : g_no_immu // Keep the socket arbiter 2-wide (so TLB_SOCKET_ID_WIDTH matches the @@ -625,7 +627,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (l2_core_if), - .mem_bus_if (l2_mem_if) + .mem_bus_if (l2_mem_if), + `UNUSED_PIN (req_empty) ); assign empty = (& dmmu_empty) && (& immu_empty) && l2tlb_empty && ptw_empty; diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index c6e97fb46f..2ace286517 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -332,16 +332,20 @@ void CommandProcessor::apply_qmd_(uint64_t qmd_addr) { // flush, so the cores flush concurrently and share one walk of the shared // caches; the per-core reads, answered once that core's flush completes, are // serial because the DCR response bus cannot carry two responses at once. -void CommandProcessor::flush_cores_(uint32_t num_cores) { +void CommandProcessor::flush_cores_(uint32_t num_cores, bool l1_only) { if (num_cores == 0 || !hooks_.vortex_dcr_read) { return; } if (hooks_.vortex_dcr_write) { - hooks_.vortex_dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); + hooks_.vortex_dcr_write(VX_DCR_BASE_CACHE_FLUSH, + l1_only ? VX_CACHE_FLUSH_L1_ONLY : 0); } for (uint32_t cid = 0; cid < num_cores; ++cid) { (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); } + if (!l1_only && hooks_.vortex_drain) { + hooks_.vortex_drain(); + } } // Execute one draw-bundle step. Inline ops (DCR_WRITE/DCR_READ/CACHE_FLUSH) @@ -367,7 +371,8 @@ bool CommandProcessor::exec_inline_cmd_(const Cmd& c) { uint32_t(c.arg0 & 0xFFF), uint32_t(c.arg1 & 0xFFFFFFFF)); return false; case OP_CACHE_FLUSH: - this->flush_cores_(uint32_t(c.arg0 & 0xFFFFFFFF)); + this->flush_cores_(uint32_t(c.arg0 & 0xFFFFFFFF), + (c.flags & FLUSH_FLAG_L1_ONLY) != 0); return false; default: // NOP / FENCE / unknown step — no-op (draws don't use MEM_*/EVENT_*). @@ -497,7 +502,8 @@ void CommandProcessor::tick_engine() { eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_CACHE_FLUSH) { // arg0 carries num_cores. - this->flush_cores_(uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF)); + this->flush_cores_(uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF), + (cur_cmd_.flags & FLUSH_FLAG_L1_ONLY) != 0); eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_EVENT_SIG) { // CMD_EVENT_SIGNAL: write arg1 (8-byte value) to arg0 diff --git a/sim/common/cmd_processor.h b/sim/common/cmd_processor.h index 38e0f80f4c..956d786e03 100644 --- a/sim/common/cmd_processor.h +++ b/sim/common/cmd_processor.h @@ -98,6 +98,11 @@ class CommandProcessor { // before retiring the CMD_LAUNCH. std::function vortex_busy; + // Block until memory has acknowledged every write Vortex issued. A + // full CMD_CACHE_FLUSH retires only then. Unset for a model whose + // writes land as they are issued. + std::function vortex_drain; + // Backend decodes the MMU fault-report DCRs. Published in DEV_CAPS // so the runtime does not read them on a target that would leave // the DCR bus waiting for a response. @@ -164,6 +169,10 @@ class CommandProcessor { // mode is its own descriptor field (modern-GPU convention). static constexpr uint8_t MEM_FLAG_PHYSICAL = 0x04; + // CMD_CACHE_FLUSH header flag: flush only the levels above the last-level + // cache. + static constexpr uint8_t FLUSH_FLAG_L1_ONLY = 0x04; + // Decoded cmd record (matches cmd_t struct layout on-wire). struct Cmd { uint8_t opcode; @@ -242,8 +251,9 @@ class CommandProcessor { // CMD_LAUNCH_QMD: read the in-memory KMU descriptor at `qmd_addr` and // replay its {dcr_addr,value} pairs through the DCR-write hook. void apply_qmd_(uint64_t qmd_addr); - // CMD_CACHE_FLUSH across cores [0, num_cores). - void flush_cores_(uint32_t num_cores); + // CMD_CACHE_FLUSH across cores [0, num_cores); l1_only leaves the + // last-level cache unwalked. + void flush_cores_(uint32_t num_cores, bool l1_only); // Advance the launch FSM one step using cur_cmd_. void tick_launch(); // Advance the engine FSM one step. diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index 416c60208b..98e1eaa40d 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -246,6 +246,12 @@ class Processor::Impl { return 0; } + void drain() { + while (device_->busy) { + this->tick(); + } + } + private: void reset() { @@ -497,4 +503,8 @@ int Processor::dcr_write(uint32_t addr, uint32_t value) { int Processor::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { return impl_->dcr_read(addr, tag, value); +} + +void Processor::drain() { + impl_->drain(); } \ No newline at end of file diff --git a/sim/rtlsim/processor.h b/sim/rtlsim/processor.h index 3293fe1823..c6ae7ae464 100644 --- a/sim/rtlsim/processor.h +++ b/sim/rtlsim/processor.h @@ -40,6 +40,10 @@ class Processor { int dcr_read(uint32_t addr, uint32_t tag, uint32_t* value); + // Clock the device until busy drops, i.e. until memory has taken every + // write it issued. + void drain(); + private: class Impl; diff --git a/sim/rtlsim/rtlsim_shim.sv b/sim/rtlsim/rtlsim_shim.sv index f5159027b9..bde1d6501f 100644 --- a/sim/rtlsim/rtlsim_shim.sv +++ b/sim/rtlsim/rtlsim_shim.sv @@ -73,6 +73,8 @@ module rtlsim_shim import VX_gpu_pkg::*; #( wire [VX_MEM_TAG_WIDTH-1:0] vx_mem_rsp_tag [VX_MEM_PORTS]; wire vx_mem_rsp_ready [VX_MEM_PORTS]; + wire vx_busy; + `SCOPE_IO_SWITCH (1); Vortex vortex ( @@ -103,7 +105,7 @@ module rtlsim_shim import VX_gpu_pkg::*; #( .dcr_rsp_data (dcr_rsp_data), .start (start), - .busy (busy) + .busy (vx_busy) ); wire mem_req_valid_a [VX_MEM_PORTS]; @@ -204,4 +206,28 @@ module rtlsim_shim import VX_gpu_pkg::*; #( .mem_rsp_ready_out (mem_rsp_ready) ); + // Memory applies a write once it takes it, so the device stays busy until + // every write Vortex issued has crossed the bank adapter. + wire [VX_MEM_PORTS-1:0] vx_wr_fire; + wire [MEM_NUM_BANKS-1:0] mem_wr_fire; + wire wr_empty; + for (genvar i = 0; i < VX_MEM_PORTS; ++i) begin : g_vx_wr_fire + assign vx_wr_fire[i] = mem_req_valid_a[i] && mem_req_ready_a[i] && mem_req_rw_a[i]; + end + for (genvar i = 0; i < MEM_NUM_BANKS; ++i) begin : g_mem_wr_fire + assign mem_wr_fire[i] = mem_req_valid[i] && mem_req_ready[i] && mem_req_rw[i]; + end + VX_mem_req_pending #( + .NUM_INPUTS (VX_MEM_PORTS), + .NUM_OUTPUTS (MEM_NUM_BANKS) + ) wr_pending ( + .clk (clk), + .reset (reset), + .in_fire (vx_wr_fire), + .out_fire (mem_wr_fire), + .empty (wr_empty) + ); + + assign busy = vx_busy || ~wr_empty; + endmodule diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index d8451f2618..0978067d9e 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -85,6 +85,7 @@ class Cluster::Impl { VX_CFG_L2_REPL_POLICY, // replacement policy (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash + L2_SHARED_LLC, // shared_llc }); @@ -476,9 +477,9 @@ class Cluster::Impl { return 0; } - void dcache_flush_begin() { + void dcache_flush_begin(bool l1_only) { for (auto& socket : sockets_) { - socket->dcache_flush_begin(); + socket->dcache_flush_begin(l1_only); } } @@ -540,8 +541,8 @@ class Cluster::Impl { } #endif - void l2_flush_begin() { - l2cache_->flush_begin(); + void l2_flush_begin(bool l1_only) { + l2cache_->flush_begin(l1_only); } bool l2_flush_done() const { @@ -659,8 +660,8 @@ Core* Cluster::get_core(uint32_t idx) const { return impl_->get_core(idx); } -void Cluster::dcache_flush_begin() { - impl_->dcache_flush_begin(); +void Cluster::dcache_flush_begin(bool l1_only) { + impl_->dcache_flush_begin(l1_only); } bool Cluster::dcache_flush_done() const { @@ -695,8 +696,8 @@ void Cluster::rtcache_flush_begin() { impl_->rtcache_flush_begin(); } bool Cluster::rtcache_flush_done() const { return impl_->rtcache_flush_done(); } #endif -void Cluster::l2_flush_begin() { - impl_->l2_flush_begin(); +void Cluster::l2_flush_begin(bool l1_only) { + impl_->l2_flush_begin(l1_only); } bool Cluster::l2_flush_done() const { diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index 2ff60836b4..bc7fa210a4 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -117,7 +117,7 @@ class Cluster : public SimObject { // → L2 → L3) to avoid downstream evictions racing the next-level walk. // L1 fanout: dcache + icache + socket-local {tcache, rtcache} (forwarded // to the sockets) + cluster-local {rcache, ocache}. - void dcache_flush_begin(); + void dcache_flush_begin(bool l1_only); bool dcache_flush_done() const; void icache_flush_begin(); bool icache_flush_done() const; @@ -137,7 +137,7 @@ class Cluster : public SimObject { void rtcache_flush_begin(); bool rtcache_flush_done() const; #endif - void l2_flush_begin(); + void l2_flush_begin(bool l1_only); bool l2_flush_done() const; #ifdef VX_CFG_EXT_RASTER_ENABLE diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 2ef860870a..9b3703bda0 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -39,6 +39,27 @@ namespace vortex { inline constexpr uint32_t XLENB = (VX_CFG_XLEN / 8); +// A last-level cache that alone serves every memory client is the point of +// coherence between kernels. The texture, ray-tracing and DXA ports join below +// the dcache and the raster and OM caches beside it, so a dcache with any of +// them is not one. +inline constexpr bool DCACHE_SHARED_LLC = (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0) + && (VX_CFG_NUM_CORES == 1) && (VX_CFG_NUM_CLUSTERS == 1) + && !VX_CFG_EXT_TEX_ENABLED && !VX_CFG_EXT_RTU_ENABLED + && !VX_CFG_EXT_RASTER_ENABLED && !VX_CFG_EXT_OM_ENABLED + && !VX_CFG_EXT_DXA_ENABLED; +inline constexpr bool L2_SHARED_LLC = (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0) && (VX_CFG_NUM_CLUSTERS == 1); +inline constexpr bool L3_SHARED_LLC = (VX_CFG_L3_ENABLED != 0); + +// A write-back LLC must be the only instance: replicated ones never make their +// dirty lines visible to each other. +static_assert(!((VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0) && VX_CFG_DCACHE_WRITEBACK) + || ((VX_CFG_NUM_CORES == 1) && (VX_CFG_NUM_CLUSTERS == 1)), + "a write-back dcache that is the last-level cache must be the only dcache"); +static_assert(!((VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0) && VX_CFG_L2_WRITEBACK) + || (VX_CFG_NUM_CLUSTERS == 1), + "a write-back L2 that is the last-level cache must be the only L2"); + inline constexpr uint32_t MAX_NUM_CORES = 1024; inline constexpr uint32_t MAX_NUM_WARPS = 64; inline constexpr uint32_t MAX_NUM_REGS = 32; diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 0bcd46007d..1a647e06ee 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -696,8 +696,9 @@ class CacheBank : public SimObject { // a writeback request for every dirty line via mem_req_out (write-back only; // write-through caches have nothing to evict). flush_done() reports when the // walk finishes AND the cache is otherwise idle. - void flush_begin() { - if (!config_.write_back) { + void flush_begin(bool keep_lines) { + keep_lines_ = keep_lines; + if (!config_.write_back || keep_lines) { flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -726,9 +727,12 @@ class CacheBank : public SimObject { pending_fill_reqs_ = 0; rand_ctr_ = 0; adm_seq_ctr_ = 0; - for (auto &set : sets_) { - set.reset(); + if (!keep_lines_) { + for (auto &set : sets_) { + set.reset(); + } } + keep_lines_ = false; mshr_.reset(); fwd_active_ = false; fwd_set_ = 0; @@ -1789,6 +1793,7 @@ class CacheBank : public SimObject { // Flush walk state. bool flushing_; + bool keep_lines_ = false; // the next launch reset keeps the lines uint32_t flush_set_idx_; uint32_t flush_way_idx_; uint32_t flush_sector_idx_; @@ -1887,7 +1892,8 @@ class Cache::Impl { if (config_.bypass) return; // calculate cache initialization cycles - init_cycles_ = params_.sets_per_bank; + init_cycles_ = keep_lines_ ? 0 : params_.sets_per_bank; + keep_lines_ = false; } void tick() { @@ -1954,10 +1960,13 @@ class Cache::Impl { return perf_stats; } - void flush_begin() { + void flush_begin(bool l1_only) { if (config_.bypass) return; + // The shared last-level cache stays the point of coherence across an + // L1-only flush and keeps its lines into the next launch. + keep_lines_ = l1_only && config_.shared_llc; for (auto &bank : banks_) { - bank->flush_begin(); + bank->flush_begin(keep_lines_); } } @@ -2004,6 +2013,7 @@ class Cache::Impl { std::vector nc_mem_arbs_; MemCrossBar::Ptr bank_core_xbar_; uint32_t init_cycles_; + bool keep_lines_ = false; }; /////////////////////////////////////////////////////////////////////////////// @@ -2027,8 +2037,8 @@ Cache::PerfStats Cache::perf_stats() const { return impl_->perf_stats(); } -void Cache::flush_begin() { - impl_->flush_begin(); +void Cache::flush_begin(bool l1_only) { + impl_->flush_begin(l1_only); } bool Cache::flush_done() const { diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 8729a52e88..4e7e8c5654 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -44,6 +44,7 @@ class Cache : public SimObject { uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache bool bank_hash = false; // hashed bank select + bool shared_llc = false; // the LLC serves every memory client alone }; struct PerfStats { @@ -81,8 +82,10 @@ class Cache : public SimObject { // Flush API: arm a write-back walk that emits dirty lines via mem_req_out. // flush_done() returns true once the walk completes; the caller is expected - // to tick the simulator until then. Write-through caches no-op. - void flush_begin(); + // to tick the simulator until then. Write-through caches no-op. On an + // L1-only flush the write-back last-level cache skips the walk and keeps + // its lines across the next launch reset. + void flush_begin(bool l1_only = false); bool flush_done() const; protected: diff --git a/sim/simx/mem/cache_cluster.cpp b/sim/simx/mem/cache_cluster.cpp index ff52d56a5f..ee88f524d9 100644 --- a/sim/simx/mem/cache_cluster.cpp +++ b/sim/simx/mem/cache_cluster.cpp @@ -82,9 +82,9 @@ Cache::PerfStats CacheCluster::perf_stats() const { return perf; } -void CacheCluster::flush_begin() { +void CacheCluster::flush_begin(bool l1_only) { for (auto cache : caches_) { - cache->flush_begin(); + cache->flush_begin(l1_only); } } diff --git a/sim/simx/mem/cache_cluster.h b/sim/simx/mem/cache_cluster.h index edbe5958c4..174c355fcd 100644 --- a/sim/simx/mem/cache_cluster.h +++ b/sim/simx/mem/cache_cluster.h @@ -33,7 +33,7 @@ class CacheCluster : public SimObject { Cache::PerfStats perf_stats() const; - void flush_begin(); + void flush_begin(bool l1_only = false); bool flush_done() const; private: diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 52ad76ca1a..bdcf97023d 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -104,6 +104,8 @@ ProcessorImpl::ProcessorImpl() VX_CFG_L3_LATENCY, // pipeline latency VX_CFG_L3_REPL_POLICY, // replacement policy VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC + false, // bank_hash + L3_SHARED_LLC, // shared_llc } ); @@ -204,7 +206,7 @@ void ProcessorImpl::attach_ram(RAM* ram) { #endif } -void ProcessorImpl::flush_caches() { +void ProcessorImpl::flush_caches(bool l1_only) { // Cache hierarchy is drained inside-out: issue all L1 flush_begin() calls // up-front so icache, dcache, and graphics caches flush in parallel, then // tick until all surfaces report flush_done(). @@ -212,7 +214,7 @@ void ProcessorImpl::flush_caches() { // L1 surfaces: dcache + icache + graphics caches. // Write-through surfaces early-exit in Cache::flush_begin(). for (auto& cluster : clusters_) { - cluster->dcache_flush_begin(); + cluster->dcache_flush_begin(l1_only); cluster->icache_flush_begin(); #ifdef VX_CFG_EXT_TEX_ENABLE cluster->tcache_flush_begin(); @@ -252,7 +254,7 @@ void ProcessorImpl::flush_caches() { // L2 caches. for (auto& cluster : clusters_) { - cluster->l2_flush_begin(); + cluster->l2_flush_begin(l1_only); } while (true) { bool all_done = true; @@ -265,7 +267,7 @@ void ProcessorImpl::flush_caches() { } // L3 cache (single instance at processor level). - l3cache_->flush_begin(); + l3cache_->flush_begin(l1_only); while (!l3cache_->flush_done() || !SimPlatform::instance().idle()) { SimPlatform::instance().tick(); } @@ -361,6 +363,11 @@ int ProcessorImpl::dcr_write(uint32_t addr, uint32_t value) { kmu_->dcr_write(addr, value); return 0; } + if (addr == VX_DCR_BASE_CACHE_FLUSH) { + // The flush itself runs on the per-core reads. + flush_l1_only_ = (value & VX_CACHE_FLUSH_L1_ONLY) != 0; + return 0; + } #ifdef VX_CFG_VM_ENABLE // Device SATP for the shared walker complex, assembled from two // 32-bit halves and fanned out to every cluster on the high write. @@ -393,10 +400,10 @@ int ProcessorImpl::dcr_write(uint32_t addr, uint32_t value) { int ProcessorImpl::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { if (addr == VX_DCR_BASE_CACHE_FLUSH) { - // Drain dirty cache lines to DRAM before the host reads back results. - // After flush_caches() returns every dirty line has reached memsim_'s - // backing RAM. - this->flush_caches(); + // After flush_caches() returns every dirty line of the flushed levels has + // reached memsim_'s backing RAM; an L1-only flush leaves the shared + // last-level cache in place. + this->flush_caches(flush_l1_only_); *value = 0; return 0; } diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index c9c7c7692d..3e2f697b87 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -72,7 +72,7 @@ class ProcessorImpl { // Drain dirty data from caches (write-back path) all the way to DRAM. // Walks L1 dcaches → L2 → L3, ticking the simulator between phases so // each level's evictions reach the next before that level itself flushes. - void flush_caches(); + void flush_caches(bool l1_only); private: @@ -103,6 +103,7 @@ class ProcessorImpl { // kmu_->start()). reset() clears it so a back-to-back kernel launch // via cycle() re-dispatches the KMU. bool is_cycle_initialized_; + bool flush_l1_only_ = false; // kind of the flush the last start write opened }; } diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 38ea84f880..640f263776 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -92,6 +92,7 @@ class Socket::Impl { VX_CFG_DCACHE_REPL_POLICY, // replacement policy (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0), // is_llc (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash + DCACHE_SHARED_LLC, // shared_llc }); #ifdef VX_CFG_EXT_TEX_ENABLE @@ -417,7 +418,7 @@ class Socket::Impl { return cores_.at(idx); } - void dcache_flush_begin() { dcaches_->flush_begin(); } + void dcache_flush_begin(bool l1_only) { dcaches_->flush_begin(l1_only); } bool dcache_flush_done() const { return dcaches_->flush_done(); } void icache_flush_begin() { icaches_->flush_begin(); } bool icache_flush_done() const { return icaches_->flush_done(); } @@ -505,8 +506,8 @@ Core::Ptr& Socket::core(uint32_t idx) { return impl_->core(idx); } -void Socket::dcache_flush_begin() { - impl_->dcache_flush_begin(); +void Socket::dcache_flush_begin(bool l1_only) { + impl_->dcache_flush_begin(l1_only); } bool Socket::dcache_flush_done() const { diff --git a/sim/simx/socket.h b/sim/simx/socket.h index f0d91a0605..15127a24e9 100644 --- a/sim/simx/socket.h +++ b/sim/simx/socket.h @@ -77,7 +77,7 @@ class Socket : public SimObject { // Forwarded cache flush (write-back eviction walk). The walk is a no-op // on write-through caches (`Cache::flush_begin` early-exits); forwarding // ensures a future write-back config exercises the same code path. - void dcache_flush_begin(); + void dcache_flush_begin(bool l1_only); bool dcache_flush_done() const; void icache_flush_begin(); bool icache_flush_done() const; diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index 0eee737e92..c8d25999af 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -84,6 +84,9 @@ constexpr uint32_t CP_STATUS_BUSY = 0x1; // is physical — the MMU-aware CP DMA skips translation. constexpr uint8_t CP_MEM_FLAG_PHYSICAL = 0x04; +// CMD_CACHE_FLUSH header flag: flush only the levels above the last-level cache. +constexpr uint8_t CP_FLUSH_FLAG_L1_ONLY = 0x04; + constexpr uint32_t CP_RING_SIZE_LOG2 = 16; // 64 KiB constexpr uint32_t CP_RING_SIZE = 1u << CP_RING_SIZE_LOG2; constexpr uint8_t CP_OPCODE_MEM_WRITE = 0x01; @@ -363,7 +366,7 @@ vx_result_t Device::cp_init() { } // Resolve the core count now, while nothing holds cp_mu_. Deferring it to - // the first cp_submit_cache_flush would reach query_caps from inside an + // the first cache flush would reach query_caps from inside an // open batch, which already owns cp_mu_ -- and it is not recursive. { auto rc = this->query_caps(VX_CAPS_NUM_CORES, &cp_num_cores_); @@ -513,7 +516,73 @@ const char* cp_opcode_name(uint8_t op) { } } // namespace +namespace { +// CMD_CACHE_FLUSH on-wire layout (cmd_size=12): +// bytes 0..3 header { opcode=0x0A, flags=CP_FLUSH_FLAG_L1_ONLY?, reserved=0 } +// bytes 4..11 arg0 number of cores to flush +// The CP sweeps a per-core flush DCR-read across [0, num_cores) and +// retires the command only when the last core's flush completes. +void build_cache_flush(uint8_t cl[CP_CL_BYTES], uint64_t num_cores, bool l1_only) { + std::memset(cl, 0, CP_CL_BYTES); + cl[0] = CP_OPCODE_CACHE_FLUSH; + cl[1] = l1_only ? CP_FLUSH_FLAG_L1_ONLY : 0; + std::memcpy(cl + 4, &num_cores, sizeof(num_cores)); +} + +// True when a CMD_MEM_* transfer reaches device memory outside the IO +// aperture. IO accesses bypass the caches, so they need no flush. +bool cp_dma_touches_device_mem(const uint8_t* cl) { + uint64_t dst = 0, src = 0, size = 0; + std::memcpy(&dst, cl + 4, sizeof(dst)); + std::memcpy(&src, cl + 12, sizeof(src)); + std::memcpy(&size, cl + 20, sizeof(size)); + auto outside_io = [size](uint64_t addr) { + return addr < VX_MEM_IO_BASE_ADDR || addr + size > VX_MEM_IO_END_ADDR; + }; + switch (cl[0]) { + case CP_OPCODE_MEM_WRITE: return outside_io(dst); + case CP_OPCODE_MEM_READ: return outside_io(src); + case CP_OPCODE_MEM_COPY: return outside_io(dst) || outside_io(src); + default: return false; + } +} +} // namespace + vx_result_t Device::cp_ring_append_(const void* cl) { + // Caller holds cp_mu_. A kernel's results are flushed lazily, by the + // next command that needs them: the next launch gets an L1-only flush + // (the last-level cache is the point of coherence between kernels), and + // the next transfer to or from device memory a full one, since the CP + // DMA reaches device memory beneath the last-level cache. Appending the + // flush under the same lock keeps any other command from slipping between. + const uint8_t* b = static_cast(cl); + const bool dma = cp_dma_touches_device_mem(b); + const bool launch = (b[0] == CP_OPCODE_LAUNCH) + || (b[0] == CP_OPCODE_LAUNCH_QMD) + || (b[0] == CP_OPCODE_DRAW); + if ((dma && (l1_dirty_ || llc_dirty_)) || (launch && l1_dirty_)) { + uint8_t flush[CP_CL_BYTES]; + build_cache_flush(flush, cp_num_cores_, /*l1_only=*/!dma); + auto r = cp_ring_write_(flush); + if (r != VX_SUCCESS) return r; + l1_dirty_ = false; + llc_dirty_ = llc_dirty_ && !dma; + } + assert(!(dma && (l1_dirty_ || llc_dirty_))); + auto r = cp_ring_write_(cl); + if (r != VX_SUCCESS) return r; + if (launch) { + // a draw descriptor ends on its own L1-only flush step + l1_dirty_ = (b[0] != CP_OPCODE_DRAW); + llc_dirty_ = true; + } else if (b[0] == CP_OPCODE_CACHE_FLUSH) { + l1_dirty_ = false; + llc_dirty_ = llc_dirty_ && (b[1] & CP_FLUSH_FLAG_L1_ONLY); + } + return VX_SUCCESS; +} + +vx_result_t Device::cp_ring_write_(const void* cl) { // Caller holds cp_mu_. Write one CL into the ring at the current tail — // a plain memcpy through the ring's CP-visible host pointer — then bump // tail + reserve the seqnum slot. No doorbell, no poll. @@ -576,9 +645,8 @@ vx_result_t Device::cp_batch_end() { if (r != VX_SUCCESS) { return r; } - // The batch's trailing CMD_CACHE_FLUSH(es) have retired, so every kernel's - // writes are coherent: drain the console rings once for the whole batch - // (deferred from each in-batch cp_submit_launch). + // Drain the console rings once for the whole batch (deferred from each + // in-batch cp_submit_launch). return drain_cout(); } @@ -811,15 +879,13 @@ vx_result_t Device::cp_submit_launch() { if (r != VX_SUCCESS) { return r; } - // Cache coherence: post an explicit cache flush right after the launch - // (ACQUIRE_MEM model) so the host observes coherent kernel results. - r = cp_submit_cache_flush(); - if (r != VX_SUCCESS) return r; - // In a batch the flush has only been appended, not retired — defer the - // COUT drain to cp_batch_end (one drain for the whole sequence). + // The kernel-boundary cache flush (ACQUIRE_MEM model) is posted by the + // next command that needs it (see cp_ring_append_). In a batch the + // launch has only been appended, not retired — defer the COUT drain to + // cp_batch_end (one drain for the whole sequence). if (tls_batch_owner == this) return VX_SUCCESS; - // Final COUT drain: the flush has made the kernel's writes coherent, so - // the tail-end console output left in the rings is now safe to read. + // Final COUT drain: the console rings live in the uncached IO aperture, + // and a launch completes only once its stores have left the device. return drain_cout(); } @@ -860,7 +926,7 @@ vx_result_t Device::cp_submit_launch_qmd(uint64_t qmd_addr) { // The CP reads the in-memory KMU descriptor (a {count,(addr,value)...} // list the caller staged) and replays it before pulsing start — one ring // command in place of the ~18 CMD_DCR_WRITEs a plain launch costs. Same - // trailing CMD_CACHE_FLUSH / COUT-drain discipline as cp_submit_launch. + // COUT-drain discipline as cp_submit_launch. uint8_t cl[CP_CL_BYTES] = {0}; auto r = ensure_mmu_satp(); if (r != VX_SUCCESS) { @@ -874,8 +940,6 @@ vx_result_t Device::cp_submit_launch_qmd(uint64_t qmd_addr) { if (r != VX_SUCCESS) { return r; } - r = cp_submit_cache_flush(); - if (r != VX_SUCCESS) return r; if (tls_batch_owner == this) return VX_SUCCESS; return drain_cout(); } @@ -905,19 +969,6 @@ vx_result_t Device::cp_submit_draw(uint64_t desc_addr) { return drain_cout(); } -vx_result_t Device::cp_submit_cache_flush() { - // CMD_CACHE_FLUSH on-wire layout (cmd_size=12): - // bytes 0..3 header { opcode=0x0A, flags=0, reserved=0 } - // bytes 4..11 arg0 number of cores to flush - // The CP sweeps a per-core flush DCR-read across [0, num_cores) and - // retires the command only when the last core's flush completes. - // No-op on write-through cache configs (the Vortex default). - uint8_t cl[CP_CL_BYTES] = {0}; - cl[0] = CP_OPCODE_CACHE_FLUSH; - std::memcpy(cl + 4, &cp_num_cores_, sizeof(cp_num_cores_)); - return cp_submit_cl_(cl); -} - vx_result_t Device::cp_submit_dcr_read(uint32_t addr, uint32_t tag, uint32_t* out_value) { if (!out_value) return VX_ERR_INVALID_VALUE; diff --git a/sw/runtime/common/queue.cpp b/sw/runtime/common/queue.cpp index df63f7a9dc..68543c0f60 100644 --- a/sw/runtime/common/queue.cpp +++ b/sw/runtime/common/queue.cpp @@ -505,6 +505,8 @@ namespace { constexpr uint8_t CP_OP_DCR_WRITE = 0x04; constexpr uint8_t CP_OP_CACHE_FLUSH = 0x0A; constexpr uint8_t CP_OP_LAUNCH_QMD = 0x0B; +// CMD_CACHE_FLUSH header flag: flush only the levels above the last-level cache. +constexpr uint8_t CP_FLUSH_FLAG_L1_ONLY = 0x04; // Fixed per-step stride in a draw descriptor (28-byte cmd-record prefix). // Must equal CommandProcessor::DRAW_STEP_BYTES. constexpr int CP_DRAW_STEP_BYTES = 28; @@ -682,7 +684,7 @@ vx_result_t cmd_stage_qmds(Device* device_, const std::vector& recs, // Submit one staged launch: CMD_LAUNCH_QMD when the CP decodes it, else // replay the descriptor pairs as CMD_DCR_WRITEs followed by a plain -// CMD_LAUNCH (same trailing cache-flush discipline in cp_submit_launch). +// CMD_LAUNCH. vx_result_t cmd_submit_launch(Device* device_, const CmdStaged& st) { if (device_->cp_supports_qmd()) { return device_->cp_submit_launch_qmd(st.qmd_addr); @@ -820,9 +822,10 @@ vx_result_t Queue::enqueue_draw(const vx_command_t* commands, if (r == VX_SUCCESS && use_op_draw) { std::vector desc(4, 0); // header placeholder uint32_t num_steps = 0; - auto put_step = [&](uint8_t opcode, uint64_t a0, uint64_t a1) { + auto put_step = [&](uint8_t opcode, uint8_t flags, uint64_t a0, uint64_t a1) { uint8_t step[CP_DRAW_STEP_BYTES] = {0}; step[0] = opcode; + step[1] = flags; std::memcpy(step + 4, &a0, sizeof(a0)); std::memcpy(step + 12, &a1, sizeof(a1)); desc.insert(desc.end(), step, step + CP_DRAW_STEP_BYTES); @@ -831,10 +834,10 @@ vx_result_t Queue::enqueue_draw(const vx_command_t* commands, for (size_t i = 0; i < recs.size(); ++i) { const CmdRec& rec = recs[i]; if (rec.is_launch) { - put_step(CP_OP_LAUNCH_QMD, staged[i].qmd_addr, 0); - put_step(CP_OP_CACHE_FLUSH, num_cores, 0); + put_step(CP_OP_LAUNCH_QMD, 0, staged[i].qmd_addr, 0); + put_step(CP_OP_CACHE_FLUSH, CP_FLUSH_FLAG_L1_ONLY, num_cores, 0); } else { - put_step(CP_OP_DCR_WRITE, rec.dcr_addr, rec.dcr_value); + put_step(CP_OP_DCR_WRITE, 0, rec.dcr_addr, rec.dcr_value); } } std::memcpy(desc.data(), &num_steps, sizeof(num_steps)); diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index ef803eefc7..23c688ad63 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -233,15 +233,15 @@ class Device : public RefCounted { vx_result_t cp_submit_dcr_write(uint32_t addr, uint32_t value); // Post one CMD_LAUNCH to the ring, commit Q_TAIL, and wait for - // Q_SEQNUM. Synchronous. Followed by an implicit CMD_CACHE_FLUSH so - // the host observes coherent results (see cp_submit_cache_flush). + // Q_SEQNUM. Synchronous. The kernel's results are flushed by the next + // command that needs them (see cp_ring_append_). vx_result_t cp_submit_launch(); // Post one CMD_LAUNCH_QMD: the CP reads the KMU descriptor (a // {count,(dcr_addr,value)...} list the caller staged to device memory at // `qmd_addr`) and replays it, then pulses start — one ring command in - // place of a launch's ~18 CMD_DCR_WRITEs (NVIDIA QMD model). Followed by an - // implicit CMD_CACHE_FLUSH, like cp_submit_launch. Batch-aware. + // place of a launch's ~18 CMD_DCR_WRITEs (NVIDIA QMD model). Flushed like + // cp_submit_launch. Batch-aware. vx_result_t cp_submit_launch_qmd(uint64_t qmd_addr); // Post one CMD_DRAW (OP_DRAW): arg0 = device address of a resident draw @@ -255,12 +255,6 @@ class Device : public RefCounted { bool cp_supports_draw() const { return cp_supports_draw_; } bool cp_supports_qmd() const { return cp_supports_qmd_; } - // Post one CMD_CACHE_FLUSH to the ring (AMD ACQUIRE_MEM model): the CP - // sweeps a per-core cache flush across all cores and retires the - // command only when the last core's flush completes. A no-op on - // write-through cache configs. Posted after every CMD_LAUNCH. - vx_result_t cp_submit_cache_flush(); - // Program the device MMU's SATP once, ahead of the first launch that // could translate. Called from every launch path. vx_result_t ensure_mmu_satp(); @@ -273,7 +267,7 @@ class Device : public RefCounted { // ----- Batched CP submission (one draw = one ring batch) ----- // cp_batch_begin holds the ring lock and switches cp_submit_* into // append-only mode: each subsequent cp_submit_dcr_write / cp_submit_launch - // / cp_submit_cache_flush writes its CL into the ring and reserves a + // writes its CL into the ring and reserves a // seqnum but does NOT ring the doorbell or poll. cp_batch_end commits // Q_TAIL once, polls Q_SEQNUM once for the last appended command, drains // COUT once, and releases the lock. The CP then retires the whole @@ -391,7 +385,12 @@ class Device : public RefCounted { // Append one CL at the current tail and reserve its seqnum (bump // cp_tail_ + cp_expected_seqnum_). Caller must hold cp_mu_ (directly in // the non-batch path, or via cp_batch_begin). No doorbell, no poll. + // Inserts the lazy kernel-boundary CMD_CACHE_FLUSH (AMD ACQUIRE_MEM + // model) under the same lock: L1-only ahead of a launch while + // l1_dirty_, full ahead of a CMD_MEM_* touching device memory while + // either flag is set. vx_result_t cp_ring_append_(const void* cl); + vx_result_t cp_ring_write_(const void* cl); // Build + submit a CMD_MEM_* descriptor (opcode, dst, src, size). // `physical` sets the CMD_MEM header flag so the CP DMA skips VM @@ -457,6 +456,13 @@ class Device : public RefCounted { uint64_t cp_tail_ = 0; uint64_t cp_expected_seqnum_ = 0; uint64_t cp_num_cores_ = 0; // cached VX_CAPS_NUM_CORES, used for CMD_CACHE_FLUSH + // A kernel ran since the last flush, so the L1s may hold its writes or + // stale lines. + bool l1_dirty_ = false; + // A kernel ran since the last full cache flush, so the last-level cache + // may hold lines the CP DMA (which reaches device memory beneath it) + // would miss or later overwrite. Both flags are guarded by cp_mu_. + bool llc_dirty_ = false; std::mutex cp_mu_; // serialize ring writes // Seqnum the last command appended to the open batch will reach; // cp_batch_end polls for it once. Touched only by the batch owner, which diff --git a/sw/runtime/firesim/vortex.cpp b/sw/runtime/firesim/vortex.cpp index 973c79bc49..6c31af7458 100644 --- a/sw/runtime/firesim/vortex.cpp +++ b/sw/runtime/firesim/vortex.cpp @@ -158,6 +158,10 @@ class vx_device { return false; return busy; }; + h.vortex_drain = [this]() { + bool busy = true; + while (busy && sim_.is_busy(&busy) == 0) {} + }; #ifdef VX_CFG_VM_ENABLE // The device MMU control surface answers the fault-report DCRs. h.mmu_fault_report = true; diff --git a/sw/runtime/rtlsim/vortex.cpp b/sw/runtime/rtlsim/vortex.cpp index e5aa68d64f..8d7f24e96f 100644 --- a/sw/runtime/rtlsim/vortex.cpp +++ b/sw/runtime/rtlsim/vortex.cpp @@ -175,6 +175,10 @@ class vx_device { if (!future_.valid()) return false; return future_.wait_for(std::chrono::seconds(0)) != std::future_status::ready; }; + h.vortex_drain = [this]() { + if (future_.valid()) future_.wait(); + processor_.drain(); + }; #ifdef VX_CFG_VM_ENABLE // The device MMU control surface answers the fault-report DCRs. h.mmu_fault_report = true; diff --git a/tests/regression/flush_coherence/common.h b/tests/regression/flush_coherence/common.h index bbdabc90a1..6b04d30d2d 100644 --- a/tests/regression/flush_coherence/common.h +++ b/tests/regression/flush_coherence/common.h @@ -8,6 +8,7 @@ typedef struct { uint32_t round; uint32_t rotate; uint32_t burst; + uint32_t dxa_slot; // 1 + DXA descriptor slot of src, 0 to load with the LSU uint64_t src_addr; uint64_t dst_addr; uint64_t acc_addr; diff --git a/tests/regression/flush_coherence/kernel.cpp b/tests/regression/flush_coherence/kernel.cpp index de656daa29..1c3e14ed9f 100644 --- a/tests/regression/flush_coherence/kernel.cpp +++ b/tests/regression/flush_coherence/kernel.cpp @@ -1,4 +1,8 @@ #include +#ifdef VX_CFG_EXT_DXA_ENABLE +#include +#include +#endif #include "common.h" __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { @@ -15,7 +19,25 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto src = reinterpret_cast(arg->src_addr); auto dst = reinterpret_cast(arg->dst_addr); auto acc = reinterpret_cast(arg->acc_addr); - uint32_t v = src[i]; + uint32_t v; +#ifdef VX_CFG_EXT_DXA_ENABLE + if (arg->dxa_slot != 0) { + // The tile copy reads memory beside the dcache, not through it, so it + // sees the previous launch's stores only once a flush has written them + // back. The host sets no rotation, so this block's tile holds element i. + auto shmem = reinterpret_cast(__local_mem()); + vortex::barrier bar(0); + if (get_sub_group_id() == 0) { + bar.expect_tx(1); + vx_dxa_issue_1d_wg(arg->dxa_slot - 1, bar.id(), shmem, blockIdx.x * blockDim.x); + } + bar.arrive_and_wait(); + v = shmem[threadIdx.x]; + } else +#endif + { + v = src[i]; + } acc[i] += v ^ arg->round; // A store burst saturates the shared levels, so the final stores are // still queued in front of them when the flushes start. diff --git a/tests/regression/flush_coherence/main.cpp b/tests/regression/flush_coherence/main.cpp index 573dbe66ad..9f45ff84ba 100644 --- a/tests/regression/flush_coherence/main.cpp +++ b/tests/regression/flush_coherence/main.cpp @@ -14,8 +14,18 @@ // With -b, each thread ends on a burst of stores to a tail buffer, so the // last stores may still be queued in front of the shared levels when the // flushes start, and every round is checked. +// With -x, each launch loads src with a DXA tile copy, which reaches memory +// beside the dcache, instead of with the LSU. +// With -c, the rounds form one command batch with no host transfer between +// kernels: each round reads the buffer the previous round wrote (ping-pong), +// and the host only reads back after the last round. Kernel-boundary flushes +// alone must then make each round's writes visible to the next, including to +// a core whose cache still holds the lines from two rounds earlier. #include +#ifdef VX_CFG_EXT_DXA_ENABLE +#include +#endif #include "common.h" #include @@ -38,17 +48,21 @@ const char* kernel_file = "kernel.vxbin"; uint32_t count = 2048; uint32_t rounds = 16; uint32_t burst = 0; +bool chain = false; +bool use_dxa = false; void parse_args(int argc, char** argv) { int c; - while ((c = getopt(argc, argv, "n:r:b:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:r:b:k:cxh")) != -1) { switch (c) { case 'n': count = (uint32_t)std::atoi(optarg); break; case 'r': rounds = (uint32_t)std::atoi(optarg); break; case 'b': burst = (uint32_t)std::atoi(optarg); break; case 'k': kernel_file = optarg; break; + case 'c': chain = true; break; + case 'x': use_dxa = true; break; default: - std::printf("Usage: [-k kernel] [-n words] [-r rounds] [-b burst] [-h]\n"); + std::printf("Usage: [-k kernel] [-n words] [-r rounds] [-b burst] [-c] [-x] [-h]\n"); std::exit(c == 'h' ? 0 : -1); } } @@ -60,6 +74,15 @@ uint32_t src_value(uint32_t i, uint32_t round) { return x * 0x2C1B3C6Du; } +#ifdef VX_CFG_EXT_DXA_ENABLE +void program_dxa(vx_device_h dev, uint32_t slot, uint64_t addr, uint32_t tile) { + if (vortex::dxa::program_1d(dev, slot, addr, count, tile, sizeof(uint32_t)) != 0) { + std::fprintf(stderr, "FAIL: DXA descriptor %u\n", slot); + std::exit(1); + } +} +#endif + uint32_t rotation(uint32_t round) { return (round % 4 == 0) ? 0 : (round * 0x9E3779B1u) % count; } @@ -84,7 +107,7 @@ int main(int argc, char** argv) { parse_args(argc, argv); const uint64_t bytes = (uint64_t)count * sizeof(uint32_t); const uint64_t tail_bytes = bytes * burst; - std::printf("flush_coherence: n=%u rounds=%u burst=%u\n", count, rounds, burst); + std::printf("flush_coherence: n=%u rounds=%u burst=%u chain=%d\n", count, rounds, burst, chain); vx_device_h dev = nullptr; CHECK(vx_device_open(0, &dev)); @@ -119,6 +142,23 @@ int main(int argc, char** argv) { uint32_t grid[1], block[1]; CHECK(vx_device_max_occupancy_grid(dev, 1, &count, grid, block)); + // DXA descriptor slot 0 covers src, 1 dst and 2 the chain's middle buffer. + uint32_t lmem_size = 0; + if (use_dxa) { +#ifdef VX_CFG_EXT_DXA_ENABLE + if (count % block[0] != 0) { + std::printf("Error: -x needs a word count that is a multiple of the block size (%u)\n", block[0]); + return -1; + } + lmem_size = block[0] * sizeof(uint32_t); + program_dxa(dev, 0, arg.src_addr, block[0]); + program_dxa(dev, 1, arg.dst_addr, block[0]); +#else + std::printf("Error: -x needs a build with VX_CFG_EXT_DXA_ENABLE\n"); + return -1; +#endif + } + // Enqueued writes and launches read their host data asynchronously, so // each round gets its own source buffer and argument block. std::vector> h_src(rounds, std::vector(count)); @@ -129,7 +169,69 @@ int main(int argc, char** argv) { CHECK(vx_enqueue_write(q, acc_buf, 0, h_acc.data(), bytes, 0, nullptr, nullptr)); int errors = 0; - for (uint32_t r = 0; r < rounds; ++r) { + vx_buffer_h mid_buf = nullptr; + if (chain) { + // round r reads the buffer round r-1 wrote: src, then dst and mid in turn + CHECK(vx_buffer_create(dev, bytes, VX_MEM_READ_WRITE, &mid_buf)); + uint64_t pp_addr[2]; + CHECK(vx_buffer_address(dst_buf, &pp_addr[0])); + CHECK(vx_buffer_address(mid_buf, &pp_addr[1])); +#ifdef VX_CFG_EXT_DXA_ENABLE + if (use_dxa) { + program_dxa(dev, 2, pp_addr[1], block[0]); + } +#endif + auto& src = h_src[0]; + for (uint32_t i = 0; i < count; ++i) { + src[i] = src_value(i, 0); + ref_dst[i] = src[i]; + } + CHECK(vx_enqueue_write(q, src_buf, 0, src.data(), bytes, 0, nullptr, nullptr)); + + std::vector lis(rounds); + std::vector cmds(rounds); + for (uint32_t r = 0; r < rounds; ++r) { + for (uint32_t i = 0; i < count; ++i) { + ref_acc[i] += ref_dst[i] ^ r; + ref_dst[i] = ref_dst[i] * 3u + r; + } + args[r].round = r; + args[r].rotate = use_dxa ? 0 : rotation(r); + args[r].src_addr = (r == 0) ? arg.src_addr : pp_addr[(r - 1) % 2]; + args[r].dxa_slot = !use_dxa ? 0 : ((r == 0) ? 1 : 2 + (r - 1) % 2); + args[r].dst_addr = pp_addr[r % 2]; + vx_launch_info_t& li = lis[r]; + li = vx_launch_info_t{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &args[r]; + li.args_size = sizeof(args[r]); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0] = block[0]; + li.lmem_size = lmem_size; + cmds[r].type = VX_COMMAND_LAUNCH; + cmds[r].data.launch = &li; + } + for (uint32_t i = 0; i < count; ++i) { + for (uint32_t k = 0; k < burst; ++k) { + ref_tail[k * count + i] = tail_value(i, k, rounds - 1); + } + } + CHECK(vx_enqueue_commands(q, cmds.data(), rounds, 0, nullptr, nullptr)); + + vx_buffer_h last_buf = ((rounds - 1) % 2 == 0) ? dst_buf : mid_buf; + CHECK(vx_enqueue_read(q, h_dst.data(), last_buf, 0, bytes, 0, nullptr, nullptr)); + if (burst) { + CHECK(vx_enqueue_read(q, h_tail.data(), tail_buf, 0, tail_bytes, 0, nullptr, nullptr)); + } + CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); + errors += check("dst", rounds - 1, h_dst, ref_dst); + if (burst) { + errors += check("tail", rounds - 1, h_tail, ref_tail); + } + } + for (uint32_t r = 0; !chain && r < rounds; ++r) { auto& src = h_src[r]; for (uint32_t i = 0; i < count; ++i) { src[i] = src_value(i, r); @@ -142,7 +244,8 @@ int main(int argc, char** argv) { CHECK(vx_enqueue_write(q, src_buf, 0, src.data(), bytes, 0, nullptr, nullptr)); args[r].round = r; - args[r].rotate = rotation(r); + args[r].rotate = use_dxa ? 0 : rotation(r); + args[r].dxa_slot = use_dxa ? 1 : 0; vx_launch_info_t li{}; li.struct_size = sizeof(li); li.kernel = kern; @@ -151,6 +254,7 @@ int main(int argc, char** argv) { li.ndim = 1; li.grid_dim[0] = grid[0]; li.block_dim[0] = block[0]; + li.lmem_size = lmem_size; CHECK(vx_enqueue_launch(q, &li, 0, nullptr, nullptr)); if (burst || r % 2 == 0 || r + 1 == rounds) { @@ -170,6 +274,9 @@ int main(int argc, char** argv) { CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); errors += check("acc", rounds, h_acc, ref_acc); + if (mid_buf) { + vx_buffer_release(mid_buf); + } if (tail_buf) { vx_buffer_release(tail_buf); } @@ -179,6 +286,7 @@ int main(int argc, char** argv) { vx_kernel_release(kern); vx_module_release(mod); vx_queue_release(q); + vx_device_dump_perf(dev, stdout); vx_device_release(dev); if (errors) { From 4f173e313f78c88f2d6ea79cc07451cfe8983df5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 03:25:36 -0700 Subject: [PATCH 035/336] Let a write-back LLC allocate on a store miss without fetching the line A store miss in a write-back last-level cache used to fetch the whole line from memory only to overwrite part of it, so streaming writes cost a full read per line. With VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH the cache instead installs the sector with its dirty-byte mask and a partial bit and issues no fill. A load or AMO that reaches a partial sector misses; its fill merges the memory data under the bytes the stores have not written. Evictions and flushes write back only the dirty bytes. Fill responses with no data use a loopback queue, and an MSHR chain that meets a partial sector parks, refetches and retries. This is the write-allocate-without-fetch policy GPU L2s use for partial writes. The dirty-byte mask is now one way-folded single-port distributed RAM that is read and written only at the commit stage, as a whole-line read-modify-write. It replaces the per-way dual-port per-byte RAMs, is cleared when a line is installed, and removes the old dirty-byte path from the critical cone. The MSHR exposes a registered dequeue_reads so the park decision does not wait on the payload read. The feature is off by default; enabling it turns on the dirty-byte mask. With it off the cache is logically identical to before and cycles are unchanged. SimX models the same mechanism behind the same knobs. 1-core vecadd with lazy fetch: DRAM reads -33%, memory requests -25%, cycles unchanged (latency-bound). Cache DUT at 300 MHz target (U55C, median of a placer-directive sweep): feature on 288.3 MHz vs 276.9 MHz for the previous dirty-byte implementation, with 1.4k fewer LUTs. Tests: lazy_fetch app, cache:lazy_fetch-1/2, parity-lazy-fetch, parity-vecadd-lazy-l2. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 13 +- ci/testcases/cache.yaml | 21 ++ ci/testcases/core.yaml | 16 ++ hw/rtl/VX_cluster.sv | 1 + hw/rtl/VX_socket.sv | 1 + hw/rtl/Vortex.sv | 1 + hw/rtl/cache/VX_cache.sv | 4 + hw/rtl/cache/VX_cache_bank.sv | 343 +++++++++++++++++++++---- hw/rtl/cache/VX_cache_cluster.sv | 3 + hw/rtl/cache/VX_cache_data.sv | 144 ++++++----- hw/rtl/cache/VX_cache_mshr.sv | 56 +++- hw/rtl/cache/VX_cache_tags.sv | 69 ++++- hw/rtl/cache/VX_cache_wrap.sv | 3 + hw/unittest/cache/VX_cache_top.sv | 4 + hw/unittest/gfx/VX_gfx_top.sv | 1 + hw/unittest/tensor/VX_tensor_top.sv | 2 + hw/unittest/vm/VX_vm_top.sv | 2 + sim/simx/cluster.cpp | 1 + sim/simx/mem/cache.cpp | 266 +++++++++++++++++-- sim/simx/mem/cache.h | 1 + sim/simx/processor.cpp | 1 + sim/simx/socket.cpp | 1 + tests/regression/Makefile | 2 +- tests/regression/lazy_fetch/Makefile | 16 ++ tests/regression/lazy_fetch/common.h | 134 ++++++++++ tests/regression/lazy_fetch/kernel.cpp | 174 +++++++++++++ tests/regression/lazy_fetch/main.cpp | 217 ++++++++++++++++ 27 files changed, 1355 insertions(+), 142 deletions(-) create mode 100644 tests/regression/lazy_fetch/Makefile create mode 100644 tests/regression/lazy_fetch/common.h create mode 100644 tests/regression/lazy_fetch/kernel.cpp create mode 100644 tests/regression/lazy_fetch/main.cpp diff --git a/VX_config.toml b/VX_config.toml index b3b89c6833..9ccfef0cc5 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -216,7 +216,12 @@ VX_CFG_DCACHE_NUM_WAYS = 4 # stays moderate: NT 1/2/4/8/16/32/64 -> word 4/8/8/16/16/32/32, banks 1/1/2/2/4/4/8. VX_CFG_DCACHE_WORD_SIZE = "expr: min($VX_CFG_L1_LINE_SIZE, min($VX_CFG_MEM_BLOCK_SIZE, ($VX_CFG_XLEN / 8) * pow(2, (clog2($VX_CFG_NUM_LSU_LANES) + 1) / 2)))" VX_CFG_DCACHE_WRITEBACK = "expr: int($dcache_is_llc and $single_core)" -VX_CFG_DCACHE_DIRTYBYTES = 0 +# LAZY_FETCH (write-back LLC only): a store miss allocates its line without a +# fill and a later read fetches and merges under the dirty bytes. It needs the +# per-byte dirty masks, so it turns DIRTYBYTES on; DIRTYBYTES alone keeps +# fetch-on-write. The same pair exists for L2 and L3. +VX_CFG_DCACHE_DIRTYBYTES = "expr: $VX_CFG_DCACHE_LAZY_FETCH" +VX_CFG_DCACHE_LAZY_FETCH = 0 VX_CFG_DCACHE_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_DCACHE_MSHR_SIZE = 16 # dcache is unsectored: line = sector = mem transaction granule = L1_LINE (which @@ -246,7 +251,8 @@ VX_CFG_L1_MEM_PORTS = "expr: min($VX_CFG_DCACHE_NUM_BANKS, $VX_CFG_PLATFORM_MEMO VX_CFG_L2_SIZE = 1048576 VX_CFG_L2_NUM_WAYS = 8 VX_CFG_L2_WRITEBACK = "expr: int($l2_is_llc and $single_cluster)" -VX_CFG_L2_DIRTYBYTES = 0 +VX_CFG_L2_DIRTYBYTES = "expr: $VX_CFG_L2_LAZY_FETCH" +VX_CFG_L2_LAZY_FETCH = 0 VX_CFG_L2_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L2_MSHR_SIZE = 16 VX_CFG_L2_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L2_SIZE) - clog2(1048576))" @@ -264,7 +270,8 @@ VX_CFG_L2_MEM_PORTS = "expr: min($VX_CFG_L2_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_N VX_CFG_L3_SIZE = 2097152 VX_CFG_L3_NUM_WAYS = 8 VX_CFG_L3_WRITEBACK = "expr: int($l3_is_llc)" -VX_CFG_L3_DIRTYBYTES = 0 +VX_CFG_L3_DIRTYBYTES = "expr: $VX_CFG_L3_LAZY_FETCH" +VX_CFG_L3_LAZY_FETCH = 0 VX_CFG_L3_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L3_MSHR_SIZE = 16 VX_CFG_L3_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L3_SIZE) - clog2(2097152))" diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index d78b7f904d..ceb79a2788 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -316,3 +316,24 @@ tests: configs: -DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_DIRTYBYTES=1 shape: threads: 16 +# Lazy fetch-on-read on a write-back LLC: a store miss allocates its line +# without a fill, and a load or AMO of a partially written line fetches it and +# merges under the dirty bytes. The default 1-core dcache is such an LLC. +- id: lazy_fetch-1 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 +# The same on a write-back L2 shared by 4 cores behind write-through L1s. +- id: lazy_fetch-2 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_LAZY_FETCH=1 + shape: + cores: 4 + l2cache: true diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index b09ebb949e..4f3f4c657e 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -131,6 +131,22 @@ tests: via: blackbox app: vecadd args: -n16384 +# Streaming stores into a lazy-fetch write-back L2: store misses allocate +# through the bank loopback instead of a memory fill. +- id: parity-vecadd-lazy-l2 + check: model_parity + via: blackbox + app: vecadd + args: -n16384 + configs: -DVX_CFG_L2_LAZY_FETCH=1 + shape: {l2cache: true} +# Lazy-fetch refetch paths on the default write-back dcache: partial-sector +# loads park and refetch, and same-set thrash evicts lines mid-refetch. +- id: parity-lazy-fetch + check: model_parity + via: blackbox + app: lazy_fetch + configs: -DVX_CFG_DCACHE_LAZY_FETCH=1 # FPU + LSU locality, multi-warp reuse (~1.29M cycles on the default shape). - id: parity-sgemm check: model_parity diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 9b817df196..be46c3f4e4 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -227,6 +227,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 3b04931533..d65f86df73 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -220,6 +220,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_DCACHE_WRITEBACK), .DIRTY_BYTES (`VX_CFG_DCACHE_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_DCACHE_LAZY_FETCH), .REPL_POLICY (`VX_CFG_DCACHE_REPL_POLICY), .NC_ENABLE (1), .CORE_OUT_BUF (3), diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 98b5fb6c8e..a605dedd04 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -152,6 +152,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L3_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L3_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L3_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L3_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index a682b600b1..3ee2bae7aa 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -40,6 +40,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -67,6 +69,7 @@ module VX_cache import VX_gpu_pkg::*; #( `STATIC_ASSERT(NUM_BANKS == (1 << `CLOG2(NUM_BANKS)), ("invalid parameter: number of banks must be power of 2")) `STATIC_ASSERT(WRITE_ENABLE || !WRITEBACK, ("invalid parameter: writeback requires write enable")) `STATIC_ASSERT(WRITEBACK || !DIRTY_BYTES, ("invalid parameter: dirty bytes require writeback")) + `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) `STATIC_ASSERT(NUM_BANKS >= MEM_PORTS, ("invalid parameter: number of banks must be greater or equal to number of memory ports")) `STATIC_ASSERT(!BANK_HASH || NUM_BANKS <= 256, ("invalid parameter: bank hash supports up to 256 banks")) localparam REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS); @@ -365,6 +368,7 @@ module VX_cache import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 4b95432384..90d8f1ddcb 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -31,6 +31,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter WRITE_ENABLE = 1, parameter WRITEBACK = 0, parameter DIRTY_BYTES = 0, + parameter LAZY_FETCH = 0, // write miss allocates without a fill; a load to a partial sector refetches parameter REPL_POLICY = `CS_REPL_FIFO, parameter TAG_WIDTH = UUID_WIDTH + 1, parameter CORE_OUT_BUF = 0, @@ -102,6 +103,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // core-rsp queue depth (pow2): registered-skid minimum (2) plus CRSQ_SIZE extra slots localparam CRSQ_QUEUE_SIZE = 1 << `CLOG2(2 + CRSQ_SIZE); `STATIC_ASSERT(LATENCY >= 2, ("invalid parameter: cache bank LATENCY must be >= 2")) + `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) `UNUSED_PARAM (MRSQ_SIZE) // ======================================================================== @@ -144,7 +146,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( req_t req; logic [`CS_WORD_WIDTH-1:0] wdata; logic [NUM_WAYS-1:0] tag_matches; - logic is_refill; // fill into an already-resident line } data_t; typedef struct packed { // response + memory request (S0 -> S1 -> stC) @@ -152,6 +153,15 @@ module VX_cache_bank import VX_gpu_pkg::*; #( lookup_t lk; } commit_t; + // Lazy-fetch request flavors ride a side pipeline beside the request; it + // exists only when the feature is enabled. + typedef struct packed { + logic is_lazy; // data-less fill of a write-miss root + logic is_refetch; // fill whose root refetches a partial sector + logic is_retry; // refetch re-issued after its sector was evicted + logic is_park; // replay parked on a partial sector: issues its refetch + } lazy_req_t; + data_t sel_req, st0, dat_in, stD; // request + fill line: sel -> S0 -> stD commit_t cmt_in, st1, stC; // request + lookup delta: S0 -> S1 -> stC lookup_t lk_st0; // S0 lookup results @@ -174,6 +184,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [`CS_LINE_ADDR_WIDTH-1:0] mem_rsp_addr; wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mem_rsp_sector; // sector this fill installs wire [MSHR_ADDR_WIDTH-1:0] mshr_alloc_id, mshr_previd; + wire mshr_fill_refetch, mshr_fill_retry, mshr_dequeue_reads; + wire lazyq_empty, lazyq_alm_full; wire mshr_pending_raw; wire mshr_pending_wr_raw; wire mshr_pending_ovl_raw; @@ -244,7 +256,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire do_read_stc = stC.req.valid && stC.req.is_creq && ~stC.req.rw; wire do_write_stc = stC.req.valid && stC.req.is_creq && stC.req.rw; - wire do_init_std = stD.req.valid && stD.req.is_init; wire do_fill_std = stD.req.valid && stD.req.is_fill; wire do_flush_std = stD.req.valid && stD.req.is_flush; wire do_read_std = stD.req.valid && stD.req.is_creq && ~stD.req.rw; @@ -285,7 +296,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign pipe_inflight = st0.req.valid || st1.req.valid || (| commit_valid); end // Queued AMO results are still writes to land in the array. - wire no_pending_req = ~pipe_inflight && mreq_queue_empty && ~amo_commit_busy; + wire no_pending_req = ~pipe_inflight && mreq_queue_empty && lazyq_empty && ~amo_commit_busy; VX_cache_flush #( .CACHE_SIZE (CACHE_SIZE), @@ -352,11 +363,39 @@ module VX_cache_bank import VX_gpu_pkg::*; #( && (replay_rw || st0.req.rw) && (replay_wsel == st0.req.word_idx); + // Lazy fetch: a write-miss root's fill loops back (lazyq) as a data-less + // fill below memory fills. A load/AMO head of a partial chain parks and + // refetches. A refetch merges into its resident sector one stage late, at + // stC; one whose sector was evicted installs nothing and retries as a + // plain fill (nothing can dirty an absent sector, so its data is current). + wire lazyq_valid = ~lazyq_empty; + wire [MSHR_ADDR_WIDTH-1:0] lazyq_id; + wire lazy_sel = ~mem_rsp_valid && lazyq_valid; + wire chain_partial, chain_retry; + // Lazy-fetch request flavors, folded to constants when the feature is off. + lazy_req_t lz_sel, lz_st0, lz_st1, lz_stc; + wire lazy_st0 = lz_st0.is_lazy; + wire refetch_st0 = lz_st0.is_refetch; + wire retry_st0 = lz_st0.is_retry; + wire park_st0 = lz_st0.is_park; + wire park_stc = lz_stc.is_park; + wire refetch_stc = lz_stc.is_refetch; + wire retry_stc = lz_stc.is_retry; + `UNUSED_VAR ({lz_st1, lz_stc.is_lazy}) + wire park_sel = (LAZY_FETCH != 0) && chain_partial && mshr_dequeue_reads; + wire refetch_hold = st0.req.valid && st0.req.is_fill && refetch_st0; + // The chain's head waits out a refetch at S0, and a head that would park + // waits for memory-request room; nothing else is selected meanwhile. Built + // from registers only and applied at the MSHR output, off the dequeue cone. + wire replay_block = refetch_hold || (park_sel && mreq_queue_alm_full); + // A fill starts a replay chain, and a chain carries at most one AMO (no // request enters behind a pending AMO), so a fill waits until the AMO // writeback queue has room for it; the queue drains meanwhile. wire replay_mux = replay_valid && ~fwd_head && ~replay_link_hold; - wire fill_mux = mem_rsp_valid && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full; + wire refetch_inflight; + wire fill_mux = (mem_rsp_valid || lazyq_valid) && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full + && ~refetch_inflight && ~replay_block; wire replay_grant = ~init_valid; wire replay_enable = replay_grant && replay_mux; @@ -374,22 +413,29 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire creq_enable = creq_grant && (amo_creq_path || amo_wb_path); assign replay_ready = replay_grant && ~fwd_head && ~replay_link_hold && ~(!WRITEBACK && replay_rw && mreq_queue_alm_full) && ~pipe_stall; - assign mem_rsp_ready = fill_grant && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; + assign mem_rsp_ready = fill_grant && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full && ~refetch_inflight + && ~replay_block && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; - assign core_req_ready = creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall + // a core request may become a write-miss root, which pushes to lazyq. + wire creq_alm_full = mreq_queue_alm_full || lazyq_alm_full || replay_block; + assign core_req_ready = creq_grant && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; wire init_fire = init_valid; wire replay_fire = replay_valid && replay_ready; wire mem_rsp_fire = mem_rsp_valid && mem_rsp_ready; + wire lazy_fire = lazy_sel && mem_rsp_ready; + wire fill_fire = mem_rsp_fire || lazy_fire; + wire park_fire = replay_fire && park_sel; wire flush_fire = flush_valid && flush_ready; // amo_wb_path already excludes the cycle a fresh AMO commits at S1, so the // writeback never races the chain update. - wire amo_wb_fire = amo_wb_path && creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall; + wire amo_wb_fire = amo_wb_path && creq_grant && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall; wire core_req_fire = (amo_creq_path || amo_wb_path) && creq_grant - && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall; + && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall; wire [MSHR_ADDR_WIDTH-1:0] mem_rsp_id = mem_rsp_tag[MSHR_ADDR_WIDTH-1:0]; + wire [MSHR_ADDR_WIDTH-1:0] fill_id = lazy_sel ? lazyq_id : mem_rsp_id; // Fill word_idx: place the installed sector in the high (sector) bits of the // in-line word offset; low (word-in-sector) bits are don't-care for a fill. @@ -441,7 +487,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // it never re-commits at S1. always @(*) begin sel_req = '0; - sel_req.req.valid = init_fire || replay_fire || mem_rsp_fire || flush_fire || core_req_fire; + sel_req.req.valid = init_fire || replay_fire || fill_fire || flush_fire || core_req_fire; sel_req.req.is_init = init_valid; sel_req.req.is_fill = fill_enable; sel_req.req.is_flush = flush_enable; @@ -462,7 +508,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( : (amo_wb_pending ? amo_wb_word_idx : core_req_wsel)); sel_req.req.req_idx = replay_mux ? replay_idx : (amo_wb_pending ? amo_wb_idx : core_req_idx); sel_req.req.tag = (init_valid | flush_valid) ? (flush_valid ? flush_tag : '0) - : (replay_mux ? replay_tag : (fill_mux ? mem_rsp_tag_s + : (replay_mux ? replay_tag : (fill_mux ? (lazy_sel ? '0 : mem_rsp_tag_s) : (amo_wb_pending ? amo_wb_tag : core_req_tag))); sel_req.req.mshr_id = replay_id; sel_req.req.amo = replay_mux ? replay_amo : (amo_wb_pending ? amo_req_t'('0) @@ -471,6 +517,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // tag_matches is computed at S0; left 0 here (overridden at the data bubble). end + assign lz_sel.is_lazy = fill_enable && lazy_sel; + assign lz_sel.is_refetch = fill_enable && ~lazy_sel && mshr_fill_refetch; + assign lz_sel.is_retry = fill_enable && ~lazy_sel && mshr_fill_retry; + assign lz_sel.is_park = replay_enable && park_sel; + // UUID extraction (debug + MSHR ordering): per stage, from the carried tag. wire [`UP(UUID_WIDTH)-1:0] req_uuid_sel, req_uuid_st0, req_uuid_st1, req_uuid_stc; if (UUID_WIDTH != 0) begin : g_req_uuid @@ -521,6 +572,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); wire [`CS_WAY_SEL_WIDTH-1:0] fill_way_st0 = line_present_any_st0 ? present_way_st0 : victim_way; wire [`CS_WAY_SEL_WIDTH-1:0] evict_way_st0 = st0.req.is_fill ? fill_way_st0 : st0.req.way_idx; + // A refetch (first attempt) uses no victim: it merges into the way that + // holds its sector, carried apart from evict_way so that the tag compare + // stays off the array write enables. + wire refetch_first_st0 = do_fill_st0 && refetch_st0 && ~retry_st0; + wire [`CS_WAY_SEL_WIDTH-1:0] merge_way_st0 = refetch_first_st0 ? hit_idx_st0 : evict_way_st0; // Fill way as one-hot, straight from the per-way presence vector: routing // it through the binary encode -> evict_way -> per-way compare round trip @@ -546,12 +602,27 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .lookup_hit (st1.lk.is_hit), .lookup_line (st1.req.addr[`CS_LINE_SEL_BITS-1:0]), .lookup_way (st1.req.way_idx), - .repl_valid (do_fill_st0 && ~st0.req.is_passthru_fill && ~pipe_stall), + // a refetch merges into its resident sector and uses no victim + .repl_valid (do_fill_st0 && ~st0.req.is_passthru_fill && ~refetch_first_st0 && ~pipe_stall), .repl_line (line_idx_st0), .repl_line_n (sel_req.req.addr[`CS_LINE_SEL_BITS-1:0]), .repl_way (victim_way) ); + wire creq_hit_order_hazard_st0; + + // Loads and AMOs read the sector, so a partial one is a miss for them. The + // array enables use the raw tag match (a partial sector is still resident); + // only the commit decisions, registered at S1, see the partial check. + wire needs_data_st0 = st0.req.is_creq && ~st0.req.rw; + wire partial_match_st0; + wire partial_miss_st0 = (LAZY_FETCH != 0) && needs_data_st0 && partial_match_st0; + // A refetch finds its partial sector resident (it merges) unless the + // sector was evicted meanwhile: its data may then predate the eviction's + // writeback, so it installs nothing and its chain retries. + wire fill_resident_st0 = (| tag_matches_st0); + wire fill_evicted_st0 = refetch_first_st0 && ~fill_resident_st0; + VX_cache_tags #( .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -560,6 +631,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .WORD_SIZE (WORD_SIZE), .SECTOR_SIZE(SECTOR_SIZE), .WRITEBACK (WRITEBACK), + .LAZY_FETCH (LAZY_FETCH), .AMO_ENABLE ((AMO_ENABLE != 0) && (IS_LLC == 0)) ) cache_tags ( .clk (clk), @@ -567,12 +639,14 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .stall (pipe_stall), .init (do_init_st0), .flush (do_flush_st0 && ~pipe_stall), - .fill (do_fill_st0 && ~st0.req.is_passthru_fill && ~pipe_stall), + .fill (do_fill_st0 && ~st0.req.is_passthru_fill && ~refetch_first_st0 && ~pipe_stall), + .refetch (refetch_first_st0 && ~pipe_stall), .read (do_read_st0 && ~pipe_stall), .write (do_write_st0 && ~pipe_stall), // non-LLC AMO forwards downstream and invalidates its own copy so the // issuer's later plain load refetches the new value. .invalidate (is_amo_fwd_st0 && lk_st0.is_hit && ~pipe_stall), + .fill_nodata (lazy_st0), .line_idx (line_idx_st0), .line_idx_n (sel_req.req.addr[`CS_LINE_SEL_BITS-1:0]), .line_tag (line_tag_st0), @@ -583,7 +657,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .line_present (line_present_st0), .evict_dirty (evict_dirty_st0), .evict_dirty_mask (evict_dirty_mask_st0), - .evict_tag (evict_tag_st0) + .evict_tag (evict_tag_st0), + .partial_match (partial_match_st0) ); VX_onehot_encoder #( @@ -606,23 +681,24 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // disjoint words — the common case in streaming, where the whole warp's // beats land on one sector — keep the fast release path, as does // read-after-read on the same word. - wire creq_hit_order_hazard_st0 = st0.req.is_creq && ~st0.req.is_replay - && mshr_pending_raw - && (mshr_pending_wr_raw || (st0.req.rw && mshr_pending_ovl_raw)) - && ~is_amo_fwd_st0 - && ~((AMO_ENABLE != 0) && st0.req.amo.amo_valid); + assign creq_hit_order_hazard_st0 = st0.req.is_creq && ~st0.req.is_replay + && mshr_pending_raw + && (mshr_pending_wr_raw || (st0.req.rw && mshr_pending_ovl_raw)) + && ~is_amo_fwd_st0 + && ~((AMO_ENABLE != 0) && st0.req.amo.amo_valid); // S0 lookup delta (single combinational driver). The AMO requester is forced // non-pending so it never coalesces onto a prior same-line entry. always @(*) begin lk_st0 = '0; - lk_st0.is_hit = (| tag_matches_st0) && ~creq_hit_order_hazard_st0; + lk_st0.is_hit = (| tag_matches_st0) && ~partial_miss_st0 && ~creq_hit_order_hazard_st0; lk_st0.is_dirty = evict_dirty_st0; lk_st0.is_refill = st0.req.is_fill && line_present_any_st0; lk_st0.evict_dirty_mask = evict_dirty_mask_st0; lk_st0.evict_tag = evict_tag_st0; lk_st0.mshr_previd = mshr_previd; - lk_st0.mshr_pending = mshr_pending_raw && ~is_amo_fwd_st0; + // a parked replay is its own chain's root: it issues the refetch. + lk_st0.mshr_pending = mshr_pending_raw && ~is_amo_fwd_st0 && ~park_st0; end // ======================================================================== @@ -646,14 +722,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( dat_in = st0; dat_in.req.way_idx = evict_way_st0; dat_in.tag_matches = tag_matches_st0 & {NUM_WAYS{~creq_hit_order_hazard_st0}}; - dat_in.is_refill = lk_st0.is_refill; end // commit path: the request (with the resolved hit/victim way and MSHR id) // plus the lookup delta. The word payload is dropped here. always @(*) begin cmt_in.req = st0.req; - cmt_in.req.way_idx = st0.req.is_creq ? hit_idx_st0 : evict_way_st0; + cmt_in.req.way_idx = st0.req.is_creq ? hit_idx_st0 : merge_way_st0; cmt_in.req.mshr_id = st0.req.is_replay ? st0.req.mshr_id : mshr_alloc_id; cmt_in.lk = lk_st0; end @@ -693,12 +768,71 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_out (stC) ); + // A refetch whose sector is still resident merges at stC; a data-less or + // refetch fill writes no data at stD. + wire fill_merge_st0 = refetch_first_st0 && fill_resident_st0; + wire fill_nodata_st0 = lazy_st0 || refetch_first_st0; + wire fill_merge_stc, fill_nodata_std, lazy_std; + if (LAZY_FETCH != 0) begin : g_lazy_pipe + wire fill_merge_st1; + VX_pipe_register #( + .DATAW ($bits(lazy_req_t)), + .RESETW ($bits(lazy_req_t)) + ) reg_lz_s0 ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (lz_sel), + .data_out (lz_st0) + ); + VX_pipe_register #( + .DATAW ($bits(lazy_req_t) + 1), + .RESETW ($bits(lazy_req_t) + 1) + ) reg_lz_s1 ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in ({lz_st0, fill_merge_st0}), + .data_out ({lz_st1, fill_merge_st1}) + ); + VX_pipe_register #( + .DATAW ($bits(lazy_req_t) + 1), + .RESETW ($bits(lazy_req_t) + 1), + .DEPTH (PIPE_EX) + ) reg_lz_cmt ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in ({lz_st1, fill_merge_st1}), + .data_out ({lz_stc, fill_merge_stc}) + ); + VX_pipe_register #( + .DATAW (2), + .RESETW (2), + .DEPTH (PIPE_EX) + ) reg_lz_dat ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in ({lazy_st0, fill_nodata_st0}), + .data_out ({lazy_std, fill_nodata_std}) + ); + end else begin : g_no_lazy_pipe + assign lz_st0 = '0; + assign lz_st1 = '0; + assign lz_stc = '0; + assign fill_merge_stc = 1'b0; + assign fill_nodata_std = 1'b0; + assign lazy_std = 1'b0; + `UNUSED_VAR ({lz_sel, fill_merge_st0, fill_nodata_st0}) + end + // a passthru-AMO replay carries its result word instead of an installed // line, so it counts as a hit at the commit stage. The AMO commit signals // (is_amo_replay_st1) resolve at stC, so the replay-must-hit check is taken // there; with PIPE_EX>0 an st1-staged check would sample them a stage early. wire eff_hit_stc = stC.lk.is_hit || is_amo_replay_st1; - `RUNTIME_ASSERT (~(stC.req.valid && stC.req.is_replay && ~eff_hit_stc), ("missed mshr replay")) + `RUNTIME_ASSERT (~(stC.req.valid && stC.req.is_replay && ~park_stc && ~eff_hit_stc), ("missed mshr replay")) // ======================================================================== // Data array (driven at stD; outputs land at stC) @@ -742,7 +876,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_in (fbuf_addr_r), .data_out (fill_addr_std) ); - `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) + `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill && ~lazy_std) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) `endif // The fill sector, replicated across the line so each sector's slices see @@ -750,6 +884,18 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // gate). One copy == the full line when 1 sector/line. wire [`CS_LINE_WIDTH-1:0] fbuf_line = {`CS_SECTORS_PER_LINE{fill_data_std}}; + // sector of the request at commit (the missed sector of a fill request). + wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx_stc; + // a refetch merges into its resident sector at stC (see refetch_hold) + wire do_merge_stc = stC.req.valid && stC.req.is_fill && fill_merge_stc; + // Dirty-byte mask updates land at stC in pipeline order, once per request: + // a fill held at stC by its writeback beats still reads the evicted mask. + wire refetch_first_stc = stC.req.is_fill && refetch_stc && ~retry_stc; + wire mask_set_stc = do_write_stc && stC.lk.is_hit; + wire mask_fill_stc = stC.req.valid && stC.req.is_fill && ~stC.req.is_passthru_fill && ~refetch_first_stc; + wire mask_clr_line_stc = (mask_fill_stc && ~stC.lk.is_refill) || (stC.req.valid && stC.req.is_flush); + wire mask_clr_sec_stc = mask_fill_stc && stC.lk.is_refill; + VX_cache_data #( .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -759,13 +905,23 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .SECTOR_SIZE (SECTOR_SIZE), .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), - .DIRTY_BYTES (DIRTY_BYTES) + .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH) ) cache_data ( .clk (clk), .reset (reset), - .init (do_init_std), .fill (do_fill_std && ~stD.req.is_passthru_fill && ~pipe_stall), - .refill (stD.is_refill), + .fill_nodata (fill_nodata_std), + .mask_write ((mask_set_stc || mask_clr_line_stc || mask_clr_sec_stc) && ~pipe_stall), + .mask_set (mask_set_stc), + .mask_clr_line (mask_clr_line_stc), + .mask_clr_sector (mask_clr_sec_stc), + .mask_way (stC.req.way_idx), + .mask_line (stC.req.addr[`CS_LINE_SEL_BITS-1:0]), + .mask_word (stC.req.word_idx), + .mask_byteen (stC.req.byteen), + .mask_sector (sector_idx_stc), + .merge_write (do_merge_stc && ~pipe_stall), .flush (do_flush_std && ~pipe_stall), .read (do_read_std && ~pipe_stall), .write (do_write_std && ~pipe_stall), @@ -776,7 +932,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .write_word (stD.wdata), .word_idx (stD.req.word_idx), .sector_idx (sector_idx_std), - .way_idx_r (stC.req.way_idx), .write_byteen (stD.req.byteen), .read_data (read_data_stc), .evict_byteen (evict_byteen_stc) @@ -829,7 +984,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mshr_release_fire = mshr_finalize_st1 && mshr_release_st1 && ~pipe_stall; wire [1:0] mshr_dequeue; - `POP_COUNT(mshr_dequeue, {(replay_fire || fwd_fire), mshr_release_fire}); + `POP_COUNT(mshr_dequeue, {((replay_fire && ~park_fire) || fwd_fire), mshr_release_fire}); VX_pending_size #( .SIZE (MSHR_SIZE), @@ -858,6 +1013,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .AMO_ENABLE (AMO_ENABLE != 0), .AMO_PASSTHRU ((AMO_ENABLE != 0) && (IS_LLC == 0)), .WORD_SEL_WIDTH (WORD_SEL_WIDTH), + .LAZY_FETCH (LAZY_FETCH), .DATA_WIDTH (WORD_SEL_WIDTH + WORD_SIZE + `CS_WORD_WIDTH + TAG_WIDTH + REQ_SEL_WIDTH + AMO_REQ_BITS) ) cache_mshr ( .clk (clk), @@ -865,10 +1021,12 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .deq_req_uuid (req_uuid_sel), .alc_req_uuid (req_uuid_st0), .fin_req_uuid (req_uuid_st1), - .fill_valid (mem_rsp_fire), - .fill_id (mem_rsp_id), + .fill_valid (fill_fire), + .fill_id (fill_id), .fill_addr (mem_rsp_addr), .fill_sector (mem_rsp_sector), + .fill_refetch (mshr_fill_refetch), + .fill_retry (mshr_fill_retry), .probe_addr (core_req_addr), .probe_pending_ld (mshr_probe_pending_ld), .probe_pending_amo (mshr_probe_pending_amo), @@ -879,9 +1037,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .dequeue_valid (replay_valid), .dequeue_addr (replay_addr), .dequeue_rw (replay_rw), + .dequeue_reads (mshr_dequeue_reads), .dequeue_data ({replay_wsel, replay_byteen, replay_data, replay_tag, replay_idx, replay_amo}), .dequeue_id (replay_id), .dequeue_ready (replay_ready || fwd_fire), + .dequeue_park (park_sel), + .dequeue_park_retry (chain_retry), + .dequeue_block (replay_block), .allocate_valid (mshr_allocate_st0 && ~pipe_stall), .allocate_addr (st0.req.addr), .allocate_sector (sector_idx_st0), @@ -890,6 +1052,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Only non-LLC AMOs must not coalesce; at the LLC same-line AMOs coalesce // and serialize their commits on the single filled line. .allocate_is_amo (AMO_ENABLE ? st0.req.amo.amo_valid : 1'b0), + .allocate_refetch (needs_data_st0 && partial_match_st0), .allocate_data ({st0.req.word_idx, st0.req.byteen, write_word_st0, st0.req.tag, st0.req.req_idx, st0.req.amo}), .allocate_id (mshr_alloc_id), .allocate_pending (mshr_pending_raw), @@ -969,7 +1132,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .is_creq_st0 (st0.req.is_creq), // Atomics are exempt from the hit-order hazard, so their S0 hit is // the raw tag match; the MSHR probe stays off the commit_busy cone. - .is_hit_st0 ((| tag_matches_st0)), + .is_hit_st0 ((| tag_matches_st0) && ~partial_miss_st0), .is_replay_st0 (st0.req.is_replay), // Commit ports are fed from stC (the deferred data-output stage), so // the AMO RMW operands and the read word align at PIPE_EX>0. At @@ -1084,7 +1247,10 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // ======================================================================== reg fwd_active_r; - wire fwd_stage = mem_rsp_fire && ~is_passthru_fill_sel; + // A refetch fill stages its sector for the array write but cannot answer + // the chain: the staged copy lacks the dirty bytes the array merges in. + wire fbuf_stage = mem_rsp_fire && ~is_passthru_fill_sel; + wire fwd_stage = fbuf_stage && ~mshr_fill_refetch; wire fwd_close = replay_valid && fwd_active_r && (replay_rw || replay_amo.amo_valid); @@ -1108,11 +1274,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( end else begin if (fwd_stage) begin fwd_active_r <= 1'b1; - end else if (fwd_close) begin + end else if (fwd_close || ((LAZY_FETCH != 0) && fill_fire)) begin fwd_active_r <= 1'b0; end end - if (fwd_stage) begin + if (fbuf_stage) begin fbuf_data_r <= mem_rsp_data; fbuf_addr_r <= mem_rsp_addr; end @@ -1163,13 +1329,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mreq_queue_rw; wire is_fill_or_flush_stc = stC.req.is_fill || (stC.req.is_flush && WRITEBACK); + // a write-miss root under lazy fetch loops its fill back instead. + wire lazy_alloc_stc = (LAZY_FETCH != 0) && do_write_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending; wire do_fill_or_flush_stc = stC.req.valid && is_fill_or_flush_stc; // a sector refill keeps the resident line (no eviction), so never writes back. - wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill; + wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill && ~refetch_first_stc; wire [`CS_LINE_ADDR_WIDTH-1:0] evict_addr_stc = {stC.lk.evict_tag, stC.req.addr[`CS_LINE_SEL_BITS-1:0]}; - // sector of the request at commit (the missed sector of a fill request). - wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx_stc; if (`CS_SECTOR_SEL_BITS != 0) begin : g_sector_idx_stc assign sector_idx_stc = stC.req.word_idx[`CS_WORD_SEL_BITS-1 -: `CS_SECTOR_SEL_BITS]; end else begin : g_sector_idx_stc0 @@ -1239,18 +1405,21 @@ module VX_cache_bank import VX_gpu_pkg::*; #( if (WRITEBACK) begin : g_wb if (DIRTY_BYTES) begin : g_dirty_bytes wire has_dirty_bytes = (| evict_byteen_stc); - `RUNTIME_ASSERT (~do_fill_or_flush_stc || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + // a refetch evicts nothing: the mask it reads is its resident way's + wire check_evict_bytes = do_fill_or_flush_stc && ~refetch_first_stc; + `RUNTIME_ASSERT (~check_evict_bytes || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) // per-sector: a dirty sector must carry dirty bytes, or its writeback beat is dropped. wire [SEC-1:0] sec_has_dirty_bytes; for (genvar s = 0; s < SEC; ++s) begin : g_sec_has_dirty_bytes assign sec_has_dirty_bytes[s] = (| evict_byteen_stc[s*SECTOR_SIZE +: SECTOR_SIZE]); end - `RUNTIME_ASSERT (~do_fill_or_flush_stc || ((stC.lk.evict_dirty_mask & ~sec_has_dirty_bytes) == '0), ("dirty sector without dirty bytes: dirty_mask=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.evict_dirty_mask, sec_has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + `RUNTIME_ASSERT (~check_evict_bytes || ((stC.lk.evict_dirty_mask & ~sec_has_dirty_bytes) == '0), ("dirty sector without dirty bytes: dirty_mask=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.evict_dirty_mask, sec_has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) end // fill request on a read/write miss (one sector); multi-beat writeback // on a dirty eviction. The two are mutually exclusive (a writeback is a // fill/flush commit; a fill request is a creq commit). - wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending && ~pipe_stall; + wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending + && ~lazy_alloc_stc && ~pipe_stall; assign mreq_queue_push = fill_req_push || wb_beat_accept; assign mreq_queue_addr = is_wb_beat ? wb_mreq_addr : rd_mreq_addr; assign mreq_queue_rw = is_wb_beat; @@ -1317,12 +1486,90 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); assign mem_req_valid = ~mreq_queue_empty; + // Lazy-fetch loopback: write-miss roots awaiting their data-less fill. + // Sized like the memory request queue's reservation: a core request is + // admitted only while every request already in the pipe still fits. + if (LAZY_FETCH != 0) begin : g_lazyq + wire lazyq_full; + localparam LAZYQ_SIZE = 1 << `CLOG2(PIPELINE_STAGES + 1); + VX_fifo_queue #( + .DATAW (MSHR_ADDR_WIDTH), + .DEPTH (LAZYQ_SIZE), + .ALM_FULL (LAZYQ_SIZE - PIPELINE_STAGES) + ) lazy_queue ( + .clk (clk), + .reset (reset), + .push (lazy_alloc_stc && ~pipe_stall), + .pop (lazy_fire), + .data_in (stC.req.mshr_id), + .data_out (lazyq_id), + .empty (lazyq_empty), + .alm_full (lazyq_alm_full), + .full (lazyq_full), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (size) + ); + + // The partial state of the draining chain's sector: set by a data-less + // fill, cleared by a data fill, and corrected at S0 when a refetch + // finds its sector evicted (the replay waits out that cycle). + reg chain_partial_r, chain_retry_r; + always @(posedge clk) begin + if (reset) begin + chain_partial_r <= 1'b0; + chain_retry_r <= 1'b0; + end else if (fill_fire) begin + chain_partial_r <= lazy_fire; + chain_retry_r <= 1'b0; + end else if (refetch_hold && ~pipe_stall) begin + chain_partial_r <= fill_evicted_st0; + chain_retry_r <= fill_evicted_st0; + end + end + assign chain_partial = chain_partial_r; + assign chain_retry = chain_retry_r; + + // A refetch's fill sector must stay staged until its stC merge. + reg refetch_inflight_r; + always @(posedge clk) begin + if (reset) begin + refetch_inflight_r <= 1'b0; + end else if (mem_rsp_fire && mshr_fill_refetch) begin + refetch_inflight_r <= 1'b1; + end else if (stC.req.valid && stC.req.is_fill && refetch_stc && ~pipe_stall) begin + refetch_inflight_r <= 1'b0; + end + end + assign refetch_inflight = refetch_inflight_r; + + `RUNTIME_ASSERT (~(lazy_alloc_stc && ~pipe_stall && lazyq_full && ~lazy_fire), + ("%t: %s lazy queue overflow", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~(park_fire && fwd_fire), ("%t: %s park and forward in one cycle", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~replay_valid || (mshr_dequeue_reads == (~replay_rw || replay_amo.amo_valid)), + ("%t: %s chain head read flag disagrees with its payload", $time, INSTANCE_ID)) + // Nothing is selected behind a refetch, so its stC merge has the array + // to itself and the next lookup reads the cleared partial bit. + `RUNTIME_ASSERT (~(refetch_hold && sel_req.req.valid), ("%t: %s request selected behind a refetch", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~(st0.req.valid && park_st0 && (| tag_matches_st0) && ~partial_miss_st0), ("%t: %s parked replay hit", $time, INSTANCE_ID)) + // Only a refetch can find its sector resident. + `RUNTIME_ASSERT (~(do_fill_st0 && ~refetch_first_st0 && ~st0.req.is_passthru_fill && fill_resident_st0), + ("%t: %s fill into a resident sector: addr=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(st0.req.addr, BANK_ID))) + end else begin : g_no_lazyq + assign lazyq_id = '0; + assign lazyq_empty = 1'b1; + assign lazyq_alm_full = 1'b0; + assign chain_partial = 1'b0; + assign chain_retry = 1'b0; + assign refetch_inflight = 1'b0; + `UNUSED_VAR ({lazy_alloc_stc, lazy_fire, refetch_hold, fill_resident_st0, fill_evicted_st0, partial_match_st0}) + end + `UNUSED_VAR (do_lookup_st0) /////////////////////////////////////////////////////////////////////////////// `ifdef PERF_ENABLE - assign perf_read_miss = do_read_st1 && ~st1.lk.is_hit; + assign perf_read_miss = do_read_st1 && ~st1.lk.is_hit && ~lz_st1.is_park; assign perf_write_miss = do_write_st1 && ~st1.lk.is_hit; assign perf_evictions = do_writeback_stc; assign perf_mshr_stall = mshr_alm_full; @@ -1348,8 +1595,16 @@ module VX_cache_bank import VX_gpu_pkg::*; #( crsp_queue_stall, mreq_queue_alm_full, mshr_alm_full)) end if (mem_rsp_fire) begin - `TRACE(2, ("%t: %s fill-rsp: addr=0x%0h, mshr_id=%0d, data=0x%h (#%0d)\n", $time, INSTANCE_ID, - mem_rsp_full_addr, mem_rsp_id, mem_rsp_data, req_uuid_sel)) + `TRACE(2, ("%t: %s fill-rsp: addr=0x%0h, mshr_id=%0d, refetch=%b, retry=%b, data=0x%h (#%0d)\n", $time, INSTANCE_ID, + mem_rsp_full_addr, mem_rsp_id, mshr_fill_refetch, mshr_fill_retry, mem_rsp_data, req_uuid_sel)) + end + if (lazy_fire) begin + `TRACE(2, ("%t: %s lazy-fill: addr=0x%0h, mshr_id=%0d\n", $time, INSTANCE_ID, + mem_rsp_full_addr, lazyq_id)) + end + if (park_fire) begin + `TRACE(2, ("%t: %s mshr-park: addr=0x%0h, tag=0x%0h, req_idx=%0d (#%0d)\n", $time, INSTANCE_ID, + replay_full_addr, replay_tag, replay_idx, req_uuid_sel)) end if (replay_fire) begin `TRACE(2, ("%t: %s mshr-pop: addr=0x%0h, tag=0x%0h, req_idx=%0d (#%0d)\n", $time, INSTANCE_ID, diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 0dfc14eca1..88dc19d40d 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -44,6 +44,8 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -164,6 +166,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index d30d289f13..bc5bf12385 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -39,14 +39,28 @@ module VX_cache_data import VX_gpu_pkg::*; #( parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (fill granule); = LINE_SIZE => 1 sector parameter WRITE_ENABLE = 1, // Enable cache writeable parameter WRITEBACK = 0, // Enable cache writeback - parameter DIRTY_BYTES = 0 // Enable dirty bytes on writeback + parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0 // Fills may be data-less or merge under the dirty bytes ) ( input wire clk, input wire reset, // inputs - input wire init, input wire fill, - input wire refill, // fill into an already-resident line + input wire fill_nodata, // data-less fill: allocate only, no array write + // dirty-byte mask access at the commit stage (stC), one stage after the + // array access: a store sets its bytes, a fill/flush reads and clears + input wire mask_write, + input wire mask_set, + input wire mask_clr_line, + input wire mask_clr_sector, + input wire [`CS_WAY_SEL_WIDTH-1:0] mask_way, + input wire [`CS_LINE_SEL_BITS-1:0] mask_line, + input wire [`UP(`CS_WORD_SEL_BITS)-1:0] mask_word, + input wire [WORD_SIZE-1:0] mask_byteen, + input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mask_sector, + // merge of a refetched sector into its resident copy at the commit stage + // (the stage behind it is empty): only the clean bytes + input wire merge_write, input wire flush, input wire read, input wire write, @@ -58,14 +72,11 @@ module VX_cache_data import VX_gpu_pkg::*; #( input wire [WORD_SIZE-1:0] write_byteen, input wire [`UP(`CS_WORD_SEL_BITS)-1:0] word_idx, input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx, // sector being filled - input wire [`CS_WAY_SEL_WIDTH-1:0] way_idx_r, // outputs output wire [`CS_LINE_WIDTH-1:0] read_data, output wire [LINE_SIZE-1:0] evict_byteen ); - // The main data array resolves the way at read-issue and folds it into the - // array address, so the S1 output-way register is only used by the (narrow, - // off-path) dirty-byte mask below. + `STATIC_ASSERT(!LAZY_FETCH || (DIRTY_BYTES && WRITEBACK), ("invalid parameter: lazy fetch requires write-back dirty bytes")) localparam WAY_SEL_BITS = `CS_WAY_SEL_BITS; localparam DATA_RAM_DEPTH = `CS_LINES_PER_BANK * NUM_WAYS; @@ -84,6 +95,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( `UNUSED_PIN (valid_out) ); + wire fill_write = fill && ~((LAZY_FETCH != 0) && fill_nodata); wire is_evict = fill || flush; wire hit_any = (| tag_matches); wire [`CS_WAY_SEL_WIDTH-1:0] way_sel = is_evict ? evict_way : hit_way; @@ -91,69 +103,66 @@ module VX_cache_data import VX_gpu_pkg::*; #( // {way, line} -> physical address wire [DATA_ADDR_WIDTH-1:0] data_addr; if (NUM_WAYS > 1) begin : g_way_addr - assign data_addr = DATA_ADDR_WIDTH'({way_sel[WAY_SEL_BITS-1:0], line_idx}); + assign data_addr = merge_write ? DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}) + : DATA_ADDR_WIDTH'({way_sel[WAY_SEL_BITS-1:0], line_idx}); end else begin : g_no_way_addr - `UNUSED_VAR (way_sel) - assign data_addr = line_idx; + `UNUSED_VAR ({way_sel, mask_way}) + assign data_addr = merge_write ? mask_line : line_idx; end - // Per-byte dirty mask (writeback only). Unlike the main data array this is - // kept as one line-indexed array per way (evict way selected at S1): it is - // narrow and off the load path, and -- crucially -- it must be cleared for - // every way during the line-only init walk, which the way-folded data - // layout cannot do in a single pass. + // Per-byte dirty mask (writeback only), one mask word per {way, line}. It + // is read and updated only at the commit stage, so each access sees every + // older update with no bypass. A line's mask is cleared when it is + // installed, which leaves nothing for the reset walk to clear. if (DIRTY_BYTES != 0) begin : g_dirty_bytes - wire [`CS_WORDS_PER_LINE-1:0][WORD_SIZE-1:0] write_mask; - for (genvar i = 0; i < `CS_WORDS_PER_LINE; ++i) begin : g_write_mask - wire word_en = (`CS_WORDS_PER_LINE == 1) || (word_idx == i); - assign write_mask[i] = write_byteen & {WORD_SIZE{word_en}}; + wire [`CS_WORDS_PER_LINE-1:0][WORD_SIZE-1:0] set_word; + for (genvar i = 0; i < `CS_WORDS_PER_LINE; ++i) begin : g_set_word + wire word_en = (`CS_WORDS_PER_LINE == 1) || (mask_word == i); + assign set_word[i] = mask_byteen & {WORD_SIZE{mask_set && word_en}}; end - - // A sector refill must clear only the installed sector's mask: the other - // resident sectors may hold dirty bytes not yet written back. - wire [`CS_SECTORS_PER_LINE-1:0][SECTOR_SIZE-1:0] fill_clr_mask; - for (genvar i = 0; i < `CS_SECTORS_PER_LINE; ++i) begin : g_fill_clr_mask - wire sec_en = ~refill || (`CS_SECTORS_PER_LINE == 1) - || (sector_idx == `UP(`CS_SECTOR_SEL_BITS)'(i)); - assign fill_clr_mask[i] = {SECTOR_SIZE{fill && sec_en}}; + // A sector refill clears only the installed sector: the other resident + // sectors may hold dirty bytes not yet written back. + wire [`CS_SECTORS_PER_LINE-1:0][SECTOR_SIZE-1:0] clr_sector; + for (genvar i = 0; i < `CS_SECTORS_PER_LINE; ++i) begin : g_clr_sector + wire sec_en = (`CS_SECTORS_PER_LINE == 1) || (mask_sector == `UP(`CS_SECTOR_SEL_BITS)'(i)); + assign clr_sector[i] = {SECTOR_SIZE{mask_clr_line || (mask_clr_sector && sec_en)}}; end - - wire [NUM_WAYS-1:0][LINE_SIZE-1:0] byteen_rdata; - for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_byteen_store - wire [LINE_SIZE-1:0] byteen_wdata = {LINE_SIZE{write}}; // only asserted on writes - wire [LINE_SIZE-1:0] byteen_wren = {LINE_SIZE{init || flush}} | fill_clr_mask | write_mask; - wire byteen_write = ((fill || flush) && ((NUM_WAYS == 1) || (evict_way == i))) - || (write && tag_matches[i]) - || init; - wire byteen_read = fill || flush; - - // The dirty mask has 1-bit (per-byte) write granularity, which - // block RAM cannot do -- inferring BRAM shatters it into dozens of - // tiny RAMB18 per way. It is small, so map it to distributed RAM. - VX_sp_ram #( - .DATAW (LINE_SIZE), - .WRENW (LINE_SIZE), - .SIZE (`CS_LINES_PER_BANK), - .OUT_REG (1), - .LUTRAM (1), - .RDW_MODE ("R") - ) byteen_store ( - .clk (clk), - .reset (reset), - .read (byteen_read), - .write (byteen_write), - .wren (byteen_wren), - .addr (line_idx), - .wdata (byteen_wdata), - .rdata (byteen_rdata[i]) - ); + wire [LINE_SIZE-1:0] set_mask = set_word; + wire [LINE_SIZE-1:0] clr_mask = clr_sector; + + wire [DATA_ADDR_WIDTH-1:0] mask_addr; + if (NUM_WAYS > 1) begin : g_mask_way_addr + assign mask_addr = DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}); + end else begin : g_mask_line_addr + assign mask_addr = mask_line; end - assign evict_byteen = byteen_rdata[way_idx_r]; + + // A whole-word read-modify-write shares one write enable across the + // mask, so it maps onto wide distributed RAM rather than one cell per + // byte (or a shattered block RAM). + wire [LINE_SIZE-1:0] mask_rdata; + VX_sp_ram #( + .DATAW (LINE_SIZE), + .SIZE (DATA_RAM_DEPTH), + .OUT_REG (0), + .LUTRAM (1), + .RDW_MODE ("W"), + .INIT_ENABLE (1), + .INIT_VALUE (0) + ) byteen_store ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (mask_write), + .wren (1'b1), + .addr (mask_addr), + .wdata ((mask_rdata & ~clr_mask) | set_mask), + .rdata (mask_rdata) + ); + assign evict_byteen = mask_rdata; end else begin : g_no_dirty_bytes - `UNUSED_VAR (init) - `UNUSED_VAR (refill) - `UNUSED_VAR (flush) - `UNUSED_VAR (way_idx_r) + `UNUSED_VAR ({mask_write, mask_set, mask_clr_line, mask_clr_sector}) + `UNUSED_VAR ({mask_line, mask_word, mask_byteen, mask_sector}) assign evict_byteen = '1; // update whole line end @@ -180,14 +189,19 @@ module VX_cache_data import VX_gpu_pkg::*; #( if (WRITE_ENABLE) begin : g_wren // fill writes the fetched sector's slices; a store writes only the hit slice. - assign slice_write = (fill && fill_sec_en) || (write && hit_any && word_en); - assign slice_wren = fill ? {WORD_SIZE{1'b1}} : write_byteen; - assign slice_wdata = fill ? fill_data[s] : write_word; + wire merge_sec_en = (`CS_SECTORS_PER_LINE == 1) + || (mask_sector == `UP(`CS_SECTOR_SEL_BITS)'(SLICE_SECTOR)); + assign slice_write = (fill_write && fill_sec_en) || (write && hit_any && word_en) + || (merge_write && merge_sec_en); + assign slice_wren = merge_write ? ~evict_byteen[s * WORD_SIZE +: WORD_SIZE] + : (fill ? {WORD_SIZE{1'b1}} : write_byteen); + assign slice_wdata = (fill || merge_write) ? fill_data[s] : write_word; end else begin : g_no_wren `UNUSED_VAR (write) `UNUSED_VAR (write_word) `UNUSED_VAR (write_byteen) `UNUSED_VAR (hit_any) + `UNUSED_VAR ({fill_write, merge_write}) assign slice_write = fill && fill_sec_en; assign slice_wren = 1'b1; assign slice_wdata = fill_data[s]; diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index e088fe95fc..c8e66e0c10 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -55,6 +55,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter AMO_ENABLE = 0, // Per-entry AMO tracking (probe outputs) parameter AMO_PASSTHRU = 0, // Non-LLC passthrough: AMO entries never coalesce parameter WORD_SEL_WIDTH = 1, // word-in-line select width (order-hazard granularity) + parameter LAZY_FETCH = 0, // Chains may park and refetch a partial sector parameter MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE) ) ( @@ -73,6 +74,10 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire [`CS_LINE_ADDR_WIDTH-1:0] fill_addr, // sector of the entry that initiated this fill (which sector to install) output wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] fill_sector, + // the entry that initiated this fill refetches a resident partial sector + output wire fill_refetch, + // ... after a refetch found it evicted: the sector it re-reads stayed clean + output wire fill_retry, // probe: pending requests for `probe_addr`'s line, split by type. // probe_pending_ld : a non-AMO (line-filling) request is pending — @@ -96,9 +101,16 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire dequeue_valid, output wire [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr, output wire dequeue_rw, + output wire dequeue_reads, // the head reads its sector (load or AMO) output wire [DATA_WIDTH-1:0] dequeue_data, output wire [MSHR_ADDR_WIDTH-1:0] dequeue_id, input wire dequeue_ready, + // park the chain at its head instead of consuming it: the entry stays + // allocated (it becomes the root of a refetch) and the walk stops. + input wire dequeue_park, + input wire dequeue_park_retry, + // withhold the head this cycle (registered-only inputs) + input wire dequeue_block, // allocate input wire allocate_valid, @@ -107,6 +119,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( input wire [WORD_SEL_WIDTH-1:0] allocate_word_idx, input wire allocate_rw, input wire allocate_is_amo, // AMO: never coalesce + input wire allocate_refetch, // reads a resident partial sector input wire [DATA_WIDTH-1:0] allocate_data, output wire [MSHR_ADDR_WIDTH-1:0] allocate_id, output wire allocate_pending, @@ -154,6 +167,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // amo_mask is the per-entry AMO flag, zero when AMO_ENABLE=0 so the // mask folds out of addr_matches and the probe logic disappears. wire [MSHR_SIZE-1:0] amo_mask; + wire [MSHR_SIZE-1:0] amo_entries; reg allocate_rdy, allocate_rdy_n; reg [MSHR_ADDR_WIDTH-1:0] allocate_id_r, allocate_id_n; @@ -165,6 +179,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( wire allocate_fire = allocate_valid && allocate_ready; wire dequeue_fire = dequeue_valid && dequeue_ready; + wire dequeue_park_fire = (LAZY_FETCH != 0) && dequeue_fire && dequeue_park; + wire dequeue_free = dequeue_fire && ~dequeue_park_fire; wire [MSHR_SIZE-1:0] addr_matches; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_addr_matches @@ -179,7 +195,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // and hits the line the releasing entry just installed. assign addr_matches[i] = valid_table[i] && (addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector) && ~amo_mask[i] - && ~(dequeue_fire && (dequeue_id == MSHR_ADDR_WIDTH'(i))) + && ~(dequeue_free && (dequeue_id == MSHR_ADDR_WIDTH'(i))) && ~(finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i))); end @@ -193,7 +209,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // the next id never re-selects the slot being allocated this cycle. wire [MSHR_SIZE-1:0] dequeue_free_oh, finalize_free_oh, allocate_set_oh; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_free_delta - assign dequeue_free_oh[i] = dequeue_fire && (dequeue_id == MSHR_ADDR_WIDTH'(i)); + assign dequeue_free_oh[i] = dequeue_free && (dequeue_id == MSHR_ADDR_WIDTH'(i)); assign finalize_free_oh[i] = finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i)); assign allocate_set_oh[i] = allocate_fire && (allocate_id == MSHR_ADDR_WIDTH'(i)); end @@ -229,7 +245,9 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( dequeue_id_n = fill_id; end - if (dequeue_fire) begin + if (dequeue_park_fire) begin + dequeue_val_n = 0; + end else if (dequeue_fire) begin valid_table_n[dequeue_id] = 0; if (next_table[dequeue_id]) begin dequeue_id_n = next_index[dequeue_id]; @@ -343,6 +361,34 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign fill_addr = addr_table[fill_id]; assign fill_sector = sector_table[fill_id]; + if (LAZY_FETCH != 0) begin : g_refetch + reg [MSHR_SIZE-1:0] refetch_table, retry_table; + always @(posedge clk) begin + if (allocate_fire) begin + refetch_table[allocate_id] <= allocate_refetch; + retry_table[allocate_id] <= 1'b0; + end + if (dequeue_park_fire) begin + refetch_table[dequeue_id] <= 1'b1; + retry_table[dequeue_id] <= dequeue_park_retry; + end + end + assign fill_refetch = refetch_table[fill_id]; + assign fill_retry = retry_table[fill_id]; + // registered with the pointer so the park decision stays off the + // payload read + reg dequeue_reads_r; + always @(posedge clk) begin + dequeue_reads_r <= ~write_table[dequeue_id_n] || amo_entries[dequeue_id_n]; + end + assign dequeue_reads = dequeue_reads_r; + end else begin : g_no_refetch + assign fill_refetch = 1'b0; + assign fill_retry = 1'b0; + assign dequeue_reads = 1'b0; + `UNUSED_VAR ({allocate_refetch, dequeue_park, dequeue_park_retry, dequeue_park_fire, amo_entries}) + end + if (AMO_ENABLE != 0) begin : g_amo reg [MSHR_SIZE-1:0] amo_table; // An entry only participates in the pending-AMO probe once it has @@ -375,6 +421,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // in arrival order like any request; the probe outputs alone provide // the same-line ordering guard. assign amo_mask = (AMO_PASSTHRU != 0) ? amo_table : '0; + assign amo_entries = amo_table; // At the LLC no request links behind a pending AMO: the bank defers // every request to a line while an AMO to it waits here. `RUNTIME_ASSERT (~((AMO_PASSTHRU == 0) && finalize_valid && ~finalize_is_release && finalize_is_pending @@ -414,6 +461,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign persist_amo = finalize_valid && ~finalize_is_release && amo_table[finalize_id]; end else begin : g_no_amo assign amo_mask = '0; + assign amo_entries = '0; assign probe_pending_ld = 1'b0; assign probe_pending_amo = 1'b0; assign {probe_any_ld, probe_any_amo, persist_ld, persist_amo} = '0; @@ -449,7 +497,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // of an older chained read of the same word. assign allocate_pending_ovl = |(addr_matches & word_matches); - assign dequeue_valid = dequeue_val; + assign dequeue_valid = dequeue_val && ~((LAZY_FETCH != 0) && dequeue_block); // Sampled with dequeue_id_n, alongside the payload in mshr_store, rather // than read live at dequeue_id_r a cycle later. The entry's slot can be // freed and reallocated while the pointer still names it -- the free set diff --git a/hw/rtl/cache/VX_cache_tags.sv b/hw/rtl/cache/VX_cache_tags.sv index 8f208a219a..5ecfc5c0be 100644 --- a/hw/rtl/cache/VX_cache_tags.sv +++ b/hw/rtl/cache/VX_cache_tags.sv @@ -35,6 +35,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( parameter WORD_SIZE = 1, // Size of a word in bytes parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (fill/eviction granule); = LINE_SIZE => 1 sector parameter WRITEBACK = 0, // Enable cache writeback + parameter LAZY_FETCH = 0, // Track per-sector partial (allocated-by-store) state parameter AMO_ENABLE = 0 // Enable the AMO-passthrough line invalidate (non-LLC banks only) ) ( input wire clk, @@ -48,6 +49,10 @@ module VX_cache_tags import VX_gpu_pkg::*; #( input wire read, input wire write, input wire invalidate, // clear valid on the hit way + // the fill installs no data: the sector becomes partial (cleared otherwise) + input wire fill_nodata, + // a refetch completes its resident partial sector: clear its partial bit + input wire refetch, input wire [`CS_LINE_SEL_BITS-1:0] line_idx, input wire [`CS_LINE_SEL_BITS-1:0] line_idx_n, input wire [`CS_TAG_SEL_BITS-1:0] line_tag, @@ -70,7 +75,9 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // per-sector dirty vector of the evict way (drives the multi-beat per-sector // writeback); a single bit == evict_dirty when 1 sector/line. output wire [`CS_SECTORS_PER_LINE-1:0] evict_dirty_mask, - output wire [`CS_TAG_SEL_BITS-1:0] evict_tag + output wire [`CS_TAG_SEL_BITS-1:0] evict_tag, + // the requested sector is resident but partial (a load/AMO to it refetches) + output wire partial_match ); // tag store: tag only (valid + dirty decoupled into side LUTRAMs) localparam SEC = `CS_SECTORS_PER_LINE; @@ -82,6 +89,8 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][`CS_TAG_SEL_BITS-1:0] read_tag; wire [NUM_WAYS-1:0][SEC-1:0] read_valid; wire [NUM_WAYS-1:0][SEC-1:0] read_dirty; + wire [NUM_WAYS-1:0][SEC-1:0] read_partial; + wire [NUM_WAYS-1:0] partial_hit; if (WRITEBACK) begin : g_evict_tag_wb assign evict_dirty = (| read_dirty[evict_way]); // dirty if any sector dirty @@ -117,6 +126,13 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][SEC-1:0] dirty_wdata; wire [NUM_WAYS-1:0][SEC-1:0] dirty_rdata; + // Per-sector partial store (lazy fetch only): a fill writes its sector's + // bit, set by a data-less fill and cleared by a data fill. The bit is only + // meaningful while the sector is valid, so no other operation touches it. + wire [NUM_WAYS-1:0][SEC-1:0] partial_wren; + wire [NUM_WAYS-1:0][SEC-1:0] partial_wdata; + wire [NUM_WAYS-1:0][SEC-1:0] partial_rdata; + for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_way_decode wire way_en = (NUM_WAYS == 1) || (evict_way == i); wire do_init = init; // init all ways @@ -163,8 +179,12 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // hit: tag match AND the requested sector is valid. read_valid folds in // a same-set fill from the previous cycle (see the bypass above), so - // this compare is exact even on the fill's write edge. - wire raw_hit = read_valid[i][sector_idx] && tag_eq; + // this compare is exact even on the fill's write edge. Whether a partial + // sector serves the lookup is decided outside, off the array enables. + wire sector_valid = read_valid[i][sector_idx] && tag_eq; + wire sector_partial = (LAZY_FETCH != 0) && read_partial[i][sector_idx]; + wire raw_hit = sector_valid; + assign partial_hit[i] = sector_valid && sector_partial; // AMO passthrough invalidate: clear the requested sector's valid. wire do_inval = (AMO_ENABLE != 0) && invalidate && raw_hit; @@ -224,6 +244,23 @@ module VX_cache_tags import VX_gpu_pkg::*; #( assign read_dirty[i] = {SEC{1'b0}}; end + if (LAZY_FETCH != 0) begin : g_partial + // Same access pattern as the valid store, so the way_filled bypass + // corrects a same-set fill from the previous cycle the same way. A + // refetch's clear needs none: nothing is looked up right behind it. + wire rdw_nodata; + `BUFFER_EX(rdw_nodata, fill_nodata, ~stall, 1, 1); + wire do_refetch = refetch && sector_valid; + assign partial_wren[i] = (do_fill || do_refetch) ? sec_oh : {SEC{1'b0}}; + assign partial_wdata[i] = {SEC{do_fill && fill_nodata}}; + assign read_partial[i] = way_filled ? ((partial_rdata[i] & ~rdw_sec_oh) | (rdw_sec_oh & {SEC{rdw_nodata}})) + : partial_rdata[i]; + end else begin : g_no_partial + assign partial_wren[i] = '0; + assign partial_wdata[i] = '0; + assign read_partial[i] = {SEC{1'b0}}; + end + assign tag_matches[i] = raw_hit; // line resident in this way: tag matches and at least one sector valid // (a same-cycle-prior fill is folded into tag_eq/read_valid). @@ -304,4 +341,30 @@ module VX_cache_tags import VX_gpu_pkg::*; #( `UNUSED_VAR ({dirty_wren, dirty_wdata, dirty_rdata}) end + if (LAZY_FETCH != 0) begin : g_partial_store + VX_dp_ram #( + .DATAW (NUM_WAYS * SEC), + .WRENW (NUM_WAYS * SEC), + .SIZE (`CS_LINES_PER_BANK), + .OUT_REG (1), + .LUTRAM (1), + .RDW_MODE ("R") + ) partial_store ( + .clk (clk), + .reset (reset), + .read (~stall), + .write (| partial_wren), + .wren (partial_wren), + .waddr (line_idx), + .raddr (line_idx_n), + .wdata (partial_wdata), + .rdata (partial_rdata) + ); + assign partial_match = (| partial_hit); + end else begin : g_no_partial_store + assign partial_rdata = '0; + assign partial_match = 1'b0; + `UNUSED_VAR ({partial_wren, partial_wdata, partial_rdata, partial_hit, read_partial, fill_nodata, refetch}) + end + endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index e9f34d17bc..0f87a80e45 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -42,6 +42,8 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -145,6 +147,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index b75215a485..5d797998c9 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -52,6 +52,9 @@ module VX_cache_top import VX_gpu_pkg::*; #( // Enable dirty bytes on writeback parameter DIRTY_BYTES = `VX_CFG_DCACHE_DIRTYBYTES, + // Write misses allocate without a fill (write-validate) + parameter LAZY_FETCH = `VX_CFG_DCACHE_LAZY_FETCH, + // Bank pipeline depth (capacity-scaled) parameter LATENCY = `VX_CFG_DCACHE_LATENCY, @@ -183,6 +186,7 @@ module VX_cache_top import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), .AMO_ENABLE (AMO_ENABLE), .IS_LLC (IS_LLC), .BANK_HASH (BANK_HASH), diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index aae201f6e1..ce4ef6876d 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -758,6 +758,7 @@ module VX_gfx_top import VX_gpu_pkg::* .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 629ab576c4..abd8b0b526 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -385,6 +385,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_DCACHE_WRITEBACK), .DIRTY_BYTES (`VX_CFG_DCACHE_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_DCACHE_LAZY_FETCH), .REPL_POLICY (`VX_CFG_DCACHE_REPL_POLICY), .NC_ENABLE (1), .CORE_OUT_BUF (3), @@ -504,6 +505,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 54e4c29b15..056711d448 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -305,6 +305,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_DCACHE_WRITEBACK), .DIRTY_BYTES (`VX_CFG_DCACHE_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_DCACHE_LAZY_FETCH), .REPL_POLICY (`VX_CFG_DCACHE_REPL_POLICY), .NC_ENABLE (1), .CORE_OUT_BUF (3), @@ -616,6 +617,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 0978067d9e..95b3669a3d 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -86,6 +86,7 @@ class Cluster::Impl { (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash L2_SHARED_LLC, // shared_llc + VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch }); diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 1a647e06ee..72124b8e3d 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -169,17 +169,24 @@ struct params_t { struct sector_t { bool valid; bool dirty; + bool partial; // allocated by a store: only dirty_mask bytes are known uint64_t dirty_mask; // per-byte dirty bits; the writeback byte-enable std::shared_ptr data; // sector bytes (= one mem_block) void reset() { valid = false; dirty = false; + partial = false; dirty_mask = 0; data.reset(); } }; +// A store may hit a partial sector; a load or AMO needs every byte. +static inline bool sector_hit(const sector_t& sec, bool needs_data) { + return sec.valid && !(needs_data && sec.partial); +} + struct line_t { uint64_t tag; uint32_t lru_ctr; @@ -369,7 +376,8 @@ struct bank_req_t { Fill = 1, Replay = 2, Core = 3, - AmoProbe = 4 // non-LLC AMO passthrough: probe-and-invalidate then forward + AmoProbe = 4, // non-LLC AMO passthrough: probe-and-invalidate then forward + Park = 5 // chain head parked on a partial sector: issues its refetch }; uint64_t addr; @@ -386,6 +394,13 @@ struct bank_req_t { // emit another response — only run sector_merge. bool skip_core_rsp; + // Fill flavors under lazy fetch: a data-less allocation for a write-miss + // root, or a refetch of a resident partial sector (merges under its dirty + // bytes). + bool nodata; + bool refetch; + bool refetch_retry; // refetch re-issued after its sector was evicted + // TLM data: // For Core writes: incoming write data + byteen. // For Fill: captured fill data from below (mem_rsp.data). @@ -425,6 +440,9 @@ struct bank_req_t { write = false; op = MemOp::LD; skip_core_rsp = false; + nodata = false; + refetch = false; + refetch_retry = false; data.reset(); byteen = 0; amo_cmp = 0; @@ -451,6 +469,8 @@ struct mshr_entry_t { uint32_t line_id; uint64_t seq; // enqueue order; replays drain oldest-first to preserve // program order among coalesced same-line accesses. + bool refetch; // root of a fill that refetches a resident partial sector + bool refetch_retry; // ... re-issued after the previous refetch found it evicted mshr_entry_t() { this->reset(); @@ -463,6 +483,8 @@ struct mshr_entry_t { sector_id = 0; line_id = 0; seq = 0; + refetch = false; + refetch_retry = false; } }; @@ -549,7 +571,8 @@ class MSHR { } // Enqueue a new core request and return the allocated entry id. - int enqueue(const bank_req_t &bank_req, uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) { + int enqueue(const bank_req_t &bank_req, uint32_t set_id, uint64_t addr_tag, uint32_t sector_id, + bool refetch = false) { assert(bank_req.type == bank_req_t::Core); for (uint32_t i = 0, n = entries_.size(); i < n; ++i) { auto &entry = entries_.at(i); @@ -560,6 +583,8 @@ class MSHR { entry.sector_id = sector_id; entry.line_id = 0; // victim is selected at Fill time entry.seq = bank_req.adm_seq; + entry.refetch = refetch; + entry.refetch_retry = false; ++size_; return i; } @@ -600,6 +625,34 @@ class MSHR { ++ready_reqs_; } + // Park the chain at its head: the oldest ready replay and the rest of its + // (set,tag,sector) chain return to waiting for a fill, which the head now + // roots as a refetch. Returns the head's id. + uint32_t park_next_replay(bool retry) { + int head = -1; + for (int i = 0, n = (int)entries_.size(); i < n; ++i) { + const auto &entry = entries_.at(i); + if (entry.bank_req.type != bank_req_t::Replay) { + continue; + } + if (head == -1 || entry.seq < entries_.at(head).seq) { + head = i; + } + } + assert(head != -1); + auto &root = entries_.at(head); + for (auto &entry : entries_) { + if (entry.bank_req.type == bank_req_t::Replay && entry.set_id == root.set_id + && entry.addr_tag == root.addr_tag && entry.sector_id == root.sector_id) { + entry.bank_req.type = bank_req_t::Core; + --ready_reqs_; + } + } + root.refetch = true; + root.refetch_retry = retry; + return head; + } + // Peek the entry dequeue() would return next (oldest ready replay), or // nullptr. Lets the fill-forward drain inspect the chain head without // consuming it. @@ -679,11 +732,14 @@ class CacheBank : public SimObject { const Cache::Config &config, const params_t ¶ms, uint32_t bank_id) - : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), rand_ctr_(0), adm_seq_ctr_(0) + : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), lazy_q_(TFifo::Create("", 1, lazyq_size(config))), rand_ctr_(0), adm_seq_ctr_(0) #if VX_CFG_EXT_A_ENABLED , amo_unit_(__MAX(2u, (uint32_t)VX_CFG_AMO_RS_SIZE)) #endif { + // Lazy allocation leaves unfetched bytes in the line: only a write-back + // LLC can hold them until a read merges or a writeback masks them out. + assert(!config.lazy_fetch || (config.write_back && config.is_llc)); this->on_reset(); } @@ -740,6 +796,12 @@ class CacheBank : public SimObject { fwd_sector_ = 0; fwd_data_.reset(); crsp_sent_ = false; + chain_partial_ = false; + chain_retry_ = false; + chain_set_ = 0; + chain_tag_ = 0; + chain_sector_ = 0; + refetch_hold_ = false; flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -788,7 +850,7 @@ class CacheBank : public SimObject { // the mem_latency accumulation exact. flush_begin() re-arms explicitly; // any packet reserved toward core_req_in/mem_rsp_in re-arms on arrival. if (this->core_req_in.size() == 0 && this->mem_rsp_in.size() == 0 - && pipe_req_->size() == 0 && !mshr_.has_ready_reqs() + && pipe_req_->size() == 0 && lazy_q_->size() == 0 && !mshr_.has_ready_reqs() && !fwd_active_ && !flushing_ && pending_fill_reqs_ == 0) { this->tick_sleep(); } @@ -812,6 +874,26 @@ class CacheBank : public SimObject { // drain completed the head this tick, this slot goes to new core // requests instead if (mshr_.has_ready_reqs() && !fwd_consumed) { + // A refetch fill learns whether its sector is still resident when it + // installs; its chain waits for that answer. + if (refetch_hold_) { + return; + } + // A chain whose sector is partial cannot serve a load or AMO: park at + // that head and refetch the sector; the rest of the chain waits. + if (chain_partial_) { + const auto *head = mshr_.peek_next_replay(); + if (head != nullptr + && head->set_id == chain_set_ && head->addr_tag == chain_tag_ && head->sector_id == chain_sector_ + && !head->bank_req.write) { + bank_req_t park_req = head->bank_req; + park_req.mshr_id = mshr_.park_next_replay(chain_retry_); + park_req.type = bank_req_t::Park; + pipe_req_->push(park_req); + DT(3, this->name() << " mshr-park: " << park_req); + return; + } + } bank_req_t bank_req; mshr_.dequeue(&bank_req); pipe_req_->push(bank_req); @@ -873,21 +955,57 @@ class CacheBank : public SimObject { bank_req.uuid = root_peek.bank_req.uuid; bank_req.mshr_id = mshr_id; bank_req.data = mem_rsp.data; + bank_req.refetch = root_peek.refetch; + bank_req.refetch_retry = root_peek.refetch_retry; pipe_req_->push(bank_req); // The chain wakes at accept: its replays enter the pipe behind the fill, // so they reach the array after the install. The forward window answers - // the chain's leading reads from the staged sector meanwhile. + // the chain's leading reads from the staged sector meanwhile; a refetch + // cannot, its staged sector lacks the dirty bytes the install merges in. mshr_.replay(mshr_id); - fwd_active_ = true; - fwd_set_ = set_id; - fwd_tag_ = addr_tag; - fwd_sector_ = sector_id; - fwd_data_ = mem_rsp.data; + this->setChain(set_id, addr_tag, sector_id, false); + if (bank_req.refetch) { + refetch_hold_ = true; + } else { + fwd_active_ = true; + fwd_set_ = set_id; + fwd_tag_ = addr_tag; + fwd_sector_ = sector_id; + fwd_data_ = mem_rsp.data; + } DT(3, this->name() << " fill-rsp: " << mem_rsp); this->mem_rsp_in.pop(); --pending_fill_reqs_; return; } + } else if (!lazy_q_->empty() && !fwd_active_) { + // Lazy allocation of a write-miss root: a data-less fill through the + // same path, below memory fills in priority. + uint32_t mshr_id = lazy_q_->peek(); + const auto &root_peek = mshr_.peek(mshr_id); + const uint32_t set_id = root_peek.set_id; + const uint64_t addr_tag = root_peek.addr_tag; + const uint32_t sector_id = root_peek.sector_id; + if (this->pipeLineBusy(set_id, addr_tag)) { + fill_held = true; + held_set_id = set_id; + held_addr_tag = addr_tag; + } else { + bank_req_t bank_req; + bank_req.reset(); + bank_req.type = bank_req_t::Fill; + bank_req.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); + bank_req.hart_id = root_peek.bank_req.hart_id; + bank_req.uuid = root_peek.bank_req.uuid; + bank_req.mshr_id = mshr_id; + bank_req.nodata = true; + pipe_req_->push(bank_req); + mshr_.replay(mshr_id); + this->setChain(set_id, addr_tag, sector_id, true); + DT(3, this->name() << " lazy-fill: " << bank_req); + lazy_q_->pop(); + return; + } } // 3) core request @@ -954,6 +1072,11 @@ class CacheBank : public SimObject { ++perf_stats_.mshr_stalls; return; } + // Any admitted request may become a write-miss root: the lazy queue + // keeps room for every request already in the pipe. + if (config_.lazy_fetch && lazy_q_->size() >= lazyq_size(config_) - config_.latency) { + return; + } // Admission-time coalescing: a request whose (set,tag,sector) chain is // still awaiting its fill must join it here, not enter the pipe as an @@ -1076,6 +1199,28 @@ class CacheBank : public SimObject { int resident_id = set.find_resident(addr_tag); const bool is_refill = (resident_id != -1); + // Only a refetch finds its sector resident (partial): it merges. A refetch + // whose sector was evicted meanwhile may carry data older than that + // eviction's writeback, so it installs nothing and its chain parks again; + // the retry then fills like any miss (an absent sector cannot be dirtied). + const bool sector_resident = is_refill && set.lines.at(resident_id).sectors.at(sector_id).valid; + const bool refetch_first = bank_req.refetch && !bank_req.refetch_retry; + assert(!sector_resident || refetch_first); + if (refetch_first) { + chain_partial_ = !sector_resident; + chain_retry_ = !sector_resident; + refetch_hold_ = false; + if (sector_resident) { + // Memory supplies the bytes no store wrote; the sector stays dirty + // in exactly the stored bytes, which are not in memory yet. + auto &sec = set.lines.at(resident_id).sectors.at(sector_id); + auto written = sec.data; + sec.data = bank_req.data; + sector_merge(sec, written, sec.dirty_mask); + sec.partial = false; + } + return true; + } int line_id; if (is_refill) { @@ -1132,13 +1277,41 @@ class CacheBank : public SimObject { line.tag = addr_tag; line.lru_ctr = 0; auto &sec = line.sectors.at(sector_id); - sec.valid = true; - sec.dirty = false; - sec.dirty_mask = 0; - sec.data = bank_req.data; + if (bank_req.nodata) { + sec.valid = true; + sec.dirty = false; + sec.partial = true; + sec.dirty_mask = 0; + sec.data.reset(); + } else { + sec.valid = true; + sec.dirty = false; + sec.partial = false; + sec.dirty_mask = 0; + sec.data = bank_req.data; + } + if (bank_req.refetch) { + chain_partial_ = false; + chain_retry_ = false; + refetch_hold_ = false; + } return true; } + // Track the partial state of the chain a fill just woke. + void setChain(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id, bool partial) { + chain_partial_ = partial; + chain_retry_ = false; + chain_set_ = set_id; + chain_tag_ = addr_tag; + chain_sector_ = sector_id; + } + + // Lazy-fetch loopback depth: room for every request in the pipe plus one. + static uint32_t lazyq_size(const Cache::Config &config) { + return 1u << log2ceil(config.latency + 1); + } + // Fill-forward drain: complete the oldest ready replay directly when it is // a plain read on the armed chain — no pipeline traversal, and the input // slot stays free for new requests. The first write/AMO head closes the @@ -1416,6 +1589,24 @@ class CacheBank : public SimObject { pipe_req_->pop(); } break; + case bank_req_t::Park: { + if (this->mem_req_out.full()) { + return; // stall + } + uint32_t set_id = params_.addr_set_id(bank_req.addr); + uint64_t addr_tag = params_.addr_tag(bank_req.addr); + uint32_t sector_id = params_.addr_sector_id(bank_req.addr); + MemReq fill; + fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); + fill.tag = bank_req.mshr_id; + fill.hart_id = bank_req.hart_id; + fill.uuid = bank_req.uuid; + this->mem_req_out.send(fill, MEM_REQ_DELAY); + DT(3, this->name() << " refetch-req: " << fill); + ++pending_fill_reqs_; + pipe_req_->pop(); + } break; + case bank_req_t::Replay: { uint32_t set_id = params_.addr_set_id(bank_req.addr); uint64_t addr_tag = params_.addr_tag(bank_req.addr); @@ -1423,8 +1614,10 @@ class CacheBank : public SimObject { auto &set = sets_.at(set_id); int32_t free_id = -1, repl_id = 0; int present_id = set.tag_match(addr_tag, config_.repl_policy, rand_ctr_, &free_id, &repl_id); - // Sector hit: line resident AND the requested sector valid. - int hit_id = (present_id != -1 && set.lines.at(present_id).sectors.at(sector_id).valid) ? present_id : -1; + // Sector hit: line resident AND the requested sector valid (and complete + // when the access reads it). + const bool needs_data = !bank_req.write; + int hit_id = (present_id != -1 && sector_hit(set.lines.at(present_id).sectors.at(sector_id), needs_data)) ? present_id : -1; // The replayed sector is normally present — the fill that woke this // replay just installed it. It can still be gone when a concurrent @@ -1452,7 +1645,8 @@ class CacheBank : public SimObject { return; // stall: need a slot to re-track the miss bank_req_t refill = bank_req; refill.type = bank_req_t::Core; - int mshr_id = mshr_.enqueue(refill, set_id, addr_tag, sector_id); + const bool refetch = (present_id != -1) && set.lines.at(present_id).sectors.at(sector_id).valid; + int mshr_id = mshr_.enqueue(refill, set_id, addr_tag, sector_id, refetch); if (!fill_pending) { MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); @@ -1530,7 +1724,11 @@ class CacheBank : public SimObject { int present_id = set.tag_match(addr_tag, config_.repl_policy, rand_ctr_, &free_id, &repl_id); // Sector hit: line resident AND the requested sector valid (a sector miss // on a resident line takes the miss path and refills just that sector). - int hit_id = (present_id != -1 && set.lines.at(present_id).sectors.at(sector_id).valid) ? present_id : -1; + // A load or AMO to a partial sector misses and refetches it. + const bool needs_data = !bank_req.write; + int hit_id = (present_id != -1 && sector_hit(set.lines.at(present_id).sectors.at(sector_id), needs_data)) ? present_id : -1; + const bool partial_resident = (present_id != -1) && (hit_id == -1) + && set.lines.at(present_id).sectors.at(sector_id).valid; if (hit_id != -1) { // Hit while a same-(set,tag,sector) chain still carries a write or @@ -1560,7 +1758,12 @@ class CacheBank : public SimObject { } else { assert(!mshr_.full()); int id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id); - mshr_.defer_to_replay(id); + // A chain still awaiting its fill (a refetch of this partial + // sector) wakes this entry itself; only a draining chain needs it + // marked ready here. + if (!mshr_.has_pending_fill(set_id, addr_tag, sector_id, id)) { + mshr_.defer_to_replay(id); + } DT(3, this->name() << " mshr-defer: " << bank_req); } --pending_mshr_size_; @@ -1663,13 +1866,23 @@ class CacheBank : public SimObject { // entries never mark new waiters, so they must not suppress the // fill for this fresh miss. bool fill_pending = mshr_.has_pending_fill(set_id, addr_tag, sector_id, -1); - if (!fill_pending && this->mem_req_out.full()) + // Lazy fetch: a write-miss root allocates through the loopback + // instead of fetching the sector. + const bool lazy_alloc = config_.lazy_fetch && bank_req.write && !fill_pending; + if (lazy_alloc && lazy_q_->full()) { return; + } + if (!fill_pending && !lazy_alloc && this->mem_req_out.full()) { + return; + } assert(!mshr_.full()); - int mshr_id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id); + int mshr_id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id, partial_resident); DT(3, this->name() << " mshr-enqueue: " << bank_req); - if (!fill_pending) { + if (lazy_alloc) { + lazy_q_->push(mshr_id); + DT(3, this->name() << " lazy-alloc: " << bank_req); + } else if (!fill_pending) { MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); // op defaults to MemOp::LD — fill is a load. @@ -1774,6 +1987,7 @@ class CacheBank : public SimObject { assert(false && "in-pipe line mirror underflow"); } TFifo::Ptr pipe_req_; + TFifo::Ptr lazy_q_; // write-miss roots awaiting their data-less fill Cache::PerfStats perf_stats_; @@ -1791,6 +2005,14 @@ class CacheBank : public SimObject { std::shared_ptr fwd_data_; bool crsp_sent_; // core-response port used this tick (pipeline priority) + // Partial state of the chain the last fill woke (lazy fetch). + bool chain_partial_; + bool chain_retry_; // the partial state came from an evicted refetch + uint32_t chain_set_; + uint64_t chain_tag_; + uint32_t chain_sector_; + bool refetch_hold_; // a refetch fill has not installed yet: its chain waits + // Flush walk state. bool flushing_; bool keep_lines_ = false; // the next launch reset keeps the lines diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 4e7e8c5654..5dd4c9bd08 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -45,6 +45,7 @@ class Cache : public SimObject { bool is_llc; // is last-level cache bool bank_hash = false; // hashed bank select bool shared_llc = false; // the LLC serves every memory client alone + bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) }; struct PerfStats { diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index bdcf97023d..aef1822cf6 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -106,6 +106,7 @@ ProcessorImpl::ProcessorImpl() VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC false, // bank_hash L3_SHARED_LLC, // shared_llc + VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch } ); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 640f263776..5a1861155a 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -93,6 +93,7 @@ class Socket::Impl { (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0), // is_llc (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash DCACHE_SHARED_LLC, // shared_llc + VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch }); #ifdef VX_CFG_EXT_TEX_ENABLE diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 3fd4c75708..bb82626739 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -15,7 +15,7 @@ TESTS := \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ async_barrier async_gbarrier gbar_phase bar_slot_phase packld wgather wsync quad_lod \ occupancy multikernel reopen module_reload single_cta \ - flush_coherence \ + flush_coherence lazy_fetch \ vm_test vm_fault vm_stress # --- common exclude list --------------------------------------------- diff --git a/tests/regression/lazy_fetch/Makefile b/tests/regression/lazy_fetch/Makefile new file mode 100644 index 0000000000..7a39f6c48c --- /dev/null +++ b/tests/regression/lazy_fetch/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := lazy_fetch + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n16 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/lazy_fetch/common.h b/tests/regression/lazy_fetch/common.h new file mode 100644 index 0000000000..308f768331 --- /dev/null +++ b/tests/regression/lazy_fetch/common.h @@ -0,0 +1,134 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Each thread owns NUM_CASES regions per round, one cache line (REGION_WORDS +// words) each, laid out round-major so neighbouring lines belong to different +// threads and the buffer outgrows the cache across rounds. +#define REGION_WORDS 16 +#define NUM_CASES 6 +#define LANE_WORDS 8 +// Same-set lines: one per thread and round, a page apart, so they compete for +// one cache set and a line being refetched is often evicted before its fill. +#define SET_STRIDE_WORDS 1024 + +enum { + CASE_HALF = 0, // store half the line, then load all of it (same thread) + CASE_BYTES = 1, // byte/halfword stores, then load the words holding them + CASE_AMO = 2, // stores, then AMO add and LR/SC on written and unwritten words + CASE_STORE = 3, // byte stores only: checked after the kernel + CASE_WARP = 4, // one warp stores half the line, the next warp loads all of it + CASE_LANE = 5 // a warp's lanes store into one line, then load its unwritten half +}; + +typedef struct { + uint64_t buf_addr; + uint64_t set_addr; + uint64_t err_addr; + uint32_t rounds; + uint32_t num_threads; // threads in the grid + uint32_t warp_size; + uint32_t cross_warp; // blocks hold an even number of warps + uint32_t amo; // the device implements the A extension + uint32_t phase; // 0: exercise the cases, 1: re-check every region + uint32_t rotate; // phase 1: check the regions of thread (gid + rotate) +} kernel_arg_t; + +// Pre-pattern the host writes over the whole buffer. +static inline uint8_t pattern_byte(uint32_t i) { + return (uint8_t)(i * 37u + (i >> 9) + 0x5Au); +} + +static inline uint32_t pattern_word(uint32_t word_idx) { + uint32_t b = word_idx * 4; + return (uint32_t)pattern_byte(b) + | ((uint32_t)pattern_byte(b + 1) << 8) + | ((uint32_t)pattern_byte(b + 2) << 16) + | ((uint32_t)pattern_byte(b + 3) << 24); +} + +static inline uint32_t store_value(uint32_t region, uint32_t w) { + return 0xC0000000u | (region << 4) | w; +} + +static inline uint32_t region_index(uint32_t round, uint32_t num_threads, uint32_t gid, uint32_t c) { + return (round * num_threads + gid) * NUM_CASES + c; +} + +static inline uint32_t set_line(uint32_t round, uint32_t num_threads, uint32_t gid) { + return round * num_threads + gid; +} + +// Expected word `word_idx` of the same-set buffer once the kernel has run: +// each line's word 0 is stored, the rest keeps the pre-pattern. +static inline uint32_t expected_set_word(uint32_t word_idx) { + if ((word_idx % SET_STRIDE_WORDS) == 0) { + return 0xA0000000u | ((word_idx / SET_STRIDE_WORDS) << 4); + } + return pattern_word(word_idx); +} + +// Owner thread of a CASE_WARP region writes it when it sits in an even warp +// of its block; the thread one warp up reads it. +static inline bool warp_producer(uint32_t gid, uint32_t warp_size) { + return ((gid / warp_size) & 1) == 0; +} + +// Expected content of word w of region `region` once the kernel has run. +static inline uint32_t expected_word(uint32_t region, uint32_t w, const kernel_arg_t* arg) { + uint32_t p = pattern_word(region * REGION_WORDS + w); + uint32_t v = store_value(region, w); + switch (region % NUM_CASES) { + case CASE_HALF: + return (w < REGION_WORDS / 2) ? v : p; + case CASE_BYTES: + if (w == 1) { + return (p & ~0x0000ff00u) | 0x0000ee00u; + } + if (w == 10) { + return (p & ~0x0000ffffu) | 0x0000beefu; + } + return p; + case CASE_AMO: + if (w < 4 && !arg->amo) { + return v; + } + if (!arg->amo) { + return p; + } + if (w == 1) { + return v + 3; + } + if (w < 4) { + return v; + } + if (w == 8) { + return p + 5; + } + if (w == 12) { + return p + 1; + } + return p; + case CASE_STORE: + return ((w & 1) == 0) ? ((p & ~0xffu) | (0x30u + w)) : p; + case CASE_WARP: { + uint32_t gid = (region / NUM_CASES) % arg->num_threads; + if (arg->cross_warp && warp_producer(gid, arg->warp_size) && w >= REGION_WORDS / 2) { + return v; + } + return p; + } + default: { + // the region of each warp's lane 0 is shared by the warp's first lanes + uint32_t gid = (region / NUM_CASES) % arg->num_threads; + uint32_t lanes = (arg->warp_size < LANE_WORDS) ? arg->warp_size : LANE_WORDS; + if ((gid % arg->warp_size) == 0 && w < lanes) { + return v; + } + return p; + } + } +} + +#endif diff --git a/tests/regression/lazy_fetch/kernel.cpp b/tests/regression/lazy_fetch/kernel.cpp new file mode 100644 index 0000000000..5f0dc5e4cd --- /dev/null +++ b/tests/regression/lazy_fetch/kernel.cpp @@ -0,0 +1,174 @@ +#include +#include +#include "common.h" + +// Every access goes through volatile pointers: the point is to send each load +// to the cache right behind the stores it must observe, not to let the +// compiler forward the stored value. +static inline volatile uint32_t* region_ptr(const kernel_arg_t* arg, uint32_t region) { + return reinterpret_cast(arg->buf_addr) + region * REGION_WORDS; +} + +static inline uint32_t check_word(volatile uint32_t* r, uint32_t region, uint32_t w, + const kernel_arg_t* arg) { + uint32_t ref = expected_word(region, w, arg); + return (r[w] != ref) ? 1 : 0; +} + +static uint32_t run_half(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + for (uint32_t w = 0; w < REGION_WORDS / 2; ++w) { + r[w] = store_value(region, w); + } + uint32_t errors = 0; + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + errors += check_word(r, region, w, arg); + } + return errors; +} + +static uint32_t run_bytes(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + reinterpret_cast(r)[5] = 0xee; + uint32_t errors = check_word(r, region, 1, arg); + reinterpret_cast(r)[20] = 0xbeef; + errors += check_word(r, region, 10, arg); + errors += check_word(r, region, 0, arg); + errors += check_word(r, region, 15, arg); + return errors; +} + +static uint32_t run_amo(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + for (uint32_t w = 0; w < 4; ++w) { + r[w] = store_value(region, w); + } + if (!arg->amo) { + return check_word(r, region, 8, arg) + check_word(r, region, 1, arg); + } + uint32_t errors = 0; + uint32_t* word8 = const_cast(&r[8]); + uint32_t* word1 = const_cast(&r[1]); + uint32_t* word12 = const_cast(&r[12]); + uint32_t old8 = __atomic_fetch_add(word8, 5u, __ATOMIC_SEQ_CST); + errors += (old8 != pattern_word(region * REGION_WORDS + 8)) ? 1 : 0; + uint32_t old1 = __atomic_fetch_add(word1, 3u, __ATOMIC_SEQ_CST); + errors += (old1 != store_value(region, 1)) ? 1 : 0; + // Only this thread touches the word, so the SC fails only spuriously. + uint32_t first = 0; + bool done = false; + for (uint32_t attempt = 0; attempt < 64 && !done; ++attempt) { + uint32_t old, sc_fail; + asm volatile ("lr.w %0, (%1)" : "=r"(old) : "r"(word12) : "memory"); + if (attempt == 0) { + first = old; + } + uint32_t newv = old + 1; + asm volatile ("sc.w %0, %2, (%1)" : "=r"(sc_fail) : "r"(word12), "r"(newv) : "memory"); + done = (sc_fail == 0); + } + errors += (!done || first != pattern_word(region * REGION_WORDS + 12)) ? 1 : 0; + return errors; +} + +static void run_store(const kernel_arg_t* arg, uint32_t region) { + auto b = reinterpret_cast(region_ptr(arg, region)); + for (uint32_t w = 0; w < REGION_WORDS; w += 2) { + b[w * 4] = (uint8_t)(0x30 + w); + } +} + +// Each lane's load follows the warp's coalesced store into the same line +// right away, so it reaches the cache while the store's allocation is still +// pending and must wait for the unwritten half to be fetched. +static uint32_t run_lane(const kernel_arg_t* arg, uint32_t round, uint32_t gid) { + uint32_t lane = gid % arg->warp_size; + uint32_t lanes = (arg->warp_size < LANE_WORDS) ? arg->warp_size : LANE_WORDS; + if (lane >= lanes) { + return 0; + } + uint32_t region = region_index(round, arg->num_threads, gid - lane, CASE_LANE); + auto r = region_ptr(arg, region); + uint32_t value = store_value(region, lane); + uint32_t loaded; + asm volatile ("sw %2, 0(%1)\n\tlw %0, %3(%1)" + : "=&r"(loaded) + : "r"(&r[lane]), "r"(value), "i"(LANE_WORDS * 4) + : "memory"); + uint32_t errors = (loaded != expected_word(region, lane + LANE_WORDS, arg)) ? 1 : 0; + errors += check_word(r, region, (lane + 1) % lanes, arg); + return errors; +} + +// Store word 0 of this thread's same-set line, then load word 8 right away. +static uint32_t run_set(const kernel_arg_t* arg, uint32_t round, uint32_t gid) { + uint32_t line = set_line(round, arg->num_threads, gid); + auto r = reinterpret_cast(arg->set_addr) + line * SET_STRIDE_WORDS; + uint32_t value = expected_set_word(line * SET_STRIDE_WORDS); + uint32_t loaded; + asm volatile ("sw %2, 0(%1)\n\tlw %0, 32(%1)" + : "=&r"(loaded) + : "r"(r), "r"(value) + : "memory"); + uint32_t errors = (loaded != expected_set_word(line * SET_STRIDE_WORDS + 8)) ? 1 : 0; + errors += (r[0] != value) ? 1 : 0; + return errors; +} + +static uint32_t check_region(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + uint32_t errors = 0; + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + errors += check_word(r, region, w, arg); + } + return errors; +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t errors = 0; + auto err = reinterpret_cast(arg->err_addr); + + if (arg->phase != 0) { + // Every region again, from another thread (another core when there are + // several), after the launch boundary flushed the partial lines. + uint32_t owner = (gid + arg->rotate) % arg->num_threads; + for (uint32_t round = 0; round < arg->rounds; ++round) { + for (uint32_t c = 0; c < NUM_CASES; ++c) { + errors += check_region(arg, region_index(round, arg->num_threads, owner, c)); + } + auto r = reinterpret_cast(arg->set_addr); + uint32_t base = set_line(round, arg->num_threads, owner) * SET_STRIDE_WORDS; + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + errors += (r[base + w] != expected_set_word(base + w)) ? 1 : 0; + } + } + err[arg->num_threads + gid] = errors; + return; + } + + for (uint32_t round = 0; round < arg->rounds; ++round) { + errors += run_half(arg, region_index(round, arg->num_threads, gid, CASE_HALF)); + errors += run_bytes(arg, region_index(round, arg->num_threads, gid, CASE_BYTES)); + errors += run_amo(arg, region_index(round, arg->num_threads, gid, CASE_AMO)); + run_store(arg, region_index(round, arg->num_threads, gid, CASE_STORE)); + errors += run_lane(arg, round, gid); + errors += run_set(arg, round, gid); + if (arg->cross_warp) { + bool producer = warp_producer(gid, arg->warp_size); + if (producer) { + uint32_t region = region_index(round, arg->num_threads, gid, CASE_WARP); + auto r = region_ptr(arg, region); + for (uint32_t w = REGION_WORDS / 2; w < REGION_WORDS; ++w) { + r[w] = store_value(region, w); + } + } + vx_fence(); + __syncthreads(); + if (!producer) { + errors += check_region(arg, region_index(round, arg->num_threads, gid - arg->warp_size, CASE_WARP)); + } + } + } + err[gid] = errors; +} diff --git a/tests/regression/lazy_fetch/main.cpp b/tests/regression/lazy_fetch/main.cpp new file mode 100644 index 0000000000..60734b7711 --- /dev/null +++ b/tests/regression/lazy_fetch/main.cpp @@ -0,0 +1,217 @@ +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.vxbin"; +uint32_t rounds = 16; + +vx_device_h device = nullptr; +vx_buffer_h buf_buffer = nullptr; +vx_buffer_h err_buffer = nullptr; +vx_buffer_h set_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n rounds] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h")) != -1) { + switch (c) { + case 'n': + rounds = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + show_usage(); + exit(0); + break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (buf_buffer) vx_buffer_release(buf_buffer); + if (err_buffer) vx_buffer_release(err_buffer); + if (set_buffer) vx_buffer_release(set_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); + vx_device_release(device); + } +} + +static int launch(const kernel_arg_t& arg, uint32_t grid, uint32_t block) { + vx_event_h ev = nullptr; + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &arg; + li.args_size = sizeof(arg); + li.ndim = 1; + li.grid_dim[0] = grid; + li.block_dim[0] = block; + int ret = vx_enqueue_launch(queue, &li, 0, nullptr, &ev); + if (ret != 0) { + return ret; + } + ret = vx_event_wait_value(ev, 1, VX_TIMEOUT_INFINITE); + vx_event_release(ev); + return ret; +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + if (rounds == 0) { + rounds = 1; + } + + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_device_open(0, &device)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_cores, num_warps, num_threads, isa_flags; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + RT_CHECK(vx_device_query(device, VX_CAPS_ISA_FLAGS, &isa_flags)); + + // One block per core, holding all of its warps. + uint32_t block = num_warps * num_threads; + uint32_t grid = num_cores; + uint32_t total_threads = grid * block; + uint32_t num_regions = rounds * total_threads * NUM_CASES; + uint32_t num_words = num_regions * REGION_WORDS; + uint32_t buf_size = num_words * sizeof(uint32_t); + uint32_t err_size = 2 * total_threads * sizeof(uint32_t); + uint32_t set_words = rounds * total_threads * SET_STRIDE_WORDS; + uint32_t set_size = set_words * sizeof(uint32_t); + + kernel_arg_t arg = {}; + arg.rounds = rounds; + arg.num_threads = total_threads; + arg.warp_size = num_threads; + arg.cross_warp = (num_warps % 2) == 0; + arg.amo = (isa_flags & VX_ISA_STD_A) != 0; + + std::cout << "threads: " << total_threads << ", amo: " << arg.amo << ", regions: " << num_regions + << ", buffer size: " << buf_size << " bytes" << std::endl; + + RT_CHECK(vx_buffer_create(device, buf_size, VX_MEM_READ_WRITE, &buf_buffer)); + RT_CHECK(vx_buffer_address(buf_buffer, &arg.buf_addr)); + RT_CHECK(vx_buffer_create(device, err_size, VX_MEM_READ_WRITE, &err_buffer)); + RT_CHECK(vx_buffer_address(err_buffer, &arg.err_addr)); + RT_CHECK(vx_buffer_create(device, set_size, VX_MEM_READ_WRITE, &set_buffer)); + RT_CHECK(vx_buffer_address(set_buffer, &arg.set_addr)); + + std::vector h_buf(buf_size); + for (uint32_t i = 0; i < buf_size; ++i) { + h_buf[i] = pattern_byte(i); + } + std::vector h_err(2 * total_threads, 0xdeadbeef); + std::vector h_set(set_words); + for (uint32_t i = 0; i < set_words; ++i) { + h_set[i] = pattern_word(i); + } + + std::cout << "upload buffers" << std::endl; + RT_CHECK(vx_enqueue_write(queue, buf_buffer, 0, h_buf.data(), buf_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, err_buffer, 0, h_err.data(), err_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, set_buffer, 0, h_set.data(), set_size, 0, nullptr, nullptr)); + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + std::cout << "launch kernel (exercise)" << std::endl; + arg.phase = 0; + RT_CHECK(launch(arg, grid, block)); + + std::cout << "launch kernel (re-check)" << std::endl; + arg.phase = 1; + arg.rotate = (num_cores > 1) ? block : num_threads; + RT_CHECK(launch(arg, grid, block)); + + std::cout << "download buffers" << std::endl; + vx_event_h read_ev = nullptr; + RT_CHECK(vx_enqueue_read(queue, h_buf.data(), buf_buffer, 0, buf_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_read(queue, h_set.data(), set_buffer, 0, set_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_read(queue, h_err.data(), err_buffer, 0, err_size, 0, nullptr, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + + std::cout << "verify result" << std::endl; + int errors = 0; + for (uint32_t t = 0; t < 2 * total_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::cout << "error: thread " << (t % total_threads) << (t < total_threads ? " (exercise)" : " (re-check)") + << " reported 0x" << std::hex << h_err[t] << std::dec << " mismatches" << std::endl; + } + ++errors; + } + } + auto words = reinterpret_cast(h_buf.data()); + for (uint32_t region = 0; region < num_regions; ++region) { + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + uint32_t ref = expected_word(region, w, &arg); + uint32_t cur = words[region * REGION_WORDS + w]; + if (cur != ref) { + if (errors < 16) { + std::cout << "error: region " << region << " (case " << (region % NUM_CASES) << ") word " << w + << std::hex << ": actual 0x" << cur << ", expected 0x" << ref << std::dec << std::endl; + } + ++errors; + } + } + } + + for (uint32_t i = 0; i < set_words; ++i) { + uint32_t ref = expected_set_word(i); + if (h_set[i] != ref) { + if (errors < 16) { + std::cout << "error: same-set word " << i << std::hex << ": actual 0x" << h_set[i] + << ", expected 0x" << ref << std::dec << std::endl; + } + ++errors; + } + } + + cleanup(); + + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + + std::cout << "PASSED!" << std::endl; + return 0; +} From e5872047882d0b81850af10a7cd8816d04f1e589 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:03 -0700 Subject: [PATCH 036/336] core: decouple fetch width from issue width (VX_CFG_FETCH_WIDTH) The core fetched for every issue slice through one scheduler and one icache port, so front-end bandwidth and the scheduler's size grew with ISSUE_WIDTH as a single block. FETCH_WIDTH (FW, a power of two dividing ISSUE_WIDTH) splits the front end into FW fetch slices. Slice f owns the warps with wid % FW == f and has its own warp scheduler, fetch, decode and icache port. RTL structure: - VX_fetch_slice = VX_warp_sched -> VX_fetch -> VX_decode. - VX_scheduler stays core-wide. It holds CTA dispatch, with one launch lane per fetch slice, VX_warp_shared_unit for barriers, wspawn and warp retirement, and the per-core state. - VX_fetch_xbar is purely combinational and routes decoded instructions, ibuffer pops, warp control, CSRs and drain status between fetch and compute partitions. - When every execution unit has one block per issue slice, the core is built from VX_compute_slice (issue + execute + commit). Otherwise it keeps the stage-sliced structure. The unit blocks (ALU/LSU/SFU/FPU/TCU) are separate modules so both structures share them. SimX mirrors the structure: FW per-slice fetch and decode latches, FW icache ports and banks, and FW CTA launch lanes. The init pause on a block-shape change is modelled when FW > 1. Where the split put a register on a path master kept combinational, master's timing is restored: - all_ibuf_full is built from next-state ibuffer flags. - Warp control registers each shared-unit request once, and the unit acts on the granted request directly. Exit and wspawn therefore reach the scheduler in the same cycle as on master. - Exits, wspawns, sync barriers and barrier waits are posted. Only a barrier arrive, which writes back the phase, waits for the unit's response. Previously every barrier held warp control for about five cycles, which spread warps' barrier release five cycles apart. - An exit retires its warp on warp_done, and the CTA dispatcher frees the slot through the slice's registered active bit, as on master. - Core busy also counts cta_fire. This closes a one-cycle idle hole that let a CTA of at most FW warps end the kernel early. Warp control acts on a granted barrier request in the same cycle (SH_DIRECT), so with FW = 1 a barrier releases on the same cycle as before the split. Fixes that come with the split: - The global-barrier core index is taken modulo the cluster's core count at full width. The truncated modulo divided by zero at 2 and 4 cores and deadlocked gbar_phase at 1 cluster x 4 cores (CI: gbar-phase-3). - SATP is a live CSR, updated by a read-modify-write in the fire cycle, and the per-block CSR requests are ordered. With two SFU blocks, a page-table switch could otherwise race a translation (CI: vm-sfu2). - The per-CTA context RAMs live in VX_scheduler, with one read port per SFU block. - Static asserts on FETCH_WIDTH, the lane dispatcher shape, and the DXA in-flight limit. CI: several config1/dxa/tensor/tensor_wg rtlsim cases now also run with FETCH_WIDTH = 2 or 4. RTL at ISSUE_WIDTH = 4, 16 warps x 16 threads, FW = 1 -> 2: sgemm 103,793 -> 70,957 cycles (-31.6%), raycast 307,307 -> 299,006 (-2.7%). At the default config (FW = 1), rtlsim moves by at most +132 cycles (printf); sgemm +9, raycast +55. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 6 +- ci/testcases/amo.yaml | 2 +- ci/testcases/config1.yaml | 14 +- ci/testcases/dxa.yaml | 4 +- ci/testcases/regression.yaml | 8 + ci/testcases/tensor.yaml | 6 +- ci/testcases/tensor_wg.yaml | 13 +- ci/testcases/vm.yaml | 9 + docs/assets/img/amo_architecture.svg | 2 +- docs/assets/img/lsu_architecture.svg | 6 +- docs/designs/atomic_memory_operations.md | 2 +- docs/designs/lsu_pipeline_design.md | 6 +- docs/designs/output_merger_architecture.md | 4 +- hw/rtl/VX_cluster.sv | 10 +- hw/rtl/VX_gpu_pkg.sv | 207 ++++- hw/rtl/VX_graphics.sv | 24 +- hw/rtl/VX_socket.sv | 27 +- hw/rtl/Vortex.sv | 2 +- hw/rtl/core/VX_alu_block.sv | 93 +++ hw/rtl/core/VX_alu_int.sv | 8 +- hw/rtl/core/VX_alu_unit.sv | 70 +- hw/rtl/core/VX_bar_unit.sv | 9 +- hw/rtl/core/VX_commit.sv | 131 +-- hw/rtl/core/VX_commit_slice.sv | 124 +++ hw/rtl/core/VX_compute_slice.sv | 192 +++++ hw/rtl/core/VX_core.sv | 767 ++++++++++++------ hw/rtl/core/VX_csr_cta.sv | 75 ++ hw/rtl/core/VX_csr_data.sv | 317 +------- hw/rtl/core/VX_csr_fpu.sv | 103 +++ hw/rtl/core/VX_csr_mpm.sv | 257 ++++++ hw/rtl/core/VX_csr_unit.sv | 49 +- hw/rtl/core/VX_cta_dispatch.sv | 616 ++++++++------ hw/rtl/core/VX_dcr_data.sv | 9 +- hw/rtl/core/VX_decompressor.sv | 66 +- hw/rtl/core/VX_dispatcher.sv | 13 +- hw/rtl/core/VX_execute.sv | 43 +- hw/rtl/core/VX_execute_slice.sv | 401 +++++++++ hw/rtl/core/VX_fetch.sv | 48 +- hw/rtl/core/VX_fetch_slice.sv | 149 ++++ hw/rtl/core/VX_fetch_xbar.sv | 229 ++++++ hw/rtl/core/VX_ibuffer.sv | 4 +- hw/rtl/core/VX_ipdom_stack.sv | 24 +- hw/rtl/core/VX_issue.sv | 22 +- hw/rtl/core/VX_issue_slice.sv | 18 +- hw/rtl/core/VX_lane_dispatch.sv | 32 +- hw/rtl/core/VX_lane_gather.sv | 32 +- hw/rtl/core/VX_lsu_agu.sv | 2 +- .../core/{VX_lsu_slice.sv => VX_lsu_block.sv} | 13 +- hw/rtl/core/VX_lsu_unit.sv | 14 +- hw/rtl/core/VX_operands.sv | 4 +- hw/rtl/core/VX_scheduler.sv | 751 ++++++----------- hw/rtl/core/VX_scoreboard.sv | 4 +- hw/rtl/core/VX_sfu_block.sv | 231 ++++++ hw/rtl/core/VX_sfu_shared.sv | 156 ++++ hw/rtl/core/VX_sfu_unit.sv | 305 ++----- hw/rtl/core/VX_split_join.sv | 20 +- hw/rtl/core/VX_uop_packld.sv | 2 +- hw/rtl/core/VX_uuid_gen.sv | 28 +- hw/rtl/core/VX_warp_sched.sv | 514 ++++++++++++ hw/rtl/core/VX_warp_shared_unit.sv | 268 ++++++ hw/rtl/core/VX_wctl_unit.sv | 150 +++- hw/rtl/dxa/VX_dxa_unit.sv | 10 +- hw/rtl/dxa/VX_dxa_worker.sv | 2 + hw/rtl/fpu/VX_fpu_block.sv | 261 ++++++ hw/rtl/fpu/VX_fpu_unit.sv | 231 +----- hw/rtl/interfaces/VX_sched_csr_if.sv | 16 +- hw/rtl/interfaces/VX_slice_sched_if.sv | 151 ++++ hw/rtl/interfaces/VX_warp_ctl_if.sv | 63 +- hw/rtl/interfaces/VX_warp_shared_if.sv | 44 + hw/rtl/om/VX_om_core.sv | 8 +- hw/rtl/om/VX_om_mem.sv | 2 +- hw/rtl/om/VX_om_steer.sv | 2 +- hw/rtl/raster/VX_raster_be.sv | 2 +- hw/rtl/raster/VX_raster_core.sv | 15 +- hw/rtl/raster/VX_raster_earlyz.sv | 2 +- hw/rtl/raster/VX_raster_mem.sv | 9 +- hw/rtl/raster/VX_raster_slice.sv | 4 +- hw/rtl/rtu/VX_rtu_unit.sv | 65 +- hw/rtl/tcu/VX_tcu_block.sv | 121 +++ hw/rtl/tcu/VX_tcu_core.sv | 4 +- hw/rtl/tcu/VX_tcu_meta.sv | 16 +- hw/rtl/tcu/VX_tcu_shared.sv | 118 +++ hw/rtl/tcu/VX_tcu_unit.sv | 241 ++---- hw/rtl/tex/VX_tex_core.sv | 8 +- hw/rtl/tex/VX_tex_mem.sv | 2 +- hw/rtl/tex/VX_tex_unit.sv | 2 - hw/syn/firesim/Makefile | 2 +- hw/syn/xilinx/aved/platforms.mk | 2 +- hw/unittest/core/VX_core_top.sv | 60 +- .../cta_dispatcher/VX_cta_dispatch_top.sv | 23 +- hw/unittest/gfx/VX_gfx_top.sv | 3 +- hw/unittest/issue/VX_issue_top.sv | 18 +- hw/unittest/raster_core/VX_raster_core_top.sv | 2 +- sim/firesim/firesim_sim.cpp | 2 +- sim/simx/constants.h | 9 + sim/simx/core.cpp | 166 ++-- sim/simx/cta_dispatcher.cpp | 96 ++- sim/simx/cta_dispatcher.h | 20 +- sim/simx/dxa/dxa_core.cpp | 4 + sim/simx/dxa/dxa_unit.cpp | 2 +- sim/simx/dxa/dxa_unit.h | 7 +- sim/simx/lsu_unit.cpp | 2 +- sim/simx/rtu/rtu_unit.cpp | 7 +- sim/simx/rtu/rtu_unit.h | 5 +- sim/simx/scheduler.cpp | 54 +- sim/simx/scheduler.h | 21 +- sim/simx/sfu_unit.cpp | 83 +- sim/simx/sfu_unit.h | 7 + sim/simx/socket.cpp | 10 +- sim/simx/tex/tex_unit.cpp | 2 +- sim/simx/tex/tex_unit.h | 7 +- sim/simx/wctl_unit.cpp | 37 + sim/simx/wctl_unit.h | 5 + 113 files changed, 6040 insertions(+), 2730 deletions(-) create mode 100644 hw/rtl/core/VX_alu_block.sv create mode 100644 hw/rtl/core/VX_commit_slice.sv create mode 100644 hw/rtl/core/VX_compute_slice.sv create mode 100644 hw/rtl/core/VX_csr_cta.sv create mode 100644 hw/rtl/core/VX_csr_fpu.sv create mode 100644 hw/rtl/core/VX_csr_mpm.sv create mode 100644 hw/rtl/core/VX_execute_slice.sv create mode 100644 hw/rtl/core/VX_fetch_slice.sv create mode 100644 hw/rtl/core/VX_fetch_xbar.sv rename hw/rtl/core/{VX_lsu_slice.sv => VX_lsu_block.sv} (98%) create mode 100644 hw/rtl/core/VX_sfu_block.sv create mode 100644 hw/rtl/core/VX_sfu_shared.sv create mode 100644 hw/rtl/core/VX_warp_sched.sv create mode 100644 hw/rtl/core/VX_warp_shared_unit.sv create mode 100644 hw/rtl/fpu/VX_fpu_block.sv create mode 100644 hw/rtl/interfaces/VX_slice_sched_if.sv create mode 100644 hw/rtl/interfaces/VX_warp_shared_if.sv create mode 100644 hw/rtl/tcu/VX_tcu_block.sv create mode 100644 hw/rtl/tcu/VX_tcu_shared.sv diff --git a/VX_config.toml b/VX_config.toml index 9ccfef0cc5..2646ef048a 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -52,9 +52,11 @@ VX_CFG_MAX_BAR_EVENTS = 32 VX_CFG_IBUF_SIZE = 4 VX_CFG_DISPATCH_QUEUE_SIZE = 4 VX_CFG_ISSUE_WIDTH = "expr: up($VX_CFG_NUM_WARPS / 16)" +# Fetch slices, each scheduling and fetching its own warps on its own icache +# port: a power of two that divides ISSUE_WIDTH. +VX_CFG_FETCH_WIDTH = 1 VX_CFG_SIMD_WIDTH = "expr: $VX_CFG_NUM_THREADS" VX_CFG_NUM_OPCS = "expr: up($VX_CFG_NUM_WARPS / (4 * $VX_CFG_ISSUE_WIDTH))" - VX_CFG_NUM_GPR_BANKS = 4 VX_CFG_NUM_VGPR_BANKS = 2 @@ -207,6 +209,8 @@ VX_CFG_ICACHE_MRSQ_SIZE = 0 VX_CFG_ICACHE_MREQ_SIZE = 0 VX_CFG_ICACHE_CRSQ_SIZE = 0 VX_CFG_ICACHE_MEM_PORTS = 1 +# banks behind the fetch-slice request ports +VX_CFG_ICACHE_NUM_BANKS = "expr: pow(2, clog2($VX_CFG_FETCH_WIDTH))" VX_CFG_ICACHE_LATENCY = "expr: 2 + max(0, clog2($VX_CFG_ICACHE_SIZE) - clog2(16384))" VX_CFG_DCACHE_SIZE = 16384 diff --git a/ci/testcases/amo.yaml b/ci/testcases/amo.yaml index de2b638690..8ce2da3fbd 100644 --- a/ci/testcases/amo.yaml +++ b/ci/testcases/amo.yaml @@ -7,7 +7,7 @@ defaults: configs: "-DVX_CFG_EXT_A_ENABLE" xlen: [32, 64] tier: smoke - touches: [hw/rtl/cache, hw/rtl/mem, hw/rtl/core/VX_decode.sv, hw/rtl/core/VX_lsu_slice.sv, sim/simx/amo, sim/simx/mem] + touches: [hw/rtl/cache, hw/rtl/mem, hw/rtl/core/VX_decode.sv, hw/rtl/core/VX_lsu_block.sv, sim/simx/amo, sim/simx/mem] tests: # RISC-V 'A' ISA conformance (rv{XLEN}ua-p-{amo*,lrsc}); shares the A-built sim. - id: isa-conformance diff --git a/ci/testcases/config1.yaml b/ci/testcases/config1.yaml index feab45fba1..cb2d49aa71 100644 --- a/ci/testcases/config1.yaml +++ b/ci/testcases/config1.yaml @@ -179,13 +179,13 @@ tests: drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=2 + configs: -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 - id: diverge-18 via: blackbox drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: diverge-19 via: blackbox drivers: @@ -254,13 +254,13 @@ tests: drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=2 -DVX_CFG_NUM_ALU_LANES=2 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=2 -DVX_CFG_NUM_ALU_LANES=2 -DVX_CFG_FETCH_WIDTH=2 - id: diverge-23 via: blackbox drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=4 -DVX_CFG_NUM_ALU_LANES=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=4 -DVX_CFG_NUM_ALU_LANES=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: diverge-24 via: blackbox drivers: @@ -284,13 +284,13 @@ tests: drivers: - rtlsim app: regression/vecadd - configs: -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_FPU_LANES=2 + configs: -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_FPU_LANES=2 -DVX_CFG_FETCH_WIDTH=2 - id: regression/vecadd-2 via: blackbox drivers: - rtlsim app: regression/vecadd - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_FPU_BLOCKS=4 -DVX_CFG_NUM_FPU_LANES=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_FPU_BLOCKS=4 -DVX_CFG_NUM_FPU_LANES=4 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: regression/vecadd-3 via: blackbox drivers: @@ -349,7 +349,7 @@ tests: drivers: - rtlsim app: regression/vecadd - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_LSU_BLOCKS=4 -DVX_CFG_NUM_LSU_LANES=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_LSU_BLOCKS=4 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: regression/vecadd-7 via: blackbox drivers: diff --git a/ci/testcases/dxa.yaml b/ci/testcases/dxa.yaml index 8cad41a2b3..913c71cb57 100644 --- a/ci/testcases/dxa.yaml +++ b/ci/testcases/dxa.yaml @@ -388,7 +388,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_dxa - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_TCU_TYPE=DPI -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_TCU_TYPE=DPI -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_FETCH_WIDTH=2 shape: threads: 32 warps: 8 @@ -419,7 +419,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_sp_dxa - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI -DVX_CFG_FETCH_WIDTH=4 -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 shape: threads: 32 diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 0d4ad00450..af35816d28 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -90,6 +90,14 @@ tests: app: gbar_phase shape: cores: 2 +# Four cores in one cluster: every core must arrive under its own index. +- id: gbar-phase-3 + via: blackbox + drivers: + - xrt + app: gbar_phase + shape: + cores: 4 # SimX<->rtlsim cycle parity on the multi-core global barrier. Until this case # existed the gate could not observe the gbar path at all: every other parity # case is single-core, so USE_GBAR is 0 and the request queue is not built. diff --git a/ci/testcases/tensor.yaml b/ci/testcases/tensor.yaml index 062a8dd331..246774db01 100644 --- a/ci/testcases/tensor.yaml +++ b/ci/testcases/tensor.yaml @@ -123,13 +123,13 @@ tests: drivers: - rtlsim app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=2 -DITYPE=fp16 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=2 -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 - id: sgemm_tcu-19 via: blackbox drivers: - rtlsim app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=4 -DITYPE=fp16 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=4 -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=4 -DPROFILE_ENABLE - id: sgemm_tcu-20 via: blackbox @@ -148,7 +148,7 @@ tests: drivers: - rtlsim app: sgemm2_tcu - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 -DITYPE=fp16 -DOTYPE=fp32 args: -w 4 diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index a08f6cc835..b9bb956479 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -44,14 +44,14 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg-6 via: blackbox drivers: - rtlsim app: sgemm_tcu_wg - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=4 -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg-7 via: blackbox @@ -72,7 +72,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg-10 via: blackbox @@ -116,7 +116,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_sp - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg_sp-5 via: blackbox @@ -130,7 +130,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_sp - configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg_sp-7 via: blackbox @@ -232,8 +232,9 @@ tests: args: -m64 -n64 -k64 # One execution block per unit: the four-block model ISSUE_WIDTH=4 implies is # several times the size, and a hosted CI runner cannot build it in a job. + # Two fetch slices: covers the fetch-slice front end on both models. configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - -DVX_CFG_TCU_FEDP2K -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS + -DVX_CFG_TCU_FEDP2K -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS -DVX_CFG_FETCH_WIDTH=2 # Cooperative sparse WGMMA (default fp16 operands, 2:4 structured sparsity). # Not warp-width swept: the tensor-core fragment layout is defined over a fixed diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index 2c4d2b7c2b..6f98894b00 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -235,6 +235,15 @@ tests: # above) are reported, via the shared walker. Re-enable when L1 permission-fault # reporting is added. +# Two SFU blocks: every warp writes SATP at entry, from either block. +- id: vm-sfu2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + configs: -DVX_CFG_VM_ENABLE -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 + # --------------------------------------------------------------------------- # check: model_parity — same VM-enabled app on both drivers (xlen 32); exact # instret + cycles within tolerance. SimX is the timing model of the RTL. diff --git a/docs/assets/img/amo_architecture.svg b/docs/assets/img/amo_architecture.svg index d95de9c2cf..6751a3d332 100644 --- a/docs/assets/img/amo_architecture.svg +++ b/docs/assets/img/amo_architecture.svg @@ -44,7 +44,7 @@ aq / rl are captured and carry no behavior - VX_lsu_slice — per-lane sideband + VX_lsu_block — per-lane sideband packs amo_req_t into each lane's mem_bus_attr_t: hart_id {core, warp, thread} amo_cmp comparand (Zacas only) diff --git a/docs/assets/img/lsu_architecture.svg b/docs/assets/img/lsu_architecture.svg index c173c657b1..ce3db01776 100644 --- a/docs/assets/img/lsu_architecture.svg +++ b/docs/assets/img/lsu_architecture.svg @@ -30,7 +30,7 @@ VX_lsu_unit execute stage - one VX_lsu_slice per + one VX_lsu_block per LSU block (the issue width) @@ -42,7 +42,7 @@ becomes packets: pid, sop, eop - VX_lsu_slice — request side + VX_lsu_block — request side VX_lsu_agu base + offset, or @@ -68,7 +68,7 @@ tag = header, operation, alignment, slot, fence - VX_lsu_slice — response side + VX_lsu_block — response side load formatter sub-word extract by the saved diff --git a/docs/designs/atomic_memory_operations.md b/docs/designs/atomic_memory_operations.md index 065ff80f0c..3388132790 100644 --- a/docs/designs/atomic_memory_operations.md +++ b/docs/designs/atomic_memory_operations.md @@ -90,7 +90,7 @@ LSU as `rs3_data` with no new operand path. ## 3. The LSU sideband -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv) packs one `amo_req_t` +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv) packs one `amo_req_t` per lane into the lane's memory attribute ([`VX_gpu_pkg.sv`](../../hw/rtl/VX_gpu_pkg.sv)): diff --git a/docs/designs/lsu_pipeline_design.md b/docs/designs/lsu_pipeline_design.md index 7ba07a2124..025c834d9f 100644 --- a/docs/designs/lsu_pipeline_design.md +++ b/docs/designs/lsu_pipeline_design.md @@ -6,7 +6,7 @@ interleave, per-lane classification and formatting, fence ordering, multi-packet tracking, the shared memory scheduler and its outstanding pool, address-class steering, coalescing, and the response path. Covers the RTL ([`VX_lsu_unit.sv`](../../hw/rtl/core/VX_lsu_unit.sv), -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv), +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv), [`VX_lsu_agu.sv`](../../hw/rtl/core/VX_lsu_agu.sv), [`VX_lsu_scheduler.sv`](../../hw/rtl/core/VX_lsu_scheduler.sv), [`VX_mem_scheduler.sv`](../../hw/rtl/libs/VX_mem_scheduler.sv), @@ -104,7 +104,7 @@ that is provably equivalent. ## 3. The slice — request side -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv). +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv). ### 3.1 Classification @@ -208,7 +208,7 @@ library block also used by the caches' other clients. | client | source | present | |---|---|---| -| 0 | the block's `VX_lsu_slice` | always | +| 0 | the block's `VX_lsu_block` | always | | 1 | the tensor unit's warp-level address generator | on block 0, when the tensor metadata path is enabled | Arbitration is round-robin, and the granted client's index is prepended to diff --git a/docs/designs/output_merger_architecture.md b/docs/designs/output_merger_architecture.md index 69a4ecff43..a8aeacb10d 100644 --- a/docs/designs/output_merger_architecture.md +++ b/docs/designs/output_merger_architecture.md @@ -8,7 +8,7 @@ and deadlock arguments, and the parallelism/scaling model. Covers the RTL ([`hw/rtl/om/`](../../hw/rtl/om/)), the SIMT-side plumbing ([`VX_decode.sv`](../../hw/rtl/core/VX_decode.sv), [`VX_gfx_uops.sv`](../../hw/rtl/core/VX_gfx_uops.sv), -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv)), the cluster +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv)), the cluster integration ([`VX_graphics.sv`](../../hw/rtl/VX_graphics.sv)), the SimX model ([`sim/simx/om/`](../../sim/simx/om/)), and the software contract ([`sw/kernel/include/vx_graphics.h`](../../sw/kernel/include/vx_graphics.h), @@ -170,7 +170,7 @@ byte-enables splitting the write (`byteen[2:0]` = depth writemask, ### 4.1 LSU attribute tagging -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv) range-checks every +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv) range-checks every lane address against the aperture and sets **two** attribute bits: `is_addr_om` (the steer's select) and `is_addr_io` (alongside it). The IO bit is not a hack — *uncached, bypassed, posted* is exactly the behaviour diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index be46c3f4e4..6204d4ee88 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -15,7 +15,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter CLUSTER_ID = 0, parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -24,6 +23,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; input wire clk, input wire reset, + input wire [CLUSTER_ID_WIDTH-1:0] cluster_id, + `ifdef PERF_ENABLE input sysmem_perf_t sysmem_perf, `endif @@ -169,7 +170,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; ); VX_gbar_unit #( - .INSTANCE_ID (`SFORMATF(("gbar%0d", CLUSTER_ID))) + .INSTANCE_ID (`SFORMATF(("%s-gbar", INSTANCE_ID))) ) gbar_unit ( .clk (clk), .reset (reset), @@ -373,13 +374,13 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; for (genvar socket_id = 0; socket_id < NUM_SOCKETS; ++socket_id) begin : g_sockets VX_socket #( - .SOCKET_ID ((CLUSTER_ID * NUM_SOCKETS) + socket_id), .INSTANCE_ID (`SFORMATF(("%s-socket%0d", INSTANCE_ID, socket_id))) ) socket ( `SCOPE_IO_BIND (scope_socket+socket_id) .clk (clk), .reset (reset), + .socket_id (CORE_ID_WIDTH'(cluster_id) * CORE_ID_WIDTH'(NUM_SOCKETS) + CORE_ID_WIDTH'(socket_id)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf_tmp), @@ -426,10 +427,11 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; assign per_socket_dcr_bus_if[DCR_GFX_IDX].rsp_data = gfx_dcr_bus_if.rsp_data; VX_graphics #( - .CLUSTER_ID (CLUSTER_ID) + .INSTANCE_ID (INSTANCE_ID) ) graphics ( .clk (clk), .reset (reset), + .cluster_id (cluster_id), `ifdef PERF_ENABLE `ifdef VX_CFG_EXT_RASTER_ENABLE .raster_perf (gfx_raster_perf), diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 39e4140148..114b2ef153 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -30,6 +30,11 @@ package VX_gpu_pkg; localparam NT_WIDTH = `UP(NT_BITS); localparam NB_WIDTH = `UP(NB_BITS); + // core index across all clusters + localparam CORE_ID_BITS = `CLOG2(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_CORES); + localparam CORE_ID_WIDTH = `UP(CORE_ID_BITS); + localparam CLUSTER_ID_WIDTH = `LOG2UP(`VX_CFG_NUM_CLUSTERS); + localparam NUM_CTA_MAX = `VX_CFG_NUM_WARPS; localparam NCTA_BITS = `CLOG2(NUM_CTA_MAX); localparam NCTA_WIDTH = `UP(NCTA_BITS); @@ -625,6 +630,104 @@ package VX_gpu_pkg; end endfunction + // Fetch slice f schedules and fetches the warps with wid mod FETCH_WIDTH == f, + // naming them by their local id wfs = wid / FETCH_WIDTH. + localparam FETCH_FSW_BITS = `CLOG2(`VX_CFG_FETCH_WIDTH); + localparam FETCH_FSW_W = `UP(FETCH_FSW_BITS); + localparam PER_FETCH_WARPS = `VX_CFG_NUM_WARPS / `VX_CFG_FETCH_WIDTH; + localparam FETCH_WFS_BITS = `CLOG2(PER_FETCH_WARPS); + localparam FETCH_WFS_W = `UP(FETCH_WFS_BITS); + + function automatic logic [NW_WIDTH-1:0] wfs_to_wid( + input logic [FETCH_WFS_W-1:0] wfs, + input logic [FETCH_FSW_W-1:0] fsw + ); + if (FETCH_WFS_BITS == 0) begin + wfs_to_wid = NW_WIDTH'(fsw); + end else if (FETCH_FSW_BITS == 0) begin + wfs_to_wid = NW_WIDTH'(wfs); + end else begin + wfs_to_wid = NW_WIDTH'({wfs, fsw}); + end + endfunction + + function automatic logic [FETCH_FSW_W-1:0] wid_to_fsw( + input logic [NW_WIDTH-1:0] wid + ); + if (FETCH_FSW_BITS != 0) begin + wid_to_fsw = wid[FETCH_FSW_W-1:0]; + end else begin + wid_to_fsw = 0; + end + endfunction + + // FU block ports per fetch slice: a unit with B blocks serves warps with + // wid mod B == b, so B >= FETCH_WIDTH gives each fetch slice B / FETCH_WIDTH + // blocks and B < FETCH_WIDTH shares each block among FETCH_WIDTH / B slices. + localparam FETCH_ALU_PORTS = (`VX_CFG_NUM_ALU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_ALU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; + localparam FETCH_SFU_PORTS = (`VX_CFG_NUM_SFU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_SFU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; + + function automatic logic [FETCH_WFS_W-1:0] wid_to_wfs( + input logic [NW_WIDTH-1:0] wid + ); + if (FETCH_WFS_BITS != 0) begin + wid_to_wfs = FETCH_WFS_W'(wid >> FETCH_FSW_BITS); + end else begin + wid_to_wfs = 0; + end + endfunction + + // A unit with B blocks serves, in block b, exactly the warps with wid mod B == b + // (lane dispatch maps issue slot {batch, block} and isw = wid mod ISSUE_WIDTH), + // so a block's warp state is indexed by the remaining high wid bits. + localparam SFU_BLOCK_BITS = `CLOG2(`VX_CFG_NUM_SFU_BLOCKS); + localparam SFU_BLOCK_W = `UP(SFU_BLOCK_BITS); + localparam SFU_BLOCK_WARPS = `VX_CFG_NUM_WARPS / `VX_CFG_NUM_SFU_BLOCKS; + localparam SFU_BLOCK_NW_BITS = `CLOG2(SFU_BLOCK_WARPS); + localparam SFU_BLOCK_NW_W = `UP(SFU_BLOCK_NW_BITS); + + function automatic logic [SFU_BLOCK_W-1:0] wid_to_sfu_block( + input logic [NW_WIDTH-1:0] wid + ); + if (SFU_BLOCK_BITS != 0) begin + wid_to_sfu_block = wid[SFU_BLOCK_W-1:0]; + end else begin + wid_to_sfu_block = 0; + end + endfunction + + function automatic logic [SFU_BLOCK_NW_W-1:0] wid_to_sfu_local( + input logic [NW_WIDTH-1:0] wid + ); + if (SFU_BLOCK_NW_BITS != 0) begin + wid_to_sfu_local = SFU_BLOCK_NW_W'(wid >> SFU_BLOCK_BITS); + end else begin + wid_to_sfu_local = 0; + end + endfunction + + function automatic logic [NW_WIDTH-1:0] sfu_local_to_wid( + input logic [SFU_BLOCK_NW_W-1:0] lid, + input logic [SFU_BLOCK_W-1:0] block + ); + if (SFU_BLOCK_NW_BITS == 0) begin + sfu_local_to_wid = NW_WIDTH'(block); + end else if (SFU_BLOCK_BITS == 0) begin + sfu_local_to_wid = NW_WIDTH'(lid); + end else begin + sfu_local_to_wid = NW_WIDTH'({lid, block}); + end + endfunction + + // The core is built from ISSUE_WIDTH compute slices (VX_compute_slice) when + // every execution unit has one block per issue slice; otherwise it keeps the + // stage-sliced issue / execute / commit structure. + localparam COMPUTE_SLICES_ENABLE = (`VX_CFG_NUM_ALU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_LSU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_SFU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (!`VX_CFG_EXT_F_ENABLED || (`VX_CFG_NUM_FPU_BLOCKS == `VX_CFG_ISSUE_WIDTH)) + && (!`VX_CFG_EXT_TCU_ENABLED || (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH)); + /////////////////////////////// TENSOR UNIT /////////////////////////////// `ifdef VX_CFG_EXT_TCU_ENABLE @@ -678,6 +781,22 @@ package VX_gpu_pkg; logic [BAR_SIZE_W-1:0] size_m1; } barrier_t; + // Core-wide warp-control requests (barrier, wspawn, warp exit), one per + // SFU block, served by VX_warp_shared_unit. + localparam WSH_BAR = 2'd0; + localparam WSH_WSPAWN = 2'd1; + localparam WSH_EXIT = 2'd2; + + typedef struct packed { + logic [1:0] kind; + logic has_rsp; // the requester waits for its completion + logic [NW_WIDTH-1:0] wid; + barrier_t bar; + logic [BAR_ADDR_W-1:0] bar_addr; + wspawn_t wspawn; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; // spawning warp's, inherited by the spawned warps + } warp_shared_req_t; + typedef struct packed { logic [NB_WIDTH-1:0] id; logic [NC_WIDTH-1:0] size_m1; @@ -885,6 +1004,72 @@ package VX_gpu_pkg; cta_lane_t [`VX_CFG_NUM_THREADS-1:0] lane_launch; } cta_warp_t; + // CTA dispatcher state broadcast to the CTA-CSR stores that serve CSR reads: + // the per-CTA context, shared by every launch lane. + typedef struct packed { + logic ctx_write; + logic [NCTA_WIDTH-1:0] ctx_waddr; + cta_ctx_t ctx_wdata; + } cta_bcast_t; + + // One launch lane's per-warp state: the warp's rank/lane record and CTA id. + typedef struct packed { + logic warp_write; + logic [NW_WIDTH-1:0] warp_waddr; + cta_warp_t warp_wdata; + logic id_write; + logic [NW_WIDTH-1:0] id_wid; + logic [NCTA_WIDTH-1:0] id_cta; + } cta_lane_bcast_t; + + // Scheduler traffic between an FU block and the fetch slice owning the warp, + // addressed by core-wide wid. + + // ALU branch resolution + typedef struct packed { + logic valid; + logic [NW_WIDTH-1:0] wid; + logic taken; + logic [PC_BITS-1:0] dest; + logic is_trap; + logic is_mret; + logic [3:0] trap_cause; + } sched_branch_t; + + // SFU warp control + typedef struct packed { + logic tmc_valid; + logic split_valid; + logic sjoin_valid; + logic wsync_valid; + logic [NW_WIDTH-1:0] wid; + tmc_t tmc; + split_t split; + join_t sjoin; + } sched_wctl_t; + + // SFU CSR writes to per-warp state + typedef struct packed { + logic valid; + logic [NW_WIDTH-1:0] wid; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] data; + logic [NUM_TRAP_CSRS-1:0] trap_valid; + logic [`VX_CFG_XLEN-1:0] trap_data; + } sched_csr_wr_t; + + // SFU CSR reads of per-warp state, selected by the read wid and CTA id + typedef struct packed { + logic [`VX_CFG_NUM_THREADS-1:0] thread_mask; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; + logic [`VX_CFG_XLEN-1:0] mstatus; + logic [`VX_CFG_XLEN-1:0] mtvec; + logic [`VX_CFG_XLEN-1:0] mepc; + logic [`VX_CFG_XLEN-1:0] mcause; + logic [`VX_CFG_XLEN-1:0] mtval; + logic [NW_WIDTH-1:0] cta_rank; + cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_lane; + } sched_csr_rd_t; + //////////////////////// instruction arguments //////////////////////////// // tcu_args_t at 27 bits = 1+1+1+2+5+5+4+4+4. Sparse variants are distinct opcodes. @@ -1402,7 +1587,7 @@ package VX_gpu_pkg; // Width of the tag carried over VX_lsu_sched_if: lsu_header + op_type // + per-lane align + packet allocator addr + fence flag (matches the - // local TAG_WIDTH built inside VX_lsu_slice). + // local TAG_WIDTH built inside VX_lsu_block). localparam LSU_CLIENT_TAG_WIDTH = $bits(lsu_header_t) + INST_LSU_BITS + (`VX_CFG_NUM_LSU_LANES * `CLOG2(LSU_WORD_SIZE)) + `LOG2UP(`VX_CFG_LSU_PENDING_SIZE) + 1; @@ -1463,8 +1648,12 @@ package VX_gpu_pkg; // Block size in bytes localparam ICACHE_LINE_SIZE = `VX_CFG_L1_LINE_SIZE; - // Core request tag Id bits - localparam ICACHE_TAG_ID_BITS = NW_WIDTH; + // Request ports per core: one per fetch slice, each fetching its own warps + localparam ICACHE_NUM_REQS = `VX_CFG_FETCH_WIDTH; + localparam ICACHE_NUM_BANKS = `VX_CFG_ICACHE_NUM_BANKS; + + // Core request tag Id bits: the warp id local to the requesting fetch slice + localparam ICACHE_TAG_ID_BITS = FETCH_WFS_W; // Core request tag bits as VX_fetch writes them (UUID + warp id). localparam ICACHE_FETCH_TAG_WIDTH = (UUID_WIDTH + ICACHE_TAG_ID_BITS); @@ -1485,9 +1674,9 @@ package VX_gpu_pkg; // Memory request tag bits `ifdef VX_CFG_ICACHE_ENABLE - localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_MEM_TAG_WIDTH(`VX_CFG_ICACHE_MSHR_SIZE, 1, 1, `VX_CFG_NUM_ICACHES, UUID_WIDTH); + localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_MEM_TAG_WIDTH(`VX_CFG_ICACHE_MSHR_SIZE, ICACHE_NUM_BANKS, 1, `VX_CFG_NUM_ICACHES, UUID_WIDTH); `else - localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_MEM_TAG_WIDTH(1, 1, ICACHE_LINE_SIZE, ICACHE_WORD_SIZE, ICACHE_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_ICACHES); + localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_MEM_TAG_WIDTH(ICACHE_NUM_REQS, 1, ICACHE_LINE_SIZE, ICACHE_WORD_SIZE, ICACHE_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_ICACHES); `endif ////////////////////////// Dcache Parameters ////////////////////////////// @@ -1556,8 +1745,10 @@ package VX_gpu_pkg; `ifdef VX_CFG_EXT_TEX_ENABLE // tex unit: per-request queue + per-request tag localparam TEX_REQ_TAG_WIDTH = (UUID_WIDTH + `CLOG2(`VX_CFG_TEX_REQ_QUEUE_SIZE)); + // core-level arb: SFU blocks -> the core's texture bus + localparam TEX_CORE_TAG_WIDTH = (TEX_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_SFU_BLOCKS, 1)); // Single socket-level arb: socket cores -> per-socket TEX units - localparam TEX_REQ_ARB1_TAG_WIDTH = (TEX_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_TEX_CORES)); + localparam TEX_REQ_ARB1_TAG_WIDTH = (TEX_CORE_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_TEX_CORES)); localparam TCACHE_WORD_SIZE = 4; localparam TCACHE_ADDR_WIDTH = (`VX_CFG_MEM_ADDR_WIDTH - `CLOG2(TCACHE_WORD_SIZE)); @@ -1679,8 +1870,10 @@ package VX_gpu_pkg; `ifdef VX_CFG_EXT_RTU_ENABLE localparam RTU_REQ_QUEUE_SIZE = 4; localparam RTU_REQ_TAG_WIDTH = (UUID_WIDTH + `CLOG2(RTU_REQ_QUEUE_SIZE)); + // core-level arb: SFU blocks -> the core's RTU bus + localparam RTU_CORE_TAG_WIDTH = (RTU_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_SFU_BLOCKS, 1)); // Single socket-level arb: socket cores -> per-socket RTU units - localparam RTU_REQ_ARB1_TAG_WIDTH = (RTU_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_RTU_CORES)); + localparam RTU_REQ_ARB1_TAG_WIDTH = (RTU_CORE_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_RTU_CORES)); // The RTU fetches a whole CW-BVH node (one 64 B cache line) per request, // so its cache is line-granular: word size == line size, one port/core. diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index a2ef22cb37..329dd7fc67 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -21,11 +21,13 @@ module VX_graphics import VX_gpu_pkg::* , VX_om_pkg::* `endif ; #( - parameter CLUSTER_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CLUSTER_ID_WIDTH-1:0] cluster_id, + `ifdef PERF_ENABLE `ifdef VX_CFG_EXT_RASTER_ENABLE output raster_perf_t raster_perf, @@ -65,7 +67,10 @@ module VX_graphics import VX_gpu_pkg::* // no in-band `done`); the cluster ORs this into the device busy aggregation. output wire busy ); - `UNUSED_PARAM (CLUSTER_ID) + `UNUSED_SPARAM (INSTANCE_ID) +`ifndef VX_CFG_EXT_RASTER_ENABLE + `UNUSED_VAR (cluster_id) +`endif wire rcache_req_empty; wire ocache_req_empty; @@ -137,8 +142,7 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < `VX_CFG_NUM_RASTER_CORES; ++i) begin : g_raster_core VX_raster_core #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-raster%0d", CLUSTER_ID, i))), - .INSTANCE_IDX (CLUSTER_ID * `VX_CFG_NUM_RASTER_CORES + i), + .INSTANCE_ID (`SFORMATF(("%s-raster%0d", INSTANCE_ID, i))), .NUM_INSTANCES (`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES), .NUM_SLICES (`VX_CFG_RASTER_NUM_SLICES), // The front end's top-level walk unit is a coarse bin: bin_x/bin_y @@ -154,6 +158,8 @@ module VX_graphics import VX_gpu_pkg::* ) raster_core ( .clk (clk), .reset (reset), + .instance_idx (`LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(cluster_id) * `LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(`VX_CFG_NUM_RASTER_CORES) + + `LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(i)), `ifdef PERF_ENABLE .perf_raster_if (per_core_raster_perf_if[i]), `endif @@ -247,7 +253,7 @@ module VX_graphics import VX_gpu_pkg::* end VX_cache_cluster #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-rcache", CLUSTER_ID))), + .INSTANCE_ID (`SFORMATF(("%s-rcache", INSTANCE_ID))), .NUM_UNITS (`VX_CFG_NUM_RCACHES), .NUM_INPUTS (`VX_CFG_NUM_RASTER_CORES), .TAG_SEL_IDX (0), @@ -338,7 +344,7 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_steer VX_om_steer #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-om-steer%0d", CLUSTER_ID, i))), + .INSTANCE_ID (`SFORMATF(("%s-om-steer%0d", INSTANCE_ID, i))), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (L2_TAG_WIDTH), .OUT_BUF (3) @@ -371,7 +377,7 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_ingress VX_om_ingress #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-om-ingress%0d", CLUSTER_ID, i))), + .INSTANCE_ID (`SFORMATF(("%s-om-ingress%0d", INSTANCE_ID, i))), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (L2_TAG_WIDTH), .NUM_LANES (OM_CORE_LANES), @@ -408,7 +414,7 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < `VX_CFG_NUM_OM_CORES; ++i) begin : g_om_core VX_om_core #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-om%0d", CLUSTER_ID, i))), + .INSTANCE_ID (`SFORMATF(("%s-om%0d", INSTANCE_ID, i))), .NUM_LANES (OM_CORE_LANES) ) om_core ( .clk (clk), @@ -588,7 +594,7 @@ module VX_graphics import VX_gpu_pkg::* `endif VX_cache_cluster #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-ocache", CLUSTER_ID))), + .INSTANCE_ID (`SFORMATF(("%s-ocache", INSTANCE_ID))), .NUM_UNITS (`VX_CFG_NUM_OCACHES), .NUM_INPUTS (OCACHE_NUM_INPUTS), .TAG_SEL_IDX (0), diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index d65f86df73..e8dd0c8248 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -18,7 +18,6 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; import VX_dxa_pkg::*; `endif #( - parameter SOCKET_ID = 0, parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -27,6 +26,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] socket_id, + `ifdef PERF_ENABLE input sysmem_perf_t sysmem_perf, `endif @@ -127,12 +128,12 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_TAG_WIDTH) - ) per_core_icache_bus_if[`VX_CFG_SOCKET_SIZE](); + ) per_core_icache_bus_if[`VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS](); VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_TAG_WIDTH) - ) icache_phys_bus_if[`VX_CFG_SOCKET_SIZE](); + ) icache_phys_bus_if[`VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS](); VX_mem_bus_if #( .DATA_SIZE (ICACHE_LINE_SIZE), @@ -149,10 +150,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .TAG_SEL_IDX (0), .CACHE_SIZE (`VX_CFG_ICACHE_SIZE), .LINE_SIZE (ICACHE_LINE_SIZE), - .NUM_BANKS (1), + .NUM_BANKS (ICACHE_NUM_BANKS), .NUM_WAYS (`VX_CFG_ICACHE_NUM_WAYS), .WORD_SIZE (ICACHE_WORD_SIZE), - .NUM_REQS (1), + .NUM_REQS (ICACHE_NUM_REQS), .MEM_PORTS (1), .CRSQ_SIZE (`VX_CFG_ICACHE_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_ICACHE_MSHR_SIZE), @@ -302,7 +303,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_mmu #( .INSTANCE_ID ("immu"), - .NUM_REQS (`VX_CFG_SOCKET_SIZE), + .NUM_REQS (`VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS), .TLB_SIZE (`VX_CFG_ITLB_SIZE), .EXEC_SIDE (1), .DATA_SIZE (ICACHE_WORD_SIZE), @@ -362,7 +363,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE * DCACHE_NUM_REQS; ++i) begin : g_dcache_phys_passthru `ASSIGN_VX_MEM_BUS_IF (dcache_phys_bus_if[i], per_core_dcache_bus_if[i]); end - for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE; ++i) begin : g_icache_phys_passthru + for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS; ++i) begin : g_icache_phys_passthru `ASSIGN_VX_MEM_BUS_IF (icache_phys_bus_if[i], per_core_icache_bus_if[i]); end `endif @@ -403,7 +404,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_EXT_TEX_ENABLE VX_tex_bus_if #( .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + .TAG_WIDTH (TEX_CORE_TAG_WIDTH) ) per_core_tex_bus_if[`VX_CFG_SOCKET_SIZE](); VX_tex_bus_if #( @@ -415,7 +416,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_INPUTS (`VX_CFG_SOCKET_SIZE), .NUM_LANES (`VX_CFG_NUM_SFU_LANES), .NUM_OUTPUTS (`VX_CFG_NUM_TEX_CORES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH), + .TAG_WIDTH (TEX_CORE_TAG_WIDTH), .ARBITER ("R"), .OUT_BUF_REQ ((`VX_CFG_SOCKET_SIZE != `VX_CFG_NUM_TEX_CORES) ? 3 : 0), // register request fan-in .OUT_BUF_RSP ((`VX_CFG_SOCKET_SIZE != `VX_CFG_NUM_TEX_CORES) ? 3 : 0) // register response distribution (skid registers the core-facing rsp and its backward ready) @@ -556,7 +557,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_rtu_bus_if #( .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .TAG_WIDTH (RTU_CORE_TAG_WIDTH), .SRC_WIDTH (1) // a core does not know its own index; the arbiter fills it in ) per_core_rtu_bus_if[`VX_CFG_SOCKET_SIZE](); @@ -570,7 +571,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_INPUTS (`VX_CFG_SOCKET_SIZE), .NUM_LANES (`VX_CFG_NUM_SFU_LANES), .NUM_OUTPUTS (`VX_CFG_NUM_RTU_CORES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .TAG_WIDTH (RTU_CORE_TAG_WIDTH), .ARBITER ("R"), // The arm may be buffered: it does not mean "the RTU has taken your ray" // (the RTU stages a ray per {src, wid}), so registering it cannot let a @@ -878,13 +879,13 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; );*/ VX_core #( - .CORE_ID ((SOCKET_ID * `VX_CFG_SOCKET_SIZE) + core_id), .INSTANCE_ID (`SFORMATF(("%s-core%0d", INSTANCE_ID, core_id))) ) core ( `SCOPE_IO_BIND (scope_core + core_id) .clk (clk), .reset (reset), + .core_id (CORE_ID_WIDTH'(socket_id * `VX_CFG_SOCKET_SIZE) + CORE_ID_WIDTH'(core_id)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf_tmp), @@ -894,7 +895,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .dcache_bus_if (per_core_dcache_bus_if[core_id * DCACHE_NUM_REQS +: DCACHE_NUM_REQS]), - .icache_bus_if (per_core_icache_bus_if[core_id]), + .icache_bus_if (per_core_icache_bus_if[core_id * ICACHE_NUM_REQS +: ICACHE_NUM_REQS]), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (per_core_dxa_req_bus_if[core_id]), diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index a605dedd04..e536ba88b8 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -335,13 +335,13 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( for (genvar cluster_id = 0; cluster_id < `VX_CFG_NUM_CLUSTERS; ++cluster_id) begin : g_clusters VX_cluster #( - .CLUSTER_ID (cluster_id), .INSTANCE_ID (`SFORMATF(("cluster%0d", cluster_id))) ) cluster ( `SCOPE_IO_BIND (scope_cluster + cluster_id) .clk (clk), .reset (reset), + .cluster_id (CLUSTER_ID_WIDTH'(cluster_id)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf), diff --git a/hw/rtl/core/VX_alu_block.sv b/hw/rtl/core/VX_alu_block.sv new file mode 100644 index 0000000000..b07a14a074 --- /dev/null +++ b/hw/rtl/core/VX_alu_block.sv @@ -0,0 +1,93 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One ALU block: the integer and mul/div datapaths behind a PE switch. +module VX_alu_block import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter `STRING BLOCK_STR = "", + parameter NUM_LANES = 1, + parameter PARTIAL_BW = 0 +) ( + input wire clk, + input wire reset, + + input wire [`LOG2UP(`VX_CFG_NUM_ALU_BLOCKS)-1:0] block_idx, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if, + VX_branch_ctl_if.master branch_ctl_if +); + localparam PE_COUNT = 1 + `VX_CFG_EXT_M_ENABLED; + localparam PE_SEL_BITS = `CLOG2(PE_COUNT); + localparam PE_IDX_INT = 0; + localparam PE_IDX_MDV = PE_IDX_INT + `VX_CFG_EXT_M_ENABLED; + + VX_execute_if #( + .data_t (alu_execute_t) + ) pe_execute_if[PE_COUNT](); + + VX_result_if#( + .data_t (alu_result_t) + ) pe_result_if[PE_COUNT](); + + reg [`UP(PE_SEL_BITS)-1:0] pe_select; + always @(*) begin + pe_select = PE_IDX_INT; + if (`VX_CFG_EXT_M_ENABLED && (execute_if.data.op_args.alu.xtype == ALU_TYPE_MULDIV)) begin + pe_select = PE_IDX_MDV; + end + end + + VX_pe_switch #( + .PE_COUNT (PE_COUNT), + .NUM_LANES (NUM_LANES), + .ARBITER ("R"), + .REQ_OUT_BUF (0), + .RSP_OUT_BUF (PARTIAL_BW ? 1 : 3) + ) pe_switch ( + .clk (clk), + .reset (reset), + .pe_sel (pe_select), + .execute_in_if (execute_if), + .result_out_if (result_if), + .execute_out_if (pe_execute_if), + .result_in_if (pe_result_if) + ); + + VX_alu_int #( + .INSTANCE_ID (`SFORMATF(("%s-int%s", INSTANCE_ID, BLOCK_STR))), + .NUM_LANES (NUM_LANES) + ) alu_int ( + .clk (clk), + .reset (reset), + .block_idx (block_idx), + .execute_if (pe_execute_if[PE_IDX_INT]), + .branch_ctl_if (branch_ctl_if), + .result_if (pe_result_if[PE_IDX_INT]) + ); + +`ifdef VX_CFG_EXT_M_ENABLE + VX_alu_muldiv #( + .INSTANCE_ID (`SFORMATF(("%s-muldiv%s", INSTANCE_ID, BLOCK_STR))), + .NUM_LANES (NUM_LANES) + ) muldiv_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_MDV]), + .result_if (pe_result_if[PE_IDX_MDV]) + ); +`endif + +endmodule diff --git a/hw/rtl/core/VX_alu_int.sv b/hw/rtl/core/VX_alu_int.sv index 3a680055bf..0c081cc07f 100644 --- a/hw/rtl/core/VX_alu_int.sv +++ b/hw/rtl/core/VX_alu_int.sv @@ -15,12 +15,13 @@ module VX_alu_int import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter BLOCK_IDX = 0, parameter NUM_LANES = 1 ) ( input wire clk, input wire reset, + input wire [`LOG2UP(`VX_CFG_NUM_ALU_BLOCKS)-1:0] block_idx, + // Inputs VX_execute_if.slave execute_if, @@ -339,7 +340,10 @@ module VX_alu_int import VX_gpu_pkg::*; #( : is_br_static ? from_fullPC(br_result) : cbr_dest_r; wire [NW_WIDTH-1:0] br_wid; - `ASSIGN_BLOCKED_WID (br_wid, result_if.data.header.wid, BLOCK_IDX, `VX_CFG_NUM_ALU_BLOCKS) + `ASSIGN_BLOCKED_WID (br_wid, result_if.data.header.wid, block_idx, `VX_CFG_NUM_ALU_BLOCKS) + if (`VX_CFG_NUM_ALU_BLOCKS == 1) begin : g_block_idx_unused + `UNUSED_VAR (block_idx) + end VX_pipe_register #( .DATAW (1 + NW_WIDTH + 1 + PC_BITS + 1 + 1 + 4), diff --git a/hw/rtl/core/VX_alu_unit.sv b/hw/rtl/core/VX_alu_unit.sv index 3128187c54..e91603251c 100644 --- a/hw/rtl/core/VX_alu_unit.sv +++ b/hw/rtl/core/VX_alu_unit.sv @@ -27,14 +27,9 @@ module VX_alu_unit import VX_gpu_pkg::*; #( VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS] ); - `UNUSED_SPARAM (INSTANCE_ID) localparam BLOCK_SIZE = `VX_CFG_NUM_ALU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_ALU_LANES; localparam PARTIAL_BW = (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) || (NUM_LANES != `VX_CFG_SIMD_WIDTH); - localparam PE_COUNT = 1 + `VX_CFG_EXT_M_ENABLED; - localparam PE_SEL_BITS = `CLOG2(PE_COUNT); - localparam PE_IDX_INT = 0; - localparam PE_IDX_MDV = PE_IDX_INT + `VX_CFG_EXT_M_ENABLED; VX_execute_if #( .data_t (alu_execute_t) @@ -51,66 +46,25 @@ module VX_alu_unit import VX_gpu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - - VX_execute_if #( - .data_t (alu_execute_t) - ) pe_execute_if[PE_COUNT](); - - VX_result_if#( - .data_t (alu_result_t) - ) pe_result_if[PE_COUNT](); - - reg [`UP(PE_SEL_BITS)-1:0] pe_select; - always @(*) begin - pe_select = PE_IDX_INT; - if (`VX_CFG_EXT_M_ENABLED && (per_block_execute_if[block_idx].data.op_args.alu.xtype == ALU_TYPE_MULDIV)) - pe_select = PE_IDX_MDV; - end - - VX_pe_switch #( - .PE_COUNT (PE_COUNT), + VX_alu_block #( + .INSTANCE_ID (INSTANCE_ID), + .BLOCK_STR (`SFORMATF(("%0d", block_idx))), .NUM_LANES (NUM_LANES), - .ARBITER ("R"), - .REQ_OUT_BUF (0), - .RSP_OUT_BUF (PARTIAL_BW ? 1 : 3) - ) pe_switch ( - .clk (clk), - .reset (reset), - .pe_sel (pe_select), - .execute_in_if (per_block_execute_if[block_idx]), - .result_out_if (per_block_result_if[block_idx]), - .execute_out_if (pe_execute_if), - .result_in_if (pe_result_if) - ); - - VX_alu_int #( - .INSTANCE_ID (`SFORMATF(("%s-int%0d", INSTANCE_ID, block_idx))), - .BLOCK_IDX (block_idx), - .NUM_LANES (NUM_LANES) - ) alu_int ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_INT]), - .branch_ctl_if (branch_ctl_if[block_idx]), - .result_if (pe_result_if[PE_IDX_INT]) - ); - - `ifdef VX_CFG_EXT_M_ENABLE - VX_alu_muldiv #( - .INSTANCE_ID (`SFORMATF(("%s-muldiv%0d", INSTANCE_ID, block_idx))), - .NUM_LANES (NUM_LANES) - ) muldiv_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_MDV]), - .result_if (pe_result_if[PE_IDX_MDV]) + .PARTIAL_BW (PARTIAL_BW) + ) alu_block ( + .clk (clk), + .reset (reset), + .block_idx (`LOG2UP(BLOCK_SIZE)'(block_idx)), + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]), + .branch_ctl_if (branch_ctl_if[block_idx]) ); - `endif end VX_lane_gather #( diff --git a/hw/rtl/core/VX_bar_unit.sv b/hw/rtl/core/VX_bar_unit.sv index f3f7313e92..f608b71e79 100644 --- a/hw/rtl/core/VX_bar_unit.sv +++ b/hw/rtl/core/VX_bar_unit.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_bar_unit import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // local barrier interface input wire req_valid, input wire [NW_WIDTH-1:0] req_wid, @@ -36,7 +37,6 @@ module VX_bar_unit import VX_gpu_pkg::*; #( output wire [`VX_CFG_NUM_WARPS-1:0] unlock_mask // warps to unlock ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) // warp mask + warp count + event count localparam EVENT_WIDTH = `CLOG2(`VX_CFG_MAX_BAR_EVENTS + 1); @@ -329,12 +329,13 @@ module VX_bar_unit import VX_gpu_pkg::*; #( assign gbar_bus_if.req_valid = gbar_req_valid_r; assign {gbar_bus_if.req_data.id, gbar_bus_if.req_data.size_m1} = req_queue_data; - assign gbar_bus_if.req_data.core_id = NC_WIDTH'(CORE_ID % `VX_CFG_NUM_CORES); + assign gbar_bus_if.req_data.core_id = NC_WIDTH'(32'(core_id) % `VX_CFG_NUM_CORES); assign gbar_bus_if.rsp_ready = gbar_rsp_ready; `RUNTIME_ASSERT(~gbar_enqueue || ~gbar_pending_r[gbar_enqueue_id], ("%s duplicate global barrier generation: id=%0d", INSTANCE_ID, gbar_enqueue_id)) `RUNTIME_ASSERT(~gbar_enqueue || (store_waddr == BAR_ADDR_W'(gbar_enqueue_id)), ("%s invalid global barrier slot: id=%0d, slot=%0d", INSTANCE_ID, gbar_enqueue_id, store_waddr)) end else begin : g_nogbar + `UNUSED_VAR (core_id) assign gbar_bus_if.req_valid = 0; assign gbar_bus_if.req_data = 'x; diff --git a/hw/rtl/core/VX_commit.sv b/hw/rtl/core/VX_commit.sv index 44a5437129..4cc0927c18 100644 --- a/hw/rtl/core/VX_commit.sv +++ b/hw/rtl/core/VX_commit.sv @@ -26,125 +26,28 @@ module VX_commit import VX_gpu_pkg::*; #( VX_writeback_if.master writeback_if [`VX_CFG_ISSUE_WIDTH], VX_commit_sched_if.master commit_sched_if ); - `UNUSED_SPARAM (INSTANCE_ID) - localparam OUT_DATAW = $bits(commit_t); - - // commit arbitration - - VX_commit_if commit_arb_if[`VX_CFG_ISSUE_WIDTH](); - wire [`VX_CFG_ISSUE_WIDTH-1:0] committed_warps; - wire [`VX_CFG_ISSUE_WIDTH-1:0][PER_ISSUE_WARPS-1:0] commit_eop_wis; - - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_commit_arbs - - wire [NUM_EX_UNITS-1:0] valid_in; - wire [NUM_EX_UNITS-1:0][PER_ISSUE_WARPS+OUT_DATAW-1:0] data_in; - wire [NUM_EX_UNITS-1:0] ready_in; - - // Decoded ahead of the output register so each warp's release reads - // its own flop. - for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_data_in - wire [PER_ISSUE_WARPS-1:0] eop_wis = PER_ISSUE_WARPS'(commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.eop) - << wid_to_wis(commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wid); - assign valid_in[j] = commit_if[j * `VX_CFG_ISSUE_WIDTH + i].valid; - assign data_in[j] = {eop_wis, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data}; - assign commit_if[j * `VX_CFG_ISSUE_WIDTH + i].ready = ready_in[j]; - end - - VX_stream_arb #( - .NUM_INPUTS (NUM_EX_UNITS), - .DATAW (PER_ISSUE_WARPS + OUT_DATAW), - .ARBITER ("P"), - .OUT_BUF (1) - ) commit_arb ( - .clk (clk), - .reset (reset), - .valid_in (valid_in), - .ready_in (ready_in), - .data_in (data_in), - .data_out ({commit_eop_wis[i], commit_arb_if[i].data}), - .valid_out (commit_arb_if[i].valid), - .ready_out (commit_arb_if[i].ready), - `UNUSED_PIN (sel_out) - ); - - wire commit_arb_fire = commit_arb_if[i].valid && commit_arb_if[i].ready; - assign committed_warps[i] = commit_arb_fire && commit_arb_if[i].data.eop; - end - - // notify scheduler: build per-warp committed mask from per-slot signals - wire [`VX_CFG_ISSUE_WIDTH-1:0][NW_WIDTH-1:0] committed_slot_wid; - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_committed_wid - assign committed_slot_wid[i] = commit_arb_if[i].data.wid; - end - - logic [`VX_CFG_NUM_WARPS-1:0] committed_warp_mask; - wire [`VX_CFG_NUM_WARPS-1:0] committed_warp_mask_r; - always_comb begin - committed_warp_mask = '0; - for (integer i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin - if (committed_warps[i]) begin - committed_warp_mask[committed_slot_wid[i]] = 1'b1; - end + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices + VX_commit_if slice_commit_if[NUM_EX_UNITS](); + for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_commit_if + `ASSIGN_VX_IF(slice_commit_if[j], commit_if[j * `VX_CFG_ISSUE_WIDTH + i]); end - end - `BUFFER(committed_warp_mask_r, committed_warp_mask); - assign commit_sched_if.committed_warps = committed_warp_mask_r; - - // Writeback - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_writeback - wire [XLENB_W-1:0] bytesel_size = commit_arb_if[i].data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; - wire [XLENB_W-1:0] bytesel_off = commit_arb_if[i].data.bytesel[0 +: XLENB_W]; - wire [`VX_CFG_SIMD_WIDTH-1:0][`VX_CFG_XLEN-1:0] writeback_data; - wire [`VX_CFG_SIMD_WIDTH-1:0][XLENB-1:0] writeback_byteen; + wire [PER_ISSUE_WARPS-1:0] committed_wis; - wire [XLENB-1:0] size_mask = (bytesel_size == XLENB_W'(7)) ? XLENB'(255) : - (bytesel_size == XLENB_W'(6)) ? XLENB'(127) : - (bytesel_size == XLENB_W'(5)) ? XLENB'(63) : - (bytesel_size == XLENB_W'(4)) ? XLENB'(31) : - (bytesel_size == XLENB_W'(3)) ? XLENB'(15) : - (bytesel_size == XLENB_W'(2)) ? XLENB'(7) : - (bytesel_size == XLENB_W'(1)) ? XLENB'(3) : XLENB'(1); - wire [XLENB-1:0] base_byteen = size_mask << bytesel_off; - - for (genvar lane = 0; lane < `VX_CFG_SIMD_WIDTH; ++lane) begin : g_bytesel - assign writeback_data[lane] = commit_arb_if[i].data.data[lane] << (8 * bytesel_off); - assign writeback_byteen[lane] = commit_arb_if[i].data.tmask[lane] ? base_byteen : '0; - end - - assign writeback_if[i].valid = commit_arb_if[i].valid; - assign writeback_if[i].data.uuid = commit_arb_if[i].data.uuid; - assign writeback_if[i].data.wis = wid_to_wis(commit_arb_if[i].data.wid); - assign writeback_if[i].data.eop_wis = commit_eop_wis[i]; - assign writeback_if[i].data.cta_id = commit_arb_if[i].data.cta_id; - assign writeback_if[i].data.sid = commit_arb_if[i].data.sid; - assign writeback_if[i].data.PC = commit_arb_if[i].data.PC; - assign writeback_if[i].data.tmask= commit_arb_if[i].data.tmask; - assign writeback_if[i].data.wb = commit_arb_if[i].data.wb; - assign writeback_if[i].data.wr_xregs = commit_arb_if[i].data.wr_xregs; - assign writeback_if[i].data.rd = commit_arb_if[i].data.rd; - assign writeback_if[i].data.byteen = writeback_byteen; - assign writeback_if[i].data.data = writeback_data; - assign writeback_if[i].data.sop = commit_arb_if[i].data.sop; - assign writeback_if[i].data.eop = commit_arb_if[i].data.eop; - assign commit_arb_if[i].ready = 1; - end + VX_commit_slice #( + .INSTANCE_ID (INSTANCE_ID) + ) commit_slice ( + .clk (clk), + .reset (reset), + .commit_if (slice_commit_if), + .writeback_if (writeback_if[i]), + .committed_wis (committed_wis) + ); -`ifdef DBG_TRACE_PIPELINE - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_trace - for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_j - always @(posedge clk) begin - if (commit_if[j * `VX_CFG_ISSUE_WIDTH + i].valid && commit_if[j * `VX_CFG_ISSUE_WIDTH + i].ready) begin - `TRACE(1, ("%t: %s commit: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wid, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.cta_id, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.sid, to_fullPC(commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.PC))) - VX_trace_pkg::trace_ex_type(1, j); - `TRACE(1, (", tmask=%b, wb=%b, wr_xregs=%b, rd=%0d, sop=%b, eop=%b, data=", commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.tmask, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wb, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wr_xregs, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.rd, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.sop, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.eop)) - `TRACE_ARRAY1D(1, "0x%0h", commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.data, `VX_CFG_SIMD_WIDTH) - `TRACE(1, (" (#%0d)\n", commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.uuid)) - end - end + // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_committed + assign commit_sched_if.committed_warps[w * `VX_CFG_ISSUE_WIDTH + i] = committed_wis[w]; end end -`endif endmodule diff --git a/hw/rtl/core/VX_commit_slice.sv b/hw/rtl/core/VX_commit_slice.sv new file mode 100644 index 0000000000..b2a1133d24 --- /dev/null +++ b/hw/rtl/core/VX_commit_slice.sv @@ -0,0 +1,124 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Commit for one issue slice: arbitrates its execution units' results onto the +// slice's writeback port and reports the warps whose instruction retired. +module VX_commit_slice import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + // inputs + VX_commit_if.slave commit_if [NUM_EX_UNITS], + + // outputs + VX_writeback_if.master writeback_if, + output wire [PER_ISSUE_WARPS-1:0] committed_wis // registered +); + `UNUSED_SPARAM (INSTANCE_ID) + localparam OUT_DATAW = $bits(commit_t); + + VX_commit_if commit_arb_if(); + wire [PER_ISSUE_WARPS-1:0] commit_eop_wis; + + wire [NUM_EX_UNITS-1:0] valid_in; + wire [NUM_EX_UNITS-1:0][PER_ISSUE_WARPS+OUT_DATAW-1:0] data_in; + wire [NUM_EX_UNITS-1:0] ready_in; + + // Decoded ahead of the output register so each warp's release reads + // its own flop. + for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_data_in + wire [PER_ISSUE_WARPS-1:0] eop_wis = PER_ISSUE_WARPS'(commit_if[j].data.eop) + << wid_to_wis(commit_if[j].data.wid); + assign valid_in[j] = commit_if[j].valid; + assign data_in[j] = {eop_wis, commit_if[j].data}; + assign commit_if[j].ready = ready_in[j]; + end + + VX_stream_arb #( + .NUM_INPUTS (NUM_EX_UNITS), + .DATAW (PER_ISSUE_WARPS + OUT_DATAW), + .ARBITER ("P"), + .OUT_BUF (1) + ) commit_arb ( + .clk (clk), + .reset (reset), + .valid_in (valid_in), + .ready_in (ready_in), + .data_in (data_in), + .data_out ({commit_eop_wis, commit_arb_if.data}), + .valid_out (commit_arb_if.valid), + .ready_out (commit_arb_if.ready), + `UNUSED_PIN (sel_out) + ); + + // notify scheduler + wire [PER_ISSUE_WARPS-1:0] committed_wis_n = commit_arb_if.valid ? commit_eop_wis : '0; + `BUFFER(committed_wis, committed_wis_n); + + // Writeback + + wire [XLENB_W-1:0] bytesel_size = commit_arb_if.data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; + wire [XLENB_W-1:0] bytesel_off = commit_arb_if.data.bytesel[0 +: XLENB_W]; + wire [`VX_CFG_SIMD_WIDTH-1:0][`VX_CFG_XLEN-1:0] writeback_data; + wire [`VX_CFG_SIMD_WIDTH-1:0][XLENB-1:0] writeback_byteen; + + wire [XLENB-1:0] size_mask = (bytesel_size == XLENB_W'(7)) ? XLENB'(255) : + (bytesel_size == XLENB_W'(6)) ? XLENB'(127) : + (bytesel_size == XLENB_W'(5)) ? XLENB'(63) : + (bytesel_size == XLENB_W'(4)) ? XLENB'(31) : + (bytesel_size == XLENB_W'(3)) ? XLENB'(15) : + (bytesel_size == XLENB_W'(2)) ? XLENB'(7) : + (bytesel_size == XLENB_W'(1)) ? XLENB'(3) : XLENB'(1); + wire [XLENB-1:0] base_byteen = size_mask << bytesel_off; + + for (genvar lane = 0; lane < `VX_CFG_SIMD_WIDTH; ++lane) begin : g_bytesel + assign writeback_data[lane] = commit_arb_if.data.data[lane] << (8 * bytesel_off); + assign writeback_byteen[lane] = commit_arb_if.data.tmask[lane] ? base_byteen : '0; + end + + assign writeback_if.valid = commit_arb_if.valid; + assign writeback_if.data.uuid = commit_arb_if.data.uuid; + assign writeback_if.data.wis = wid_to_wis(commit_arb_if.data.wid); + assign writeback_if.data.eop_wis = commit_eop_wis; + assign writeback_if.data.cta_id = commit_arb_if.data.cta_id; + assign writeback_if.data.sid = commit_arb_if.data.sid; + assign writeback_if.data.PC = commit_arb_if.data.PC; + assign writeback_if.data.tmask= commit_arb_if.data.tmask; + assign writeback_if.data.wb = commit_arb_if.data.wb; + assign writeback_if.data.wr_xregs = commit_arb_if.data.wr_xregs; + assign writeback_if.data.rd = commit_arb_if.data.rd; + assign writeback_if.data.byteen = writeback_byteen; + assign writeback_if.data.data = writeback_data; + assign writeback_if.data.sop = commit_arb_if.data.sop; + assign writeback_if.data.eop = commit_arb_if.data.eop; + assign commit_arb_if.ready = 1; + +`ifdef DBG_TRACE_PIPELINE + for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_trace + always @(posedge clk) begin + if (commit_if[j].valid && commit_if[j].ready) begin + `TRACE(1, ("%t: %s commit: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, commit_if[j].data.wid, commit_if[j].data.cta_id, commit_if[j].data.sid, to_fullPC(commit_if[j].data.PC))) + VX_trace_pkg::trace_ex_type(1, j); + `TRACE(1, (", tmask=%b, wb=%b, wr_xregs=%b, rd=%0d, sop=%b, eop=%b, data=", commit_if[j].data.tmask, commit_if[j].data.wb, commit_if[j].data.wr_xregs, commit_if[j].data.rd, commit_if[j].data.sop, commit_if[j].data.eop)) + `TRACE_ARRAY1D(1, "0x%0h", commit_if[j].data.data, `VX_CFG_SIMD_WIDTH) + `TRACE(1, (" (#%0d)\n", commit_if[j].data.uuid)) + end + end + end +`endif + +endmodule diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv new file mode 100644 index 0000000000..33c771fdc1 --- /dev/null +++ b/hw/rtl/core/VX_compute_slice.sv @@ -0,0 +1,192 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One issue slice carried through issue, execute and commit, serving the warps +// with wid mod ISSUE_WIDTH == slice_id with one block of every execution unit. +// Every slice is the same module: its identity arrives on slice_id/core_id, and +// only the core-wide resources (fetch slices, warp-control resources, memory +// system, SFU buses, WGMMA tile buffers and the metadata AGU) are reached through +// its ports. +module VX_compute_slice import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [ISSUE_ISW_W-1:0] slice_id, + +`ifdef PERF_ENABLE + output issue_perf_t issue_perf, +`endif + + // decoded instructions of the slice's warps + VX_decode_if.slave decode_if, + + // issued and retired warps (registered), indexed by wid / ISSUE_WIDTH + output wire warp_issued, + output wire [ISSUE_WIS_W-1:0] warp_issued_wis, + output wire [PER_ISSUE_WARPS-1:0] committed_wis, + + // scheduler traffic of the slice's ALU and SFU blocks + VX_branch_ctl_if.master branch_ctl_if, + VX_warp_ctl_if.master warp_ctl_if, + VX_sched_csr_if.slave sched_csr_if, +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if.master sched_unlock_if, +`endif + + // LSU + VX_lsu_sched_if.master lsu_client_if, + + // SFU +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, +`endif + + // TCU +`ifdef VX_CFG_TCU_WGMMA_ENABLE + output wire tcu_exec_valid, + output wire tcu_exec_ready, + output tcu_execute_t tcu_exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire tcu_tbuf_ready, +`endif +`ifdef TCU_META_ENABLE + output wire tcu_agu_ld_valid, + output tcu_execute_t tcu_agu_ld_data, + input wire tcu_agu_ld_ready, + input wire tcu_agu_result_valid, + input tcu_result_t tcu_agu_result_data, + output wire tcu_agu_result_ready, + input wire tcu_meta_wr_en, + input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, + input wire [4:0] tcu_meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, +`endif + + // core-wide warp control + VX_warp_shared_if.master warp_shared_if +); + `STATIC_ASSERT (COMPUTE_SLICES_ENABLE, ("invalid parameter")) + + `SCOPE_IO_SWITCH (2); + + VX_dispatch_if dispatch_if [NUM_EX_UNITS](); + VX_commit_if commit_if [NUM_EX_UNITS](); + VX_writeback_if writeback_if(); + + // issue + + wire issue_fire; + wire [ISSUE_WIS_W-1:0] issue_wis; + + VX_issue_slice #( + .INSTANCE_ID (`SFORMATF(("%s-issue%s", INSTANCE_ID, SLICE_STR))) + ) issue_slice ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .slice_id (slice_id), + `ifdef PERF_ENABLE + .issue_perf (issue_perf), + `endif + .decode_if (decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .warp_issued (issue_fire), + .warp_issued_wis (issue_wis) + ); + + `BUFFER(warp_issued, issue_fire); + `BUFFER(warp_issued_wis, issue_wis); + + // execute + + VX_execute_slice #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), + .SLICE_STR (SLICE_STR) + ) execute_slice ( + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (slice_id), + .dispatch_if (dispatch_if), + .commit_if (commit_if), + .branch_ctl_if (branch_ctl_if), + .lsu_client_if (lsu_client_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if (sched_unlock_if), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid), + .tcu_exec_ready (tcu_exec_ready), + .tcu_exec_data (tcu_exec_data), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), + .tcu_tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid), + .tcu_agu_ld_data (tcu_agu_ld_data), + .tcu_agu_ld_ready (tcu_agu_ld_ready), + .tcu_agu_result_valid (tcu_agu_result_valid), + .tcu_agu_result_data (tcu_agu_result_data), + .tcu_agu_result_ready (tcu_agu_result_ready), + .tcu_meta_wr_en (tcu_meta_wr_en), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if) + ); + + // commit + + VX_commit_slice #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit_slice ( + .clk (clk), + .reset (reset), + .commit_if (commit_if), + .writeback_if (writeback_if), + .committed_wis (committed_wis) + ); + +endmodule diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 618fabade9..b8aaf1d528 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -17,8 +17,11 @@ `include "VX_fpu_define.vh" `endif -module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter CORE_ID = 0, +module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -27,6 +30,8 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + `ifdef PERF_ENABLE input sysmem_perf_t sysmem_perf, `endif @@ -35,13 +40,11 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], - VX_mem_bus_if.master icache_bus_if, + VX_mem_bus_if.master icache_bus_if [ICACHE_NUM_REQS], `ifdef VX_CFG_VM_ENABLE - // Address translation is relocated to the socket (one MMU per L1 cache); the - // core emits virtual addresses on its cache buses. The socket returns its - // MMU drain state so the core's busy/barrier logic still waits for in-flight - // translations to retire before completion/suspend. + // Translation happens at the socket (one MMU per L1 cache): the core emits + // virtual addresses, and busy/barrier logic waits on the socket MMU draining. input wire mmu_drained, `endif @@ -54,12 +57,6 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_tex_bus_if.master tex_bus_if, `endif -`ifdef VX_CFG_EXT_OM_ENABLE -`endif - -`ifdef VX_CFG_EXT_RASTER_ENABLE -`endif - `ifdef VX_CFG_EXT_RTU_ENABLE VX_rtu_bus_if.master rtu_bus_if, `endif @@ -77,40 +74,19 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // Status output wire busy ); - VX_schedule_if schedule_if(); - VX_fetch_if fetch_if(); - VX_decode_if decode_if(); - VX_sched_csr_if sched_csr_if(); - VX_decode_sched_if decode_sched_if(); - VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); - VX_commit_sched_if commit_sched_if(); - VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); - VX_warp_ctl_if warp_ctl_if(); -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if sched_unlock_if(); // RTU TRACE wstall release -> scheduler -`endif - - VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); + VX_warp_shared_if warp_shared_if[`VX_CFG_NUM_SFU_BLOCKS](); `ifdef VX_CFG_EXT_DXA_ENABLE VX_txbar_bus_if dxa_txbar_bus_if(); `endif - // cta_table_if removed: cluster-contiguous LMEM placement lets the - // DXA multicast writer compute receiver addresses as - // `issuer_base + r × smem_stride`, eliminating the per-slot lookup table. - VX_lsu_mem_if #( .NUM_LANES (`VX_CFG_NUM_LSU_LANES), .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) ) lsu_mem_if[`VX_CFG_NUM_LSU_BLOCKS](); - // VX_lsu_scheduler instantiated per-LSU-block; all blocks have NUM_CLIENTS=2. - // Block 0 wires client 1 to the warp-level TCU AGU; other blocks tie it off. - // LSU client interfaces flow from execute as client 0. + // per-LSU-block memory scheduler clients from execute VX_lsu_sched_if lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS](); wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; @@ -118,24 +94,11 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_lsu_sched_if tcu_mem_if(); `endif - VX_mem_bus_if #( - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_TAG_WIDTH_BASE) - ) mmu_dcache_if[DCACHE_NUM_REQS](); - - // VX_fetch -> VX_dcr_flush -> mmu_icache_if -> (i)MMU -> icache_bus_if. - // The flush wrapper consumes the dcr-triggered cache-flush request and - // injects a synthetic flush request into the icache stream, mirroring - // the dcache path in VX_mem_unit. + // fetch-slice icache requests; port 0 also carries the injected flush VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) - ) fetch_icache_if[1](); - - VX_mem_bus_if #( - .DATA_SIZE (ICACHE_WORD_SIZE), - .TAG_WIDTH (ICACHE_TAG_WIDTH_BASE) - ) mmu_icache_if[1](); + ) fetch_icache_if[ICACHE_NUM_REQS](); `ifdef VX_CFG_TCU_WGMMA_ENABLE localparam TCU_LMEM_BANK_ADDR_W = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE) - `CLOG2(`VX_CFG_LMEM_NUM_BANKS); @@ -147,7 +110,6 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( ) tcu_lmem_if(); `endif - `ifdef PERF_ENABLE lmem_perf_t lmem_perf; coalescer_perf_t coalescer_perf; @@ -166,35 +128,25 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_dcr_csr_if dcr_csr_if(); - // Single DCR-flush trigger from VX_dcr_data, fanned out below to BOTH - // the dcache (via VX_mem_unit) and the icache (via the wrapper inserted - // between VX_fetch and the iMMU). Without invalidating the icache too, - // a kernel re-loaded to the same VMA after a CACHE_FLUSH would execute - // stale lines from the previous launch. + // One DCR flush request invalidates both L1s: a kernel re-loaded at the same + // address must not execute stale icache lines. VX_dcr_flush_if dcr_flush_if(); VX_dcr_flush_if dcr_flush_dcache_if(); VX_dcr_flush_if dcr_flush_icache_if(); wire dcr_flush_l1_only; - // High when the in-core global-store path is empty (no store still in - // flight toward the dcache). Gates `busy` so the flush waits for the tail. + // the in-core global-store path is empty wire mem_unit_empty; - // Hold the dcache flush request until this core's stores have all reached L1. - // Stores are ack-less and lag warp-exit by many cycles, and the flush arbiter - // sits at the adapter output with no upstream visibility, so without this gate a - // flush could inject ahead of a store still in the LSU/coalescer and lose it. + // Stores are ack-less and lag warp exit, so the dcache flush waits until every + // store of this core has reached L1. wire store_drained = (&lsu_sched_empty) && mem_unit_empty; assign dcr_flush_dcache_if.req = dcr_flush_if.req && store_drained; // Only the dcache flush walks the shared levels (the icache does not // forward its flush), so the icache flushes alongside it. assign dcr_flush_icache_if.req = dcr_flush_if.req; - // Each VX_dcr_flush holds .done level-high until its req drops, so a - // straight AND of the two dones reports the combined completion to - // VX_dcr_data — which then drops req, re-arming both for the next flush. - // When gfx extensions are enabled, also gate on the cluster-shared - // gfx-cache flush done so that VX_dcr_data doesn't retire the flush DCR - // before tcache/rcache/ocache have actually invalidated. + // Each flush holds done until its req drops, so the dones AND into the combined + // completion (with the cluster graphics caches when present). `ifdef EXT_GFX_ANY_ENABLE assign cluster_flush_if.req = dcr_flush_if.req; assign dcr_flush_if.done = dcr_flush_dcache_if.done & dcr_flush_icache_if.done & cluster_flush_if.done; @@ -203,179 +155,531 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `endif VX_dcr_data #( - .INSTANCE_ID (`SFORMATF(("%s-dcr_data", INSTANCE_ID))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("%s-dcr_data", INSTANCE_ID))) ) dcr_data ( .clk (clk), .reset (reset), + .core_id (core_id), .dcr_bus_if (dcr_bus_if), .dcr_csr_if (dcr_csr_if), .dcr_flush_if(dcr_flush_if), .flush_l1_only(dcr_flush_l1_only) ); - `SCOPE_IO_SWITCH (3); - - wire sched_busy; - VX_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) scheduler ( - .clk (clk), - .reset (reset), + wire [PERF_CTR_BITS-1:0] sched_cycles; + wire [PERF_CTR_BITS-1:0] sched_instret; + wire lsu_drained; + VX_csr_mpm #( + .INSTANCE_ID (`SFORMATF(("%s-mpm", INSTANCE_ID))) + ) csr_mpm ( `ifdef PERF_ENABLE - .sched_perf (pipeline_perf.sched), - `endif - - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if), - `ifdef VX_CFG_EXT_RTU_ENABLE - .sched_unlock_if (sched_unlock_if), + .sysmem_perf (sysmem_perf_tmp), + .pipeline_perf (pipeline_perf), `endif - - .decode_sched_if(decode_sched_if), - .issue_sched_if (issue_sched_if), - .commit_sched_if(commit_sched_if), - - .kmu_bus_if (kmu_bus_if), - - .schedule_if (schedule_if), - .sched_csr_if (sched_csr_if), - .gbar_bus_if (gbar_bus_if), - - .busy (sched_busy) + .cycles (sched_cycles), + .instret (sched_instret), + .dcr_csr_if (dcr_csr_if) ); - VX_fetch #( - .INSTANCE_ID (`SFORMATF(("%s-fetch", INSTANCE_ID))) - ) fetch ( - `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .fetch_perf (pipeline_perf.fetch), - `endif - .icache_bus_if (fetch_icache_if[0]), - .schedule_if (schedule_if), - .fetch_if (fetch_if) - ); + `SCOPE_IO_SWITCH (3); - // Inject CACHE_FLUSH onto the icache stream so a host-side CMD_CACHE_FLUSH - // (issued after every kernel launch) invalidates instruction-cache lines - // belonging to the previous kernel image. + // Port 0 injects the cache flush into its request stream; the cache flushes + // every bank. The merged request leaves the core registered. VX_dcr_flush #( - .WORD_SIZE (ICACHE_WORD_SIZE), - .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH), - .REQ_OUT_BUF (3) // register core icache-request boundary; rsp already registered by L1 CORE_OUT_BUF + .WORD_SIZE (ICACHE_WORD_SIZE), + .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH), + .REQ_OUT_BUF (3) ) icache_dcr_flush ( .clk (clk), .reset (reset), .dcr_flush_if (dcr_flush_icache_if), .l1_only (dcr_flush_l1_only), .core_bus_if (fetch_icache_if[0]), - .cache_bus_if (mmu_icache_if[0]) + .cache_bus_if (icache_bus_if[0]) ); - VX_decode #( - .INSTANCE_ID (`SFORMATF(("%s-decode", INSTANCE_ID))) - ) decode ( - .clk (clk), - .reset (reset), - .fetch_if (fetch_if), - .decode_if (decode_if), - .decode_sched_if(decode_sched_if) - ); + // The other ports carry their fetch slice's registered requests, with the + // flush arbiter's select bit, the tag's LSB, clear. + for (genvar i = 1; i < ICACHE_NUM_REQS; ++i) begin : g_icache_ports + assign icache_bus_if[i].req_valid = fetch_icache_if[i].req_valid; + assign icache_bus_if[i].req_data.rw = fetch_icache_if[i].req_data.rw; + assign icache_bus_if[i].req_data.addr = fetch_icache_if[i].req_data.addr; + assign icache_bus_if[i].req_data.data = fetch_icache_if[i].req_data.data; + assign icache_bus_if[i].req_data.byteen = fetch_icache_if[i].req_data.byteen; + assign icache_bus_if[i].req_data.attr = fetch_icache_if[i].req_data.attr; + assign icache_bus_if[i].req_data.tag = {fetch_icache_if[i].req_data.tag, 1'b0}; + assign fetch_icache_if[i].req_ready = icache_bus_if[i].req_ready; + + assign fetch_icache_if[i].rsp_valid = icache_bus_if[i].rsp_valid; + assign fetch_icache_if[i].rsp_data.data = icache_bus_if[i].rsp_data.data; + assign fetch_icache_if[i].rsp_data.tag = icache_bus_if[i].rsp_data.tag[ICACHE_TAG_WIDTH-1:1]; + assign icache_bus_if[i].rsp_ready = fetch_icache_if[i].rsp_ready; + `UNUSED_VAR (icache_bus_if[i].rsp_data.tag[0]) + end - VX_issue #( - .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) - ) issue ( - `SCOPE_IO_BIND (1) + // scheduling and the front end: the core scheduler, FETCH_WIDTH fetch slices + // and the crossbar to the issue slices and ALU/SFU blocks - .clk (clk), - .reset (reset), + localparam FW = `VX_CFG_FETCH_WIDTH; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; - `ifdef PERF_ENABLE - .issue_perf (pipeline_perf.issue), - `endif + VX_slice_sched_if slice_sched_if[FW](); + VX_decode_if fs_decode_if[FW](); + VX_decode_if is_decode_if[`VX_CFG_ISSUE_WIDTH](); + VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); + VX_warp_ctl_if warp_ctl_if[`VX_CFG_NUM_SFU_BLOCKS](); + VX_sched_csr_if sched_csr_if[`VX_CFG_NUM_SFU_BLOCKS](); +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if sched_unlock_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif - .decode_if (decode_if), - .writeback_if (writeback_if), - .dispatch_if (dispatch_if), - .issue_sched_if (issue_sched_if) - ); + wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid; + wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis; + wire [`VX_CFG_NUM_WARPS-1:0] committed_warps; + wire [`VX_CFG_NUM_WARPS-1:0] sched_active_warps; +`ifdef VX_CFG_VM_ENABLE + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] sched_satp; + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid; + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data; +`endif - VX_execute #( - .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) execute ( - `SCOPE_IO_BIND (2) + sched_branch_t [FW-1:0][NPA-1:0] fs_branch; + sched_wctl_t [FW-1:0][NPS-1:0] fs_wctl; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_dvstack_wid; + wire [FW-1:0][NPS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr; + wire [FW-1:0][NPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid; + sched_csr_rd_t [FW-1:0][NPS-1:0] fs_csr_rd; + sched_csr_wr_t [FW-1:0][NPS-1:0] fs_csr_wr; +`ifdef VX_CFG_EXT_RTU_ENABLE + wire [FW-1:0][NPS-1:0] fs_rtu_unlock_valid; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid; +`endif + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_issued; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_committed; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty; + + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id; + cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx; + for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_csr_rd_cta_id + assign csr_rd_cta_id[b] = sched_csr_if[b].csr_rd_cta_id; + end + VX_scheduler #( + .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))) + ) scheduler ( .clk (clk), .reset (reset), - + .core_id (core_id), `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf_tmp), - .pipeline_perf (pipeline_perf), - `ifdef VX_CFG_EXT_TCU_ENABLE - .tcu_perf (tcu_perf), + .sched_perf (pipeline_perf.sched), `endif + .slice_if (slice_sched_if), + .warp_shared_if (warp_shared_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .txbar_bus_if (dxa_txbar_bus_if), `endif + .committed_warps(committed_warps), + .csr_rd_cta_id (csr_rd_cta_id), + .csr_rd_ctx (csr_rd_ctx), + .kmu_bus_if (kmu_bus_if), + .gbar_bus_if (gbar_bus_if), + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp), + .satp_wr_valid (satp_wr_valid), + .satp_wr_data (satp_wr_data), + `endif + .busy (sched_busy) + ); - // execute exposes LSU client interfaces; lsu_scheduler sits between - // execute and lsu_mem_if (which connects to mem_unit). - .lsu_client_if (lsu_client_if), - - `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + VX_fetch_xbar fetch_xbar ( + .fs_decode_if (fs_decode_if), + .is_decode_if (is_decode_if), + .issue_valid (issue_valid), + .issue_wis (issue_wis), + .committed_warps (committed_warps), + .fs_issued (fs_issued), + .fs_committed (fs_committed), + .fs_pending_alm_empty (fs_pending_alm_empty), + .branch_ctl_if (branch_ctl_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if), `endif + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + .lsu_sched_drained (lsu_drained), + .csr_rd_ctx (csr_rd_ctx), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp), + .satp_wr_valid (satp_wr_valid), + .satp_wr_data (satp_wr_data), + `endif + .fs_branch (fs_branch), + .fs_wctl (fs_wctl), + .fs_dvstack_wid (fs_dvstack_wid), + .fs_dvstack_ptr (fs_dvstack_ptr), + .fs_dvstack_mscratch (fs_dvstack_mscratch), + .fs_csr_rd_wid (fs_csr_rd_wid), + .fs_csr_rd (fs_csr_rd), + `ifdef VX_CFG_EXT_RTU_ENABLE + .fs_rtu_unlock_valid (fs_rtu_unlock_valid), + .fs_rtu_unlock_wid (fs_rtu_unlock_wid), + `endif + .fs_csr_wr (fs_csr_wr) + ); + +`ifdef PERF_ENABLE + fetch_perf_t per_slice_fetch_perf [FW]; + `PERF_COUNTER_ADD (pipeline_perf.fetch, per_slice_fetch_perf, stalls, PERF_CTR_BITS, FW, (FW > 2)) +`endif + +`ifdef SCOPE + wire [FW-1:0] fetch_scope_bus_in, fetch_scope_bus_out; + VX_scope_switch #( + .N (FW) + ) fetch_scope_switch ( + .clk (clk), + .reset (scope_reset_w[0]), + .req_in (scope_bus_in_w[0]), + .rsp_out (scope_bus_out_w[0]), + .req_out (fetch_scope_bus_in), + .rsp_in (fetch_scope_bus_out) + ); +`endif - .dispatch_if (dispatch_if), - .commit_if (commit_if), + for (genvar f = 0; f < FW; ++f) begin : g_fetch_slices + VX_fetch_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR ((FW > 1) ? `SFORMATF(("%0d", f)) : "") + ) fetch_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[0]), + .scope_bus_in (fetch_scope_bus_in[f]), + .scope_bus_out (fetch_scope_bus_out[f]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (FETCH_FSW_W'(f)), + `ifdef PERF_ENABLE + .fetch_perf (per_slice_fetch_perf[f]), + `endif + .slice_sched_if (slice_sched_if[f]), + .icache_bus_if (fetch_icache_if[f]), + .decode_if (fs_decode_if[f]), + .branch (fs_branch[f]), + .wctl (fs_wctl[f]), + .dvstack_wid (fs_dvstack_wid[f]), + .dvstack_ptr (fs_dvstack_ptr[f]), + .dvstack_mscratch (fs_dvstack_mscratch[f]), + .csr_rd_wid (fs_csr_rd_wid[f]), + .csr_rd (fs_csr_rd[f]), + .csr_wr (fs_csr_wr[f]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (fs_rtu_unlock_valid[f]), + .rtu_unlock_wid (fs_rtu_unlock_wid[f]), + `endif + .issued (fs_issued[f]), + .committed (fs_committed[f]), + .pending_alm_empty (fs_pending_alm_empty[f]) + ); + end - .sched_csr_if (sched_csr_if), + // issue, execute and commit - .dcr_csr_if (dcr_csr_if), + if (COMPUTE_SLICES_ENABLE) begin : g_compute_slices - `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + `ifdef PERF_ENABLE + issue_perf_t per_slice_issue_perf [`VX_CFG_ISSUE_WIDTH]; + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, ibf_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, scb_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, opd_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + for (genvar i = 0; i < NUM_EX_UNITS; ++i) begin : g_issue_perf_units_uses + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_stalls[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_instrs[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + end `endif + `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_req_bus_if (dxa_req_bus_if), - .dxa_txbar_bus_if(dxa_txbar_bus_if), + VX_dxa_req_bus_if slice_dxa_req_bus_if[`VX_CFG_ISSUE_WIDTH](); `endif `ifdef VX_CFG_EXT_TEX_ENABLE - .tex_bus_if (tex_bus_if), - `endif - `ifdef VX_CFG_EXT_OM_ENABLE + VX_tex_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + ) slice_tex_bus_if[`VX_CFG_ISSUE_WIDTH](); `endif `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_bus_if (rtu_bus_if), - .sched_unlock_if (sched_unlock_if), + VX_rtu_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH) + ) slice_rtu_bus_if[`VX_CFG_ISSUE_WIDTH](); `endif - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if) - ); + `ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_ISSUE_WIDTH]; + wire [`VX_CFG_ISSUE_WIDTH-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_ISSUE_WIDTH-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_tbuf_ready; + `endif + `ifdef TCU_META_ENABLE + wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; + tcu_execute_t tcu_agu_ld_data [`VX_CFG_ISSUE_WIDTH]; + wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_agu_result_valid, tcu_agu_result_ready; + tcu_result_t tcu_agu_result_data [`VX_CFG_ISSUE_WIDTH]; + wire tcu_meta_wr_en; + wire [NW_WIDTH-1:0] tcu_meta_wr_wid; + wire [4:0] tcu_meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; + `endif - VX_commit #( - .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) - ) commit ( - .clk (clk), - .reset (reset), + `ifdef SCOPE + wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_scope_bus_in, slice_scope_bus_out; + VX_scope_switch #( + .N (`VX_CFG_ISSUE_WIDTH) + ) slice_scope_switch ( + .clk (clk), + .reset (scope_reset_w[1]), + .req_in (scope_bus_in_w[1]), + .rsp_out (scope_bus_out_w[1]), + .req_out (slice_scope_bus_in), + .rsp_in (slice_scope_bus_out) + ); + `endif + `SCOPE_IO_UNUSED (2) + + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices + wire [PER_ISSUE_WARPS-1:0] committed_wis; + + VX_compute_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR (`SFORMATF(("%0d", i))) + ) compute_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[1]), + .scope_bus_in (slice_scope_bus_in[i]), + .scope_bus_out (slice_scope_bus_out[i]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (ISSUE_ISW_W'(i)), + `ifdef PERF_ENABLE + .issue_perf (per_slice_issue_perf[i]), + `endif + .decode_if (is_decode_if[i]), + .warp_issued (issue_valid[i]), + .warp_issued_wis (issue_wis[i]), + .committed_wis (committed_wis), + .branch_ctl_if (branch_ctl_if[i]), + .warp_ctl_if (warp_ctl_if[i]), + .sched_csr_if (sched_csr_if[i]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if[i]), + `endif + .lsu_client_if (lsu_client_if[i]), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (slice_dxa_req_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (slice_tex_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (slice_rtu_bus_if[i]), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid[i]), + .tcu_exec_ready (tcu_exec_ready[i]), + .tcu_exec_data (tcu_exec_data[i]), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), + .tcu_tbuf_ready (tcu_tbuf_ready[i]), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid[i]), + .tcu_agu_ld_data (tcu_agu_ld_data[i]), + .tcu_agu_ld_ready (tcu_agu_ld_ready[i]), + .tcu_agu_result_valid (tcu_agu_result_valid[i]), + .tcu_agu_result_data (tcu_agu_result_data[i]), + .tcu_agu_result_ready (tcu_agu_result_ready[i]), + .tcu_meta_wr_en (tcu_meta_wr_en && (NW_WIDTH'(tcu_meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(i))), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + .warp_shared_if (warp_shared_if[i]) + ); + + // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_committed + assign committed_warps[w * `VX_CFG_ISSUE_WIDTH + i] = committed_wis[w]; + end + end + + VX_sfu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-execute-sfu-shared", INSTANCE_ID))) + ) sfu_shared ( + `ifdef VX_CFG_EXT_DXA_ENABLE + .block_dxa_req_bus_if (slice_dxa_req_bus_if), + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .block_tex_bus_if (slice_tex_bus_if), + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .block_rtu_bus_if (slice_rtu_bus_if), + .rtu_bus_if (rtu_bus_if), + `endif + .clk (clk), + .reset (reset) + ); + + `ifdef VX_CFG_EXT_TCU_ENABLE + VX_tcu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-execute-tcu", INSTANCE_ID))), + .BLOCK_SIZE (`VX_CFG_ISSUE_WIDTH) + ) tcu_shared ( + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), + `endif + .clk (clk), + .reset (reset) + ); + `endif - .commit_if (commit_if), + end else begin : g_stage_sliced - .writeback_if (writeback_if), + VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); + VX_commit_sched_if commit_sched_if(); - .commit_sched_if(commit_sched_if) - ); + VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); + + VX_issue #( + .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) + ) issue ( + `SCOPE_IO_BIND (1) + + .clk (clk), + .reset (reset), + + `ifdef PERF_ENABLE + .issue_perf (pipeline_perf.issue), + `endif + + .decode_if (is_decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .issue_sched_if (issue_sched_if) + ); - // Per-block VX_lsu_scheduler instances: all parameterized NUM_CLIENTS=2. - // Block 0 wires client 1 to the warp-level TCU AGU; other blocks tie it off. - // Symmetric NUM_CLIENTS keeps module generation uniform — tied-off clients - // cost only a few muxes inside the round-robin arbiter. + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_issue_sched + assign issue_valid[i] = issue_sched_if[i].valid; + assign issue_wis[i] = issue_sched_if[i].wis; + end + + VX_execute #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))) + ) execute ( + `SCOPE_IO_BIND (2) + + .clk (clk), + .reset (reset), + .core_id (core_id), + + `ifdef PERF_ENABLE + `ifdef VX_CFG_EXT_TCU_ENABLE + .tcu_perf (tcu_perf), + `endif + `endif + + // execute exposes LSU client interfaces; lsu_scheduler sits between + // execute and lsu_mem_if (which connects to mem_unit). + .lsu_client_if (lsu_client_if), + + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if), + `endif + + .dispatch_if (dispatch_if), + .commit_if (commit_if), + + .sched_csr_if (sched_csr_if), + + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_lmem_if (tcu_lmem_if), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if(sched_unlock_if), + `endif + + .warp_shared_if (warp_shared_if), + + .warp_ctl_if (warp_ctl_if), + .branch_ctl_if (branch_ctl_if) + ); + + VX_commit #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit ( + .clk (clk), + .reset (reset), + + .commit_if (commit_if), + + .writeback_if (writeback_if), + + .commit_sched_if(commit_sched_if) + ); + + assign committed_warps = commit_sched_if.committed_warps; + + end + + // One memory scheduler per LSU block. With the TCU metadata AGU, every block + // has a second client: block 0 serves the AGU and the others tie it off, which + // keeps the blocks identical at the cost of a few arbiter muxes. `ifdef TCU_META_ENABLE localparam LSU_SCHED_NUM_CLIENTS = 2; `else @@ -384,18 +688,16 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( for (genvar block_idx = 0; block_idx < `VX_CFG_NUM_LSU_BLOCKS; ++block_idx) begin : g_lsu_scheduler VX_lsu_sched_if sched_client_if [LSU_SCHED_NUM_CLIENTS](); - // Client 0 — LSU (per-block). Forward the per-block lsu_client_if - // master onto the scheduler's slave port via per-signal assigns - // (interface arrays can't be wired with a single assign). - assign sched_client_if[0].req_valid = lsu_client_if[block_idx].req_valid; - assign sched_client_if[0].req_data = lsu_client_if[block_idx].req_data; - assign lsu_client_if[block_idx].req_ready = sched_client_if[0].req_ready; - assign lsu_client_if[block_idx].rsp_valid = sched_client_if[0].rsp_valid; - assign lsu_client_if[block_idx].rsp_data = sched_client_if[0].rsp_data; - assign sched_client_if[0].rsp_ready = lsu_client_if[block_idx].rsp_ready; + // client 0: the block's LSU + assign sched_client_if[0].req_valid = lsu_client_if[block_idx].req_valid; + assign sched_client_if[0].req_data = lsu_client_if[block_idx].req_data; + assign lsu_client_if[block_idx].req_ready = sched_client_if[0].req_ready; + assign lsu_client_if[block_idx].rsp_valid = sched_client_if[0].rsp_valid; + assign lsu_client_if[block_idx].rsp_data = sched_client_if[0].rsp_data; + assign sched_client_if[0].rsp_ready = lsu_client_if[block_idx].rsp_ready; `ifdef TCU_META_ENABLE - // Client 1 — TCU AGU on block 0; tied off on other blocks. + // client 1: the TCU metadata AGU on block 0 if (block_idx == 0) begin : g_tcu_client assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; assign sched_client_if[1].req_data = tcu_mem_if.req_data; @@ -414,12 +716,12 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `endif VX_lsu_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d", INSTANCE_ID, block_idx))), - .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .CORE_QUEUE_SIZE(`VX_CFG_LSU_QUEUE_IN_SIZE), - .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) + .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d", INSTANCE_ID, block_idx))), + .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .CORE_QUEUE_SIZE (`VX_CFG_LSU_QUEUE_IN_SIZE), + .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), + .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) ) lsu_scheduler ( .clk (clk), .reset (reset), @@ -432,46 +734,33 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_mem_unit #( .INSTANCE_ID (INSTANCE_ID) ) mem_unit ( - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), `ifdef PERF_ENABLE - .lmem_perf (lmem_perf), - .coalescer_perf(coalescer_perf), + .lmem_perf (lmem_perf), + .coalescer_perf (coalescer_perf), `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .tcu_lmem_if (tcu_lmem_if), `endif `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_lmem_bus_if(dxa_lmem_bus_if), + .dxa_lmem_bus_if (dxa_lmem_bus_if), .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif - .empty (mem_unit_empty), - .lsu_mem_if (lsu_mem_if), - .dcr_flush_if (dcr_flush_dcache_if), - .flush_l1_only (dcr_flush_l1_only), - .dcache_bus_if (mmu_dcache_if) + .empty (mem_unit_empty), + .lsu_mem_if (lsu_mem_if), + .dcr_flush_if (dcr_flush_dcache_if), + .flush_l1_only (dcr_flush_l1_only), + .dcache_bus_if (dcache_bus_if) ); - // Address translation is relocated to the socket (one MMU per L1 cache). - // The core's dcache/icache buses carry the addresses the mem_unit and fetch - // produce straight through — virtual under VM, physical otherwise. Same - // widths on both sides (BASE == full for both L1s). - for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_passthru - `ASSIGN_VX_MEM_BUS_IF (dcache_bus_if[i], mmu_dcache_if[i]); - end - `ASSIGN_VX_MEM_BUS_IF (icache_bus_if, mmu_icache_if[0]); - `ifdef VX_CFG_VM_ENABLE `ifdef PERF_ENABLE - // The relocated MMU's perf counters live at the socket and are not yet routed - // through sysmem_perf, so the in-core pipeline view reads zero. + // the MMU perf counters are kept at the socket assign pipeline_perf.mmu = '0; `endif // Translation drains at the socket MMU; the socket returns its empty status. wire mmu_empty = mmu_drained; - // The socket MMU takes its satp from the shared DCR root, so the per-core CSR - // satp is not a translation source. - `UNUSED_VAR (sched_csr_if.csr_satp) `else wire mmu_empty = 1'b1; `endif @@ -485,12 +774,12 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // buffer, which mem_unit_empty does not cover. wire [DCACHE_NUM_REQS-1:0] dcache_req_valid; for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_req_valid - assign dcache_req_valid[i] = mmu_dcache_if[i].req_valid; + assign dcache_req_valid[i] = dcache_bus_if[i].req_valid; end assign busy = sched_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || (| dcache_req_valid) || ~mmu_empty; // BAR (vx_barrier / vx_barrier_arrive) drains LSU + MMU before suspending or registering arrival. - assign warp_ctl_if.lsu_sched_drained = (&lsu_sched_empty) && mmu_empty; + assign lsu_drained = (&lsu_sched_empty) && mmu_empty; `ifdef PERF_ENABLE @@ -498,7 +787,9 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [`CLOG2(LSU_NUM_REQS+1)-1:0] perf_dcache_wr_req_per_cycle; wire [`CLOG2(LSU_NUM_REQS+1)-1:0] perf_dcache_rsp_per_cycle; - wire [1:0] perf_icache_pending_read_cycle; + wire [`CLOG2(ICACHE_NUM_REQS+1)-1:0] perf_icache_req_per_cycle; + wire [`CLOG2(ICACHE_NUM_REQS+1)-1:0] perf_icache_rsp_per_cycle; + wire [`CLOG2(ICACHE_NUM_REQS+1)+1-1:0] perf_icache_pending_read_cycle; wire [`CLOG2(LSU_NUM_REQS+1)+1-1:0] perf_dcache_pending_read_cycle; reg [PERF_CTR_BITS-1:0] perf_icache_pending_reads; @@ -508,8 +799,14 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( reg [PERF_CTR_BITS-1:0] perf_loads; reg [PERF_CTR_BITS-1:0] perf_stores; - wire perf_icache_req_fire = icache_bus_if.req_valid && icache_bus_if.req_ready; - wire perf_icache_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; + wire [ICACHE_NUM_REQS-1:0] perf_icache_req_fire; + wire [ICACHE_NUM_REQS-1:0] perf_icache_rsp_fire; + for (genvar i = 0; i < ICACHE_NUM_REQS; ++i) begin : g_perf_icache + assign perf_icache_req_fire[i] = icache_bus_if[i].req_valid && icache_bus_if[i].req_ready; + assign perf_icache_rsp_fire[i] = icache_bus_if[i].rsp_valid && icache_bus_if[i].rsp_ready; + end + `POP_COUNT(perf_icache_req_per_cycle, perf_icache_req_fire); + `POP_COUNT(perf_icache_rsp_per_cycle, perf_icache_rsp_fire); wire [LSU_NUM_REQS-1:0] perf_dcache_rd_req_fire, perf_dcache_rd_req_fire_r; wire [LSU_NUM_REQS-1:0] perf_dcache_wr_req_fire, perf_dcache_wr_req_fire_r; @@ -530,7 +827,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `POP_COUNT(perf_dcache_wr_req_per_cycle, perf_dcache_wr_req_fire_r); `POP_COUNT(perf_dcache_rsp_per_cycle, perf_dcache_rsp_fire); - assign perf_icache_pending_read_cycle = perf_icache_req_fire - perf_icache_rsp_fire; + assign perf_icache_pending_read_cycle = perf_icache_req_per_cycle - perf_icache_rsp_per_cycle; assign perf_dcache_pending_read_cycle = perf_dcache_rd_req_per_cycle - perf_dcache_rsp_per_cycle; always @(posedge clk) begin @@ -554,7 +851,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( perf_icache_lat <= '0; perf_dcache_lat <= '0; end else begin - perf_ifetches <= perf_ifetches + PERF_CTR_BITS'(perf_icache_req_fire); + perf_ifetches <= perf_ifetches + PERF_CTR_BITS'(perf_icache_req_per_cycle); perf_loads <= perf_loads + PERF_CTR_BITS'(perf_dcache_rd_req_per_cycle); perf_stores <= perf_stores + PERF_CTR_BITS'(perf_dcache_wr_req_per_cycle); perf_icache_lat <= perf_icache_lat + perf_icache_pending_reads; diff --git a/hw/rtl/core/VX_csr_cta.sv b/hw/rtl/core/VX_csr_cta.sv new file mode 100644 index 0000000000..8ceb33a7c4 --- /dev/null +++ b/hw/rtl/core/VX_csr_cta.sv @@ -0,0 +1,75 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Per-warp CTA state for one group of warps, addressed by group-local warp id: +// the rank, lane record and CTA id of the group's own warps. Written by the +// dispatcher's broadcast; the parent presents only writes that belong to this +// group. The CTA contexts every warp may read are held core-wide. +module VX_csr_cta import VX_gpu_pkg::*; #( + parameter NUM_WARPS = 1, + parameter LW_WIDTH = `LOG2UP(NUM_WARPS) +) ( + input wire clk, + input wire reset, + + // this group's warps + input wire warp_write, + input wire [LW_WIDTH-1:0] warp_waddr, + input cta_warp_t warp_wdata, + input wire id_write, + input wire [LW_WIDTH-1:0] id_waddr, + input wire [NCTA_WIDTH-1:0] id_wdata, + + // CSR read: rdata is valid one cycle after the address, which the CSR unit + // holds stable for that cycle + input wire [LW_WIDTH-1:0] rd_lid, + output wire [NW_WIDTH-1:0] rd_cta_rank, + output cta_lane_t [`VX_CFG_NUM_THREADS-1:0] rd_lane, + + // CTA id of a warp, combinational + input wire [LW_WIDTH-1:0] sched_lid, + output wire [NCTA_WIDTH-1:0] sched_cta_id +); + cta_warp_t warp_rdata; + + VX_dp_ram #( + .DATAW ($bits(cta_warp_t)), + .SIZE (NUM_WARPS), + .RDW_MODE ("R"), + .OUT_REG (1) + ) cta_warp_ram ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (warp_write), + .wren (1'b1), + .waddr (warp_waddr), + .wdata (warp_wdata), + .raddr (rd_lid), + .rdata (warp_rdata) + ); + + assign rd_cta_rank = warp_rdata.cta_rank; + assign rd_lane = warp_rdata.lane_launch; + + reg [NUM_WARPS-1:0][NCTA_WIDTH-1:0] cta_id_r; + always @(posedge clk) begin + if (id_write) begin + cta_id_r[id_waddr] <= id_wdata; + end + end + assign sched_cta_id = cta_id_r[sched_lid]; + +endmodule diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index f09c17bb7e..e7a8799dc6 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -17,34 +17,24 @@ `include "VX_fpu_define.vh" `endif -`ifdef VX_CFG_XLEN_64 - `define CSR_READ_64(addr, dst, src) \ - addr : dst = `VX_CFG_XLEN'(src) -`else - `define CSR_READ_64(addr, dst, src) \ - addr : dst = src[31:0]; \ - addr+12'h80 : dst = 32'(src[$bits(src)-1:32]) -`endif - module VX_csr_data import VX_gpu_pkg::*; `ifdef VX_CFG_EXT_F_ENABLE import VX_fpu_pkg::*; `endif #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, -`ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`endif + input wire [CORE_ID_WIDTH-1:0] core_id, `ifdef VX_CFG_EXT_F_ENABLE - VX_fpu_csr_if.slave fpu_csr_if [`VX_CFG_NUM_FPU_BLOCKS], + // FP CSR of the requesting warp: read live, written at fire + input wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data, + output wire [2:0] fcsr_wr_sel, // {fcsr, frm, fflags} + output wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data, `endif VX_sched_csr_if.slave sched_csr_if, @@ -58,12 +48,7 @@ import VX_fpu_pkg::*; input wire [`VX_CSR_ADDR_BITS-1:0] req_addr, input wire [INST_SFU_BITS-1:0] req_op, input wire [`VX_CFG_XLEN-1:0] req_src, - output wire [`VX_CFG_XLEN-1:0] read_data, - - // Host performance-counter reads - input wire [7:0] dcr_mpm_class, - input wire [`VX_CSR_ADDR_BITS-1:0] dcr_addr, - output wire [`VX_CFG_XLEN-1:0] dcr_data + output wire [`VX_CFG_XLEN-1:0] read_data ); `UNUSED_SPARAM (INSTANCE_ID) @@ -91,7 +76,8 @@ import VX_fpu_pkg::*; localparam LIVE_LMEM_ADDR = 20; localparam LIVE_CLUSTER = 21; localparam LIVE_ENTRY = 22; - localparam NUM_LIVE = 23; + localparam LIVE_SATP = 23; + localparam NUM_LIVE = 24; // Writable CSRs localparam WR_MSCRATCH = 0; @@ -114,10 +100,6 @@ import VX_fpu_pkg::*; endcase endfunction -`ifdef VX_CFG_VM_ENABLE - reg [`VX_CFG_XLEN-1:0] satp; -`endif - // Decode //////////////////////////////////////////////////////////////// // Scheduler CSRs read interface @@ -161,7 +143,7 @@ import VX_fpu_pkg::*; `VX_CSR_CTA_ENTRY : live_sel_w[LIVE_ENTRY] = 1; `VX_CSR_WARP_ID : stable_w = `VX_CFG_XLEN'(req_wid); - `VX_CSR_CORE_ID : stable_w = `VX_CFG_XLEN'(CORE_ID); + `VX_CSR_CORE_ID : stable_w = `VX_CFG_XLEN'(core_id); `VX_CSR_ACTIVE_THREADS: live_sel_w[LIVE_THREADS] = 1; `VX_CSR_ACTIVE_WARPS: live_sel_w[LIVE_WARPS] = 1; `VX_CSR_NUM_THREADS: stable_w = `VX_CFG_XLEN'(`VX_CFG_NUM_THREADS); @@ -178,7 +160,7 @@ import VX_fpu_pkg::*; `endif `ifdef VX_CFG_VM_ENABLE - `VX_CSR_SATP : begin stable_w = satp; wr_sel_w[WR_SATP] = 1; end + `VX_CSR_SATP : begin live_sel_w[LIVE_SATP] = 1; wr_sel_w[WR_SATP] = 1; end `endif // Machine-mode trap CSRs (stored in the scheduler). @@ -228,8 +210,7 @@ import VX_fpu_pkg::*; // Live values /////////////////////////////////////////////////////////// `ifdef VX_CFG_EXT_F_ENABLE - reg [`VX_CFG_NUM_WARPS-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr, fcsr_n; - wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] req_fcsr = fcsr[req_wid]; + wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] req_fcsr = fcsr_rd_data; `endif wire [NUM_LIVE-1:0][`VX_CFG_XLEN-1:0] live_src; @@ -245,7 +226,7 @@ import VX_fpu_pkg::*; assign live_src[LIVE_INSTRET] = sched_csr_if.instret[31:0]; assign live_src[LIVE_INSTRET_H] = 32'(sched_csr_if.instret[PERF_CTR_BITS-1:32]); `endif - assign live_src[LIVE_THREADS] = `VX_CFG_XLEN'(sched_csr_if.thread_masks[req_wid]); + assign live_src[LIVE_THREADS] = `VX_CFG_XLEN'(sched_csr_if.thread_mask); assign live_src[LIVE_WARPS] = `VX_CFG_XLEN'(sched_csr_if.active_warps); `ifdef VX_CFG_EXT_F_ENABLE assign live_src[LIVE_FFLAGS] = `VX_CFG_XLEN'(req_fcsr[`FP_FLAGS_BITS-1:0]); @@ -268,6 +249,11 @@ import VX_fpu_pkg::*; assign live_src[LIVE_LMEM_ADDR] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.lmem_addr); assign live_src[LIVE_CLUSTER] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cluster_size); assign live_src[LIVE_ENTRY] = `VX_CFG_XLEN'(to_fullPC(sched_csr_if.cta_csrs.entry)); +`ifdef VX_CFG_VM_ENABLE + assign live_src[LIVE_SATP] = sched_csr_if.csr_satp; +`else + assign live_src[LIVE_SATP] = '0; +`endif reg [`VX_CFG_XLEN-1:0] live_data; always @(*) begin @@ -294,66 +280,17 @@ import VX_fpu_pkg::*; `ifdef VX_CFG_EXT_F_ENABLE // The FP flags keep moving under in-flight FP instructions, so their // read-modify-write stays in the fire cycle. - wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data = - (INST_FRM_BITS+`FP_FLAGS_BITS)'(csr_rmw(req_op, live_data, req_src)); - - wire [`VX_CFG_NUM_FPU_BLOCKS-1:0] fpu_write_enable; - wire [`VX_CFG_NUM_FPU_BLOCKS-1:0][NW_WIDTH-1:0] fpu_write_wid; - fflags_t [`VX_CFG_NUM_FPU_BLOCKS-1:0] fpu_write_fflags; - - for (genvar i = 0; i < `VX_CFG_NUM_FPU_BLOCKS; ++i) begin : g_fpu_write - assign fpu_write_enable[i] = fpu_csr_if[i].write_enable; - assign fpu_write_wid[i] = fpu_csr_if[i].write_wid; - assign fpu_write_fflags[i] = fpu_csr_if[i].write_fflags; - end - - always @(*) begin - fcsr_n = fcsr; - for (integer i = 0; i < `VX_CFG_NUM_FPU_BLOCKS; ++i) begin - if (fpu_write_enable[i]) begin - fcsr_n[fpu_write_wid[i]][`FP_FLAGS_BITS-1:0] = fcsr[fpu_write_wid[i]][`FP_FLAGS_BITS-1:0] - | fpu_write_fflags[i]; - end - end - if (write_sel[WR_FFLAGS]) begin - fcsr_n[req_wid][`FP_FLAGS_BITS-1:0] = fcsr_wr_data[`FP_FLAGS_BITS-1:0]; - end - if (write_sel[WR_FRM]) begin - fcsr_n[req_wid][INST_FRM_BITS+`FP_FLAGS_BITS-1:`FP_FLAGS_BITS] = fcsr_wr_data[INST_FRM_BITS-1:0]; - end - if (write_sel[WR_FCSR]) begin - fcsr_n[req_wid] = fcsr_wr_data; - end - end - - for (genvar i = 0; i < `VX_CFG_NUM_FPU_BLOCKS; ++i) begin : g_fpu_csr_read_frm - assign fpu_csr_if[i].read_frm = fcsr[fpu_csr_if[i].read_wid][INST_FRM_BITS+`FP_FLAGS_BITS-1:`FP_FLAGS_BITS]; - end - - always @(posedge clk) begin - if (reset) begin - fcsr <= '0; - end else begin - fcsr <= fcsr_n; - end - end + assign fcsr_wr_data = (INST_FRM_BITS+`FP_FLAGS_BITS)'(csr_rmw(req_op, live_data, req_src)); + assign fcsr_wr_sel = {write_sel[WR_FCSR], write_sel[WR_FRM], write_sel[WR_FFLAGS]}; `else `UNUSED_VAR ({write_sel[WR_FCSR], write_sel[WR_FRM], write_sel[WR_FFLAGS]}) `endif `ifdef VX_CFG_VM_ENABLE - // Per-core SATP CSR. Initialized to 0 (BARE mode); kernel writes - // it from vx_start.S after the runtime has installed the page table. - // Surfaced on sched_csr_if.satp so VX_core can pick it up directly - // off the shared interface instead of routing through SFU/execute. - always @(posedge clk) begin - if (reset) begin - satp <= '0; - end else if (write_sel[WR_SATP]) begin - satp <= wr_data_r; - end - end - assign sched_csr_if.csr_satp = satp; + // The per-core SATP CSR lives in the scheduler, shared by every SFU block, + // so its read-modify-write stays in the fire cycle. + assign sched_csr_if.satp_wr_valid = write_sel[WR_SATP]; + assign sched_csr_if.satp_wr_data = csr_rmw(req_op, live_data, req_src); `else `UNUSED_VAR (write_sel[WR_SATP]) `endif @@ -386,210 +323,4 @@ import VX_fpu_pkg::*; end end - // Host performance-counter reads //////////////////////////////////////// - - reg [`VX_CFG_XLEN-1:0] dcr_data_w; - - always @(*) begin - dcr_data_w = '0; - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MCYCLE, dcr_data_w, sched_csr_if.cycles); - `CSR_READ_64(`VX_CSR_MINSTRET, dcr_data_w, sched_csr_if.instret); - default: begin - `ifdef PERF_ENABLE - if ((dcr_addr >= `VX_CSR_MPM_USER && dcr_addr < (`VX_CSR_MPM_USER + 32)) - || (dcr_addr >= `VX_CSR_MPM_USER_H && dcr_addr < (`VX_CSR_MPM_USER_H + 32))) begin - case (dcr_mpm_class) - `VX_DCR_MPM_CLASS_CORE: begin - case (dcr_addr) - // PERF: pipeline - `CSR_READ_64(`VX_CSR_MPM_SCHED_IDLE, dcr_data_w, pipeline_perf.sched.idles); - `CSR_READ_64(`VX_CSR_MPM_ACTIVE_WARPS, dcr_data_w, pipeline_perf.sched.active_warps); - `CSR_READ_64(`VX_CSR_MPM_STALLED_WARPS, dcr_data_w, pipeline_perf.sched.stalled_warps); - `CSR_READ_64(`VX_CSR_MPM_ISSUED_WARPS, dcr_data_w, pipeline_perf.sched.issued_warps); - `CSR_READ_64(`VX_CSR_MPM_ISSUED_THREADS, dcr_data_w, pipeline_perf.sched.issued_threads); - `CSR_READ_64(`VX_CSR_MPM_STALL_FETCH, dcr_data_w, pipeline_perf.fetch.stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_IBUF, dcr_data_w, pipeline_perf.issue.ibf_stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_SCRB, dcr_data_w, pipeline_perf.issue.scb_stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_OPDS, dcr_data_w, pipeline_perf.issue.opd_stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_ALU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_ALU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_ALU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_ALU]); - `CSR_READ_64(`VX_CSR_MPM_STALL_LSU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_LSU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_LSU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_LSU]); - `CSR_READ_64(`VX_CSR_MPM_STALL_SFU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_SFU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_SFU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_SFU]); - `ifdef VX_CFG_EXT_F_ENABLE - `CSR_READ_64(`VX_CSR_MPM_STALL_FPU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_FPU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_FPU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_FPU]); - `endif - `ifdef VX_CFG_EXT_TCU_ENABLE - `CSR_READ_64(`VX_CSR_MPM_STALL_TCU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_TCU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_TCU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_TCU]); - `endif - // PERF: branches - `CSR_READ_64(`VX_CSR_MPM_BRANCHES, dcr_data_w, pipeline_perf.sched.branches); - `CSR_READ_64(`VX_CSR_MPM_DIVERGENCE, dcr_data_w, pipeline_perf.sched.divergence); - // PERF: memory (core-issued requests; DRAM traffic is in the MEM class) - `CSR_READ_64(`VX_CSR_MPM_IFETCHES, dcr_data_w, pipeline_perf.ifetches); - `CSR_READ_64(`VX_CSR_MPM_LOADS, dcr_data_w, pipeline_perf.loads); - `CSR_READ_64(`VX_CSR_MPM_STORES, dcr_data_w, pipeline_perf.stores); - `CSR_READ_64(`VX_CSR_MPM_IFETCH_LT, dcr_data_w, pipeline_perf.ifetch_latency); - `CSR_READ_64(`VX_CSR_MPM_LOAD_LT, dcr_data_w, pipeline_perf.load_latency); - default:; - endcase - end - `VX_DCR_MPM_CLASS_ICACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_ICACHE_READS, dcr_data_w, sysmem_perf.icache.reads); - `CSR_READ_64(`VX_CSR_MPM_ICACHE_MISS_R, dcr_data_w, sysmem_perf.icache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_ICACHE_MSHR_ST, dcr_data_w, sysmem_perf.icache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_DCACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_DCACHE_READS, dcr_data_w, sysmem_perf.dcache.reads); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_WRITES, dcr_data_w, sysmem_perf.dcache.writes); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_R, dcr_data_w, sysmem_perf.dcache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_W, dcr_data_w, sysmem_perf.dcache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_EVICTS, dcr_data_w, sysmem_perf.dcache.evictions); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_BANK_ST, dcr_data_w, sysmem_perf.dcache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_MSHR_ST, dcr_data_w, sysmem_perf.dcache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_L2CACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_READS, dcr_data_w, sysmem_perf.l2cache.reads); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_WRITES, dcr_data_w, sysmem_perf.l2cache.writes); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_R, dcr_data_w, sysmem_perf.l2cache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_W, dcr_data_w, sysmem_perf.l2cache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_EVICTS, dcr_data_w, sysmem_perf.l2cache.evictions); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_BANK_ST, dcr_data_w, sysmem_perf.l2cache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l2cache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_L3CACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_READS, dcr_data_w, sysmem_perf.l3cache.reads); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_WRITES, dcr_data_w, sysmem_perf.l3cache.writes); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_R, dcr_data_w, sysmem_perf.l3cache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_W, dcr_data_w, sysmem_perf.l3cache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_EVICTS, dcr_data_w, sysmem_perf.l3cache.evictions); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_BANK_ST, dcr_data_w, sysmem_perf.l3cache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l3cache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_MEM: begin - case (dcr_addr) - // PERF: off-chip memory - `CSR_READ_64(`VX_CSR_MPM_MEM_READS, dcr_data_w, sysmem_perf.mem.reads); - `CSR_READ_64(`VX_CSR_MPM_MEM_WRITES, dcr_data_w, sysmem_perf.mem.writes); - `CSR_READ_64(`VX_CSR_MPM_MEM_LT, dcr_data_w, sysmem_perf.mem.latency); - // PERF: lmem - `CSR_READ_64(`VX_CSR_MPM_LMEM_READS, dcr_data_w, sysmem_perf.lmem.reads); - `CSR_READ_64(`VX_CSR_MPM_LMEM_WRITES, dcr_data_w, sysmem_perf.lmem.writes); - `CSR_READ_64(`VX_CSR_MPM_LMEM_BANK_ST, dcr_data_w, sysmem_perf.lmem.bank_stalls); - // PERF: coalescer - `CSR_READ_64(`VX_CSR_MPM_COALESCER_MISS, dcr_data_w, sysmem_perf.coalescer.misses); - `ifdef VX_CFG_VM_ENABLE - // PERF: VM/MMU (icache + dcache MMU summed) - `CSR_READ_64(`VX_CSR_MPM_TLB_READS, dcr_data_w, pipeline_perf.mmu.tlb_reads); - `CSR_READ_64(`VX_CSR_MPM_TLB_HITS, dcr_data_w, pipeline_perf.mmu.tlb_hits); - `CSR_READ_64(`VX_CSR_MPM_TLB_MISSES, dcr_data_w, pipeline_perf.mmu.tlb_misses); - `CSR_READ_64(`VX_CSR_MPM_TLB_EVICTS, dcr_data_w, pipeline_perf.mmu.tlb_evictions); - `CSR_READ_64(`VX_CSR_MPM_PTW_WALKS, dcr_data_w, pipeline_perf.mmu.ptw_walks); - `CSR_READ_64(`VX_CSR_MPM_PTW_LATENCY, dcr_data_w, pipeline_perf.mmu.ptw_latency); - `endif - default:; - endcase - end - `ifdef VX_CFG_EXT_DXA_ENABLE - `VX_DCR_MPM_CLASS_DXA: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_DXA_TRANSFERS, dcr_data_w, sysmem_perf.dxa.transfers); - `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_READS, dcr_data_w, sysmem_perf.dxa.gmem_reads); - `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_DEDUP, dcr_data_w, sysmem_perf.dxa.gmem_dedup); - `CSR_READ_64(`VX_CSR_MPM_DXA_LMEM_WRITES,dcr_data_w, sysmem_perf.dxa.lmem_writes); - `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_LT, dcr_data_w, sysmem_perf.dxa.gmem_latency); - `CSR_READ_64(`VX_CSR_MPM_DXA_NOSLOT_STALLS, dcr_data_w, sysmem_perf.dxa.noslot_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_TCU_ENABLE - `VX_DCR_MPM_CLASS_TCU: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_STALLS, dcr_data_w, pipeline_perf.tcu.tbuf_stalls); - `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_CACHE_HITS, dcr_data_w, pipeline_perf.tcu.tbuf_cache_hits); - `CSR_READ_64(`VX_CSR_MPM_TCU_LMEM_READS, dcr_data_w, pipeline_perf.tcu.lmem_reads); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - `VX_DCR_MPM_CLASS_TEX: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_TEX_READS, dcr_data_w, sysmem_perf.tex.mem_reads); - `CSR_READ_64(`VX_CSR_MPM_TEX_LAT, dcr_data_w, sysmem_perf.tex.mem_latency); - `CSR_READ_64(`VX_CSR_MPM_TEX_ST, dcr_data_w, sysmem_perf.tex.stall_cycles); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_READS, dcr_data_w, sysmem_perf.tcache.reads); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_MISS_R, dcr_data_w, sysmem_perf.tcache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_BANK_ST, dcr_data_w, sysmem_perf.tcache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_MSHR_ST, dcr_data_w, sysmem_perf.tcache.mshr_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_RASTER_ENABLE - `VX_DCR_MPM_CLASS_RASTER: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_RASTER_READS, dcr_data_w, sysmem_perf.raster.mem_reads); - `CSR_READ_64(`VX_CSR_MPM_RASTER_LAT, dcr_data_w, sysmem_perf.raster.mem_latency); - `CSR_READ_64(`VX_CSR_MPM_RASTER_ST, dcr_data_w, sysmem_perf.raster.stall_cycles); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_READS, dcr_data_w, sysmem_perf.rcache.reads); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_MISS_R, dcr_data_w, sysmem_perf.rcache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_BANK_ST, dcr_data_w, sysmem_perf.rcache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_MSHR_ST, dcr_data_w, sysmem_perf.rcache.mshr_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_OM_ENABLE - `VX_DCR_MPM_CLASS_OM: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_OM_READS, dcr_data_w, sysmem_perf.om.mem_reads); - `CSR_READ_64(`VX_CSR_MPM_OM_WRITES, dcr_data_w, sysmem_perf.om.mem_writes); - `CSR_READ_64(`VX_CSR_MPM_OM_LAT, dcr_data_w, sysmem_perf.om.mem_latency); - `CSR_READ_64(`VX_CSR_MPM_OM_ST, dcr_data_w, sysmem_perf.om.stall_cycles); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_READS, dcr_data_w, sysmem_perf.ocache.reads); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_WRITES, dcr_data_w, sysmem_perf.ocache.writes); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_R, dcr_data_w, sysmem_perf.ocache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_W, dcr_data_w, sysmem_perf.ocache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_BANK_ST, dcr_data_w, sysmem_perf.ocache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_MSHR_ST, dcr_data_w, sysmem_perf.ocache.mshr_stalls); - default:; - endcase - end - `endif - default:; - endcase - end - `endif - end - endcase - end - - assign dcr_data = dcr_data_w; -`ifndef PERF_ENABLE - `UNUSED_VAR (dcr_mpm_class) -`endif - -`ifdef PERF_ENABLE - `UNUSED_VAR (sysmem_perf.icache); - `UNUSED_VAR (sysmem_perf.lmem); -`endif - endmodule diff --git a/hw/rtl/core/VX_csr_fpu.sv b/hw/rtl/core/VX_csr_fpu.sv new file mode 100644 index 0000000000..7e3889c40f --- /dev/null +++ b/hw/rtl/core/VX_csr_fpu.sv @@ -0,0 +1,103 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" +`include "VX_fpu_define.vh" + +// Per-warp FP CSRs (frm + fflags) for one residue class of warps modulo WID_STRIDE, +// stored by wid / WID_STRIDE; every request presented belongs to that class. +// CSR port c serves the warps with (wid / WID_STRIDE) mod NUM_CSR_PORTS == c and +// FPU port f those with (wid / WID_STRIDE) mod NUM_FPU_PORTS == f, so each warp's +// state is written by exactly one port of each kind. +module VX_csr_fpu import VX_gpu_pkg::*, VX_fpu_pkg::*; #( + parameter WID_STRIDE = 1, + parameter NUM_CSR_PORTS = 1, + parameter NUM_FPU_PORTS = 1 +) ( + input wire clk, + input wire reset, + + input wire [NUM_CSR_PORTS-1:0][NW_WIDTH-1:0] csr_wid, + output wire [NUM_CSR_PORTS-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] csr_rd_data, + input wire [NUM_CSR_PORTS-1:0][2:0] csr_wr_sel, // {fcsr, frm, fflags} + input wire [NUM_CSR_PORTS-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] csr_wr_data, + + VX_fpu_csr_if.slave fpu_csr_if [NUM_FPU_PORTS] +); + localparam FCSR_W = INST_FRM_BITS + `FP_FLAGS_BITS; + localparam NUM_ENTRIES = `VX_CFG_NUM_WARPS / WID_STRIDE; + localparam STRIDE_BITS = `CLOG2(WID_STRIDE); + localparam ENTRY_W = `LOG2UP(NUM_ENTRIES); + + `STATIC_ASSERT (`IS_POW2(WID_STRIDE), ("invalid parameter")) + + function automatic logic [ENTRY_W-1:0] entry_of(input logic [NW_WIDTH-1:0] wid); + entry_of = ENTRY_W'(wid >> STRIDE_BITS); + endfunction + + wire [NUM_FPU_PORTS-1:0] fpu_wr_enable; + wire [NUM_FPU_PORTS-1:0][NW_WIDTH-1:0] fpu_wr_wid; + fflags_t [NUM_FPU_PORTS-1:0] fpu_wr_fflags; + for (genvar f = 0; f < NUM_FPU_PORTS; ++f) begin : g_fpu_ports + assign fpu_wr_enable[f] = fpu_csr_if[f].write_enable; + assign fpu_wr_wid[f] = fpu_csr_if[f].write_wid; + assign fpu_wr_fflags[f] = fpu_csr_if[f].write_fflags; + end + + wire [NUM_ENTRIES-1:0][FCSR_W-1:0] fcsr; + + for (genvar l = 0; l < NUM_ENTRIES; ++l) begin : g_entries + localparam C = l % NUM_CSR_PORTS; + localparam F = l % NUM_FPU_PORTS; + localparam [ENTRY_W-1:0] ENTRY = ENTRY_W'(l); + + reg [FCSR_W-1:0] fcsr_r, fcsr_n; + + always @(*) begin + fcsr_n = fcsr_r; + if (fpu_wr_enable[F] && (entry_of(fpu_wr_wid[F]) == ENTRY)) begin + fcsr_n[`FP_FLAGS_BITS-1:0] = fcsr_r[`FP_FLAGS_BITS-1:0] | fpu_wr_fflags[F]; + end + if (entry_of(csr_wid[C]) == ENTRY) begin + if (csr_wr_sel[C][0]) begin + fcsr_n[`FP_FLAGS_BITS-1:0] = csr_wr_data[C][`FP_FLAGS_BITS-1:0]; + end + if (csr_wr_sel[C][1]) begin + fcsr_n[FCSR_W-1:`FP_FLAGS_BITS] = csr_wr_data[C][INST_FRM_BITS-1:0]; + end + if (csr_wr_sel[C][2]) begin + fcsr_n = csr_wr_data[C]; + end + end + end + + always @(posedge clk) begin + if (reset) begin + fcsr_r <= '0; + end else begin + fcsr_r <= fcsr_n; + end + end + + assign fcsr[l] = fcsr_r; + end + + for (genvar c = 0; c < NUM_CSR_PORTS; ++c) begin : g_csr_rd + assign csr_rd_data[c] = fcsr[entry_of(csr_wid[c])]; + end + + for (genvar f = 0; f < NUM_FPU_PORTS; ++f) begin : g_fpu_rd + assign fpu_csr_if[f].read_frm = fcsr[entry_of(fpu_csr_if[f].read_wid)][FCSR_W-1:`FP_FLAGS_BITS]; + end + +endmodule diff --git a/hw/rtl/core/VX_csr_mpm.sv b/hw/rtl/core/VX_csr_mpm.sv new file mode 100644 index 0000000000..9105e11851 --- /dev/null +++ b/hw/rtl/core/VX_csr_mpm.sv @@ -0,0 +1,257 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_XLEN_64 + `define CSR_READ_64(addr, dst, src) \ + addr : dst = `VX_CFG_XLEN'(src) +`else + `define CSR_READ_64(addr, dst, src) \ + addr : dst = src[31:0]; \ + addr+12'h80 : dst = 32'(src[$bits(src)-1:32]) +`endif + +// Host-side machine performance-monitor reads (DCR path). The counters are not +// reachable from kernel CSR instructions, so one instance serves the whole core. +module VX_csr_mpm import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( +`ifdef PERF_ENABLE + input sysmem_perf_t sysmem_perf, + input pipeline_perf_t pipeline_perf, +`endif + input wire [PERF_CTR_BITS-1:0] cycles, + input wire [PERF_CTR_BITS-1:0] instret, + + VX_dcr_csr_if.slave dcr_csr_if +); + `UNUSED_SPARAM (INSTANCE_ID) + + wire [7:0] dcr_mpm_class = dcr_csr_if.mpm_class; + wire [`VX_CSR_ADDR_BITS-1:0] dcr_addr = dcr_csr_if.addr; + wire [`VX_CFG_XLEN-1:0] dcr_data; + + // Host counter reads decode their own address and never wait on a request. + assign dcr_csr_if.ready = 1'b1; + assign dcr_csr_if.value = VX_DCR_DATA_WIDTH'(dcr_data); + `UNUSED_VAR (dcr_data) + `UNUSED_VAR (dcr_csr_if.valid) + + // Host performance-counter reads //////////////////////////////////////// + + reg [`VX_CFG_XLEN-1:0] dcr_data_w; + + always @(*) begin + dcr_data_w = '0; + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MCYCLE, dcr_data_w, cycles); + `CSR_READ_64(`VX_CSR_MINSTRET, dcr_data_w, instret); + default: begin + `ifdef PERF_ENABLE + if ((dcr_addr >= `VX_CSR_MPM_USER && dcr_addr < (`VX_CSR_MPM_USER + 32)) + || (dcr_addr >= `VX_CSR_MPM_USER_H && dcr_addr < (`VX_CSR_MPM_USER_H + 32))) begin + case (dcr_mpm_class) + `VX_DCR_MPM_CLASS_CORE: begin + case (dcr_addr) + // PERF: pipeline + `CSR_READ_64(`VX_CSR_MPM_SCHED_IDLE, dcr_data_w, pipeline_perf.sched.idles); + `CSR_READ_64(`VX_CSR_MPM_ACTIVE_WARPS, dcr_data_w, pipeline_perf.sched.active_warps); + `CSR_READ_64(`VX_CSR_MPM_STALLED_WARPS, dcr_data_w, pipeline_perf.sched.stalled_warps); + `CSR_READ_64(`VX_CSR_MPM_ISSUED_WARPS, dcr_data_w, pipeline_perf.sched.issued_warps); + `CSR_READ_64(`VX_CSR_MPM_ISSUED_THREADS, dcr_data_w, pipeline_perf.sched.issued_threads); + `CSR_READ_64(`VX_CSR_MPM_STALL_FETCH, dcr_data_w, pipeline_perf.fetch.stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_IBUF, dcr_data_w, pipeline_perf.issue.ibf_stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_SCRB, dcr_data_w, pipeline_perf.issue.scb_stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_OPDS, dcr_data_w, pipeline_perf.issue.opd_stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_ALU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_ALU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_ALU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_ALU]); + `CSR_READ_64(`VX_CSR_MPM_STALL_LSU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_LSU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_LSU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_LSU]); + `CSR_READ_64(`VX_CSR_MPM_STALL_SFU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_SFU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_SFU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_SFU]); + `ifdef VX_CFG_EXT_F_ENABLE + `CSR_READ_64(`VX_CSR_MPM_STALL_FPU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_FPU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_FPU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_FPU]); + `endif + `ifdef VX_CFG_EXT_TCU_ENABLE + `CSR_READ_64(`VX_CSR_MPM_STALL_TCU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_TCU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_TCU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_TCU]); + `endif + // PERF: branches + `CSR_READ_64(`VX_CSR_MPM_BRANCHES, dcr_data_w, pipeline_perf.sched.branches); + `CSR_READ_64(`VX_CSR_MPM_DIVERGENCE, dcr_data_w, pipeline_perf.sched.divergence); + // PERF: memory (core-issued requests; DRAM traffic is in the MEM class) + `CSR_READ_64(`VX_CSR_MPM_IFETCHES, dcr_data_w, pipeline_perf.ifetches); + `CSR_READ_64(`VX_CSR_MPM_LOADS, dcr_data_w, pipeline_perf.loads); + `CSR_READ_64(`VX_CSR_MPM_STORES, dcr_data_w, pipeline_perf.stores); + `CSR_READ_64(`VX_CSR_MPM_IFETCH_LT, dcr_data_w, pipeline_perf.ifetch_latency); + `CSR_READ_64(`VX_CSR_MPM_LOAD_LT, dcr_data_w, pipeline_perf.load_latency); + default:; + endcase + end + `VX_DCR_MPM_CLASS_ICACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_ICACHE_READS, dcr_data_w, sysmem_perf.icache.reads); + `CSR_READ_64(`VX_CSR_MPM_ICACHE_MISS_R, dcr_data_w, sysmem_perf.icache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_ICACHE_MSHR_ST, dcr_data_w, sysmem_perf.icache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_DCACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_DCACHE_READS, dcr_data_w, sysmem_perf.dcache.reads); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_WRITES, dcr_data_w, sysmem_perf.dcache.writes); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_R, dcr_data_w, sysmem_perf.dcache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_W, dcr_data_w, sysmem_perf.dcache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_EVICTS, dcr_data_w, sysmem_perf.dcache.evictions); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_BANK_ST, dcr_data_w, sysmem_perf.dcache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_MSHR_ST, dcr_data_w, sysmem_perf.dcache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_L2CACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_READS, dcr_data_w, sysmem_perf.l2cache.reads); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_WRITES, dcr_data_w, sysmem_perf.l2cache.writes); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_R, dcr_data_w, sysmem_perf.l2cache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_W, dcr_data_w, sysmem_perf.l2cache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_EVICTS, dcr_data_w, sysmem_perf.l2cache.evictions); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_BANK_ST, dcr_data_w, sysmem_perf.l2cache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l2cache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_L3CACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_READS, dcr_data_w, sysmem_perf.l3cache.reads); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_WRITES, dcr_data_w, sysmem_perf.l3cache.writes); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_R, dcr_data_w, sysmem_perf.l3cache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_W, dcr_data_w, sysmem_perf.l3cache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_EVICTS, dcr_data_w, sysmem_perf.l3cache.evictions); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_BANK_ST, dcr_data_w, sysmem_perf.l3cache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l3cache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_MEM: begin + case (dcr_addr) + // PERF: off-chip memory + `CSR_READ_64(`VX_CSR_MPM_MEM_READS, dcr_data_w, sysmem_perf.mem.reads); + `CSR_READ_64(`VX_CSR_MPM_MEM_WRITES, dcr_data_w, sysmem_perf.mem.writes); + `CSR_READ_64(`VX_CSR_MPM_MEM_LT, dcr_data_w, sysmem_perf.mem.latency); + // PERF: lmem + `CSR_READ_64(`VX_CSR_MPM_LMEM_READS, dcr_data_w, sysmem_perf.lmem.reads); + `CSR_READ_64(`VX_CSR_MPM_LMEM_WRITES, dcr_data_w, sysmem_perf.lmem.writes); + `CSR_READ_64(`VX_CSR_MPM_LMEM_BANK_ST, dcr_data_w, sysmem_perf.lmem.bank_stalls); + // PERF: coalescer + `CSR_READ_64(`VX_CSR_MPM_COALESCER_MISS, dcr_data_w, sysmem_perf.coalescer.misses); + `ifdef VX_CFG_VM_ENABLE + // PERF: VM/MMU (icache + dcache MMU summed) + `CSR_READ_64(`VX_CSR_MPM_TLB_READS, dcr_data_w, pipeline_perf.mmu.tlb_reads); + `CSR_READ_64(`VX_CSR_MPM_TLB_HITS, dcr_data_w, pipeline_perf.mmu.tlb_hits); + `CSR_READ_64(`VX_CSR_MPM_TLB_MISSES, dcr_data_w, pipeline_perf.mmu.tlb_misses); + `CSR_READ_64(`VX_CSR_MPM_TLB_EVICTS, dcr_data_w, pipeline_perf.mmu.tlb_evictions); + `CSR_READ_64(`VX_CSR_MPM_PTW_WALKS, dcr_data_w, pipeline_perf.mmu.ptw_walks); + `CSR_READ_64(`VX_CSR_MPM_PTW_LATENCY, dcr_data_w, pipeline_perf.mmu.ptw_latency); + `endif + default:; + endcase + end + `ifdef VX_CFG_EXT_DXA_ENABLE + `VX_DCR_MPM_CLASS_DXA: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_DXA_TRANSFERS, dcr_data_w, sysmem_perf.dxa.transfers); + `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_READS, dcr_data_w, sysmem_perf.dxa.gmem_reads); + `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_DEDUP, dcr_data_w, sysmem_perf.dxa.gmem_dedup); + `CSR_READ_64(`VX_CSR_MPM_DXA_LMEM_WRITES,dcr_data_w, sysmem_perf.dxa.lmem_writes); + `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_LT, dcr_data_w, sysmem_perf.dxa.gmem_latency); + `CSR_READ_64(`VX_CSR_MPM_DXA_NOSLOT_STALLS, dcr_data_w, sysmem_perf.dxa.noslot_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_TCU_ENABLE + `VX_DCR_MPM_CLASS_TCU: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_STALLS, dcr_data_w, pipeline_perf.tcu.tbuf_stalls); + `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_CACHE_HITS, dcr_data_w, pipeline_perf.tcu.tbuf_cache_hits); + `CSR_READ_64(`VX_CSR_MPM_TCU_LMEM_READS, dcr_data_w, pipeline_perf.tcu.lmem_reads); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + `VX_DCR_MPM_CLASS_TEX: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_TEX_READS, dcr_data_w, sysmem_perf.tex.mem_reads); + `CSR_READ_64(`VX_CSR_MPM_TEX_LAT, dcr_data_w, sysmem_perf.tex.mem_latency); + `CSR_READ_64(`VX_CSR_MPM_TEX_ST, dcr_data_w, sysmem_perf.tex.stall_cycles); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_READS, dcr_data_w, sysmem_perf.tcache.reads); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_MISS_R, dcr_data_w, sysmem_perf.tcache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_BANK_ST, dcr_data_w, sysmem_perf.tcache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_MSHR_ST, dcr_data_w, sysmem_perf.tcache.mshr_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_RASTER_ENABLE + `VX_DCR_MPM_CLASS_RASTER: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_RASTER_READS, dcr_data_w, sysmem_perf.raster.mem_reads); + `CSR_READ_64(`VX_CSR_MPM_RASTER_LAT, dcr_data_w, sysmem_perf.raster.mem_latency); + `CSR_READ_64(`VX_CSR_MPM_RASTER_ST, dcr_data_w, sysmem_perf.raster.stall_cycles); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_READS, dcr_data_w, sysmem_perf.rcache.reads); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_MISS_R, dcr_data_w, sysmem_perf.rcache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_BANK_ST, dcr_data_w, sysmem_perf.rcache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_MSHR_ST, dcr_data_w, sysmem_perf.rcache.mshr_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_OM_ENABLE + `VX_DCR_MPM_CLASS_OM: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_OM_READS, dcr_data_w, sysmem_perf.om.mem_reads); + `CSR_READ_64(`VX_CSR_MPM_OM_WRITES, dcr_data_w, sysmem_perf.om.mem_writes); + `CSR_READ_64(`VX_CSR_MPM_OM_LAT, dcr_data_w, sysmem_perf.om.mem_latency); + `CSR_READ_64(`VX_CSR_MPM_OM_ST, dcr_data_w, sysmem_perf.om.stall_cycles); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_READS, dcr_data_w, sysmem_perf.ocache.reads); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_WRITES, dcr_data_w, sysmem_perf.ocache.writes); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_R, dcr_data_w, sysmem_perf.ocache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_W, dcr_data_w, sysmem_perf.ocache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_BANK_ST, dcr_data_w, sysmem_perf.ocache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_MSHR_ST, dcr_data_w, sysmem_perf.ocache.mshr_stalls); + default:; + endcase + end + `endif + default:; + endcase + end + `endif + end + endcase + end + + assign dcr_data = dcr_data_w; +`ifndef PERF_ENABLE + `UNUSED_VAR (dcr_mpm_class) +`endif + +`ifdef PERF_ENABLE + `UNUSED_VAR (sysmem_perf.icache); + `UNUSED_VAR (sysmem_perf.lmem); +`endif + +endmodule diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index 925d1e7cff..d4f501574b 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -13,25 +13,27 @@ `include "VX_define.vh" +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + module VX_csr_unit import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0, parameter NUM_LANES = 1 ) ( input wire clk, input wire reset, -`ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`endif + input wire [CORE_ID_WIDTH-1:0] core_id, `ifdef VX_CFG_EXT_F_ENABLE - VX_fpu_csr_if.slave fpu_csr_if [`VX_CFG_NUM_FPU_BLOCKS], + output wire [NW_WIDTH-1:0] fcsr_wid, + input wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data, + output wire [2:0] fcsr_wr_sel, + output wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data, `endif VX_sched_csr_if.slave sched_csr_if, - VX_dcr_csr_if.slave dcr_csr_if, VX_execute_if.slave execute_if, VX_result_if.master result_if ); @@ -75,30 +77,21 @@ module VX_csr_unit import VX_gpu_pkg::*; #( wire [`VX_CFG_XLEN-1:0] csr_req_src = execute_if.data.op_args.csr.use_imm ? `VX_CFG_XLEN'(csr_imm) : rs1_data[0]; - wire [`VX_CFG_XLEN-1:0] csr_scalar_data, dcr_read_data; - - // Host counter reads decode their own address and never wait on a request. - assign dcr_csr_if.ready = 1'b1; - assign dcr_csr_if.value = VX_DCR_DATA_WIDTH'(dcr_read_data); - `UNUSED_VAR (dcr_read_data) - `UNUSED_VAR (dcr_csr_if.valid) + wire [`VX_CFG_XLEN-1:0] csr_scalar_data; VX_csr_data #( - .INSTANCE_ID (INSTANCE_ID), - .CORE_ID (CORE_ID) + .INSTANCE_ID (INSTANCE_ID) ) csr_data ( .clk (clk), .reset (reset), - - `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf), - .pipeline_perf (pipeline_perf), - `endif + .core_id (core_id), .sched_csr_if (sched_csr_if), `ifdef VX_CFG_EXT_F_ENABLE - .fpu_csr_if (fpu_csr_if), + .fcsr_rd_data (fcsr_rd_data), + .fcsr_wr_sel (fcsr_wr_sel), + .fcsr_wr_data (fcsr_wr_data), `endif .req_fire (csr_req_fire), @@ -108,18 +101,18 @@ module VX_csr_unit import VX_gpu_pkg::*; #( .req_addr (csr_addr), .req_op (execute_if.data.op_type), .req_src (csr_req_src), - .read_data (csr_scalar_data), - - .dcr_mpm_class (dcr_csr_if.mpm_class), - .dcr_addr (dcr_csr_if.addr), - .dcr_data (dcr_read_data) + .read_data (csr_scalar_data) ); +`ifdef VX_CFG_EXT_F_ENABLE + assign fcsr_wid = execute_if.data.header.wid; +`endif + // Per-lane CSRs // Thread ids are the lane index on top of a per-request base. wire [`VX_CFG_XLEN-1:0] wtid_base = (PID_BITS != 0) ? `VX_CFG_XLEN'(execute_if.data.header.pid * NUM_LANES) : '0; - wire [`VX_CFG_XLEN-1:0] gtid_base = (`VX_CFG_XLEN'(CORE_ID) << (NW_BITS + NT_BITS)) + wire [`VX_CFG_XLEN-1:0] gtid_base = (`VX_CFG_XLEN'(core_id) << (NW_BITS + NT_BITS)) + (`VX_CFG_XLEN'(execute_if.data.header.wid) << NT_BITS) + wtid_base; diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index 4991f97899..8dd1147587 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -13,9 +13,12 @@ `include "VX_define.vh" -// Dispatches warps for incoming CTAs from the KMU. +// Dispatches warps for incoming CTAs from the KMU. Warps launch on NUM_LANES +// lanes in parallel: lane l owns the warps with wid % NUM_LANES == l, and each +// lane launches one of its free warps per cycle until the CTA is fully launched. module VX_cta_dispatch import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 1 ) ( input wire clk, input wire reset, @@ -28,31 +31,27 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( input wire warp_done, input wire [NW_WIDTH-1:0] warp_done_wid, - // to scheduler (one warp per cycle) - output wire cta_fire, - output wire [NW_WIDTH-1:0] cta_wid, + // to scheduler (one warp per lane per cycle) + output wire [NUM_LANES-1:0] cta_fire, + output wire [NUM_LANES-1:0][NW_WIDTH-1:0] cta_wid, output wire [PC_BITS-1:0] cta_PC, - output wire [`VX_CFG_NUM_THREADS-1:0] cta_tmask, + output wire [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] cta_tmask, output wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param, - output wire cta_init, + output wire [NUM_LANES-1:0] cta_init, - // CTA-CSR read-back - input wire [NW_WIDTH-1:0] csr_rd_wid, - input wire [NCTA_WIDTH-1:0] csr_rd_cta_id, - output cta_csrs_t cta_rd_csrs, - // The per-lane launch record (see cta_lane_t): a compute warp's thread index and a - // fragment warp's stamp overlay the same bits, and the CSR unit reads the view the - // CSR being accessed calls for. - output cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_rd_lane, - - // CTA id of the scheduled warp - input wire [NW_WIDTH-1:0] schedule_wid, - output wire [NCTA_WIDTH-1:0] schedule_cta_id, + // CTA state for the CTA-CSR stores (see VX_csr_cta): the CTA context, + // and each lane's warp records + output cta_bcast_t cta_bcast, + output cta_lane_bcast_t [NUM_LANES-1:0] cta_lane_bcast, output wire busy ); `UNUSED_SPARAM (INSTANCE_ID) + `STATIC_ASSERT ((`VX_CFG_NUM_WARPS % NUM_LANES) == 0, ("invalid parameter")) localparam NUM_CTA_SLOTS= `VX_CFG_NUM_WARPS; + localparam LANE_WARPS = `VX_CFG_NUM_WARPS / NUM_LANES; + localparam LANE_WIDTH = `LOG2UP(NUM_LANES); + localparam LCNT_WIDTH = `CLOG2(NUM_LANES+1); localparam CS_BITS = NW_WIDTH; // UP(NW_BITS): at least 1 to avoid zero-width when NUM_WARPS=1 localparam LMEM_SIZE = (1 << `VX_CFG_LMEM_LOG_SIZE); @@ -114,7 +113,6 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // so a slot relaunched with lanes it has not run before must run it again. reg [7:0] cur_ctx_id_r; reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_NUM_THREADS-1:0] warp_init_lanes_r; - reg warp_skip_init_r; // ------------------------------------------------------------------------- // FSM + per-dispatch registers @@ -131,28 +129,50 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( reg [CTA_TID_WIDTH:0] block_size_r; reg [2:0][CTA_TID_WIDTH-1:0] warp_step_r; reg [NW_WIDTH:0] cluster_size_r; - reg warp_fire_r; - reg [NW_WIDTH-1:0] warp_id_r; - reg [`VX_CFG_NUM_THREADS-1:0] warp_tmask_r; reg [NW_WIDTH-1:0] cta_rank_r; - reg [2:0][CTA_TID_WIDTH-1:0] thread_idx_r; reg [CS_BITS-1:0] cur_slot_r; + // Warps launched last cycle, one per lane; `order` is the warp's position + // among them, so its CTA rank is cta_rank_r + order. + reg [NUM_LANES-1:0] warp_fire_r; + reg [NUM_LANES-1:0][NW_WIDTH-1:0] warp_id_r; + reg [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] warp_tmask_r; + reg [NUM_LANES-1:0][LANE_WIDTH-1:0] warp_order_r; + reg [NUM_LANES-1:0] warp_skip_init_r; + reg [LCNT_WIDTH-1:0] nfired_r; + wire fire_any_r = (| warp_fire_r); + + // Thread-index window: thread_idx_r[k] is the base thread index of the warp + // ranked cta_rank_r + k. Launching n warps shifts the window by n; the entries + // shifted in advance by group_step_r (NUM_LANES warps), so no two advances + // chain within a cycle. + reg [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_r; + reg [2:0][CTA_TID_WIDTH-1:0] group_step_r; + // ------------------------------------------------------------------------- - // Free-warp selection + // Free-warp selection: one candidate per lane // ------------------------------------------------------------------------- reg [`VX_CFG_NUM_WARPS-1:0] dispatched_warps; - wire [NW_WIDTH-1:0] warp_id_n; - wire warp_ready; - VX_priority_encoder #( - .N (`VX_CFG_NUM_WARPS), - .REVERSE (0) - ) priority_enc ( - .data_in (~(active_warps | dispatched_warps)), - `UNUSED_PIN(onehot_out), - .index_out (warp_id_n), - .valid_out (warp_ready) - ); + wire [`VX_CFG_NUM_WARPS-1:0] free_warps = ~(active_warps | dispatched_warps); + wire [NUM_LANES-1:0] lane_ready; + wire [NUM_LANES-1:0][NW_WIDTH-1:0] lane_wid; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_select + wire [LANE_WARPS-1:0] lane_free; + for (genvar j = 0; j < LANE_WARPS; ++j) begin : g_free + assign lane_free[j] = free_warps[j * NUM_LANES + l]; + end + wire [`LOG2UP(LANE_WARPS)-1:0] lane_idx; + VX_priority_encoder #( + .N (LANE_WARPS), + .REVERSE (0) + ) priority_enc ( + .data_in (lane_free), + `UNUSED_PIN(onehot_out), + .index_out (lane_idx), + .valid_out (lane_ready[l]) + ); + assign lane_wid[l] = NW_WIDTH'(lane_idx) * NW_WIDTH'(NUM_LANES) + NW_WIDTH'(l); + end wire kmu_bus_if_fire = kmu_bus_if.valid && kmu_bus_if.ready; @@ -191,7 +211,6 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // pipeline; `is_frag_warp`/`frag_stamps_sel` pick the lane-record view being // written. All benign (0) in a build with no rasterizer. wire frag_inflight; - wire is_frag_warp; wire [`VX_CFG_NUM_THREADS-1:0][FRAG_LANE_BITS-1:0] frag_stamps_sel; `ifdef VX_CFG_EXT_RASTER_ENABLE @@ -213,54 +232,126 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // ------------------------------------------------------------------------- // Power-of-two NUM_THREADS arithmetic — all combinational, zero adders // ------------------------------------------------------------------------- - wire [NW_WIDTH:0] cta_num_warps; wire [NW_WIDTH:0] kmu_num_warps; wire [CTA_TID_WIDTH:0] block_size_next; wire [CTA_TID_WIDTH:0] block_size_cur; wire [`VX_CFG_NUM_THREADS-1:0] partial_tmask; - // block_size_r retires the previously latched warp's threads a cycle late, so - // the count left for the warp being latched now excludes them. - assign block_size_cur = warp_fire_r ? block_size_next : block_size_r; + // block_size_r retires the warps latched last cycle a cycle late, so the count + // left for the warps being latched now excludes them. + assign block_size_cur = fire_any_r ? block_size_next : block_size_r; if (NT_BITS > 0) begin : g_nt_nonzero // Ceiling division block_size / NUM_THREADS: upper bits + OR of lower bits. - assign cta_num_warps = (NW_WIDTH+1)'(block_size_r[CTA_TID_WIDTH:NT_BITS]) + (NW_WIDTH+1)'(|block_size_r[NT_BITS-1:0]); // From KMU data at accept time (used to initialise table + cta_size output) assign kmu_num_warps = (NW_WIDTH+1)'(eff_block_size[CTA_TID_WIDTH:NT_BITS]) + (NW_WIDTH+1)'(|eff_block_size[NT_BITS-1:0]); - // Shared block_size decrement: low NT_BITS bits unchanged; upper bits decrement by 1. - assign block_size_next = {block_size_r[CTA_TID_WIDTH:NT_BITS] - 1'b1, block_size_r[NT_BITS-1:0]}; + // Shared block_size decrement: low NT_BITS bits unchanged; upper bits drop by + // the warps launched. + assign block_size_next = {block_size_r[CTA_TID_WIDTH:NT_BITS] - (CTA_TID_WIDTH+1-NT_BITS)'(nfired_r), block_size_r[NT_BITS-1:0]}; // Partial-warp mask: (1 << count) - 1 where count = block_size_cur[NT_BITS-1:0] assign partial_tmask = (`VX_CFG_NUM_THREADS'(1) << block_size_cur[NT_BITS-1:0]) - `VX_CFG_NUM_THREADS'(1); end else begin : g_nt_zero // NT_BITS=0: NUM_THREADS=1, each warp has exactly 1 thread, no partial warps. - assign cta_num_warps = (NW_WIDTH+1)'(block_size_r); assign kmu_num_warps = (NW_WIDTH+1)'(eff_block_size); - assign block_size_next = (CTA_TID_WIDTH+1)'(block_size_r - 1'b1); + assign block_size_next = (CTA_TID_WIDTH+1)'(block_size_r - (CTA_TID_WIDTH+1)'(nfired_r)); assign partial_tmask = `VX_CFG_NUM_THREADS'(0); end - // Full-warp test: upper bits non-zero (no comparator) - wire is_full_warp = |block_size_cur[CTA_TID_WIDTH:NT_BITS]; - // The lanes this launch activates: full warp = all ones, partial = (1< 0) begin : g_rem_nt + assign rem_warps_cur = (NW_WIDTH+1)'(full_warps_cur) + (NW_WIDTH+1)'(|block_size_cur[NT_BITS-1:0]); + end else begin : g_rem_nt1 + assign rem_warps_cur = (NW_WIDTH+1)'(full_warps_cur); + end - // Does slot w still owe the startup for any lane this launch activates? Resolved - // per slot, off the warp_id_n path, so the encoder still feeds only a 1-bit mux. + // Launch selection: lanes with a free warp, taken in round-robin order from + // lane_rr_r, up to the warps left. order[l] is the lane's position among + // the selected lanes. + reg [LANE_WIDTH-1:0] lane_rr_r; + wire launch_en; + reg [NUM_LANES-1:0] lane_sel; + reg [NUM_LANES-1:0][LANE_WIDTH-1:0] lane_order; + reg [LCNT_WIDTH-1:0] lane_cnt; + always @(*) begin + lane_sel = '0; + lane_order = '0; + lane_cnt = '0; + for (integer o = 0; o < NUM_LANES; ++o) begin + if (launch_en && lane_ready[(int'(lane_rr_r) + o) % NUM_LANES] + && ((NW_WIDTH+1)'(lane_cnt) < rem_warps_cur)) begin + lane_sel[(int'(lane_rr_r) + o) % NUM_LANES] = 1'b1; + lane_order[(int'(lane_rr_r) + o) % NUM_LANES] = LANE_WIDTH'(lane_cnt); + lane_cnt = lane_cnt + LCNT_WIDTH'(1); + end + end + end + + // A warp is full unless it is the CTA's last and the block size is not a + // multiple of the warp width. + wire [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] lane_tmask; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_tmask + wire is_full_warp = ((CTA_TID_WIDTH+1-NT_BITS)'(lane_order[l]) < full_warps_cur); + assign lane_tmask[l] = is_full_warp ? {`VX_CFG_NUM_THREADS{1'b1}} : partial_tmask; + end + + // Does slot w still owe the startup for any lane a full launch activates? A + // partial last warp re-runs the check on its own mask. wire [`VX_CFG_NUM_WARPS-1:0] slot_needs_init; for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_needs_init - assign slot_needs_init[w] = |(launch_tmask & ~warp_init_lanes_r[w]); + localparam l = w % NUM_LANES; + assign slot_needs_init[w] = |(lane_tmask[l] & ~warp_init_lanes_r[w]); end - // Last-warp test: ceiling(remaining/NUM_THREADS) == 1. - // Covers (upper==1, lower==0) full-last and (upper==0, lower!=0) partial-only cases. - wire is_last_warp = (cta_num_warps == (NW_WIDTH+1)'(1)); - // 3D thread-index carry-propagation (combinational on registered state) - wire [CTA_TID_WIDTH:0] next_x = {1'b0, thread_idx_r[0]} + {1'b0, warp_step_r[0]}; - wire wrap_x = (next_x >= block_dim_r[0]); - wire [CTA_TID_WIDTH:0] next_y = ({1'b0, thread_idx_r[1]} + {1'b0, warp_step_r[1]}) + (CTA_TID_WIDTH+1)'(wrap_x); - wire wrap_y = (next_y >= block_dim_r[1]); + // One thread-index advance by `step`, with a single wrap per dimension: every + // step is normalized below the block dimensions. + function automatic logic [2:0][CTA_TID_WIDTH-1:0] tid_advance ( + input logic [2:0][CTA_TID_WIDTH-1:0] base, + input logic [2:0][CTA_TID_WIDTH-1:0] step, + input logic [CTA_TID_WIDTH:0] bdim_x, + input logic [CTA_TID_WIDTH:0] bdim_y + ); + logic [CTA_TID_WIDTH:0] nx, ny; + logic wx, wy; + nx = {1'b0, base[0]} + {1'b0, step[0]}; + wx = (nx >= bdim_x); + ny = ({1'b0, base[1]} + {1'b0, step[1]}) + (CTA_TID_WIDTH+1)'(wx); + wy = (ny >= bdim_y); + tid_advance[0] = wx ? CTA_TID_WIDTH'(nx - bdim_x) : CTA_TID_WIDTH'(nx); + tid_advance[1] = wy ? CTA_TID_WIDTH'(ny - bdim_y) : CTA_TID_WIDTH'(ny); + tid_advance[2] = base[2] + step[2] + CTA_TID_WIDTH'(wy); + endfunction + + // The window after the warps latched last cycle launch: entries shift down by + // nfired_r, and the ones shifted in are the entries they replace advanced by + // one group step. + wire [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_grp; + wire [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_adv; + for (genvar k = 0; k < NUM_LANES; ++k) begin : g_tid_window + assign thread_idx_grp[k] = tid_advance(thread_idx_r[k], group_step_r, block_dim_r[0], block_dim_r[1]); + reg [2:0][CTA_TID_WIDTH-1:0] shifted; + always @(*) begin + shifted = thread_idx_r[k]; + for (integer n = 1; n <= NUM_LANES; ++n) begin + if (int'(nfired_r) == n) begin + shifted = ((k + n) < NUM_LANES) ? thread_idx_r[(k + n) % NUM_LANES] + : thread_idx_grp[(k + n) % NUM_LANES]; + end + end + end + assign thread_idx_adv[k] = shifted; + end + + // Kernel constants of the window: the base thread indices of ranks 0..NUM_LANES-1 + // and the NUM_LANES-warp step, built by stepping warp_step_r once per cycle + // whenever a launch's block shape differs from the last one's. + reg [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_init_r; + reg [2:0][CTA_TID_WIDTH-1:0] shape_step_r; + reg [2:0][CTA_TID_WIDTH:0] shape_dim_r; + reg [LCNT_WIDTH-1:0] init_cnt_r; + reg [2:0][CTA_TID_WIDTH-1:0] init_acc_r; + wire init_busy = (init_cnt_r != '0); // ------------------------------------------------------------------------- // Retirement decode — pipeline: @@ -411,12 +502,21 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( always_ff @(posedge clk) begin if (reset) begin state <= IDLE; - warp_fire_r <= 0; + warp_fire_r <= '0; warp_id_r <= '0; warp_tmask_r <= '0; + warp_order_r <= '0; + nfired_r <= '0; + lane_rr_r <= '0; + init_cnt_r <= '0; + shape_step_r <= '0; + shape_dim_r <= '0; + thread_idx_init_r <= '0; + group_step_r <= '0; + init_acc_r <= '0; cur_ctx_id_r <= '0; warp_init_lanes_r <= '0; - warp_skip_init_r <= 0; + warp_skip_init_r <= '0; tail_r <= '0; cur_lmem_base_r <= '0; slot_valid_r <= '0; @@ -447,8 +547,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // ---- wid → cta-slot map: latch on warp dispatch ---------------- // (Used internally by the retirement path to recover the slot // index of a finishing warp; no longer exposed externally.) - if ((state == DISPATCH) && warp_ready) begin - cta_slot_per_warp_r[warp_id_n] <= cur_slot_r; + for (integer l = 0; l < NUM_LANES; ++l) begin + if (lane_sel[l]) begin + cta_slot_per_warp_r[lane_wid[l]] <= cur_slot_r; + end end // ---- Warp retirement ------------------------------------------- @@ -487,7 +589,18 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( warp_step_r <= kmu_req.args.compute.warp_step; cluster_size_r <= eff_cluster_size; cta_rank_r <= '0; - thread_idx_r <= '0; + thread_idx_r <= thread_idx_init_r; + group_step_r <= (NUM_LANES == 1) ? kmu_req.args.compute.warp_step : group_step_r; + // A new block shape rebuilds the window constants before launching. + if ((NUM_LANES > 1) && !kmu_is_frag + && ((kmu_req.args.compute.warp_step != shape_step_r) + || (kmu_req.args.compute.block_dim != shape_dim_r))) begin + shape_step_r <= kmu_req.args.compute.warp_step; + shape_dim_r <= kmu_req.args.compute.block_dim; + init_cnt_r <= LCNT_WIDTH'(1); + init_acc_r <= '0; + thread_idx_r <= '0; + end // Fixed-stride placement: this CTA lands in slot base_slot // (the round-robin tail, pre-wrapped for a cluster window) @@ -506,30 +619,44 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( end DISPATCH: begin - if (warp_ready) begin - warp_fire_r <= 1; - warp_id_r <= warp_id_n; - dispatched_warps[warp_id_n] <= 1'b1; - warp_tmask_r <= launch_tmask; - warp_skip_init_r <= ~slot_needs_init[warp_id_n]; - end else begin - warp_fire_r <= 0; + for (integer l = 0; l < NUM_LANES; ++l) begin + warp_fire_r[l] <= lane_sel[l]; + if (lane_sel[l]) begin + warp_id_r[l] <= lane_wid[l]; + warp_tmask_r[l] <= lane_tmask[l]; + warp_order_r[l] <= lane_order[l]; + warp_skip_init_r[l] <= ~slot_needs_init[lane_wid[l]]; + dispatched_warps[lane_wid[l]] <= 1'b1; + end end + nfired_r <= lane_cnt; + lane_rr_r <= LANE_WIDTH'((int'(lane_rr_r) + int'(lane_cnt)) % NUM_LANES); - if (warp_fire_r) begin - cta_rank_r <= cta_rank_r + NW_WIDTH'(1); - // Single shared adder result used for both decrement and last-warp test + if (fire_any_r) begin + cta_rank_r <= cta_rank_r + NW_WIDTH'(nfired_r); block_size_r <= block_size_next; + thread_idx_r <= thread_idx_adv; + for (integer l = 0; l < NUM_LANES; ++l) begin + if (warp_fire_r[l]) begin + warp_init_lanes_r[warp_id_r[l]] <= warp_init_lanes_r[warp_id_r[l]] | warp_tmask_r[l]; + end + end + if (rem_warps_cur == '0) begin + state <= IDLE; + end + end - warp_init_lanes_r[warp_id_r] <= warp_init_lanes_r[warp_id_r] | warp_tmask_r; - thread_idx_r[0] <= wrap_x ? CTA_TID_WIDTH'(next_x - block_dim_r[0]) : CTA_TID_WIDTH'(next_x); - thread_idx_r[1] <= wrap_y ? CTA_TID_WIDTH'(next_y - block_dim_r[1]) : CTA_TID_WIDTH'(next_y); - thread_idx_r[2] <= thread_idx_r[2] + warp_step_r[2] + CTA_TID_WIDTH'(wrap_y); - - if (is_last_warp) begin - warp_fire_r <= 0; - state <= IDLE; + // window constants: step warp_step_r once per cycle + if (init_busy) begin + if (int'(init_cnt_r) < NUM_LANES) begin + thread_idx_r[init_cnt_r] <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + thread_idx_init_r[init_cnt_r] <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + init_cnt_r <= init_cnt_r + LCNT_WIDTH'(1); + end else begin + group_step_r <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + init_cnt_r <= '0; end + init_acc_r <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); end end default:; @@ -544,6 +671,8 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // and cta_param. cta_csrs_t cta_csrs; + assign launch_en = (state == DISPATCH) && !init_busy; + assign cta_fire = warp_fire_r; assign cta_wid = warp_id_r; assign cta_PC = warp_PC; @@ -584,67 +713,21 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // ------------------------------------------------------------------------- // CTA context storage + per-thread coordinate expansion // ------------------------------------------------------------------------- - // Per-CTA context (cta_ctx_ram) and per-warp residue (cta_warp_ram), written - // at launch and read back via csr_rd_wid / csr_rd_cta_id so CTA CSRs resolve - // in one cycle with no divider. BRAM-backed to avoid per-warp full-record flops. + // Per-CTA context and per-warp residue, broadcast at launch to the CTA-CSR + // stores, which resolve CTA CSRs in one cycle with no divider. - // Per-CTA context. + // Per-CTA context write port, shared by every lane. wire cta_ctx_write; wire [NCTA_WIDTH-1:0] cta_ctx_waddr; cta_ctx_t cta_ctx_wdata; - wire [NCTA_WIDTH-1:0] cta_ctx_raddr; - cta_ctx_t cta_ctx_rdata; - - VX_dp_ram #( - .DATAW ($bits(cta_ctx_t)), - .SIZE (NUM_CTA_MAX), - .RDW_MODE ("R"), - .OUT_REG (1) - ) cta_ctx_ram ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (cta_ctx_write), - .wren (1'b1), - .waddr (cta_ctx_waddr), - .wdata (cta_ctx_wdata), - .raddr (cta_ctx_raddr), - .rdata (cta_ctx_rdata) - ); - - // Per-warp residue. - wire cta_warp_write; - wire [NW_WIDTH-1:0] cta_warp_waddr; - cta_warp_t cta_warp_wdata; - wire [NW_WIDTH-1:0] cta_warp_raddr; - cta_warp_t cta_warp_rdata; - - VX_dp_ram #( - .DATAW ($bits(cta_warp_t)), - .SIZE (`VX_CFG_NUM_WARPS), - .RDW_MODE ("R"), - .OUT_REG (1) - ) cta_warp_ram ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (cta_warp_write), - .wren (1'b1), - .waddr (cta_warp_waddr), - .wdata (cta_warp_wdata), - .raddr (cta_warp_raddr), - .rdata (cta_warp_rdata) - ); // Per-lane CTA thread coordinates, expanded divide-free from the warp base // (lane 0 = base; each later lane steps +1 along X with a single wrap into Y // then Z) and stored in cta_warp_ram so CTA_THREAD_ID reads cost one cycle // with no divider. The expansion is a serial ripple, so it is pipelined - // TID_STEP lanes/cycle to meet timing. CTA dispatch is infrequent and + // TID_STEP lanes/cycle to meet timing; each launch lane has its own pipeline. // cta_warp_ram is read many cycles after a warp launches (fetch/decode/issue // latency >> TID_STAGES), so the added write latency is hidden. - wire [2:0][CTA_TID_WIDTH-1:0] cta_base_tid = thread_idx_r; - localparam TID_STEP = 2; localparam TID_STAGES = (`VX_CFG_NUM_THREADS - 1 + TID_STEP - 1) / TID_STEP; @@ -666,121 +749,139 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( typedef logic [`VX_CFG_NUM_THREADS-1:0][2:0][CTA_TID_WIDTH-1:0] cta_tid_arr_t; - cta_tid_arr_t tidp_tid [TID_STAGES+1]; - wire [TID_STAGES:0] tidp_valid; - wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_wid; - wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_rank; - // block_dim spans 1..NUM_WARPS*NUM_THREADS inclusive, one bit more than a - // thread index -- truncating it to CTA_TID_WIDTH turns a full-width dimension - // into 0 and makes every lane wrap. - wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdx; - wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdy; - - // stage 0: combinational inputs; seed lane 0 with the warp base. - cta_tid_arr_t tidp_tid0; - always @(*) begin - tidp_tid0 = '0; - tidp_tid0[0] = cta_base_tid; - end - assign tidp_tid[0] = tidp_tid0; - assign tidp_valid[0] = cta_fire; - assign tidp_wid[0] = cta_wid; - assign tidp_rank[0] = cta_csrs.cta_rank; - assign tidp_bdx[0] = cta_csrs.block_dim[0]; - assign tidp_bdy[0] = cta_csrs.block_dim[1]; - - for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_tid_pipe - cta_tid_arr_t nxt; + wire [NUM_LANES-1:0] lane_frag_inflight; + + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lanes + cta_tid_arr_t tidp_tid [TID_STAGES+1]; + wire [TID_STAGES:0] tidp_valid; + wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_wid; + wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_rank; + // block_dim spans 1..NUM_WARPS*NUM_THREADS inclusive, one bit more than a + // thread index -- truncating it to CTA_TID_WIDTH turns a full-width dimension + // into 0 and makes every lane wrap. + wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdx; + wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdy; + + // stage 0: combinational inputs; seed lane 0 with the warp base. + cta_tid_arr_t tidp_tid0; always @(*) begin - nxt = tidp_tid[s-1]; - for (integer k = 0; k < TID_STEP; k = k + 1) begin - if (((s-1)*TID_STEP + k + 1) < `VX_CFG_NUM_THREADS) begin - nxt[(s-1)*TID_STEP + k + 1] = tid_next(nxt[(s-1)*TID_STEP + k], tidp_bdx[s-1], tidp_bdy[s-1]); + tidp_tid0 = '0; + tidp_tid0[0] = thread_idx_r[warp_order_r[l]]; + end + assign tidp_tid[0] = tidp_tid0; + assign tidp_valid[0] = warp_fire_r[l]; + assign tidp_wid[0] = warp_id_r[l]; + assign tidp_rank[0] = cta_csrs.cta_rank + NW_WIDTH'(warp_order_r[l]); + assign tidp_bdx[0] = cta_csrs.block_dim[0]; + assign tidp_bdy[0] = cta_csrs.block_dim[1]; + + for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_tid_pipe + cta_tid_arr_t nxt; + always @(*) begin + nxt = tidp_tid[s-1]; + for (integer k = 0; k < TID_STEP; k = k + 1) begin + if (((s-1)*TID_STEP + k + 1) < `VX_CFG_NUM_THREADS) begin + nxt[(s-1)*TID_STEP + k + 1] = tid_next(nxt[(s-1)*TID_STEP + k], tidp_bdx[s-1], tidp_bdy[s-1]); + end end end - end - reg v_r; - reg [NW_WIDTH-1:0] wid_r, rank_r; - reg [CTA_TID_WIDTH:0] bdx_r, bdy_r; - cta_tid_arr_t tid_r; - always @(posedge clk) begin - if (reset) begin - v_r <= 1'b0; - end else begin - v_r <= tidp_valid[s-1]; + reg v_r; + reg [NW_WIDTH-1:0] wid_r, rank_r; + reg [CTA_TID_WIDTH:0] bdx_r, bdy_r; + cta_tid_arr_t tid_r; + always @(posedge clk) begin + if (reset) begin + v_r <= 1'b0; + end else begin + v_r <= tidp_valid[s-1]; + end + tid_r <= nxt; + wid_r <= tidp_wid[s-1]; + rank_r <= tidp_rank[s-1]; + bdx_r <= tidp_bdx[s-1]; + bdy_r <= tidp_bdy[s-1]; end - tid_r <= nxt; - wid_r <= tidp_wid[s-1]; - rank_r <= tidp_rank[s-1]; - bdx_r <= tidp_bdx[s-1]; - bdy_r <= tidp_bdy[s-1]; + assign tidp_tid[s] = tid_r; + assign tidp_valid[s] = v_r; + assign tidp_wid[s] = wid_r; + assign tidp_rank[s] = rank_r; + assign tidp_bdx[s] = bdx_r; + assign tidp_bdy[s] = bdy_r; end - assign tidp_tid[s] = tid_r; - assign tidp_valid[s] = v_r; - assign tidp_wid[s] = wid_r; - assign tidp_rank[s] = rank_r; - assign tidp_bdx[s] = bdx_r; - assign tidp_bdy[s] = bdy_r; - end -`ifdef VX_CFG_EXT_RASTER_ENABLE - // Which in-flight warps came from a fragment launch. One bit per stage rather - // than the stamps themselves: the write side needs to know WHICH view of the - // lane record to land, and the header side needs to know whether frag_stamps_r - // is still owed to someone. Shaped exactly like tidp_valid -- stage 0 - // combinational, the rest registered -- so the bit and the warp it describes - // stay in step. - wire [TID_STAGES:0] tidp_frag; - assign tidp_frag[0] = cta_fire && is_frag_r; - for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_frag_pipe - reg f_r; - always @(posedge clk) begin - if (reset) begin - f_r <= 1'b0; - end else begin - f_r <= tidp_frag[s-1]; + wire is_frag_warp; + `ifdef VX_CFG_EXT_RASTER_ENABLE + // Which in-flight warps came from a fragment launch. One bit per stage rather + // than the stamps themselves: the write side needs to know WHICH view of the + // lane record to land, and the header side needs to know whether frag_stamps_r + // is still owed to someone. Shaped exactly like tidp_valid -- stage 0 + // combinational, the rest registered -- so the bit and the warp it describes + // stay in step. + wire [TID_STAGES:0] tidp_frag; + assign tidp_frag[0] = warp_fire_r[l] && is_frag_r; + for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_frag_pipe + reg f_r; + always @(posedge clk) begin + if (reset) begin + f_r <= 1'b0; + end else begin + f_r <= tidp_frag[s-1]; + end end + assign tidp_frag[s] = f_r; end - assign tidp_frag[s] = f_r; + assign lane_frag_inflight[l] = |tidp_frag; + assign is_frag_warp = tidp_frag[TID_STAGES]; + `else + assign lane_frag_inflight[l] = 1'b0; + assign is_frag_warp = 1'b0; + // its only consumer (the lane-launch view mux) is RASTER-only + `UNUSED_VAR (is_frag_warp) + `endif + + // block_dim is consumed only by stages still rippling; the final stage's + // forwarded copy is unused. + `UNUSED_VAR (tidp_bdx[TID_STAGES]) + `UNUSED_VAR (tidp_bdy[TID_STAGES]) + + cta_warp_t cta_warp_wdata; + assign cta_warp_wdata.cta_rank = tidp_rank[TID_STAGES]; + + // The lane record is an overlay: a fragment launch lands its stamp slice, and a + // compute launch lands the expanded thread index. Which of the two applies is a + // property of the WARP being written, not of whatever launch the header stage + // happens to be holding now, so it rides the pipeline with it. + for (genvar i = 0; i < `VX_CFG_NUM_THREADS; ++i) begin : g_lane_launch + cta_lane_t lane_compute; + assign lane_compute = LANE_LAUNCH_BITS'(tidp_tid[TID_STAGES][i]); + `ifdef VX_CFG_EXT_RASTER_ENABLE + cta_lane_t lane_fragment; + assign lane_fragment = LANE_LAUNCH_BITS'(frag_stamps_sel[i]); + assign cta_warp_wdata.lane_launch[i] = is_frag_warp ? lane_fragment : lane_compute; + `else + assign cta_warp_wdata.lane_launch[i] = lane_compute; + `endif + end + + assign cta_lane_bcast[l].warp_write = tidp_valid[TID_STAGES]; + assign cta_lane_bcast[l].warp_waddr = tidp_wid[TID_STAGES]; + assign cta_lane_bcast[l].warp_wdata = cta_warp_wdata; + assign cta_lane_bcast[l].id_write = warp_fire_r[l]; + assign cta_lane_bcast[l].id_wid = warp_id_r[l]; + assign cta_lane_bcast[l].id_cta = cta_csrs.cta_id; end - assign frag_inflight = |tidp_frag; - assign is_frag_warp = tidp_frag[TID_STAGES]; + +`ifdef VX_CFG_EXT_RASTER_ENABLE + assign frag_inflight = |lane_frag_inflight; assign frag_stamps_sel = frag_stamps_r; `else assign frag_inflight = 1'b0; - assign is_frag_warp = 1'b0; assign frag_stamps_sel = '0; - // their only consumer (the lane-launch view mux) is RASTER-only - `UNUSED_VAR (is_frag_warp) + `UNUSED_VAR (lane_frag_inflight) `UNUSED_VAR (frag_stamps_sel) `endif - // block_dim is consumed only by stages still rippling; the final stage's - // forwarded copy is unused. - `UNUSED_VAR (tidp_bdx[TID_STAGES]) - `UNUSED_VAR (tidp_bdy[TID_STAGES]) - - assign cta_warp_write = tidp_valid[TID_STAGES]; - assign cta_warp_waddr = tidp_wid[TID_STAGES]; - assign cta_warp_wdata.cta_rank = tidp_rank[TID_STAGES]; - - // The lane record is an overlay: a fragment launch lands its stamp slice, and a - // compute launch lands the expanded thread index. Which of the two applies is a - // property of the WARP being written, not of whatever launch the header stage - // happens to be holding now, so it rides the pipeline with it. - for (genvar i = 0; i < `VX_CFG_NUM_THREADS; ++i) begin : g_lane_launch - cta_lane_t lane_compute; - assign lane_compute = LANE_LAUNCH_BITS'(tidp_tid[TID_STAGES][i]); - `ifdef VX_CFG_EXT_RASTER_ENABLE - cta_lane_t lane_fragment; - assign lane_fragment = LANE_LAUNCH_BITS'(frag_stamps_sel[i]); - assign cta_warp_wdata.lane_launch[i] = is_frag_warp ? lane_fragment : lane_compute; - `else - assign cta_warp_wdata.lane_launch[i] = lane_compute; - `endif - end - - assign cta_ctx_write = cta_fire; + assign cta_ctx_write = fire_any_r; assign cta_ctx_waddr = cta_csrs.cta_id; assign cta_ctx_wdata.cta_size = cta_csrs.cta_size; assign cta_ctx_wdata.block_idx = cta_csrs.block_idx; @@ -791,32 +892,9 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( assign cta_ctx_wdata.cluster_size = cta_csrs.cluster_size; assign cta_ctx_wdata.entry = cta_csrs.entry; - // rdata is valid one cycle after raddr; csr_unit holds its read address - // stable for that cycle. - assign cta_warp_raddr = csr_rd_wid; - assign cta_ctx_raddr = csr_rd_cta_id; - - assign cta_rd_csrs.cta_id = csr_rd_cta_id; - assign cta_rd_csrs.cta_rank = cta_warp_rdata.cta_rank; - - assign cta_rd_lane = cta_warp_rdata.lane_launch; - assign cta_rd_csrs.cta_size = cta_ctx_rdata.cta_size; - assign cta_rd_csrs.block_idx = cta_ctx_rdata.block_idx; - assign cta_rd_csrs.block_dim = cta_ctx_rdata.block_dim; - assign cta_rd_csrs.grid_dim = cta_ctx_rdata.grid_dim; - assign cta_rd_csrs.param = cta_ctx_rdata.param; - assign cta_rd_csrs.lmem_addr = cta_ctx_rdata.lmem_addr; - assign cta_rd_csrs.cluster_size = cta_ctx_rdata.cluster_size; - assign cta_rd_csrs.entry = cta_ctx_rdata.entry; - - // Per-warp -> CTA-id map, latched on launch and indexed by schedule_wid. - reg [`VX_CFG_NUM_WARPS-1:0][NCTA_WIDTH-1:0] cta_id_per_warp_r; - always @(posedge clk) begin - if (cta_fire) begin - cta_id_per_warp_r[cta_wid] <= cta_csrs.cta_id; - end - end - assign schedule_cta_id = cta_id_per_warp_r[schedule_wid]; + assign cta_bcast.ctx_write = cta_ctx_write; + assign cta_bcast.ctx_waddr = cta_ctx_waddr; + assign cta_bcast.ctx_wdata = cta_ctx_wdata; `ifdef DBG_TRACE_PIPELINE // Pipeline warp_done_wid alongside the retirement chain for trace logging. @@ -840,13 +918,15 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( kmu_req.param, stride, kmu_num_warps, usable_slots_r)) end - // Warp dispatched to scheduler - if (warp_fire_r) begin - `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, param=0x%0h, lmem_addr=0x%0h, init=%b\n", - $time, INSTANCE_ID, warp_id_r, cur_slot_r, to_fullPC(warp_PC), - warp_tmask_r, param_r, - (`VX_CFG_MEM_ADDR_WIDTH'(`VX_MEM_LMEM_BASE_ADDR) | `VX_CFG_MEM_ADDR_WIDTH'(cur_lmem_base_r)), - ~warp_skip_init_r)) + // Warps dispatched to the schedulers + for (integer l = 0; l < NUM_LANES; ++l) begin + if (warp_fire_r[l]) begin + `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, rank=%0d, PC=0x%0h, tmask=%b, param=0x%0h, lmem_addr=0x%0h, init=%b\n", + $time, INSTANCE_ID, warp_id_r[l], cur_slot_r, cta_rank_r + NW_WIDTH'(warp_order_r[l]), to_fullPC(warp_PC), + warp_tmask_r[l], param_r, + (`VX_CFG_MEM_ADDR_WIDTH'(`VX_MEM_LMEM_BASE_ADDR) | `VX_CFG_MEM_ADDR_WIDTH'(cur_lmem_base_r)), + ~warp_skip_init_r[l])) + end end // Warp retirement / CTA done if (warp_done_r_dly && slot_valid_r[done_slot_r_dly]) begin diff --git a/hw/rtl/core/VX_dcr_data.sv b/hw/rtl/core/VX_dcr_data.sv index 44f5624b82..c835d176a5 100644 --- a/hw/rtl/core/VX_dcr_data.sv +++ b/hw/rtl/core/VX_dcr_data.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_dcr_data import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // Inputs VX_dcr_bus_if.slave dcr_bus_if, @@ -50,7 +51,7 @@ module VX_dcr_data import VX_gpu_pkg::*; #( wire is_mpm_read = dcr_bus_if.req_valid && ~dcr_bus_if.req_data.rw && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_MPM_VALUE) - && (mpm_target_cid == 16'(CORE_ID)) + && (mpm_target_cid == 16'(core_id)) && ~dcr_csr_pending_r; always @(posedge clk) begin @@ -89,7 +90,7 @@ module VX_dcr_data import VX_gpu_pkg::*; #( && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); wire is_flush_write = is_flush_req && dcr_bus_if.req_data.rw; wire is_flush_read = is_flush_req && ~dcr_bus_if.req_data.rw - && (mpm_target_cid == 16'(CORE_ID)); + && (mpm_target_cid == 16'(core_id)); wire flush_done = flush_pending_r && dcr_flush_if.done; wire flush_start = ~flush_pending_r && is_flush_write; diff --git a/hw/rtl/core/VX_decompressor.sv b/hw/rtl/core/VX_decompressor.sv index de3aea297e..229400d48e 100644 --- a/hw/rtl/core/VX_decompressor.sv +++ b/hw/rtl/core/VX_decompressor.sv @@ -66,7 +66,9 @@ // state[]/buf_pc[]/BRAM/follow_req/rsp_ready and sched_buffered_match all // gated on s1_can_accept. module VX_decompressor import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_WARPS = `VX_CFG_NUM_WARPS, + parameter WID_W = `LOG2UP(NUM_WARPS) ) ( input wire clk, input wire reset, @@ -76,7 +78,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( input wire [PC_BITS-1:0] sched_PC, input wire [`VX_CFG_NUM_THREADS-1:0] sched_tmask, input wire [NCTA_WIDTH-1:0] sched_cta_id, - input wire [NW_WIDTH-1:0] sched_wid, + input wire [WID_W-1:0] sched_wid, output wire sched_buffered_match, // Ungated buffered-hit: high whenever the scheduled PC is already in the // decompressor buffer, regardless of stage-0 stall. Used by VX_fetch to @@ -89,19 +91,19 @@ module VX_decompressor import VX_gpu_pkg::*; #( input wire [PC_BITS-1:0] rsp_PC, input wire [`VX_CFG_NUM_THREADS-1:0] rsp_tmask, input wire [NCTA_WIDTH-1:0] rsp_cta_id, - input wire [NW_WIDTH-1:0] rsp_wid, + input wire [WID_W-1:0] rsp_wid, input wire [UUID_WIDTH-1:0] rsp_uuid, output logic rsp_ready, // One-outstanding-per-warp tracking (from VX_fetch): high while a warp has // an icache request in flight. Used to gate the persistent follow arbiter. - input wire [`VX_CFG_NUM_WARPS-1:0] inflight, + input wire [NUM_WARPS-1:0] inflight, // Follow-up icache request (cross-word 32-bit). output logic follow_req_valid, output logic [PC_BITS-1:0] follow_req_PC, output logic [`VX_CFG_NUM_THREADS-1:0] follow_req_tmask, - output logic [NW_WIDTH-1:0] follow_req_wid, + output logic [WID_W-1:0] follow_req_wid, output logic [UUID_WIDTH-1:0] follow_req_uuid, // Decoded fetch interface. @@ -151,7 +153,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( typedef struct packed { logic [31:0] payload32; logic [PC_BITS-1:0] PC; - logic [NW_WIDTH-1:0] wid; + logic [WID_W-1:0] wid; logic [UUID_WIDTH-1:0] uuid; logic [`VX_CFG_NUM_THREADS-1:0] tmask; logic [NCTA_WIDTH-1:0] cta_id; @@ -338,25 +340,25 @@ module VX_decompressor import VX_gpu_pkg::*; #( // State storage // ------------------------------------------------------------------ - buf_state_e state [`VX_CFG_NUM_WARPS]; - buf_state_e state_n [`VX_CFG_NUM_WARPS]; - logic [PC_BITS-1:0] buf_pc [`VX_CFG_NUM_WARPS]; - logic [PC_BITS-1:0] buf_pc_n [`VX_CFG_NUM_WARPS]; + buf_state_e state [NUM_WARPS]; + buf_state_e state_n [NUM_WARPS]; + logic [PC_BITS-1:0] buf_pc [NUM_WARPS]; + logic [PC_BITS-1:0] buf_pc_n [NUM_WARPS]; // Follow metadata captured on BUF_32HI entry (the persistent follow arbiter // re-issues the request from registered state, so it needs its own copy). - logic [UUID_WIDTH-1:0] hi_uuid [`VX_CFG_NUM_WARPS]; - logic [UUID_WIDTH-1:0] hi_uuid_n [`VX_CFG_NUM_WARPS]; - logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask [`VX_CFG_NUM_WARPS]; - logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask_n [`VX_CFG_NUM_WARPS]; + logic [UUID_WIDTH-1:0] hi_uuid [NUM_WARPS]; + logic [UUID_WIDTH-1:0] hi_uuid_n [NUM_WARPS]; + logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask [NUM_WARPS]; + logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask_n [NUM_WARPS]; // High half of the cross-word 32-bit, latched from the follow response and // held (BUF_32RDY) until the scheduler-gated emit combines it with the // BRAM-stashed low half. - logic [15:0] hi_hw [`VX_CFG_NUM_WARPS]; - logic [15:0] hi_hw_n [`VX_CFG_NUM_WARPS]; + logic [15:0] hi_hw [NUM_WARPS]; + logic [15:0] hi_hw_n [NUM_WARPS]; // drain[w]: the BUF_32HI straddle on warp w was squashed by a redirect // while its follow was in flight; discard the follow response (no emit). - logic [`VX_CFG_NUM_WARPS-1:0] drain, drain_n; + logic [NUM_WARPS-1:0] drain, drain_n; // spec[w]: the BUF_32HI straddle is speculative — it trails an RVC emitted // from the same word (have_D_emit_lo), which may be a taken branch, so the // warp is NOT yet committed to the straddle PC. Speculative straddles defer @@ -365,18 +367,18 @@ module VX_decompressor import VX_gpu_pkg::*; #( // (have_D_xword: the warp is already at the straddle PC and was acked by its // own fetch, so the scheduler will not re-present it) emit on the follow // response (have_B_resp). - logic [`VX_CFG_NUM_WARPS-1:0] spec, spec_n; + logic [NUM_WARPS-1:0] spec, spec_n; logic buf_we; - logic [NW_WIDTH-1:0] buf_waddr; + logic [WID_W-1:0] buf_waddr; buf_data_t buf_wdata; - logic [NW_WIDTH-1:0] buf_raddr; + logic [WID_W-1:0] buf_raddr; buf_data_t buf_rdata; // registered (OUT_REG=1) logic buf_read; // gates rdata_r update during stall VX_dp_ram #( .DATAW ($bits(buf_data_t)), - .SIZE (`VX_CFG_NUM_WARPS), + .SIZE (NUM_WARPS), .OUT_REG (1), .LUTRAM (1), .RDW_MODE("R") @@ -530,7 +532,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( buf_we = 1'b0; buf_waddr = rsp_wid; buf_wdata = '0; - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin + for (int w = 0; w < NUM_WARPS; ++w) begin state_n[w] = state[w]; buf_pc_n[w] = buf_pc[w]; hi_uuid_n[w] = hi_uuid[w]; @@ -552,8 +554,8 @@ module VX_decompressor import VX_gpu_pkg::*; #( // BUF_RVC : no outstanding fetch -> clear directly. // (buf_pc == sched_PC is normal straddler continuation after an // RVC-then-straddler word — must NOT drop.) - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin - if (sched_valid && (sched_wid == w[NW_WIDTH-1:0]) + for (int w = 0; w < NUM_WARPS; ++w) begin + if (sched_valid && (sched_wid == WID_W'(w)) && (buf_pc[w] != sched_PC)) begin if (state[w] == BUF_32HI) begin drain_n[w] = 1'b1; @@ -684,14 +686,14 @@ module VX_decompressor import VX_gpu_pkg::*; #( // alias. Lowest-wid priority (reverse scan). // ------------------------------------------------------------------ logic follow_sel_valid; - logic [NW_WIDTH-1:0] follow_sel_wid; + logic [WID_W-1:0] follow_sel_wid; always_comb begin follow_sel_valid = 1'b0; follow_sel_wid = '0; - for (int w = `VX_CFG_NUM_WARPS-1; w >= 0; --w) begin - if ((state[w] == BUF_32HI) && ~inflight[w[NW_WIDTH-1:0]]) begin + for (int w = NUM_WARPS-1; w >= 0; --w) begin + if ((state[w] == BUF_32HI) && ~inflight[w]) begin follow_sel_valid = 1'b1; - follow_sel_wid = w[NW_WIDTH-1:0]; + follow_sel_wid = WID_W'(w); end end end @@ -716,7 +718,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( always_comb begin fsm_data = '0; fsm_data.PC = s1_ctx.PC; - fsm_data.wid = s1_ctx.wid; + fsm_data.wid = NW_WIDTH'(s1_ctx.wid); // cta_id: response-gated straddle (S1_BR) carries the BRAM-stashed CTA; // all other kinds captured the CTA into ctx (sched or rsp) in stage 0. fsm_data.cta_id = (s1_kind == S1_BR) ? buf_rdata.cta_id : s1_ctx.cta_id; @@ -761,7 +763,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( always_comb begin fast_data = '0; fast_data.uuid = rsp_uuid; - fast_data.wid = rsp_wid; + fast_data.wid = NW_WIDTH'(rsp_wid); fast_data.tmask = rsp_tmask; fast_data.cta_id = rsp_cta_id; fast_data.PC = rsp_PC; @@ -782,7 +784,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( always_ff @(posedge clk) begin if (reset) begin - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin + for (int w = 0; w < NUM_WARPS; ++w) begin state[w] <= BUF_EMPTY; buf_pc[w] <= '0; hi_uuid[w] <= '0; @@ -794,7 +796,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( s1_kind <= S1_NONE; s1_ctx <= '0; end else begin - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin + for (int w = 0; w < NUM_WARPS; ++w) begin state[w] <= state_n[w]; buf_pc[w] <= buf_pc_n[w]; hi_uuid[w] <= hi_uuid_n[w]; diff --git a/hw/rtl/core/VX_dispatcher.sv b/hw/rtl/core/VX_dispatcher.sv index d0c3a81d62..9c17114d77 100644 --- a/hw/rtl/core/VX_dispatcher.sv +++ b/hw/rtl/core/VX_dispatcher.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_dispatcher import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [ISSUE_ISW_W-1:0] slice_id, + `ifdef PERF_ENABLE output wire [NUM_EX_UNITS-1:0][PERF_CTR_BITS-1:0] perf_stalls, output wire [NUM_EX_UNITS-1:0][PERF_CTR_BITS-1:0] perf_instrs, @@ -32,7 +33,9 @@ module VX_dispatcher import VX_gpu_pkg::*; #( VX_dispatch_if.master dispatch_if [NUM_EX_UNITS] ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (ISSUE_ID) +`ifndef DBG_TRACE_PIPELINE + `UNUSED_VAR (slice_id) +`endif localparam OUT_DATAW = $bits(dispatch_t); @@ -45,7 +48,7 @@ module VX_dispatcher import VX_gpu_pkg::*; #( end // All execution units pass operand data straight through; LSU address - // generation (pack-load included) lives in VX_lsu_slice. + // generation (pack-load included) lives in VX_lsu_block. for (genvar i = 0; i < NUM_EX_UNITS; ++i) begin : g_buffers VX_elastic_buffer #( .DATAW (OUT_DATAW), @@ -107,7 +110,7 @@ module VX_dispatcher import VX_gpu_pkg::*; #( for (genvar ex = 0; ex < NUM_EX_UNITS; ++ex) begin : g_dispatch_trace always @(posedge clk) begin if (dispatch_if[ex].valid && dispatch_if[ex].ready) begin - `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, wis_to_wid(dispatch_if[ex].data.wis, ISSUE_ID), dispatch_if[ex].data.cta_id, dispatch_if[ex].data.sid, to_fullPC(dispatch_if[ex].data.PC))) + `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, wis_to_wid(dispatch_if[ex].data.wis, slice_id), dispatch_if[ex].data.cta_id, dispatch_if[ex].data.sid, to_fullPC(dispatch_if[ex].data.PC))) VX_trace_pkg::trace_ex_type(1, ex); `TRACE(1, (", op=")) VX_trace_pkg::trace_ex_op(1, ex, dispatch_if[ex].data.op_type, dispatch_if[ex].data.op_args); diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 4f0222e066..b1601ff6da 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -14,17 +14,16 @@ `include "VX_define.vh" module VX_execute import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + `ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, `ifdef VX_CFG_EXT_TCU_ENABLE output tcu_perf_t tcu_perf, `endif @@ -49,7 +48,6 @@ module VX_execute import VX_gpu_pkg::*; #( `ifdef VX_CFG_EXT_DXA_ENABLE VX_dxa_req_bus_if.master dxa_req_bus_if, - VX_txbar_bus_if.slave dxa_txbar_bus_if, `endif `ifdef VX_CFG_EXT_TEX_ENABLE @@ -65,21 +63,19 @@ module VX_execute import VX_gpu_pkg::*; #( `ifdef VX_CFG_EXT_RTU_ENABLE VX_rtu_bus_if.master rtu_bus_if, - VX_sched_unlock_if.master sched_unlock_if, + VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif - // scheduler interfaces - VX_sched_csr_if.slave sched_csr_if, - VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], - VX_warp_ctl_if.master warp_ctl_if, - `ifdef VX_CFG_TCU_WGMMA_ENABLE // TCU tile-buffer local-memory read port VX_mem_bus_if.master tcu_lmem_if, `endif - // DCR-CSR interface - VX_dcr_csr_if dcr_csr_if + // scheduler interfaces + VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], + VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS] ); `ifdef VX_CFG_EXT_F_ENABLE @@ -99,12 +95,12 @@ module VX_execute import VX_gpu_pkg::*; #( `SCOPE_IO_SWITCH (1); VX_lsu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))) ) lsu_unit ( `SCOPE_IO_BIND (0) .clk (clk), .reset (reset), + .core_id (core_id), .dispatch_if (dispatch_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .per_block_client_if (lsu_client_if) @@ -143,15 +139,11 @@ module VX_execute import VX_gpu_pkg::*; #( `endif VX_sfu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-sfu", INSTANCE_ID))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("%s-sfu", INSTANCE_ID))) ) sfu_unit ( .clk (clk), .reset (reset), - `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf), - .pipeline_perf (pipeline_perf), - `endif + .core_id (core_id), .dispatch_if (dispatch_if[EX_SFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_SFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), `ifdef VX_CFG_EXT_F_ENABLE @@ -159,22 +151,17 @@ module VX_execute import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (dxa_req_bus_if), - .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif `ifdef VX_CFG_EXT_TEX_ENABLE .tex_bus_if (tex_bus_if), `endif - `ifdef VX_CFG_EXT_OM_ENABLE - `endif - `ifdef VX_CFG_EXT_RASTER_ENABLE - `endif `ifdef VX_CFG_EXT_RTU_ENABLE .rtu_bus_if (rtu_bus_if), .sched_unlock_if (sched_unlock_if), `endif .sched_csr_if (sched_csr_if), - .warp_ctl_if (warp_ctl_if), - .dcr_csr_if (dcr_csr_if) + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if) ); endmodule diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv new file mode 100644 index 0000000000..541c84efe7 --- /dev/null +++ b/hw/rtl/core/VX_execute_slice.sv @@ -0,0 +1,401 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + +// The execution units of one issue slice: one ALU, LSU, SFU, FPU and TCU block +// each, and the FP CSRs of the slice's warps. +module VX_execute_slice import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_F_ENABLE +import VX_fpu_pkg::*; +`endif +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [ISSUE_ISW_W-1:0] slice_id, + + VX_dispatch_if.slave dispatch_if [NUM_EX_UNITS], + VX_commit_if.master commit_if [NUM_EX_UNITS], + + // ALU + VX_branch_ctl_if.master branch_ctl_if, + + // LSU + VX_lsu_sched_if.master lsu_client_if, + + // SFU +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, + VX_sched_unlock_if.master sched_unlock_if, +`endif + + // TCU +`ifdef VX_CFG_TCU_WGMMA_ENABLE + output wire tcu_exec_valid, + output wire tcu_exec_ready, + output tcu_execute_t tcu_exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire tcu_tbuf_ready, +`endif +`ifdef TCU_META_ENABLE + output wire tcu_agu_ld_valid, + output tcu_execute_t tcu_agu_ld_data, + input wire tcu_agu_ld_ready, + input wire tcu_agu_result_valid, + input tcu_result_t tcu_agu_result_data, + output wire tcu_agu_result_ready, + input wire tcu_meta_wr_en, + input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, + input wire [4:0] tcu_meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, +`endif + + // SFU warp control and CSRs + VX_warp_shared_if.master warp_shared_if, + VX_warp_ctl_if.master warp_ctl_if, + VX_sched_csr_if.slave sched_csr_if +); + localparam ALU_PARTIAL_BW = (`VX_CFG_NUM_ALU_LANES != `VX_CFG_SIMD_WIDTH); +`ifdef VX_CFG_EXT_F_ENABLE + localparam FPU_PARTIAL_BW = (`VX_CFG_NUM_FPU_LANES != `VX_CFG_SIMD_WIDTH); +`endif + +`ifdef SCOPE + localparam SCOPE_COUNT = 1 + `VX_CFG_EXT_TCU_ENABLED; +`endif + `SCOPE_IO_SWITCH (SCOPE_COUNT); + + // ALU + + VX_execute_if #( + .data_t (alu_execute_t) + ) alu_execute_if[1](); + + VX_result_if #( + .data_t (alu_result_t) + ) alu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_ALU_LANES), + .OUT_BUF (ALU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) alu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_ALU +: 1]), + .execute_if (alu_execute_if) + ); + + VX_alu_block #( + .INSTANCE_ID (`SFORMATF(("%s-alu", INSTANCE_ID))), + .BLOCK_STR (SLICE_STR), + .NUM_LANES (`VX_CFG_NUM_ALU_LANES), + .PARTIAL_BW (ALU_PARTIAL_BW) + ) alu_block ( + .clk (clk), + .reset (reset), + .block_idx (slice_id), + .execute_if (alu_execute_if[0]), + .result_if (alu_result_if[0]), + .branch_ctl_if (branch_ctl_if) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_ALU_LANES), + .OUT_BUF (ALU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) alu_gather ( + .clk (clk), + .reset (reset), + .result_if (alu_result_if), + .commit_if (commit_if[EX_ALU +: 1]) + ); + + // LSU + + VX_execute_if #( + .data_t (lsu_execute_t) + ) lsu_execute_if[1](); + + VX_result_if #( + .data_t (lsu_result_t) + ) lsu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) lsu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_LSU +: 1]), + .execute_if (lsu_execute_if) + ); + + VX_lsu_block #( + .INSTANCE_ID (`SFORMATF(("%s-lsu%s", INSTANCE_ID, SLICE_STR))) + ) lsu_block ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (lsu_execute_if[0]), + .result_if (lsu_result_if[0]), + .client_if (lsu_client_if) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) lsu_gather ( + .clk (clk), + .reset (reset), + .result_if (lsu_result_if), + .commit_if (commit_if[EX_LSU +: 1]) + ); + + // SFU + + VX_execute_if #( + .data_t (sfu_execute_t) + ) sfu_execute_if[1](); + + VX_result_if #( + .data_t (sfu_result_t) + ) sfu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) sfu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_SFU +: 1]), + .execute_if (sfu_execute_if) + ); + +`ifdef VX_CFG_EXT_F_ENABLE + wire [NW_WIDTH-1:0] fcsr_wid; + wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data; + wire [2:0] fcsr_wr_sel; + wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data; +`endif + + VX_sfu_block #( + .INSTANCE_ID ((`VX_CFG_NUM_SFU_BLOCKS > 1) ? `SFORMATF(("%s-sfu%s", INSTANCE_ID, SLICE_STR)) : `SFORMATF(("%s-sfu", INSTANCE_ID))) + ) sfu_block ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .block_idx (SFU_BLOCK_W'(slice_id)), + .execute_if (sfu_execute_if[0]), + .result_if (sfu_result_if[0]), + `ifdef VX_CFG_EXT_F_ENABLE + .fcsr_wid (fcsr_wid), + .fcsr_rd_data (fcsr_rd_data), + .fcsr_wr_sel (fcsr_wr_sel), + .fcsr_wr_data (fcsr_wr_data), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if(sched_unlock_if), + `endif + .sched_csr_if (sched_csr_if), + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) sfu_gather ( + .clk (clk), + .reset (reset), + .result_if (sfu_result_if), + .commit_if (commit_if[EX_SFU +: 1]) + ); + + // FPU + +`ifdef VX_CFG_EXT_F_ENABLE + VX_fpu_csr_if fpu_csr_if[1](); + + VX_execute_if #( + .data_t (fpu_execute_t) + ) fpu_execute_if[1](); + + VX_result_if #( + .data_t (fpu_result_t) + ) fpu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_FPU_LANES), + .OUT_BUF (FPU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) fpu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_FPU +: 1]), + .execute_if (fpu_execute_if) + ); + + VX_fpu_block #( + .INSTANCE_ID (`SFORMATF(("%s-fpu", INSTANCE_ID))), + .NUM_LANES (`VX_CFG_NUM_FPU_LANES), + .PARTIAL_BW (FPU_PARTIAL_BW) + ) fpu_block ( + .clk (clk), + .reset (reset), + .block_idx (slice_id), + .execute_if (fpu_execute_if[0]), + .result_if (fpu_result_if[0]), + .fpu_csr_if (fpu_csr_if[0]) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_FPU_LANES), + .OUT_BUF (FPU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) fpu_gather ( + .clk (clk), + .reset (reset), + .result_if (fpu_result_if), + .commit_if (commit_if[EX_FPU +: 1]) + ); + + VX_csr_fpu #( + .WID_STRIDE (`VX_CFG_ISSUE_WIDTH), + .NUM_CSR_PORTS (1), + .NUM_FPU_PORTS (1) + ) csr_fpu ( + .clk (clk), + .reset (reset), + .csr_wid (fcsr_wid), + .csr_rd_data (fcsr_rd_data), + .csr_wr_sel (fcsr_wr_sel), + .csr_wr_data (fcsr_wr_data), + .fpu_csr_if (fpu_csr_if) + ); +`endif + + // TCU + +`ifdef VX_CFG_EXT_TCU_ENABLE + VX_execute_if #( + .data_t (tcu_execute_t) + ) tcu_execute_if[1](); + + VX_result_if #( + .data_t (tcu_result_t) + ) tcu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_TCU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) tcu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_TCU +: 1]), + .execute_if (tcu_execute_if) + ); + + VX_tcu_block #( + .INSTANCE_ID (`SFORMATF(("%s-tcu", INSTANCE_ID))), + .BLOCK_STR (SLICE_STR) + ) tcu_block ( + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + `endif + .execute_if (tcu_execute_if[0]), + .result_if (tcu_result_if[0]) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_TCU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) tcu_gather ( + .clk (clk), + .reset (reset), + .result_if (tcu_result_if), + .commit_if (commit_if[EX_TCU +: 1]) + ); +`endif + +endmodule diff --git a/hw/rtl/core/VX_fetch.sv b/hw/rtl/core/VX_fetch.sv index 72cc0b6754..a4f8847e7f 100644 --- a/hw/rtl/core/VX_fetch.sv +++ b/hw/rtl/core/VX_fetch.sv @@ -13,8 +13,11 @@ `include "VX_define.vh" +// NUM_WARPS is the number of warps this unit fetches for; their ids arrive in +// the low bits of schedule_if.data.wid and leave the same way on fetch_if. module VX_fetch import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_WARPS = `VX_CFG_NUM_WARPS ) ( `SCOPE_IO_DECL @@ -37,6 +40,14 @@ module VX_fetch import VX_gpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_VAR (reset) + localparam WID_W = `LOG2UP(NUM_WARPS); + `STATIC_ASSERT (WID_W == ICACHE_TAG_ID_BITS, ("invalid parameter")) + + wire [WID_W-1:0] sched_wid = schedule_if.data.wid[WID_W-1:0]; + if (NW_WIDTH > WID_W) begin : g_sched_wid_unused + `UNUSED_VAR (schedule_if.data.wid[NW_WIDTH-1:WID_W]) + end + wire icache_req_valid; wire icache_req_ready; wire [ICACHE_ADDR_WIDTH-1:0] icache_req_addr; @@ -44,25 +55,25 @@ module VX_fetch import VX_gpu_pkg::*; #( // tag bits VX_mem_bus_arb adds downstream live in ICACHE_TAG_WIDTH but // are not driven by this stage. wire [ICACHE_FETCH_TAG_WIDTH-1:0] icache_req_tag; - wire [NW_WIDTH-1:0] icache_req_wid; + wire [WID_W-1:0] icache_req_wid; wire [UUID_WIDTH-1:0] icache_req_uuid; wire [UUID_WIDTH-1:0] rsp_uuid; wire [PC_BITS-1:0] rsp_PC; wire [`VX_CFG_NUM_THREADS-1:0] rsp_tmask; - wire [NW_WIDTH-1:0] req_tag; - wire [NW_WIDTH-1:0] rsp_tag; + wire [WID_W-1:0] req_tag; + wire [WID_W-1:0] rsp_tag; wire [NCTA_WIDTH-1:0] rsp_cta_id; wire icache_req_fire = icache_req_valid && icache_req_ready; - assign req_tag = schedule_if.data.wid; + assign req_tag = sched_wid; assign {rsp_uuid, rsp_tag} = icache_bus_if.rsp_data.tag; VX_dp_ram #( .DATAW (PC_BITS + `VX_CFG_NUM_THREADS + NCTA_WIDTH), - .SIZE (`VX_CFG_NUM_WARPS), + .SIZE (NUM_WARPS), .RDW_MODE ("R"), .LUTRAM (1) ) tag_store ( @@ -87,12 +98,12 @@ module VX_fetch import VX_gpu_pkg::*; #( wire [PC_BITS-1:0] icache_req_PC; wire [`VX_CFG_NUM_THREADS-1:0] icache_req_tmask; - wire [NW_WIDTH-1:0] rsp_wid; + wire [WID_W-1:0] rsp_wid; wire follow_req_valid; wire [PC_BITS-1:0] follow_req_PC; wire [`VX_CFG_NUM_THREADS-1:0] follow_req_tmask; - wire [NW_WIDTH-1:0] follow_req_wid; + wire [WID_W-1:0] follow_req_wid; wire [UUID_WIDTH-1:0] follow_req_uuid; wire sched_buffered_match; @@ -102,23 +113,23 @@ module VX_fetch import VX_gpu_pkg::*; #( // One-outstanding-per-warp (Invariant B): a warp with an icache request in // flight must not issue another, else its two responses alias the single // tag_store slot. Set on request fire, cleared on response fire. - reg [`VX_CFG_NUM_WARPS-1:0] inflight; + reg [NUM_WARPS-1:0] inflight; wire icache_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; always @(posedge clk) begin if (reset) begin inflight <= '0; end else begin - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin - if (icache_req_fire && (icache_req_wid == w[NW_WIDTH-1:0])) + for (int w = 0; w < NUM_WARPS; ++w) begin + if (icache_req_fire && (icache_req_wid == WID_W'(w))) inflight[w] <= 1'b1; - else if (icache_rsp_fire && (rsp_wid == w[NW_WIDTH-1:0])) + else if (icache_rsp_fire && (rsp_wid == WID_W'(w))) inflight[w] <= 1'b0; end end end `UNUSED_VAR (icache_req_tmask) - // ibuffer occupancy is already gated by VX_scheduler (schedule_warps + // ibuffer occupancy is already gated by VX_warp_sched (schedule_warps // masks out warps with full ibufs), so schedule_if.valid implies space. // Suppress with the UNGATED sched_buffered (not the st0-gated match): a // word already in the decompressor buffer must never be re-fetched, even @@ -135,7 +146,7 @@ module VX_fetch import VX_gpu_pkg::*; #( assign icache_req_valid = follow_req_valid || sched_req_valid; assign icache_req_PC = follow_req_valid ? follow_req_PC : schedule_if.data.PC; assign icache_req_tmask = follow_req_valid ? follow_req_tmask : schedule_if.data.tmask; - assign icache_req_wid = follow_req_valid ? follow_req_wid : schedule_if.data.wid; + assign icache_req_wid = follow_req_valid ? follow_req_wid : sched_wid; assign icache_req_uuid = follow_req_valid ? follow_req_uuid : schedule_if.data.uuid; // NOTE: bind the full PC to a net before part-selecting — Vivado synthesis // rejects a bit-select applied directly to a function-call result. @@ -156,7 +167,8 @@ module VX_fetch import VX_gpu_pkg::*; #( || sched_buffered_match; VX_decompressor #( - .INSTANCE_ID (INSTANCE_ID) + .INSTANCE_ID (INSTANCE_ID), + .NUM_WARPS (NUM_WARPS) ) decompressor ( .clk (clk), .reset (reset), @@ -164,7 +176,7 @@ module VX_fetch import VX_gpu_pkg::*; #( .sched_PC (schedule_if.data.PC), .sched_tmask (schedule_if.data.tmask), .sched_cta_id (schedule_if.data.cta_id), - .sched_wid (schedule_if.data.wid), + .sched_wid (sched_wid), .sched_buffered_match (sched_buffered_match), .sched_buffered (sched_buffered), .inflight (inflight), @@ -191,7 +203,7 @@ module VX_fetch import VX_gpu_pkg::*; #( assign icache_req_valid = schedule_if.valid; assign icache_req_addr = schedule_if.data.PC[2-(`VX_CFG_XLEN-PC_BITS) +: ICACHE_ADDR_WIDTH]; - assign icache_req_wid = schedule_if.data.wid; + assign icache_req_wid = sched_wid; assign icache_req_uuid = schedule_if.data.uuid; assign icache_req_tag = {icache_req_uuid, icache_req_wid}; assign schedule_if.ibuf_pop = fetch_if.ibuf_pop; @@ -199,7 +211,7 @@ module VX_fetch import VX_gpu_pkg::*; #( assign fetch_if.valid = icache_bus_if.rsp_valid; assign fetch_if.data.tmask = rsp_tmask; - assign fetch_if.data.wid = rsp_tag; + assign fetch_if.data.wid = NW_WIDTH'(rsp_tag); assign fetch_if.data.cta_id = rsp_cta_id; assign fetch_if.data.PC = rsp_PC; assign fetch_if.data.instr = icache_bus_if.rsp_data.data; diff --git a/hw/rtl/core/VX_fetch_slice.sv b/hw/rtl/core/VX_fetch_slice.sv new file mode 100644 index 0000000000..41d788c064 --- /dev/null +++ b/hw/rtl/core/VX_fetch_slice.sv @@ -0,0 +1,149 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One fetch slice: warp scheduling, fetch and decode of the warps with +// wid mod FETCH_WIDTH == slice_id, on the slice's own icache port. The front end +// names the slice's warps by wfs; decoded instructions leave with their +// core-wide wid, and ibuffer pops return indexed by wfs. +module VX_fetch_slice import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [FETCH_FSW_W-1:0] slice_id, + +`ifdef PERF_ENABLE + output fetch_perf_t fetch_perf, +`endif + + // core scheduler + VX_slice_sched_if.slave slice_sched_if, + + // instruction fetch + VX_mem_bus_if.master icache_bus_if, + + // decoded instructions + VX_decode_if.master decode_if, + + // ALU ports + input sched_branch_t [FETCH_ALU_PORTS-1:0] branch, + + // SFU ports + input sched_wctl_t [FETCH_SFU_PORTS-1:0] wctl, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] dvstack_wid, + output wire [FETCH_SFU_PORTS-1:0][DV_STACK_SIZEW-1:0] dvstack_ptr, + output wire [FETCH_SFU_PORTS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] dvstack_mscratch, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] csr_rd_wid, + output wire sched_csr_rd_t [FETCH_SFU_PORTS-1:0] csr_rd, + input sched_csr_wr_t [FETCH_SFU_PORTS-1:0] csr_wr, +`ifdef VX_CFG_EXT_RTU_ENABLE + input wire [FETCH_SFU_PORTS-1:0] rtu_unlock_valid, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] rtu_unlock_wid, +`endif + + // per-warp pipeline events, indexed by wfs + input wire [PER_FETCH_WARPS-1:0] issued, + input wire [PER_FETCH_WARPS-1:0] committed, + output wire [PER_FETCH_WARPS-1:0] pending_alm_empty +); + localparam SW = PER_FETCH_WARPS; + localparam LW = FETCH_WFS_W; + + `SCOPE_IO_SWITCH (1); + + VX_schedule_if schedule_if(); + VX_fetch_if fetch_if(); + VX_decode_if fe_decode_if(); + VX_decode_sched_if decode_sched_if(); + + VX_warp_sched #( + .INSTANCE_ID (`SFORMATF(("%s-scheduler%s", INSTANCE_ID, SLICE_STR))) + ) warp_sched ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (slice_id), + .sched_if (slice_sched_if), + .branch (branch), + .wctl (wctl), + .dvstack_wid (dvstack_wid), + .dvstack_ptr (dvstack_ptr), + .dvstack_mscratch (dvstack_mscratch), + .csr_rd_wid (csr_rd_wid), + .csr_rd (csr_rd), + .csr_wr (csr_wr), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (rtu_unlock_valid), + .rtu_unlock_wid (rtu_unlock_wid), + `endif + .decode_sched_if (decode_sched_if), + .issued (issued), + .committed (committed), + .ibuf_pop (decode_if.ibuf_pop[SW-1:0]), + .pending_alm_empty(pending_alm_empty), + .schedule_if (schedule_if) + ); + + VX_fetch #( + .INSTANCE_ID (`SFORMATF(("%s-fetch%s", INSTANCE_ID, SLICE_STR))), + .NUM_WARPS (SW) + ) fetch ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .fetch_perf (fetch_perf), + `endif + .icache_bus_if (icache_bus_if), + .schedule_if (schedule_if), + .fetch_if (fetch_if) + ); + + VX_decode #( + .INSTANCE_ID (`SFORMATF(("%s-decode%s", INSTANCE_ID, SLICE_STR))) + ) decode ( + .clk (clk), + .reset (reset), + .fetch_if (fetch_if), + .decode_if (fe_decode_if), + .decode_sched_if(decode_sched_if) + ); + + // decoded instructions take their core-wide wid + decode_t decode_data; + always @(*) begin + decode_data = fe_decode_if.data; + decode_data.wid = wfs_to_wid(fe_decode_if.data.wid[LW-1:0], slice_id); + end + if (NW_WIDTH > LW) begin : g_fe_wid_unused + `UNUSED_VAR (fe_decode_if.data.wid[NW_WIDTH-1:LW]) + end + + assign decode_if.valid = fe_decode_if.valid; + assign decode_if.data = decode_data; + assign fe_decode_if.ready = decode_if.ready; + + assign fe_decode_if.ibuf_pop[SW-1:0] = decode_if.ibuf_pop[SW-1:0]; + if (SW < `VX_CFG_NUM_WARPS) begin : g_ibuf_pop_pad + assign fe_decode_if.ibuf_pop[`VX_CFG_NUM_WARPS-1:SW] = '0; + `UNUSED_VAR (decode_if.ibuf_pop[`VX_CFG_NUM_WARPS-1:SW]) + end + +endmodule diff --git a/hw/rtl/core/VX_fetch_xbar.sv b/hw/rtl/core/VX_fetch_xbar.sv new file mode 100644 index 0000000000..85c7babd07 --- /dev/null +++ b/hw/rtl/core/VX_fetch_xbar.sv @@ -0,0 +1,229 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Connects the fetch slices (fetch slice f owns wid mod FETCH_WIDTH == f) to the +// issue slices (wid mod ISSUE_WIDTH) and the ALU/SFU blocks (wid mod B). Issue +// slice i takes its instructions from fetch slice i mod FETCH_WIDTH. A block +// serving several fetch slices sends each its own warps' events and answers its +// reads from the slice owning the read wid. Purely combinational. +module VX_fetch_xbar import VX_gpu_pkg::*; ( + // instruction path + VX_decode_if.slave fs_decode_if [`VX_CFG_FETCH_WIDTH], + VX_decode_if.master is_decode_if [`VX_CFG_ISSUE_WIDTH], + + // issue and commit events, and per-warp drain status + input wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid, + input wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis, + input wire [`VX_CFG_NUM_WARPS-1:0] committed_warps, + output wire [`VX_CFG_FETCH_WIDTH-1:0][PER_FETCH_WARPS-1:0] fs_issued, + output wire [`VX_CFG_FETCH_WIDTH-1:0][PER_FETCH_WARPS-1:0] fs_committed, + input wire [`VX_CFG_FETCH_WIDTH-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty, + + // FU blocks + VX_branch_ctl_if.slave branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], + VX_warp_ctl_if.slave warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_sched_csr_if.master sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if.slave sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], +`endif + + // core scheduler state + input wire [PERF_CTR_BITS-1:0] cycles, + input wire [PERF_CTR_BITS-1:0] instret, + input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, + input wire lsu_sched_drained, + input cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, +`ifdef VX_CFG_VM_ENABLE + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, + output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid, + output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data, +`endif + + // fetch slices + output sched_branch_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_ALU_PORTS-1:0] fs_branch, + output sched_wctl_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_wctl, + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_dvstack_wid, + input wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr, + input wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch, + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid, + input sched_csr_rd_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_csr_rd, +`ifdef VX_CFG_EXT_RTU_ENABLE + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_rtu_unlock_valid, + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid, +`endif + output sched_csr_wr_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_csr_wr +); + localparam FW = `VX_CFG_FETCH_WIDTH; + localparam IW = `VX_CFG_ISSUE_WIDTH; + localparam NW = `VX_CFG_NUM_WARPS; + localparam SW = PER_FETCH_WARPS; + localparam BA = `VX_CFG_NUM_ALU_BLOCKS; + localparam BS = `VX_CFG_NUM_SFU_BLOCKS; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; + + `STATIC_ASSERT (`IS_POW2(FW) && (FW <= IW) && ((IW % FW) == 0), ("invalid parameter")) + + // instruction path + + wire [IW-1:0] is_ready; + for (genvar i = 0; i < IW; ++i) begin : g_issue_slices + localparam F = i % FW; + assign is_decode_if[i].valid = fs_decode_if[F].valid && (wid_to_isw(fs_decode_if[F].data.wid) == ISSUE_ISW_W'(i)); + assign is_decode_if[i].data = fs_decode_if[F].data; + assign is_ready[i] = is_decode_if[i].ready; + if (PER_ISSUE_WARPS < NW) begin : g_ibuf_pop_unused + `UNUSED_VAR (is_decode_if[i].ibuf_pop[NW-1:PER_ISSUE_WARPS]) + end + end + + for (genvar f = 0; f < FW; ++f) begin : g_fetch_slices + assign fs_decode_if[f].ready = is_ready[wid_to_isw(fs_decode_if[f].data.wid)]; + for (genvar l = 0; l < SW; ++l) begin : g_ibuf_pop + localparam W = l * FW + f; + assign fs_decode_if[f].ibuf_pop[l] = is_decode_if[W % IW].ibuf_pop[W / IW]; + end + if (SW < NW) begin : g_ibuf_pop_pad + assign fs_decode_if[f].ibuf_pop[NW-1:SW] = '0; + end + end + + // issue and commit events, and per-warp drain status + + wire [NW-1:0] issued_warps; + wire [NW-1:0] pending_alm_empty; + for (genvar w = 0; w < NW; ++w) begin : g_warps + assign issued_warps[w] = issue_valid[w % IW] && (issue_wis[w % IW] == ISSUE_WIS_W'(w / IW)); + assign fs_issued[w % FW][w / FW] = issued_warps[w]; + assign fs_committed[w % FW][w / FW] = committed_warps[w]; + assign pending_alm_empty[w] = fs_pending_alm_empty[w % FW][w / FW]; + end + + // ALU blocks: fetch slice f port q is block q * FW + f, or block f mod BA when shared + + for (genvar f = 0; f < FW; ++f) begin : g_alu_slices + for (genvar q = 0; q < NPA; ++q) begin : g_ports + localparam B = (BA >= FW) ? (q * FW + f) : (f % BA); + wire own; + if (BA >= FW) begin : g_own + assign own = 1'b1; + end else begin : g_shared + assign own = (wid_to_fsw(branch_ctl_if[B].wid) == FETCH_FSW_W'(f)); + end + assign fs_branch[f][q].valid = branch_ctl_if[B].valid && own; + assign fs_branch[f][q].wid = branch_ctl_if[B].wid; + assign fs_branch[f][q].taken = branch_ctl_if[B].taken; + assign fs_branch[f][q].dest = branch_ctl_if[B].dest; + assign fs_branch[f][q].is_trap = branch_ctl_if[B].is_trap; + assign fs_branch[f][q].is_mret = branch_ctl_if[B].is_mret; + assign fs_branch[f][q].trap_cause = branch_ctl_if[B].trap_cause; + end + end + + // SFU blocks: same port mapping; reads are answered by the slice owning the read wid + + for (genvar f = 0; f < FW; ++f) begin : g_sfu_slices + for (genvar q = 0; q < NPS; ++q) begin : g_ports + localparam B = (BS >= FW) ? (q * FW + f) : (f % BS); + wire own_ctl, own_wr; + `ifdef VX_CFG_EXT_RTU_ENABLE + wire own_unlock; + `endif + if (BS >= FW) begin : g_own + assign own_ctl = 1'b1; + assign own_wr = 1'b1; + `ifdef VX_CFG_EXT_RTU_ENABLE + assign own_unlock = 1'b1; + `endif + end else begin : g_shared + assign own_ctl = (wid_to_fsw(warp_ctl_if[B].wid) == FETCH_FSW_W'(f)); + assign own_wr = (wid_to_fsw(sched_csr_if[B].csr_wr_wid) == FETCH_FSW_W'(f)); + `ifdef VX_CFG_EXT_RTU_ENABLE + assign own_unlock = (wid_to_fsw(sched_unlock_if[B].wid) == FETCH_FSW_W'(f)); + `endif + end + + assign fs_wctl[f][q].tmc_valid = warp_ctl_if[B].tmc_valid && own_ctl; + assign fs_wctl[f][q].split_valid = warp_ctl_if[B].split_valid && own_ctl; + assign fs_wctl[f][q].sjoin_valid = warp_ctl_if[B].sjoin_valid && own_ctl; + assign fs_wctl[f][q].wsync_valid = warp_ctl_if[B].wsync_valid && own_ctl; + assign fs_wctl[f][q].wid = warp_ctl_if[B].wid; + assign fs_wctl[f][q].tmc = warp_ctl_if[B].tmc; + assign fs_wctl[f][q].split = warp_ctl_if[B].split; + assign fs_wctl[f][q].sjoin = warp_ctl_if[B].sjoin; + assign fs_dvstack_wid[f][q] = warp_ctl_if[B].dvstack_wid; + + assign fs_csr_rd_wid[f][q] = sched_csr_if[B].csr_rd_wid; + assign fs_csr_wr[f][q].valid = sched_csr_if[B].csr_wr_valid && own_wr; + assign fs_csr_wr[f][q].wid = sched_csr_if[B].csr_wr_wid; + assign fs_csr_wr[f][q].data = sched_csr_if[B].csr_wr_data; + assign fs_csr_wr[f][q].trap_valid = sched_csr_if[B].trap_csr_wr_valid & {NUM_TRAP_CSRS{own_wr}}; + assign fs_csr_wr[f][q].trap_data = sched_csr_if[B].trap_csr_wr_data; + + `ifdef VX_CFG_EXT_RTU_ENABLE + assign fs_rtu_unlock_valid[f][q] = sched_unlock_if[B].valid && own_unlock; + assign fs_rtu_unlock_wid[f][q] = sched_unlock_if[B].wid; + `endif + end + end + + for (genvar b = 0; b < BS; ++b) begin : g_sfu_blocks + sched_csr_rd_t csr_rd; + if (BS >= FW) begin : g_own + assign warp_ctl_if[b].dvstack_ptr = fs_dvstack_ptr[b % FW][b / FW]; + assign warp_ctl_if[b].mscratch = fs_dvstack_mscratch[b % FW][b / FW]; + assign csr_rd = fs_csr_rd[b % FW][b / FW]; + end else begin : g_shared + wire [FETCH_FSW_W-1:0] stack_fsw = wid_to_fsw(warp_ctl_if[b].dvstack_wid); + wire [FETCH_FSW_W-1:0] csr_fsw = wid_to_fsw(sched_csr_if[b].csr_rd_wid); + assign warp_ctl_if[b].dvstack_ptr = fs_dvstack_ptr[stack_fsw][0]; + assign warp_ctl_if[b].mscratch = fs_dvstack_mscratch[stack_fsw][0]; + assign csr_rd = fs_csr_rd[csr_fsw][0]; + end + + for (genvar k = 0; k < SFU_BLOCK_WARPS; ++k) begin : g_pending + assign warp_ctl_if[b].warp_pending_alm_empty[k] = pending_alm_empty[k * BS + b]; + end + assign warp_ctl_if[b].lsu_sched_drained = lsu_sched_drained; + + assign sched_csr_if[b].cycles = cycles; + assign sched_csr_if[b].instret = instret; + assign sched_csr_if[b].active_warps = active_warps; + assign sched_csr_if[b].thread_mask = csr_rd.thread_mask; + assign sched_csr_if[b].mscratch = csr_rd.mscratch; + assign sched_csr_if[b].csr_mstatus = csr_rd.mstatus; + assign sched_csr_if[b].csr_mtvec = csr_rd.mtvec; + assign sched_csr_if[b].csr_mepc = csr_rd.mepc; + assign sched_csr_if[b].csr_mcause = csr_rd.mcause; + assign sched_csr_if[b].csr_mtval = csr_rd.mtval; + assign sched_csr_if[b].cta_csrs.cta_id = sched_csr_if[b].csr_rd_cta_id; + assign sched_csr_if[b].cta_csrs.cta_rank = csr_rd.cta_rank; + assign sched_csr_if[b].cta_csrs.cta_size = csr_rd_ctx[b].cta_size; + assign sched_csr_if[b].cta_csrs.block_idx = csr_rd_ctx[b].block_idx; + assign sched_csr_if[b].cta_csrs.block_dim = csr_rd_ctx[b].block_dim; + assign sched_csr_if[b].cta_csrs.grid_dim = csr_rd_ctx[b].grid_dim; + assign sched_csr_if[b].cta_csrs.entry = csr_rd_ctx[b].entry; + assign sched_csr_if[b].cta_csrs.param = csr_rd_ctx[b].param; + assign sched_csr_if[b].cta_csrs.lmem_addr = csr_rd_ctx[b].lmem_addr; + assign sched_csr_if[b].cta_csrs.cluster_size = csr_rd_ctx[b].cluster_size; + assign sched_csr_if[b].cta_lane = csr_rd.cta_lane; + `ifdef VX_CFG_VM_ENABLE + assign sched_csr_if[b].csr_satp = satp[b]; + assign satp_wr_valid[b] = sched_csr_if[b].satp_wr_valid; + assign satp_wr_data[b] = sched_csr_if[b].satp_wr_data; + `endif + end + +endmodule diff --git a/hw/rtl/core/VX_ibuffer.sv b/hw/rtl/core/VX_ibuffer.sv index 377075c7f1..6cfd72f7bd 100644 --- a/hw/rtl/core/VX_ibuffer.sv +++ b/hw/rtl/core/VX_ibuffer.sv @@ -14,8 +14,7 @@ `include "VX_define.vh" module VX_ibuffer import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, @@ -30,7 +29,6 @@ module VX_ibuffer import VX_gpu_pkg::*; #( // outputs VX_ibuffer_if.master ibuffer_if [PER_ISSUE_WARPS] ); - `UNUSED_PARAM (ISSUE_ID) localparam OUT_DATAW = $bits(ibuffer_t); diff --git a/hw/rtl/core/VX_ipdom_stack.sv b/hw/rtl/core/VX_ipdom_stack.sv index a702d61d09..a5b8a8b3f7 100644 --- a/hw/rtl/core/VX_ipdom_stack.sv +++ b/hw/rtl/core/VX_ipdom_stack.sv @@ -14,31 +14,33 @@ `include "VX_define.vh" module VX_ipdom_stack import VX_gpu_pkg::*; #( - parameter WIDTH = 1, - parameter DEPTH = 1, - parameter ADDRW = `LOG2UP(DEPTH) + parameter WIDTH = 1, + parameter DEPTH = 1, + parameter NUM_WARPS = `VX_CFG_NUM_WARPS, + parameter WID_W = `LOG2UP(NUM_WARPS), + parameter ADDRW = `LOG2UP(DEPTH) ) ( input wire clk, input wire reset, - input wire [NW_WIDTH-1:0] wid, + input wire [WID_W-1:0] wid, input wire [WIDTH-1:0] d_val, input wire [ADDRW-1:0] rd_ptr, input wire push, input wire pop, output wire [WIDTH-1:0] q_val, output wire q_idx, - output wire [`VX_CFG_NUM_WARPS-1:0][ADDRW-1:0] wr_ptr, + output wire [NUM_WARPS-1:0][ADDRW-1:0] wr_ptr, output wire empty, output wire full ); localparam BRAM_DATAW = 1 + WIDTH; - localparam BRAM_SIZE = DEPTH * `VX_CFG_NUM_WARPS; + localparam BRAM_SIZE = DEPTH * NUM_WARPS; localparam BRAW_ADDRW = `LOG2UP(BRAM_SIZE); - wire [`VX_CFG_NUM_WARPS-1:0][ADDRW-1:0] wr_ptr_w; - wire [`VX_CFG_NUM_WARPS-1:0] empty_w, full_w; + wire [NUM_WARPS-1:0][ADDRW-1:0] wr_ptr_w; + wire [NUM_WARPS-1:0] empty_w, full_w; - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; i++) begin : g_addressing + for (genvar i = 0; i < NUM_WARPS; i++) begin : g_addressing reg [ADDRW-1:0] wr_ptr_r; reg empty_r, full_r; @@ -75,14 +77,14 @@ module VX_ipdom_stack import VX_gpu_pkg::*; #( wire [BRAW_ADDRW-1:0] raddr, waddr; - if (DEPTH > 1 && `VX_CFG_NUM_WARPS > 1) begin : g_DW + if (DEPTH > 1 && NUM_WARPS > 1) begin : g_DW assign waddr = push ? {wr_ptr_w[wid], wid} : {rd_ptr, wid}; assign raddr = {rd_ptr, wid}; end else if (DEPTH > 1) begin : g_D `UNUSED_VAR (wid) assign waddr = push ? wr_ptr_w : rd_ptr; assign raddr = rd_ptr; - end else if (`VX_CFG_NUM_WARPS > 1) begin : g_W + end else if (NUM_WARPS > 1) begin : g_W `UNUSED_VAR (rd_ptr) `UNUSED_VAR (wr_ptr_w) assign waddr = wid; diff --git a/hw/rtl/core/VX_issue.sv b/hw/rtl/core/VX_issue.sv index ddc745b577..03bff6cd98 100644 --- a/hw/rtl/core/VX_issue.sv +++ b/hw/rtl/core/VX_issue.sv @@ -25,7 +25,7 @@ module VX_issue import VX_gpu_pkg::*; #( output issue_perf_t issue_perf, `endif - VX_decode_if.slave decode_if, + VX_decode_if.slave decode_if [`VX_CFG_ISSUE_WIDTH], VX_writeback_if.slave writeback_if [`VX_CFG_ISSUE_WIDTH], VX_dispatch_if.master dispatch_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], VX_issue_sched_if.master issue_sched_if [`VX_CFG_ISSUE_WIDTH] @@ -43,11 +43,6 @@ module VX_issue import VX_gpu_pkg::*; #( end `endif - wire [ISSUE_ISW_W-1:0] decode_isw = wid_to_isw(decode_if.data.wid); - - wire [`VX_CFG_ISSUE_WIDTH-1:0] decode_ready_in; - assign decode_if.ready = decode_ready_in[decode_isw]; - `SCOPE_IO_SWITCH (`VX_CFG_ISSUE_WIDTH); wire [`VX_CFG_ISSUE_WIDTH-1:0] issued_warps; @@ -56,27 +51,18 @@ module VX_issue import VX_gpu_pkg::*; #( for (genvar issue_id = 0; issue_id < `VX_CFG_ISSUE_WIDTH; ++issue_id) begin : g_slices VX_dispatch_if per_issue_dispatch_if[NUM_EX_UNITS](); - VX_decode_if slice_decode_if(); - - assign slice_decode_if.valid = decode_if.valid && (decode_isw == issue_id); - assign slice_decode_if.data = decode_if.data; - assign decode_ready_in[issue_id] = slice_decode_if.ready; - - for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_ibuf_pop - assign decode_if.ibuf_pop[wis_to_wid(ISSUE_WIS_W'(w), ISSUE_ISW_W'(issue_id))] = slice_decode_if.ibuf_pop[w]; - end VX_issue_slice #( - .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, issue_id))), - .ISSUE_ID (issue_id) + .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, issue_id))) ) issue_slice ( `SCOPE_IO_BIND(issue_id) .clk (clk), .reset (reset), + .slice_id (ISSUE_ISW_W'(issue_id)), `ifdef PERF_ENABLE .issue_perf (per_issue_perf[issue_id]), `endif - .decode_if (slice_decode_if), + .decode_if (decode_if[issue_id]), .writeback_if (writeback_if[issue_id]), .dispatch_if (per_issue_dispatch_if), .warp_issued (issued_warps[issue_id]), diff --git a/hw/rtl/core/VX_issue_slice.sv b/hw/rtl/core/VX_issue_slice.sv index b827b1e0ee..4f147c68e2 100644 --- a/hw/rtl/core/VX_issue_slice.sv +++ b/hw/rtl/core/VX_issue_slice.sv @@ -14,14 +14,15 @@ `include "VX_define.vh" module VX_issue_slice import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [ISSUE_ISW_W-1:0] slice_id, + `ifdef PERF_ENABLE output issue_perf_t issue_perf, `endif @@ -39,8 +40,7 @@ module VX_issue_slice import VX_gpu_pkg::*; #( wire [NUM_EX_UNITS-1:0] fu_release; VX_ibuffer #( - .INSTANCE_ID (`SFORMATF(("%s-ibuffer", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-ibuffer", INSTANCE_ID))) ) ibuffer ( .clk (clk), .reset (reset), @@ -52,8 +52,7 @@ module VX_issue_slice import VX_gpu_pkg::*; #( ); VX_scoreboard #( - .INSTANCE_ID (`SFORMATF(("%s-scoreboard", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-scoreboard", INSTANCE_ID))) ) scoreboard ( .clk (clk), .reset (reset), @@ -67,8 +66,7 @@ module VX_issue_slice import VX_gpu_pkg::*; #( ); VX_operands #( - .INSTANCE_ID (`SFORMATF(("%s-operands", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-operands", INSTANCE_ID))) ) operands ( .clk (clk), .reset (reset), @@ -81,11 +79,11 @@ module VX_issue_slice import VX_gpu_pkg::*; #( ); VX_dispatcher #( - .INSTANCE_ID (`SFORMATF(("%s-dispatcher", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-dispatcher", INSTANCE_ID))) ) dispatcher ( .clk (clk), .reset (reset), + .slice_id (slice_id), `ifdef PERF_ENABLE .perf_stalls (issue_perf.dispatch_stalls), .perf_instrs (issue_perf.dispatch_instrs), diff --git a/hw/rtl/core/VX_lane_dispatch.sv b/hw/rtl/core/VX_lane_dispatch.sv index b6ae6126a1..5ff16da692 100644 --- a/hw/rtl/core/VX_lane_dispatch.sv +++ b/hw/rtl/core/VX_lane_dispatch.sv @@ -17,19 +17,25 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( parameter BLOCK_SIZE = 1, parameter NUM_LANES = 1, parameter OUT_BUF = 0, - parameter MAX_FANOUT = `MAX_FANOUT + parameter MAX_FANOUT = `MAX_FANOUT, + // dispatch ports served, starting at issue slice port_base + parameter NUM_PORTS = `VX_CFG_ISSUE_WIDTH ) ( input wire clk, input wire reset, + input wire [ISSUE_ISW_W-1:0] port_base, + // inputs - VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], + VX_dispatch_if.slave dispatch_if [NUM_PORTS], // outputs VX_execute_if.master execute_if [BLOCK_SIZE] ); - `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_ISSUE_WIDTH, BLOCK_SIZE), ("invalid parameter")) + `STATIC_ASSERT (`IS_DIVISBLE(NUM_PORTS, BLOCK_SIZE), ("invalid parameter")) `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_SIMD_WIDTH, NUM_LANES), ("invalid parameter")) + // a batched dispatch derives the issue slot from the batch index alone + `STATIC_ASSERT ((NUM_PORTS == `VX_CFG_ISSUE_WIDTH) || (BLOCK_SIZE == NUM_PORTS), ("invalid parameter")) `DECL_EXECUTE_T (pe, NUM_LANES); localparam IN_DATAW = $bits(dispatch_t); @@ -41,9 +47,9 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( localparam LPID_WIDTH = `UP(LPID_BITS); localparam GPID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); localparam GPID_WIDTH = `UP(GPID_BITS); - localparam BATCH_COUNT = `VX_CFG_ISSUE_WIDTH / BLOCK_SIZE; + localparam BATCH_COUNT = NUM_PORTS / BLOCK_SIZE; localparam BATCH_COUNT_W= `LOG2UP(BATCH_COUNT); - localparam ISSUE_W = `LOG2UP(`VX_CFG_ISSUE_WIDTH); + localparam ISSUE_W = `LOG2UP(NUM_PORTS); localparam FANOUT_ENABLE= (`VX_CFG_SIMD_WIDTH > (MAX_FANOUT + MAX_FANOUT /2)); localparam DATA_IN_TMASK_OFF = IN_DATAW - (UUID_WIDTH + ISSUE_WIS_W + NCTA_WIDTH + SIMD_IDX_W + `VX_CFG_SIMD_WIDTH); @@ -54,11 +60,11 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( logic [NUM_LANES-1:0] tmask; } packet_t; - wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_valid; - wire [`VX_CFG_ISSUE_WIDTH-1:0][IN_DATAW-1:0] dispatch_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_ready; + wire [NUM_PORTS-1:0] dispatch_valid; + wire [NUM_PORTS-1:0][IN_DATAW-1:0] dispatch_data; + wire [NUM_PORTS-1:0] dispatch_ready; - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_dispatch_data + for (genvar i = 0; i < NUM_PORTS; ++i) begin : g_dispatch_data assign dispatch_valid[i] = dispatch_if[i].valid; assign dispatch_data[i] = dispatch_if[i].data; assign dispatch_if[i].ready = dispatch_ready[i]; @@ -110,6 +116,10 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( `UNUSED_VAR (batch_done) end + if (BATCH_COUNT != 1) begin : g_no_port_base + `UNUSED_VAR (port_base) + end + wire [BLOCK_SIZE-1:0][ISSUE_W-1:0] issue_indices; for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_issue_indices assign issue_indices[block_idx] = ISSUE_W'(batch_idx * BLOCK_SIZE) + ISSUE_W'(block_idx); @@ -198,7 +208,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( assign isw = batch_idx; end end else begin : g_isw - assign isw = block_idx; + assign isw = port_base + ISSUE_ISW_W'(block_idx); end wire [NW_WIDTH-1:0] block_wid = wis_to_wid(dispatch_wis, isw); @@ -236,7 +246,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( end // release the dispatch interface when all packets are sent - reg [`VX_CFG_ISSUE_WIDTH-1:0] ready_in; + reg [NUM_PORTS-1:0] ready_in; always @(*) begin ready_in = 0; for (integer block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin diff --git a/hw/rtl/core/VX_lane_gather.sv b/hw/rtl/core/VX_lane_gather.sv index 0803f31bc8..905049c5e5 100644 --- a/hw/rtl/core/VX_lane_gather.sv +++ b/hw/rtl/core/VX_lane_gather.sv @@ -16,7 +16,9 @@ module VX_lane_gather import VX_gpu_pkg::*; #( parameter BLOCK_SIZE = 1, parameter NUM_LANES = 1, - parameter OUT_BUF = 0 + parameter OUT_BUF = 0, + // commit ports served + parameter NUM_PORTS = `VX_CFG_ISSUE_WIDTH ) ( input wire clk, input wire reset, @@ -25,9 +27,9 @@ module VX_lane_gather import VX_gpu_pkg::*; #( VX_result_if.slave result_if [BLOCK_SIZE], // outputs - VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH] + VX_commit_if.master commit_if [NUM_PORTS] ); - `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_ISSUE_WIDTH, BLOCK_SIZE), ("invalid parameter")) + `STATIC_ASSERT (`IS_DIVISBLE(NUM_PORTS, BLOCK_SIZE), ("invalid parameter")) `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_SIMD_WIDTH, NUM_LANES), ("invalid parameter")) `DECL_EXECUTE_T (pe, NUM_LANES); @@ -38,34 +40,38 @@ module VX_lane_gather import VX_gpu_pkg::*; #( localparam LPID_WIDTH = `UP(LPID_BITS); localparam DATAW = $bits(pe_result_t); localparam DATA_WIS_OFF = DATAW - (UUID_WIDTH + NW_WIDTH); + localparam PORT_W = `LOG2UP(NUM_PORTS); + + // a partial gather recovers the issue slice from the warp id + `STATIC_ASSERT ((BLOCK_SIZE == NUM_PORTS) || (NUM_PORTS == `VX_CFG_ISSUE_WIDTH), ("invalid parameter")) wire [BLOCK_SIZE-1:0] result_in_valid; wire [BLOCK_SIZE-1:0][DATAW-1:0] result_in_data; wire [BLOCK_SIZE-1:0] result_in_ready; - wire [BLOCK_SIZE-1:0][ISSUE_ISW_W-1:0] result_in_isw; + wire [BLOCK_SIZE-1:0][PORT_W-1:0] result_in_isw; for (genvar i = 0; i < BLOCK_SIZE; ++i) begin : g_commit_in assign result_in_valid[i] = result_if[i].valid; assign result_in_data[i] = result_if[i].data; assign result_if[i].ready = result_in_ready[i]; - if (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) begin : g_result_in_isw_partial + if (BLOCK_SIZE != NUM_PORTS) begin : g_result_in_isw_partial if (BLOCK_SIZE != 1) begin : g_block - assign result_in_isw[i] = {result_in_data[i][DATA_WIS_OFF+BLOCK_SIZE_W +: (ISSUE_ISW_W-BLOCK_SIZE_W)], BLOCK_SIZE_W'(i)}; + assign result_in_isw[i] = PORT_W'({result_in_data[i][DATA_WIS_OFF+BLOCK_SIZE_W +: (ISSUE_ISW_W-BLOCK_SIZE_W)], BLOCK_SIZE_W'(i)}); end else begin : g_no_block - assign result_in_isw[i] = result_in_data[i][DATA_WIS_OFF +: ISSUE_ISW_W]; + assign result_in_isw[i] = PORT_W'(result_in_data[i][DATA_WIS_OFF +: ISSUE_ISW_W]); end end else begin : g_result_in_isw_full - assign result_in_isw[i] = BLOCK_SIZE_W'(i); + assign result_in_isw[i] = PORT_W'(i); end end - reg [`VX_CFG_ISSUE_WIDTH-1:0] result_out_valid; - reg [`VX_CFG_ISSUE_WIDTH-1:0][DATAW-1:0] result_out_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0] result_out_ready; + reg [NUM_PORTS-1:0] result_out_valid; + reg [NUM_PORTS-1:0][DATAW-1:0] result_out_data; + wire [NUM_PORTS-1:0] result_out_ready; always @(*) begin result_out_valid = '0; - for (integer i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin + for (integer i = 0; i < NUM_PORTS; ++i) begin result_out_data[i] = 'x; end for (integer i = 0; i < BLOCK_SIZE; ++i) begin @@ -78,7 +84,7 @@ module VX_lane_gather import VX_gpu_pkg::*; #( assign result_in_ready[i] = result_out_ready[result_in_isw[i]]; end - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin: g_out_bufs + for (genvar i = 0; i < NUM_PORTS; ++i) begin: g_out_bufs VX_result_if #( .data_t (pe_result_t) ) result_tmp_if(); diff --git a/hw/rtl/core/VX_lsu_agu.sv b/hw/rtl/core/VX_lsu_agu.sv index a858877ee9..c9dde01271 100644 --- a/hw/rtl/core/VX_lsu_agu.sv +++ b/hw/rtl/core/VX_lsu_agu.sv @@ -14,7 +14,7 @@ `include "VX_define.vh" // VX_lsu_agu: per-lane LSU address generator. Owns all LSU address forms -// so VX_lsu_slice contains no address arithmetic: +// so VX_lsu_block contains no address arithmetic: // - plain load/store/fence: addr = base + sext(offset) // - pack-load uop: addr = base + uop_idx * stride // (uop_idx = offset[1:0], stride = rs2; a 2-bit shift-and-add diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_block.sv similarity index 98% rename from hw/rtl/core/VX_lsu_slice.sv rename to hw/rtl/core/VX_lsu_block.sv index 066df5073e..f127d9f260 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -13,15 +13,16 @@ `include "VX_define.vh" -module VX_lsu_slice import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 +module VX_lsu_block import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // Inputs VX_execute_if.slave execute_if, @@ -47,7 +48,9 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( localparam [MEM_ADDRW-1:0] LMEM_BLK_BASE = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> MEM_ASHIFT); `STATIC_ASSERT (MEM_ADDRW > LMEM_BLK_LSB, ("LMEM window wider than the block address space")) `endif - `UNUSED_PARAM (CORE_ID) +`ifndef VX_CFG_EXT_A_ENABLE + `UNUSED_VAR (core_id) +`endif `UNUSED_SPARAM (INSTANCE_ID) // tag_width = header + op_type + align + pkt_addr + fence @@ -141,7 +144,7 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( `endif // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid. assign lane_amo.hart_id = HART_ID_WIDTH'( - (HART_ID_WIDTH'(CORE_ID) << (NW_BITS + NT_BITS)) + (HART_ID_WIDTH'(core_id) << (NW_BITS + NT_BITS)) | (HART_ID_WIDTH'(execute_if.data.header.wid) << NT_BITS) | HART_ID_WIDTH'(i) ); diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index 67e5d22eea..2623bc8162 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -14,14 +14,15 @@ `include "VX_define.vh" module VX_lsu_unit import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // Inputs VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], @@ -47,6 +48,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); @@ -56,13 +58,13 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( ) per_block_result_if[BLOCK_SIZE](); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - VX_lsu_slice #( - .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, block_idx))), - .CORE_ID (CORE_ID) - ) lsu_slice( + VX_lsu_block #( + .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, block_idx))) + ) lsu_block( `SCOPE_IO_BIND (block_idx) .clk (clk), .reset (reset), + .core_id (core_id), .execute_if (per_block_execute_if[block_idx]), .result_if (per_block_result_if[block_idx]), .client_if (per_block_client_if[block_idx]) diff --git a/hw/rtl/core/VX_operands.sv b/hw/rtl/core/VX_operands.sv index 56e1ba7723..2c1f11b9bc 100644 --- a/hw/rtl/core/VX_operands.sv +++ b/hw/rtl/core/VX_operands.sv @@ -14,8 +14,7 @@ `include "VX_define.vh" module VX_operands import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, @@ -28,7 +27,6 @@ module VX_operands import VX_gpu_pkg::*; #( VX_scoreboard_if.slave scoreboard_if, VX_operands_if.master operands_if ); - `UNUSED_SPARAM (ISSUE_ID) localparam OUT_DATAW = $bits(operands_t); diff --git a/hw/rtl/core/VX_scheduler.sv b/hw/rtl/core/VX_scheduler.sv index 66e02120de..c2d36fda22 100644 --- a/hw/rtl/core/VX_scheduler.sv +++ b/hw/rtl/core/VX_scheduler.sv @@ -13,559 +13,261 @@ `include "VX_define.vh" +// The core-level part of scheduling: CTA dispatch and the core-wide warp-control +// resources. Each fetch slice schedules and fetches its own warps (VX_warp_sched, +// VX_fetch); core-wide events are handed to it with slice-local warp ids. module VX_scheduler import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + `ifdef PERF_ENABLE output sched_perf_t sched_perf, `endif - // inputs - VX_warp_ctl_if.slave warp_ctl_if, -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if.slave sched_unlock_if, // RTU TRACE wstall release + VX_slice_sched_if.master slice_if [`VX_CFG_FETCH_WIDTH], + VX_warp_shared_if.slave warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_txbar_bus_if.slave txbar_bus_if, `endif - VX_branch_ctl_if.slave branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], - VX_decode_sched_if.slave decode_sched_if, - VX_issue_sched_if.slave issue_sched_if [`VX_CFG_ISSUE_WIDTH], - VX_commit_sched_if.slave commit_sched_if, + input wire [`VX_CFG_NUM_WARPS-1:0] committed_warps, + + // CTA context read by each SFU block's CSR unit, valid a cycle after the address + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id, + output cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, - // KMU bus VX_kmu_bus_if.slave kmu_bus_if, - // outputs - VX_schedule_if.master schedule_if, - VX_sched_csr_if.master sched_csr_if, VX_gbar_bus_if.master gbar_bus_if, - // status + // per-core CSR state + output wire [PERF_CTR_BITS-1:0] cycles, + output wire [PERF_CTR_BITS-1:0] instret, + output wire [`VX_CFG_NUM_WARPS-1:0] active_warps, +`ifdef VX_CFG_VM_ENABLE + output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid, + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data, +`endif + output wire busy ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) - - reg [`VX_CFG_NUM_WARPS-1:0] active_warps, active_warps_n; // updated when a warp is activated or disabled - reg [`VX_CFG_NUM_WARPS-1:0] stalled_warps, stalled_warps_n; // set when branch/gpgpu instructions are issued - - reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_NUM_THREADS-1:0] thread_masks, thread_masks_n; - reg [`VX_CFG_NUM_WARPS-1:0][PC_BITS-1:0] warp_pcs, warp_pcs_n; - reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch_r; - - // Per-warp machine-mode trap CSRs. csrw writes arrive on - // sched_csr_if.trap_csr_wr_*; ECALL/EBREAK hardware-write mepc/mcause/ - // mtval; MRET restores the warp PC from mepc. - reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_XLEN-1:0] mstatus_r, mtvec_r, mepc_r, mcause_r, mtval_r; - wire [NW_WIDTH-1:0] schedule_wid; - wire [`VX_CFG_NUM_THREADS-1:0] schedule_tmask; - wire [PC_BITS-1:0] schedule_pc; - wire schedule_valid; - wire schedule_ready; + localparam FW = `VX_CFG_FETCH_WIDTH; + localparam SW = PER_FETCH_WARPS; + + // slice status in wid order (wid = {wfs, fsw}) + wire [`VX_CFG_NUM_WARPS-1:0] stalled_warps; + wire [`VX_CFG_NUM_WARPS-1:0] ibuf_full_n, pending_warp_empty; + for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_interleave + assign active_warps[w] = slice_if[w % FW].active[w / FW]; + assign stalled_warps[w] = slice_if[w % FW].stalled[w / FW]; + assign ibuf_full_n[w] = slice_if[w % FW].ibuf_full_n[w / FW]; + assign pending_warp_empty[w] = slice_if[w % FW].pending_empty[w / FW]; + end - // CTA dispatcher - wire cta_fire; - wire [NW_WIDTH-1:0] cta_wid; + // CTA dispatcher: lane p launches fetch slice p's warps + wire [FW-1:0] cta_fire; + wire [FW-1:0][NW_WIDTH-1:0] cta_wid; wire [PC_BITS-1:0] cta_PC; - wire [`VX_CFG_NUM_THREADS-1:0] cta_tmask; + wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] cta_tmask; wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param; + wire [FW-1:0] cta_init; + cta_bcast_t cta_bcast; + cta_lane_bcast_t [FW-1:0] cta_lane_bcast; wire cta_dispatcher_busy; - wire cta_init; - - // CTA-CSR read-back from the dispatcher (it owns the per-CTA/per-warp tables). - cta_csrs_t cta_rd_csrs; - cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_rd_lane; - wire [NCTA_WIDTH-1:0] schedule_cta_id; - - // Warp retirement: TMC with tmask==0 permanently deactivates the warp - wire cta_warp_done = warp_ctl_if.tmc_valid && (warp_ctl_if.tmc.tmask == 0); + wire cta_warp_done; + wire [NW_WIDTH-1:0] cta_warp_done_wid; VX_cta_dispatch #( - .INSTANCE_ID (`SFORMATF(("%s-cta_dispatch", INSTANCE_ID))) + .INSTANCE_ID (`SFORMATF(("%s-cta_dispatch", INSTANCE_ID))), + .NUM_LANES (FW) ) cta_dispatcher ( - .clk (clk), - .reset (reset), - .kmu_bus_if (kmu_bus_if), - .active_warps(active_warps), - .warp_done (cta_warp_done), - .warp_done_wid(warp_ctl_if.wid), - .cta_fire (cta_fire), - .cta_wid (cta_wid), - .cta_PC (cta_PC), - .cta_tmask (cta_tmask), - .cta_param (cta_param), - .cta_init (cta_init), - .csr_rd_wid (sched_csr_if.csr_rd_wid), - .csr_rd_cta_id(sched_csr_if.csr_rd_cta_id), - .cta_rd_csrs(cta_rd_csrs), - .cta_rd_lane(cta_rd_lane), - .schedule_wid(schedule_wid), - .schedule_cta_id(schedule_cta_id), - .busy (cta_dispatcher_busy) + .clk (clk), + .reset (reset), + .kmu_bus_if (kmu_bus_if), + .active_warps (active_warps), + .warp_done (cta_warp_done), + .warp_done_wid(cta_warp_done_wid), + .cta_fire (cta_fire), + .cta_wid (cta_wid), + .cta_PC (cta_PC), + .cta_tmask (cta_tmask), + .cta_param (cta_param), + .cta_init (cta_init), + .cta_bcast (cta_bcast), + .cta_lane_bcast(cta_lane_bcast), + .busy (cta_dispatcher_busy) ); - assign sched_csr_if.cta_csrs = cta_rd_csrs; - assign sched_csr_if.cta_lane = cta_rd_lane; - - assign sched_csr_if.mscratch = mscratch_r[sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mstatus = mstatus_r[sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mtvec = mtvec_r [sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mepc = mepc_r [sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mcause = mcause_r [sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mtval = mtval_r [sched_csr_if.csr_rd_wid]; - - // split/join - wire join_valid; - wire join_is_dvg; - wire join_is_else; - wire [NW_WIDTH-1:0] join_wid; - wire [`VX_CFG_NUM_THREADS-1:0] join_tmask; - wire [PC_BITS-1:0] join_pc; - - reg [PERF_CTR_BITS-1:0] cycles; - - wire schedule_fire = schedule_valid && schedule_ready; - wire schedule_if_fire = schedule_if.valid && schedule_if.ready; -`ifdef VX_CFG_EXT_C_ENABLE - // PC advance is driven by decompress_finished under EXT_C; - `UNUSED_VAR (schedule_if_fire) -`endif - - // branch - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_valid; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0][NW_WIDTH-1:0] branch_wid; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_taken; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0][PC_BITS-1:0] branch_dest; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_is_trap; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_is_mret; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0][3:0] branch_trap_cause; - for (genvar i = 0; i < `VX_CFG_NUM_ALU_BLOCKS; ++i) begin : g_branch_init - assign branch_valid[i] = branch_ctl_if[i].valid; - assign branch_wid[i] = branch_ctl_if[i].wid; - assign branch_taken[i] = branch_ctl_if[i].taken; - assign branch_dest[i] = branch_ctl_if[i].dest; - assign branch_is_trap[i] = branch_ctl_if[i].is_trap; - assign branch_is_mret[i] = branch_ctl_if[i].is_mret; - assign branch_trap_cause[i] = branch_ctl_if[i].trap_cause; + // CTA contexts, one read port per SFU block + for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_cta_ctx + VX_dp_ram #( + .DATAW ($bits(cta_ctx_t)), + .SIZE (NUM_CTA_MAX), + .RDW_MODE ("R"), + .OUT_REG (1) + ) cta_ctx_ram ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (cta_bcast.ctx_write), + .wren (1'b1), + .waddr (cta_bcast.ctx_waddr), + .wdata (cta_bcast.ctx_wdata), + .raddr (csr_rd_cta_id[b]), + .rdata (csr_rd_ctx[b]) + ); end - // barriers - wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask; + // core-wide warp-control resources wire bar_unlock_valid; - - // wspawn - wspawn_t wspawn; - reg wspawn_valid; - reg [NW_WIDTH-1:0] wspawn_wid; - reg is_single_warp; - - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] active_warps_cnt; - `POP_COUNT(active_warps_cnt, active_warps); - - always @(*) begin - active_warps_n = active_warps; - stalled_warps_n = stalled_warps; - thread_masks_n = thread_masks; - warp_pcs_n = warp_pcs; - - // dispatch warps - if (cta_fire) begin - active_warps_n[cta_wid] = 1; - // Reusing a warp for the next CTA skips the one-time prologue and rewinds to the - // kernel's per-CTA dispatch window: a fixed 20-byte (5-instruction) sequence that - // reloads the entry pointer and kargs before re-calling. - warp_pcs_n[cta_wid] = cta_init ? cta_PC : (warp_pcs[cta_wid] - from_fullPC(`VX_CFG_XLEN'(20))); - thread_masks_n[cta_wid] = cta_tmask; - end - - // decode unlock - if (decode_sched_if.valid && decode_sched_if.unlock) begin - stalled_warps_n[decode_sched_if.wid] = 0; - end - - // wspawn handling - if (wspawn_valid && is_single_warp) begin - active_warps_n |= wspawn.wmask; - for (integer i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin - if (wspawn.wmask[i] && (NW_WIDTH'(i) != wspawn_wid)) begin - thread_masks_n[i][0] = 1; - warp_pcs_n[i] = wspawn.pc; - end - end - stalled_warps_n[wspawn_wid] = 0; // unlock warp - end - - // TMC handling - if (warp_ctl_if.tmc_valid) begin - active_warps_n[warp_ctl_if.wid] = (warp_ctl_if.tmc.tmask != 0); - thread_masks_n[warp_ctl_if.wid] = warp_ctl_if.tmc.tmask; - stalled_warps_n[warp_ctl_if.wid] = 0; // unlock warp - end - - // split handling - if (warp_ctl_if.split_valid) begin - if (warp_ctl_if.split.is_dvg) begin - thread_masks_n[warp_ctl_if.wid] = warp_ctl_if.split.then_tmask; - end - stalled_warps_n[warp_ctl_if.wid] = 0; // unlock warp - end - - // join handling - if (join_valid) begin - if (join_is_dvg) begin - if (join_is_else) begin - warp_pcs_n[join_wid] = join_pc; - end - thread_masks_n[join_wid] = join_tmask; - end - stalled_warps_n[join_wid] = 0; // unlock warp - end - - // barrier unlock handling - if (bar_unlock_valid) begin - stalled_warps_n &= ~bar_unlock_mask; - end - - // wsync unlock: warp pipeline drained - if (warp_ctl_if.wsync_valid) begin - stalled_warps_n[warp_ctl_if.wid] = 0; - end - - // Branch handling - for (integer i = 0; i < `VX_CFG_NUM_ALU_BLOCKS; ++i) begin - if (branch_valid[i]) begin - if (branch_is_trap[i]) begin - // ECALL/EBREAK: redirect to trap vector (mtvec[1:0] = MODE field; mask off to get base address). - warp_pcs_n[branch_wid[i]] = from_fullPC(mtvec_r[branch_wid[i]] & ~`VX_CFG_XLEN'(3)); - end else if (branch_is_mret[i]) begin - // MRET/SRET/URET: restore the saved PC from mepc. ECALL/EBREAK - // are the only traps and they do not narrow the tmask, so there - // is nothing to restore beyond the PC. - warp_pcs_n[branch_wid[i]] = from_fullPC(mepc_r[branch_wid[i]]); - end else if (branch_taken[i]) begin - warp_pcs_n[branch_wid[i]] = branch_dest[i]; - end - stalled_warps_n[branch_wid[i]] = 0; // unlock warp - end - end - - // stall the warp until decode stage - if (schedule_fire) begin - stalled_warps_n[schedule_wid] = 1; - end - - // advance PC. - `ifdef VX_CFG_EXT_C_ENABLE - // With RVC, the decompressor may emit a 2-byte instruction. Advance - // from the committed warp PC rather than the redirect-muxed next-PC: - // every redirect source (branch/trap/mret, split/join, wspawn) stalls - // its warp from schedule until it resolves, so no redirect writes a - // warp's PC on the same cycle that warp decode-advances. Reading the - // registered PC keeps the +2/+4 adder off the branch/trap redirect - // cone, matching the non-RVC path which advances the pipeline-carried - // PC instead of the combinational next-PC. - if (decode_sched_if.valid) begin - warp_pcs_n[decode_sched_if.wid] = - warp_pcs[decode_sched_if.wid] - + from_fullPC(decode_sched_if.is_rvc ? `VX_CFG_XLEN'(2) : `VX_CFG_XLEN'(4)); - end - `else - if (schedule_if_fire) begin - warp_pcs_n[schedule_if.data.wid] = schedule_if.data.PC + from_fullPC(`VX_CFG_XLEN'(4)); - end + wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask; + wire wspawn_apply; + wire [`VX_CFG_NUM_WARPS-1:0] wspawn_wmask; + wire [PC_BITS-1:0] wspawn_pc; + wire [NW_WIDTH-1:0] wspawn_wid; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch; + + VX_warp_shared_unit #( + .INSTANCE_ID (INSTANCE_ID), + .NUM_REQS (`VX_CFG_NUM_SFU_BLOCKS) + ) warp_shared_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .warp_shared_if (warp_shared_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .txbar_bus_if (txbar_bus_if), `endif + .active_warps (active_warps), + .bar_unlock_valid(bar_unlock_valid), + .bar_unlock_mask (bar_unlock_mask), + .gbar_bus_if (gbar_bus_if), + .wspawn_apply (wspawn_apply), + .wspawn_wmask (wspawn_wmask), + .wspawn_pc (wspawn_pc), + .wspawn_wid (wspawn_wid), + .wspawn_mscratch (wspawn_mscratch), + .warp_done (cta_warp_done), + .warp_done_wid (cta_warp_done_wid), + `UNUSED_PIN (wspawn_pending) + ); - `ifdef VX_CFG_EXT_RTU_ENABLE - // A wstall'd TRACE retires (its traversal's first response landed and the - // arm op wrote back the handle): resume the warp so it proceeds to WAIT, - // which returns the response status (terminal or candidate). No trap, no - // redirect — the candidate is serviced inline by the warp's loop. - if (sched_unlock_if.valid) begin - stalled_warps_n[sched_unlock_if.wid] = 1'b0; - end - `endif - end + // counters + reg [PERF_CTR_BITS-1:0] cycles_r; + reg [PERF_CTR_BITS-1:0] instret_r; + wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] committed_warps_cnt; + `POP_COUNT(committed_warps_cnt, committed_warps); always @(posedge clk) begin if (reset) begin - stalled_warps <= '0; - warp_pcs <= '0; - active_warps <= '0; - thread_masks <= '0; - cycles <= '0; - wspawn_valid <= 0; - warp_pcs <= '0; - active_warps <= '0; - thread_masks <= '0; - is_single_warp <= 0; - mscratch_r <= '0; - mstatus_r <= '0; - mtvec_r <= '0; - mepc_r <= '0; - mcause_r <= '0; - mtval_r <= '0; + cycles_r <= '0; + instret_r <= '0; end else begin - active_warps <= active_warps_n; - stalled_warps <= stalled_warps_n; - thread_masks <= thread_masks_n; - warp_pcs <= warp_pcs_n; - is_single_warp <= (active_warps_cnt == $bits(active_warps_cnt)'(1)); - - // wspawn handling - if (warp_ctl_if.wspawn_valid) begin - wspawn_valid <= 1; - wspawn.wmask <= warp_ctl_if.wspawn.wmask; - wspawn.pc <= warp_ctl_if.wspawn.pc; - wspawn_wid <= warp_ctl_if.wid; - end - if (wspawn_valid && is_single_warp) begin - wspawn_valid <= 0; - // copy mscratch from spawning warp to all newly spawned warps - for (integer i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin - if (wspawn.wmask[i] && (NW_WIDTH'(i) != wspawn_wid)) begin - mscratch_r[i] <= mscratch_r[wspawn_wid]; - end - end - end - - // CTA dispatch: latch this warp's mscratch (param). The per-CTA / - // per-warp tables and the wid->cta_id map live in VX_cta_dispatch. - if (cta_fire) begin - mscratch_r[cta_wid] <= cta_param; - end - - // MSCRATCH write-back from CSR unit (CSR instruction) - if (sched_csr_if.csr_wr_valid) begin - mscratch_r[sched_csr_if.csr_wr_wid] <= sched_csr_if.csr_wr_data; - end - - // Trap CSR write-back from CSR unit (csrw mstatus/mtvec/mepc/...) - if (sched_csr_if.trap_csr_wr_valid[0]) begin - mstatus_r[sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[1]) begin - mtvec_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[2]) begin - mepc_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[3]) begin - mcause_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[4]) begin - mtval_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - - // Hardware trap entry (ECALL/EBREAK): snapshot the faulting PC - // into mepc and the cause into mcause. Ordered after the - // software write so a hardware trap wins a same-cycle conflict. - for (integer i = 0; i < `VX_CFG_NUM_ALU_BLOCKS; ++i) begin - if (branch_valid[i] && branch_is_trap[i]) begin - mepc_r [branch_wid[i]] <= to_fullPC(branch_dest[i]); - mcause_r[branch_wid[i]] <= `VX_CFG_XLEN'(branch_trap_cause[i]); - mtval_r [branch_wid[i]] <= '0; - end - end - if (busy) begin - cycles <= cycles + 1; + cycles_r <= cycles_r + 1; end + instret_r <= instret_r + PERF_CTR_BITS'(committed_warps_cnt); end end + assign cycles = cycles_r; + assign instret = instret_r; - // Barrier unit - - VX_bar_unit #( - .INSTANCE_ID (`SFORMATF(("%s-barrier", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) bar_unit ( - .clk (clk), - .reset (reset), - .req_valid (warp_ctl_if.bar_valid), - .req_wid (warp_ctl_if.wid), - .req_data (warp_ctl_if.bar), - .read_addr (warp_ctl_if.bar_addr), - .read_phase (warp_ctl_if.bar_phase), - .active_warps(active_warps), - .gbar_bus_if(gbar_bus_if), - .unlock_valid(bar_unlock_valid), - .unlock_mask(bar_unlock_mask) - ); - - // split/join handling - - VX_split_join #( - .INSTANCE_ID (`SFORMATF(("%s-splitjoin", INSTANCE_ID))), - .OUT_REG (1) - ) split_join ( - .clk (clk), - .reset (reset), - .split_valid(warp_ctl_if.split_valid), - .sjoin_valid(warp_ctl_if.sjoin_valid), - .wid (warp_ctl_if.wid), - .split (warp_ctl_if.split), - .sjoin (warp_ctl_if.sjoin), - .join_valid (join_valid), - .join_is_dvg(join_is_dvg), - .join_is_else(join_is_else), - .join_wid (join_wid), - .join_tmask (join_tmask), - .join_pc (join_pc), - .stack_wid (warp_ctl_if.dvstack_wid), - .stack_ptr (warp_ctl_if.dvstack_ptr) - ); - - // schedule the next ready warp - - wire [`VX_CFG_NUM_WARPS-1:0] ready_warps = active_warps & ~stalled_warps; - - // Per-warp ibuffer occupancy counter (registered full[i] keeps arbitration - // off the critical path; full_n feeds an externally registered aggregate - // so all_full is valid the same cycle as full[i]). - localparam IBUF_CW = $clog2(`VX_CFG_IBUF_SIZE + 1); - - wire [`VX_CFG_NUM_WARPS-1:0] schedule_onehot; - logic [`VX_CFG_NUM_WARPS-1:0] ibuf_full, ibuf_full_n; - - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_ibuf_cnt - logic [IBUF_CW-1:0] size_r, size_n; - wire incr = schedule_fire && schedule_onehot[i]; - wire decr = schedule_if.ibuf_pop[i]; - assign size_n = size_r + IBUF_CW'(incr) - IBUF_CW'(decr); - assign ibuf_full_n[i] = (size_n == IBUF_CW'(`VX_CFG_IBUF_SIZE)); - always @(posedge clk) begin - if (reset) begin - size_r <= '0; - ibuf_full[i] <= 1'b0; - end else begin - size_r <= size_n; - ibuf_full[i] <= ibuf_full_n[i]; - end + // from the slices' next-state full, so it is valid the same cycle as their full flags + reg all_ibuf_full; + always @(posedge clk) begin + if (reset) begin + all_ibuf_full <= 1'b0; + end else begin + all_ibuf_full <= (& ibuf_full_n); end end - wire [`VX_CFG_NUM_WARPS-1:0] preferred_warps = ready_warps & ~ibuf_full; -`ifndef L1_ENABLE - // without L1, we should ensure the icache never stalls, - // because it could deadlock dcache response since they share the same bus. - wire [`VX_CFG_NUM_WARPS-1:0] schedule_warps = preferred_warps; -`else - reg all_ibuf_full; +`ifdef VX_CFG_VM_ENABLE + // Per-core SATP CSR, written by any SFU block's CSR unit. Accesses in one + // cycle take effect in block order: each block reads the value the lower + // blocks leave. + reg [`VX_CFG_XLEN-1:0] satp_r; + wire [`VX_CFG_NUM_SFU_BLOCKS:0][`VX_CFG_XLEN-1:0] satp_chain; + assign satp_chain[0] = satp_r; + for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_satp + assign satp[b] = satp_chain[b]; + assign satp_chain[b+1] = satp_wr_valid[b] ? satp_wr_data[b] : satp_chain[b]; + end always @(posedge clk) begin - if (reset) all_ibuf_full <= 1'b0; - else all_ibuf_full <= (& ibuf_full_n); + if (reset) begin + satp_r <= '0; + end else begin + satp_r <= satp_chain[`VX_CFG_NUM_SFU_BLOCKS]; + end end - wire [`VX_CFG_NUM_WARPS-1:0] schedule_warps = all_ibuf_full ? ready_warps : preferred_warps; `endif - VX_priority_encoder #( - .N (`VX_CFG_NUM_WARPS) - ) wid_select ( - .data_in (schedule_warps), - .index_out (schedule_wid), - .valid_out (schedule_valid), - .onehot_out(schedule_onehot) - ); + // per-slice view of the core-wide events - wire [`VX_CFG_NUM_WARPS-1:0][(`VX_CFG_NUM_THREADS + PC_BITS)-1:0] schedule_data; - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_schedule_data - assign schedule_data[i] = {thread_masks[i], warp_pcs[i]}; - end + wire [FETCH_FSW_W-1:0] wspawn_fsw = wid_to_fsw(wspawn_wid); - assign {schedule_tmask, schedule_pc} = { - schedule_data[schedule_wid][(`VX_CFG_NUM_THREADS + PC_BITS)-1:(`VX_CFG_NUM_THREADS + PC_BITS)-4], - schedule_data[schedule_wid][(`VX_CFG_NUM_THREADS + PC_BITS)-5:0] - }; - - wire [UUID_WIDTH-1:0] instr_uuid; -`ifdef UUID_ENABLE - VX_uuid_gen #( - .CORE_ID (CORE_ID) - ) uuid_gen ( - .clk (clk), - .reset (reset), - .incr (schedule_fire), - .wid (schedule_wid), - .uuid (instr_uuid) - ); -`else - assign instr_uuid = '0; -`endif + for (genvar p = 0; p < FW; ++p) begin : g_slices + localparam [FETCH_FSW_W-1:0] FSW = FETCH_FSW_W'(p); - // schedule_cta_id is produced by VX_cta_dispatch from its wid->cta_id map. - - VX_elastic_buffer #( - .DATAW (`VX_CFG_NUM_THREADS + PC_BITS + NW_WIDTH + NCTA_WIDTH + UUID_WIDTH), - .SIZE (2), // need to buffer out ready_in - .OUT_REG (1) // should be registered for BRAM acces in fetch unit - ) out_buf ( - .clk (clk), - .reset (reset), - .valid_in (schedule_valid), - .ready_in (schedule_ready), - .data_in ({schedule_tmask, schedule_pc, schedule_wid, schedule_cta_id, instr_uuid}), - .data_out ({schedule_if.data.tmask, schedule_if.data.PC, schedule_if.data.wid, schedule_if.data.cta_id, schedule_if.data.uuid}), - .valid_out (schedule_if.valid), - .ready_out (schedule_if.ready) - ); + assign slice_if[p].cta_fire = cta_fire[p]; + assign slice_if[p].cta_lid = wid_to_wfs(cta_wid[p]); + assign slice_if[p].cta_PC = cta_PC; + assign slice_if[p].cta_tmask = cta_tmask[p]; + assign slice_if[p].cta_param = cta_param; + assign slice_if[p].cta_init = cta_init[p]; - // Track committed instructions + assign slice_if[p].warp_write = cta_lane_bcast[p].warp_write; + assign slice_if[p].warp_waddr = wid_to_wfs(cta_lane_bcast[p].warp_waddr); + assign slice_if[p].warp_wdata = cta_lane_bcast[p].warp_wdata; + assign slice_if[p].id_write = cta_lane_bcast[p].id_write; + assign slice_if[p].id_waddr = wid_to_wfs(cta_lane_bcast[p].id_wid); + assign slice_if[p].id_wdata = cta_lane_bcast[p].id_cta; + `UNUSED_VAR ({wid_to_fsw(cta_wid[p]), wid_to_fsw(cta_lane_bcast[p].warp_waddr), wid_to_fsw(cta_lane_bcast[p].id_wid)}) - reg [PERF_CTR_BITS-1:0] instret; + assign slice_if[p].wspawn_apply = wspawn_apply; + assign slice_if[p].wspawn_pc = wspawn_pc; + assign slice_if[p].wspawn_src = (wspawn_fsw == FSW); + assign slice_if[p].wspawn_src_lid = wid_to_wfs(wspawn_wid); + assign slice_if[p].wspawn_mscratch = wspawn_mscratch; - wire [`VX_CFG_NUM_WARPS-1:0] committed_warps_v = commit_sched_if.committed_warps; - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] committed_warps_cnt_v; - `POP_COUNT(committed_warps_cnt_v, committed_warps_v); + assign slice_if[p].bar_unlock_valid = bar_unlock_valid; - always @(posedge clk) begin - if (reset) begin - instret <= '0; - end else begin - instret <= instret + PERF_CTR_BITS'(committed_warps_cnt_v); + assign slice_if[p].retire_valid = cta_warp_done && (wid_to_fsw(cta_warp_done_wid) == FSW); + assign slice_if[p].retire_lid = wid_to_wfs(cta_warp_done_wid); + + for (genvar l = 0; l < SW; ++l) begin : g_lw + assign slice_if[p].wspawn_wmask[l] = wspawn_wmask[l * FW + p]; + assign slice_if[p].bar_unlock_mask[l] = bar_unlock_mask[l * FW + p]; end + + assign slice_if[p].all_ibuf_full = all_ibuf_full; end - // Track pending instructions per warp - - wire [`VX_CFG_NUM_WARPS-1:0] pending_warp_empty; - wire [`VX_CFG_NUM_WARPS-1:0] pending_warp_alm_empty; - - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_pending_warps - localparam logic [ISSUE_ISW_W-1:0] isw = wid_to_isw(i); - localparam logic [ISSUE_WIS_W-1:0] wis = wid_to_wis(i); - - VX_pending_size #( - .SIZE (256), - .ALM_EMPTY (1) - ) per_warp_ctr ( - .clk (clk), - .reset (reset), - .incr (issue_sched_if[isw].valid && (issue_sched_if[isw].wis == ISSUE_WIS_W'(wis))), - .decr (commit_sched_if.committed_warps[i]), - .empty (pending_warp_empty[i]), - .alm_empty (pending_warp_alm_empty[i]), - `UNUSED_PIN (full), - `UNUSED_PIN (alm_full), - `UNUSED_PIN (size) - ); + wire [FW-1:0] sched_valid, sched_fire; + wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] sched_tmask; + for (genvar p = 0; p < FW; ++p) begin : g_sched_status + assign sched_valid[p] = slice_if[p].sched_valid; + assign sched_fire[p] = slice_if[p].sched_fire; + assign sched_tmask[p] = slice_if[p].sched_tmask; end +`ifndef PERF_ENABLE + `UNUSED_VAR ({sched_valid, sched_fire, sched_tmask}) +`endif + // active_warps is the slices' registered state, so a CTA launch shows there one cycle after + // cta_fire; counting the launch itself keeps busy from dropping between the dispatcher going + // idle and the launched warps becoming visible (a CTA launched in a single cycle). wire busy_buf; - `BUFFER_EX(busy_buf, (active_warps_n != 0 || ~(&pending_warp_empty)), 1'b1, 1, 1); + `BUFFER_EX(busy_buf, (active_warps != 0 || (| cta_fire) || ~(&pending_warp_empty)), 1'b1, 1, 1); assign busy = busy_buf || cta_dispatcher_busy; - assign warp_ctl_if.warp_pending_alm_empty = pending_warp_alm_empty; - - // export CSRs - assign sched_csr_if.cycles = cycles; - assign sched_csr_if.instret = instret; - assign sched_csr_if.active_warps = active_warps; - assign sched_csr_if.thread_masks = thread_masks; - - // timeout handling + // timeout handling reg [31:0] timeout_ctr; reg timeout_enable; always @(posedge clk) begin @@ -573,7 +275,7 @@ module VX_scheduler import VX_gpu_pkg::*; #( timeout_ctr <= '0; timeout_enable <= 0; end else begin - if (decode_sched_if.valid && decode_sched_if.unlock) begin + if (| cta_fire) begin timeout_enable <= 1; end if (timeout_enable && active_warps !=0 && active_warps == stalled_warps) begin @@ -590,15 +292,15 @@ module VX_scheduler import VX_gpu_pkg::*; #( localparam SCHED_STALL_TIMEOUT = STALL_TIMEOUT; `endif `ifdef EXT_SCHED_TIMEOUT_DUMP + wire timeout_dump = !reset && (timeout_ctr == (SCHED_STALL_TIMEOUT - 1)); always @(posedge clk) begin - if (!reset && (timeout_ctr == (SCHED_STALL_TIMEOUT - 1))) begin + if (timeout_dump) begin $display("*** %s scheduler-timeout dump: active=%b stalled=%b", INSTANCE_ID, active_warps, stalled_warps); - for (integer wi = 0; wi < `VX_CFG_NUM_WARPS; ++wi) begin - $display(" wid=%0d stalled=%0d pc=0x%0h tmask=%b", - wi, stalled_warps[wi], to_fullPC(warp_pcs[wi]), thread_masks[wi]); - end end end + for (genvar p = 0; p < FW; ++p) begin : g_timeout_dump + assign slice_if[p].timeout_dump = timeout_dump; + end `endif `RUNTIME_ASSERT(timeout_ctr < SCHED_STALL_TIMEOUT, ("*** %s timeout: active_warps=%b, stalled_warps=%b", INSTANCE_ID, active_warps, stalled_warps)) @@ -611,17 +313,34 @@ module VX_scheduler import VX_gpu_pkg::*; #( reg [PERF_CTR_BITS-1:0] perf_branches; reg [PERF_CTR_BITS-1:0] perf_divergence; - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] stalled_warps_cnt; - wire [`CLOG2(`VX_CFG_NUM_ALU_BLOCKS+1)-1:0] branches_cnt; - wire [`CLOG2(`VX_CFG_NUM_THREADS+1)-1:0] issued_threads_cnt; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; - wire schedule_idle = ~schedule_valid; - wire has_divergence = warp_ctl_if.split_valid && warp_ctl_if.split.is_dvg; - wire [`VX_CFG_NUM_THREADS-1:0] issued_threads = {`VX_CFG_NUM_THREADS{schedule_if_fire}} & schedule_if.data.tmask; + wire [FW-1:0][NPA-1:0] slice_branch; + wire [FW-1:0][NPS-1:0] slice_divergence; + for (genvar p = 0; p < FW; ++p) begin : g_perf_slices + assign slice_branch[p] = slice_if[p].branch_valid; + assign slice_divergence[p] = slice_if[p].divergence; + end + wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] active_warps_cnt, stalled_warps_cnt; + wire [`CLOG2(FW*NPA+1)-1:0] branches_cnt; + wire [`CLOG2(FW*NPS+1)-1:0] divergence_cnt; + wire [`CLOG2(FW+1)-1:0] issued_warps_cnt; + wire [`CLOG2(FW*`VX_CFG_NUM_THREADS+1)-1:0] issued_threads_cnt; + wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] issued_threads; + for (genvar p = 0; p < FW; ++p) begin : g_issued_threads + assign issued_threads[p] = {`VX_CFG_NUM_THREADS{sched_fire[p]}} & sched_tmask[p]; + end + + `POP_COUNT(active_warps_cnt, active_warps); `POP_COUNT(stalled_warps_cnt, stalled_warps); `POP_COUNT(issued_threads_cnt, issued_threads); - `POP_COUNT(branches_cnt, branch_valid); + `POP_COUNT(issued_warps_cnt, sched_fire); + `POP_COUNT(branches_cnt, slice_branch); + `POP_COUNT(divergence_cnt, slice_divergence); + + wire sched_idle = ~(| sched_valid); always @(posedge clk) begin if (reset) begin @@ -633,13 +352,13 @@ module VX_scheduler import VX_gpu_pkg::*; #( perf_branches <= '0; perf_divergence <= '0; end else begin - perf_sched_idles <= perf_sched_idles + PERF_CTR_BITS'(schedule_idle); + perf_sched_idles <= perf_sched_idles + PERF_CTR_BITS'(sched_idle); perf_active_warps <= perf_active_warps + PERF_CTR_BITS'(active_warps_cnt); perf_stalled_warps <= perf_stalled_warps + PERF_CTR_BITS'(stalled_warps_cnt); - perf_issued_warps <= perf_issued_warps + PERF_CTR_BITS'(schedule_if_fire); + perf_issued_warps <= perf_issued_warps + PERF_CTR_BITS'(issued_warps_cnt); perf_issued_threads<= perf_issued_threads + PERF_CTR_BITS'(issued_threads_cnt); perf_branches <= perf_branches + PERF_CTR_BITS'(branches_cnt); - perf_divergence <= perf_divergence + PERF_CTR_BITS'(has_divergence); + perf_divergence <= perf_divergence + PERF_CTR_BITS'(divergence_cnt); end end @@ -650,25 +369,9 @@ module VX_scheduler import VX_gpu_pkg::*; #( assign sched_perf.issued_threads= perf_issued_threads; assign sched_perf.branches = perf_branches; assign sched_perf.divergence = perf_divergence; -`endif - -`ifdef DBG_TRACE_PIPELINE - for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_trace_warp_status - always @(posedge clk) begin - if (active_warps_n[w] != active_warps[w] - || (active_warps[w] && (stalled_warps_n[w] != stalled_warps[w] - || thread_masks_n[w] != thread_masks[w]))) begin - `TRACE(1, ("%t: %s warp-state: wid=%0d, active=%b, stalled=%b, tmask=%b\n", - $time, INSTANCE_ID, w, active_warps_n[w], stalled_warps_n[w], thread_masks_n[w] - )) - end - end - end - - always @(posedge clk) begin - if (schedule_fire) begin - `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b (#%0d)\n", $time, INSTANCE_ID, schedule_wid, schedule_cta_id, to_fullPC(schedule_pc), schedule_tmask, instr_uuid)) - end +`else + for (genvar p = 0; p < FW; ++p) begin : g_perf_unused + `UNUSED_VAR ({slice_if[p].branch_valid, slice_if[p].divergence}) end `endif diff --git a/hw/rtl/core/VX_scoreboard.sv b/hw/rtl/core/VX_scoreboard.sv index 54f9f1e6e8..922ef621d9 100644 --- a/hw/rtl/core/VX_scoreboard.sv +++ b/hw/rtl/core/VX_scoreboard.sv @@ -14,8 +14,7 @@ `include "VX_define.vh" module VX_scoreboard import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, @@ -30,7 +29,6 @@ module VX_scoreboard import VX_gpu_pkg::*; #( VX_scoreboard_if.master scoreboard_if ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (ISSUE_ID) `UNUSED_VAR ({writeback_if.data.sop, writeback_if.data.eop, writeback_if.data.wis}) localparam NUM_OPDS = NUM_SRC_OPDS + 1; diff --git a/hw/rtl/core/VX_sfu_block.sv b/hw/rtl/core/VX_sfu_block.sv new file mode 100644 index 0000000000..15ce7ad91d --- /dev/null +++ b/hw/rtl/core/VX_sfu_block.sv @@ -0,0 +1,231 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + +// One SFU block: the PEs serving the warps with wid mod NUM_SFU_BLOCKS == block_idx. +module VX_sfu_block import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [SFU_BLOCK_W-1:0] block_idx, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if, + +`ifdef VX_CFG_EXT_F_ENABLE + output wire [NW_WIDTH-1:0] fcsr_wid, + input wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data, + output wire [2:0] fcsr_wr_sel, + output wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data, +`endif + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, + VX_sched_unlock_if.master sched_unlock_if, +`endif + + VX_sched_csr_if.slave sched_csr_if, + VX_warp_shared_if.master warp_shared_if, + VX_warp_ctl_if.master warp_ctl_if +); + localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; +`ifndef VX_CFG_EXT_RTU_ENABLE + `UNUSED_VAR (block_idx) +`endif + localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; + localparam PE_SEL_BITS = `CLOG2(PE_COUNT); + localparam PE_IDX_WCTL = 0; + localparam PE_IDX_CSRS = 1; +`ifdef VX_CFG_EXT_DXA_ENABLE + localparam PE_IDX_DXA = 2; +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + localparam PE_IDX_TEX = 2 + `VX_CFG_EXT_DXA_ENABLED; +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + localparam PE_IDX_OM = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED; +`endif +`ifdef VX_CFG_EXT_RASTER_ENABLE + localparam PE_IDX_RASTER = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED; +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + localparam PE_IDX_RTUW = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED; +`endif + + VX_execute_if #( + .data_t (sfu_execute_t) + ) pe_execute_if[PE_COUNT](); + + VX_result_if#( + .data_t (sfu_result_t) + ) pe_result_if[PE_COUNT](); + + reg [PE_SEL_BITS-1:0] pe_select; + always @(*) begin + pe_select = PE_SEL_BITS'(PE_IDX_WCTL); + if (inst_sfu_is_csr(execute_if.data.op_type)) begin + pe_select = PE_SEL_BITS'(PE_IDX_CSRS); + end + `ifdef VX_CFG_EXT_DXA_ENABLE + if (execute_if.data.op_type == INST_SFU_DXA) begin + pe_select = PE_SEL_BITS'(PE_IDX_DXA); + end + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + if (execute_if.data.op_type == INST_SFU_TEX) begin + pe_select = PE_SEL_BITS'(PE_IDX_TEX); + end + `endif + `ifdef VX_CFG_EXT_OM_ENABLE + if (execute_if.data.op_type == INST_SFU_OM) begin + pe_select = PE_SEL_BITS'(PE_IDX_OM); + end + `endif + `ifdef VX_CFG_EXT_RASTER_ENABLE + if (execute_if.data.op_type == INST_SFU_RASTER) begin + pe_select = PE_SEL_BITS'(PE_IDX_RASTER); + end + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + if (execute_if.data.op_type == INST_SFU_RTUW) begin + pe_select = PE_SEL_BITS'(PE_IDX_RTUW); + end + `endif + end + + VX_pe_switch #( + .PE_COUNT (PE_COUNT), + .NUM_LANES (NUM_LANES), + .ARBITER ("R"), + .REQ_OUT_BUF(0), + .RSP_OUT_BUF(3) + ) pe_switch ( + .clk (clk), + .reset (reset), + .pe_sel (pe_select), + .execute_in_if (execute_if), + .result_out_if (result_if), + .execute_out_if (pe_execute_if), + .result_in_if (pe_result_if) + ); + + VX_wctl_unit #( + .INSTANCE_ID (`SFORMATF(("%s-wctl", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) wctl_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_WCTL]), + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if), + .result_if (pe_result_if[PE_IDX_WCTL]) + ); + + VX_csr_unit #( + .INSTANCE_ID (`SFORMATF(("%s-csr", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) csr_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (pe_execute_if[PE_IDX_CSRS]), + `ifdef VX_CFG_EXT_F_ENABLE + .fcsr_wid (fcsr_wid), + .fcsr_rd_data (fcsr_rd_data), + .fcsr_wr_sel (fcsr_wr_sel), + .fcsr_wr_data (fcsr_wr_data), + `endif + .sched_csr_if (sched_csr_if), + .result_if (pe_result_if[PE_IDX_CSRS]) + ); + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_unit #( + .INSTANCE_ID (`SFORMATF(("%s-dxa", INSTANCE_ID))) + ) dxa_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (pe_execute_if[PE_IDX_DXA]), + .result_if (pe_result_if[PE_IDX_DXA]), + .dxa_req_bus_if (dxa_req_bus_if) + ); +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_unit #( + .INSTANCE_ID (`SFORMATF(("%s-tex", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) tex_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_TEX]), + .result_if (pe_result_if[PE_IDX_TEX]), + .tex_bus_if (tex_bus_if) + ); +`endif + +`ifdef VX_CFG_EXT_OM_ENABLE + // A fragment leaves the shader as a store to the OM aperture through the LSU, + // so the SFU services no OM op; the PE slot is kept for index stability. + assign pe_execute_if[PE_IDX_OM].ready = 1'b1; + assign pe_result_if[PE_IDX_OM].valid = 1'b0; + assign pe_result_if[PE_IDX_OM].data = '0; + `UNUSED_VAR (pe_execute_if[PE_IDX_OM].valid) + `UNUSED_VAR (pe_execute_if[PE_IDX_OM].data) +`endif + +`ifdef VX_CFG_EXT_RASTER_ENABLE + // The fragment payload reaches the warp at launch, so the SFU services no + // raster op; the PE slot is kept for index stability. + assign pe_execute_if[PE_IDX_RASTER].ready = 1'b1; + assign pe_result_if[PE_IDX_RASTER].valid = 1'b0; + assign pe_result_if[PE_IDX_RASTER].data = '0; + `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].valid) + `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].data) + `UNUSED_VAR (pe_result_if[PE_IDX_RASTER].ready) +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_unit #( + .INSTANCE_ID (`SFORMATF(("%s-rtuw", INSTANCE_ID))), + .RTU_TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .NUM_LANES (NUM_LANES) + ) rtu_unit ( + .clk (clk), + .reset (reset), + .block_idx (block_idx), + .execute_if (pe_execute_if[PE_IDX_RTUW]), + .result_if (pe_result_if[PE_IDX_RTUW]), + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if (sched_unlock_if) + ); +`endif + +endmodule diff --git a/hw/rtl/core/VX_sfu_shared.sv b/hw/rtl/core/VX_sfu_shared.sv new file mode 100644 index 0000000000..2d8cfb9c64 --- /dev/null +++ b/hw/rtl/core/VX_sfu_shared.sv @@ -0,0 +1,156 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Merges the SFU blocks' requests onto the core's single DXA, TEX and RTU buses. +module VX_sfu_shared import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.slave block_dxa_req_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.slave block_tex_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_tex_bus_if.master tex_bus_if, +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.slave block_rtu_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_rtu_bus_if.master rtu_bus_if, +`endif + + input wire clk, + input wire reset +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam NB = `VX_CFG_NUM_SFU_BLOCKS; + // the merged request leaves the core: register it when it is a real merge + localparam REQ_OUT_BUF = (NB > 1) ? 3 : 0; + +`ifndef VX_CFG_EXT_DXA_ENABLE +`ifndef VX_CFG_EXT_TEX_ENABLE +`ifndef VX_CFG_EXT_RTU_ENABLE + `UNUSED_VAR ({clk, reset}) + `UNUSED_PARAM (REQ_OUT_BUF) +`endif +`endif +`endif + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if dxa_arb_out_if[1](); + + VX_dxa_req_arb #( + .NUM_INPUTS (NB), + .NUM_OUTPUTS (1), + .OUT_BUF (REQ_OUT_BUF), + .ARBITER ("R") + ) dxa_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (block_dxa_req_bus_if), + .bus_out_if (dxa_arb_out_if) + ); + + assign dxa_req_bus_if.req_valid = dxa_arb_out_if[0].req_valid; + assign dxa_req_bus_if.req_data = dxa_arb_out_if[0].req_data; + assign dxa_arb_out_if[0].req_ready = dxa_req_bus_if.req_ready; +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_CORE_TAG_WIDTH) + ) tex_arb_out_if[1](); + + VX_tex_bus_arb #( + .NUM_INPUTS (NB), + .NUM_OUTPUTS (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH), + .ARBITER ("R"), + .OUT_BUF_REQ (REQ_OUT_BUF), + .OUT_BUF_RSP (0) + ) tex_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (block_tex_bus_if), + .bus_out_if (tex_arb_out_if) + ); + + assign tex_bus_if.req_valid = tex_arb_out_if[0].req_valid; + assign tex_bus_if.req_data = tex_arb_out_if[0].req_data; + assign tex_arb_out_if[0].req_ready = tex_bus_if.req_ready; + assign tex_arb_out_if[0].rsp_valid = tex_bus_if.rsp_valid; + assign tex_arb_out_if[0].rsp_data = tex_bus_if.rsp_data; + assign tex_bus_if.rsp_ready = tex_arb_out_if[0].rsp_ready; +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + localparam RTU_SEL_W = `UP(`ARB_SEL_BITS(NB, 1)); + + VX_rtu_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_CORE_TAG_WIDTH), + .SRC_WIDTH (RTU_SEL_W) + ) rtu_arb_out_if[1](); + + VX_rtu_bus_arb #( + .NUM_INPUTS (NB), + .NUM_OUTPUTS (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .ARBITER ("R"), + .OUT_BUF_ARM (REQ_OUT_BUF), + .OUT_BUF_REQ (REQ_OUT_BUF), + .OUT_BUF_WIN (0) + ) rtu_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (block_rtu_bus_if), + .bus_out_if (rtu_arb_out_if) + ); + + // A warp lives in exactly one SFU block, so {wid} alone names a beat's owner + // on the core's bus; the socket re-derives the core index. + `UNUSED_VAR (rtu_arb_out_if[0].arm_data.src) + `UNUSED_VAR (rtu_arb_out_if[0].req_data.src) + + assign rtu_bus_if.arm_valid = rtu_arb_out_if[0].arm_valid; + assign rtu_bus_if.arm_data.src = 1'b0; + assign rtu_bus_if.arm_data.wid = rtu_arb_out_if[0].arm_data.wid; + assign rtu_bus_if.arm_data.mask = rtu_arb_out_if[0].arm_data.mask; + assign rtu_bus_if.arm_data.scene_base = rtu_arb_out_if[0].arm_data.scene_base; + assign rtu_bus_if.arm_data.flags = rtu_arb_out_if[0].arm_data.flags; + assign rtu_bus_if.arm_data.cull_mask = rtu_arb_out_if[0].arm_data.cull_mask; + assign rtu_bus_if.arm_data.payload_ptr = rtu_arb_out_if[0].arm_data.payload_ptr; + assign rtu_bus_if.arm_data.tag = rtu_arb_out_if[0].arm_data.tag; + assign rtu_arb_out_if[0].arm_ready = rtu_bus_if.arm_ready; + + assign rtu_bus_if.req_valid = rtu_arb_out_if[0].req_valid; + assign rtu_bus_if.req_data.kind = rtu_arb_out_if[0].req_data.kind; + assign rtu_bus_if.req_data.src = 1'b0; + assign rtu_bus_if.req_data.wid = rtu_arb_out_if[0].req_data.wid; + assign rtu_bus_if.req_data.data = rtu_arb_out_if[0].req_data.data; + assign rtu_bus_if.req_data.cb_action = rtu_arb_out_if[0].req_data.cb_action; + assign rtu_arb_out_if[0].req_ready = rtu_bus_if.req_ready; + + assign rtu_arb_out_if[0].win_valid = rtu_bus_if.win_valid; + assign rtu_arb_out_if[0].win_data = rtu_bus_if.win_data; + assign rtu_bus_if.win_ready = rtu_arb_out_if[0].win_ready; +`endif + +endmodule diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index 660303ee31..ecbbf46146 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -13,21 +13,17 @@ `include "VX_define.vh" -module VX_sfu_unit import VX_gpu_pkg::*; -`ifdef VX_CFG_EXT_RASTER_ENABLE -import VX_raster_pkg::*; +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" `endif -#( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + +module VX_sfu_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, -`ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`endif + input wire [CORE_ID_WIDTH-1:0] core_id, // Inputs VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], @@ -38,55 +34,26 @@ import VX_raster_pkg::*; `ifdef VX_CFG_EXT_DXA_ENABLE VX_dxa_req_bus_if.master dxa_req_bus_if, - VX_txbar_bus_if.slave dxa_txbar_bus_if, `endif `ifdef VX_CFG_EXT_TEX_ENABLE VX_tex_bus_if.master tex_bus_if, `endif -`ifdef VX_CFG_EXT_OM_ENABLE -`endif - - // RASTER has no SFU port at all now: the fragment shader is launched by the - // raster engine (push, not pull) and its stamp arrives in the launch, so - // nothing about a fragment passes through this unit. - `ifdef VX_CFG_EXT_RTU_ENABLE VX_rtu_bus_if.master rtu_bus_if, - VX_sched_unlock_if.master sched_unlock_if, + VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif - VX_sched_csr_if.slave sched_csr_if, - - VX_dcr_csr_if dcr_csr_if, + VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], // Outputs VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH], - VX_warp_ctl_if.master warp_ctl_if + VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS] ); - `UNUSED_SPARAM (INSTANCE_ID) - localparam BLOCK_SIZE = 1; - localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; - localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; - localparam PE_SEL_BITS = `CLOG2(PE_COUNT); - localparam PE_IDX_WCTL = 0; - localparam PE_IDX_CSRS = 1; -`ifdef VX_CFG_EXT_DXA_ENABLE - localparam PE_IDX_DXA = 2; -`endif -`ifdef VX_CFG_EXT_TEX_ENABLE - localparam PE_IDX_TEX = 2 + `VX_CFG_EXT_DXA_ENABLED; -`endif -`ifdef VX_CFG_EXT_OM_ENABLE - localparam PE_IDX_OM = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED; -`endif -`ifdef VX_CFG_EXT_RASTER_ENABLE - localparam PE_IDX_RASTER = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED; -`endif -`ifdef VX_CFG_EXT_RTU_ENABLE - localparam PE_IDX_RTUW = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED; -`endif + localparam BLOCK_SIZE = `VX_CFG_NUM_SFU_BLOCKS; + localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; VX_execute_if #( .data_t (sfu_execute_t) @@ -103,193 +70,99 @@ import VX_raster_pkg::*; ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); - VX_execute_if #( - .data_t (sfu_execute_t) - ) pe_execute_if[PE_COUNT](); - - VX_result_if#( - .data_t (sfu_result_t) - ) pe_result_if[PE_COUNT](); - - reg [PE_SEL_BITS-1:0] pe_select; - always @(*) begin - pe_select = PE_SEL_BITS'(PE_IDX_WCTL); - if (inst_sfu_is_csr(per_block_execute_if[0].data.op_type)) begin - pe_select = PE_SEL_BITS'(PE_IDX_CSRS); - end - `ifdef VX_CFG_EXT_DXA_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_DXA) begin - pe_select = PE_SEL_BITS'(PE_IDX_DXA); - end - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_TEX) begin - pe_select = PE_SEL_BITS'(PE_IDX_TEX); - end - `endif - `ifdef VX_CFG_EXT_OM_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_OM) begin - pe_select = PE_SEL_BITS'(PE_IDX_OM); - end - `endif - `ifdef VX_CFG_EXT_RASTER_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_RASTER) begin - pe_select = PE_SEL_BITS'(PE_IDX_RASTER); - end - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_RTUW) begin - pe_select = PE_SEL_BITS'(PE_IDX_RTUW); - end - `endif - end - - VX_pe_switch #( - .PE_COUNT (PE_COUNT), - .NUM_LANES (NUM_LANES), - .ARBITER ("R"), - .REQ_OUT_BUF(0), - .RSP_OUT_BUF(3) - ) pe_switch ( - .clk (clk), - .reset (reset), - .pe_sel (pe_select), - .execute_in_if (per_block_execute_if[0]), - .result_out_if (per_block_result_if[0]), - .execute_out_if (pe_execute_if), - .result_in_if (pe_result_if) - ); - - VX_txbar_bus_if txbar_bus_if(); - - VX_wctl_unit #( - .INSTANCE_ID (`SFORMATF(("%s-wctl", INSTANCE_ID))), - .NUM_LANES (NUM_LANES) - ) wctl_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_WCTL]), - .warp_ctl_if(warp_ctl_if), - .txbar_bus_if(txbar_bus_if), - .result_if (pe_result_if[PE_IDX_WCTL]) - ); - - VX_csr_unit #( - .INSTANCE_ID (`SFORMATF(("%s-csr", INSTANCE_ID))), - .CORE_ID (CORE_ID), - .NUM_LANES (NUM_LANES) - ) csr_unit ( - .clk (clk), - .reset (reset), - - .execute_if (pe_execute_if[PE_IDX_CSRS]), - - `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf), - .pipeline_perf (pipeline_perf), - `endif - - `ifdef VX_CFG_EXT_F_ENABLE - .fpu_csr_if (fpu_csr_if), - `endif - - .sched_csr_if (sched_csr_if), - .result_if (pe_result_if[PE_IDX_CSRS]), - .dcr_csr_if (dcr_csr_if) - ); +`ifdef VX_CFG_EXT_F_ENABLE + wire [BLOCK_SIZE-1:0][NW_WIDTH-1:0] fcsr_wid; + wire [BLOCK_SIZE-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data; + wire [BLOCK_SIZE-1:0][2:0] fcsr_wr_sel; + wire [BLOCK_SIZE-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data; +`endif `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_unit #( - .INSTANCE_ID (`SFORMATF(("%s-dxa", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) dxa_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_DXA]), - .result_if (pe_result_if[PE_IDX_DXA]), - .dxa_req_bus_if (dxa_req_bus_if) - ); - - // The only txbar producer is the SMEM-completion path through - // dxa_txbar_bus_if (DXA release); no arbitration is needed. - assign txbar_bus_if.valid = dxa_txbar_bus_if.valid; - assign txbar_bus_if.data = dxa_txbar_bus_if.data; - assign dxa_txbar_bus_if.ready = txbar_bus_if.ready; -`else - assign txbar_bus_if.valid = 1'b0; - assign txbar_bus_if.data = 'x; - `UNUSED_VAR (txbar_bus_if.ready) + VX_dxa_req_bus_if block_dxa_req_bus_if[BLOCK_SIZE](); `endif - -// The hit window has no FF consumers left. TEX was the last one — it spilled -// its 2x2 quad's eight (u,v) operands into window slots because no RISC-V encoding -// holds eight inputs, and that cost the window two full RAM mirrors. It now takes -// u/v/lod in registers (vx_tex, R4-type) and its shader computes the mip LOD with -// vx_tex_quad_lod(), so the window is down to a single mirror and its only tenant -// is the RTU. - `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_unit #( - .INSTANCE_ID (`SFORMATF(("%s-tex", INSTANCE_ID))), - .CORE_ID (CORE_ID), - .NUM_LANES (NUM_LANES) - ) tex_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_TEX]), - .result_if (pe_result_if[PE_IDX_TEX]), - .tex_bus_if (tex_bus_if) - ); -`endif - -`ifdef VX_CFG_EXT_OM_ENABLE - // OM export (v2): a fragment leaves the shader as a STORE to the OM aperture - // (vx_om_export -> VX_gfx_uops -> the LSU), so the SFU services no OM op. The - // PE slot is retained for index stability and tied off, as RASTER's is. - assign pe_execute_if[PE_IDX_OM].ready = 1'b1; - assign pe_result_if[PE_IDX_OM].valid = 1'b0; - assign pe_result_if[PE_IDX_OM].data = '0; - `UNUSED_VAR (pe_execute_if[PE_IDX_OM].valid) - `UNUSED_VAR (pe_execute_if[PE_IDX_OM].data) + VX_tex_bus_if #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + ) block_tex_bus_if[BLOCK_SIZE](); `endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH) + ) block_rtu_bus_if[BLOCK_SIZE](); +`endif + + for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks + VX_sfu_block #( + .INSTANCE_ID ((BLOCK_SIZE > 1) ? `SFORMATF(("%s%0d", INSTANCE_ID, block_idx)) : INSTANCE_ID) + ) sfu_block ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .block_idx (SFU_BLOCK_W'(block_idx)), + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]), + `ifdef VX_CFG_EXT_F_ENABLE + .fcsr_wid (fcsr_wid[block_idx]), + .fcsr_rd_data (fcsr_rd_data[block_idx]), + .fcsr_wr_sel (fcsr_wr_sel[block_idx]), + .fcsr_wr_data (fcsr_wr_data[block_idx]), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (block_dxa_req_bus_if[block_idx]), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (block_tex_bus_if[block_idx]), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (block_rtu_bus_if[block_idx]), + .sched_unlock_if(sched_unlock_if[block_idx]), + `endif + .sched_csr_if (sched_csr_if[block_idx]), + .warp_shared_if (warp_shared_if[block_idx]), + .warp_ctl_if (warp_ctl_if[block_idx]) + ); + end -`ifdef VX_CFG_EXT_RASTER_ENABLE - // RASTER push (v2): the fragment payload reaches the warp at launch via the - // core-level distributor's window seed (rast_win_wr_*), so the SFU services - // no raster op. The PE slot is retained for index stability and tied off — - // no kernel issues INST_SFU_RASTER under the push model. - assign pe_execute_if[PE_IDX_RASTER].ready = 1'b1; - assign pe_result_if[PE_IDX_RASTER].valid = 1'b0; - assign pe_result_if[PE_IDX_RASTER].data = '0; - `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].valid) - `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].data) - `UNUSED_VAR (pe_result_if[PE_IDX_RASTER].ready) +`ifdef VX_CFG_EXT_F_ENABLE + VX_csr_fpu #( + .NUM_CSR_PORTS (BLOCK_SIZE), + .NUM_FPU_PORTS (`VX_CFG_NUM_FPU_BLOCKS) + ) csr_fpu ( + .clk (clk), + .reset (reset), + .csr_wid (fcsr_wid), + .csr_rd_data (fcsr_rd_data), + .csr_wr_sel (fcsr_wr_sel), + .csr_wr_data (fcsr_wr_data), + .fpu_csr_if (fpu_csr_if) + ); `endif -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-rtuw", INSTANCE_ID))), - .CORE_ID (CORE_ID), - `ifdef VX_CFG_EXT_RTU_ENABLE - .RTU_TAG_WIDTH (RTU_REQ_TAG_WIDTH), + VX_sfu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-shared", INSTANCE_ID))) + ) sfu_shared ( + `ifdef VX_CFG_EXT_DXA_ENABLE + .block_dxa_req_bus_if (block_dxa_req_bus_if), + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .block_tex_bus_if (block_tex_bus_if), + .tex_bus_if (tex_bus_if), `endif - .NUM_LANES (NUM_LANES) - ) rtu_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_RTUW]), - .result_if (pe_result_if[PE_IDX_RTUW]) `ifdef VX_CFG_EXT_RTU_ENABLE - , - .rtu_bus_if (rtu_bus_if), - .sched_unlock_if (sched_unlock_if) + .block_rtu_bus_if (block_rtu_bus_if), + .rtu_bus_if (rtu_bus_if), `endif + .clk (clk), + .reset (reset) ); -`endif VX_lane_gather #( .BLOCK_SIZE (BLOCK_SIZE), diff --git a/hw/rtl/core/VX_split_join.sv b/hw/rtl/core/VX_split_join.sv index 7da6d4df6d..36fe514539 100644 --- a/hw/rtl/core/VX_split_join.sv +++ b/hw/rtl/core/VX_split_join.sv @@ -15,20 +15,22 @@ module VX_split_join import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter OUT_REG = 0 + parameter NUM_WARPS = `VX_CFG_NUM_WARPS, + parameter WID_W = `LOG2UP(NUM_WARPS), + parameter OUT_REG = 0 ) ( input wire clk, input wire reset, input wire split_valid, input wire sjoin_valid, - input wire [NW_WIDTH-1:0] wid, + input wire [WID_W-1:0] wid, input split_t split, input join_t sjoin, - input wire [NW_WIDTH-1:0] stack_wid, + input wire [WID_W-1:0] stack_wid, output wire join_valid, output wire join_is_dvg, output wire join_is_else, - output wire [NW_WIDTH-1:0] join_wid, + output wire [WID_W-1:0] join_wid, output wire [`VX_CFG_NUM_THREADS-1:0] join_tmask, output wire [PC_BITS-1:0] join_pc, output wire [DV_STACK_SIZEW-1:0] stack_ptr @@ -36,7 +38,7 @@ module VX_split_join import VX_gpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) if (NT_BITS != 0) begin : g_enable - wire [`VX_CFG_NUM_WARPS-1:0][DV_STACK_SIZEW-1:0] ipdom_wr_ptr; + wire [NUM_WARPS-1:0][DV_STACK_SIZEW-1:0] ipdom_wr_ptr; wire [`VX_CFG_NUM_THREADS-1:0] orig_tmask; wire [PC_BITS-1:0] next_pc; wire ipdom_idx; @@ -46,8 +48,10 @@ module VX_split_join import VX_gpu_pkg::*; #( wire sjoin_is_dvg = (sjoin.stack_ptr != ipdom_wr_ptr[wid]); VX_ipdom_stack #( - .WIDTH (`VX_CFG_NUM_THREADS + PC_BITS), - .DEPTH (DV_STACK_SIZE) + .WIDTH (`VX_CFG_NUM_THREADS + PC_BITS), + .DEPTH (DV_STACK_SIZE), + .NUM_WARPS (NUM_WARPS), + .WID_W (WID_W) ) ipdom_stack ( .clk (clk), .reset (reset), @@ -66,7 +70,7 @@ module VX_split_join import VX_gpu_pkg::*; #( wire [`VX_CFG_NUM_THREADS-1:0] join_tmask_n = ipdom_idx ? orig_tmask : (~sjoin.tmask & orig_tmask); VX_pipe_register #( - .DATAW (1 + NW_WIDTH + 1 + 1 + `VX_CFG_NUM_THREADS + PC_BITS), + .DATAW (1 + WID_W + 1 + 1 + `VX_CFG_NUM_THREADS + PC_BITS), .RESETW (1), .DEPTH (OUT_REG) ) pipe_reg ( diff --git a/hw/rtl/core/VX_uop_packld.sv b/hw/rtl/core/VX_uop_packld.sv index f33483f579..9ec971e633 100644 --- a/hw/rtl/core/VX_uop_packld.sv +++ b/hw/rtl/core/VX_uop_packld.sv @@ -24,7 +24,7 @@ // eff_rs1[lane] = rs1[lane] + rs2[lane] * uop_idx (shift-and-add) // The pack field is cleared in op_args forwarded to the LSU slice so it sees a // plain LBU or LHU. The NaN-boxing of the FLW result is handled automatically -// because is_float==1 and rd is a float register (triggers nan-box in lsu_slice). +// because is_float==1 and rd is a float register (triggers nan-box in lsu_block). module VX_uop_packld import VX_gpu_pkg::*; ( input clk, diff --git a/hw/rtl/core/VX_uuid_gen.sv b/hw/rtl/core/VX_uuid_gen.sv index 61ec538e35..34ea3aab1d 100644 --- a/hw/rtl/core/VX_uuid_gen.sv +++ b/hw/rtl/core/VX_uuid_gen.sv @@ -13,31 +13,45 @@ `include "VX_define.vh" +// Per-warp instruction ids. NUM_WARPS is the number of warps counted here: all +// of the core's, or the PER_FETCH_WARPS of one fetch slice (wid = {wfs, slice}). module VX_uuid_gen import VX_gpu_pkg::*; #( - parameter CORE_ID = 0 + parameter NUM_WARPS = `VX_CFG_NUM_WARPS ) ( input wire clk, input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, input wire incr, input wire [NW_WIDTH-1:0] wid, output wire [UUID_WIDTH-1:0] uuid ); localparam GNW_WIDTH = UUID_WIDTH - 32; - reg [31:0] uuid_cntrs [0:`VX_CFG_NUM_WARPS-1]; - reg [`VX_CFG_NUM_WARPS-1:0] has_uuid_cntrs; + localparam LW = `LOG2UP(NUM_WARPS); + + wire [LW-1:0] lid; + if (NUM_WARPS == `VX_CFG_NUM_WARPS) begin : g_core_warps + assign lid = LW'(wid); + end else begin : g_slice_warps + `STATIC_ASSERT (NUM_WARPS == PER_FETCH_WARPS, ("invalid parameter")) + assign lid = wid_to_wfs(wid); + end + + reg [31:0] uuid_cntrs [0:NUM_WARPS-1]; + reg [NUM_WARPS-1:0] has_uuid_cntrs; always @(posedge clk) begin if (reset) begin has_uuid_cntrs <= '0; end else if (incr) begin - has_uuid_cntrs[wid] <= 1; + has_uuid_cntrs[lid] <= 1; end if (incr) begin - uuid_cntrs[wid] <= has_uuid_cntrs[wid] ? (uuid_cntrs[wid] + 1) : 1; + uuid_cntrs[lid] <= has_uuid_cntrs[lid] ? (uuid_cntrs[lid] + 1) : 1; end end - wire [GNW_WIDTH-1:0] g_wid = (GNW_WIDTH'(CORE_ID) << NW_BITS) + GNW_WIDTH'(wid); - assign uuid = {g_wid, (has_uuid_cntrs[wid] ? uuid_cntrs[wid] : 0)}; + wire [GNW_WIDTH-1:0] g_wid = (GNW_WIDTH'(core_id) << NW_BITS) + GNW_WIDTH'(wid); + assign uuid = {g_wid, (has_uuid_cntrs[lid] ? uuid_cntrs[lid] : 0)}; endmodule diff --git a/hw/rtl/core/VX_warp_sched.sv b/hw/rtl/core/VX_warp_sched.sv new file mode 100644 index 0000000000..eeeb663bc4 --- /dev/null +++ b/hw/rtl/core/VX_warp_sched.sv @@ -0,0 +1,514 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// The warp scheduler of one fetch slice: the state of the slice's warps +// (wid = {wfs, slice_id}), its warp selection and a registered fetch request to +// the slice's fetch unit, which names warps by wfs. FU block port q carries the +// traffic of the local warps with wfs mod PORTS == q, addressed by core-wide wid; +// core-wide events arrive through sched_if already localized. +module VX_warp_sched import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [FETCH_FSW_W-1:0] slice_id, + + VX_slice_sched_if.slave sched_if, + + // ALU ports + input sched_branch_t [FETCH_ALU_PORTS-1:0] branch, + + // SFU ports + input sched_wctl_t [FETCH_SFU_PORTS-1:0] wctl, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] dvstack_wid, + output wire [FETCH_SFU_PORTS-1:0][DV_STACK_SIZEW-1:0] dvstack_ptr, + output wire [FETCH_SFU_PORTS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] dvstack_mscratch, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] csr_rd_wid, + output wire sched_csr_rd_t [FETCH_SFU_PORTS-1:0] csr_rd, + input sched_csr_wr_t [FETCH_SFU_PORTS-1:0] csr_wr, +`ifdef VX_CFG_EXT_RTU_ENABLE + input wire [FETCH_SFU_PORTS-1:0] rtu_unlock_valid, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] rtu_unlock_wid, +`endif + + // per-warp pipeline events, indexed by wfs + VX_decode_sched_if.slave decode_sched_if, + input wire [PER_FETCH_WARPS-1:0] issued, + input wire [PER_FETCH_WARPS-1:0] committed, + input wire [PER_FETCH_WARPS-1:0] ibuf_pop, + output wire [PER_FETCH_WARPS-1:0] pending_alm_empty, + + // fetch request + VX_schedule_if.master schedule_if +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam SW = PER_FETCH_WARPS; + localparam LW = FETCH_WFS_W; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; + localparam NPS_BITS = `CLOG2(NPS); + localparam PSW = SW / NPS; // warps per SFU port + localparam KW = `LOG2UP(PSW); + localparam IBUF_CW = $clog2(`VX_CFG_IBUF_SIZE + 1); + + `UNUSED_VAR (schedule_if.ibuf_pop) + + // wfs -> SFU port-local index + function automatic logic [KW-1:0] wfs_to_k(input logic [LW-1:0] wfs); + wfs_to_k = KW'(wfs >> NPS_BITS); + endfunction + + wire [LW-1:0] dec_lid = decode_sched_if.wid[LW-1:0]; + if (NW_WIDTH > LW) begin : g_dec_wid_unused + `UNUSED_VAR (decode_sched_if.wid[NW_WIDTH-1:LW]) + end + + wire [NPA-1:0][LW-1:0] br_lid; + for (genvar q = 0; q < NPA; ++q) begin : g_br_lid + assign br_lid[q] = wid_to_wfs(branch[q].wid); + `UNUSED_VAR (wid_to_fsw(branch[q].wid)) + end + + wire [NPS-1:0][LW-1:0] ctl_lid, csr_rd_lid, csr_wr_lid; +`ifdef VX_CFG_EXT_RTU_ENABLE + wire [NPS-1:0][LW-1:0] rtu_lid; +`endif + + // split/join, one stack per SFU port over the port's warps + wire [NPS-1:0] join_valid; + wire [NPS-1:0] join_is_dvg; + wire [NPS-1:0] join_is_else; + wire [NPS-1:0][KW-1:0] join_k; + wire [NPS-1:0][`VX_CFG_NUM_THREADS-1:0] join_tmask; + wire [NPS-1:0][PC_BITS-1:0] join_pc; + + for (genvar q = 0; q < NPS; ++q) begin : g_sfu_ports + assign ctl_lid[q] = wid_to_wfs(wctl[q].wid); + assign csr_rd_lid[q] = wid_to_wfs(csr_rd_wid[q]); + assign csr_wr_lid[q] = wid_to_wfs(csr_wr[q].wid); + `UNUSED_VAR ({wid_to_fsw(wctl[q].wid), wid_to_fsw(csr_rd_wid[q]), wid_to_fsw(csr_wr[q].wid), wid_to_fsw(dvstack_wid[q])}) + `ifdef VX_CFG_EXT_RTU_ENABLE + assign rtu_lid[q] = wid_to_wfs(rtu_unlock_wid[q]); + `UNUSED_VAR (wid_to_fsw(rtu_unlock_wid[q])) + `endif + + VX_split_join #( + .INSTANCE_ID ((NPS > 1) ? `SFORMATF(("%s-splitjoin%0d", INSTANCE_ID, q)) : `SFORMATF(("%s-splitjoin", INSTANCE_ID))), + .NUM_WARPS (PSW), + .WID_W (KW), + .OUT_REG (1) + ) split_join ( + .clk (clk), + .reset (reset), + .split_valid (wctl[q].split_valid), + .sjoin_valid (wctl[q].sjoin_valid), + .wid (wfs_to_k(ctl_lid[q])), + .split (wctl[q].split), + .sjoin (wctl[q].sjoin), + .join_valid (join_valid[q]), + .join_is_dvg (join_is_dvg[q]), + .join_is_else(join_is_else[q]), + .join_wid (join_k[q]), + .join_tmask (join_tmask[q]), + .join_pc (join_pc[q]), + .stack_wid (wfs_to_k(wid_to_wfs(dvstack_wid[q]))), + .stack_ptr (dvstack_ptr[q]) + ); + end + + wire [SW-1:0] active_warps, stalled_warps; + wire [SW-1:0][`VX_CFG_NUM_THREADS-1:0] thread_masks; + wire [SW-1:0][PC_BITS-1:0] warp_pcs; + wire [SW-1:0] ibuf_full, ibuf_full_n; + wire [SW-1:0] pending_empty; + wire [SW-1:0] cand_onehot; + wire cand_valid; + wire sched_fire; + + reg [SW-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch_r; + reg [SW-1:0][`VX_CFG_XLEN-1:0] mstatus_r, mtvec_r, mepc_r, mcause_r, mtval_r; + + for (genvar l = 0; l < SW; ++l) begin : g_warps + localparam [LW-1:0] LID = LW'(l); + localparam QA = l % NPA; + localparam QS = l % NPS; + localparam [KW-1:0] K = KW'(l / NPS); + + reg active_r, active_n; + reg stalled_r, stalled_n; + reg [`VX_CFG_NUM_THREADS-1:0] tmask_r, tmask_n; + reg [PC_BITS-1:0] pc_r, pc_n; + + wire ctl_hit = (ctl_lid[QS] == LID); + wire br_hit = branch[QA].valid && (br_lid[QA] == LID); + wire join_hit = join_valid[QS] && (join_k[QS] == K); + + always @(*) begin + active_n = active_r; + stalled_n = stalled_r; + tmask_n = tmask_r; + pc_n = pc_r; + + if (sched_if.cta_fire && (sched_if.cta_lid == LID)) begin + active_n = 1; + // Reusing a warp for the next CTA skips the one-time prologue and rewinds to the + // kernel's per-CTA dispatch window: a fixed 20-byte (5-instruction) sequence that + // reloads the entry pointer and kargs before re-calling. + pc_n = sched_if.cta_init ? sched_if.cta_PC : (pc_r - from_fullPC(`VX_CFG_XLEN'(20))); + tmask_n = sched_if.cta_tmask; + end + + if (decode_sched_if.valid && decode_sched_if.unlock && (dec_lid == LID)) begin + stalled_n = 0; + end + + if (sched_if.wspawn_apply) begin + if (sched_if.wspawn_wmask[l]) begin + active_n = 1; + if (!(sched_if.wspawn_src && (sched_if.wspawn_src_lid == LID))) begin + tmask_n[0] = 1; + pc_n = sched_if.wspawn_pc; + end + end + if (sched_if.wspawn_src && (sched_if.wspawn_src_lid == LID)) begin + stalled_n = 0; + end + end + + if (wctl[QS].tmc_valid && ctl_hit) begin + active_n = (wctl[QS].tmc.tmask != 0); + tmask_n = wctl[QS].tmc.tmask; + stalled_n = 0; + end + + if (wctl[QS].split_valid && ctl_hit) begin + if (wctl[QS].split.is_dvg) begin + tmask_n = wctl[QS].split.then_tmask; + end + stalled_n = 0; + end + + if (join_hit) begin + if (join_is_dvg[QS]) begin + if (join_is_else[QS]) begin + pc_n = join_pc[QS]; + end + tmask_n = join_tmask[QS]; + end + stalled_n = 0; + end + + if (sched_if.bar_unlock_valid && sched_if.bar_unlock_mask[l]) begin + stalled_n = 0; + end + + // an exiting warp is retired as its CTA accounting takes it, before + // the slot can be relaunched + if (sched_if.retire_valid && (sched_if.retire_lid == LID)) begin + active_n = 0; + tmask_n = '0; + stalled_n = 0; + end + + // wsync unlock: warp pipeline drained + if (wctl[QS].wsync_valid && ctl_hit) begin + stalled_n = 0; + end + + if (br_hit) begin + if (branch[QA].is_trap) begin + // ECALL/EBREAK: redirect to trap vector (mtvec[1:0] = MODE field; mask off to get base address). + pc_n = from_fullPC(mtvec_r[l] & ~`VX_CFG_XLEN'(3)); + end else if (branch[QA].is_mret) begin + // MRET/SRET/URET: restore the saved PC from mepc. + pc_n = from_fullPC(mepc_r[l]); + end else if (branch[QA].taken) begin + pc_n = branch[QA].dest; + end + stalled_n = 0; + end + + // stall the warp until decode stage + if (sched_fire && cand_onehot[l]) begin + stalled_n = 1; + `ifndef VX_CFG_EXT_C_ENABLE + pc_n = pc_r + from_fullPC(`VX_CFG_XLEN'(4)); + `endif + end + + `ifdef VX_CFG_EXT_C_ENABLE + // Every redirect source stalls its warp from schedule until it resolves, so + // no redirect writes a warp's PC on the cycle that warp decode-advances. + if (decode_sched_if.valid && (dec_lid == LID)) begin + pc_n = pc_r + from_fullPC(decode_sched_if.is_rvc ? `VX_CFG_XLEN'(2) : `VX_CFG_XLEN'(4)); + end + `endif + + `ifdef VX_CFG_EXT_RTU_ENABLE + // A wstall'd TRACE retires: resume the warp so it proceeds to WAIT. + if (rtu_unlock_valid[QS] && (rtu_lid[QS] == LID)) begin + stalled_n = 1'b0; + end + `endif + end + + always @(posedge clk) begin + if (reset) begin + active_r <= 0; + stalled_r <= 0; + tmask_r <= '0; + pc_r <= '0; + mscratch_r[l] <= '0; + mstatus_r[l] <= '0; + mtvec_r[l] <= '0; + mepc_r[l] <= '0; + mcause_r[l] <= '0; + mtval_r[l] <= '0; + end else begin + active_r <= active_n; + stalled_r <= stalled_n; + tmask_r <= tmask_n; + pc_r <= pc_n; + + // spawned warps inherit the spawning warp's mscratch + if (sched_if.wspawn_apply && sched_if.wspawn_wmask[l] + && !(sched_if.wspawn_src && (sched_if.wspawn_src_lid == LID))) begin + mscratch_r[l] <= sched_if.wspawn_mscratch; + end + // CTA dispatch: latch this warp's mscratch (param) + if (sched_if.cta_fire && (sched_if.cta_lid == LID)) begin + mscratch_r[l] <= sched_if.cta_param; + end + if (csr_wr[QS].valid && (csr_wr_lid[QS] == LID)) begin + mscratch_r[l] <= csr_wr[QS].data; + end + + if (csr_wr_lid[QS] == LID) begin + if (csr_wr[QS].trap_valid[0]) begin + mstatus_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[1]) begin + mtvec_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[2]) begin + mepc_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[3]) begin + mcause_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[4]) begin + mtval_r[l] <= csr_wr[QS].trap_data; + end + end + + // Hardware trap entry (ECALL/EBREAK) wins a same-cycle software write. + if (br_hit && branch[QA].is_trap) begin + mepc_r[l] <= to_fullPC(branch[QA].dest); + mcause_r[l] <= `VX_CFG_XLEN'(branch[QA].trap_cause); + mtval_r[l] <= '0; + end + end + end + + assign active_warps[l] = active_r; + assign stalled_warps[l] = stalled_r; + assign thread_masks[l] = tmask_r; + assign warp_pcs[l] = pc_r; + + // ibuffer occupancy: registered full keeps arbitration off the critical path + logic [IBUF_CW-1:0] ibuf_size_r, ibuf_size_n; + logic ibuf_full_r; + wire ibuf_incr = sched_fire && cand_onehot[l]; + assign ibuf_size_n = ibuf_size_r + IBUF_CW'(ibuf_incr) - IBUF_CW'(ibuf_pop[l]); + always @(posedge clk) begin + if (reset) begin + ibuf_size_r <= '0; + ibuf_full_r <= 1'b0; + end else begin + ibuf_size_r <= ibuf_size_n; + ibuf_full_r <= ibuf_full_n[l]; + end + end + assign ibuf_full_n[l] = (ibuf_size_n == IBUF_CW'(`VX_CFG_IBUF_SIZE)); + assign ibuf_full[l] = ibuf_full_r; + + VX_pending_size #( + .SIZE (256), + .ALM_EMPTY (1) + ) pending_ctr ( + .clk (clk), + .reset (reset), + .incr (issued[l]), + .decr (committed[l]), + .empty (pending_empty[l]), + .alm_empty (pending_alm_empty[l]), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + + `ifdef DBG_TRACE_PIPELINE + always @(posedge clk) begin + if (active_n != active_r + || (active_r && (stalled_n != stalled_r || tmask_n != tmask_r))) begin + `TRACE(1, ("%t: %s warp-state: wid=%0d, active=%b, stalled=%b, tmask=%b\n", + $time, INSTANCE_ID, wfs_to_wid(LID, slice_id), active_n, stalled_n, tmask_n)) + end + end + `endif + end + + // the slice's lowest ready warp + wire [SW-1:0] ready_warps = active_warps & ~stalled_warps; + wire [SW-1:0] preferred_warps = ready_warps & ~ibuf_full; +`ifndef L1_ENABLE + // without L1, the icache must never stall: it shares the bus with dcache responses + wire [SW-1:0] schedule_warps = preferred_warps; + `UNUSED_VAR (sched_if.all_ibuf_full) +`else + wire [SW-1:0] schedule_warps = sched_if.all_ibuf_full ? ready_warps : preferred_warps; +`endif + + wire [`UP(FETCH_WFS_BITS)-1:0] cand_index; + VX_priority_encoder #( + .N (SW) + ) wid_select ( + .data_in (schedule_warps), + .index_out (cand_index), + .valid_out (cand_valid), + .onehot_out(cand_onehot) + ); + wire [LW-1:0] cand_lid = LW'(cand_index); + + // CTA CSRs and CTA ids, one store per SFU port over the port's warps + wire [NPS-1:0][NCTA_WIDTH-1:0] port_cta_id; + + for (genvar q = 0; q < NPS; ++q) begin : g_csr_cta + wire [LW-1:0] warp_waddr = sched_if.warp_waddr; + wire [LW-1:0] id_waddr = sched_if.id_waddr; + + VX_csr_cta #( + .NUM_WARPS (PSW) + ) csr_cta ( + .clk (clk), + .reset (reset), + .warp_write (sched_if.warp_write && ((32'(warp_waddr) % NPS) == q)), + .warp_waddr (wfs_to_k(warp_waddr)), + .warp_wdata (sched_if.warp_wdata), + .id_write (sched_if.id_write && ((32'(id_waddr) % NPS) == q)), + .id_waddr (wfs_to_k(id_waddr)), + .id_wdata (sched_if.id_wdata), + .rd_lid (wfs_to_k(csr_rd_lid[q])), + .rd_cta_rank (csr_rd[q].cta_rank), + .rd_lane (csr_rd[q].cta_lane), + .sched_lid (wfs_to_k(cand_lid)), + .sched_cta_id (port_cta_id[q]) + ); + + assign csr_rd[q].thread_mask = thread_masks[csr_rd_lid[q]]; + assign csr_rd[q].mscratch = mscratch_r[csr_rd_lid[q]]; + assign csr_rd[q].mstatus = mstatus_r[csr_rd_lid[q]]; + assign csr_rd[q].mtvec = mtvec_r[csr_rd_lid[q]]; + assign csr_rd[q].mepc = mepc_r[csr_rd_lid[q]]; + assign csr_rd[q].mcause = mcause_r[csr_rd_lid[q]]; + assign csr_rd[q].mtval = mtval_r[csr_rd_lid[q]]; + + assign dvstack_mscratch[q] = mscratch_r[wid_to_wfs(dvstack_wid[q])]; + end + + wire [NCTA_WIDTH-1:0] cand_cta_id = port_cta_id[32'(cand_lid) % NPS]; + + // fetch request, registered at the slice boundary + wire sched_ready; + assign sched_fire = cand_valid && sched_ready; + + wire [UUID_WIDTH-1:0] sched_uuid; +`ifdef UUID_ENABLE + VX_uuid_gen #( + .NUM_WARPS (SW) + ) uuid_gen ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .incr (sched_fire), + .wid (wfs_to_wid(cand_lid, slice_id)), + .uuid (sched_uuid) + ); +`else + assign sched_uuid = '0; + `UNUSED_VAR (core_id) +`ifndef DBG_TRACE_PIPELINE + `UNUSED_VAR (slice_id) +`endif +`endif + + VX_elastic_buffer #( + .DATAW (`VX_CFG_NUM_THREADS + PC_BITS + NW_WIDTH + NCTA_WIDTH + UUID_WIDTH), + .SIZE (2), + .OUT_REG (1) + ) sched_buf ( + .clk (clk), + .reset (reset), + .valid_in (cand_valid), + .ready_in (sched_ready), + .data_in ({thread_masks[cand_lid], warp_pcs[cand_lid], NW_WIDTH'(cand_lid), cand_cta_id, sched_uuid}), + .data_out ({schedule_if.data.tmask, schedule_if.data.PC, schedule_if.data.wid, schedule_if.data.cta_id, schedule_if.data.uuid}), + .valid_out (schedule_if.valid), + .ready_out (schedule_if.ready) + ); + + wire schedule_if_fire = schedule_if.valid && schedule_if.ready; + + // status + assign sched_if.active = active_warps; + assign sched_if.stalled = stalled_warps; + assign sched_if.ibuf_full_n = ibuf_full_n; + assign sched_if.pending_empty = pending_empty; + + // performance events + for (genvar q = 0; q < NPA; ++q) begin : g_branch_valid + assign sched_if.branch_valid[q] = branch[q].valid; + end + for (genvar q = 0; q < NPS; ++q) begin : g_divergence + assign sched_if.divergence[q] = wctl[q].split_valid && wctl[q].split.is_dvg; + end + assign sched_if.sched_valid = cand_valid; + assign sched_if.sched_fire = schedule_if_fire; + assign sched_if.sched_tmask = schedule_if.data.tmask; + +`ifdef EXT_SCHED_TIMEOUT_DUMP + always @(posedge clk) begin + if (sched_if.timeout_dump) begin + for (integer l = 0; l < SW; ++l) begin + $display(" wid=%0d stalled=%0d pc=0x%0h tmask=%b", + wfs_to_wid(LW'(l), slice_id), stalled_warps[l], to_fullPC(warp_pcs[l]), thread_masks[l]); + end + end + end +`endif + +`ifdef DBG_TRACE_PIPELINE + always @(posedge clk) begin + if (sched_fire) begin + `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b (#%0d)\n", $time, INSTANCE_ID, + wfs_to_wid(cand_lid, slice_id), cand_cta_id, to_fullPC(warp_pcs[cand_lid]), thread_masks[cand_lid], sched_uuid)) + end + end +`endif + +endmodule diff --git a/hw/rtl/core/VX_warp_shared_unit.sv b/hw/rtl/core/VX_warp_shared_unit.sv new file mode 100644 index 0000000000..de5ed3db5f --- /dev/null +++ b/hw/rtl/core/VX_warp_shared_unit.sv @@ -0,0 +1,268 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// The core-wide warp-control resources: barrier unit, wspawn and warp retirement. +// One request per cycle across the requesters, round-robin; a request that +// asks for it is completed back to its requester. Transaction-barrier +// completions use the barrier unit when no request does. +// +// Several requesters register their requests toward the arbiter, which acts on +// the grant and completes it registered. A sole requester is connected +// directly: it is completed in the cycle it asks, and its request is registered +// here instead, so every request takes effect at the same distance from its +// instruction in both cases. +module VX_warp_shared_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter NUM_REQS = 1 +) ( + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + + VX_warp_shared_if.slave warp_shared_if [NUM_REQS], +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_txbar_bus_if.slave txbar_bus_if, +`endif + input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, + + // barrier release + output wire bar_unlock_valid, + output wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask, + VX_gbar_bus_if.master gbar_bus_if, + + // wspawn, applied once a single warp remains + output wire wspawn_apply, + output wire [`VX_CFG_NUM_WARPS-1:0] wspawn_wmask, + output wire [PC_BITS-1:0] wspawn_pc, + output wire [NW_WIDTH-1:0] wspawn_wid, + output wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch, + + // warp retirement + output wire warp_done, + output wire [NW_WIDTH-1:0] warp_done_wid, + + output wire wspawn_pending +); + localparam REQ_DATAW = $bits(warp_shared_req_t); + localparam SEL_W = `LOG2UP(NUM_REQS); + localparam DIRECT = (NUM_REQS == 1); + + wire [NUM_REQS-1:0] req_valid_in; + wire [NUM_REQS-1:0][REQ_DATAW-1:0] req_data_in; + wire [NUM_REQS-1:0] req_ready_in; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_in + assign req_valid_in[i] = warp_shared_if[i].req_valid; + assign req_data_in[i] = warp_shared_if[i].req_data; + assign warp_shared_if[i].req_ready = req_ready_in[i]; + end + + wire req_valid; + warp_shared_req_t req; + wire [SEL_W-1:0] req_sel; + + VX_stream_arb #( + .NUM_INPUTS (NUM_REQS), + .NUM_OUTPUTS (1), + .DATAW (REQ_DATAW), + .ARBITER ("R"), + .OUT_BUF (0) + ) req_arb ( + .clk (clk), + .reset (reset), + .valid_in (req_valid_in), + .ready_in (req_ready_in), + .data_in (req_data_in), + .data_out (req), + .valid_out (req_valid), + .ready_out (1'b1), + .sel_out (req_sel) + ); + + wire is_bar = req_valid && (req.kind == WSH_BAR); + wire is_wspawn = req_valid && (req.kind == WSH_WSPAWN); + wire is_exit = req_valid && (req.kind == WSH_EXIT); + + // wspawn and retirement act on the registered request + localparam ACT_DATAW = NW_WIDTH + $bits(wspawn_t) + `VX_CFG_MEM_ADDR_WIDTH; + wire act_wspawn, act_exit; + wire [NW_WIDTH-1:0] act_wid; + wspawn_t act_wspawn_data; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] act_mscratch; + if (DIRECT) begin : g_act_reg + VX_pipe_register #( + .DATAW (2 + ACT_DATAW), + .RESETW (2) + ) act_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({is_wspawn, is_exit, req.wid, req.wspawn, req.mscratch}), + .data_out ({act_wspawn, act_exit, act_wid, act_wspawn_data, act_mscratch}) + ); + end else begin : g_act + assign {act_wspawn, act_exit, act_wid, act_wspawn_data, act_mscratch} + = {is_wspawn, is_exit, req.wid, req.wspawn, req.mscratch}; + end + + // barrier unit + + wire txbar_req_valid; + barrier_t txbar_req; + wire [BAR_ADDR_W-1:0] bar_read_addr; +`ifdef VX_CFG_EXT_DXA_ENABLE + assign txbar_bus_if.ready = ~is_bar; + wire txbar_fire = txbar_bus_if.valid && txbar_bus_if.ready; + assign bar_read_addr = is_bar ? req.bar_addr : txbar_bus_if.data.addr; + // a completion releases one pending transaction event + barrier_t txbar_req_w; + assign txbar_req_w.id = '0; + assign txbar_req_w.is_event = 1'b1; + assign txbar_req_w.is_sync = 1'b0; + assign txbar_req_w.is_global = 1'b0; + assign txbar_req_w.is_arrive = 1'b0; + assign txbar_req_w.phase = ~txbar_bus_if.data.is_done; + assign txbar_req_w.size_m1 = '0; + VX_pipe_register #( + .DATAW (1 + $bits(barrier_t)), + .RESETW (1) + ) txbar_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({txbar_fire, txbar_req_w}), + .data_out ({txbar_req_valid, txbar_req}) + ); +`else + assign bar_read_addr = req.bar_addr; + assign txbar_req_valid = 1'b0; + assign txbar_req = '0; +`endif + + // the barrier request applies the cycle after its phase is read, as the + // read must see the barrier before this arrival + reg bar_valid_r; + barrier_t bar_r; + reg [NW_WIDTH-1:0] bar_wid_r; + always @(posedge clk) begin + if (reset) begin + bar_valid_r <= 1'b0; + end else begin + bar_valid_r <= is_bar; + end + bar_r <= req.bar; + bar_wid_r <= req.wid; + end + + wire bar_read_phase; + + VX_bar_unit #( + .INSTANCE_ID (`SFORMATF(("%s-barrier", INSTANCE_ID))) + ) bar_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .req_valid (bar_valid_r || txbar_req_valid), + .req_wid (bar_wid_r), + .req_data (bar_valid_r ? bar_r : txbar_req), + .read_addr (bar_read_addr), + .read_phase (bar_read_phase), + .active_warps (active_warps), + .gbar_bus_if (gbar_bus_if), + .unlock_valid (bar_unlock_valid), + .unlock_mask (bar_unlock_mask) + ); + + // wspawn + + reg wspawn_valid_r; + wspawn_t wspawn_r; + reg [NW_WIDTH-1:0] wspawn_wid_r; + reg [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch_r; + reg is_single_warp; + + wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] active_warps_cnt; + `POP_COUNT(active_warps_cnt, active_warps); + + assign wspawn_apply = wspawn_valid_r && is_single_warp; + + always @(posedge clk) begin + if (reset) begin + wspawn_valid_r <= 1'b0; + is_single_warp <= 1'b0; + end else begin + is_single_warp <= (active_warps_cnt == $bits(active_warps_cnt)'(1)); + if (act_wspawn) begin + wspawn_valid_r <= 1'b1; + end + if (wspawn_apply) begin + wspawn_valid_r <= 1'b0; + end + end + if (act_wspawn) begin + wspawn_r <= act_wspawn_data; + wspawn_wid_r <= act_wid; + wspawn_mscratch_r <= act_mscratch; + end + end + + assign wspawn_wmask = wspawn_r.wmask; + assign wspawn_pc = wspawn_r.pc; + assign wspawn_wid = wspawn_wid_r; + assign wspawn_mscratch = wspawn_mscratch_r; + assign wspawn_pending = wspawn_valid_r; + + // warp retirement + + assign warp_done = act_exit; + assign warp_done_wid = act_wid; + + // completion + + wire [NUM_REQS-1:0] rsp_valid = NUM_REQS'(req_valid && req.has_rsp) << req_sel; + wire [NUM_REQS-1:0] rsp_valid_out; + wire rsp_phase_out; + if (DIRECT) begin : g_rsp_direct + assign rsp_valid_out = rsp_valid; + assign rsp_phase_out = bar_read_phase; + end else begin : g_rsp_reg + reg [NUM_REQS-1:0] rsp_valid_r; + reg rsp_phase_r; + always @(posedge clk) begin + if (reset) begin + rsp_valid_r <= '0; + end else begin + rsp_valid_r <= rsp_valid; + end + rsp_phase_r <= bar_read_phase; + end + assign rsp_valid_out = rsp_valid_r; + assign rsp_phase_out = rsp_phase_r; + end + + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_rsp + assign warp_shared_if[i].rsp_valid = rsp_valid_out[i]; + assign warp_shared_if[i].rsp_phase = rsp_phase_out; + end + +`ifdef DBG_TRACE_PIPELINE + always @(posedge clk) begin + if (req_valid) begin + `TRACE(1, ("%t: %s request: sel=%0d, kind=%0d, wid=%0d\n", $time, INSTANCE_ID, req_sel, req.kind, req.wid)) + end + end +`endif + +endmodule diff --git a/hw/rtl/core/VX_wctl_unit.sv b/hw/rtl/core/VX_wctl_unit.sv index f2a3857ab5..7a2a0117de 100644 --- a/hw/rtl/core/VX_wctl_unit.sv +++ b/hw/rtl/core/VX_wctl_unit.sv @@ -22,16 +22,16 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( // Inputs VX_execute_if.slave execute_if, - VX_txbar_bus_if.slave txbar_bus_if, // Outputs VX_warp_ctl_if.master warp_ctl_if, + VX_warp_shared_if.master warp_shared_if, VX_result_if.master result_if ); `UNUSED_SPARAM (INSTANCE_ID) localparam LANE_BITS = `CLOG2(NUM_LANES); localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); - localparam WCTL_WIDTH = $bits(tmc_t) + $bits(wspawn_t) + $bits(split_t) + $bits(join_t) + $bits(barrier_t); + localparam WCTL_WIDTH = $bits(tmc_t) + $bits(split_t) + $bits(join_t); `UNUSED_VAR (execute_if.data.rs3_data) @@ -50,7 +50,7 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire is_wsync = (execute_if.data.op_type == INST_SFU_WSYNC); wire wctl_valid; - wire wspawn_valid, tmc_valid, split_valid, sjoin_valid, bar_valid, wsync_valid; + wire tmc_valid, split_valid, sjoin_valid, wsync_valid; wire [`UP(LANE_BITS)-1:0] last_tid; if (LANE_BITS != 0) begin : g_last_tid @@ -82,9 +82,10 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( logic [`VX_CFG_NUM_THREADS-1:0] else_tmask; if (PID_BITS != 0) begin : g_pid - reg [`VX_CFG_NUM_WARPS-1:0][2*`VX_CFG_NUM_THREADS-1:0] tmask_table; + reg [SFU_BLOCK_WARPS-1:0][2*`VX_CFG_NUM_THREADS-1:0] tmask_table; - wire [2*`VX_CFG_NUM_THREADS-1:0] tmask_r = tmask_table[execute_if.data.header.wid]; + wire [SFU_BLOCK_NW_W-1:0] tmask_lid = wid_to_sfu_local(execute_if.data.header.wid); + wire [2*`VX_CFG_NUM_THREADS-1:0] tmask_r = tmask_table[tmask_lid]; always @(*) begin {else_tmask, then_tmask} = execute_if.data.header.sop ? '0 : tmask_r; @@ -94,7 +95,7 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( always @(posedge clk) begin if (execute_if.valid) begin - tmask_table[execute_if.data.header.wid] <= {else_tmask, then_tmask}; + tmask_table[tmask_lid] <= {else_tmask, then_tmask}; end end end else begin : g_no_pid @@ -108,7 +109,6 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( // tmc / pred wire [`VX_CFG_NUM_THREADS-1:0] pred_mask = has_then ? then_tmask : rs2_data[`VX_CFG_NUM_THREADS-1:0]; - assign tmc_valid = wctl_valid && (is_tmc || is_pred); assign tmc.tmask = is_pred ? pred_mask : rs1_data[`VX_CFG_NUM_THREADS-1:0]; // split @@ -137,62 +137,144 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire [BAR_ADDR_W-1:0] wctl_bar_addr; `CONCAT(wctl_bar_addr, rs1_data[NW_BITS-1:0], rs1_data[BAR_ID_SHIFT +: NB_BITS], NW_BITS, NB_BITS) - wire wctl_bar_enable = wctl_valid && is_bar; - // expect_tx variant of barrier_arrive: rs2[31]=1 flags pre-registration // of pending transaction events. Routed through the existing event path // (is_event + phase=1) so VX_bar_unit's increment branch handles it. - wire is_tx_expect = wctl_bar_enable + wire is_tx_expect = is_bar && execute_if.data.op_args.wctl.is_bar_arrive && rs2_data[31]; - assign bar_valid = wctl_bar_enable || txbar_bus_if.valid; assign bar.id = rs1_data[BAR_ID_SHIFT +: NB_BITS]; - assign bar.is_event = (txbar_bus_if.valid && ~wctl_bar_enable) || is_tx_expect; + assign bar.is_event = is_tx_expect; assign bar.is_sync = execute_if.data.op_args.wctl.is_sync_bar; - assign bar.is_global= wctl_bar_enable && rs1_data[31]; + assign bar.is_global= rs1_data[31]; // expect_tx is not a normal arrival (it does not advance arrive_count) assign bar.is_arrive= (execute_if.data.op_args.wctl.is_bar_arrive && ~rs2_data[31]) || execute_if.data.op_args.wctl.is_sync_bar; - // expect_tx → phase=1 (attach/increment); txbar release → phase=0 - assign bar.phase = is_tx_expect ? 1'b1 : (wctl_bar_enable ? rs2_data[0] : ~txbar_bus_if.data.is_done); + // expect_tx → phase=1 (attach/increment) + assign bar.phase = is_tx_expect ? 1'b1 : rs2_data[0]; // For expect_tx: size_m1 carries (count - 1); for normal arrive: (num_warps - 1). // bar_unit will + 1 when consuming on the event path so the actual increment is `count`. assign bar.size_m1 = rs2_data[BAR_SIZE_W-1:0] - BAR_SIZE_W'(1); - assign txbar_bus_if.ready = ~wctl_bar_enable; - // wspawn wire [`VX_CFG_NUM_WARPS-1:0] wspawn_wmask; for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_wspawn_wmask assign wspawn_wmask[i] = (i < rs1_data[NW_BITS:0]) && (i != execute_if.data.header.wid); end - assign wspawn_valid = wctl_valid && is_wspawn; assign wspawn.wmask = wspawn_wmask; assign wspawn.pc = from_fullPC(rs2_data); // Scheduler response setup assign warp_ctl_if.dvstack_wid = execute_if.data.header.wid; - assign warp_ctl_if.bar_addr = wctl_bar_enable ? wctl_bar_addr : txbar_bus_if.data.addr; - - // Send scheduler request // WSYNC: hold execute until this warp's pipeline is drained (alm_empty=1 // means only WSYNC itself is pending — all prior instructions committed). wire wsync_drain = execute_if.valid && is_wsync - && !warp_ctl_if.warp_pending_alm_empty[execute_if.data.header.wid]; + && !warp_ctl_if.warp_pending_alm_empty[wid_to_sfu_local(execute_if.data.header.wid)]; // BAR (vx_barrier or vx_barrier_arrive) drains LSU before continuing. wire bar_drain = execute_if.valid && is_bar && !warp_ctl_if.lsu_sched_drained; + // Barriers, wspawn and warp retirement are core-wide requests to the shared + // unit. Only a barrier arrive writes a result, the barrier phase, so it alone + // completes once the unit has served it; every other request is posted, + // completing as it is accepted, and the warp resumes through its unlock path. + wire bar_rsp_valid = is_bar && execute_if.data.op_args.wctl.is_bar_arrive; + wire is_exit = (is_tmc || is_pred) && (tmc.tmask == 0); + wire is_shared = execute_if.data.header.eop && (is_bar || is_wspawn || is_exit); + wire sh_posted = is_shared && ~bar_rsp_valid; + + // an exit is retired by the shared unit, which deactivates the warp itself + assign tmc_valid = wctl_valid && (is_tmc || is_pred) && !is_shared; + + localparam SH_IDLE = 2'd0, SH_WAIT = 2'd1, SH_DONE = 2'd2; + reg [1:0] sh_state; + reg sh_phase; + + warp_shared_req_t sh_req; + assign sh_req.kind = is_bar ? WSH_BAR : (is_wspawn ? WSH_WSPAWN : WSH_EXIT); + assign sh_req.has_rsp = ~sh_posted; + assign sh_req.wid = execute_if.data.header.wid; + assign sh_req.bar = bar; + assign sh_req.bar_addr = wctl_bar_addr; + assign sh_req.wspawn = wspawn; + assign sh_req.mscratch = warp_ctl_if.mscratch; + + // A sole requester reaches the shared unit directly and is answered in the + // cycle it asks; with several, requests are registered toward the unit's + // arbiter and answered two cycles later. + localparam SH_DIRECT = (`VX_CFG_NUM_SFU_BLOCKS == 1); + + wire rsp_buf_ready; + // a posted or directly answered request is sent only in the cycle its instruction completes + wire sh_req_valid = execute_if.valid && is_shared && (sh_state == SH_IDLE) && !bar_drain + && ((!sh_posted && !SH_DIRECT) || rsp_buf_ready); + wire sh_req_ready; + + if (SH_DIRECT) begin : g_sh_direct + assign warp_shared_if.req_valid = sh_req_valid; + assign warp_shared_if.req_data = sh_req; + assign sh_req_ready = warp_shared_if.req_ready; + end else begin : g_sh_buf + VX_elastic_buffer #( + .DATAW ($bits(warp_shared_req_t)), + .SIZE (2), + .OUT_REG (1) + ) sh_req_buf ( + .clk (clk), + .reset (reset), + .valid_in (sh_req_valid), + .ready_in (sh_req_ready), + .data_in (sh_req), + .data_out (warp_shared_if.req_data), + .valid_out (warp_shared_if.req_valid), + .ready_out (warp_shared_if.req_ready) + ); + end + + wire sh_req_fire = sh_req_valid && sh_req_ready; + wire sh_ok = ~is_shared || (sh_state == SH_DONE) + || ((sh_state == SH_WAIT) && warp_shared_if.rsp_valid) + || (sh_req_fire && (sh_posted || warp_shared_if.rsp_valid)); + wire exec_ok = rsp_buf_ready && !wsync_drain && sh_ok && (is_shared || !bar_drain); + assign execute_if.ready = exec_ok; + wire execute_fire = execute_if.valid && execute_if.ready; assign wctl_valid = execute_fire && execute_if.data.header.eop; + always @(posedge clk) begin + if (reset) begin + sh_state <= SH_IDLE; + end else begin + case (sh_state) + SH_IDLE: begin + if (sh_req_fire && !sh_posted && !execute_fire) begin + sh_state <= warp_shared_if.rsp_valid ? SH_DONE : SH_WAIT; + end + end + SH_WAIT: begin + if (warp_shared_if.rsp_valid) begin + sh_state <= execute_fire ? SH_IDLE : SH_DONE; + end + end + default: begin + if (execute_fire) begin + sh_state <= SH_IDLE; + end + end + endcase + end + if (warp_shared_if.rsp_valid) begin + sh_phase <= warp_shared_if.rsp_phase; + end + end + assign wsync_valid = wctl_valid && is_wsync; VX_pipe_register #( - .DATAW (6 + NW_WIDTH + WCTL_WIDTH), + .DATAW (4 + NW_WIDTH + WCTL_WIDTH), .RESETW (1) ) wctl_reg ( .clk (clk), @@ -200,56 +282,46 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( .enable (1'b1), .data_in ({ tmc_valid, - wspawn_valid, split_valid, sjoin_valid, - bar_valid, wsync_valid, execute_if.data.header.wid, tmc, - wspawn, split, - sjoin, - bar + sjoin }), .data_out ({ warp_ctl_if.tmc_valid, - warp_ctl_if.wspawn_valid, warp_ctl_if.split_valid, warp_ctl_if.sjoin_valid, - warp_ctl_if.bar_valid, warp_ctl_if.wsync_valid, warp_ctl_if.wid, warp_ctl_if.tmc, - warp_ctl_if.wspawn, warp_ctl_if.split, - warp_ctl_if.sjoin, - warp_ctl_if.bar + warp_ctl_if.sjoin }) ); // Send result + wire rsp_phase = warp_shared_if.rsp_valid ? warp_shared_if.rsp_phase : sh_phase; wire [DV_STACK_SIZEW-1:0] dvstack_ptr_r; - wire bar_rsp_valid = is_bar && execute_if.data.op_args.wctl.is_bar_arrive; wire bar_rsp_valid_r; wire bar_phase_r; - wire rsp_buf_ready; VX_elastic_buffer #( .DATAW ($bits(sfu_header_t) + DV_STACK_SIZEW + 1 + 1), .SIZE (2) ) rsp_buf ( .clk (clk), .reset (reset), - .valid_in (execute_if.valid && !wsync_drain && !bar_drain), + .valid_in (execute_if.valid && exec_ok), .ready_in (rsp_buf_ready), - .data_in ({execute_if.data.header, warp_ctl_if.dvstack_ptr, warp_ctl_if.bar_phase, bar_rsp_valid}), - .data_out ({result_if.data.header, dvstack_ptr_r, bar_phase_r, bar_rsp_valid_r}), + .data_in ({execute_if.data.header, warp_ctl_if.dvstack_ptr, rsp_phase, bar_rsp_valid}), + .data_out ({result_if.data.header, dvstack_ptr_r, bar_phase_r, bar_rsp_valid_r}), .valid_out (result_if.valid), .ready_out (result_if.ready) ); - assign execute_if.ready = rsp_buf_ready && !wsync_drain && !bar_drain; // Result data for (genvar i = 0; i < NUM_LANES; ++i) begin : g_result_if diff --git a/hw/rtl/dxa/VX_dxa_unit.sv b/hw/rtl/dxa/VX_dxa_unit.sv index ed9136fd95..cd3d8c8b13 100644 --- a/hw/rtl/dxa/VX_dxa_unit.sv +++ b/hw/rtl/dxa/VX_dxa_unit.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + VX_execute_if.slave execute_if, VX_result_if.master result_if, VX_dxa_req_bus_if.master dxa_req_bus_if @@ -58,7 +59,10 @@ module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Build dxa_req payload dxa_req_data_t dxa_req_data_in; - assign dxa_req_data_in.core_id = NC_WIDTH'(CORE_ID); + assign dxa_req_data_in.core_id = NC_WIDTH'(core_id); + if (CORE_ID_WIDTH > NC_WIDTH) begin : g_core_id_hi_unused + `UNUSED_VAR (core_id[CORE_ID_WIDTH-1:NC_WIDTH]) + end assign dxa_req_data_in.uuid = execute_if.data.header.uuid; assign dxa_req_data_in.wid = execute_if.data.header.wid; assign dxa_req_data_in.smem_addr = lmem_rel_byte_addr[DXA_SMEM_ADDR_W-1:0]; diff --git a/hw/rtl/dxa/VX_dxa_worker.sv b/hw/rtl/dxa/VX_dxa_worker.sv index 8f2a9500ea..fd8f0d5e25 100644 --- a/hw/rtl/dxa/VX_dxa_worker.sv +++ b/hw/rtl/dxa/VX_dxa_worker.sv @@ -45,6 +45,8 @@ module VX_dxa_worker import VX_gpu_pkg::*, VX_dxa_pkg::*; #( localparam MAX_OUTSTANDING = `VX_CFG_DXA_MAX_INFLIGHT; localparam TAG_W = `CLOG2(MAX_OUTSTANDING); localparam SEQ_W = `CLOG2(MAX_OUTSTANDING + 1); + // the outstanding-request queue and its tags need a power of two of at least 2 + `STATIC_ASSERT (`IS_POW2(MAX_OUTSTANDING) && (MAX_OUTSTANDING >= 2), ("invalid parameter")) // ════════════════════════════════════════════════════════════════════ // Inter-module wires diff --git a/hw/rtl/fpu/VX_fpu_block.sv b/hw/rtl/fpu/VX_fpu_block.sv new file mode 100644 index 0000000000..eb97c427e6 --- /dev/null +++ b/hw/rtl/fpu/VX_fpu_block.sv @@ -0,0 +1,261 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_fpu_define.vh" + +// One FPU block: the FP datapath with its tag store and fflags accumulation. +module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 1, + parameter PARTIAL_BW = 0 +) ( + input wire clk, + input wire reset, + + input wire [`LOG2UP(`VX_CFG_NUM_FPU_BLOCKS)-1:0] block_idx, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if, + VX_fpu_csr_if.master fpu_csr_if +); + `UNUSED_SPARAM (INSTANCE_ID) + localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); + localparam TAG_WIDTH = `LOG2UP(`VX_CFG_FPU_QUEUE_SIZE); + + wire fpu_req_valid, fpu_req_ready; + wire fpu_rsp_valid, fpu_rsp_ready; + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_rsp_result; + fflags_t fpu_rsp_fflags; + wire fpu_rsp_has_fflags; + + fpu_header_t fpu_hdr, fpu_hdr_wb, fpu_hdr_store; + + // Carry the decoded wb through the tag store: FPU compares may target + // integer x0 (e.g. feq.s x0, ...), which decode does not reserve in the + // scoreboard, so forcing wb=1 on readback would trip the invalid + // writeback assertion. + always_comb begin + fpu_hdr_store = execute_if.data.header; + end + + always_comb begin + fpu_hdr_wb = fpu_hdr; + end + + wire [TAG_WIDTH-1:0] fpu_req_tag, fpu_rsp_tag; + wire mdata_full; + + wire [INST_FMT_BITS-1:0] fpu_fmt = execute_if.data.op_args.fpu.fmt; + wire [INST_FRM_BITS-1:0] fpu_frm = execute_if.data.op_args.fpu.frm; + + wire execute_fire = execute_if.valid && execute_if.ready; + wire fpu_rsp_fire = fpu_rsp_valid && fpu_rsp_ready; + + VX_index_buffer #( + .DATAW ($bits(fpu_header_t)), + .SIZE (`VX_CFG_FPU_QUEUE_SIZE) + ) tag_store ( + .clk (clk), + .reset (reset), + .acquire_en (execute_fire), + .write_addr (fpu_req_tag), + .write_data (fpu_hdr_store), + .read_data (fpu_hdr), + .read_addr (fpu_rsp_tag), + .release_en (fpu_rsp_fire), + .full (mdata_full), + `UNUSED_PIN (empty) + ); + + // resolve dynamic FRM from CSR + wire [INST_FRM_BITS-1:0] fpu_req_frm; + `ASSIGN_BLOCKED_WID (fpu_csr_if.read_wid, execute_if.data.header.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) + if (`VX_CFG_NUM_FPU_BLOCKS == 1) begin : g_block_idx_unused + `UNUSED_VAR (block_idx) + end + assign fpu_req_frm = (execute_if.data.op_type != INST_FPU_MISC + && fpu_frm == INST_FRM_DYN) ? fpu_csr_if.read_frm : fpu_frm; + + // submit FPU request + + assign fpu_req_valid = execute_if.valid && ~mdata_full; + assign execute_if.ready = fpu_req_ready && ~mdata_full; + +`ifdef VX_CFG_FPU_TYPE_DPI + + VX_fpu_dpi #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_dpi ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`elsif VX_CFG_FPU_TYPE_FPNEW + + VX_fpu_fpnew #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_fpnew ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`elsif VX_CFG_FPU_TYPE_DSP + + VX_fpu_dsp #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_dsp ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`elsif VX_CFG_FPU_TYPE_STD + + VX_fpu_std #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_std ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`endif + + // handle CSR update + fflags_t fpu_rsp_fflags_q; + + if (PID_BITS != 0) begin : g_fflags_pid + fflags_t fpu_rsp_fflags_r; + always @(posedge clk) begin + if (reset) begin + fpu_rsp_fflags_r <= '0; + end else if (fpu_rsp_fire) begin + fpu_rsp_fflags_r <= fpu_hdr.eop ? '0 : (fpu_rsp_fflags_r | fpu_rsp_fflags); + end + end + assign fpu_rsp_fflags_q = fpu_rsp_fflags_r | fpu_rsp_fflags; + end else begin : g_fflags_no_pid + assign fpu_rsp_fflags_q = fpu_rsp_fflags; + end + + VX_fpu_csr_if fpu_csr_tmp_if(); + assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_hdr.eop && fpu_rsp_has_fflags; + `ASSIGN_BLOCKED_WID (fpu_csr_tmp_if.write_wid, fpu_hdr.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) + assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags_q; + + VX_pipe_register #( + .DATAW (1 + NW_WIDTH + $bits(fflags_t)), + .RESETW (1) + ) fpu_csr_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({fpu_csr_tmp_if.write_enable, fpu_csr_tmp_if.write_wid, fpu_csr_tmp_if.write_fflags}), + .data_out ({fpu_csr_if.write_enable, fpu_csr_if.write_wid, fpu_csr_if.write_fflags}) + ); + + // send response + + VX_elastic_buffer #( + .DATAW ($bits(fpu_header_t) + (NUM_LANES * `VX_CFG_XLEN)), + .SIZE (0) + ) rsp_buf ( + .clk (clk), + .reset (reset), + .valid_in (fpu_rsp_valid), + .ready_in (fpu_rsp_ready), + .data_in ({fpu_hdr_wb, fpu_rsp_result}), + .data_out (result_if.data), + .valid_out (result_if.valid), + .ready_out (result_if.ready) + ); + +endmodule diff --git a/hw/rtl/fpu/VX_fpu_unit.sv b/hw/rtl/fpu/VX_fpu_unit.sv index 9e2386d54f..b64bf5825e 100644 --- a/hw/rtl/fpu/VX_fpu_unit.sv +++ b/hw/rtl/fpu/VX_fpu_unit.sv @@ -29,8 +29,6 @@ module VX_fpu_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) localparam BLOCK_SIZE = `VX_CFG_NUM_FPU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_FPU_LANES; - localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); - localparam TAG_WIDTH = `LOG2UP(`VX_CFG_FPU_QUEUE_SIZE); localparam PARTIAL_BW = (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) || (NUM_LANES != `VX_CFG_SIMD_WIDTH); VX_execute_if #( @@ -44,6 +42,7 @@ module VX_fpu_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); @@ -53,227 +52,17 @@ module VX_fpu_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( ) per_block_result_if[BLOCK_SIZE](); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - wire fpu_req_valid, fpu_req_ready; - wire fpu_rsp_valid, fpu_rsp_ready; - wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_rsp_result; - fflags_t fpu_rsp_fflags; - wire fpu_rsp_has_fflags; - - fpu_header_t fpu_hdr, fpu_hdr_wb, fpu_hdr_store; - - // Carry the decoded wb through the tag store: FPU compares may target - // integer x0 (e.g. feq.s x0, ...), which decode does not reserve in the - // scoreboard, so forcing wb=1 on readback would trip the invalid - // writeback assertion. - always_comb begin - fpu_hdr_store = per_block_execute_if[block_idx].data.header; - end - - always_comb begin - fpu_hdr_wb = fpu_hdr; - end - - wire [TAG_WIDTH-1:0] fpu_req_tag, fpu_rsp_tag; - wire mdata_full; - - wire [INST_FMT_BITS-1:0] fpu_fmt = per_block_execute_if[block_idx].data.op_args.fpu.fmt; - wire [INST_FRM_BITS-1:0] fpu_frm = per_block_execute_if[block_idx].data.op_args.fpu.frm; - - wire execute_fire = per_block_execute_if[block_idx].valid && per_block_execute_if[block_idx].ready; - wire fpu_rsp_fire = fpu_rsp_valid && fpu_rsp_ready; - - VX_index_buffer #( - .DATAW ($bits(fpu_header_t)), - .SIZE (`VX_CFG_FPU_QUEUE_SIZE) - ) tag_store ( - .clk (clk), - .reset (reset), - .acquire_en (execute_fire), - .write_addr (fpu_req_tag), - .write_data (fpu_hdr_store), - .read_data (fpu_hdr), - .read_addr (fpu_rsp_tag), - .release_en (fpu_rsp_fire), - .full (mdata_full), - `UNUSED_PIN (empty) - ); - - // resolve dynamic FRM from CSR - wire [INST_FRM_BITS-1:0] fpu_req_frm; - `ASSIGN_BLOCKED_WID (fpu_csr_if[block_idx].read_wid, per_block_execute_if[block_idx].data.header.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) - assign fpu_req_frm = (per_block_execute_if[block_idx].data.op_type != INST_FPU_MISC - && fpu_frm == INST_FRM_DYN) ? fpu_csr_if[block_idx].read_frm : fpu_frm; - - // submit FPU request - - assign fpu_req_valid = per_block_execute_if[block_idx].valid && ~mdata_full; - assign per_block_execute_if[block_idx].ready = fpu_req_ready && ~mdata_full; - - `ifdef VX_CFG_FPU_TYPE_DPI - - VX_fpu_dpi #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_dpi ( + VX_fpu_block #( + .INSTANCE_ID (INSTANCE_ID), + .NUM_LANES (NUM_LANES), + .PARTIAL_BW (PARTIAL_BW) + ) fpu_block ( .clk (clk), .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `elsif VX_CFG_FPU_TYPE_FPNEW - - VX_fpu_fpnew #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_fpnew ( - .clk (clk), - .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `elsif VX_CFG_FPU_TYPE_DSP - - VX_fpu_dsp #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_dsp ( - .clk (clk), - .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `elsif VX_CFG_FPU_TYPE_STD - - VX_fpu_std #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_std ( - .clk (clk), - .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `endif - - // handle CSR update - fflags_t fpu_rsp_fflags_q; - - if (PID_BITS != 0) begin : g_fflags_pid - fflags_t fpu_rsp_fflags_r; - always @(posedge clk) begin - if (reset) begin - fpu_rsp_fflags_r <= '0; - end else if (fpu_rsp_fire) begin - fpu_rsp_fflags_r <= fpu_hdr.eop ? '0 : (fpu_rsp_fflags_r | fpu_rsp_fflags); - end - end - assign fpu_rsp_fflags_q = fpu_rsp_fflags_r | fpu_rsp_fflags; - end else begin : g_fflags_no_pid - assign fpu_rsp_fflags_q = fpu_rsp_fflags; - end - - VX_fpu_csr_if fpu_csr_tmp_if(); - assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_hdr.eop && fpu_rsp_has_fflags; - `ASSIGN_BLOCKED_WID (fpu_csr_tmp_if.write_wid, fpu_hdr.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) - assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags_q; - - VX_pipe_register #( - .DATAW (1 + NW_WIDTH + $bits(fflags_t)), - .RESETW (1) - ) fpu_csr_reg ( - .clk (clk), - .reset (reset), - .enable (1'b1), - .data_in ({fpu_csr_tmp_if.write_enable, fpu_csr_tmp_if.write_wid, fpu_csr_tmp_if.write_fflags}), - .data_out ({fpu_csr_if[block_idx].write_enable, fpu_csr_if[block_idx].write_wid, fpu_csr_if[block_idx].write_fflags}) - ); - - // send response - - VX_elastic_buffer #( - .DATAW ($bits(fpu_header_t) + (NUM_LANES * `VX_CFG_XLEN)), - .SIZE (0) - ) rsp_buf ( - .clk (clk), - .reset (reset), - .valid_in (fpu_rsp_valid), - .ready_in (fpu_rsp_ready), - .data_in ({fpu_hdr_wb, fpu_rsp_result}), - .data_out (per_block_result_if[block_idx].data), - .valid_out (per_block_result_if[block_idx].valid), - .ready_out (per_block_result_if[block_idx].ready) + .block_idx (`LOG2UP(BLOCK_SIZE)'(block_idx)), + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]), + .fpu_csr_if (fpu_csr_if[block_idx]) ); end diff --git a/hw/rtl/interfaces/VX_sched_csr_if.sv b/hw/rtl/interfaces/VX_sched_csr_if.sv index 5ac508cbfc..d773df084d 100644 --- a/hw/rtl/interfaces/VX_sched_csr_if.sv +++ b/hw/rtl/interfaces/VX_sched_csr_if.sv @@ -18,7 +18,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); wire [PERF_CTR_BITS-1:0] cycles; wire [PERF_CTR_BITS-1:0] instret; wire [`VX_CFG_NUM_WARPS-1:0] active_warps; - wire [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_NUM_THREADS-1:0] thread_masks; + wire [`VX_CFG_NUM_THREADS-1:0] thread_mask; // selected by csr_rd_wid // Read port: slave sends wid + cta_id, master returns selected mscratch // and cta_csrs. csr_rd_cta_id must carry the executing instruction's @@ -33,6 +33,8 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); `ifdef VX_CFG_VM_ENABLE logic [`VX_CFG_XLEN-1:0] csr_satp; + logic satp_wr_valid; + logic [`VX_CFG_XLEN-1:0] satp_wr_data; `endif // Write port: slave notifies scheduler of MSCRATCH CSR writes @@ -57,7 +59,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output cycles, output instret, output active_warps, - output thread_masks, + output thread_mask, output mscratch, output cta_csrs, output cta_lane, @@ -67,7 +69,9 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output csr_mcause, output csr_mtval, `ifdef VX_CFG_VM_ENABLE - input csr_satp, + output csr_satp, + input satp_wr_valid, + input satp_wr_data, `endif input csr_rd_wid, input csr_rd_cta_id, @@ -82,7 +86,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input cycles, input instret, input active_warps, - input thread_masks, + input thread_mask, input mscratch, input cta_csrs, input cta_lane, @@ -92,7 +96,9 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input csr_mcause, input csr_mtval, `ifdef VX_CFG_VM_ENABLE - output csr_satp, + input csr_satp, + output satp_wr_valid, + output satp_wr_data, `endif output csr_rd_wid, output csr_rd_cta_id, diff --git a/hw/rtl/interfaces/VX_slice_sched_if.sv b/hw/rtl/interfaces/VX_slice_sched_if.sv new file mode 100644 index 0000000000..c5e056ab43 --- /dev/null +++ b/hw/rtl/interfaces/VX_slice_sched_if.sv @@ -0,0 +1,151 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Core scheduler state exchanged with one fetch slice's warp scheduler. +// Core-wide events arrive already addressed to the slice: warp ids are +// slice-local (wid = {wfs, fsw}) and per-warp masks hold only its warps. +interface VX_slice_sched_if import VX_gpu_pkg::*; (); + + localparam SW = PER_FETCH_WARPS; + localparam LW = FETCH_WFS_W; + + // CTA launch + logic cta_fire; + logic [LW-1:0] cta_lid; + logic [PC_BITS-1:0] cta_PC; + logic [`VX_CFG_NUM_THREADS-1:0] cta_tmask; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param; + logic cta_init; + + // CTA-CSR store writes + logic warp_write; + logic [LW-1:0] warp_waddr; + cta_warp_t warp_wdata; + logic id_write; + logic [LW-1:0] id_waddr; + logic [NCTA_WIDTH-1:0] id_wdata; + + // wspawn + logic wspawn_apply; + logic [SW-1:0] wspawn_wmask; + logic [PC_BITS-1:0] wspawn_pc; + logic wspawn_src; // the spawning warp is in this slice + logic [LW-1:0] wspawn_src_lid; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch; + + // barrier release + logic bar_unlock_valid; + logic [SW-1:0] bar_unlock_mask; + + // warp retirement, as the core-wide CTA accounting takes it + logic retire_valid; + logic [LW-1:0] retire_lid; + + // every warp of the core has a full ibuffer + logic all_ibuf_full; +`ifdef EXT_SCHED_TIMEOUT_DUMP + // the core-wide stall timeout is about to fire + logic timeout_dump; +`endif + + // slice status and performance events + logic [SW-1:0] active; + logic [SW-1:0] stalled; + logic [SW-1:0] pending_empty; + // next-state ibuffer full, registered core-wide into all_ibuf_full + logic [SW-1:0] ibuf_full_n; + logic [FETCH_ALU_PORTS-1:0] branch_valid; + logic [FETCH_SFU_PORTS-1:0] divergence; + logic sched_valid; + logic sched_fire; + logic [`VX_CFG_NUM_THREADS-1:0] sched_tmask; + + modport master ( + output cta_fire, + output cta_lid, + output cta_PC, + output cta_tmask, + output cta_param, + output cta_init, + output warp_write, + output warp_waddr, + output warp_wdata, + output id_write, + output id_waddr, + output id_wdata, + output wspawn_apply, + output wspawn_wmask, + output wspawn_pc, + output wspawn_src, + output wspawn_src_lid, + output wspawn_mscratch, + output bar_unlock_valid, + output bar_unlock_mask, + output retire_valid, + output retire_lid, + output all_ibuf_full, +`ifdef EXT_SCHED_TIMEOUT_DUMP + output timeout_dump, +`endif + input active, + input stalled, + input ibuf_full_n, + input pending_empty, + input branch_valid, + input divergence, + input sched_valid, + input sched_fire, + input sched_tmask + ); + + modport slave ( + input cta_fire, + input cta_lid, + input cta_PC, + input cta_tmask, + input cta_param, + input cta_init, + input warp_write, + input warp_waddr, + input warp_wdata, + input id_write, + input id_waddr, + input id_wdata, + input wspawn_apply, + input wspawn_wmask, + input wspawn_pc, + input wspawn_src, + input wspawn_src_lid, + input wspawn_mscratch, + input bar_unlock_valid, + input bar_unlock_mask, + input retire_valid, + input retire_lid, + input all_ibuf_full, +`ifdef EXT_SCHED_TIMEOUT_DUMP + input timeout_dump, +`endif + output active, + output stalled, + output ibuf_full_n, + output pending_empty, + output branch_valid, + output divergence, + output sched_valid, + output sched_fire, + output sched_tmask + ); + +endinterface diff --git a/hw/rtl/interfaces/VX_warp_ctl_if.sv b/hw/rtl/interfaces/VX_warp_ctl_if.sv index 9da68980c9..0e49757335 100644 --- a/hw/rtl/interfaces/VX_warp_ctl_if.sv +++ b/hw/rtl/interfaces/VX_warp_ctl_if.sv @@ -15,23 +15,17 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); - wire wspawn_valid; wire tmc_valid; wire split_valid; wire sjoin_valid; - wire bar_valid; wire wsync_valid; - wire [NW_WIDTH-1:0] wid; - tmc_t tmc; - wspawn_t wspawn; split_t split; join_t sjoin; - barrier_t bar; - // Per-warp pipeline drain status (from scheduler) - wire [`VX_CFG_NUM_WARPS-1:0] warp_pending_alm_empty; + // Pipeline drain status of the scheduler's warps, by SFU-block-local warp id + wire [SFU_BLOCK_WARPS-1:0] warp_pending_alm_empty; // LSU memory-scheduler drained signal (AND-reduced across all LSU blocks). // Used by VX_wctl_unit to gate BAR (both vx_barrier and vx_barrier_arrive) @@ -40,64 +34,41 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); // the CUDA __syncthreads / OpenCL barrier(CLK_LOCAL_MEM_FENCE) contract. wire lsu_sched_drained; - // Return IPDOM stack address + // IPDOM stack address and mscratch of the warp at dvstack_wid wire [NW_WIDTH-1:0] dvstack_wid; wire [DV_STACK_SIZEW-1:0] dvstack_ptr; - - // Return barrier phase - wire [BAR_ADDR_W-1:0] bar_addr; - wire bar_phase; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; modport master ( - output wspawn_valid, output tmc_valid, output split_valid, output sjoin_valid, - output bar_valid, output wsync_valid, - output wid, - - output wspawn, output tmc, output split, output sjoin, - output bar, - - output bar_addr, - input bar_phase, - input warp_pending_alm_empty, input lsu_sched_drained, - output dvstack_wid, - input dvstack_ptr + input dvstack_ptr, + input mscratch ); modport slave ( - input wspawn_valid, - input tmc_valid, - input split_valid, - input sjoin_valid, - input bar_valid, - input wsync_valid, - - input wid, - - input wspawn, - input tmc, - input split, - input sjoin, - input bar, - - input bar_addr, - output bar_phase, - + input tmc_valid, + input split_valid, + input sjoin_valid, + input wsync_valid, + input wid, + input tmc, + input split, + input sjoin, output warp_pending_alm_empty, output lsu_sched_drained, - - input dvstack_wid, - output dvstack_ptr + input dvstack_wid, + output dvstack_ptr, + output mscratch ); endinterface diff --git a/hw/rtl/interfaces/VX_warp_shared_if.sv b/hw/rtl/interfaces/VX_warp_shared_if.sv new file mode 100644 index 0000000000..86d1f13786 --- /dev/null +++ b/hw/rtl/interfaces/VX_warp_shared_if.sv @@ -0,0 +1,44 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// A core-wide warp-control request and its completion. A requester waits on at +// most one request with has_rsp set; rsp_valid completes it and carries the +// barrier phase. Other requests are posted and get no completion. +interface VX_warp_shared_if import VX_gpu_pkg::*; (); + + logic req_valid; + warp_shared_req_t req_data; + logic req_ready; + + logic rsp_valid; + logic rsp_phase; + + modport master ( + output req_valid, + output req_data, + input req_ready, + input rsp_valid, + input rsp_phase + ); + + modport slave ( + input req_valid, + input req_data, + output req_ready, + output rsp_valid, + output rsp_phase + ); + +endinterface diff --git a/hw/rtl/om/VX_om_core.sv b/hw/rtl/om/VX_om_core.sv index 459708e0d6..f8f610d39f 100644 --- a/hw/rtl/om/VX_om_core.sv +++ b/hw/rtl/om/VX_om_core.sv @@ -51,7 +51,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( om_rt_dcrs_t om_rt_dcrs [`VX_OM_MAX_RT]; VX_om_dcr #( - .INSTANCE_ID ($sformatf("%s-dcr", INSTANCE_ID)) + .INSTANCE_ID (`SFORMATF(("%s-dcr", INSTANCE_ID))) ) om_dcr ( .clk (clk), .reset (reset), @@ -88,7 +88,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( `UNUSED_VAR (mem_write_notify) VX_om_mem #( - .INSTANCE_ID ($sformatf("%s-mem", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-mem", INSTANCE_ID))), .NUM_LANES (NUM_LANES), .TAG_WIDTH (MEM_TAG_WIDTH) ) om_mem ( @@ -144,7 +144,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( wire [NUM_LANES-1:0] ds_pass_out; VX_om_ds #( - .INSTANCE_ID ($sformatf("%s-ds", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-ds", INSTANCE_ID))), .NUM_LANES (NUM_LANES), .TAG_WIDTH (DS_TAG_WIDTH) ) om_ds ( @@ -187,7 +187,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( wire [OM_RT_IDX_BITS-1:0] mem_rsp_rt; VX_om_blend #( - .INSTANCE_ID ($sformatf("%s-blend", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-blend", INSTANCE_ID))), .NUM_LANES (NUM_LANES), .TAG_WIDTH (BLEND_TAG_WIDTH) ) om_blend ( diff --git a/hw/rtl/om/VX_om_mem.sv b/hw/rtl/om/VX_om_mem.sv index 67cc03cb6f..45841002fb 100644 --- a/hw/rtl/om/VX_om_mem.sv +++ b/hw/rtl/om/VX_om_mem.sv @@ -245,7 +245,7 @@ module VX_om_mem import VX_gpu_pkg::*; import VX_om_pkg::*; #( ) lsu_mem_if(); VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (NUM_REQS), .MEM_CHANNELS (MEM_CHANNELS), .WORD_SIZE (4), diff --git a/hw/rtl/om/VX_om_steer.sv b/hw/rtl/om/VX_om_steer.sv index 0fd8ce9c8e..26310185c8 100644 --- a/hw/rtl/om/VX_om_steer.sv +++ b/hw/rtl/om/VX_om_steer.sv @@ -23,7 +23,7 @@ // // The select is the is_addr_om attr bit alone. No cache level decodes it -- the // aperture store is presented to the caches as an ordinary IO store (they carry -// attr through verbatim), so this bit arrives untouched. See VX_lsu_slice. +// attr through verbatim), so this bit arrives untouched. See VX_lsu_block. // // Flush ordering: the steer is a fork, so a flush token bound for L2 is held // here until the OM side has been flushed. Every aperture store this trunk diff --git a/hw/rtl/raster/VX_raster_be.sv b/hw/rtl/raster/VX_raster_be.sv index e33ca50d9c..8426c46cf0 100644 --- a/hw/rtl/raster/VX_raster_be.sv +++ b/hw/rtl/raster/VX_raster_be.sv @@ -119,7 +119,7 @@ module VX_raster_be import VX_raster_pkg::*; #( wire [PER_BLOCK_QUADS-1:0][`VX_RASTER_DIM_BITS-1:0] qe_yloc; VX_raster_qe #( - .INSTANCE_ID ($sformatf("%s-qe", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-qe", INSTANCE_ID))), .NUM_QUADS (PER_BLOCK_QUADS) ) quad_evaluator ( .clk (clk), diff --git a/hw/rtl/raster/VX_raster_core.sv b/hw/rtl/raster/VX_raster_core.sv index 470e8ea205..be08681dbc 100644 --- a/hw/rtl/raster/VX_raster_core.sv +++ b/hw/rtl/raster/VX_raster_core.sv @@ -17,7 +17,6 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter INSTANCE_IDX = 0, parameter NUM_INSTANCES = 1, parameter NUM_SLICES = 1, // number of slices parameter TILE_LOGSIZE = 5, // tile log size @@ -32,6 +31,8 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( input wire clk, input wire reset, + input wire [`LOG2UP(NUM_INSTANCES)-1:0] instance_idx, + // PERF `ifdef PERF_ENABLE VX_raster_perf_if.master perf_raster_if, @@ -79,7 +80,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( raster_dcrs_t raster_dcrs; VX_raster_dcr #( - .INSTANCE_ID ($sformatf("%s-dcr", INSTANCE_ID)) + .INSTANCE_ID (`SFORMATF(("%s-dcr", INSTANCE_ID))) ) raster_dcr ( .clk (clk), .reset (reset), @@ -134,14 +135,14 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( // Memory unit VX_raster_mem #( - .INSTANCE_ID ($sformatf("%s-mem", INSTANCE_ID)), - .INSTANCE_IDX (INSTANCE_IDX), + .INSTANCE_ID (`SFORMATF(("%s-mem", INSTANCE_ID))), .NUM_INSTANCES (NUM_INSTANCES), .TILE_LOGSIZE (TILE_LOGSIZE), .QUEUE_SIZE (MEM_FIFO_DEPTH) ) raster_mem ( .clk (clk), .reset (reset), + .instance_idx (instance_idx), .start (mem_unit_start), .busy (mem_unit_busy), @@ -330,7 +331,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `endif VX_raster_slice #( - .INSTANCE_ID ($sformatf("%s-slice%d", INSTANCE_ID, slice_id)), + .INSTANCE_ID (`SFORMATF(("%s-slice%d", INSTANCE_ID, slice_id))), .TILE_LOGSIZE (TILE_LOGSIZE), .BLOCK_LOGSIZE (BLOCK_LOGSIZE), .OUTPUT_QUADS (OUTPUT_QUADS), @@ -372,7 +373,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( // depth read from the ocache (elastic, variable-latency). Pass-through // when the per-draw earlyz_safe DCR is clear. VX_raster_earlyz #( - .INSTANCE_ID ($sformatf("%s-earlyz%d", INSTANCE_ID, slice_id)), + .INSTANCE_ID (`SFORMATF(("%s-earlyz%d", INSTANCE_ID, slice_id))), .OUTPUT_QUADS (OUTPUT_QUADS) ) raster_earlyz ( .clk (clk), @@ -529,7 +530,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( // no-work case so such an engine drains immediately. localparam LOG2_NUM_INSTANCES = `CLOG2(NUM_INSTANCES); wire [`RASTER_TILE_BITS-1:0] my_tile_count = - (raster_dcrs.tile_count + `RASTER_TILE_BITS'(NUM_INSTANCES - 1 - INSTANCE_IDX)) >> LOG2_NUM_INSTANCES; + (raster_dcrs.tile_count + (`RASTER_TILE_BITS'(NUM_INSTANCES - 1) - `RASTER_TILE_BITS'(instance_idx))) >> LOG2_NUM_INSTANCES; wire has_no_tiles = (my_tile_count == '0); always @(posedge clk) begin diff --git a/hw/rtl/raster/VX_raster_earlyz.sv b/hw/rtl/raster/VX_raster_earlyz.sv index 2cb0529650..7759ba7232 100644 --- a/hw/rtl/raster/VX_raster_earlyz.sv +++ b/hw/rtl/raster/VX_raster_earlyz.sv @@ -297,7 +297,7 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX ) lsu_mem_if(); VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (NUM_PIX), .MEM_CHANNELS (OCACHE_NUM_REQS), .WORD_SIZE (OCACHE_WORD_SIZE), diff --git a/hw/rtl/raster/VX_raster_mem.sv b/hw/rtl/raster/VX_raster_mem.sv index d4e5a82bda..002a9bc5ce 100644 --- a/hw/rtl/raster/VX_raster_mem.sv +++ b/hw/rtl/raster/VX_raster_mem.sv @@ -23,7 +23,6 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter INSTANCE_IDX = 0, parameter NUM_INSTANCES = 1, parameter TILE_LOGSIZE = 5, parameter QUEUE_SIZE = 8 @@ -31,6 +30,8 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( input wire clk, input wire reset, + input wire [`LOG2UP(NUM_INSTANCES)-1:0] instance_idx, + // Device configurations raster_dcrs_t dcrs, @@ -144,8 +145,8 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `UNUSED_VAR (th_pids_count) // calculate tile start info - wire [`RASTER_TILE_BITS-1:0] start_tile_count = (dcrs.tile_count + `RASTER_TILE_BITS'(NUM_INSTANCES - 1 - INSTANCE_IDX)) >> LOG2_NUM_INSTANCES; - wire [W_ADDR_BITS-1:0] start_tbuf_addr = {dcrs.tbuf_addr, 4'b0} + W_ADDR_BITS'(INSTANCE_IDX * TILE_HEADER_SIZEW); + wire [`RASTER_TILE_BITS-1:0] start_tile_count = (dcrs.tile_count + (`RASTER_TILE_BITS'(NUM_INSTANCES - 1) - `RASTER_TILE_BITS'(instance_idx))) >> LOG2_NUM_INSTANCES; + wire [W_ADDR_BITS-1:0] start_tbuf_addr = {dcrs.tbuf_addr, 4'b0} + (W_ADDR_BITS'(instance_idx) * W_ADDR_BITS'(TILE_HEADER_SIZEW)); // The sorted-pid array is a single dense block that follows ALL bin headers // (num_bins == dcrs.tile_count of them, 3 words each), shared across the N @@ -345,7 +346,7 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `UNUSED_VAR (sched_rsp_tag) VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (NUM_REQS), .MEM_CHANNELS (RCACHE_NUM_REQS), .WORD_SIZE (`RASTER_DATA_BITS / 8), diff --git a/hw/rtl/raster/VX_raster_slice.sv b/hw/rtl/raster/VX_raster_slice.sv index ae1e0a243f..4f1f920660 100644 --- a/hw/rtl/raster/VX_raster_slice.sv +++ b/hw/rtl/raster/VX_raster_slice.sv @@ -79,7 +79,7 @@ module VX_raster_slice import VX_raster_pkg::*; #( `endif VX_raster_te #( - .INSTANCE_ID ($sformatf("%s-te", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-te", INSTANCE_ID))), .TILE_LOGSIZE (TILE_LOGSIZE), .BLOCK_LOGSIZE (BLOCK_LOGSIZE) ) tile_evaluator ( @@ -147,7 +147,7 @@ module VX_raster_slice import VX_raster_pkg::*; #( `endif VX_raster_be #( - .INSTANCE_ID ($sformatf("%s-be", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-be", INSTANCE_ID))), .BLOCK_LOGSIZE (BLOCK_LOGSIZE), .OUTPUT_QUADS (OUTPUT_QUADS), .QUAD_FIFO_DEPTH (QUAD_FIFO_DEPTH) diff --git a/hw/rtl/rtu/VX_rtu_unit.sv b/hw/rtl/rtu/VX_rtu_unit.sv index cfeb22a315..ab93aceecf 100644 --- a/hw/rtl/rtu/VX_rtu_unit.sv +++ b/hw/rtl/rtu/VX_rtu_unit.sv @@ -38,13 +38,14 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0, parameter NUM_LANES = `VX_CFG_NUM_THREADS, parameter RTU_TAG_WIDTH = 1 ) ( input wire clk, input wire reset, + input wire [SFU_BLOCK_W-1:0] block_idx, + // SFU PE-style request/response interfaces VX_execute_if.slave execute_if, VX_result_if.master result_if, @@ -56,9 +57,12 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( VX_sched_unlock_if.master sched_unlock_if ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) - localparam RAM_SIZE = `VX_CFG_NUM_WARPS * RTUW_SLOT_COUNT; + // This SFU block serves the warps with wid mod NUM_SFU_BLOCKS == block_idx; + // its per-warp state is indexed by the block-local warp id. + localparam NLW = SFU_BLOCK_WARPS; + localparam RAM_SIZE = NLW * RTUW_SLOT_COUNT; + localparam RAM_ADDRW = `LOG2UP(RAM_SIZE); localparam RAM_DATAW = 32 * NUM_LANES; // The address concatenates {warp, slot}; the slot stride must be a power of @@ -80,6 +84,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire [2:0] uop = execute_if.data.op_args.rtuw.uop; wire [RTUW_SLOT_BITS-1:0] slot = execute_if.data.op_args.rtuw.slot[RTUW_SLOT_BITS-1:0]; wire [NW_WIDTH-1:0] wid = execute_if.data.header.wid; + wire [SFU_BLOCK_NW_W-1:0] lid = wid_to_sfu_local(wid); // Register the outgoing RTU bus at this module boundary so the core->socket // seam launches at a flop (see VX_rtu_bus_slice). We drive the internal @@ -107,7 +112,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // Written by the RTU, read by the shader. Neither port is ever contended, so // the RTU is never made to wait for a write and a read never waits for it. wire win_v = rtu_bus_w.win_valid; - wire [NW_WIDTH-1:0] win_wid = rtu_bus_w.win_data.wid; + wire [SFU_BLOCK_NW_W-1:0] win_lid = wid_to_sfu_local(rtu_bus_w.win_data.wid); wire [RTUW_SLOT_BITS-1:0] win_slot = rtu_bus_w.win_data.slot; `UNUSED_VAR (rtu_bus_w.win_data.tag) @@ -115,7 +120,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire core_rden; wire [RTUW_SLOT_BITS-1:0] core_slot; - wire [NW_WIDTH-1:0] core_wid; + wire [SFU_BLOCK_NW_W-1:0] core_lid; wire [NUM_LANES-1:0][31:0] core_rdata; VX_dp_ram #( @@ -130,9 +135,9 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( .read (core_rden), .write (win_v), .wren (rtu_bus_w.win_data.mask), - .waddr ({win_wid, win_slot}), + .waddr (RAM_ADDRW'({win_lid, win_slot})), .wdata (rtu_bus_w.win_data.data), - .raddr ({core_wid, core_slot}), + .raddr (RAM_ADDRW'({core_lid, core_slot})), .rdata (core_rdata) ); @@ -147,7 +152,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // longer one of them — see the parked-WAIT table below. wire is_read = (op == RTUW_OP_GETWF) || (op == RTUW_OP_GETW); - reg [`VX_CFG_NUM_WARPS-1:0] response_ready, trace_open; + reg [NLW-1:0] response_ready, trace_open; // ── the parked WAIT — a long-latency op MUST NOT hold the execute stage ──── // A WAIT used to sit at execute_if until its record landed, and execute_if is @@ -165,8 +170,8 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // WAIT and the unlock below is what releases it — so the semantics are unchanged; // only the execute port is freed. A warp holds at most one trace, so one entry // per warp is exact, not a heuristic. - reg [`VX_CFG_NUM_WARPS-1:0] wait_pend; - sfu_header_t wait_hdr [`VX_CFG_NUM_WARPS]; + reg [NLW-1:0] wait_pend; + sfu_header_t wait_hdr [NLW]; // ── result stage (the RAM read is synchronous) ───────────────────────── // One op is accepted per cycle into s1; its word arrives from the RAM on the @@ -227,7 +232,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // // A CONTINUE with no open candidate retires as a no-op (defensive — correct // kernels only issue one inside the yield loop). - wire cont_want = is_cont && trace_open[wid]; + wire cont_want = is_cont && trace_open[lid]; wire is_stream = is_ray || cont_want; wire is_2beat = cont_want || (is_trace && (uop == RTUW_UOP_ARM)); @@ -281,20 +286,22 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // the two arbitrate for the result stage. The parked WAIT wins: it has already // retired from execute_if and the warp behind it is stalled on nothing else, // while a GETW at execute_if can simply wait a cycle. - wire [`VX_CFG_NUM_WARPS-1:0] wait_done = wait_pend & response_ready; + wire [NLW-1:0] wait_done = wait_pend & response_ready; - wire [NW_WIDTH-1:0] wake_wid; - wire wake_valid; - wire [`VX_CFG_NUM_WARPS-1:0] wake_1h; + wire [`LOG2UP(NLW)-1:0] wake_index; + wire wake_valid; + wire [NLW-1:0] wake_1h; VX_priority_encoder #( - .N (`VX_CFG_NUM_WARPS) + .N (NLW) ) wake_picker ( .data_in (wait_done), .onehot_out (wake_1h), - .index_out (wake_wid), + .index_out (wake_index), .valid_out (wake_valid) ); `UNUSED_VAR (wake_1h) + wire [SFU_BLOCK_NW_W-1:0] wake_lid = SFU_BLOCK_NW_W'(wake_index); + wire [NW_WIDTH-1:0] wake_wid = sfu_local_to_wid(wake_lid, block_idx); assign wake_fire = wake_valid && s1_ready; @@ -302,12 +309,12 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // A stream op retires with its last beat; everything else needs only the // result stage. A WAIT retires the moment it is parked — it never blocks. wire read_fire = execute_if.valid && is_read && s1_grant; - wire wait_fire = execute_if.valid && is_wait && ~wait_pend[wid]; + wire wait_fire = execute_if.valid && is_wait && ~wait_pend[lid]; wire stream_fire = rb_fire && rb_end; - wire cont_nop = execute_if.valid && is_cont && ~trace_open[wid] && s1_grant; + wire cont_nop = execute_if.valid && is_cont && ~trace_open[lid] && s1_grant; wire op_fire = read_fire || wait_fire || stream_fire || cont_nop || arm_fire; - `RUNTIME_ASSERT(~(execute_if.valid && is_wait && wait_pend[wid]), + `RUNTIME_ASSERT(~(execute_if.valid && is_wait && wait_pend[lid]), ("%t: *** %s: wid=%0d issued a second WAIT with one already parked", $time, INSTANCE_ID, wid)) @@ -331,7 +338,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // read reads the slot its op names. assign core_rden = read_fire || wake_fire; assign core_slot = wake_fire ? RTUW_SLOT_BITS'(`VX_RT_STATUS) : slot; - assign core_wid = wake_fire ? wake_wid : wid; + assign core_lid = wake_fire ? wake_lid : lid; // ── sequential state ─────────────────────────────────────────────────── always @(posedge clk) begin @@ -350,7 +357,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( if (wake_fire) begin s1_valid <= 1'b1; s1_from_ram <= 1'b1; - s1_header <= wait_hdr[wake_wid]; + s1_header <= wait_hdr[wake_lid]; end else if (op_fire && ~wait_fire) begin s1_valid <= 1'b1; s1_from_ram <= is_read; @@ -359,8 +366,8 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // park the WAIT: it retires now, and answers later if (wait_fire) begin - wait_pend[wid] <= 1'b1; - wait_hdr[wid] <= execute_if.data.header; + wait_pend[lid] <= 1'b1; + wait_hdr[lid] <= execute_if.data.header; end // The last beat retires the op, so the counter always lands back at 0. @@ -370,18 +377,18 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // The waking WAIT consumes the record, so the next one waits again. if (wake_fire) begin - wait_pend[wake_wid] <= 1'b0; - response_ready[wake_wid] <= 1'b0; + wait_pend[wake_lid] <= 1'b0; + response_ready[wake_lid] <= 1'b0; end // A CONTINUE resolves the open candidate once its last beat is away. if (stream_fire && cont_want) begin - trace_open[wid] <= 1'b0; + trace_open[lid] <= 1'b0; end // Last, so a record landing in the same cycle as the op it unblocks // wins: the record is the newer event. if (status_wr) begin - response_ready[win_wid] <= 1'b1; - trace_open[win_wid] <= rtu_bus_w.win_data.is_cand; + response_ready[win_lid] <= 1'b1; + trace_open[win_lid] <= rtu_bus_w.win_data.is_cand; end end end diff --git a/hw/rtl/tcu/VX_tcu_block.sv b/hw/rtl/tcu/VX_tcu_block.sv new file mode 100644 index 0000000000..ee6e19a637 --- /dev/null +++ b/hw/rtl/tcu/VX_tcu_block.sv @@ -0,0 +1,121 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One TCU block: the MMA datapath, with TCU_LD handed to the core-wide AGU and +// WGMMA operands supplied by the core-wide tile buffers (VX_tcu_shared). +module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter `STRING BLOCK_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + // MMA execute handshake observed by the WGMMA tile buffers + output wire exec_valid, + output wire exec_ready, + output tcu_execute_t exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, + input wire tbuf_ready, +`endif + +`ifdef TCU_META_ENABLE + // TCU_LD to and from the AGU + output wire agu_ld_valid, + output tcu_execute_t agu_ld_data, + input wire agu_ld_ready, + input wire agu_result_valid, + input tcu_result_t agu_result_data, + output wire agu_result_ready, + input wire meta_wr_en, + input wire [NW_WIDTH-1:0] meta_wr_wid, + input wire [4:0] meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, +`endif + + VX_execute_if.slave execute_if, + VX_result_if.master result_if +); + `STATIC_ASSERT (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH, ("must be full issue execution")); + `STATIC_ASSERT (`VX_CFG_NUM_TCU_LANES == `VX_CFG_NUM_THREADS, ("must be full warp execution")); + + VX_execute_if #( + .data_t (tcu_execute_t) + ) core_execute_if(); + + VX_result_if #( + .data_t (tcu_result_t) + ) core_result_if(); + +`ifdef TCU_META_ENABLE + // TCU_LD goes to the AGU, every MMA op to the core + wire is_tcu_ld = (execute_if.data.op_type == INST_TCU_LD); + + assign agu_ld_valid = execute_if.valid && is_tcu_ld; + assign agu_ld_data = execute_if.data; + assign core_execute_if.valid = execute_if.valid && !is_tcu_ld; + assign core_execute_if.data = execute_if.data; + assign execute_if.ready = is_tcu_ld ? agu_ld_ready : core_execute_if.ready; + + // AGU wins same-cycle conflicts; the core stalls and retries next cycle. + assign result_if.valid = agu_result_valid || core_result_if.valid; + assign result_if.data = agu_result_valid ? agu_result_data : core_result_if.data; + assign agu_result_ready = result_if.ready; + assign core_result_if.ready = result_if.ready && !agu_result_valid; +`else + assign core_execute_if.valid = execute_if.valid; + assign core_execute_if.data = execute_if.data; + assign execute_if.ready = core_execute_if.ready; + + assign result_if.valid = core_result_if.valid; + assign result_if.data = core_result_if.data; + assign core_result_if.ready = result_if.ready; +`endif + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + assign exec_valid = core_execute_if.valid; + assign exec_ready = core_execute_if.ready; + assign exec_data = core_execute_if.data; +`endif + + VX_tcu_core #( + .INSTANCE_ID (`SFORMATF(("%s-fused%s", INSTANCE_ID, BLOCK_STR))) + ) tcu_core ( + `ifdef SCOPE + .scope_reset (scope_reset), + .scope_bus_in (scope_bus_in), + .scope_bus_out (scope_bus_out), + `endif + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tbuf_rs1_data (tbuf_rs1_data), + .tbuf_rs2_data (tbuf_rs2_data), + .tbuf_ready (tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .ext_meta_wr_en (meta_wr_en), + .ext_meta_wr_wid (meta_wr_wid), + .ext_meta_wr_idx (meta_wr_idx), + .ext_meta_wr_data (meta_wr_data), + `endif + .execute_if (core_execute_if), + .result_if (core_result_if) + ); + +endmodule diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index f1a3c91b19..1762caafd7 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -303,8 +303,10 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `ifdef TCU_META_ENABLE + // a TCU block only executes one residue class of warps modulo NUM_TCU_BLOCKS VX_tcu_meta #( - .INSTANCE_ID (INSTANCE_ID) + .INSTANCE_ID (INSTANCE_ID), + .WID_STRIDE (`VX_CFG_NUM_TCU_BLOCKS) ) tcu_meta ( .clk (clk), .reset (reset), diff --git a/hw/rtl/tcu/VX_tcu_meta.sv b/hw/rtl/tcu/VX_tcu_meta.sv index 5c55e58bb0..8a31ca6bad 100644 --- a/hw/rtl/tcu/VX_tcu_meta.sv +++ b/hw/rtl/tcu/VX_tcu_meta.sv @@ -31,7 +31,10 @@ // VX_tcu_core consume the result in that same cycle. module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + // stores one residue class of warps modulo WID_STRIDE, by wid / WID_STRIDE; + // the parent only presents writes for that class + parameter WID_STRIDE = 1 ) ( input wire clk, input wire reset, @@ -74,8 +77,11 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam MERGED_COLS = SP_COLS + 2 * MX_COLS; localparam MERGED_BITS = MERGED_COLS * 32; - localparam META_DEPTH = `VX_CFG_NUM_WARPS; - localparam META_ADDRW = `CLOG2(META_DEPTH); + localparam META_DEPTH = `VX_CFG_NUM_WARPS / WID_STRIDE; + localparam META_ADDRW = `CLOG2(META_DEPTH); + localparam STRIDE_BITS = `CLOG2(WID_STRIDE); + + `STATIC_ASSERT (`IS_POW2(WID_STRIDE), ("invalid parameter")) // Namespace decode (mirrors the AGU / vx_tensor.h slot encoding). The // per-region write strobes are defined inside each region's block below so @@ -85,8 +91,8 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [MERGED_BITS-1:0] merged_wdata; wire [MERGED_BITS-1:0] merged_rdata; - wire [`UP(META_ADDRW)-1:0] wr_addr = `UP(META_ADDRW)'(wr_wid); - wire [`UP(META_ADDRW)-1:0] rd_addr = `UP(META_ADDRW)'(rd_wid); + wire [`UP(META_ADDRW)-1:0] wr_addr = `UP(META_ADDRW)'(wr_wid >> STRIDE_BITS); + wire [`UP(META_ADDRW)-1:0] rd_addr = `UP(META_ADDRW)'(rd_wid >> STRIDE_BITS); // ----------------------------------------------------------------------- // Sparse region: per-thread metadata packed into PER_WARP_DEPTH banks of diff --git a/hw/rtl/tcu/VX_tcu_shared.sv b/hw/rtl/tcu/VX_tcu_shared.sv new file mode 100644 index 0000000000..77ff231a72 --- /dev/null +++ b/hw/rtl/tcu/VX_tcu_shared.sv @@ -0,0 +1,118 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// TCU state shared by every TCU block: the WGMMA tile buffers (a warpgroup's +// B operand is common to its blocks) and the warp-level metadata AGU. +module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS +) ( +`ifdef PERF_ENABLE + output tcu_perf_t tcu_perf, +`endif + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + input wire [BLOCK_SIZE-1:0] exec_valid, + input wire [BLOCK_SIZE-1:0] exec_ready, + input tcu_execute_t exec_data [BLOCK_SIZE], + output wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + output wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, + output wire [BLOCK_SIZE-1:0] tbuf_ready_eff, + // Bank-parallel LMEM read port + VX_mem_bus_if.master tcu_lmem_if, +`endif + +`ifdef TCU_META_ENABLE + input wire [BLOCK_SIZE-1:0] agu_ld_valid, + input tcu_execute_t agu_ld_data [BLOCK_SIZE], + output wire [BLOCK_SIZE-1:0] agu_ld_ready, + output wire [BLOCK_SIZE-1:0] agu_result_valid, + output tcu_result_t agu_result_data [BLOCK_SIZE], + input wire [BLOCK_SIZE-1:0] agu_result_ready, + output wire meta_wr_en, + output wire [NW_WIDTH-1:0] meta_wr_wid, + output wire [4:0] meta_wr_idx, + output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, + // TCU_LD memory client of the LSU scheduler + VX_lsu_sched_if.master tcu_mem_if, +`endif + + input wire clk, + input wire reset +); + `UNUSED_SPARAM (INSTANCE_ID) +`ifndef VX_CFG_TCU_WGMMA_ENABLE +`ifndef TCU_META_ENABLE + `UNUSED_PARAM (BLOCK_SIZE) + `UNUSED_VAR ({clk, reset}) +`endif +`endif + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + VX_tcu_wgmma #( + .INSTANCE_ID (`SFORMATF(("%s-wgmma", INSTANCE_ID))), + .BLOCK_SIZE (BLOCK_SIZE) + ) wgmma ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + .exec_valid (exec_valid), + .exec_ready (exec_ready), + .exec_data (exec_data), + .tcu_lmem_if (tcu_lmem_if), + .tbuf_rs1_data (tbuf_rs1_data), + .tbuf_rs2_data (tbuf_rs2_data), + .tbuf_ready_eff (tbuf_ready_eff) + ); +`else +`ifdef PERF_ENABLE + assign tcu_perf.tbuf_stalls = '0; + assign tcu_perf.tbuf_cache_hits = '0; + assign tcu_perf.lmem_reads = '0; + assign tcu_perf.wgmma_instrs = '0; + assign tcu_perf.wgmma_stalls = '0; +`endif +`endif + +`ifdef TCU_META_ENABLE + // The AGU streams NUM_TCU_LANES operands over the LSU memory client, whose + // request/response mask and data are sized by NUM_LSU_LANES; the two lane + // counts must match for the per-lane completion tracking to be correct. + `STATIC_ASSERT (`VX_CFG_NUM_TCU_LANES == `VX_CFG_NUM_LSU_LANES, ("VX_tcu_agu requires NUM_TCU_LANES == NUM_LSU_LANES")); + + VX_tcu_agu #( + .INSTANCE_ID (`SFORMATF(("%s-agu", INSTANCE_ID))), + .BLOCK_SIZE (BLOCK_SIZE), + .NUM_LANES (`VX_CFG_NUM_TCU_LANES) + ) agu ( + .clk (clk), + .reset (reset), + .per_block_ld_valid (agu_ld_valid), + .per_block_ld_data (agu_ld_data), + .per_block_ld_ready (agu_ld_ready), + .client_if (tcu_mem_if), + .meta_wr_en (meta_wr_en), + .meta_wr_wid (meta_wr_wid), + .meta_wr_idx (meta_wr_idx), + .meta_wr_data (meta_wr_data), + .result_valid (agu_result_valid), + .result_data (agu_result_data), + .result_ready (agu_result_ready) + ); +`endif + +endmodule diff --git a/hw/rtl/tcu/VX_tcu_unit.sv b/hw/rtl/tcu/VX_tcu_unit.sv index 593e39eee7..fe0e4838e7 100644 --- a/hw/rtl/tcu/VX_tcu_unit.sv +++ b/hw/rtl/tcu/VX_tcu_unit.sv @@ -59,6 +59,7 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); @@ -67,188 +68,96 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .data_t (tcu_result_t) ) per_block_result_if[BLOCK_SIZE](); - // ----------------------------------------------------------------------- - // Split each per_block_execute_if between two consumers: - // - VX_tcu_agu: handles INST_TCU_LD (warp-level memory load). - // - VX_tcu_core: handles every MMA op_type. - // The ready signal is muxed by op_type so only one consumer drives at a time. - // ----------------------------------------------------------------------- - VX_execute_if #( - .data_t (tcu_execute_t) - ) core_execute_if[BLOCK_SIZE](); - - VX_result_if #( - .data_t (tcu_result_t) - ) core_result_if[BLOCK_SIZE](); - -`ifdef TCU_META_ENABLE - wire [BLOCK_SIZE-1:0] agu_ld_valid; - wire [BLOCK_SIZE-1:0] agu_ld_ready; - tcu_execute_t agu_ld_data [BLOCK_SIZE]; - - wire [BLOCK_SIZE-1:0] agu_result_valid; - tcu_result_t agu_result_data [BLOCK_SIZE]; - wire [BLOCK_SIZE-1:0] agu_result_ready; -`endif - - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_split - `ifdef TCU_META_ENABLE - wire is_tcu_ld = (per_block_execute_if[bi].data.op_type == INST_TCU_LD); - - // To AGU when TCU_LD - assign agu_ld_valid[bi] = per_block_execute_if[bi].valid && is_tcu_ld; - assign agu_ld_data[bi] = per_block_execute_if[bi].data; - - // To tcu_core when NOT TCU_LD - assign core_execute_if[bi].valid = per_block_execute_if[bi].valid && !is_tcu_ld; - assign core_execute_if[bi].data = per_block_execute_if[bi].data; - - // Parent .ready: route to AGU on TCU_LD, otherwise to tcu_core - assign per_block_execute_if[bi].ready = is_tcu_ld - ? agu_ld_ready[bi] - : core_execute_if[bi].ready; - `else - // No sparse: pass-through to tcu_core - assign core_execute_if[bi].valid = per_block_execute_if[bi].valid; - assign core_execute_if[bi].data = per_block_execute_if[bi].data; - assign per_block_execute_if[bi].ready = core_execute_if[bi].ready; - `endif - end - - // ----------------------------------------------------------------------- - // Result_if merge: AGU result and tcu_core result are mutually exclusive - // in time per block; OR-mux with priority arbiter (AGU wins: TCU_LD is rare). - // ----------------------------------------------------------------------- -`ifdef TCU_META_ENABLE - // AGU wins same-cycle conflicts; tcu_core stalls (ready=0) and retries next cycle. - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_result_merge - assign per_block_result_if[bi].valid = agu_result_valid[bi] || core_result_if[bi].valid; - assign per_block_result_if[bi].data = agu_result_valid[bi] - ? agu_result_data[bi] - : core_result_if[bi].data; - assign agu_result_ready[bi] = per_block_result_if[bi].ready; - assign core_result_if[bi].ready = per_block_result_if[bi].ready && !agu_result_valid[bi]; - end -`else - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_result_passthru - assign per_block_result_if[bi].valid = core_result_if[bi].valid; - assign per_block_result_if[bi].data = core_result_if[bi].data; - assign core_result_if[bi].ready = per_block_result_if[bi].ready; - end -`endif - - // ----------------------------------------------------------------------- - // WGMMA feature (orchestrator): VX_tcu_tbuf + VX_tcu_lockstep + perf. - // ----------------------------------------------------------------------- - `ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [BLOCK_SIZE-1:0] exec_valid_w; - wire [BLOCK_SIZE-1:0] exec_ready_w; - tcu_execute_t exec_data_w [BLOCK_SIZE]; - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_obs - assign exec_valid_w[bi] = core_execute_if[bi].valid; - assign exec_ready_w[bi] = core_execute_if[bi].ready; - assign exec_data_w[bi] = core_execute_if[bi].data; - end - + wire [BLOCK_SIZE-1:0] exec_valid, exec_ready; + tcu_execute_t exec_data [BLOCK_SIZE]; wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data; - wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data; - wire [BLOCK_SIZE-1:0] tbuf_ready_eff; - - VX_tcu_wgmma #( - .INSTANCE_ID (`SFORMATF(("%s-wgmma", INSTANCE_ID))), - .BLOCK_SIZE (BLOCK_SIZE) - ) wgmma ( - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif - .exec_valid (exec_valid_w), - .exec_ready (exec_ready_w), - .exec_data (exec_data_w), - .tcu_lmem_if (tcu_lmem_if), - .tbuf_rs1_data (tbuf_rs1_data), - .tbuf_rs2_data (tbuf_rs2_data), - .tbuf_ready_eff (tbuf_ready_eff) - ); - -`else // !VX_CFG_TCU_WGMMA_ENABLE - -`ifdef PERF_ENABLE - assign tcu_perf.tbuf_stalls = '0; - assign tcu_perf.tbuf_cache_hits = '0; - assign tcu_perf.lmem_reads = '0; - assign tcu_perf.wgmma_instrs = '0; - assign tcu_perf.wgmma_stalls = '0; + wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data; + wire [BLOCK_SIZE-1:0] tbuf_ready_eff; `endif -`endif // VX_CFG_TCU_WGMMA_ENABLE - - // ----------------------------------------------------------------------- - // VX_tcu_agu — warp-level AGU for TCU_LD instructions. - // Drives meta_wr signals broadcast to every tcu_core so wmma_sp on - // any block sees the loaded metadata. - // ----------------------------------------------------------------------- `ifdef TCU_META_ENABLE - // The AGU streams NUM_TCU_LANES operands over the LSU memory client, whose - // request/response mask and data are sized by NUM_LSU_LANES; the two lane - // counts must match for the per-lane completion tracking to be correct. - `STATIC_ASSERT (NUM_LANES == `VX_CFG_NUM_LSU_LANES, ("VX_tcu_agu requires NUM_TCU_LANES == NUM_LSU_LANES")); - - wire agu_meta_wr_en; - wire [NW_WIDTH-1:0] agu_meta_wr_wid; - wire [4:0] agu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] agu_meta_wr_data; - - VX_tcu_agu #( - .INSTANCE_ID (`SFORMATF(("%s-agu", INSTANCE_ID))), - .BLOCK_SIZE (BLOCK_SIZE), - .NUM_LANES (NUM_LANES) - ) agu ( - .clk (clk), - .reset (reset), - .per_block_ld_valid (agu_ld_valid), - .per_block_ld_data (agu_ld_data), - .per_block_ld_ready (agu_ld_ready), - .client_if (tcu_mem_if), - .meta_wr_en (agu_meta_wr_en), - .meta_wr_wid (agu_meta_wr_wid), - .meta_wr_idx (agu_meta_wr_idx), - .meta_wr_data (agu_meta_wr_data), - .result_valid (agu_result_valid), - .result_data (agu_result_data), - .result_ready (agu_result_ready) - ); + wire [BLOCK_SIZE-1:0] agu_ld_valid, agu_ld_ready; + tcu_execute_t agu_ld_data [BLOCK_SIZE]; + wire [BLOCK_SIZE-1:0] agu_result_valid, agu_result_ready; + tcu_result_t agu_result_data [BLOCK_SIZE]; + wire meta_wr_en; + wire [NW_WIDTH-1:0] meta_wr_wid; + wire [4:0] meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data; `endif - // ----------------------------------------------------------------------- - // TCU core instances - // ----------------------------------------------------------------------- - for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - VX_tcu_core #( - .INSTANCE_ID (`SFORMATF(("%s-fused%0d", INSTANCE_ID, block_idx))) - ) tcu_core ( + `ifdef TCU_META_ENABLE + wire block_meta_wr_en = meta_wr_en + && (NW_WIDTH'(meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(block_idx)); + `endif + + VX_tcu_block #( + .INSTANCE_ID (INSTANCE_ID), + .BLOCK_STR (`SFORMATF(("%0d", block_idx))) + ) tcu_block ( `SCOPE_IO_BIND (block_idx) - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tbuf_rs1_data (tbuf_rs1_data[block_idx]), - .tbuf_rs2_data (tbuf_rs2_data[block_idx]), - .tbuf_ready (tbuf_ready_eff[block_idx]), + .exec_valid (exec_valid[block_idx]), + .exec_ready (exec_ready[block_idx]), + .exec_data (exec_data[block_idx]), + .tbuf_rs1_data (tbuf_rs1_data[block_idx]), + .tbuf_rs2_data (tbuf_rs2_data[block_idx]), + .tbuf_ready (tbuf_ready_eff[block_idx]), `endif `ifdef TCU_META_ENABLE - .ext_meta_wr_en (agu_meta_wr_en), - .ext_meta_wr_wid (agu_meta_wr_wid), - .ext_meta_wr_idx (agu_meta_wr_idx), - .ext_meta_wr_data (agu_meta_wr_data), + .agu_ld_valid (agu_ld_valid[block_idx]), + .agu_ld_data (agu_ld_data[block_idx]), + .agu_ld_ready (agu_ld_ready[block_idx]), + .agu_result_valid (agu_result_valid[block_idx]), + .agu_result_data (agu_result_data[block_idx]), + .agu_result_ready (agu_result_ready[block_idx]), + .meta_wr_en (block_meta_wr_en), + .meta_wr_wid (meta_wr_wid), + .meta_wr_idx (meta_wr_idx), + .meta_wr_data (meta_wr_data), `endif - .execute_if (core_execute_if[block_idx]), - .result_if (core_result_if[block_idx]) + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]) ); end + VX_tcu_shared #( + .INSTANCE_ID (INSTANCE_ID), + .BLOCK_SIZE (BLOCK_SIZE) + ) tcu_shared ( + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (exec_valid), + .exec_ready (exec_ready), + .exec_data (exec_data), + .tbuf_rs1_data (tbuf_rs1_data), + .tbuf_rs2_data (tbuf_rs2_data), + .tbuf_ready_eff (tbuf_ready_eff), + .tcu_lmem_if (tcu_lmem_if), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (agu_ld_valid), + .agu_ld_data (agu_ld_data), + .agu_ld_ready (agu_ld_ready), + .agu_result_valid (agu_result_valid), + .agu_result_data (agu_result_data), + .agu_result_ready (agu_result_ready), + .meta_wr_en (meta_wr_en), + .meta_wr_wid (meta_wr_wid), + .meta_wr_idx (meta_wr_idx), + .meta_wr_data (meta_wr_data), + .tcu_mem_if (tcu_mem_if), + `endif + .clk (clk), + .reset (reset) + ); + // ----------------------------------------------------------------------- // Lane gather // ----------------------------------------------------------------------- diff --git a/hw/rtl/tex/VX_tex_core.sv b/hw/rtl/tex/VX_tex_core.sv index 7ffec13f8c..c8e427df62 100644 --- a/hw/rtl/tex/VX_tex_core.sv +++ b/hw/rtl/tex/VX_tex_core.sv @@ -48,7 +48,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( tex_dcrs_t tex_dcrs; VX_tex_dcr #( - .INSTANCE_ID ($sformatf("%s-dcr", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-dcr", INSTANCE_ID))), .NUM_STAGES (`VX_TEX_STAGE_COUNT) ) tex_dcr ( .clk (clk), @@ -132,7 +132,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( // sampler, for the same reason the format does: the state a sample merges // under is the state it was issued under. VX_tex_addr #( - .INSTANCE_ID ($sformatf("%s-addr", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-addr", INSTANCE_ID))), .REQ_TAGW (TAG_WIDTH + TEX_FORMAT_BITS + 32), .NUM_LANES (NUM_LANES) ) tex_addr ( @@ -176,7 +176,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( wire mem_rsp_ready; VX_tex_mem #( - .INSTANCE_ID ($sformatf("%s-mem", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-mem", INSTANCE_ID))), .REQ_TAGW (TAG_WIDTH + TEX_FORMAT_BITS + 32 + BLEND_FRAC_W + BORDER_W + LODFRAC_W), .NUM_LANES (NUM_LANES) ) tex_mem ( @@ -218,7 +218,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( wire [TAG_WIDTH-1:0] mem_rsp_tag = mem_rsp_info[(LODFRAC_W + BORDER_W + BLEND_FRAC_W + 32 + TEX_FORMAT_BITS) +: TAG_WIDTH]; VX_tex_sampler #( - .INSTANCE_ID ($sformatf("%s-sampler", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-sampler", INSTANCE_ID))), .REQ_TAGW (TAG_WIDTH), .NUM_LANES (NUM_LANES) ) tex_sampler ( diff --git a/hw/rtl/tex/VX_tex_mem.sv b/hw/rtl/tex/VX_tex_mem.sv index d712174204..8907ff267d 100644 --- a/hw/rtl/tex/VX_tex_mem.sv +++ b/hw/rtl/tex/VX_tex_mem.sv @@ -128,7 +128,7 @@ module VX_tex_mem import VX_gpu_pkg::*; import VX_tex_pkg::*; #( ) lsu_mem_if(); VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (TEX_MEM_REQS), .MEM_CHANNELS(TCACHE_NUM_REQS), .WORD_SIZE (4), diff --git a/hw/rtl/tex/VX_tex_unit.sv b/hw/rtl/tex/VX_tex_unit.sv index 56c19b2be8..d49391f3e1 100644 --- a/hw/rtl/tex/VX_tex_unit.sv +++ b/hw/rtl/tex/VX_tex_unit.sv @@ -32,7 +32,6 @@ module VX_tex_unit import VX_gpu_pkg::*, VX_tex_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0, parameter NUM_LANES = `VX_CFG_NUM_THREADS ) ( input wire clk, @@ -46,7 +45,6 @@ module VX_tex_unit import VX_gpu_pkg::*, VX_tex_pkg::*; #( VX_tex_bus_if.master tex_bus_if ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) localparam REQ_QUEUE_BITS = `LOG2UP(`VX_CFG_TEX_REQ_QUEUE_SIZE); localparam PID_W = `LOG2UP(`VX_CFG_NUM_THREADS / NUM_LANES); diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 10e96cd472..b945a90189 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -123,7 +123,7 @@ PLATFORM_DEFINES += -DPLATFORM_MEMORY_ID_WIDTH=4 # them. The AXI boundary cannot show any of that -- a core waiting on a response # it never retired looks exactly like a core with nothing left to do. They are # read over the DCR response path, which both the harness and the driver already -# drive, but VX_csr_data.sv gates them behind PERF_ENABLE. Off by default because +# drive, but VX_csr_mpm.sv gates them behind PERF_ENABLE. Off by default because # they cost area; build with PERF_DEFINES=-DPERF_ENABLE to read them, and note # that without it every counter reads zero, which is indistinguishable from an # idle core rather than an absent instrument. diff --git a/hw/syn/xilinx/aved/platforms.mk b/hw/syn/xilinx/aved/platforms.mk index 9a6b5b2a90..80ced1dd35 100644 --- a/hw/syn/xilinx/aved/platforms.mk +++ b/hw/syn/xilinx/aved/platforms.mk @@ -33,7 +33,7 @@ override CONFIGS += -DPLATFORM_MEMORY_OFFSET=40\'h4000000000 # which the compute shell maps as 1 GB at 0x40_0000_0000. Vortex's 32-bit memory # map puts the stack at the top of a 4 GB space -- vx_start.S sets sp to # VX_MEM_STACK_BASE_ADDR (0xFFFF0000) and grows down, and that lands just below -# the local-memory window (VX_lsu_slice.sv decodes LMEM *upward* from the same +# the local-memory window (VX_lsu_block.sv decodes LMEM *upward* from the same # base), so it is a global-memory access. Rebased it becomes 0x40_FFFE_Fxxx, # which a 1 GB aperture cannot answer: the read gets no response, the LSU stalls # forever, and the AFU never goes idle -- taking the shell's AXI path with it. diff --git a/hw/unittest/core/VX_core_top.sv b/hw/unittest/core/VX_core_top.sv index 1e0621d0a7..b0dbb9b44a 100644 --- a/hw/unittest/core/VX_core_top.sv +++ b/hw/unittest/core/VX_core_top.sv @@ -42,18 +42,18 @@ module VX_core_top import VX_gpu_pkg::*; input wire [DCACHE_NUM_REQS-1:0][DCACHE_TAG_WIDTH-1:0] dcache_rsp_tag, output wire [DCACHE_NUM_REQS-1:0] dcache_rsp_ready, - output wire icache_req_valid, - output wire icache_req_rw, - output wire [ICACHE_WORD_SIZE-1:0] icache_req_byteen, - output wire [ICACHE_ADDR_WIDTH-1:0] icache_req_addr, - output wire [ICACHE_WORD_SIZE*8-1:0] icache_req_data, - output wire [ICACHE_TAG_WIDTH-1:0] icache_req_tag, - input wire icache_req_ready, - - input wire icache_rsp_valid, - input wire [ICACHE_WORD_SIZE*8-1:0] icache_rsp_data, - input wire [ICACHE_TAG_WIDTH-1:0] icache_rsp_tag, - output wire icache_rsp_ready, + output wire [ICACHE_NUM_REQS-1:0] icache_req_valid, + output wire [ICACHE_NUM_REQS-1:0] icache_req_rw, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_WORD_SIZE-1:0] icache_req_byteen, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_ADDR_WIDTH-1:0] icache_req_addr, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_WORD_SIZE*8-1:0] icache_req_data, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_TAG_WIDTH-1:0] icache_req_tag, + input wire [ICACHE_NUM_REQS-1:0] icache_req_ready, + + input wire [ICACHE_NUM_REQS-1:0] icache_rsp_valid, + input wire [ICACHE_NUM_REQS-1:0][ICACHE_WORD_SIZE*8-1:0] icache_rsp_data, + input wire [ICACHE_NUM_REQS-1:0][ICACHE_TAG_WIDTH-1:0] icache_rsp_tag, + output wire [ICACHE_NUM_REQS-1:0] icache_rsp_ready, output wire gbar_req_valid, output wire [NB_WIDTH-1:0] gbar_req_id, @@ -174,21 +174,23 @@ module VX_core_top import VX_gpu_pkg::*; VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_TAG_WIDTH) - ) icache_bus_if(); - - assign icache_req_valid = icache_bus_if.req_valid; - assign icache_req_rw = icache_bus_if.req_data.rw; - assign icache_req_byteen = icache_bus_if.req_data.byteen; - assign icache_req_addr = icache_bus_if.req_data.addr; - assign icache_req_data = icache_bus_if.req_data.data; - assign icache_req_tag = icache_bus_if.req_data.tag; - assign icache_bus_if.req_ready = icache_req_ready; - `UNUSED_VAR (icache_bus_if.req_data.attr) - - assign icache_bus_if.rsp_valid = icache_rsp_valid; - assign icache_bus_if.rsp_data.tag = icache_rsp_tag; - assign icache_bus_if.rsp_data.data = icache_rsp_data; - assign icache_rsp_ready = icache_bus_if.rsp_ready; + ) icache_bus_if[ICACHE_NUM_REQS](); + + for (genvar i = 0; i < ICACHE_NUM_REQS; ++i) begin : g_icache_bus_if + assign icache_req_valid[i] = icache_bus_if[i].req_valid; + assign icache_req_rw[i] = icache_bus_if[i].req_data.rw; + assign icache_req_byteen[i] = icache_bus_if[i].req_data.byteen; + assign icache_req_addr[i] = icache_bus_if[i].req_data.addr; + assign icache_req_data[i] = icache_bus_if[i].req_data.data; + assign icache_req_tag[i] = icache_bus_if[i].req_data.tag; + assign icache_bus_if[i].req_ready = icache_req_ready[i]; + `UNUSED_VAR (icache_bus_if[i].req_data.attr) + + assign icache_bus_if[i].rsp_valid = icache_rsp_valid[i]; + assign icache_bus_if[i].rsp_data.tag = icache_rsp_tag[i]; + assign icache_bus_if[i].rsp_data.data = icache_rsp_data[i]; + assign icache_rsp_ready[i] = icache_bus_if[i].rsp_ready; + end `ifdef PERF_ENABLE sysmem_perf_t mem_perf; @@ -208,12 +210,12 @@ module VX_core_top import VX_gpu_pkg::*; `endif VX_core #( - .INSTANCE_ID (`SFORMATF(("core"))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("core"))) ) core ( `SCOPE_IO_BIND (0) .clk (clk), .reset (reset), + .core_id (CORE_ID_WIDTH'(CORE_ID)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf), diff --git a/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv b/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv index 564634269b..9062fd6110 100644 --- a/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv +++ b/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv @@ -101,21 +101,18 @@ module VX_cta_dispatch_top import VX_gpu_pkg::*; wire cta_init; wire busy; - // CTA-CSR read-back / schedule lookup ports are unexercised by this bench - // (it only checks warp dispatch via cta_fire/cta_wid), so the read indices - // are tied off and the read-back outputs are left unused. - cta_csrs_t cta_rd_csrs; - cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_rd_lane; - wire [NCTA_WIDTH-1:0] schedule_cta_id; + // The CTA-CSR broadcast is unexercised by this bench (it only checks warp + // dispatch via cta_fire/cta_wid). + cta_bcast_t cta_bcast; + cta_lane_bcast_t cta_lane_bcast; `UNUSED_VAR (cta_PC) `UNUSED_VAR (cta_tmask) `UNUSED_VAR (cta_param) `UNUSED_VAR (cta_init) `UNUSED_VAR (busy) - `UNUSED_VAR (cta_rd_csrs) - `UNUSED_VAR (cta_rd_lane) - `UNUSED_VAR (schedule_cta_id) + `UNUSED_VAR (cta_bcast) + `UNUSED_VAR (cta_lane_bcast) VX_cta_dispatch cta_dispatch ( .clk (clk), @@ -130,12 +127,8 @@ module VX_cta_dispatch_top import VX_gpu_pkg::*; .cta_tmask (cta_tmask), .cta_param (cta_param), .cta_init (cta_init), - .csr_rd_wid ('0), - .csr_rd_cta_id ('0), - .cta_rd_csrs (cta_rd_csrs), - .cta_rd_lane (cta_rd_lane), - .schedule_wid ('0), - .schedule_cta_id (schedule_cta_id), + .cta_bcast (cta_bcast), + .cta_lane_bcast(cta_lane_bcast), .busy (busy) ); diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index ce4ef6876d..09159d41da 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -664,10 +664,11 @@ module VX_gfx_top import VX_gpu_pkg::* `endif VX_graphics #( - .CLUSTER_ID (0) + .INSTANCE_ID ("cluster0") ) graphics ( .clk (clk), .reset (reset), + .cluster_id ('0), `ifdef VX_CFG_EXT_RASTER_ENABLE .raster_kmu_bus_if (raster_kmu_bus_if), .rcache_mem_bus_if (rcache_l2_bus_if), diff --git a/hw/unittest/issue/VX_issue_top.sv b/hw/unittest/issue/VX_issue_top.sv index 0b34f25478..01de117a5e 100644 --- a/hw/unittest/issue/VX_issue_top.sv +++ b/hw/unittest/issue/VX_issue_top.sv @@ -90,7 +90,21 @@ module VX_issue_top import VX_gpu_pkg::*; #( assign decode_if.data.rs1 = decode_rs1; assign decode_if.data.rs2 = decode_rs2; assign decode_if.data.rs3 = decode_rs3; - assign decode_ready = decode_if.ready; + + // issue slice i takes the warps with wid mod ISSUE_WIDTH == i + VX_decode_if slice_decode_if[`VX_CFG_ISSUE_WIDTH](); + wire [ISSUE_ISW_W-1:0] decode_isw = wid_to_isw(decode_wid); + wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_decode_ready; + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_decode_if + assign slice_decode_if[i].valid = decode_if.valid && (decode_isw == ISSUE_ISW_W'(i)); + assign slice_decode_if[i].data = decode_if.data; + assign slice_decode_ready[i] = slice_decode_if[i].ready; + `UNUSED_VAR (slice_decode_if[i].ibuf_pop) + end + assign decode_if.ready = slice_decode_ready[decode_isw]; + assign decode_if.ibuf_pop = '0; + assign decode_ready = decode_if.ready; + `UNUSED_VAR (decode_if.ibuf_pop) for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_writeback_if assign writeback_if[i].valid = writeback_valid[i]; @@ -157,7 +171,7 @@ module VX_issue_top import VX_gpu_pkg::*; #( .issue_perf (issue_perf), `endif - .decode_if (decode_if), + .decode_if (slice_decode_if), .writeback_if (writeback_if), .dispatch_if (dispatch_if), .issue_sched_if (issue_sched_if) diff --git a/hw/unittest/raster_core/VX_raster_core_top.sv b/hw/unittest/raster_core/VX_raster_core_top.sv index 9e1665a093..fb64f41c63 100644 --- a/hw/unittest/raster_core/VX_raster_core_top.sv +++ b/hw/unittest/raster_core/VX_raster_core_top.sv @@ -108,7 +108,6 @@ module VX_raster_core_top import VX_gpu_pkg::*; import VX_raster_pkg::*; #( VX_raster_core #( .INSTANCE_ID (INSTANCE_ID), - .INSTANCE_IDX (INSTANCE_IDX), .NUM_INSTANCES (NUM_INSTANCES), .NUM_SLICES (NUM_SLICES), .TILE_LOGSIZE (TILE_LOGSIZE), @@ -120,6 +119,7 @@ module VX_raster_core_top import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `SCOPE_IO_BIND (0) .clk (clk), .reset (reset), + .instance_idx (`LOG2UP(NUM_INSTANCES)'(INSTANCE_IDX)), `ifdef PERF_ENABLE .perf_raster_if(perf_raster_if), `endif diff --git a/sim/firesim/firesim_sim.cpp b/sim/firesim/firesim_sim.cpp index 1ff8723c8c..9604c3a01e 100644 --- a/sim/firesim/firesim_sim.cpp +++ b/sim/firesim/firesim_sim.cpp @@ -483,7 +483,7 @@ void firesim_sim::report_progress(const char *why) { // they are parked at a barrier, and which pipeline is holding them. // // Requires the RTL to be built with PERF_ENABLE (see hw/syn/firesim/Makefile); -// without it VX_csr_data.sv ties the counters off and every read returns zero, +// without it VX_csr_mpm.sv ties the counters off and every read returns zero, // so an all-zero line here means the instrument is absent, not that the core is // idle. That distinction is exactly the kind this project has been bitten by, // so it is called out rather than left to be inferred. diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 9b3703bda0..da7a62eaa3 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -105,4 +105,13 @@ inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_ inline constexpr uint32_t PER_ISSUE_WARPS = VX_CFG_NUM_WARPS / VX_CFG_ISSUE_WIDTH; inline constexpr uint32_t ISSUE_WIS_BITS = log2ceil(PER_ISSUE_WARPS); +// Fetch slice f owns the warps with wid % VX_CFG_FETCH_WIDTH == f; each has its +// own warp scheduler, fetch unit, decoder and icache request port. +inline constexpr uint32_t PER_FETCH_WARPS = VX_CFG_NUM_WARPS / VX_CFG_FETCH_WIDTH; +inline constexpr uint32_t ICACHE_NUM_REQS = VX_CFG_FETCH_WIDTH; +static_assert(VX_CFG_FETCH_WIDTH != 0 && (VX_CFG_FETCH_WIDTH & (VX_CFG_FETCH_WIDTH - 1)) == 0 + && VX_CFG_FETCH_WIDTH <= VX_CFG_NUM_WARPS + && (VX_CFG_ISSUE_WIDTH % VX_CFG_FETCH_WIDTH) == 0, + "FETCH_WIDTH must be a power of two that divides ISSUE_WIDTH and is at most NUM_WARPS"); + } // namespace vortex diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index ec729d83e3..447314ce80 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -66,9 +66,6 @@ class Core::Impl { , func_units_((uint32_t)FUType::Count) , lmem_switch_(VX_CFG_NUM_LSU_BLOCKS) , mem_coalescers_(VX_CFG_NUM_LSU_BLOCKS) - , fetch_latch_(ctx, "fetch_latch", 2, 2) - , decode_latch_(ctx, "decode_latch", 1, 2) - , pending_icache_(VX_CFG_NUM_WARPS) , ibuffer_arbs_(VX_CFG_ISSUE_WIDTH, {ArbiterType::GreedyRR, PER_ISSUE_WARPS}) , issue_lock_owner_(VX_CFG_ISSUE_WIDTH, kLockOpen) , fu_credits_(VX_CFG_ISSUE_WIDTH, std::vector((uint32_t)FUType::Count, 0)) @@ -81,6 +78,16 @@ class Core::Impl { const std::string& name = simobject_->name(); char sname[100]; + // per fetch slice: scheduler-to-fetch and fetch-to-decode latches, and the + // icache tags of the slice's in-flight fetches + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + snprintf(sname, 100, "fetch_latch%d", f); + fetch_latch_.emplace_back(std::make_unique>(ctx, sname, 2, 2)); + snprintf(sname, 100, "decode_latch%d", f); + decode_latch_.emplace_back(std::make_unique>(ctx, sname, 1, 2)); + pending_icache_.emplace_back(PER_FETCH_WARPS); + } + // create scheduler (SimObject) — also creates the CTA dispatcher. snprintf(sname, 100, "%s-scheduler", name.c_str()); scheduler_ = SimPlatform::instance().create_object(sname, simobject_); @@ -92,8 +99,10 @@ class Core::Impl { #ifdef VX_CFG_EXT_C_ENABLE // create RVC fetch decompressor (SimObject) — owns per-warp half-word // buffer + refetch queue. Resets via SimObject::on_reset. - snprintf(sname, 100, "%s-decompressor", name.c_str()); - decompressor_ = SimPlatform::instance().create_object(sname, VX_CFG_NUM_WARPS); + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + snprintf(sname, 100, "%s-decompressor%d", name.c_str(), f); + decompressors_.push_back(SimPlatform::instance().create_object(sname, VX_CFG_NUM_WARPS)); + } #endif // create scoreboard (SimObject) @@ -209,11 +218,11 @@ class Core::Impl { dcache_mmu_ = Mmu::Create(sname, VX_CFG_DCACHE_NUM_REQS, VX_CFG_DTLB_SIZE, local_core * 2 + 0, false); - // Per-core icache MMU (1 port). Fetch reads/writes its upstream - // channels (ReqIn[0]/RspOut[0]) directly; the downstream side is - // bound to the Core's icache port below. + // Per-core icache MMU, one port per fetch slice. Fetch reads/writes its + // upstream channels directly; the downstream side is bound to the Core's + // icache ports below. snprintf(sname, 100, "%s-icache_mmu", name.c_str()); - icache_mmu_ = Mmu::Create(sname, 1, + icache_mmu_ = Mmu::Create(sname, ICACHE_NUM_REQS, VX_CFG_ITLB_SIZE, local_core * 2 + 1, true); // LSU memory side -> dcache MMU -> core's dcache port @@ -224,10 +233,12 @@ class Core::Impl { simobject_->dcache_rsp_in.at(p).bind(&dcache_mmu_->RspIn.at(p)); } - // icache MMU downstream side -> core's icache port. Upstream side - // (ReqIn[0]/RspOut[0]) is consumed directly by fetch() in on_tick. - icache_mmu_->ReqOut.at(0).bind(&simobject_->icache_req_out.at(0)); - simobject_->icache_rsp_in.at(0).bind(&icache_mmu_->RspIn.at(0)); + // icache MMU downstream side -> core's icache ports. Upstream side is + // consumed directly by fetch() in on_tick. + for (uint32_t f = 0; f < ICACHE_NUM_REQS; ++f) { + icache_mmu_->ReqOut.at(f).bind(&simobject_->icache_req_out.at(f)); + simobject_->icache_rsp_in.at(f).bind(&icache_mmu_->RspIn.at(f)); + } #else // No-VM: direct passthrough. for (uint32_t p = 0; p < VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS; ++p) { @@ -360,61 +371,88 @@ class Core::Impl { perf_stats_.active_warps += scheduler_->active_warps().count(); perf_stats_.stalled_warps += scheduler_->stalled_warps().count(); - // stop when fetch_latch cannot accept - if (fetch_latch_.full()) - return; + scheduler_->schedule_begin(); + + // each fetch slice picks among its own warps whose ibuffer is not yet saturated + // the cycle is idle when no slice has a warp to fetch, its latch free or not + bool any_cand = false; + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + auto& fetch_latch = fetch_latch_.at(f); + WarpMask warp_mask; + for (uint32_t w = f; w < VX_CFG_NUM_WARPS; w += VX_CFG_FETCH_WIDTH) { + if (ibuf_inflight_.at(w) < VX_CFG_IBUF_SIZE) { + warp_mask.set(w); + } + } - // eligible warps: ibuffer not yet saturated - WarpMask warp_mask; - for (uint32_t w = 0, nw = VX_CFG_NUM_WARPS; w < nw; ++w) { - if (ibuf_inflight_.at(w) < VX_CFG_IBUF_SIZE) { - warp_mask.set(w); + if (fetch_latch->full()) { + any_cand |= scheduler_->has_ready(warp_mask); + continue; } - } - auto trace = scheduler_->schedule(warp_mask); - if (trace == nullptr) { + auto trace = scheduler_->schedule(warp_mask); + if (trace == nullptr) { + continue; + } + any_cand = true; + + fetch_latch->push(trace); + DT(3, simobject_->name() << "-pipeline schedule: " << *trace); + pending_instrs_.push_back(trace); + ++ibuf_inflight_.at(trace->wid); + perf_stats_.issued_warps += 1; + perf_stats_.issued_threads += trace->tmask.count(); + } + if (!any_cand) { ++perf_stats_.sched_idle; - return; } - fetch_latch_.push(trace); - DT(3, simobject_->name() << "-pipeline schedule: " << *trace); - pending_instrs_.push_back(trace); - ++ibuf_inflight_.at(trace->wid); - perf_stats_.issued_warps += 1; - perf_stats_.issued_threads += trace->tmask.count(); + scheduler_->schedule_end(); } void fetch() { perf_stats_.ifetch_latency += pending_ifetches_; + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + this->fetch_slice(f); + } + } + + // Fetch slice f: its icache response into its decode latch, then its next + // fetch request out of its fetch latch, on icache port f. + void fetch_slice(uint32_t f) { + auto& fetch_latch = *fetch_latch_.at(f); + auto& decode_latch = *decode_latch_.at(f); + auto& pending_icache = pending_icache_.at(f); + #ifdef VX_CFG_EXT_C_ENABLE + auto& decompressor = decompressors_.at(f); + #endif #ifdef VX_CFG_VM_ENABLE // With VM, fetch reads/writes the icache MMU's upstream channels. // The MMU translates and forwards to simobject_->icache_req_out // downstream (driven by SimChannel binding in the constructor). - auto& icache_rsp = icache_mmu_->RspOut.at(0); - auto& icache_req = icache_mmu_->ReqIn.at(0); + auto& icache_rsp = icache_mmu_->RspOut.at(f); + auto& icache_req = icache_mmu_->ReqIn.at(f); #else - auto& icache_rsp = simobject_->icache_rsp_in.at(0); - auto& icache_req = simobject_->icache_req_out.at(0); + auto& icache_rsp = simobject_->icache_rsp_in.at(f); + auto& icache_req = simobject_->icache_req_out.at(f); #endif // handle icache response — extract instruction word from the TLM line payload if (!icache_rsp.empty()){ auto& mem_rsp = icache_rsp.peek(); - auto trace = pending_icache_.at(mem_rsp.tag); + auto trace = pending_icache.at(mem_rsp.tag); assert(mem_rsp.data && "icache response must carry line payload"); #ifdef VX_CFG_EXT_C_ENABLE // Decompressor classifies the rsp: emit-ready (sets trace->code) or // queues a refetch internally for cross-word 32-bit completion. - bool emit = decompressor_->on_icache_rsp(trace, *mem_rsp.data); - bool progressed = !emit || decode_latch_.try_push(trace); + bool emit = decompressor->on_icache_rsp(trace, *mem_rsp.data); + bool progressed = !emit || decode_latch.try_push(trace); #else bool emit = true; bool progressed = false; - if (decode_latch_.try_push(trace)) { + if (decode_latch.try_push(trace)) { uint32_t offset = trace->PC & (VX_CFG_MEM_BLOCK_SIZE - 1); std::memcpy(&trace->code, mem_rsp.data->data() + offset, sizeof(uint32_t)); progressed = true; @@ -427,7 +465,7 @@ class Core::Impl { << ", PC=0x" << std::hex << trace->PC << " (#" << std::dec << trace->uuid << ")"); } DT(3, simobject_->name() << " icache-rsp: addr=0x" << std::hex << trace->PC << ", tag=0x" << mem_rsp.tag << std::dec << ", " << *trace); - pending_icache_.release(mem_rsp.tag); + pending_icache.release(mem_rsp.tag); icache_rsp.pop(); --pending_ifetches_; } @@ -440,15 +478,15 @@ class Core::Impl { #ifdef VX_CFG_EXT_C_ENABLE bool from_refetch = false; { - auto pick = decompressor_->pick_request(fetch_latch_.empty() ? nullptr - : fetch_latch_.peek()); + auto pick = decompressor->pick_request(fetch_latch.empty() ? nullptr + : fetch_latch.peek()); trace = pick.trace; req_addr = pick.addr; from_refetch = pick.from_refetch; } #else - if (!fetch_latch_.empty()) { - trace = fetch_latch_.peek(); + if (!fetch_latch.empty()) { + trace = fetch_latch.peek(); req_addr = trace->PC; } #endif @@ -456,7 +494,7 @@ class Core::Impl { return; // Avoid leaking icache tags when the request port back-pressures. - if (pending_icache_.full()) { + if (pending_icache.full()) { ++perf_stats_.fetch_stalls; return; } @@ -466,31 +504,39 @@ class Core::Impl { // substitutes PA before forwarding; with VM off, this is just the PA. mem_req.addr = req_addr; // op defaults to MemOp::LD — icache request is a load. - uint32_t tag = pending_icache_.allocate(trace); + uint32_t tag = pending_icache.allocate(trace); mem_req.tag = tag; mem_req.hart_id = trace->cid; mem_req.uuid = trace->uuid; if (icache_req.try_send(mem_req)) { DT(3, simobject_->name() << " icache-req: addr=0x" << std::hex << mem_req.addr << ", tag=0x" << mem_req.tag << std::dec << ", " << *trace); #ifdef VX_CFG_EXT_C_ENABLE - decompressor_->commit_request(from_refetch); - if (!from_refetch) fetch_latch_.pop(); + decompressor->commit_request(from_refetch); + if (!from_refetch) { fetch_latch.pop(); } #else - fetch_latch_.pop(); + fetch_latch.pop(); #endif ++perf_stats_.ifetches; ++pending_ifetches_; } else { - pending_icache_.release(tag); + pending_icache.release(tag); ++perf_stats_.fetch_stalls; } } void decode() { - if (decode_latch_.empty()) + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + this->decode_slice(*decode_latch_.at(f)); + } + } + + // One fetch slice's decoder: it feeds the issue slice of each decoded warp. + void decode_slice(TFifo& decode_latch) { + if (decode_latch.empty()) { return; + } - auto trace = decode_latch_.peek(); + auto trace = decode_latch.peek(); // check ibuffer capacity auto& ibuffer = ibuffers_.at(trace->wid); @@ -549,7 +595,7 @@ class Core::Impl { trace->ibuf_time = SimPlatform::instance().cycles(); ibuffer->push(trace); - decode_latch_.pop(); + decode_latch.pop(); } void issue() { @@ -1092,10 +1138,10 @@ class Core::Impl { std::vector lmem_switch_; std::vector mem_coalescers_; - TFifo fetch_latch_; - TFifo decode_latch_; + std::vector>> fetch_latch_; + std::vector>> decode_latch_; - HashTable pending_icache_; + std::vector> pending_icache_; std::list> pending_instrs_; uint64_t pending_ifetches_; @@ -1121,9 +1167,9 @@ class Core::Impl { PoolAllocator trace_pool_; #ifdef VX_CFG_EXT_C_ENABLE - // Per-core RVC fetch SimObject (per-warp halfword buffer + refetch + // RVC fetch SimObject per fetch slice (per-warp halfword buffer + refetch // queue + pick/commit_request helpers). See decompressor.h. - Decompressor::Ptr decompressor_; + std::vector decompressors_; #endif #ifdef VX_CFG_VM_ENABLE Mmu::Ptr dcache_mmu_; @@ -1139,8 +1185,8 @@ Core::Core(const SimContext& ctx, Socket* socket ) : SimObject(ctx, name) - , icache_req_out(1, this) - , icache_rsp_in(1, this) + , icache_req_out(ICACHE_NUM_REQS, this) + , icache_rsp_in(ICACHE_NUM_REQS, this) , dcache_req_out(VX_CFG_DCACHE_NUM_REQS, this) , dcache_rsp_in(VX_CFG_DCACHE_NUM_REQS, this) , branch_ctl_in(this, VX_CFG_NUM_WARPS) diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index cb8c10f5e0..950c972df9 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -39,11 +39,17 @@ CtaDispatcher::CtaDispatcher(const SimContext& ctx, const char* name, Core* core , rank_(0) , block_size_rem_(0) , lmem_addr_(0) + , lane_rr_(0) + , init_wait_(0) , cur_kernel_pc_(0) , cur_ctx_id_(0) , warp_init_lanes_(num_warps_) { thread_idx_[0] = thread_idx_[1] = thread_idx_[2] = 0; + for (uint32_t i = 0; i < 3; ++i) { + shape_step_[i] = 0; + shape_dim_[i] = 0; + } for (auto& lanes : warp_init_lanes_) { lanes.resize(num_threads_); } @@ -54,11 +60,13 @@ CtaDispatcher::~CtaDispatcher() {} void CtaDispatcher::on_reset() { has_cta_ = false; tail_slot_ = 0; + init_wait_ = 0; for (uint32_t i = 0; i < num_warps_; ++i) { slot_rem_warps_[i] = 0; wid_to_slot_[i] = num_warps_; // invalid } cur_kernel_pc_ = 0; + // lane_rr_ and the cached block shape are device state, like warp_init_lanes_. // warp_init_lanes_ and cur_ctx_id_ are excluded: which lanes of a slot have // already run the startup is keyed on the context, and a launch is not a // device reset. Clearing them here would make the context check dead and @@ -91,14 +99,14 @@ uint32_t CtaDispatcher::usable_slots(uint32_t stride) const { return fit; } -bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_warp_record_t* rec_out) { +uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs) { if (!has_cta_) { // The KMU pushes CTAs down a registered, credit-gated launch lane; the // endpoint holds the one CTA committed to this core. It is popped only on // admission — an unadmitted CTA keeps the lane's credit, so the KMU // cannot over-commit work to a busy core. if (bus_in.empty()) - return false; + return 0; const kmu_req_t& next = bus_in.peek(); // Fixed-stride admission. Round the per-CTA LMEM size up to a @@ -133,13 +141,13 @@ bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_wa k = max_slots; // cluster larger than co-residency: clamp (degenerate) for (uint32_t i = 0; i < k; ++i) { if (slot_rem_warps_[base + i] != 0) - return false; // window not free yet — wait + return 0; // window not free yet — wait } } else { // Standalone CTA, or a following cluster member (its slot was reserved by // the first-of-cluster window above and is guaranteed free). if (slot_rem_warps_[base] != 0) - return false; + return 0; } // A new launch is a new context: every slot owes the startup again. Keyed on @@ -169,39 +177,65 @@ bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_wa tail_slot_ = (base + 1) % max_slots; has_cta_ = true; - } - // Find lowest-index free warp slot. - int free_wid = -1; - for (int wid = 0; wid < (int)num_warps_; ++wid) { - if (!active_warps.test(wid)) { - free_wid = wid; - break; + if (VX_CFG_FETCH_WIDTH > 1) { + bool same_shape = true; + for (uint32_t d = 0; d < 3; ++d) { + same_shape = same_shape && (shape_step_[d] == cta_.warp_step[d]) + && (shape_dim_[d] == cta_.block_dim[d]); + } + if (!same_shape) { + for (uint32_t d = 0; d < 3; ++d) { + shape_step_[d] = cta_.warp_step[d]; + shape_dim_[d] = cta_.block_dim[d]; + } + init_wait_ = VX_CFG_FETCH_WIDTH; + } } } - if (free_wid < 0) return false; - // Skip the startup only if every lane this launch activates has already run it. - uint32_t active = (block_size_rem_ >= num_threads_) ? num_threads_ : block_size_rem_; - auto& inited = warp_init_lanes_[free_wid]; - bool needs_init = false; - for (uint32_t t = 0; t < active; ++t) { - if (!inited.test(t)) { - needs_init = true; - break; - } + if (init_wait_ != 0) { + --init_wait_; + return 0; } - if (!next_warp(needs_init, rec_out)) - return false; - - for (uint32_t t = 0; t < active; ++t) { - inited.set(t); + // Lanes with a free slot, round-robin from lane_rr_, up to the warps left; + // each launches its lowest free slot. + uint32_t count = 0; + for (uint32_t o = 0; o < VX_CFG_FETCH_WIDTH && has_cta_; ++o) { + uint32_t lane = (lane_rr_ + o) % VX_CFG_FETCH_WIDTH; + int free_wid = -1; + for (uint32_t wid = lane; wid < num_warps_; wid += VX_CFG_FETCH_WIDTH) { + if (!active_warps.test(wid)) { + free_wid = int(wid); + break; + } + } + if (free_wid < 0) { + continue; + } + // Skip the startup only if every lane this launch activates has already run it. + uint32_t active = (block_size_rem_ >= num_threads_) ? num_threads_ : block_size_rem_; + auto& inited = warp_init_lanes_[free_wid]; + bool needs_init = false; + for (uint32_t t = 0; t < active; ++t) { + if (!inited.test(t)) { + needs_init = true; + break; + } + } + if (!next_warp(needs_init, &recs[count])) { + break; + } + for (uint32_t t = 0; t < active; ++t) { + inited.set(t); + } + wid_to_slot_[free_wid] = cur_slot_; + wids[count] = uint32_t(free_wid); + ++count; } - - wid_to_slot_[free_wid] = cur_slot_; - *wid_out = uint32_t(free_wid); - return true; + lane_rr_ = (lane_rr_ + count) % VX_CFG_FETCH_WIDTH; + return count; } void CtaDispatcher::warp_done(uint32_t wid) { @@ -263,7 +297,9 @@ bool CtaDispatcher::next_warp(bool do_init, cta_warp_record_t* out) { out->cluster_size = cta_.cluster_dim[0] * cta_.cluster_dim[1] * cta_.cluster_dim[2]; // Thread mask: full warp or partial last warp. + // The record is reused across launches, so clear the lanes of the last one. out->tmask.resize(num_threads_); + out->tmask.reset(); uint32_t active = (block_size_rem_ >= num_threads_) ? num_threads_ : block_size_rem_; for (uint32_t t = 0; t < active; ++t) { out->tmask.set(t); diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 74d266c2e3..2404f75055 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -40,8 +40,9 @@ struct cta_warp_record_t { uint32_t cluster_size; }; -// CTA dispatcher: consumes CTAs pushed by the KMU and admits one warp rank -// per step() into a free warp slot. +// CTA dispatcher: consumes CTAs pushed by the KMU and, per step(), launches up +// to one warp of the CTA on each launch lane, into a free warp slot of that +// lane's frontend slice. class CtaDispatcher : public SimObject { public: // Launch-lane endpoint (capacity 1): the CTA the KMU has committed to this @@ -55,9 +56,10 @@ class CtaDispatcher : public SimObject { CtaDispatcher(const SimContext& ctx, const char* name, Core* core); ~CtaDispatcher(); - // Try to dispatch one warp rank into a free warp slot. - // Returns true on success and sets *wid_out / *rec_out. - bool step(const WarpMask& active_warps, uint32_t* wid_out, cta_warp_record_t* rec_out); + // Dispatch warps into free warp slots, at most one per launch lane: lane l + // owns the slots with wid % VX_CFG_FETCH_WIDTH == l. Fills wids[] / recs[] in + // launch order and returns how many warps were dispatched. + uint32_t step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs); // Notify that a warp has exited; frees its lmem slot when all warps of the CTA are done. void warp_done(uint32_t wid); @@ -120,6 +122,14 @@ class CtaDispatcher : public SimObject { uint32_t thread_idx_[3]; uint64_t lmem_addr_; + // Launch lanes are taken round-robin from lane_rr_. Several lanes need the + // per-lane thread-index window rebuilt when the block shape changes, which + // holds the launch for one cycle per lane. + uint32_t lane_rr_; + uint32_t init_wait_; + uint32_t shape_step_[3]; + uint32_t shape_dim_[3]; + Word cur_kernel_pc_; uint8_t cur_ctx_id_; // Which lanes of each warp slot have run the startup stub. Per LANE, not per diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 5f3c357644..abbaf45ced 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -27,6 +27,10 @@ using namespace vortex; +// response tags index the in-flight slots by masking +static_assert(VX_CFG_DXA_MAX_INFLIGHT >= 2 && (VX_CFG_DXA_MAX_INFLIGHT & (VX_CFG_DXA_MAX_INFLIGHT - 1)) == 0, + "DXA_MAX_INFLIGHT must be a power of two of at least 2"); + namespace { // One GMEM port: the DXA joins its socket's L2-facing arb as a single peer. diff --git a/sim/simx/dxa/dxa_unit.cpp b/sim/simx/dxa/dxa_unit.cpp index 42db10595f..e84f2ea0d2 100644 --- a/sim/simx/dxa/dxa_unit.cpp +++ b/sim/simx/dxa/dxa_unit.cpp @@ -62,7 +62,7 @@ instr_trace_t* DxaUnit::process(instr_trace_t* trace) { // completion; pre-registration happens explicitly per-CTA so multicast // destinations correctly wait. - req_out_.send(req); + req_out_.send(req, 1 + bus_delay_); DT(4, "dxa-unit submit: core=" << core_->id() << ", wid=" << trace->wid << ", slot=" << desc_slot << ", bar=" << raw_bar << ", cta_mask=0x" << std::hex << cta_mask << std::dec); diff --git a/sim/simx/dxa/dxa_unit.h b/sim/simx/dxa/dxa_unit.h index c076a14b53..04ab7c3435 100644 --- a/sim/simx/dxa/dxa_unit.h +++ b/sim/simx/dxa/dxa_unit.h @@ -43,8 +43,10 @@ struct DxaReq { class DxaUnit { public: // The outbound channel lives on SfuUnit (which is the SimObject). - DxaUnit(Core* core, SimChannel& req_out) - : core_(core), req_out_(req_out) {} + // bus_delay: extra cycles a request spends merging with other SFU blocks' + // requests onto the core's DXA bus. + DxaUnit(Core* core, SimChannel& req_out, uint32_t bus_delay = 0) + : core_(core), req_out_(req_out), bus_delay_(bus_delay) {} // Decode lanes 0..3 from the trace's source operands and try to push a // DxaReq on the SFU's outbound channel. Returns the trace on success @@ -55,6 +57,7 @@ class DxaUnit { private: Core* core_; SimChannel& req_out_; + uint32_t bus_delay_; }; } // namespace vortex diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 78fd3fd466..2119b6d76f 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -176,7 +176,7 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { // The datapath has no misalignment support: only a naturally aligned // access is guaranteed to sit inside one memory block, which is what // lets a lane's payload pack into a single block downstream. - // VX_lsu_slice.sv enforces the same contract in hardware, so modelling + // The hardware LSU enforces the same contract, so modelling // anything looser here would let a shader pass in simulation and fail // on the device. if ((e.addr % e.size) != 0) { diff --git a/sim/simx/rtu/rtu_unit.cpp b/sim/simx/rtu/rtu_unit.cpp index 2d8cef624f..4f8878c3ba 100644 --- a/sim/simx/rtu/rtu_unit.cpp +++ b/sim/simx/rtu/rtu_unit.cpp @@ -35,10 +35,11 @@ inline uint32_t float_to_bits(float f) { } } // namespace -RtuUnit::RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window) +RtuUnit::RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window, uint32_t bus_delay) : window_(window) , core_(core) , req_out_(req_out) + , bus_delay_(bus_delay) { trace_slot_.fill(-1); for (auto& s : trace_scene_) s.fill(0); @@ -252,7 +253,7 @@ instr_trace_t* RtuUnit::process_cb_ret(instr_trace_t* trace, uint32_t block_id) trace->dst_data[t].u = 0; // no writeback } req.tmask_bits = bits; - req_out_.send(req); + req_out_.send(req, 1 + bus_delay_); DT(3, "rtu-cb_ret submit: core=" << core_->id() << ", wid=" << trace->wid << ", tmask=0x" << std::hex << bits << std::dec); return trace; @@ -527,7 +528,7 @@ instr_trace_t* RtuUnit::process_trace_uop(instr_trace_t* trace, uint32_t block_i req.cull_mask[t] = trace_cull_.at(wid); } req.tmask_bits = bits; - req_out_.send(req); + req_out_.send(req, 1 + bus_delay_); trace_slot_.at(wid) = -1; DT(3, "rtu-trace arm: core=" << core_->id() << ", wid=" << wid << ", slot=" << slot << ", tmask=0x" << std::hex << bits << std::dec); diff --git a/sim/simx/rtu/rtu_unit.h b/sim/simx/rtu/rtu_unit.h index 978e7ea511..a1ad4bdc4f 100644 --- a/sim/simx/rtu/rtu_unit.h +++ b/sim/simx/rtu/rtu_unit.h @@ -69,7 +69,9 @@ class RtuUopGen { // register file. Plain (non-SimObject) helper owned by SfuUnit. class RtuUnit { public: - RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window); + // bus_delay: extra cycles a request spends merging with other SFU blocks' + // requests onto the core's RTU bus. + RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window, uint32_t bus_delay = 0); // Async ray pool. process_wait either: // - returns the trace with the per-lane status word written @@ -187,6 +189,7 @@ class RtuUnit { Core* core_; SimChannel& req_out_; + uint32_t bus_delay_; // Async ray pool. Borrowed from the Socket via set_rtu_core(); // null until the Socket has wired it (TRACE/WAIT paths must NEVER // dereference rtu_core_ before that — but in practice the Socket diff --git a/sim/simx/scheduler.cpp b/sim/simx/scheduler.cpp index 551adf6976..793b1c4074 100644 --- a/sim/simx/scheduler.cpp +++ b/sim/simx/scheduler.cpp @@ -89,7 +89,7 @@ void Scheduler::on_reset() { stalled_warps_next_.reset(); active_warps_.reset(); cta_active_view_.reset(); - cta_fire_ = false; + cta_fire_cnt_ = 0; // Sequencers live on Core now; Core::on_reset() resets them. wspawn_.valid = false; @@ -155,26 +155,15 @@ void Scheduler::activate_warp(uint32_t wid, const cta_warp_record_t& rec) { << ", mscratch=0x" << std::hex << warp.mscratch << std::dec); } -instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { - int scheduled_warp = -1; - - // Dispatch one CTA warp. The dispatcher picks from the registered active - // set and its pick lands the next cycle; the warp it activates becomes - // schedulable the cycle after that. - if (cta_fire_) { - activate_warp(cta_fire_wid_, cta_fire_rec_); - stalled_warps_.set(cta_fire_wid_); - cta_fire_ = false; - } - { - uint32_t wid; - cta_warp_record_t rec; - if (cta_dispatcher_->step(cta_active_view_, &wid, &rec)) { - cta_fire_ = true; - cta_fire_wid_ = wid; - cta_fire_rec_ = rec; - } +void Scheduler::schedule_begin() { + // Dispatch CTA warps, one per launch lane. The dispatcher picks from the + // registered active set and its picks land the next cycle; a warp it + // activates becomes schedulable the cycle after that. + for (uint32_t i = 0; i < cta_fire_cnt_; ++i) { + activate_warp(cta_fire_wids_[i], cta_fire_recs_[i]); + stalled_warps_.set(cta_fire_wids_[i]); } + cta_fire_cnt_ = cta_dispatcher_->step(cta_active_view_, cta_fire_wids_.data(), cta_fire_recs_.data()); #ifdef VX_CFG_EXT_RASTER_ENABLE // Inject ready fragment waves into free warp slots. @@ -200,6 +189,14 @@ instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { wspawn_.valid = false; this->resume(0); } +} + +bool Scheduler::has_ready(const WarpMask& warp_mask) const { + return (active_warps_ & ~stalled_warps_ & warp_mask).any(); +} + +instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { + int scheduled_warp = -1; // pick next ready warp for (size_t wid = 0, nw = VX_CFG_NUM_WARPS; wid < nw; ++wid) { @@ -244,20 +241,23 @@ instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { this->suspend(scheduled_warp); } + return trace; +} + +void Scheduler::schedule_end() { // Clock the registered warp-stall state. The suspend above, or a resume from // decode/commit/FU earlier this cycle, drives stalled_warps_next_ and only // becomes visible to the pick loop next cycle — so a warp released as its // instruction resolves is never re-scheduled the same cycle. stalled_warps_ = stalled_warps_next_; cta_active_view_ = active_warps_; - if (cta_fire_) { - cta_active_view_.set(cta_fire_wid_); + for (uint32_t i = 0; i < cta_fire_cnt_; ++i) { + cta_active_view_.set(cta_fire_wids_[i]); } - return trace; } bool Scheduler::running() const { - return active_warps_.any() || cta_fire_ || cta_dispatcher_->running() + return active_warps_.any() || (cta_fire_cnt_ != 0) || cta_dispatcher_->running() #ifdef VX_CFG_EXT_RASTER_ENABLE || fwd_armed_ #endif @@ -417,7 +417,11 @@ void Scheduler::fwd_try_inject() { // Find any free warp slot — there is no driver warp to skip in the push model. int wid = -1; for (uint32_t w = 0; w < VX_CFG_NUM_WARPS; ++w) { - if (!active_warps_.test(w) && !(cta_fire_ && w == cta_fire_wid_)) { wid = int(w); break; } + bool firing = false; + for (uint32_t i = 0; i < cta_fire_cnt_; ++i) { + firing = firing || (cta_fire_wids_[i] == w); + } + if (!active_warps_.test(w) && !firing) { wid = int(w); break; } } if (wid < 0) break; // no free slot this cycle diff --git a/sim/simx/scheduler.h b/sim/simx/scheduler.h index a397eb60ba..58f10378b0 100644 --- a/sim/simx/scheduler.h +++ b/sim/simx/scheduler.h @@ -118,7 +118,18 @@ class Scheduler : public SimObject { ~Scheduler(); // ----- Warp lifecycle ----- + // Once per cycle, before the fetch slices pick: activate the warps the CTA + // dispatcher launched last cycle, launch the next ones, and apply a wspawn. + void schedule_begin(); + + // One fetch slice's pick: its lowest ready warp in warp_mask. instr_trace_t* schedule(const WarpMask& warp_mask); + + // Whether warp_mask holds a ready warp, without picking it. + bool has_ready(const WarpMask& warp_mask) const; + + // Once per cycle, after the picks: clock the registered warp state. + void schedule_end(); void suspend(uint32_t wid); void resume(uint32_t wid); // Advance the warp's PC by `inc` bytes (called at decode with 2 or 4 @@ -211,12 +222,12 @@ class Scheduler : public SimObject { WarpMask active_warps_; WarpMask stalled_warps_; // registered (current) state read by schedule() WarpMask stalled_warps_next_; // next-state written by suspend()/resume() - // CTA dispatch pipeline: the active set the dispatcher sees, and the warp - // it picked last cycle. + // the active set the CTA dispatcher sees WarpMask cta_active_view_; - bool cta_fire_ = false; - uint32_t cta_fire_wid_ = 0; - cta_warp_record_t cta_fire_rec_; + // Warps the CTA dispatcher launched this cycle, one per launch lane at most. + uint32_t cta_fire_cnt_ = 0; + std::array cta_fire_wids_; + std::array cta_fire_recs_; uint32_t ipdom_size_; wspawn_t wspawn_; uint32_t mpm_class_; diff --git a/sim/simx/sfu_unit.cpp b/sim/simx/sfu_unit.cpp index de61756629..edf9311c5f 100644 --- a/sim/simx/sfu_unit.cpp +++ b/sim/simx/sfu_unit.cpp @@ -30,6 +30,10 @@ using namespace vortex; +// With several SFU blocks, their TEX/RTU/DXA requests merge onto the core's bus +// through one registered arbiter stage. +static constexpr uint32_t kBusMergeDelay = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 1 : 0; + SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) : FuncUnit(ctx, name, core, 6) , warp_ctl_out(this, VX_CFG_NUM_WARPS) @@ -54,16 +58,16 @@ SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) , wctl_unit_(new WctlUnit(core)) , csr_unit_(new CsrUnit(core)) #ifdef VX_CFG_EXT_DXA_ENABLE - , dxa_unit_(new DxaUnit(core, dxa_req_out)) + , dxa_unit_(new DxaUnit(core, dxa_req_out, kBusMergeDelay)) #endif #ifdef VX_CFG_EXT_TEX_ENABLE - , tex_unit_(new TexUnit(core, tex_req_out)) + , tex_unit_(new TexUnit(core, tex_req_out, kBusMergeDelay)) #endif #ifdef VX_CFG_EXT_OM_ENABLE , om_unit_(new OmUnit(core, om_req_out)) #endif #ifdef VX_CFG_EXT_RTU_ENABLE - , rtu_unit_(new RtuUnit(core, rtu_req_out, rtu_window_)) + , rtu_unit_(new RtuUnit(core, rtu_req_out, rtu_window_, kBusMergeDelay)) #endif { } @@ -256,12 +260,36 @@ void SfuUnit::on_tick() { } #endif + // Grant each shared resource to one requesting block this cycle. + SharedRes want[VX_CFG_NUM_SFU_BLOCKS]; + int winner[RES_COUNT]; + instr_trace_t* winner_head[RES_COUNT] = {}; + for (uint32_t r = 0; r < RES_COUNT; ++r) { + winner[r] = -1; + } + for (uint32_t b = 0; b < VX_CFG_NUM_SFU_BLOCKS; ++b) { + want[b] = this->shared_res(b); + } + for (uint32_t r = RES_WCTL; r < RES_COUNT; ++r) { + for (uint32_t i = 0; i < VX_CFG_NUM_SFU_BLOCKS; ++i) { + uint32_t b = (res_rr_[r] + i) % VX_CFG_NUM_SFU_BLOCKS; + if (want[b] == r) { + winner[r] = b; + winner_head[r] = Inputs.at(b).peek(); + break; + } + } + } + // PE switch: peek input, route to the matching sub-unit (WCTL / CSR / // DXA / TEX / OM / RASTER) by op_type, gather to the single result port. for (uint32_t b = 0; b < VX_CFG_NUM_SFU_BLOCKS; ++b) { auto& input = Inputs.at(b); if (input.empty()) continue; + if (want[b] != RES_NONE && winner[want[b]] != int(b)) { + continue; // another block holds the shared resource this cycle + } auto& output = Outputs.at(b); auto trace = input.peek(); @@ -441,4 +469,53 @@ void SfuUnit::on_tick() { input.pop(); } + + // A granted request that went through moves its resource's pointer past it. + for (uint32_t r = RES_WCTL; r < RES_COUNT; ++r) { + if (winner[r] < 0) { + continue; + } + auto& input = Inputs.at(winner[r]); + if (input.empty() || input.peek() != winner_head[r]) { + res_rr_[r] = (winner[r] + 1) % VX_CFG_NUM_SFU_BLOCKS; + } + } +} + +SfuUnit::SharedRes SfuUnit::shared_res(uint32_t block) { + auto& input = Inputs.at(block); + if (input.empty()) { + return RES_NONE; + } + auto trace = input.peek(); +#ifdef VX_CFG_EXT_TEX_ENABLE + if (std::get_if(&trace->op_type)) { + return RES_TEX; + } +#endif +#ifdef VX_CFG_EXT_RTU_ENABLE + if (auto rtu_p = std::get_if(&trace->op_type)) { + if (*rtu_p == GfxwType::TRACE || *rtu_p == GfxwType::CB_RET) { + return RES_RTU; + } + } +#endif +#ifdef VX_CFG_EXT_DXA_ENABLE + if (std::get_if(&trace->op_type)) { + return RES_DXA; + } +#endif + if (auto wctl_p = std::get_if(&trace->op_type)) { + // a block requests only when its op could fire this cycle + if (Outputs.at(block).full()) { + return RES_NONE; + } + if (*wctl_p == WctlType::BAR && trace->eop && !core_->lsu_drained()) { + return RES_NONE; + } + if (wctl_unit_->is_shared(trace)) { + return RES_WCTL; + } + } + return RES_NONE; } diff --git a/sim/simx/sfu_unit.h b/sim/simx/sfu_unit.h index 1eb74a31f1..ffc26c22b0 100644 --- a/sim/simx/sfu_unit.h +++ b/sim/simx/sfu_unit.h @@ -123,6 +123,13 @@ class SfuUnit : public FuncUnit { private: uint32_t latency_of(const instr_trace_t* trace) const; + // Core-wide resources the SFU blocks share: the warp-control resources + // (barrier unit, wspawn, warp retirement) and the TEX/RTU/DXA buses each + // accept one request per cycle, granted round-robin per resource. + enum SharedRes { RES_NONE = 0, RES_WCTL, RES_TEX, RES_RTU, RES_DXA, RES_COUNT }; + SharedRes shared_res(uint32_t block); + uint32_t res_rr_[RES_COUNT] = {}; + std::unique_ptr wctl_unit_; std::unique_ptr csr_unit_; #ifdef VX_CFG_EXT_RTU_ENABLE diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 5a1861155a..536dfe4d1e 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -60,9 +60,9 @@ class Socket::Impl { log2ceil(VX_CFG_L1_LINE_SIZE), // S (no sectoring) log2ceil(sizeof(uint32_t)), // W log2ceil(VX_CFG_ICACHE_NUM_WAYS),// A - log2ceil(1), // B + log2ceil(VX_CFG_ICACHE_NUM_BANKS), // B VX_CFG_XLEN, // address bits - 1, // number of inputs + ICACHE_NUM_REQS, // number of inputs (one per fetch slice) VX_CFG_ICACHE_MEM_PORTS, // memory ports false, // write-back false, // write response @@ -276,8 +276,10 @@ class Socket::Impl { // connect cores to caches for (uint32_t i = 0; i < cores_per_socket; ++i) { - cores_.at(i)->icache_req_out.at(0).bind(&icaches_->core_req_in.at(i).at(0)); - icaches_->core_rsp_out.at(i).at(0).bind(&cores_.at(i)->icache_rsp_in.at(0)); + for (uint32_t j = 0; j < ICACHE_NUM_REQS; ++j) { + cores_.at(i)->icache_req_out.at(j).bind(&icaches_->core_req_in.at(i).at(j)); + icaches_->core_rsp_out.at(i).at(j).bind(&cores_.at(i)->icache_rsp_in.at(j)); + } for (uint32_t j = 0; j < VX_CFG_DCACHE_NUM_REQS; ++j) { cores_.at(i)->dcache_req_out.at(j).bind(&dcaches_->core_req_in.at(i).at(j)); diff --git a/sim/simx/tex/tex_unit.cpp b/sim/simx/tex/tex_unit.cpp index fd88c0731b..462a8f924f 100644 --- a/sim/simx/tex/tex_unit.cpp +++ b/sim/simx/tex/tex_unit.cpp @@ -49,7 +49,7 @@ instr_trace_t* TexUnit::process(instr_trace_t* trace, uint32_t block_id) { // Combinational into the shared arbiter; the arbiter's forward is the // registered boundary. - req_out_.send(req, 0); + req_out_.send(req, bus_delay_); DT(3, "tex-unit submit: core=" << core_->id() << ", wid=" << trace->wid << ", stage=" << args.stage); return trace; diff --git a/sim/simx/tex/tex_unit.h b/sim/simx/tex/tex_unit.h index 710c011701..65b1319eb5 100644 --- a/sim/simx/tex/tex_unit.h +++ b/sim/simx/tex/tex_unit.h @@ -84,8 +84,10 @@ class TexCore; // backpressure (SfuUnit retries idempotently next cycle). class TexUnit { public: - TexUnit(Core* core, SimChannel& req_out) - : core_(core), req_out_(req_out) {} + // bus_delay: extra cycles a request spends merging with other SFU blocks' + // requests onto the core's texture bus. + TexUnit(Core* core, SimChannel& req_out, uint32_t bus_delay = 0) + : core_(core), req_out_(req_out), bus_delay_(bus_delay) {} // Submit one sample request: u/v/lod ride src_data[0..2], the texel returns in // rd. Returns the trace if accepted, or nullptr on full output channel. @@ -94,6 +96,7 @@ class TexUnit { private: Core* core_; SimChannel& req_out_; + uint32_t bus_delay_; }; } // namespace vortex diff --git a/sim/simx/wctl_unit.cpp b/sim/simx/wctl_unit.cpp index 4fbe8a531e..e989360acc 100644 --- a/sim/simx/wctl_unit.cpp +++ b/sim/simx/wctl_unit.cpp @@ -20,6 +20,43 @@ using namespace vortex; +bool WctlUnit::is_shared(const instr_trace_t* trace) const { + if (!trace->eop) { + return false; + } + auto wctl_type = std::get(trace->op_type); + if (wctl_type == WctlType::BAR || wctl_type == WctlType::WSPAWN) { + return true; + } + if (wctl_type != WctlType::TMC && wctl_type != WctlType::PRED) { + return false; + } + auto& warp = core_->scheduler().warp(trace->wid); + uint32_t num_threads = VX_CFG_NUM_THREADS; + int32_t thread_last = num_threads - 1; + for (; thread_last >= 0; --thread_last) { + if (warp.tmask.test(thread_last)) { break; } + } + auto& rs1_data = trace->src_data[0]; + auto& rs2_data = trace->src_data[1]; + ThreadMask next_tmask(num_threads); + if (wctl_type == WctlType::TMC) { + for (uint32_t t = 0; t < num_threads; ++t) { + next_tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); + } + } else { + auto wctlArgs = std::get(trace->instr_ptr->get_args()); + auto not_pred = wctlArgs.is_cond_neg; + ThreadMask pred(num_threads); + for (uint32_t t = 0; t < num_threads; ++t) { + auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; + pred[t] = warp.tmask.test(t) && cond; + } + next_tmask = pred.any() ? (warp.tmask & pred) : ThreadMask(num_threads, rs2_data.at(thread_last).u); + } + return next_tmask.none(); +} + bool WctlUnit::process(instr_trace_t* trace, bool* exit) { bool release_warp = trace->fetch_stall; *exit = false; diff --git a/sim/simx/wctl_unit.h b/sim/simx/wctl_unit.h index 799c34647e..e688a59abd 100644 --- a/sim/simx/wctl_unit.h +++ b/sim/simx/wctl_unit.h @@ -34,6 +34,11 @@ class WctlUnit { // the latency. bool process(instr_trace_t* trace, bool* exit); + // Whether `trace` needs a core-wide warp-control resource (the barrier unit, + // wspawn, or warp retirement), which takes one request per cycle across all + // SFU blocks. No side effects. + bool is_shared(const instr_trace_t* trace) const; + private: Core* core_; }; From 285e94684974675a67fbb089e302c07f1f0c1725 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:11 -0700 Subject: [PATCH 037/336] simx: model the RTL's buffer structure at every pipeline boundary SimX matched the RTL's uncontended latencies but not its structure: buffers had different capacities and ready timing, so under contention the two models waited at different points, and several end-to-end latencies matched only because errors on the request and response sides cancelled. A cycle-level trace diff of both models on targeted microbenchmarks located each difference; each change below matches the RTL element it models. SimX: - SimChannel gains an opt-in registered ready: a slot freed by a pop is offered to producers from the next cycle, independent of tick order. - Operand collector: an instruction holds its input slot until its last operand read (the 2-entry skid buffer after the scoreboard), with registered ready; bank-conflict cost is max-per-bank - 1. - Issue grants only when the collector input has room. - Arbiter copies get their own state. std::vector copies shared one implementation, so every issue slice shared grant state. - SFU: CSR result one cycle later (the hold was counted twice); registered ready on the block inputs; a barrier arrive is non-posted (request after the LSU drains, applied two cycles later, completes the cycle after) and frees the shared warp-control port once sent. - Barrier releases, local and global, go through one registered unlock path, as the RTL's unlock_valid/unlock_mask do. - LSU drained() includes pending requests, as the RTL's barrier drain check does; stores commit at queue accept. - dcache: a port arbiter grants one core port per bank per cycle with a round-robin pointer per bank (the RTL core crossbar), only when the bank's input buffer has room; the non-cacheable bypass layer adds its response register on the dcache, L2 and L3 (NC_ENABLE in the RTL). - icache port 0 carries the flush register in front of the cache. - Local memory: the request and response crossbars become one cycle deeper above 1.5x the fan-out limit (the RTL's arbiter tree). - CTA dispatch: the launch pipeline's init delay. - DXA: SMEM write timing per layout and the engine's request staging. - Functional builds (-DSIMX_FUNCTIONAL) accept the registered-ready option and ignore it. RTL: - VX_dxa_gmem_req buffers memory responses per slot, so one slow response no longer blocks the others (head-of-line). It holds one entry per busy slot, so it never overflows. Testbench: - rtlsim completes DRAM requests in per-bank arrival order, as the AXI platform does. Configuration: - NUM_SFU_BLOCKS defaults to 1, as before the split. With one SFU block the stage-sliced back end is selected for ISSUE_WIDTH > 1, so the CI cases that cover the compute-slice back end set the SFU block count explicitly. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/dxa/VX_dxa_gmem_req.sv | 50 +++++++++++- sim/common/simobject.h | 35 ++++++++- sim/common/simobject_functional.h | 4 + sim/rtlsim/processor.cpp | 40 ++++++---- sim/simx/barrier_unit.cpp | 33 +++++--- sim/simx/barrier_unit.h | 9 ++- sim/simx/cluster.cpp | 1 + sim/simx/core.cpp | 74 +++++++++++++----- sim/simx/core.h | 3 +- sim/simx/cta_dispatcher.cpp | 18 ++++- sim/simx/cta_dispatcher.h | 2 + sim/simx/dxa/dxa_core.cpp | 44 +++++++++-- sim/simx/lsu_unit.cpp | 30 ++++++-- sim/simx/mem/cache.cpp | 123 +++++++++++++++++++++++++----- sim/simx/mem/cache.h | 1 + sim/simx/mem/local_mem.cpp | 12 ++- sim/simx/opc_unit.cpp | 38 +++++---- sim/simx/opc_unit.h | 2 + sim/simx/processor.cpp | 1 + sim/simx/sfu_unit.cpp | 77 +++++++++++++++++-- sim/simx/sfu_unit.h | 12 +++ sim/simx/socket.cpp | 1 + sim/simx/types.h | 19 ++++- sim/simx/wctl_unit.cpp | 75 ++++++++---------- sim/simx/wctl_unit.h | 3 + 25 files changed, 556 insertions(+), 151 deletions(-) diff --git a/hw/rtl/dxa/VX_dxa_gmem_req.sv b/hw/rtl/dxa/VX_dxa_gmem_req.sv index 43e467ab46..929b02c3e7 100644 --- a/hw/rtl/dxa/VX_dxa_gmem_req.sv +++ b/hw/rtl/dxa/VX_dxa_gmem_req.sv @@ -15,9 +15,9 @@ // // Manages a slot-table-indexed pool of MAX_OUTSTANDING in-flight tags. On // each accept, allocates a free slot and writes per-tag metadata. GMEM -// responses (out-of-order from the L1) are streamed directly to smem_wr -// via the `sw_*` channel — no rsp_buf BRAM. OOB entries skip the bus and -// are presented to smem_wr via an oob_pending[] bitvector. +// responses (out-of-order from the L1) queue in a response buffer with one +// entry per slot and drain to smem_wr via the `sw_*` channel. OOB entries +// skip the bus and are presented to smem_wr via an oob_pending[] bitvector. // // Tag reuse is per-slot via a busy[] bitvector and outstanding counter; // release order is independent of issue order (OOO drain in smem_wr). @@ -242,6 +242,48 @@ module VX_dxa_gmem_req import VX_gpu_pkg::*, VX_dxa_pkg::*; #( end end + VX_mem_bus_if #( + .DATA_SIZE (GMEM_BYTES), + .TAG_WIDTH (GMEM_TAG_WIDTH) + ) mem_bus_raw (); + + assign mem_bus_raw.req_valid = mem_bus_w.req_valid; + assign mem_bus_raw.req_data = mem_bus_w.req_data; + assign mem_bus_w.req_ready = mem_bus_raw.req_ready; + + // smem_wr takes a scattered line one element per cycle. The memory + // response path is shared with the L1 fills, so a response must never + // wait on it: one entry per slot means the buffer is never full. A + // response passes straight through when the buffer is empty and smem_wr + // is ready. + wire rsp_buf_empty, rsp_buf_full; + wire [GMEM_DATAW + GMEM_TAG_WIDTH-1:0] rsp_buf_data; + wire rsp_bypass = rsp_buf_empty && mem_bus_w.rsp_ready; + + VX_fifo_queue #( + .DATAW (GMEM_DATAW + GMEM_TAG_WIDTH), + .DEPTH (MAX_OUTSTANDING) + ) rsp_buf ( + .clk (clk), + .reset (reset), + .push (mem_bus_raw.rsp_valid && ~rsp_bypass), + .pop (~rsp_buf_empty && mem_bus_w.rsp_ready), + .data_in (mem_bus_raw.rsp_data), + .data_out (rsp_buf_data), + .empty (rsp_buf_empty), + .full (rsp_buf_full), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + + `RUNTIME_ASSERT(~(mem_bus_raw.rsp_valid && ~rsp_bypass && rsp_buf_full), + ("%t: GMEM response buffer overflow", $time)) + + assign mem_bus_raw.rsp_ready = 1'b1; + assign mem_bus_w.rsp_valid = mem_bus_raw.rsp_valid || ~rsp_buf_empty; + assign mem_bus_w.rsp_data = rsp_buf_empty ? mem_bus_raw.rsp_data : rsp_buf_data; + VX_mem_bus_slice #( .DATA_SIZE (GMEM_BYTES), .TAG_WIDTH (GMEM_TAG_WIDTH), @@ -250,7 +292,7 @@ module VX_dxa_gmem_req import VX_gpu_pkg::*, VX_dxa_pkg::*; #( ) gmem_out_slice ( .clk (clk), .reset (reset), - .bus_in_if (mem_bus_w), + .bus_in_if (mem_bus_raw), .bus_out_if (gmem_bus_if) ); diff --git a/sim/common/simobject.h b/sim/common/simobject.h index b3c9879dc8..225dab157a 100644 --- a/sim/common/simobject.h +++ b/sim/common/simobject.h @@ -245,6 +245,10 @@ class SimPlatform { return (tl_exec_domain_ == HOST_DOMAIN) ? cycles_ : tl_cycles_; } + // Counts resets: cycle numbers repeat across runs, so per-cycle state is + // tagged with the run it belongs to. + uint64_t reset_gen() const { return reset_gen_; } + // Sends issued outside any object tick (driver/host calls). static constexpr uint32_t HOST_DOMAIN = 0xffffffffu; @@ -387,6 +391,7 @@ class SimPlatform { std::vector> domains_; uint64_t cycles_; + uint64_t reset_gen_ = 0; uint32_t num_domains_ = 1; uint32_t build_domain_ = 0; @@ -651,6 +656,7 @@ class SimChannel : public SimChannelBase { : SimChannelBase(other.module_) , storage_(other.storage_.capacity()) , pending_count_(0) + , reg_ready_(other.reg_ready_) , convert_fn_(other.convert_fn_) , tx_cb_(other.tx_cb_) { sink_ = nullptr; @@ -666,6 +672,9 @@ class SimChannel : public SimChannelBase { pop_cb_ = {}; storage_ = RingQueue(other.storage_.capacity()); pending_count_ = 0; + reg_ready_ = other.reg_ready_; + freed_cycle_ = ~uint64_t(0); + freed_count_ = 0; convert_fn_ = other.convert_fn_; tx_cb_ = other.tx_cb_; } @@ -704,9 +713,14 @@ class SimChannel : public SimChannelBase { return sink_->full(); } assert(!SimPlatform::instance().mt_cross_access(module_)); - return this->occupancy() >= storage_.capacity(); + return this->occupancy() + this->freed_now() >= storage_.capacity(); } + // A registered ready: a slot freed by a pop is offered to producers from the + // next cycle, as a buffer whose full flag is a register does, whatever order + // the producer and the consumer tick in. + void set_registered_ready(bool enable) { reg_ready_ = enable; } + void send(const Pkt& pkt, uint64_t delay = 1) { auto& platform = SimPlatform::instance(); // A send whose endpoint lives in another domain is handed to that @@ -849,7 +863,21 @@ class SimChannel : public SimChannelBase { bool queue_empty() const { return storage_.empty(); } uint32_t queue_size() const { return storage_.size(); } const Pkt& queue_front() const { return storage_.front(); } + uint32_t freed_now() const { + auto& platform = SimPlatform::instance(); + return (reg_ready_ && freed_cycle_ == platform.cycles() && freed_gen_ == platform.reset_gen()) + ? freed_count_ : 0; + } void queue_pop() { + if (reg_ready_) { + auto& platform = SimPlatform::instance(); + if (freed_cycle_ != platform.cycles() || freed_gen_ != platform.reset_gen()) { + freed_cycle_ = platform.cycles(); + freed_gen_ = platform.reset_gen(); + freed_count_ = 0; + } + ++freed_count_; + } storage_.pop(); --SimChannelBase::inflight_count(); if (pop_cb_) { @@ -860,6 +888,10 @@ class SimChannel : public SimChannelBase { RingQueue storage_; uint32_t pending_count_; + bool reg_ready_ = false; + uint64_t freed_cycle_ = ~uint64_t(0); + uint64_t freed_gen_ = 0; + uint32_t freed_count_ = 0; SmallFunction convert_fn_; TxCallback tx_cb_; @@ -1198,6 +1230,7 @@ inline void SimPlatform::reset() { // Reset timing cycles_ = 0; + ++reset_gen_; } inline void SimPlatform::tick() { diff --git a/sim/common/simobject_functional.h b/sim/common/simobject_functional.h index 8074a179a1..924f09ba56 100644 --- a/sim/common/simobject_functional.h +++ b/sim/common/simobject_functional.h @@ -717,6 +717,10 @@ class SimChannel : public SimChannelBase { template void tx_callback(F&& callback) { tx_cb_ = std::forward(callback); } + // Ready timing is a timing-model property; accepted for API compatibility + // and ignored. + void set_registered_ready(bool /*enable*/) {} + // Back-pressure is disabled: producers never stall. bool full() const override { return false; } diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index 98e1eaa40d..eaef68cc0e 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -74,6 +74,7 @@ typedef uint64_t Word; using namespace vortex; +static uint64_t g_mem_done_seq = 0; static uint32_t g_mem_bank_addr_width = (VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS)); static uint64_t timestamp = 0; @@ -315,6 +316,7 @@ class Processor::Impl { // mark completed request as ready auto orig_req = reinterpret_cast(arg); orig_req->ready = true; + orig_req->done_seq = g_mem_done_seq++; return true; }, mem_req); dram_queue_[b].pop(); @@ -361,22 +363,29 @@ class Processor::Impl { if (device_->mem_rsp_valid[b] && mem_rd_rsp_ready_[b]) { device_->mem_rsp_valid[b] = 0; } - if (device_->mem_rsp_valid[b] == 0) { - if (!pending_mem_reqs_[b].empty()) { - auto mem_rsp_it = pending_mem_reqs_[b].begin(); - auto mem_rsp = *mem_rsp_it; - if (mem_rsp->ready) { - if (!mem_rsp->write) { - // return read responses - device_->mem_rsp_valid[b] = 1; - memcpy(VDataCast::get(device_->mem_rsp_data[b]), mem_rsp->data.data(), VX_CFG_PLATFORM_MEMORY_DATA_SIZE); - device_->mem_rsp_tag[b] = mem_rsp->tag; - } - // delete the request - pending_mem_reqs_[b].erase(mem_rsp_it); - delete mem_rsp; - } + // Reads carry distinct AXI IDs, so the memory returns them in the order + // they complete; writes are posted and never hold back a read. + auto& pending = pending_mem_reqs_[b]; + auto next_rd = pending.end(); + for (auto it = pending.begin(); it != pending.end();) { + auto mem_req = *it; + if (mem_req->ready && mem_req->write) { + it = pending.erase(it); + delete mem_req; + continue; } + if (mem_req->ready && (next_rd == pending.end() || mem_req->done_seq < (*next_rd)->done_seq)) { + next_rd = it; + } + ++it; + } + if (next_rd != pending.end() && !device_->mem_rsp_valid[b]) { + auto mem_rsp = *next_rd; + device_->mem_rsp_valid[b] = 1; + memcpy(VDataCast::get(device_->mem_rsp_data[b]), mem_rsp->data.data(), VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + device_->mem_rsp_tag[b] = mem_rsp->tag; + pending.erase(next_rd); + delete mem_rsp; } // process memory requests @@ -454,6 +463,7 @@ class Processor::Impl { uint64_t tag; bool write; bool ready; + uint64_t done_seq; } mem_req_t; std::unordered_map print_bufs_; diff --git a/sim/simx/barrier_unit.cpp b/sim/simx/barrier_unit.cpp index 92349d7add..6538afc2a7 100644 --- a/sim/simx/barrier_unit.cpp +++ b/sim/simx/barrier_unit.cpp @@ -21,8 +21,17 @@ using namespace vortex; +// Cycles from an SFU barrier op to its effect in the barrier store: the store +// is read a cycle before an arrival applies, and with several SFU blocks the +// request is also registered out of its block. A transaction-barrier +// completion is registered once before it applies; a global-barrier release +// applies as it arrives. A release then reaches the warp scheduler registered. +static constexpr uint32_t kOpApplyLag = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 2 : 1; +static constexpr uint32_t kEventApplyLag = 1; + BarrierUnit::BarrierUnit(const SimContext& ctx, const char* name, Core* core, Scheduler* scheduler) : SimObject(ctx, name) + , unlock_out(this, VX_CFG_NUM_WARPS) , gbar_send_next_(0) , core_(core) , scheduler_(scheduler) @@ -34,13 +43,17 @@ BarrierUnit::~BarrierUnit() {} // One request leaves the queue per cycle, so a completion that lands while the // queue is still draining is delivered on the next free cycle rather than // immediately. Mirrors the RTL request FIFO's drain rate. -void BarrierUnit::gbar_send(uint32_t bar_id, uint32_t count) { +void BarrierUnit::gbar_send(uint32_t bar_id, uint32_t count, uint32_t lag) { uint64_t now = SimPlatform::instance().cycles(); - uint64_t due = std::max(now + 1, gbar_send_next_); + uint64_t due = std::max(now + lag + 1, gbar_send_next_); gbar_send_next_ = due + 1; core_->gbar_arrive_out.send({bar_id, count, core_->id()}, due - now); } +void BarrierUnit::release(uint32_t wid, uint32_t lag) { + unlock_out.send(wid, lag + 1); +} + void BarrierUnit::on_reset() { gbar_send_next_ = 0; for (auto& barrier : barriers_) { @@ -74,7 +87,7 @@ void BarrierUnit::arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_ if (barrier.arrival_mask.count() == active_warps.count() && barrier.events == 0) { // wait_mask is preserved across the async hop (holds warps resumed by // global_resume) and cleared there. arrival_mask is local-only and reset here. - this->gbar_send(bar_id, count); + this->gbar_send(bar_id, count, kOpApplyLag); barrier.arrival_mask.reset(); barrier.count = 0; barrier.events = 0; @@ -90,7 +103,7 @@ void BarrierUnit::arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_ for (uint32_t i = 0; i < VX_CFG_NUM_WARPS; ++i) { if (barrier.wait_mask.test(i)) { DP(3, "*** Resume core #" << core_->id() << ", warp #" << i << " at barrier #" << bar_id); - scheduler_->resume(i); + this->release(i, kOpApplyLag); } } // reset barrier and advance phase @@ -106,7 +119,7 @@ void BarrierUnit::arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_ } } -bool BarrierUnit::wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { +void BarrierUnit::wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { auto bar_index = bar_id & 0x7fffffff; auto& barrier = barriers_.at(bar_index); @@ -118,9 +131,9 @@ bool BarrierUnit::wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { // add warp to wait list barrier.wait_mask.set(wid); DP(3, "*** Suspend core #" << core_->id() << ", warp #" << wid << " at barrier #" << bar_id); + } else { + this->release(wid, kOpApplyLag); } - - return wait; } void BarrierUnit::global_resume(uint32_t bar_id) { @@ -132,7 +145,7 @@ void BarrierUnit::global_resume(uint32_t bar_id) { for (uint32_t i = 0; i < VX_CFG_NUM_WARPS; ++i) { if (barrier.wait_mask.test(i)) { DP(3, "*** Resume core #" << core_->id() << ", warp #" << i << " at barrier #" << bar_id); - scheduler_->resume(i); + this->release(i, 0); } } barrier.wait_mask.reset(); @@ -156,7 +169,7 @@ void BarrierUnit::event_release(uint32_t bar_id) { if (is_global) { if (barrier.arrival_mask.count() == active_warps.count()) { uint32_t num_cores = barrier.count; - this->gbar_send(bar_id, num_cores); + this->gbar_send(bar_id, num_cores, kEventApplyLag); barrier.arrival_mask.reset(); barrier.count = 0; barrier.events = 0; @@ -167,7 +180,7 @@ void BarrierUnit::event_release(uint32_t bar_id) { for (uint32_t i = 0; i < VX_CFG_NUM_WARPS; ++i) { if (barrier.wait_mask.test(i)) { DP(3, "*** Resume core #" << core_->id() << ", warp #" << i << " at barrier #" << bar_id); - scheduler_->resume(i); + this->release(i, kEventApplyLag); } } // reset barrier and advance phase diff --git a/sim/simx/barrier_unit.h b/sim/simx/barrier_unit.h index 253fb601c7..d75222612f 100644 --- a/sim/simx/barrier_unit.h +++ b/sim/simx/barrier_unit.h @@ -30,11 +30,14 @@ class BarrierUnit : public SimObject { uint32_t get_phase(uint32_t bar_id) const; void arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_sync_bar); - bool wait(uint32_t bar_id, uint32_t phase, uint32_t wid); + void wait(uint32_t bar_id, uint32_t phase, uint32_t wid); void global_resume(uint32_t bar_id); void event_attach(uint32_t bar_id, uint32_t count = 1); void event_release(uint32_t bar_id); + // warps a local barrier releases, to the core's warp scheduler + SimChannel unlock_out; + protected: void on_reset(); @@ -44,7 +47,9 @@ class BarrierUnit : public SimObject { // send on its own cycle so back-to-back completions serialize here too. uint64_t gbar_send_next_; - void gbar_send(uint32_t bar_id, uint32_t count); + // lag: cycles until the triggering request applies in the barrier store + void gbar_send(uint32_t bar_id, uint32_t count, uint32_t lag); + void release(uint32_t wid, uint32_t lag); Core* core_; Scheduler* scheduler_; diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 95b3669a3d..8674f73591 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -84,6 +84,7 @@ class Cluster::Impl { VX_CFG_L2_LATENCY, // pipeline latency VX_CFG_L2_REPL_POLICY, // replacement policy (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + true, // non-cacheable bypass (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash L2_SHARED_LLC, // shared_llc VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 447314ce80..a6fcfdff90 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -74,6 +74,7 @@ class Core::Impl { , ibuf_inflight_(VX_CFG_NUM_WARPS, 0) , last_issue_(VX_CFG_NUM_WARPS, 0) , staged_since_(VX_CFG_NUM_WARPS, 0) + , head_staged_(VX_CFG_NUM_WARPS, false) { const std::string& name = simobject_->name(); char sname[100]; @@ -258,6 +259,7 @@ class Core::Impl { func_units_.at((int)FUType::SFU) = SimPlatform::instance().create_object(sname, simobject_); std::static_pointer_cast(func_units_.at((int)FUType::ALU))->branch_ctl_out.bind(&simobject_->branch_ctl_in); std::static_pointer_cast(func_units_.at((int)FUType::SFU))->warp_ctl_out.bind(&simobject_->warp_ctl_in); + scheduler_->barrier_unit().unlock_out.bind(&simobject_->bar_unlock_in); #ifdef VX_CFG_EXT_TCU_ENABLE snprintf(sname, 100, "%s-tcu", name.c_str()); tcu_unit_ = SimPlatform::instance().create_object(sname, simobject_); @@ -338,6 +340,7 @@ class Core::Impl { std::fill(issue_lock_owner_.begin(), issue_lock_owner_.end(), kLockOpen); std::fill(last_issue_.begin(), last_issue_.end(), 0); std::fill(staged_since_.begin(), staged_since_.end(), 0); + std::fill(head_staged_.begin(), head_staged_.end(), false); pending_instrs_.clear(); pending_ifetches_ = 0; @@ -373,14 +376,20 @@ class Core::Impl { scheduler_->schedule_begin(); - // each fetch slice picks among its own warps whose ibuffer is not yet saturated + // Each fetch slice picks among its own warps whose ibuffer is not full; a + // buffer is full at exactly IBUF_SIZE entries. With an L1, once every warp's + // buffer is full any ready warp may fetch, its instruction waiting at decode. + bool all_full = (VX_CFG_ICACHE_ENABLED || VX_CFG_DCACHE_ENABLED); + for (uint32_t w = 0; w < VX_CFG_NUM_WARPS && all_full; ++w) { + all_full = (ibuf_inflight_.at(w) == VX_CFG_IBUF_SIZE); + } // the cycle is idle when no slice has a warp to fetch, its latch free or not bool any_cand = false; for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { auto& fetch_latch = fetch_latch_.at(f); WarpMask warp_mask; for (uint32_t w = f; w < VX_CFG_NUM_WARPS; w += VX_CFG_FETCH_WIDTH) { - if (ibuf_inflight_.at(w) < VX_CFG_IBUF_SIZE) { + if (all_full || ibuf_inflight_.at(w) != VX_CFG_IBUF_SIZE) { warp_mask.set(w); } } @@ -508,7 +517,9 @@ class Core::Impl { mem_req.tag = tag; mem_req.hart_id = trace->cid; mem_req.uuid = trace->uuid; - if (icache_req.try_send(mem_req)) { + // The first icache port also carries the cache-flush injection, which + // registers its requests once more. + if (icache_req.try_send(mem_req, (f == 0) ? 2 : 1)) { DT(3, simobject_->name() << " icache-req: addr=0x" << std::hex << mem_req.addr << ", tag=0x" << mem_req.tag << std::dec << ", " << *trace); #ifdef VX_CFG_EXT_C_ENABLE decompressor->commit_request(from_refetch); @@ -634,6 +645,15 @@ class Core::Impl { // check scoreboard against sequencer micro-op (handles uop WAW/RAW) auto trace = ibuffer->peek(); + + // The head moves into the warp's staging register, freeing its buffer + // slot, once it has arrived and the warp's previous instruction has + // issued; the scheduler sees the freed slot the cycle after. + if (!head_staged_.at(wid) + && SimPlatform::instance().cycles() > std::max(last_issue_.at(wid), trace->ibuf_time)) { + head_staged_.at(wid) = true; + this->ibuf_release(wid); + } auto seq = sequencers_.at(wid); auto uop_trace = seq->get(trace); // returns cached uop or generates next @@ -694,7 +714,8 @@ class Core::Impl { } } - if (ready_set.any()) { + // The arbiter advances only on an issue that fires. + if (ready_set.any() && !operands_.at(iw)->Input.full()) { // select one instruction from ready set auto w = ibuffer_arbs_.at(iw).grant(ready_set); uint32_t wid = w * VX_CFG_ISSUE_WIDTH + iw; @@ -749,19 +770,10 @@ class Core::Impl { trace_pool_.deallocate(trace, 1); } ibuffer->pop(); - // Release ibuffer backpressure slot (matches ++ in schedule()). - // Saturating decrement: flush_warp_pipeline only drains the - // ibuffer + resets ibuf_inflight to 0, but fetch_latch_ / - // decode-stage entries scheduled pre-trap can still arrive - // at issue() post-flush and decrement the (now-zeroed) - // counter. Async-trap users (RTU CB_YIELD path: AHS, CHS, - // MISS) routinely hit this; a non-saturating decrement - // underflows to UINT32_MAX and stalls scheduling. Proper - // fix is to extend flush_warp_pipeline to also drain - // fetch_latch_ + pending icache fetches for the warp. - if (ibuf_inflight_.at(wid) > 0) { - --ibuf_inflight_.at(wid); + if (!head_staged_.at(wid)) { + this->ibuf_release(wid); } + head_staged_.at(wid) = false; } } } @@ -773,6 +785,21 @@ class Core::Impl { } } + // Release an ibuffer backpressure slot (matches ++ in schedule()). + // Saturating decrement: flush_warp_pipeline only drains the ibuffer and + // resets ibuf_inflight to 0, but fetch_latch_ / decode-stage entries + // scheduled pre-trap can still arrive at issue() post-flush and decrement + // the (now-zeroed) counter. Async-trap users (RTU CB_YIELD path: AHS, CHS, + // MISS) routinely hit this; a non-saturating decrement underflows to + // UINT32_MAX and stalls scheduling. Proper fix is to extend + // flush_warp_pipeline to also drain fetch_latch_ + pending icache fetches + // for the warp. + void ibuf_release(uint32_t wid) { + if (ibuf_inflight_.at(wid) > 0) { + --ibuf_inflight_.at(wid); + } + } + void update_queue_full() { for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { fu_full_seen_.at(iw) = fu_full_.at(iw); @@ -818,6 +845,11 @@ class Core::Impl { } warp_ctl.pop(); } + auto& bar_unlock = simobject_->bar_unlock_in; + while (!bar_unlock.empty()) { + scheduler_->resume(bar_unlock.peek()); + bar_unlock.pop(); + } // Fan-in: route per-block FU outputs to per-iw commit queues by trace->wid. // Each FU has NUM_*_BLOCKS outputs; the original iw was lost during @@ -934,7 +966,8 @@ class Core::Impl { // ----- Public-facing API (called from outer Core trampolines) ----- bool running() const { - if (scheduler_->running() || !pending_instrs_.empty()) { + // committed stores wait in the LSU request queue until sent to memory + if (scheduler_->running() || !pending_instrs_.empty() || !this->lsu_drained()) { return true; } #ifdef VX_CFG_VM_ENABLE @@ -977,6 +1010,7 @@ class Core::Impl { ibuffer->pop(); } ibuf_inflight_.at(wid) = 0; + head_staged_.at(wid) = false; uint32_t iw = wid % VX_CFG_ISSUE_WIDTH; if (issue_lock_owner_.at(iw) == wid / VX_CFG_ISSUE_WIDTH) { issue_lock_owner_.at(iw) = kLockOpen; @@ -1163,6 +1197,7 @@ class Core::Impl { std::vector ibuf_inflight_; std::vector last_issue_; // [wid] cycle of the warp's last issue std::vector staged_since_; // [wid] cycle the last issued candidate was staged + std::vector head_staged_; // [wid] the buffer head has left for staging PoolAllocator trace_pool_; @@ -1191,6 +1226,7 @@ Core::Core(const SimContext& ctx, , dcache_rsp_in(VX_CFG_DCACHE_NUM_REQS, this) , branch_ctl_in(this, VX_CFG_NUM_WARPS) , warp_ctl_in(this, VX_CFG_NUM_WARPS) + , bar_unlock_in(this, VX_CFG_NUM_WARPS) , gbar_arrive_out(this) , gbar_resume_in(this) #ifdef VX_CFG_EXT_RASTER_ENABLE @@ -1243,8 +1279,8 @@ void Core::barrier_arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is impl_->scheduler()->barrier_unit().arrive(bar_id, count, wid, is_sync_bar); } -bool Core::barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { - return impl_->scheduler()->barrier_unit().wait(bar_id, phase, wid); +void Core::barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { + impl_->scheduler()->barrier_unit().wait(bar_id, phase, wid); } void Core::global_barrier_resume(uint32_t bar_id) { diff --git a/sim/simx/core.h b/sim/simx/core.h index df1e117b7b..49462541bd 100644 --- a/sim/simx/core.h +++ b/sim/simx/core.h @@ -82,6 +82,7 @@ class Core : public SimObject { // A warp has at most one release in flight, so NUM_WARPS bounds each. SimChannel branch_ctl_in; SimChannel warp_ctl_in; + SimChannel bar_unlock_in; // Global-barrier event links, wired core <-> cluster at elaboration. SimEventLink gbar_arrive_out; @@ -110,7 +111,7 @@ class Core : public SimObject { void barrier_arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_sync_bar); - bool barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid); + void barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid); void global_barrier_resume(uint32_t bar_id); diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index 950c972df9..1d0fb6883a 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -34,6 +34,7 @@ CtaDispatcher::CtaDispatcher(const SimContext& ctx, const char* name, Core* core , wid_to_slot_(num_warps_, num_warps_) // num_warps_ = invalid/unassigned , tail_slot_(0) , has_cta_(false) + , cta_done_(false) , cur_slot_(0) , cta_size_(0) , rank_(0) @@ -59,6 +60,7 @@ CtaDispatcher::~CtaDispatcher() {} void CtaDispatcher::on_reset() { has_cta_ = false; + cta_done_ = false; tail_slot_ = 0; init_wait_ = 0; for (uint32_t i = 0; i < num_warps_; ++i) { @@ -100,6 +102,11 @@ uint32_t CtaDispatcher::usable_slots(uint32_t stride) const { } uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs) { + if (cta_done_) { + cta_done_ = false; + has_cta_ = false; + return 0; + } if (!has_cta_) { // The KMU pushes CTAs down a registered, credit-gated launch lane; the // endpoint holds the one CTA committed to this core. It is popped only on @@ -178,6 +185,9 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w has_cta_ = true; + // The accept moves the dispatcher to its launch state. + init_wait_ = 1; + if (VX_CFG_FETCH_WIDTH > 1) { bool same_shape = true; for (uint32_t d = 0; d < 3; ++d) { @@ -189,7 +199,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w shape_step_[d] = cta_.warp_step[d]; shape_dim_[d] = cta_.block_dim[d]; } - init_wait_ = VX_CFG_FETCH_WIDTH; + init_wait_ += VX_CFG_FETCH_WIDTH; } } } @@ -202,7 +212,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w // Lanes with a free slot, round-robin from lane_rr_, up to the warps left; // each launches its lowest free slot. uint32_t count = 0; - for (uint32_t o = 0; o < VX_CFG_FETCH_WIDTH && has_cta_; ++o) { + for (uint32_t o = 0; o < VX_CFG_FETCH_WIDTH && !cta_done_; ++o) { uint32_t lane = (lane_rr_ + o) % VX_CFG_FETCH_WIDTH; int free_wid = -1; for (uint32_t wid = lane; wid < num_warps_; wid += VX_CFG_FETCH_WIDTH) { @@ -266,7 +276,7 @@ void CtaDispatcher::mark_slot_inited(uint32_t wid, const ThreadMask& tmask) { } bool CtaDispatcher::next_warp(bool do_init, cta_warp_record_t* out) { - if (!has_cta_) return false; + if (!has_cta_ || cta_done_) return false; out->do_init = do_init; out->PC = Word(cta_.PC); @@ -320,7 +330,7 @@ bool CtaDispatcher::next_warp(bool do_init, cta_warp_record_t* out) { ++rank_; if (rank_ >= cta_size_) { - has_cta_ = false; + cta_done_ = true; } return true; diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 2404f75055..7c74930292 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -114,6 +114,8 @@ class CtaDispatcher : public SimObject { // Currently-in-flight CTA (being dispatched warp-rank by warp-rank) bool has_cta_; + // the last warp has launched; the dispatcher returns to accepting next cycle + bool cta_done_; uint32_t cur_slot_; // slot index assigned to the current in-flight CTA kmu_req_t cta_; uint32_t cta_size_; diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index abbaf45ced..3ad602dd27 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -48,6 +48,8 @@ constexpr uint64_t kGmemLineMask = ~uint64_t(VX_CFG_L1_LINE_SIZE - 1); // Cores served by one (socket-resident) DXA engine. constexpr uint32_t kCoresPerSocket = VX_CFG_SOCKET_SIZE; +// A delivered line reaches its SMEM write through two pipeline registers. +constexpr uint32_t kSmemWrLatency = 2; } // namespace @@ -95,6 +97,8 @@ class DxaCore::Impl { struct InflightSlot { bool allocated = false; bool rsp_arrived = false; + uint64_t ready_cycle = 0; // first cycle the line can be written to SMEM + uint64_t free_cycle = 0; // first cycle the slot can carry a new read LineWork work; // captured at allocation std::shared_ptr rsp_data; }; @@ -112,6 +116,11 @@ class DxaCore::Impl { uint32_t smem_stride = 0; uint64_t issue_cycle = 0; + // setup engine: the next transfer is set up while this one drains + bool staged = false; + DxaReq staged_req; + uint64_t staged_ready = 0; + // addr_gen: pre-enumerated work list (one entry per GMEM CL + // multicast replays handled at smem_wr stage). std::vector work_list; @@ -149,7 +158,8 @@ class DxaCore::Impl { w.cta_indices.clear(); w.issued_order.clear(); w.drain_slot = UINT32_MAX; - for (auto& s : w.inflight) { s.allocated = false; s.rsp_arrived = false; s.rsp_data.reset(); } + for (auto& s : w.inflight) { s = InflightSlot(); } + w.staged = false; w.ag_idx = 0; w.mc_cta_idx = 0; w.km_elem_idx = 0; @@ -223,6 +233,7 @@ class DxaCore::Impl { auto& w = workers_[worker_id]; if (slot_id < w.inflight.size() && w.inflight[slot_id].allocated) { w.inflight[slot_id].rsp_arrived = true; + w.inflight[slot_id].ready_cycle = cycle_ + kSmemWrLatency; w.inflight[slot_id].rsp_data = rsp.data; } ch.pop(); @@ -237,10 +248,18 @@ class DxaCore::Impl { } } - // 3) Dispatch from queue to idle workers. + // 3) Start a set-up transfer on an idle worker; otherwise let the + // worker's setup engine take the next request. for (auto& w : workers_) { - if (w.state == WState::IDLE && !queue_.empty()) - start_worker(w, queue_.front()), queue_.pop_front(); + if (w.staged && w.state == WState::IDLE && cycle_ >= w.staged_ready) { + w.staged = false; + start_worker(w, w.staged_req); + } else if (!w.staged && !queue_.empty()) { + w.staged_req = queue_.front(); + w.staged_ready = cycle_ + this->setup_latency(w.staged_req); + w.staged = true; + queue_.pop_front(); + } } // 4) Drain DxaUnit channels → req queue (round-robin). @@ -323,6 +342,14 @@ class DxaCore::Impl { return (k_blk * n_steps + n_blk) * b_blk_elems + n_in * kw + r_in; } + // Parameter setup takes longer for higher-rank descriptors: each extra + // dimension adds a multiply phase. + uint32_t setup_latency(const DxaReq& req) const { + uint32_t rank = (req.desc_slot < VX_DCR_DXA_DESC_COUNT) + ? desc_rank(descriptors_.at(req.desc_slot).meta) : 1; + return 2 * std::max(rank, 2) + 2; + } + // ── Pull from per-core dxa_req_in[] into queue_ (round-robin) ──────── void drain_req_in() { auto& chs = simobject_->dxa_req_in; @@ -548,7 +575,7 @@ class DxaCore::Impl { // Find a free slot. uint32_t slot = UINT32_MAX; for (uint32_t s = 0; s < w.inflight.size(); ++s) { - if (!w.inflight[s].allocated) { + if (!w.inflight[s].allocated && cycle_ >= w.inflight[s].free_cycle) { slot = s; break; } @@ -570,6 +597,7 @@ class DxaCore::Impl { if (lw.oob) { w.inflight[slot].allocated = true; w.inflight[slot].rsp_arrived = true; + w.inflight[slot].ready_cycle = cycle_ + kSmemWrLatency; w.inflight[slot].rsp_data.reset(); w.inflight[slot].work = lw; w.issued_order.push_back(slot); @@ -614,7 +642,8 @@ class DxaCore::Impl { if (slot == UINT32_MAX) { for (uint32_t i = 0; i < w.inflight.size(); ++i) { const auto& candidate = w.inflight[i]; - if (candidate.allocated && candidate.rsp_arrived && !candidate.work.last) { + if (candidate.allocated && candidate.rsp_arrived && !candidate.work.last + && cycle_ >= candidate.ready_cycle) { slot = i; break; } @@ -623,7 +652,7 @@ class DxaCore::Impl { if (slot == UINT32_MAX && w.issued_order.size() == 1) { uint32_t last_slot = w.issued_order.front(); const auto& candidate = w.inflight[last_slot]; - if (candidate.allocated && candidate.rsp_arrived) + if (candidate.allocated && candidate.rsp_arrived && cycle_ >= candidate.ready_cycle) slot = last_slot; } @@ -787,6 +816,7 @@ class DxaCore::Impl { // Slot done — release. s.allocated = false; s.rsp_arrived = false; + s.free_cycle = cycle_ + 1; s.rsp_data.reset(); auto it = std::find(w.issued_order.begin(), w.issued_order.end(), slot); if (it != w.issued_order.end()) w.issued_order.erase(it); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 2119b6d76f..c81c3458aa 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -87,7 +87,9 @@ LsuUnit::LsuUnit(const SimContext& ctx, const char* name, Core* core) LsuUnit::~LsuUnit() {} -// Returns true when all blocks' input queues are empty and no reads are outstanding. +// Returns true when no block holds an instruction with requests still to send +// or a read awaiting its response; ops still in the dispatch buffer do not hold +// a barrier. bool LsuUnit::drained() const { #ifdef TCU_META_ENABLE // TcuRspOut forwards to the TCU's endpoint, so query via size() (empty() @@ -97,7 +99,6 @@ bool LsuUnit::drained() const { } #endif for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - if (!Inputs.at(b).empty()) return false; if (!states_.at(b).req_queue.empty()) return false; if (!states_.at(b).pending_reqs.empty()) return false; } @@ -294,8 +295,7 @@ void LsuUnit::process_response_step(uint32_t b) { state.pending_reqs.release(lsu_rsp.tag); } if (is_final && entry.eop) { - // Load writeback crosses one more registered stage than the - // direct-commit path before reaching the commit arbiter. + // the response buffer, then the result arbiter's register output.send(trace, 2); } else { // A non-final fragment still spends a commit-port slot: it drains @@ -335,6 +335,18 @@ void LsuUnit::ingest_inputs(uint32_t b) { } ++state.staged_reads; } + if (lsu_type_tag && *lsu_type_tag == LsuType::STORE) { + // A store commits as the request queue accepts it, through the + // no-response path, and is sent to memory later. The commit stage + // retires the trace, so the queue keeps a copy of its own. + if (Outputs.at(b).full()) { + return; + } + auto copy = new (core_->trace_pool().allocate(1)) instr_trace_t(*trace); + // the no-response buffer, then the result arbiter's register + Outputs.at(b).send(trace, 2); + trace = copy; + } state.req_queue.push(trace); input.pop(); } @@ -451,9 +463,9 @@ void LsuUnit::process_request_step(uint32_t b) { state.head_staged = !is_write || is_amo; } - // AMO always returns to rd, so it is not direct-commit even though it - // carries a side-effect write. - bool direct_commit = (is_write || 0 == state.addr_list.size()) && !is_amo; + // A store committed when the queue accepted it. AMO always returns to rd, + // so it is not direct-commit even though it carries a side-effect write. + bool direct_commit = !is_write && 0 == state.addr_list.size() && !is_amo; // Skip beats whose tid group is entirely inactive — nothing to issue. while (state.remain_addrs != 0) { @@ -582,6 +594,10 @@ void LsuUnit::process_request_step(uint32_t b) { state.head_staged = false; } state.req_queue.pop(); + if (is_write) { + trace->~instr_trace_t(); + core_->trace_pool().deallocate(trace, 1); + } } } diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 72124b8e3d..f7c9c82bc0 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2044,6 +2044,93 @@ class CacheBank : public SimObject { /////////////////////////////////////////////////////////////////////////////// +// The cache's core-port arbiter: a port's request is accepted only when its +// bank grants it, one per bank per cycle, round-robin over the ports, and only +// while the bank's input buffer has room; non-cacheable requests are split off +// for the bypass path. It ticks ahead of the banks, so a grant reaches its bank +// in the same cycle unless the crossbar registers its output. +class CachePortArb : public SimObject { +public: + std::vector> ReqIn; // core ports + std::vector> ReqOut; // banks + std::vector> IoOut; // non-cacheable requests, per port + + CachePortArb(const SimContext& ctx, const char* name, const params_t& params, + uint32_t num_inputs, uint32_t num_banks) + : SimObject(ctx, name) + , ReqIn(num_inputs, this) + , ReqOut(num_banks, this) + , IoOut(num_inputs, this) + , params_(params) + , lg2_inputs_(log2ceil(num_inputs)) + // the request crossbar registers its output when it has more than two inputs + , delay_((num_inputs > 2) ? 1 : 0) + , rr_(num_banks, num_inputs - 1) + { + assert(num_inputs <= 64); + } + + uint64_t collisions() const { return collisions_; } + + // The cache's line initialization holds the core ports for these cycles. + void set_init_cycles(uint32_t cycles) { init_cycles_ = cycles; } + +protected: + void on_reset() { + std::fill(rr_.begin(), rr_.end(), ReqIn.size() - 1); + collisions_ = 0; + } + + void on_tick() { + if (init_cycles_ != 0) { + --init_cycles_; + return; + } + uint32_t n = ReqIn.size(); + // requesting ports per bank; non-cacheable requests leave on their own port + std::vector want(ReqOut.size(), 0); + for (uint32_t i = 0; i < n; ++i) { + auto& in = ReqIn.at(i); + if (in.empty()) { + continue; + } + auto& req = in.peek(); + if (req.flags.io) { + if (IoOut.at(i).try_send(req, 0)) { + in.pop(); + } + continue; + } + want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; + } + for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { + if (want[b] == 0 || ReqOut.at(b).full()) { + continue; + } + uint32_t i = rr_[b]; + do { + i = (i + 1) % n; + } while (!(want[b] & (1ull << i))); + collisions_ += __builtin_popcountll(want[b]) - 1; + auto req = ReqIn.at(i).peek(); + req.tag = (req.tag << lg2_inputs_) | i; + ReqOut.at(b).send(req, delay_); + ReqIn.at(i).pop(); + rr_[b] = i; + } + } + +private: + params_t params_; + uint32_t lg2_inputs_; + uint32_t delay_; + uint32_t init_cycles_ = 0; + std::vector rr_; + uint64_t collisions_ = 0; + + friend class SimObject; +}; + class Cache::Impl { public: Impl(Cache *simobject, const Config &config) @@ -2088,7 +2175,15 @@ class Cache::Impl { nc_mem_arbs_.at(i)->RspOut.at(0).bind(&bank_mem_arb->RspIn.at(i)); } - // Create bank's core crossbar + // Create the core-port arbiter ahead of the banks it feeds + snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks); + for (uint32_t i = 0; i < config_.num_inputs; ++i) { + simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); + } + + // Create bank's core crossbar; it carries the bank responses back to the + // core ports snprintf(sname, 100, "%s-core_xbar", simobject->name().c_str()); bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, [&](const MemCrossBar::ReqType &req) { @@ -2101,7 +2196,7 @@ class Cache::Impl { banks_.at(i) = CacheBank::Create(sname, config, params_, i); // bind core ports - bank_core_xbar_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); + port_arb_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); banks_.at(i)->core_rsp_out.bind(&bank_core_xbar_->RspIn.at(i)); // bind memory ports @@ -2115,6 +2210,7 @@ class Cache::Impl { return; // calculate cache initialization cycles init_cycles_ = keep_lines_ ? 0 : params_.sets_per_bank; + port_arb_->set_init_cycles(init_cycles_); keep_lines_ = false; } @@ -2147,26 +2243,18 @@ class Cache::Impl { if (bank_rsp.empty()) continue; auto &core_rsp = bank_rsp.peek(); - if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, 0)) { + // the non-cacheable bypass layer's response arbiter registers it once more + if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, config_.nc_enable ? 1 : 0)) { DT(3, simobject_->name() << " core-rsp: " << core_rsp); bank_rsp.pop(); } } - // schedule core requests + // non-cacheable core requests split off by the port arbiter for (uint32_t req_id = 0, n = config_.num_inputs; req_id < n; ++req_id) { - auto &core_req_in = simobject_->core_req_in.at(req_id); - if (core_req_in.empty()) - continue; - auto &core_req = core_req_in.peek(); - if (core_req.flags.io) { - if (this->processBypassRequest(core_req, req_id)) { - core_req_in.pop(); - } - } else { - if (bank_core_xbar_->ReqIn.at(req_id).try_send(core_req, 0)) { - core_req_in.pop(); - } + auto &io_req = port_arb_->IoOut.at(req_id); + if (!io_req.empty() && this->processBypassRequest(io_req.peek(), req_id)) { + io_req.pop(); } } } @@ -2177,7 +2265,7 @@ class Cache::Impl { for (const auto &bank : banks_) { perf_stats += bank->perf_stats(); } - perf_stats.bank_stalls = bank_core_xbar_->collisions(); + perf_stats.bank_stalls = port_arb_->collisions(); } return perf_stats; } @@ -2233,6 +2321,7 @@ class Cache::Impl { std::vector banks_; MemArbiter::Ptr bank_arb_; std::vector nc_mem_arbs_; + CachePortArb::Ptr port_arb_; MemCrossBar::Ptr bank_core_xbar_; uint32_t init_cycles_; bool keep_lines_ = false; diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 5dd4c9bd08..40d7890501 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -43,6 +43,7 @@ class Cache : public SimObject { uint16_t latency; // pipeline latency uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache + bool nc_enable = false; // non-cacheable bypass layer in front of the cache bool bank_hash = false; // hashed bank select bool shared_llc = false; // the LLC serves every memory client alone bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index d2b6d60598..9e3dc16884 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -85,11 +85,21 @@ class LocalMem::Impl { snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); uint32_t lg2_line_size = log2ceil(config_.line_size); uint32_t num_banks = 1 << config.B; + // The crossbar's registered request and response outputs are taken by + // the switch and bank stages around it, so its delays here are only the + // extra tree level: an arbiter over more than 1.5x the fan-out limit is + // split into a buffered tree, one cycle deeper. The request side + // arbitrates the lanes at each bank, the response side the banks at + // each lane. + auto tree_delay = [](uint32_t requesters) -> uint32_t { + constexpr uint32_t kMaxFanout = 8; + return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; + }; mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::Priority, config.num_reqs, num_banks, [lg2_line_size, num_banks](const MemCrossBar::ReqType& req) { // Custom logic to calculate the output index using bank interleaving return (uint32_t)((req.addr >> lg2_line_size) & (num_banks-1)); - }); + }, tree_delay(config.num_reqs), tree_delay(num_banks)); for (uint32_t i = 0; i < config.num_reqs; ++i) { simobject->Inputs.at(i).bind(&mem_xbar_->ReqIn.at(i)); mem_xbar_->RspOut.at(i).bind(&simobject->Outputs.at(i)); diff --git a/sim/simx/opc_unit.cpp b/sim/simx/opc_unit.cpp index e5b1f8aac1..9585ba21fb 100644 --- a/sim/simx/opc_unit.cpp +++ b/sim/simx/opc_unit.cpp @@ -13,6 +13,7 @@ #include "opc_unit.h" #include "core.h" +#include #include #include #include "types.h" @@ -30,6 +31,9 @@ OpcUnit::OpcUnit(const SimContext &ctx, const char* name, , Input(this) , Output(this) , num_threads_(num_threads) { + // The scoreboard's output skid buffer: two entries, the one being collected + // among them, with a registered ready. + Input.set_registered_ready(true); regs_.reserve(num_warp_slots); for (uint32_t i = 0; i < num_warp_slots; ++i) { regs_.emplace_back(num_threads); @@ -43,29 +47,29 @@ void OpcUnit::on_reset() { total_stalls_ = 0; cur_trace_ = nullptr; release_cycle_ = 0; + consume_cycle_ = 0; + consumed_ = false; // The register files are excluded: they are storage, not pipeline state, and // a launch is not a device reset. A slot that skips its startup relies on the // stack pointer that startup left behind, so clearing them here would strand // it at zero. They are zero-initialized at construction. } +// Each bank serves one operand per cycle. static uint32_t compute_bank_conflicts(const instr_trace_t* trace) { - uint32_t stalls = 0; + uint32_t per_bank[VX_CFG_NUM_GPR_BANKS] = {}; + uint32_t most = 0; for (uint32_t i = 0; i < NUM_SRC_REGS; ++i) { - for (uint32_t j = i + 1; j < NUM_SRC_REGS; ++j) { - if ((trace->src_regs[i].type == RegType::None) - || (trace->src_regs[j].type == RegType::None)) - continue; - if ((trace->src_regs[i].type == RegType::Integer && trace->src_regs[i].id() == 0) - || (trace->src_regs[j].type == RegType::Integer && trace->src_regs[j].id() == 0)) - continue; // skip x0 - uint32_t bank_i = trace->src_regs[i].idx % VX_CFG_NUM_GPR_BANKS; - uint32_t bank_j = trace->src_regs[j].idx % VX_CFG_NUM_GPR_BANKS; - if (bank_i == bank_j) - ++stalls; + auto& reg = trace->src_regs[i]; + if (reg.type == RegType::None) { + continue; + } + if (reg.type == RegType::Integer && reg.id() == 0) { + continue; // skip x0 } + most = std::max(most, ++per_bank[reg.idx % VX_CFG_NUM_GPR_BANKS]); } - return stalls; + return most ? (most - 1) : 0; } void OpcUnit::on_tick() { @@ -79,14 +83,20 @@ void OpcUnit::on_tick() { cur_trace_ = nullptr; } - // accept next uop into the holding slot + // accept next uop into the holding slot; it stays in the input buffer + // until its last operand read if (cur_trace_ == nullptr && !Input.empty()) { auto trace = Input.peek(); uint32_t stalls = compute_bank_conflicts(trace); total_stalls_ += stalls; cur_trace_ = trace; + consume_cycle_ = cur_cycle + stalls; release_cycle_ = cur_cycle + 1 + stalls; + consumed_ = false; + } + if (cur_trace_ != nullptr && !consumed_ && cur_cycle >= consume_cycle_) { Input.pop(); + consumed_ = true; } } diff --git a/sim/simx/opc_unit.h b/sim/simx/opc_unit.h index af8f201468..ed15f26819 100644 --- a/sim/simx/opc_unit.h +++ b/sim/simx/opc_unit.h @@ -76,6 +76,8 @@ class OpcUnit : public SimObject { uint32_t total_stalls_ = 0; instr_trace_t* cur_trace_ = nullptr; uint64_t release_cycle_ = 0; + uint64_t consume_cycle_ = 0; + bool consumed_ = false; friend class SimObject; friend class Operands; // Operands::get_exit_code reads regs_ directly diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index aef1822cf6..b7d31c1705 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -104,6 +104,7 @@ ProcessorImpl::ProcessorImpl() VX_CFG_L3_LATENCY, // pipeline latency VX_CFG_L3_REPL_POLICY, // replacement policy VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC + true, // non-cacheable bypass false, // bank_hash L3_SHARED_LLC, // shared_llc VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch diff --git a/sim/simx/sfu_unit.cpp b/sim/simx/sfu_unit.cpp index edf9311c5f..4c57d1ab8b 100644 --- a/sim/simx/sfu_unit.cpp +++ b/sim/simx/sfu_unit.cpp @@ -33,6 +33,9 @@ using namespace vortex; // With several SFU blocks, their TEX/RTU/DXA requests merge onto the core's bus // through one registered arbiter stage. static constexpr uint32_t kBusMergeDelay = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 1 : 0; +// With several SFU blocks, warp-control requests are registered toward the +// core-wide warp-control unit; a sole block reaches it directly. +static constexpr bool kWctlBuffered = (VX_CFG_NUM_SFU_BLOCKS > 1); SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) : FuncUnit(ctx, name, core, 6) @@ -70,6 +73,10 @@ SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) , rtu_unit_(new RtuUnit(core, rtu_req_out, rtu_window_, kBusMergeDelay)) #endif { + // the dispatch skid buffer's ready is registered + for (auto& input : Inputs) { + input.set_registered_ready(true); + } } // Cycles from processing to the output: the result buffer and the PE @@ -418,6 +425,43 @@ void SfuUnit::on_tick() { // The fragment work distributor (above + scheduler) launches fragment // warps directly from the autonomously-pulled covered-quad waves. + // A CSR op is held a cycle before it may fire: the CTA context RAMs are + // read and the CSR decode registered in it. The wait restarts per op, so + // the CSR path takes one op every two cycles. + const bool is_csr = std::get_if(&trace->op_type) != nullptr; + if (is_csr && csr_read_done_.at(b) != trace) { + csr_read_done_.at(b) = trace; + continue; + } + + // A barrier arrive is the one shared request with a response. With + // several SFU blocks its request is registered toward the shared unit, + // which answers it two cycles after it is sent; it is sent once the LSU + // has drained, and the arrival takes effect as it is sent, as the + // unit's one grant per cycle orders it among the other arrivals. A sole + // block is answered in the cycle it asks, like any other op. + bool bar_arrive = false; + if (auto wctl_p = std::get_if(&trace->op_type); + kWctlBuffered && wctl_p && *wctl_p == WctlType::BAR && trace->eop) { + auto args = std::get(trace->instr_ptr->get_args()); + bar_arrive = args.is_bar_arrive; + } + if (bar_arrive) { + auto& ba = bar_arrive_.at(b); + uint64_t now = SimPlatform::instance().cycles(); + if (ba.trace != trace) { + if (!core_->lsu_drained()) { + continue; // drain LSU before the barrier + } + ba = BarArrive{trace, now, false, false}; + ba.release_warp = wctl_unit_->process(trace, &ba.warp_exit); + continue; + } + if (now < ba.sent + 2 || output.full()) { + continue; + } + } + if (output.full()) continue; // stall — no side effects this tick @@ -429,7 +473,7 @@ void SfuUnit::on_tick() { if (*wctl_p == WctlType::WSYNC) { if (core_->has_pending_instrs(trace->wid)) continue; // wait for the warp's prior instrs to retire - } else if (*wctl_p == WctlType::BAR) { + } else if (*wctl_p == WctlType::BAR && !bar_arrive) { if (!core_->lsu_drained()) continue; // drain LSU before the barrier } @@ -438,10 +482,15 @@ void SfuUnit::on_tick() { bool release_warp = trace->fetch_stall; bool warp_exit = false; - if (std::get_if(&trace->op_type)) { + if (bar_arrive) { + release_warp = bar_arrive_.at(b).release_warp; + warp_exit = bar_arrive_.at(b).warp_exit; + bar_arrive_.at(b).trace = nullptr; + } else if (std::get_if(&trace->op_type)) { release_warp = wctl_unit_->process(trace, &warp_exit); - } else if (std::get_if(&trace->op_type)) { + } else if (is_csr) { csr_unit_->process(trace); + csr_read_done_.at(b) = nullptr; #ifdef VX_CFG_EXT_DXA_ENABLE } else if (std::get_if(&trace->op_type)) { // process() returns nullptr on backpressure (idempotent retry next @@ -476,12 +525,19 @@ void SfuUnit::on_tick() { continue; } auto& input = Inputs.at(winner[r]); - if (input.empty() || input.peek() != winner_head[r]) { + if (input.empty() || input.peek() != winner_head[r] + || bar_arrive_.at(winner[r]).trace == winner_head[r]) { res_rr_[r] = (winner[r] + 1) % VX_CFG_NUM_SFU_BLOCKS; } } } +void SfuUnit::on_reset() { + std::fill(std::begin(res_rr_), std::end(res_rr_), 0); + csr_read_done_.fill(nullptr); + bar_arrive_.fill(BarArrive{}); +} + SfuUnit::SharedRes SfuUnit::shared_res(uint32_t block) { auto& input = Inputs.at(block); if (input.empty()) { @@ -506,8 +562,17 @@ SfuUnit::SharedRes SfuUnit::shared_res(uint32_t block) { } #endif if (auto wctl_p = std::get_if(&trace->op_type)) { - // a block requests only when its op could fire this cycle - if (Outputs.at(block).full()) { + // a barrier arrive whose request has left no longer needs the port + if (bar_arrive_.at(block).trace == trace) { + return RES_NONE; + } + // A block requests only when its op could fire this cycle. A barrier + // arrive's request does not wait on the output: it completes later. + bool bar_arrive = false; + if (*wctl_p == WctlType::BAR && trace->eop) { + bar_arrive = std::get(trace->instr_ptr->get_args()).is_bar_arrive; + } + if (!bar_arrive && Outputs.at(block).full()) { return RES_NONE; } if (*wctl_p == WctlType::BAR && trace->eop && !core_->lsu_drained()) { diff --git a/sim/simx/sfu_unit.h b/sim/simx/sfu_unit.h index ffc26c22b0..5e208b614e 100644 --- a/sim/simx/sfu_unit.h +++ b/sim/simx/sfu_unit.h @@ -118,6 +118,7 @@ class SfuUnit : public FuncUnit { #endif protected: + void on_reset() override; void on_tick() override; private: @@ -132,6 +133,17 @@ class SfuUnit : public FuncUnit { std::unique_ptr wctl_unit_; std::unique_ptr csr_unit_; + // Per block, the CSR op that has spent its one-cycle CTA-context read. + std::array csr_read_done_{}; + // Per block, the barrier arrive waiting for the shared barrier unit's + // response, which carries the phase it writes back. + struct BarArrive { + const instr_trace_t* trace = nullptr; + uint64_t sent = 0; + bool release_warp = false; + bool warp_exit = false; + }; + std::array bar_arrive_{}; #ifdef VX_CFG_EXT_RTU_ENABLE // Shared graphics register window (SETW/GETW/GETWF slot file) used by the RTU // ray/hit stream, TEX (vx_tex4) u,v payload + texel, and OM (vx_om4) payload. diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 536dfe4d1e..83eac594da 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -91,6 +91,7 @@ class Socket::Impl { VX_CFG_DCACHE_LATENCY, // pipeline latency (capacity-scaled, matches hardware) VX_CFG_DCACHE_REPL_POLICY, // replacement policy (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + true, // non-cacheable bypass (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash DCACHE_SHARED_LLC, // shared_llc VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch diff --git a/sim/simx/types.h b/sim/simx/types.h index 1658771c6e..85b248ade7 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -1052,7 +1052,8 @@ class GreedyRRArbiter : public IArbiterImpl { class Arbiter { public: - Arbiter(ArbiterType type = ArbiterType::Priority, uint32_t size = 0) { + Arbiter(ArbiterType type = ArbiterType::Priority, uint32_t size = 0) + : type_(type), size_(size) { switch (type) { case ArbiterType::Priority: impl_ = std::make_shared(size); @@ -1071,6 +1072,20 @@ class Arbiter { } } + // A copy is a separate arbiter with its own grant state, so a vector of + // arbiters built from one prototype does not share a single state. + Arbiter(const Arbiter& other) : Arbiter(other.type_, other.size_) {} + + Arbiter& operator=(const Arbiter& other) { + if (this != &other) { + *this = Arbiter(other.type_, other.size_); + } + return *this; + } + + Arbiter(Arbiter&&) = default; + Arbiter& operator=(Arbiter&&) = default; + virtual ~Arbiter() {} uint32_t grant(const BitVector<>& requests) { @@ -1082,6 +1097,8 @@ class Arbiter { } private: + ArbiterType type_; + uint32_t size_; std::shared_ptr impl_; }; diff --git a/sim/simx/wctl_unit.cpp b/sim/simx/wctl_unit.cpp index e989360acc..e31d612e72 100644 --- a/sim/simx/wctl_unit.cpp +++ b/sim/simx/wctl_unit.cpp @@ -20,17 +20,8 @@ using namespace vortex; -bool WctlUnit::is_shared(const instr_trace_t* trace) const { - if (!trace->eop) { - return false; - } +ThreadMask WctlUnit::next_tmask(const instr_trace_t* trace) const { auto wctl_type = std::get(trace->op_type); - if (wctl_type == WctlType::BAR || wctl_type == WctlType::WSPAWN) { - return true; - } - if (wctl_type != WctlType::TMC && wctl_type != WctlType::PRED) { - return false; - } auto& warp = core_->scheduler().warp(trace->wid); uint32_t num_threads = VX_CFG_NUM_THREADS; int32_t thread_last = num_threads - 1; @@ -39,22 +30,35 @@ bool WctlUnit::is_shared(const instr_trace_t* trace) const { } auto& rs1_data = trace->src_data[0]; auto& rs2_data = trace->src_data[1]; - ThreadMask next_tmask(num_threads); if (wctl_type == WctlType::TMC) { + ThreadMask tmask(num_threads); for (uint32_t t = 0; t < num_threads; ++t) { - next_tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); + tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); } - } else { - auto wctlArgs = std::get(trace->instr_ptr->get_args()); - auto not_pred = wctlArgs.is_cond_neg; - ThreadMask pred(num_threads); - for (uint32_t t = 0; t < num_threads; ++t) { - auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; - pred[t] = warp.tmask.test(t) && cond; - } - next_tmask = pred.any() ? (warp.tmask & pred) : ThreadMask(num_threads, rs2_data.at(thread_last).u); + return tmask; + } + auto wctlArgs = std::get(trace->instr_ptr->get_args()); + auto not_pred = wctlArgs.is_cond_neg; + ThreadMask pred(num_threads); + for (uint32_t t = 0; t < num_threads; ++t) { + auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; + pred[t] = warp.tmask.test(t) && cond; + } + return pred.any() ? (warp.tmask & pred) : ThreadMask(num_threads, rs2_data.at(thread_last).u); +} + +bool WctlUnit::is_shared(const instr_trace_t* trace) const { + if (!trace->eop) { + return false; + } + auto wctl_type = std::get(trace->op_type); + if (wctl_type == WctlType::BAR || wctl_type == WctlType::WSPAWN) { + return true; } - return next_tmask.none(); + if (wctl_type != WctlType::TMC && wctl_type != WctlType::PRED) { + return false; + } + return this->next_tmask(trace).none(); } bool WctlUnit::process(instr_trace_t* trace, bool* exit) { @@ -80,16 +84,13 @@ bool WctlUnit::process(instr_trace_t* trace, bool* exit) { switch (wctl_type) { case WctlType::TMC: { - ThreadMask next_tmask(num_threads); - for (uint32_t t = 0; t < num_threads; ++t) { - next_tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); - } if (trace->eop) { - if (next_tmask.none()) { + auto tmask = this->next_tmask(trace); + if (tmask.none()) { *exit = true; release_warp = false; } else { - release_warp = core_->setTmask(trace->wid, next_tmask); + release_warp = core_->setTmask(trace->wid, tmask); } } } break; @@ -180,25 +181,15 @@ bool WctlUnit::process(instr_trace_t* trace, bool* exit) { release_warp = false; } } else { - release_warp = !core_->barrier_wait(bar_id, arg2, trace->wid); + // the barrier unit releases the warp once its phase has passed + core_->barrier_wait(bar_id, arg2, trace->wid); + release_warp = false; } } } break; case WctlType::PRED: { - ThreadMask pred(num_threads); - auto not_pred = wctlArgs.is_cond_neg; - for (uint32_t t = 0; t < num_threads; ++t) { - auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; - pred[t] = warp.tmask.test(t) && cond; - } - ThreadMask next_tmask = warp.tmask; - if (pred.any()) { - next_tmask &= pred; - } else { - next_tmask = ThreadMask(num_threads, rs2_data.at(thread_last).u); - } if (trace->eop) { - release_warp = core_->setTmask(trace->wid, next_tmask); + release_warp = core_->setTmask(trace->wid, this->next_tmask(trace)); } } break; case WctlType::WSYNC: diff --git a/sim/simx/wctl_unit.h b/sim/simx/wctl_unit.h index e688a59abd..d1ab897a28 100644 --- a/sim/simx/wctl_unit.h +++ b/sim/simx/wctl_unit.h @@ -40,6 +40,9 @@ class WctlUnit { bool is_shared(const instr_trace_t* trace) const; private: + // Thread mask a TMC or PRED installs. + ThreadMask next_tmask(const instr_trace_t* trace) const; + Core* core_; }; From 397d7d24836af24024aaab8359c0e8a65e0ed3d0 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:12 -0700 Subject: [PATCH 038/336] rtl: fix -DSCOPE builds of fetch, issue, LSU, TCU and graphics SCOPE_IO_SWITCH was declared inside the DBG_SCOPE_* blocks of VX_fetch, VX_issue_slice and VX_lsu_block, so a -DSCOPE build without the matching DBG_SCOPE_* tap left the module's scope bus undeclared and failed to elaborate. Declare the switch whenever SCOPE is set, and keep only the taps under DBG_SCOPE_*. With the TCU enabled, VX_execute never connected tcu_unit's scope port and VX_tcu_core never drove its scope output. VX_execute now switches 1 + EXT_TCU_ENABLED scope buses and binds the TCU to the second one; VX_tcu_core terminates its bus with SCOPE_IO_UNUSED. The texture, raster and OM cores terminated their scope bus with an empty SCOPE_IO_UNUSED(), which does not parse; they now pass the bus index. Their parents never bound those buses: VX_graphics now switches one bus per raster and OM core, VX_socket adds one per texture core, and VX_cluster gives VX_graphics its own slot. A -DSCOPE build with TEX, RASTER and OM enabled now lints clean. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_cluster.sv | 6 ++++++ hw/rtl/VX_graphics.sv | 18 ++++++++++++++++++ hw/rtl/VX_socket.sv | 6 ++++++ hw/rtl/core/VX_execute.sv | 6 +++++- hw/rtl/core/VX_fetch.sv | 2 +- hw/rtl/core/VX_issue_slice.sv | 2 +- hw/rtl/core/VX_lsu_block.sv | 2 +- hw/rtl/om/VX_om_core.sv | 4 ++-- hw/rtl/raster/VX_raster_core.sv | 4 ++-- hw/rtl/tcu/VX_tcu_core.sv | 5 +++++ hw/rtl/tex/VX_tex_core.sv | 4 ++-- 11 files changed, 49 insertions(+), 10 deletions(-) diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 6204d4ee88..79f625eda4 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -59,8 +59,13 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef SCOPE localparam scope_socket = 0; +`ifdef EXT_GFX_CLUSTER_ENABLE + localparam scope_gfx = NUM_SOCKETS; + `SCOPE_IO_SWITCH (NUM_SOCKETS + 1); +`else `SCOPE_IO_SWITCH (NUM_SOCKETS); `endif +`endif `ifdef PERF_ENABLE cache_perf_t l2_perf; @@ -429,6 +434,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_graphics #( .INSTANCE_ID (INSTANCE_ID) ) graphics ( + `SCOPE_IO_BIND (scope_gfx) .clk (clk), .reset (reset), .cluster_id (cluster_id), diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index 329dd7fc67..37a035506d 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -23,6 +23,8 @@ module VX_graphics import VX_gpu_pkg::* ; #( parameter `STRING INSTANCE_ID = "" ) ( + `SCOPE_IO_DECL + input wire clk, input wire reset, @@ -67,6 +69,20 @@ module VX_graphics import VX_gpu_pkg::* // no in-band `done`); the cluster ORs this into the device busy aggregation. output wire busy ); + +`ifdef SCOPE +`ifdef VX_CFG_EXT_RASTER_ENABLE + localparam scope_raster_cnt = `VX_CFG_NUM_RASTER_CORES; +`else + localparam scope_raster_cnt = 0; +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + localparam scope_om_cnt = `VX_CFG_NUM_OM_CORES; +`else + localparam scope_om_cnt = 0; +`endif + `SCOPE_IO_SWITCH (scope_raster_cnt + scope_om_cnt); +`endif `UNUSED_SPARAM (INSTANCE_ID) `ifndef VX_CFG_EXT_RASTER_ENABLE `UNUSED_VAR (cluster_id) @@ -156,6 +172,7 @@ module VX_graphics import VX_gpu_pkg::* .QUAD_FIFO_DEPTH (`VX_CFG_RASTER_QUAD_FIFO_DEPTH), .OUTPUT_QUADS (`VX_CFG_NUM_THREADS) ) raster_core ( + `SCOPE_IO_BIND (i) .clk (clk), .reset (reset), .instance_idx (`LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(cluster_id) * `LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(`VX_CFG_NUM_RASTER_CORES) @@ -417,6 +434,7 @@ module VX_graphics import VX_gpu_pkg::* .INSTANCE_ID (`SFORMATF(("%s-om%0d", INSTANCE_ID, i))), .NUM_LANES (OM_CORE_LANES) ) om_core ( + `SCOPE_IO_BIND (scope_raster_cnt + i) .clk (clk), .reset (reset), `ifdef PERF_ENABLE diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index e8dd0c8248..08aa35fc86 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -65,7 +65,12 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef SCOPE localparam scope_core = 0; +`ifdef VX_CFG_EXT_TEX_ENABLE + localparam scope_tex = `VX_CFG_SOCKET_SIZE; + `SCOPE_IO_SWITCH (`VX_CFG_SOCKET_SIZE + `VX_CFG_NUM_TEX_CORES); +`else `SCOPE_IO_SWITCH (`VX_CFG_SOCKET_SIZE); +`endif `endif VX_kmu_bus_if per_core_kmu_bus_if[`VX_CFG_SOCKET_SIZE](); @@ -442,6 +447,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_LANES (`VX_CFG_NUM_SFU_LANES), .TAG_WIDTH (TEX_REQ_ARB1_TAG_WIDTH) ) tex_core ( + `SCOPE_IO_BIND (scope_tex + i) .clk (clk), .reset (reset), `ifdef PERF_ENABLE diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index b1601ff6da..7454c4d92e 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -92,7 +92,10 @@ module VX_execute import VX_gpu_pkg::*; #( .branch_ctl_if (branch_ctl_if) ); - `SCOPE_IO_SWITCH (1); +`ifdef SCOPE + localparam SCOPE_COUNT = 1 + `VX_CFG_EXT_TCU_ENABLED; +`endif + `SCOPE_IO_SWITCH (SCOPE_COUNT); VX_lsu_unit #( .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))) @@ -122,6 +125,7 @@ module VX_execute import VX_gpu_pkg::*; #( VX_tcu_unit #( .INSTANCE_ID (`SFORMATF(("%s-tcu", INSTANCE_ID))) ) tcu_unit ( + `SCOPE_IO_BIND (1) .clk (clk), .reset (reset), `ifdef PERF_ENABLE diff --git a/hw/rtl/core/VX_fetch.sv b/hw/rtl/core/VX_fetch.sv index a4f8847e7f..43442d9bb0 100644 --- a/hw/rtl/core/VX_fetch.sv +++ b/hw/rtl/core/VX_fetch.sv @@ -260,8 +260,8 @@ module VX_fetch import VX_gpu_pkg::*; #( `endif `ifdef SCOPE -`ifdef DBG_SCOPE_FETCH `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_FETCH wire schedule_fire = schedule_if.valid && schedule_if.ready; wire icache_bus_req_fire = icache_bus_if.req_valid && icache_bus_if.req_ready; wire icache_bus_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; diff --git a/hw/rtl/core/VX_issue_slice.sv b/hw/rtl/core/VX_issue_slice.sv index 4f147c68e2..2ad54391fd 100644 --- a/hw/rtl/core/VX_issue_slice.sv +++ b/hw/rtl/core/VX_issue_slice.sv @@ -99,8 +99,8 @@ module VX_issue_slice import VX_gpu_pkg::*; #( assign warp_issued_wis = scoreboard_if.data.wis; `ifdef SCOPE -`ifdef DBG_SCOPE_ISSUE `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_ISSUE wire decode_fire = decode_if.valid && decode_if.ready; wire operands_fire = operands_if.valid && operands_if.ready; wire reset_negedge; diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index f127d9f260..3d9406851a 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -543,8 +543,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `ifdef SCOPE -`ifdef DBG_SCOPE_LSU `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_LSU wire reset_negedge; `NEG_EDGE (reset_negedge, reset); `SCOPE_TAP_EX (0, 3, 4, 2, ( diff --git a/hw/rtl/om/VX_om_core.sv b/hw/rtl/om/VX_om_core.sv index f8f610d39f..e1821560c3 100644 --- a/hw/rtl/om/VX_om_core.sv +++ b/hw/rtl/om/VX_om_core.sv @@ -575,8 +575,8 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( || ~pending_reads_empty || mem_rsp_valid || mem_unit_busy; `ifdef SCOPE -`ifdef DBG_SCOPE_OM `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_OM wire cache_bus_req_fire_0 = cache_bus_if[0].req_valid && cache_bus_if[0].req_ready; wire cache_bus_rsp_fire_0 = cache_bus_if[0].rsp_valid && cache_bus_if[0].rsp_ready; wire [OCACHE_TAG_WIDTH-UUID_WIDTH-1:0] cache_bus_req_tag = cache_bus_if[0].req_data.tag[OCACHE_TAG_WIDTH-UUID_WIDTH-1:0]; @@ -628,7 +628,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( reset_negedge, 1'b0, 4096 ); `else - `SCOPE_IO_UNUSED() + `SCOPE_IO_UNUSED(0) `endif `endif `ifdef CHIPSCOPE diff --git a/hw/rtl/raster/VX_raster_core.sv b/hw/rtl/raster/VX_raster_core.sv index be08681dbc..372dd007a0 100644 --- a/hw/rtl/raster/VX_raster_core.sv +++ b/hw/rtl/raster/VX_raster_core.sv @@ -560,8 +560,8 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( assign busy = armed_r | frame_kick; `ifdef SCOPE -`ifdef DBG_SCOPE_RASTER `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_RASTER wire cache_bus_req_fire_0 = cache_bus_if[0].req_valid && cache_bus_if[0].req_ready; wire cache_bus_rsp_fire_0 = cache_bus_if[0].rsp_valid && cache_bus_if[0].rsp_ready; wire raster_bus_fire = raster_bus_tmp_if[0].req_valid && raster_bus_tmp_if[0].req_ready; @@ -615,7 +615,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( reset_negedge, 1'b0, 4096 ); `else - `SCOPE_IO_UNUSED() + `SCOPE_IO_UNUSED(0) `endif `endif `ifdef CHIPSCOPE diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index 1762caafd7..e4fc89786f 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -41,6 +41,11 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Outputs VX_result_if.master result_if ); + +`ifdef SCOPE + `SCOPE_IO_SWITCH (1); + `SCOPE_IO_UNUSED (0) +`endif `UNUSED_SPARAM (INSTANCE_ID); // FEDP dot-product width in 32-bit words (doubles under FEDP2K). diff --git a/hw/rtl/tex/VX_tex_core.sv b/hw/rtl/tex/VX_tex_core.sv index c8e427df62..490b48965f 100644 --- a/hw/rtl/tex/VX_tex_core.sv +++ b/hw/rtl/tex/VX_tex_core.sv @@ -259,8 +259,8 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( ); `ifdef SCOPE -`ifdef DBG_SCOPE_TEX `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_TEX wire cache_bus_req_fire_0 = cache_bus_if[0].req_valid && cache_bus_if[0].req_ready; wire cache_bus_rsp_fire_0 = cache_bus_if[0].rsp_valid && cache_bus_if[0].rsp_ready; wire [TCACHE_TAG_WIDTH-UUID_WIDTH-1:0] cache_bus_req_tag = cache_bus_if[0].req_data.tag[TCACHE_TAG_WIDTH-UUID_WIDTH-1:0]; @@ -324,7 +324,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( reset_negedge, 1'b0, 4096 ); `else - `SCOPE_IO_UNUSED() + `SCOPE_IO_UNUSED(0) `endif `endif `ifdef CHIPSCOPE From 8705e2b7b666e3ed2664b7254257bfa26f6c889a Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:13 -0700 Subject: [PATCH 039/336] core: derive the core partitioning from the widths The core now has three structures, selected by VX_gpu_pkg from the fetch width (FW), issue width (IW) and per-unit block counts, with no config switch: - some unit has blocks != IW: FW frontend slices, the frontend crossbar, and the stage-sliced VX_issue / VX_execute / VX_commit; - every unit has blocks == IW, FW < IW (BACKEND_SLICES): FW frontend slices, the frontend crossbar, and IW backend slices; - every unit has blocks == IW == FW (COMPUTE_SLICES): IW compute slices, each a frontend slice joined directly to its backend slice, with no crossbar. This is the NVIDIA processing-block organization. Modules: - VX_frontend_slice (was VX_fetch_slice): warp scheduler, fetch, decode. - VX_frontend_xbar (was VX_fetch_xbar): routes instructions, branch and warp-control events and CSR traffic by warp id in the first two structures. - VX_backend_slice (was VX_compute_slice): issue, one block of every unit, and commit for one issue slice. - VX_compute_slice (new): VX_frontend_slice + VX_backend_slice. VX_CFG_COMPUTE_SLICES_ENABLE is removed: it could only turn off a structure that the widths already select. VX_sfu_shared (DXA/TEX/RTU request arbitration) and VX_tcu_shared (WGMMA tile buffers, AGU and sparse metadata) serve every block of the core, so they move from VX_sfu_unit / VX_tcu_unit to VX_core and are shared by all three structures. The units expose per-block ports instead. The tcu_unit and tensor unit-test wrappers instantiate VX_tcu_shared themselves, so the measured design still includes it. SimX already models this timing, so only its comments change to the new terminology. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor_wg.yaml | 2 +- docs/designs/microarchitecture.md | 35 + docs/designs/tensor_core_wgmma_engine.md | 3 +- hw/rtl/VX_gpu_pkg.sv | 31 +- hw/rtl/core/VX_backend_slice.sv | 192 +++++ hw/rtl/core/VX_compute_slice.sv | 221 ++++-- hw/rtl/core/VX_core.sv | 688 ++++++++++-------- hw/rtl/core/VX_execute.sv | 74 +- ...VX_fetch_slice.sv => VX_frontend_slice.sv} | 4 +- .../{VX_fetch_xbar.sv => VX_frontend_xbar.sv} | 14 +- hw/rtl/core/VX_scheduler.sv | 4 +- hw/rtl/core/VX_sfu_unit.sv | 47 +- hw/rtl/core/VX_uuid_gen.sv | 2 +- hw/rtl/core/VX_warp_sched.sv | 2 +- hw/rtl/interfaces/VX_slice_sched_if.sv | 2 +- hw/rtl/tcu/VX_tcu_bbuf.sv | 2 +- hw/rtl/tcu/VX_tcu_unit.sv | 80 +- hw/rtl/tcu/VX_tcu_wgmma.sv | 4 +- hw/unittest/tcu_unit/VX_tcu_unit_top.sv | 77 +- hw/unittest/tensor/VX_tensor_top.sv | 76 +- sim/simx/constants.h | 2 +- sim/simx/core.cpp | 12 +- sim/simx/scheduler.h | 4 +- sim/simx/socket.cpp | 2 +- 24 files changed, 1045 insertions(+), 535 deletions(-) create mode 100644 hw/rtl/core/VX_backend_slice.sv rename hw/rtl/core/{VX_fetch_slice.sv => VX_frontend_slice.sv} (97%) rename hw/rtl/core/{VX_fetch_xbar.sv => VX_frontend_xbar.sv} (95%) diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index b9bb956479..937d898bff 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -232,7 +232,7 @@ tests: args: -m64 -n64 -k64 # One execution block per unit: the four-block model ISSUE_WIDTH=4 implies is # several times the size, and a hosted CI runner cannot build it in a job. - # Two fetch slices: covers the fetch-slice front end on both models. + # Two frontend slices: covers the frontend crossbar on both models. configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS -DVX_CFG_FETCH_WIDTH=2 diff --git a/docs/designs/microarchitecture.md b/docs/designs/microarchitecture.md index 3a9913d82d..10516946c5 100644 --- a/docs/designs/microarchitecture.md +++ b/docs/designs/microarchitecture.md @@ -75,6 +75,41 @@ Vortex has a 6-stage pipeline: - **Commit** - Write result back to the register file and update the Scoreboard. +### Core partitioning + +The core splits its warps into slices, each a constant bit-slice of the warp id: +`VX_CFG_FETCH_WIDTH` (FW) frontend slices (`wid mod FW`), `VX_CFG_ISSUE_WIDTH` (IW) +issue slices (`wid mod IW`), and B blocks per execution unit (`wid mod B`), with +FW and B dividing IW. A slice owns its warps' pipeline state; the core scheduler +(CTA launch, barriers, wspawn, retirement), the shared SFU/TCU services and the +memory unit stay core-wide. + +The structure follows from the widths: + +| Widths | Structure | +|---|---| +| every B = IW = FW | IW compute slices (`VX_compute_slice`), no frontend crossbar | +| every B = IW, FW < IW | FW frontend slices, the frontend crossbar, IW backend slices (`VX_backend_slice`) | +| some unit has B ≠ IW | FW frontend slices, the frontend crossbar, stage-sliced `VX_issue` / `VX_execute` / `VX_commit` | + +- **Frontend slice** (`VX_frontend_slice`): warp scheduler, fetch and decode for + `wid mod FW`, on its own icache port. +- **Backend slice** (`VX_backend_slice`): issue, execute (one block of every + unit) and commit for one issue slice. +- **Compute slice** (`VX_compute_slice`): a frontend slice and its backend slice, + joined directly: one end-to-end pipeline for `wid mod IW`. This is the + counterpart of an NVIDIA SM processing block. +- **Frontend crossbar** (`VX_frontend_xbar`): combinational routing between FW + frontend slices and the back end whenever the core is not built from + compute slices. It sends each instruction to its issue slice, and each + branch, warp-control and CSR event to the frontend slice that owns the warp. +- **Core-wide services**, reached through one port per block in every structure: + - `VX_sfu_shared` arbitrates the SFU blocks onto the core's single DXA, TEX + and RTU buses. + - `VX_tcu_shared` holds the state that spans TCU blocks: the WGMMA tile + buffers, whose B operand is common to a warpgroup's blocks, and the sparse + metadata AGU. + ### Vortex clustering architecture - Sockets - Grouping multiple cores sharing L1 cache diff --git a/docs/designs/tensor_core_wgmma_engine.md b/docs/designs/tensor_core_wgmma_engine.md index d6eefb36f1..ad890708d8 100644 --- a/docs/designs/tensor_core_wgmma_engine.md +++ b/docs/designs/tensor_core_wgmma_engine.md @@ -192,7 +192,8 @@ is muxed by `op_type`: `VX_tcu_wgmma` is a third party that only **observes** the dispatch path. It builds the tile-buffer request from each WGMMA micro-op and returns operand data and a ready signal, but never drives the handshake itself; the core -does. +does. `VX_tcu_agu` and `VX_tcu_wgmma` serve every block, so they live in +the core-level `VX_tcu_shared` and reach each block through per-block ports. --- diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 114b2ef153..ec92eca60a 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -630,7 +630,7 @@ package VX_gpu_pkg; end endfunction - // Fetch slice f schedules and fetches the warps with wid mod FETCH_WIDTH == f, + // Frontend slice f schedules and fetches the warps with wid mod FETCH_WIDTH == f, // naming them by their local id wfs = wid / FETCH_WIDTH. localparam FETCH_FSW_BITS = `CLOG2(`VX_CFG_FETCH_WIDTH); localparam FETCH_FSW_W = `UP(FETCH_FSW_BITS); @@ -661,8 +661,8 @@ package VX_gpu_pkg; end endfunction - // FU block ports per fetch slice: a unit with B blocks serves warps with - // wid mod B == b, so B >= FETCH_WIDTH gives each fetch slice B / FETCH_WIDTH + // FU block ports per frontend slice: a unit with B blocks serves warps with + // wid mod B == b, so B >= FETCH_WIDTH gives each frontend slice B / FETCH_WIDTH // blocks and B < FETCH_WIDTH shares each block among FETCH_WIDTH / B slices. localparam FETCH_ALU_PORTS = (`VX_CFG_NUM_ALU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_ALU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; localparam FETCH_SFU_PORTS = (`VX_CFG_NUM_SFU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_SFU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; @@ -719,14 +719,17 @@ package VX_gpu_pkg; end endfunction - // The core is built from ISSUE_WIDTH compute slices (VX_compute_slice) when - // every execution unit has one block per issue slice; otherwise it keeps the - // stage-sliced issue / execute / commit structure. - localparam COMPUTE_SLICES_ENABLE = (`VX_CFG_NUM_ALU_BLOCKS == `VX_CFG_ISSUE_WIDTH) - && (`VX_CFG_NUM_LSU_BLOCKS == `VX_CFG_ISSUE_WIDTH) - && (`VX_CFG_NUM_SFU_BLOCKS == `VX_CFG_ISSUE_WIDTH) - && (!`VX_CFG_EXT_F_ENABLED || (`VX_CFG_NUM_FPU_BLOCKS == `VX_CFG_ISSUE_WIDTH)) - && (!`VX_CFG_EXT_TCU_ENABLED || (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH)); + // The core structure follows from the widths. When every execution unit has + // one block per issue slice, the back end is ISSUE_WIDTH backend slices + // (VX_backend_slice: issue, execute, commit); otherwise it is stage-sliced. + // When the fetch width matches too, each backend slice and its frontend slice + // form one compute slice (VX_compute_slice), with no frontend crossbar. + localparam BACKEND_SLICES = (`VX_CFG_NUM_ALU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_LSU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_SFU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (!`VX_CFG_EXT_F_ENABLED || (`VX_CFG_NUM_FPU_BLOCKS == `VX_CFG_ISSUE_WIDTH)) + && (!`VX_CFG_EXT_TCU_ENABLED || (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH)); + localparam COMPUTE_SLICES = BACKEND_SLICES && (`VX_CFG_FETCH_WIDTH == `VX_CFG_ISSUE_WIDTH); /////////////////////////////// TENSOR UNIT /////////////////////////////// @@ -1022,7 +1025,7 @@ package VX_gpu_pkg; logic [NCTA_WIDTH-1:0] id_cta; } cta_lane_bcast_t; - // Scheduler traffic between an FU block and the fetch slice owning the warp, + // Scheduler traffic between an FU block and the frontend slice owning the warp, // addressed by core-wide wid. // ALU branch resolution @@ -1648,11 +1651,11 @@ package VX_gpu_pkg; // Block size in bytes localparam ICACHE_LINE_SIZE = `VX_CFG_L1_LINE_SIZE; - // Request ports per core: one per fetch slice, each fetching its own warps + // Request ports per core: one per frontend slice, each fetching its own warps localparam ICACHE_NUM_REQS = `VX_CFG_FETCH_WIDTH; localparam ICACHE_NUM_BANKS = `VX_CFG_ICACHE_NUM_BANKS; - // Core request tag Id bits: the warp id local to the requesting fetch slice + // Core request tag Id bits: the warp id local to the requesting frontend slice localparam ICACHE_TAG_ID_BITS = FETCH_WFS_W; // Core request tag bits as VX_fetch writes them (UUID + warp id). diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv new file mode 100644 index 0000000000..93732a84cd --- /dev/null +++ b/hw/rtl/core/VX_backend_slice.sv @@ -0,0 +1,192 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One issue slice carried through issue, execute and commit, serving the warps +// with wid mod ISSUE_WIDTH == slice_id with one block of every execution unit. +// Every slice is the same module: its identity arrives on slice_id/core_id, and +// only the core-wide resources (frontend slices, warp-control resources, memory +// system, SFU buses, WGMMA tile buffers and the metadata AGU) are reached through +// its ports. +module VX_backend_slice import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [ISSUE_ISW_W-1:0] slice_id, + +`ifdef PERF_ENABLE + output issue_perf_t issue_perf, +`endif + + // decoded instructions of the slice's warps + VX_decode_if.slave decode_if, + + // issued and retired warps (registered), indexed by wid / ISSUE_WIDTH + output wire warp_issued, + output wire [ISSUE_WIS_W-1:0] warp_issued_wis, + output wire [PER_ISSUE_WARPS-1:0] committed_wis, + + // scheduler traffic of the slice's ALU and SFU blocks + VX_branch_ctl_if.master branch_ctl_if, + VX_warp_ctl_if.master warp_ctl_if, + VX_sched_csr_if.slave sched_csr_if, +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if.master sched_unlock_if, +`endif + + // LSU + VX_lsu_sched_if.master lsu_client_if, + + // SFU +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, +`endif + + // TCU +`ifdef VX_CFG_TCU_WGMMA_ENABLE + output wire tcu_exec_valid, + output wire tcu_exec_ready, + output tcu_execute_t tcu_exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire tcu_tbuf_ready, +`endif +`ifdef TCU_META_ENABLE + output wire tcu_agu_ld_valid, + output tcu_execute_t tcu_agu_ld_data, + input wire tcu_agu_ld_ready, + input wire tcu_agu_result_valid, + input tcu_result_t tcu_agu_result_data, + output wire tcu_agu_result_ready, + input wire tcu_meta_wr_en, + input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, + input wire [4:0] tcu_meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, +`endif + + // core-wide warp control + VX_warp_shared_if.master warp_shared_if +); + `STATIC_ASSERT (BACKEND_SLICES, ("invalid parameter")) + + `SCOPE_IO_SWITCH (2); + + VX_dispatch_if dispatch_if [NUM_EX_UNITS](); + VX_commit_if commit_if [NUM_EX_UNITS](); + VX_writeback_if writeback_if(); + + // issue + + wire issue_fire; + wire [ISSUE_WIS_W-1:0] issue_wis; + + VX_issue_slice #( + .INSTANCE_ID (`SFORMATF(("%s-issue%s", INSTANCE_ID, SLICE_STR))) + ) issue_slice ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .slice_id (slice_id), + `ifdef PERF_ENABLE + .issue_perf (issue_perf), + `endif + .decode_if (decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .warp_issued (issue_fire), + .warp_issued_wis (issue_wis) + ); + + `BUFFER(warp_issued, issue_fire); + `BUFFER(warp_issued_wis, issue_wis); + + // execute + + VX_execute_slice #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), + .SLICE_STR (SLICE_STR) + ) execute_slice ( + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (slice_id), + .dispatch_if (dispatch_if), + .commit_if (commit_if), + .branch_ctl_if (branch_ctl_if), + .lsu_client_if (lsu_client_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if (sched_unlock_if), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid), + .tcu_exec_ready (tcu_exec_ready), + .tcu_exec_data (tcu_exec_data), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), + .tcu_tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid), + .tcu_agu_ld_data (tcu_agu_ld_data), + .tcu_agu_ld_ready (tcu_agu_ld_ready), + .tcu_agu_result_valid (tcu_agu_result_valid), + .tcu_agu_result_data (tcu_agu_result_data), + .tcu_agu_result_ready (tcu_agu_result_ready), + .tcu_meta_wr_en (tcu_meta_wr_en), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if) + ); + + // commit + + VX_commit_slice #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit_slice ( + .clk (clk), + .reset (reset), + .commit_if (commit_if), + .writeback_if (writeback_if), + .committed_wis (committed_wis) + ); + +endmodule diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index 33c771fdc1..0111b36d12 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -13,12 +13,12 @@ `include "VX_define.vh" -// One issue slice carried through issue, execute and commit, serving the warps -// with wid mod ISSUE_WIDTH == slice_id with one block of every execution unit. -// Every slice is the same module: its identity arrives on slice_id/core_id, and -// only the core-wide resources (fetch slices, warp-control resources, memory -// system, SFU buses, WGMMA tile buffers and the metadata AGU) are reached through -// its ports. +// A frontend slice and its backend slice: the warps with wid mod ISSUE_WIDTH == +// slice_id carried through fetch, decode, issue, execute and commit. Used when +// the fetch, issue and every execution-unit width are equal, so frontend slice i +// serves only issue slice i and the blocks of slice i, and the two meet +// directly. Only the core scheduler, the icache, the memory system and the +// shared SFU/TCU services are reached through the ports. module VX_compute_slice import VX_gpu_pkg::*; `ifdef VX_CFG_EXT_TCU_ENABLE import VX_tcu_pkg::*; @@ -36,23 +36,26 @@ import VX_tcu_pkg::*; input wire [ISSUE_ISW_W-1:0] slice_id, `ifdef PERF_ENABLE + output fetch_perf_t fetch_perf, output issue_perf_t issue_perf, `endif - // decoded instructions of the slice's warps - VX_decode_if.slave decode_if, - - // issued and retired warps (registered), indexed by wid / ISSUE_WIDTH - output wire warp_issued, - output wire [ISSUE_WIS_W-1:0] warp_issued_wis, + // core scheduler and icache + VX_slice_sched_if.slave slice_sched_if, + VX_mem_bus_if.master icache_bus_if, output wire [PER_ISSUE_WARPS-1:0] committed_wis, - // scheduler traffic of the slice's ALU and SFU blocks - VX_branch_ctl_if.master branch_ctl_if, - VX_warp_ctl_if.master warp_ctl_if, - VX_sched_csr_if.slave sched_csr_if, -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if.master sched_unlock_if, + // core scheduler state + input wire [PERF_CTR_BITS-1:0] cycles, + input wire [PERF_CTR_BITS-1:0] instret, + input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, + input wire lsu_sched_drained, + output wire [NCTA_WIDTH-1:0] csr_rd_cta_id, + input cta_ctx_t csr_rd_ctx, +`ifdef VX_CFG_VM_ENABLE + input wire [`VX_CFG_XLEN-1:0] satp, + output wire satp_wr_valid, + output wire [`VX_CFG_XLEN-1:0] satp_wr_data, `endif // LSU @@ -94,53 +97,160 @@ import VX_tcu_pkg::*; // core-wide warp control VX_warp_shared_if.master warp_shared_if ); - `STATIC_ASSERT (COMPUTE_SLICES_ENABLE, ("invalid parameter")) + `STATIC_ASSERT (COMPUTE_SLICES, ("invalid parameter")) + + localparam SW = PER_FETCH_WARPS; `SCOPE_IO_SWITCH (2); - VX_dispatch_if dispatch_if [NUM_EX_UNITS](); - VX_commit_if commit_if [NUM_EX_UNITS](); - VX_writeback_if writeback_if(); + VX_decode_if decode_if(); + VX_branch_ctl_if branch_ctl_if(); + VX_warp_ctl_if warp_ctl_if(); + VX_sched_csr_if sched_csr_if(); +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if sched_unlock_if(); +`endif - // issue + wire warp_issued; + wire [ISSUE_WIS_W-1:0] warp_issued_wis; - wire issue_fire; - wire [ISSUE_WIS_W-1:0] issue_wis; + // slice-local view of the frontend slice's ports: block i serves only frontend slice + // i, and a warp's fetch-local and issue-local ids are the same + sched_branch_t [0:0] branch; + sched_wctl_t [0:0] wctl; + wire [0:0][NW_WIDTH-1:0] dvstack_wid; + wire [0:0][DV_STACK_SIZEW-1:0] dvstack_ptr; + wire [0:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] dvstack_mscratch; + wire [0:0][NW_WIDTH-1:0] csr_rd_wid; + sched_csr_rd_t [0:0] csr_rd; + sched_csr_wr_t [0:0] csr_wr; +`ifdef VX_CFG_EXT_RTU_ENABLE + wire [0:0] rtu_unlock_valid; + wire [0:0][NW_WIDTH-1:0] rtu_unlock_wid; +`endif + wire [SW-1:0] issued; + wire [SW-1:0] pending_alm_empty; - VX_issue_slice #( - .INSTANCE_ID (`SFORMATF(("%s-issue%s", INSTANCE_ID, SLICE_STR))) - ) issue_slice ( - `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), - .slice_id (slice_id), + VX_frontend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR ((`VX_CFG_FETCH_WIDTH > 1) ? SLICE_STR : "") + ) frontend_slice ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (FETCH_FSW_W'(slice_id)), `ifdef PERF_ENABLE - .issue_perf (issue_perf), + .fetch_perf (fetch_perf), `endif - .decode_if (decode_if), - .writeback_if (writeback_if), - .dispatch_if (dispatch_if), - .warp_issued (issue_fire), - .warp_issued_wis (issue_wis) + .slice_sched_if (slice_sched_if), + .icache_bus_if (icache_bus_if), + .decode_if (decode_if), + .branch (branch), + .wctl (wctl), + .dvstack_wid (dvstack_wid), + .dvstack_ptr (dvstack_ptr), + .dvstack_mscratch (dvstack_mscratch), + .csr_rd_wid (csr_rd_wid), + .csr_rd (csr_rd), + .csr_wr (csr_wr), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (rtu_unlock_valid), + .rtu_unlock_wid (rtu_unlock_wid), + `endif + .issued (issued), + .committed (committed_wis), + .pending_alm_empty (pending_alm_empty) ); - `BUFFER(warp_issued, issue_fire); - `BUFFER(warp_issued_wis, issue_wis); + for (genvar l = 0; l < SW; ++l) begin : g_issued + assign issued[l] = warp_issued && (warp_issued_wis == ISSUE_WIS_W'(l)); + end + + assign branch[0].valid = branch_ctl_if.valid; + assign branch[0].wid = branch_ctl_if.wid; + assign branch[0].taken = branch_ctl_if.taken; + assign branch[0].dest = branch_ctl_if.dest; + assign branch[0].is_trap = branch_ctl_if.is_trap; + assign branch[0].is_mret = branch_ctl_if.is_mret; + assign branch[0].trap_cause = branch_ctl_if.trap_cause; + + assign wctl[0].tmc_valid = warp_ctl_if.tmc_valid; + assign wctl[0].split_valid = warp_ctl_if.split_valid; + assign wctl[0].sjoin_valid = warp_ctl_if.sjoin_valid; + assign wctl[0].wsync_valid = warp_ctl_if.wsync_valid; + assign wctl[0].wid = warp_ctl_if.wid; + assign wctl[0].tmc = warp_ctl_if.tmc; + assign wctl[0].split = warp_ctl_if.split; + assign wctl[0].sjoin = warp_ctl_if.sjoin; + assign dvstack_wid[0] = warp_ctl_if.dvstack_wid; + assign warp_ctl_if.dvstack_ptr = dvstack_ptr[0]; + assign warp_ctl_if.mscratch = dvstack_mscratch[0]; + assign warp_ctl_if.warp_pending_alm_empty = pending_alm_empty; + assign warp_ctl_if.lsu_sched_drained = lsu_sched_drained; - // execute + assign csr_rd_wid[0] = sched_csr_if.csr_rd_wid; + assign csr_rd_cta_id = sched_csr_if.csr_rd_cta_id; + assign csr_wr[0].valid = sched_csr_if.csr_wr_valid; + assign csr_wr[0].wid = sched_csr_if.csr_wr_wid; + assign csr_wr[0].data = sched_csr_if.csr_wr_data; + assign csr_wr[0].trap_valid = sched_csr_if.trap_csr_wr_valid; + assign csr_wr[0].trap_data = sched_csr_if.trap_csr_wr_data; - VX_execute_slice #( - .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), + assign sched_csr_if.cycles = cycles; + assign sched_csr_if.instret = instret; + assign sched_csr_if.active_warps = active_warps; + assign sched_csr_if.thread_mask = csr_rd[0].thread_mask; + assign sched_csr_if.mscratch = csr_rd[0].mscratch; + assign sched_csr_if.csr_mstatus = csr_rd[0].mstatus; + assign sched_csr_if.csr_mtvec = csr_rd[0].mtvec; + assign sched_csr_if.csr_mepc = csr_rd[0].mepc; + assign sched_csr_if.csr_mcause = csr_rd[0].mcause; + assign sched_csr_if.csr_mtval = csr_rd[0].mtval; + assign sched_csr_if.cta_csrs.cta_id = sched_csr_if.csr_rd_cta_id; + assign sched_csr_if.cta_csrs.cta_rank = csr_rd[0].cta_rank; + assign sched_csr_if.cta_csrs.cta_size = csr_rd_ctx.cta_size; + assign sched_csr_if.cta_csrs.block_idx = csr_rd_ctx.block_idx; + assign sched_csr_if.cta_csrs.block_dim = csr_rd_ctx.block_dim; + assign sched_csr_if.cta_csrs.grid_dim = csr_rd_ctx.grid_dim; + assign sched_csr_if.cta_csrs.entry = csr_rd_ctx.entry; + assign sched_csr_if.cta_csrs.param = csr_rd_ctx.param; + assign sched_csr_if.cta_csrs.lmem_addr = csr_rd_ctx.lmem_addr; + assign sched_csr_if.cta_csrs.cluster_size = csr_rd_ctx.cluster_size; + assign sched_csr_if.cta_lane = csr_rd[0].cta_lane; +`ifdef VX_CFG_VM_ENABLE + assign sched_csr_if.csr_satp = satp; + assign satp_wr_valid = sched_csr_if.satp_wr_valid; + assign satp_wr_data = sched_csr_if.satp_wr_data; +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + assign rtu_unlock_valid[0] = sched_unlock_if.valid; + assign rtu_unlock_wid[0] = sched_unlock_if.wid; +`endif + + VX_backend_slice #( + .INSTANCE_ID (INSTANCE_ID), .SLICE_STR (SLICE_STR) - ) execute_slice ( - `SCOPE_IO_BIND (1) + ) backend_slice ( + `SCOPE_IO_BIND (1) .clk (clk), .reset (reset), .core_id (core_id), .slice_id (slice_id), - .dispatch_if (dispatch_if), - .commit_if (commit_if), + `ifdef PERF_ENABLE + .issue_perf (issue_perf), + `endif + .decode_if (decode_if), + .warp_issued (warp_issued), + .warp_issued_wis (warp_issued_wis), + .committed_wis (committed_wis), .branch_ctl_if (branch_ctl_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if), + `endif .lsu_client_if (lsu_client_if), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (dxa_req_bus_if), @@ -150,7 +260,6 @@ import VX_tcu_pkg::*; `endif `ifdef VX_CFG_EXT_RTU_ENABLE .rtu_bus_if (rtu_bus_if), - .sched_unlock_if (sched_unlock_if), `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE .tcu_exec_valid (tcu_exec_valid), @@ -172,21 +281,7 @@ import VX_tcu_pkg::*; .tcu_meta_wr_idx (tcu_meta_wr_idx), .tcu_meta_wr_data (tcu_meta_wr_data), `endif - .warp_shared_if (warp_shared_if), - .warp_ctl_if (warp_ctl_if), - .sched_csr_if (sched_csr_if) - ); - - // commit - - VX_commit_slice #( - .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) - ) commit_slice ( - .clk (clk), - .reset (reset), - .commit_if (commit_if), - .writeback_if (writeback_if), - .committed_wis (committed_wis) + .warp_shared_if (warp_shared_if) ); endmodule diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index b8aaf1d528..4b1b9c6e42 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -94,7 +94,7 @@ import VX_tcu_pkg::*; VX_lsu_sched_if tcu_mem_if(); `endif - // fetch-slice icache requests; port 0 also carries the injected flush + // frontend-slice icache requests; port 0 also carries the injected flush VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) @@ -200,7 +200,7 @@ import VX_tcu_pkg::*; .cache_bus_if (icache_bus_if[0]) ); - // The other ports carry their fetch slice's registered requests, with the + // The other ports carry their frontend slice's registered requests, with the // flush arbiter's select bit, the tag's LSB, clear. for (genvar i = 1; i < ICACHE_NUM_REQS; ++i) begin : g_icache_ports assign icache_bus_if[i].req_valid = fetch_icache_if[i].req_valid; @@ -219,25 +219,12 @@ import VX_tcu_pkg::*; `UNUSED_VAR (icache_bus_if[i].rsp_data.tag[0]) end - // scheduling and the front end: the core scheduler, FETCH_WIDTH fetch slices - // and the crossbar to the issue slices and ALU/SFU blocks + // scheduling: the core scheduler and its per-frontend-slice interfaces - localparam FW = `VX_CFG_FETCH_WIDTH; - localparam NPA = FETCH_ALU_PORTS; - localparam NPS = FETCH_SFU_PORTS; + localparam FW = `VX_CFG_FETCH_WIDTH; VX_slice_sched_if slice_sched_if[FW](); - VX_decode_if fs_decode_if[FW](); - VX_decode_if is_decode_if[`VX_CFG_ISSUE_WIDTH](); - VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); - VX_warp_ctl_if warp_ctl_if[`VX_CFG_NUM_SFU_BLOCKS](); - VX_sched_csr_if sched_csr_if[`VX_CFG_NUM_SFU_BLOCKS](); -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if sched_unlock_if[`VX_CFG_NUM_SFU_BLOCKS](); -`endif - wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid; - wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis; wire [`VX_CFG_NUM_WARPS-1:0] committed_warps; wire [`VX_CFG_NUM_WARPS-1:0] sched_active_warps; `ifdef VX_CFG_VM_ENABLE @@ -246,27 +233,8 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data; `endif - sched_branch_t [FW-1:0][NPA-1:0] fs_branch; - sched_wctl_t [FW-1:0][NPS-1:0] fs_wctl; - wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_dvstack_wid; - wire [FW-1:0][NPS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr; - wire [FW-1:0][NPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch; - wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid; - sched_csr_rd_t [FW-1:0][NPS-1:0] fs_csr_rd; - sched_csr_wr_t [FW-1:0][NPS-1:0] fs_csr_wr; -`ifdef VX_CFG_EXT_RTU_ENABLE - wire [FW-1:0][NPS-1:0] fs_rtu_unlock_valid; - wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid; -`endif - wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_issued; - wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_committed; - wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty; - wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id; cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx; - for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_csr_rd_cta_id - assign csr_rd_cta_id[b] = sched_csr_if[b].csr_rd_cta_id; - end VX_scheduler #( .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))) @@ -298,105 +266,103 @@ import VX_tcu_pkg::*; .busy (sched_busy) ); - VX_fetch_xbar fetch_xbar ( - .fs_decode_if (fs_decode_if), - .is_decode_if (is_decode_if), - .issue_valid (issue_valid), - .issue_wis (issue_wis), - .committed_warps (committed_warps), - .fs_issued (fs_issued), - .fs_committed (fs_committed), - .fs_pending_alm_empty (fs_pending_alm_empty), - .branch_ctl_if (branch_ctl_if), - .warp_ctl_if (warp_ctl_if), - .sched_csr_if (sched_csr_if), - `ifdef VX_CFG_EXT_RTU_ENABLE - .sched_unlock_if (sched_unlock_if), +`ifdef PERF_ENABLE + fetch_perf_t per_slice_fetch_perf [FW]; + `PERF_COUNTER_ADD (pipeline_perf.fetch, per_slice_fetch_perf, stalls, PERF_CTR_BITS, FW, (FW > 2)) +`endif + + // core-wide services, reached through one port per SFU or TCU block + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if block_dxa_req_bus_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + ) block_tex_bus_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH) + ) block_rtu_bus_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif +`ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; +`endif +`ifdef TCU_META_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; + tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; + tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire tcu_meta_wr_en; + wire [NW_WIDTH-1:0] tcu_meta_wr_wid; + wire [4:0] tcu_meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; +`endif + + VX_sfu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-sfu-shared", INSTANCE_ID))) + ) sfu_shared ( + `ifdef VX_CFG_EXT_DXA_ENABLE + .block_dxa_req_bus_if (block_dxa_req_bus_if), + .dxa_req_bus_if (dxa_req_bus_if), `endif - .cycles (sched_cycles), - .instret (sched_instret), - .active_warps (sched_active_warps), - .lsu_sched_drained (lsu_drained), - .csr_rd_ctx (csr_rd_ctx), - `ifdef VX_CFG_VM_ENABLE - .satp (sched_satp), - .satp_wr_valid (satp_wr_valid), - .satp_wr_data (satp_wr_data), + `ifdef VX_CFG_EXT_TEX_ENABLE + .block_tex_bus_if (block_tex_bus_if), + .tex_bus_if (tex_bus_if), `endif - .fs_branch (fs_branch), - .fs_wctl (fs_wctl), - .fs_dvstack_wid (fs_dvstack_wid), - .fs_dvstack_ptr (fs_dvstack_ptr), - .fs_dvstack_mscratch (fs_dvstack_mscratch), - .fs_csr_rd_wid (fs_csr_rd_wid), - .fs_csr_rd (fs_csr_rd), `ifdef VX_CFG_EXT_RTU_ENABLE - .fs_rtu_unlock_valid (fs_rtu_unlock_valid), - .fs_rtu_unlock_wid (fs_rtu_unlock_wid), + .block_rtu_bus_if (block_rtu_bus_if), + .rtu_bus_if (rtu_bus_if), `endif - .fs_csr_wr (fs_csr_wr) + .clk (clk), + .reset (reset) ); -`ifdef PERF_ENABLE - fetch_perf_t per_slice_fetch_perf [FW]; - `PERF_COUNTER_ADD (pipeline_perf.fetch, per_slice_fetch_perf, stalls, PERF_CTR_BITS, FW, (FW > 2)) -`endif - -`ifdef SCOPE - wire [FW-1:0] fetch_scope_bus_in, fetch_scope_bus_out; - VX_scope_switch #( - .N (FW) - ) fetch_scope_switch ( - .clk (clk), - .reset (scope_reset_w[0]), - .req_in (scope_bus_in_w[0]), - .rsp_out (scope_bus_out_w[0]), - .req_out (fetch_scope_bus_in), - .rsp_in (fetch_scope_bus_out) +`ifdef VX_CFG_EXT_TCU_ENABLE + VX_tcu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-tcu-shared", INSTANCE_ID))), + .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) + ) tcu_shared ( + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), + `endif + .clk (clk), + .reset (reset) ); `endif - for (genvar f = 0; f < FW; ++f) begin : g_fetch_slices - VX_fetch_slice #( - .INSTANCE_ID (INSTANCE_ID), - .SLICE_STR ((FW > 1) ? `SFORMATF(("%0d", f)) : "") - ) fetch_slice ( - `ifdef SCOPE - .scope_reset (scope_reset_w[0]), - .scope_bus_in (fetch_scope_bus_in[f]), - .scope_bus_out (fetch_scope_bus_out[f]), - `endif - .clk (clk), - .reset (reset), - .core_id (core_id), - .slice_id (FETCH_FSW_W'(f)), - `ifdef PERF_ENABLE - .fetch_perf (per_slice_fetch_perf[f]), - `endif - .slice_sched_if (slice_sched_if[f]), - .icache_bus_if (fetch_icache_if[f]), - .decode_if (fs_decode_if[f]), - .branch (fs_branch[f]), - .wctl (fs_wctl[f]), - .dvstack_wid (fs_dvstack_wid[f]), - .dvstack_ptr (fs_dvstack_ptr[f]), - .dvstack_mscratch (fs_dvstack_mscratch[f]), - .csr_rd_wid (fs_csr_rd_wid[f]), - .csr_rd (fs_csr_rd[f]), - .csr_wr (fs_csr_wr[f]), - `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_unlock_valid (fs_rtu_unlock_valid[f]), - .rtu_unlock_wid (fs_rtu_unlock_wid[f]), - `endif - .issued (fs_issued[f]), - .committed (fs_committed[f]), - .pending_alm_empty (fs_pending_alm_empty[f]) - ); - end - - // issue, execute and commit + // fetch, issue, execute and commit, structured by COMPUTE_SLICES / BACKEND_SLICES - if (COMPUTE_SLICES_ENABLE) begin : g_compute_slices + if (COMPUTE_SLICES) begin : g_compute_slices `ifdef PERF_ENABLE issue_perf_t per_slice_issue_perf [`VX_CFG_ISSUE_WIDTH]; @@ -409,40 +375,6 @@ import VX_tcu_pkg::*; end `endif - `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if slice_dxa_req_bus_if[`VX_CFG_ISSUE_WIDTH](); - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if #( - .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH) - ) slice_tex_bus_if[`VX_CFG_ISSUE_WIDTH](); - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if #( - .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH) - ) slice_rtu_bus_if[`VX_CFG_ISSUE_WIDTH](); - `endif - - `ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_exec_valid, tcu_exec_ready; - tcu_execute_t tcu_exec_data [`VX_CFG_ISSUE_WIDTH]; - wire [`VX_CFG_ISSUE_WIDTH-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_tbuf_ready; - `endif - `ifdef TCU_META_ENABLE - wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; - tcu_execute_t tcu_agu_ld_data [`VX_CFG_ISSUE_WIDTH]; - wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_agu_result_valid, tcu_agu_result_ready; - tcu_result_t tcu_agu_result_data [`VX_CFG_ISSUE_WIDTH]; - wire tcu_meta_wr_en; - wire [NW_WIDTH-1:0] tcu_meta_wr_wid; - wire [4:0] tcu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; - `endif - `ifdef SCOPE wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_scope_bus_in, slice_scope_bus_out; VX_scope_switch #( @@ -456,6 +388,7 @@ import VX_tcu_pkg::*; .rsp_in (slice_scope_bus_out) ); `endif + `SCOPE_IO_UNUSED (0) `SCOPE_IO_UNUSED (2) for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices @@ -475,27 +408,32 @@ import VX_tcu_pkg::*; .core_id (core_id), .slice_id (ISSUE_ISW_W'(i)), `ifdef PERF_ENABLE + .fetch_perf (per_slice_fetch_perf[i]), .issue_perf (per_slice_issue_perf[i]), `endif - .decode_if (is_decode_if[i]), - .warp_issued (issue_valid[i]), - .warp_issued_wis (issue_wis[i]), + .slice_sched_if (slice_sched_if[i]), + .icache_bus_if (fetch_icache_if[i]), .committed_wis (committed_wis), - .branch_ctl_if (branch_ctl_if[i]), - .warp_ctl_if (warp_ctl_if[i]), - .sched_csr_if (sched_csr_if[i]), - `ifdef VX_CFG_EXT_RTU_ENABLE - .sched_unlock_if (sched_unlock_if[i]), + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + .lsu_sched_drained (lsu_drained), + .csr_rd_cta_id (csr_rd_cta_id[i]), + .csr_rd_ctx (csr_rd_ctx[i]), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp[i]), + .satp_wr_valid (satp_wr_valid[i]), + .satp_wr_data (satp_wr_data[i]), `endif .lsu_client_if (lsu_client_if[i]), `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_req_bus_if (slice_dxa_req_bus_if[i]), + .dxa_req_bus_if (block_dxa_req_bus_if[i]), `endif `ifdef VX_CFG_EXT_TEX_ENABLE - .tex_bus_if (slice_tex_bus_if[i]), + .tex_bus_if (block_tex_bus_if[i]), `endif `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_bus_if (slice_rtu_bus_if[i]), + .rtu_bus_if (block_rtu_bus_if[i]), `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE .tcu_exec_valid (tcu_exec_valid[i]), @@ -526,155 +464,331 @@ import VX_tcu_pkg::*; end end - VX_sfu_shared #( - .INSTANCE_ID (`SFORMATF(("%s-execute-sfu-shared", INSTANCE_ID))) - ) sfu_shared ( - `ifdef VX_CFG_EXT_DXA_ENABLE - .block_dxa_req_bus_if (slice_dxa_req_bus_if), - .dxa_req_bus_if (dxa_req_bus_if), + end else begin : g_crossbar + + VX_decode_if fs_decode_if[FW](); + VX_decode_if is_decode_if[`VX_CFG_ISSUE_WIDTH](); + VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); + VX_warp_ctl_if warp_ctl_if[`VX_CFG_NUM_SFU_BLOCKS](); + VX_sched_csr_if sched_csr_if[`VX_CFG_NUM_SFU_BLOCKS](); + for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_csr_rd_cta_id + assign csr_rd_cta_id[b] = sched_csr_if[b].csr_rd_cta_id; + end + `ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if sched_unlock_if[`VX_CFG_NUM_SFU_BLOCKS](); + `endif + wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid; + wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis; + + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; + + sched_branch_t [FW-1:0][NPA-1:0] fs_branch; + sched_wctl_t [FW-1:0][NPS-1:0] fs_wctl; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_dvstack_wid; + wire [FW-1:0][NPS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr; + wire [FW-1:0][NPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid; + sched_csr_rd_t [FW-1:0][NPS-1:0] fs_csr_rd; + sched_csr_wr_t [FW-1:0][NPS-1:0] fs_csr_wr; + `ifdef VX_CFG_EXT_RTU_ENABLE + wire [FW-1:0][NPS-1:0] fs_rtu_unlock_valid; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid; + `endif + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_issued; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_committed; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty; + + VX_frontend_xbar frontend_xbar ( + .fs_decode_if (fs_decode_if), + .is_decode_if (is_decode_if), + .issue_valid (issue_valid), + .issue_wis (issue_wis), + .committed_warps (committed_warps), + .fs_issued (fs_issued), + .fs_committed (fs_committed), + .fs_pending_alm_empty (fs_pending_alm_empty), + .branch_ctl_if (branch_ctl_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if), `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - .block_tex_bus_if (slice_tex_bus_if), - .tex_bus_if (tex_bus_if), + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + .lsu_sched_drained (lsu_drained), + .csr_rd_ctx (csr_rd_ctx), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp), + .satp_wr_valid (satp_wr_valid), + .satp_wr_data (satp_wr_data), `endif + .fs_branch (fs_branch), + .fs_wctl (fs_wctl), + .fs_dvstack_wid (fs_dvstack_wid), + .fs_dvstack_ptr (fs_dvstack_ptr), + .fs_dvstack_mscratch (fs_dvstack_mscratch), + .fs_csr_rd_wid (fs_csr_rd_wid), + .fs_csr_rd (fs_csr_rd), `ifdef VX_CFG_EXT_RTU_ENABLE - .block_rtu_bus_if (slice_rtu_bus_if), - .rtu_bus_if (rtu_bus_if), + .fs_rtu_unlock_valid (fs_rtu_unlock_valid), + .fs_rtu_unlock_wid (fs_rtu_unlock_wid), `endif - .clk (clk), - .reset (reset) + .fs_csr_wr (fs_csr_wr) ); - `ifdef VX_CFG_EXT_TCU_ENABLE - VX_tcu_shared #( - .INSTANCE_ID (`SFORMATF(("%s-execute-tcu", INSTANCE_ID))), - .BLOCK_SIZE (`VX_CFG_ISSUE_WIDTH) - ) tcu_shared ( - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif - `ifdef VX_CFG_TCU_WGMMA_ENABLE - .exec_valid (tcu_exec_valid), - .exec_ready (tcu_exec_ready), - .exec_data (tcu_exec_data), - .tbuf_rs1_data (tcu_tbuf_rs1_data), - .tbuf_rs2_data (tcu_tbuf_rs2_data), - .tbuf_ready_eff (tcu_tbuf_ready), - .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (tcu_mem_if), - `endif - .clk (clk), - .reset (reset) + `ifdef SCOPE + wire [FW-1:0] frontend_scope_bus_in, frontend_scope_bus_out; + VX_scope_switch #( + .N (FW) + ) frontend_scope_switch ( + .clk (clk), + .reset (scope_reset_w[0]), + .req_in (scope_bus_in_w[0]), + .rsp_out (scope_bus_out_w[0]), + .req_out (frontend_scope_bus_in), + .rsp_in (frontend_scope_bus_out) ); `endif - end else begin : g_stage_sliced + for (genvar f = 0; f < FW; ++f) begin : g_frontend_slices + VX_frontend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR ((FW > 1) ? `SFORMATF(("%0d", f)) : "") + ) frontend_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[0]), + .scope_bus_in (frontend_scope_bus_in[f]), + .scope_bus_out (frontend_scope_bus_out[f]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (FETCH_FSW_W'(f)), + `ifdef PERF_ENABLE + .fetch_perf (per_slice_fetch_perf[f]), + `endif + .slice_sched_if (slice_sched_if[f]), + .icache_bus_if (fetch_icache_if[f]), + .decode_if (fs_decode_if[f]), + .branch (fs_branch[f]), + .wctl (fs_wctl[f]), + .dvstack_wid (fs_dvstack_wid[f]), + .dvstack_ptr (fs_dvstack_ptr[f]), + .dvstack_mscratch (fs_dvstack_mscratch[f]), + .csr_rd_wid (fs_csr_rd_wid[f]), + .csr_rd (fs_csr_rd[f]), + .csr_wr (fs_csr_wr[f]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (fs_rtu_unlock_valid[f]), + .rtu_unlock_wid (fs_rtu_unlock_wid[f]), + `endif + .issued (fs_issued[f]), + .committed (fs_committed[f]), + .pending_alm_empty (fs_pending_alm_empty[f]) + ); + end - VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); - VX_commit_sched_if commit_sched_if(); + if (BACKEND_SLICES) begin : g_backend_slices - VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); + `ifdef PERF_ENABLE + issue_perf_t per_slice_issue_perf [`VX_CFG_ISSUE_WIDTH]; + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, ibf_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, scb_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, opd_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + for (genvar i = 0; i < NUM_EX_UNITS; ++i) begin : g_issue_perf_units_uses + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_stalls[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_instrs[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + end + `endif - VX_issue #( - .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) - ) issue ( - `SCOPE_IO_BIND (1) + `ifdef SCOPE + wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_scope_bus_in, slice_scope_bus_out; + VX_scope_switch #( + .N (`VX_CFG_ISSUE_WIDTH) + ) slice_scope_switch ( + .clk (clk), + .reset (scope_reset_w[1]), + .req_in (scope_bus_in_w[1]), + .rsp_out (scope_bus_out_w[1]), + .req_out (slice_scope_bus_in), + .rsp_in (slice_scope_bus_out) + ); + `endif + `SCOPE_IO_UNUSED (2) + + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices + wire [PER_ISSUE_WARPS-1:0] committed_wis; + + VX_backend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR (`SFORMATF(("%0d", i))) + ) backend_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[1]), + .scope_bus_in (slice_scope_bus_in[i]), + .scope_bus_out (slice_scope_bus_out[i]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (ISSUE_ISW_W'(i)), + `ifdef PERF_ENABLE + .issue_perf (per_slice_issue_perf[i]), + `endif + .decode_if (is_decode_if[i]), + .warp_issued (issue_valid[i]), + .warp_issued_wis (issue_wis[i]), + .committed_wis (committed_wis), + .branch_ctl_if (branch_ctl_if[i]), + .warp_ctl_if (warp_ctl_if[i]), + .sched_csr_if (sched_csr_if[i]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if[i]), + `endif + .lsu_client_if (lsu_client_if[i]), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (block_dxa_req_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (block_tex_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (block_rtu_bus_if[i]), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid[i]), + .tcu_exec_ready (tcu_exec_ready[i]), + .tcu_exec_data (tcu_exec_data[i]), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), + .tcu_tbuf_ready (tcu_tbuf_ready[i]), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid[i]), + .tcu_agu_ld_data (tcu_agu_ld_data[i]), + .tcu_agu_ld_ready (tcu_agu_ld_ready[i]), + .tcu_agu_result_valid (tcu_agu_result_valid[i]), + .tcu_agu_result_data (tcu_agu_result_data[i]), + .tcu_agu_result_ready (tcu_agu_result_ready[i]), + .tcu_meta_wr_en (tcu_meta_wr_en && (NW_WIDTH'(tcu_meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(i))), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + .warp_shared_if (warp_shared_if[i]) + ); + + // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_committed + assign committed_warps[w * `VX_CFG_ISSUE_WIDTH + i] = committed_wis[w]; + end + end - .clk (clk), - .reset (reset), + end else begin : g_stage_sliced - `ifdef PERF_ENABLE - .issue_perf (pipeline_perf.issue), - `endif + VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); + VX_commit_sched_if commit_sched_if(); - .decode_if (is_decode_if), - .writeback_if (writeback_if), - .dispatch_if (dispatch_if), - .issue_sched_if (issue_sched_if) - ); + VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_issue_sched - assign issue_valid[i] = issue_sched_if[i].valid; - assign issue_wis[i] = issue_sched_if[i].wis; - end + VX_issue #( + .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) + ) issue ( + `SCOPE_IO_BIND (1) - VX_execute #( - .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))) - ) execute ( - `SCOPE_IO_BIND (2) + .clk (clk), + .reset (reset), - .clk (clk), - .reset (reset), - .core_id (core_id), + `ifdef PERF_ENABLE + .issue_perf (pipeline_perf.issue), + `endif - `ifdef PERF_ENABLE - `ifdef VX_CFG_EXT_TCU_ENABLE - .tcu_perf (tcu_perf), - `endif - `endif + .decode_if (is_decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .issue_sched_if (issue_sched_if) + ); - // execute exposes LSU client interfaces; lsu_scheduler sits between - // execute and lsu_mem_if (which connects to mem_unit). - .lsu_client_if (lsu_client_if), + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_issue_sched + assign issue_valid[i] = issue_sched_if[i].valid; + assign issue_wis[i] = issue_sched_if[i].wis; + end - `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), - `endif + VX_execute #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))) + ) execute ( + `SCOPE_IO_BIND (2) - .dispatch_if (dispatch_if), - .commit_if (commit_if), + .clk (clk), + .reset (reset), + .core_id (core_id), - .sched_csr_if (sched_csr_if), + // execute exposes LSU client interfaces; lsu_scheduler sits between + // execute and lsu_mem_if (which connects to mem_unit). + .lsu_client_if (lsu_client_if), - `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_req_bus_if (dxa_req_bus_if), - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - .tex_bus_if (tex_bus_if), - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_bus_if (rtu_bus_if), - .sched_unlock_if(sched_unlock_if), - `endif + .dispatch_if (dispatch_if), + .commit_if (commit_if), - .warp_shared_if (warp_shared_if), + .sched_csr_if (sched_csr_if), - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if) - ); + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid), + .tcu_exec_ready (tcu_exec_ready), + .tcu_exec_data (tcu_exec_data), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), + .tcu_tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid), + .tcu_agu_ld_data (tcu_agu_ld_data), + .tcu_agu_ld_ready (tcu_agu_ld_ready), + .tcu_agu_result_valid (tcu_agu_result_valid), + .tcu_agu_result_data (tcu_agu_result_data), + .tcu_agu_result_ready (tcu_agu_result_ready), + .tcu_meta_wr_en (tcu_meta_wr_en), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (block_dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (block_tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (block_rtu_bus_if), + .sched_unlock_if (sched_unlock_if), + `endif - VX_commit #( - .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) - ) commit ( - .clk (clk), - .reset (reset), + .warp_shared_if (warp_shared_if), - .commit_if (commit_if), + .warp_ctl_if (warp_ctl_if), + .branch_ctl_if (branch_ctl_if) + ); - .writeback_if (writeback_if), + VX_commit #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit ( + .clk (clk), + .reset (reset), - .commit_sched_if(commit_sched_if) - ); + .commit_if (commit_if), - assign committed_warps = commit_sched_if.committed_warps; + .writeback_if (writeback_if), + .commit_sched_if(commit_sched_if) + ); + + assign committed_warps = commit_sched_if.committed_warps; + + end end // One memory scheduler per LSU block. With the TCU metadata AGU, every block diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 7454c4d92e..94e33c191a 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -13,7 +13,11 @@ `include "VX_define.vh" -module VX_execute import VX_gpu_pkg::*; #( +module VX_execute import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -23,35 +27,24 @@ module VX_execute import VX_gpu_pkg::*; #( input wire [CORE_ID_WIDTH-1:0] core_id, -`ifdef PERF_ENABLE -`ifdef VX_CFG_EXT_TCU_ENABLE - output tcu_perf_t tcu_perf, -`endif -`endif - // Per-block LSU client connections to VX_lsu_scheduler (at VX_core). // The dcache port (lsu_mem_if) lives at VX_core; execute exposes // only the LSU client interfaces. VX_lsu_sched_if.master lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS], -`ifdef TCU_META_ENABLE - // TCU AGU memory client (single warp-level AGU shared across blocks). - // VX_core wires this to client 1 of block 0's lsu_scheduler. - VX_lsu_sched_if.master tcu_mem_if, -`endif - // dispatch interface VX_dispatch_if.slave dispatch_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], // commit interface VX_commit_if.master commit_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], + // per-block ports of the core-wide SFU and TCU services `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if.master dxa_req_bus_if, + VX_dxa_req_bus_if.master dxa_req_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if.master tex_bus_if, + VX_tex_bus_if.master tex_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_OM_ENABLE @@ -62,13 +55,29 @@ module VX_execute import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if.master rtu_bus_if, + VX_rtu_bus_if.master rtu_bus_if [`VX_CFG_NUM_SFU_BLOCKS], VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE - // TCU tile-buffer local-memory read port - VX_mem_bus_if.master tcu_lmem_if, + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_ready, + output tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready, +`endif +`ifdef TCU_META_ENABLE + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, + output tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_ready, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, + input tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS], + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_ready, + input wire tcu_meta_wr_en, + input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, + input wire [4:0] tcu_meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, `endif // scheduler interfaces @@ -126,19 +135,30 @@ module VX_execute import VX_gpu_pkg::*; #( .INSTANCE_ID (`SFORMATF(("%s-tcu", INSTANCE_ID))) ) tcu_unit ( `SCOPE_IO_BIND (1) - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif + .clk (clk), + .reset (reset), `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), `endif - .dispatch_if (dispatch_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), - .commit_if (commit_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) + .dispatch_if (dispatch_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), + .commit_if (commit_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) ); `endif diff --git a/hw/rtl/core/VX_fetch_slice.sv b/hw/rtl/core/VX_frontend_slice.sv similarity index 97% rename from hw/rtl/core/VX_fetch_slice.sv rename to hw/rtl/core/VX_frontend_slice.sv index 41d788c064..9bc2eed602 100644 --- a/hw/rtl/core/VX_fetch_slice.sv +++ b/hw/rtl/core/VX_frontend_slice.sv @@ -13,11 +13,11 @@ `include "VX_define.vh" -// One fetch slice: warp scheduling, fetch and decode of the warps with +// One frontend slice: warp scheduling, fetch and decode of the warps with // wid mod FETCH_WIDTH == slice_id, on the slice's own icache port. The front end // names the slice's warps by wfs; decoded instructions leave with their // core-wide wid, and ibuffer pops return indexed by wfs. -module VX_fetch_slice import VX_gpu_pkg::*; #( +module VX_frontend_slice import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter `STRING SLICE_STR = "" ) ( diff --git a/hw/rtl/core/VX_fetch_xbar.sv b/hw/rtl/core/VX_frontend_xbar.sv similarity index 95% rename from hw/rtl/core/VX_fetch_xbar.sv rename to hw/rtl/core/VX_frontend_xbar.sv index 85c7babd07..8f4d7cf3c7 100644 --- a/hw/rtl/core/VX_fetch_xbar.sv +++ b/hw/rtl/core/VX_frontend_xbar.sv @@ -13,12 +13,12 @@ `include "VX_define.vh" -// Connects the fetch slices (fetch slice f owns wid mod FETCH_WIDTH == f) to the +// Connects the frontend slices (frontend slice f owns wid mod FETCH_WIDTH == f) to the // issue slices (wid mod ISSUE_WIDTH) and the ALU/SFU blocks (wid mod B). Issue -// slice i takes its instructions from fetch slice i mod FETCH_WIDTH. A block -// serving several fetch slices sends each its own warps' events and answers its +// slice i takes its instructions from frontend slice i mod FETCH_WIDTH. A block +// serving several frontend slices sends each its own warps' events and answers its // reads from the slice owning the read wid. Purely combinational. -module VX_fetch_xbar import VX_gpu_pkg::*; ( +module VX_frontend_xbar import VX_gpu_pkg::*; ( // instruction path VX_decode_if.slave fs_decode_if [`VX_CFG_FETCH_WIDTH], VX_decode_if.master is_decode_if [`VX_CFG_ISSUE_WIDTH], @@ -51,7 +51,7 @@ module VX_fetch_xbar import VX_gpu_pkg::*; ( output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data, `endif - // fetch slices + // frontend slices output sched_branch_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_ALU_PORTS-1:0] fs_branch, output sched_wctl_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_wctl, output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_dvstack_wid, @@ -89,7 +89,7 @@ module VX_fetch_xbar import VX_gpu_pkg::*; ( end end - for (genvar f = 0; f < FW; ++f) begin : g_fetch_slices + for (genvar f = 0; f < FW; ++f) begin : g_frontend_slices assign fs_decode_if[f].ready = is_ready[wid_to_isw(fs_decode_if[f].data.wid)]; for (genvar l = 0; l < SW; ++l) begin : g_ibuf_pop localparam W = l * FW + f; @@ -111,7 +111,7 @@ module VX_fetch_xbar import VX_gpu_pkg::*; ( assign pending_alm_empty[w] = fs_pending_alm_empty[w % FW][w / FW]; end - // ALU blocks: fetch slice f port q is block q * FW + f, or block f mod BA when shared + // ALU blocks: frontend slice f port q is block q * FW + f, or block f mod BA when shared for (genvar f = 0; f < FW; ++f) begin : g_alu_slices for (genvar q = 0; q < NPA; ++q) begin : g_ports diff --git a/hw/rtl/core/VX_scheduler.sv b/hw/rtl/core/VX_scheduler.sv index c2d36fda22..246274c7ca 100644 --- a/hw/rtl/core/VX_scheduler.sv +++ b/hw/rtl/core/VX_scheduler.sv @@ -14,7 +14,7 @@ `include "VX_define.vh" // The core-level part of scheduling: CTA dispatch and the core-wide warp-control -// resources. Each fetch slice schedules and fetches its own warps (VX_warp_sched, +// resources. Each frontend slice schedules and fetches its own warps (VX_warp_sched, // VX_fetch); core-wide events are handed to it with slice-local warp ids. module VX_scheduler import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "" @@ -70,7 +70,7 @@ module VX_scheduler import VX_gpu_pkg::*; #( assign pending_warp_empty[w] = slice_if[w % FW].pending_empty[w / FW]; end - // CTA dispatcher: lane p launches fetch slice p's warps + // CTA dispatcher: lane p launches frontend slice p's warps wire [FW-1:0] cta_fire; wire [FW-1:0][NW_WIDTH-1:0] cta_wid; wire [PC_BITS-1:0] cta_PC; diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index ecbbf46146..a7f14fe32d 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -33,15 +33,15 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if.master dxa_req_bus_if, + VX_dxa_req_bus_if.master dxa_req_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if.master tex_bus_if, + VX_tex_bus_if.master tex_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if.master rtu_bus_if, + VX_rtu_bus_if.master rtu_bus_if [`VX_CFG_NUM_SFU_BLOCKS], VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif @@ -82,22 +82,6 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( wire [BLOCK_SIZE-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data; `endif -`ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if block_dxa_req_bus_if[BLOCK_SIZE](); -`endif -`ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH) - ) block_tex_bus_if[BLOCK_SIZE](); -`endif -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH) - ) block_rtu_bus_if[BLOCK_SIZE](); -`endif - for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks VX_sfu_block #( .INSTANCE_ID ((BLOCK_SIZE > 1) ? `SFORMATF(("%s%0d", INSTANCE_ID, block_idx)) : INSTANCE_ID) @@ -115,13 +99,13 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( .fcsr_wr_data (fcsr_wr_data[block_idx]), `endif `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_req_bus_if (block_dxa_req_bus_if[block_idx]), + .dxa_req_bus_if (dxa_req_bus_if[block_idx]), `endif `ifdef VX_CFG_EXT_TEX_ENABLE - .tex_bus_if (block_tex_bus_if[block_idx]), + .tex_bus_if (tex_bus_if[block_idx]), `endif `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_bus_if (block_rtu_bus_if[block_idx]), + .rtu_bus_if (rtu_bus_if[block_idx]), .sched_unlock_if(sched_unlock_if[block_idx]), `endif .sched_csr_if (sched_csr_if[block_idx]), @@ -145,25 +129,6 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( ); `endif - VX_sfu_shared #( - .INSTANCE_ID (`SFORMATF(("%s-shared", INSTANCE_ID))) - ) sfu_shared ( - `ifdef VX_CFG_EXT_DXA_ENABLE - .block_dxa_req_bus_if (block_dxa_req_bus_if), - .dxa_req_bus_if (dxa_req_bus_if), - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - .block_tex_bus_if (block_tex_bus_if), - .tex_bus_if (tex_bus_if), - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - .block_rtu_bus_if (block_rtu_bus_if), - .rtu_bus_if (rtu_bus_if), - `endif - .clk (clk), - .reset (reset) - ); - VX_lane_gather #( .BLOCK_SIZE (BLOCK_SIZE), .NUM_LANES (NUM_LANES), diff --git a/hw/rtl/core/VX_uuid_gen.sv b/hw/rtl/core/VX_uuid_gen.sv index 34ea3aab1d..5e2da82927 100644 --- a/hw/rtl/core/VX_uuid_gen.sv +++ b/hw/rtl/core/VX_uuid_gen.sv @@ -14,7 +14,7 @@ `include "VX_define.vh" // Per-warp instruction ids. NUM_WARPS is the number of warps counted here: all -// of the core's, or the PER_FETCH_WARPS of one fetch slice (wid = {wfs, slice}). +// of the core's, or the PER_FETCH_WARPS of one frontend slice (wid = {wfs, slice}). module VX_uuid_gen import VX_gpu_pkg::*; #( parameter NUM_WARPS = `VX_CFG_NUM_WARPS ) ( diff --git a/hw/rtl/core/VX_warp_sched.sv b/hw/rtl/core/VX_warp_sched.sv index eeeb663bc4..70cdac2baa 100644 --- a/hw/rtl/core/VX_warp_sched.sv +++ b/hw/rtl/core/VX_warp_sched.sv @@ -13,7 +13,7 @@ `include "VX_define.vh" -// The warp scheduler of one fetch slice: the state of the slice's warps +// The warp scheduler of one frontend slice: the state of the slice's warps // (wid = {wfs, slice_id}), its warp selection and a registered fetch request to // the slice's fetch unit, which names warps by wfs. FU block port q carries the // traffic of the local warps with wfs mod PORTS == q, addressed by core-wide wid; diff --git a/hw/rtl/interfaces/VX_slice_sched_if.sv b/hw/rtl/interfaces/VX_slice_sched_if.sv index c5e056ab43..4f66b97f24 100644 --- a/hw/rtl/interfaces/VX_slice_sched_if.sv +++ b/hw/rtl/interfaces/VX_slice_sched_if.sv @@ -13,7 +13,7 @@ `include "VX_define.vh" -// Core scheduler state exchanged with one fetch slice's warp scheduler. +// Core scheduler state exchanged with one frontend slice's warp scheduler. // Core-wide events arrive already addressed to the slice: warp ids are // slice-local (wid = {wfs, fsw}) and per-warp masks hold only its warps. interface VX_slice_sched_if import VX_gpu_pkg::*; (); diff --git a/hw/rtl/tcu/VX_tcu_bbuf.sv b/hw/rtl/tcu/VX_tcu_bbuf.sv index 11a4624a26..e87fb7e9c6 100644 --- a/hw/rtl/tcu/VX_tcu_bbuf.sv +++ b/hw/rtl/tcu/VX_tcu_bbuf.sv @@ -18,7 +18,7 @@ // // TB-shared B buffer (block-major SMEM, 1 bank-row storage). // -// Single instance per VX_tcu_unit. Holds the bank-row of B that contains +// Single instance per core (in VX_tcu_shared). Holds the bank-row of B that contains // the current (step_k, step_n) block. All Q tcu_cores read from the // same buffer (structural fan-out, not arbitrated). // diff --git a/hw/rtl/tcu/VX_tcu_unit.sv b/hw/rtl/tcu/VX_tcu_unit.sv index fe0e4838e7..91dd5fd292 100644 --- a/hw/rtl/tcu/VX_tcu_unit.sv +++ b/hw/rtl/tcu/VX_tcu_unit.sv @@ -21,18 +21,28 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire clk, input wire reset, -`ifdef PERF_ENABLE - output tcu_perf_t tcu_perf, -`endif - `ifdef VX_CFG_TCU_WGMMA_ENABLE - // Bank-parallel LMEM read port - VX_mem_bus_if.master tcu_lmem_if, + // per-block ports of the core-wide WGMMA tile buffers (VX_tcu_shared) + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] exec_valid, + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] exec_ready, + output tcu_execute_t exec_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tbuf_ready, `endif `ifdef TCU_META_ENABLE - // TCU_LD memory client connection to VX_lsu_scheduler at VX_core. - VX_lsu_sched_if.master tcu_mem_if, + // per-block ports of the core-wide metadata AGU (VX_tcu_shared) + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_ld_valid, + output tcu_execute_t agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_ld_ready, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_result_valid, + input tcu_result_t agu_result_data [`VX_CFG_NUM_TCU_BLOCKS], + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_result_ready, + input wire meta_wr_en, + input wire [NW_WIDTH-1:0] meta_wr_wid, + input wire [4:0] meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, `endif // Inputs @@ -68,25 +78,6 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .data_t (tcu_result_t) ) per_block_result_if[BLOCK_SIZE](); -`ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [BLOCK_SIZE-1:0] exec_valid, exec_ready; - tcu_execute_t exec_data [BLOCK_SIZE]; - wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data; - wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data; - wire [BLOCK_SIZE-1:0] tbuf_ready_eff; -`endif - -`ifdef TCU_META_ENABLE - wire [BLOCK_SIZE-1:0] agu_ld_valid, agu_ld_ready; - tcu_execute_t agu_ld_data [BLOCK_SIZE]; - wire [BLOCK_SIZE-1:0] agu_result_valid, agu_result_ready; - tcu_result_t agu_result_data [BLOCK_SIZE]; - wire meta_wr_en; - wire [NW_WIDTH-1:0] meta_wr_wid; - wire [4:0] meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data; -`endif - for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks `ifdef TCU_META_ENABLE wire block_meta_wr_en = meta_wr_en @@ -106,7 +97,7 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .exec_data (exec_data[block_idx]), .tbuf_rs1_data (tbuf_rs1_data[block_idx]), .tbuf_rs2_data (tbuf_rs2_data[block_idx]), - .tbuf_ready (tbuf_ready_eff[block_idx]), + .tbuf_ready (tbuf_ready[block_idx]), `endif `ifdef TCU_META_ENABLE .agu_ld_valid (agu_ld_valid[block_idx]), @@ -125,39 +116,6 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ); end - VX_tcu_shared #( - .INSTANCE_ID (INSTANCE_ID), - .BLOCK_SIZE (BLOCK_SIZE) - ) tcu_shared ( - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif - `ifdef VX_CFG_TCU_WGMMA_ENABLE - .exec_valid (exec_valid), - .exec_ready (exec_ready), - .exec_data (exec_data), - .tbuf_rs1_data (tbuf_rs1_data), - .tbuf_rs2_data (tbuf_rs2_data), - .tbuf_ready_eff (tbuf_ready_eff), - .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (agu_ld_valid), - .agu_ld_data (agu_ld_data), - .agu_ld_ready (agu_ld_ready), - .agu_result_valid (agu_result_valid), - .agu_result_data (agu_result_data), - .agu_result_ready (agu_result_ready), - .meta_wr_en (meta_wr_en), - .meta_wr_wid (meta_wr_wid), - .meta_wr_idx (meta_wr_idx), - .meta_wr_data (meta_wr_data), - .tcu_mem_if (tcu_mem_if), - `endif - .clk (clk), - .reset (reset) - ); - // ----------------------------------------------------------------------- // Lane gather // ----------------------------------------------------------------------- diff --git a/hw/rtl/tcu/VX_tcu_wgmma.sv b/hw/rtl/tcu/VX_tcu_wgmma.sv index 671a1a8d61..4df2e33579 100644 --- a/hw/rtl/tcu/VX_tcu_wgmma.sv +++ b/hw/rtl/tcu/VX_tcu_wgmma.sv @@ -20,8 +20,8 @@ // CTA lockstep gate, and WGMMA-specific perf counters. // // Observation-only on the dispatch path: this module does not drive -// .ready. The wrapper (VX_tcu_unit) feeds tbuf_ready_eff into tcu_core, -// which produces the actual handshake .ready going back to dispatch. +// .ready. VX_tcu_shared exports tbuf_ready_eff per block; the block's +// tcu_core produces the actual handshake .ready going back to dispatch. module VX_tcu_wgmma import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", diff --git a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv index 192fb19b5a..637ba9bd9a 100644 --- a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv +++ b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv @@ -13,7 +13,7 @@ `include "VX_define.vh" -// Flat-port testbench wrapper around VX_tcu_unit. +// Flat-port testbench wrapper around VX_tcu_unit and VX_tcu_shared. // All VX_dispatch_if and VX_commit_if signals are exposed as individual // ports so that the Verilator C++ testbench can drive them directly. // Only ISSUE_WIDTH == 1 is supported by this wrapper. @@ -164,20 +164,83 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( `UNUSED_VAR (commit_if[0].data.cta_id) // ---- DUT ------------------------------------------------------------- +`ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; +`endif +`ifdef TCU_META_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; + tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; + tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire tcu_meta_wr_en; + wire [NW_WIDTH-1:0] tcu_meta_wr_wid; + wire [4:0] tcu_meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; +`endif + VX_tcu_unit #( .INSTANCE_ID ("tcu_unit_top") ) tcu_unit ( `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + `endif + .dispatch_if (dispatch_if), + .commit_if (commit_if) + ); + + // the core-wide TCU state the blocks share: WGMMA tile buffers, metadata AGU + VX_tcu_shared #( + .INSTANCE_ID ("tcu_unit_top"), + .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) + ) tcu_shared ( `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), `endif `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), `endif - .dispatch_if (dispatch_if), - .commit_if (commit_if) + .clk (clk), + .reset (reset) ); endmodule diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index abd8b0b526..e56e5c14b6 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -20,6 +20,7 @@ // // per core (VX_CFG_SOCKET_SIZE): // VX_tcu_unit : tensor-core unit (dispatch/commit from top ports) +// VX_tcu_shared : WGMMA tile buffers + metadata AGU // VX_lsu_scheduler : per-LSU-block, 2 clients — the ordinary LSU stream // (top ports) and the TCU AGU metadata-load port // VX_mem_unit : LMEM + coalescer + LMEM-DMA arb (TCU WGMMA + DXA @@ -175,20 +176,83 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( VX_lsu_sched_if tcu_mem_if(); `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; + `endif + `ifdef TCU_META_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; + tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; + tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire tcu_meta_wr_en; + wire [NW_WIDTH-1:0] tcu_meta_wr_wid; + wire [4:0] tcu_meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; + `endif + VX_tcu_unit #( .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))) ) tcu_unit ( `SCOPE_IO_BIND (c) - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + `endif + .dispatch_if (dispatch_if), + .commit_if (commit_if) + ); + + // the core-wide TCU state the blocks share: WGMMA tile buffers, metadata AGU + VX_tcu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))), + .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) + ) tcu_shared ( `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), `endif `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), `endif - .dispatch_if (dispatch_if), - .commit_if (commit_if) + .clk (clk), + .reset (reset) ); // --- LSU schedulers: client 0 from top ports, client 1 = TCU AGU --- diff --git a/sim/simx/constants.h b/sim/simx/constants.h index da7a62eaa3..64a61c14d2 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -105,7 +105,7 @@ inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_ inline constexpr uint32_t PER_ISSUE_WARPS = VX_CFG_NUM_WARPS / VX_CFG_ISSUE_WIDTH; inline constexpr uint32_t ISSUE_WIS_BITS = log2ceil(PER_ISSUE_WARPS); -// Fetch slice f owns the warps with wid % VX_CFG_FETCH_WIDTH == f; each has its +// Frontend slice f owns the warps with wid % VX_CFG_FETCH_WIDTH == f; each has its // own warp scheduler, fetch unit, decoder and icache request port. inline constexpr uint32_t PER_FETCH_WARPS = VX_CFG_NUM_WARPS / VX_CFG_FETCH_WIDTH; inline constexpr uint32_t ICACHE_NUM_REQS = VX_CFG_FETCH_WIDTH; diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index a6fcfdff90..c81fc2709b 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -79,7 +79,7 @@ class Core::Impl { const std::string& name = simobject_->name(); char sname[100]; - // per fetch slice: scheduler-to-fetch and fetch-to-decode latches, and the + // per frontend slice: scheduler-to-fetch and fetch-to-decode latches, and the // icache tags of the slice's in-flight fetches for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { snprintf(sname, 100, "fetch_latch%d", f); @@ -219,7 +219,7 @@ class Core::Impl { dcache_mmu_ = Mmu::Create(sname, VX_CFG_DCACHE_NUM_REQS, VX_CFG_DTLB_SIZE, local_core * 2 + 0, false); - // Per-core icache MMU, one port per fetch slice. Fetch reads/writes its + // Per-core icache MMU, one port per frontend slice. Fetch reads/writes its // upstream channels directly; the downstream side is bound to the Core's // icache ports below. snprintf(sname, 100, "%s-icache_mmu", name.c_str()); @@ -376,7 +376,7 @@ class Core::Impl { scheduler_->schedule_begin(); - // Each fetch slice picks among its own warps whose ibuffer is not full; a + // Each frontend slice picks among its own warps whose ibuffer is not full; a // buffer is full at exactly IBUF_SIZE entries. With an L1, once every warp's // buffer is full any ready warp may fetch, its instruction waiting at decode. bool all_full = (VX_CFG_ICACHE_ENABLED || VX_CFG_DCACHE_ENABLED); @@ -426,7 +426,7 @@ class Core::Impl { } } - // Fetch slice f: its icache response into its decode latch, then its next + // Frontend slice f: its icache response into its decode latch, then its next // fetch request out of its fetch latch, on icache port f. void fetch_slice(uint32_t f) { auto& fetch_latch = *fetch_latch_.at(f); @@ -541,7 +541,7 @@ class Core::Impl { } } - // One fetch slice's decoder: it feeds the issue slice of each decoded warp. + // One frontend slice's decoder: it feeds the issue slice of each decoded warp. void decode_slice(TFifo& decode_latch) { if (decode_latch.empty()) { return; @@ -1202,7 +1202,7 @@ class Core::Impl { PoolAllocator trace_pool_; #ifdef VX_CFG_EXT_C_ENABLE - // RVC fetch SimObject per fetch slice (per-warp halfword buffer + refetch + // RVC fetch SimObject per frontend slice (per-warp halfword buffer + refetch // queue + pick/commit_request helpers). See decompressor.h. std::vector decompressors_; #endif diff --git a/sim/simx/scheduler.h b/sim/simx/scheduler.h index 58f10378b0..7a3fd6dee2 100644 --- a/sim/simx/scheduler.h +++ b/sim/simx/scheduler.h @@ -118,11 +118,11 @@ class Scheduler : public SimObject { ~Scheduler(); // ----- Warp lifecycle ----- - // Once per cycle, before the fetch slices pick: activate the warps the CTA + // Once per cycle, before the frontend slices pick: activate the warps the CTA // dispatcher launched last cycle, launch the next ones, and apply a wspawn. void schedule_begin(); - // One fetch slice's pick: its lowest ready warp in warp_mask. + // One frontend slice's pick: its lowest ready warp in warp_mask. instr_trace_t* schedule(const WarpMask& warp_mask); // Whether warp_mask holds a ready warp, without picking it. diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 83eac594da..c919ce378f 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -62,7 +62,7 @@ class Socket::Impl { log2ceil(VX_CFG_ICACHE_NUM_WAYS),// A log2ceil(VX_CFG_ICACHE_NUM_BANKS), // B VX_CFG_XLEN, // address bits - ICACHE_NUM_REQS, // number of inputs (one per fetch slice) + ICACHE_NUM_REQS, // number of inputs (one per frontend slice) VX_CFG_ICACHE_MEM_PORTS, // memory ports false, // write-back false, // write response From e849c567cf580ba238172995c8c1a6b938b7b976 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:14 -0700 Subject: [PATCH 040/336] simx: register the cache core-response port as the RTL does The RTL caches register their core-response port (CORE_OUT_BUF) and the LSU-side coalescer and lane adapter answer combinationally, while SimX put the register in the LSU adapters and the memory coalescer instead. The total load latency matched, but under contention the models stalled at different points. The cache config gains core_out_buf. The dcache, L2 and L3 set it, and a cache response is delayed by that register plus the non-cacheable bypass layer's response register. The memory coalescer and the LSU lane adapter now send responses with no delay, and both tick ahead of the local-memory switch. Co-Authored-By: Claude Opus 5.5 --- sim/simx/cluster.cpp | 1 + sim/simx/core.cpp | 15 ++++++++++----- sim/simx/mem/cache.cpp | 6 ++++-- sim/simx/mem/cache.h | 1 + sim/simx/mem/lsu_mem_adapter.cpp | 6 ++++-- sim/simx/mem/lsu_mem_adapter.h | 4 +++- sim/simx/mem/mem_coalescer.cpp | 4 ++-- sim/simx/processor.cpp | 1 + sim/simx/socket.cpp | 1 + 9 files changed, 27 insertions(+), 12 deletions(-) diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 8674f73591..88ab5dbb20 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -88,6 +88,7 @@ class Cluster::Impl { (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash L2_SHARED_LLC, // shared_llc VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) }); diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index c81fc2709b..558bf54a6e 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -128,10 +128,18 @@ class Core::Impl { ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE); } - // create the memory coalescer + // create the memory coalescer, or the lane adapter that replaces it. + // Both tick ahead of the lmem switch: their response paths are + // combinational, and the dcache registers its output. + const bool dcache_coalescing = (VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE); + std::vector lsu_dcache_adapter(VX_CFG_NUM_LSU_BLOCKS); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { snprintf(sname, 100, "%s-coalescer%d", name.c_str(), b); mem_coalescers_.at(b) = MemCoalescer::Create(sname, LSU_CHANNELS, DCACHE_CHANNELS, DCACHE_WORD_SIZE, LSU_QUEUE_OUT_SIZE, 1); + if (!dcache_coalescing) { + snprintf(sname, 100, "%s-lsu_dcache_adapter%d", name.c_str(), b); + lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1, 0); + } } // create local memory. @@ -180,7 +188,7 @@ class Core::Impl { // The first data-cache port also carries the cache-flush injection, which // registers its requests once more. - if ((VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE)) { + if (dcache_coalescing) { // connect memory coalescer; its memory side drives the dcache // channels directly (combinational lane fan-out/fan-in). for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { @@ -195,10 +203,7 @@ class Core::Impl { } else { // bypass memory coalescer: per-block lane adapter (channel-fused // pass-through when DCACHE_CHANNELS == 1) - std::vector lsu_dcache_adapter(VX_CFG_NUM_LSU_BLOCKS); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - snprintf(sname, 100, "%s-lsu_dcache_adapter%d", name.c_str(), b); - lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1); lmem_switch_.at(b)->ReqOutDC.bind(&lsu_dcache_adapter.at(b)->ReqIn); lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); for (uint32_t c = 0; c < DCACHE_CHANNELS; ++c) { diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index f7c9c82bc0..ecd6b16448 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2243,8 +2243,10 @@ class Cache::Impl { if (bank_rsp.empty()) continue; auto &core_rsp = bank_rsp.peek(); - // the non-cacheable bypass layer's response arbiter registers it once more - if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, config_.nc_enable ? 1 : 0)) { + // the core-response port's register, then the non-cacheable bypass + // layer's response arbiter + uint32_t rsp_delay = (config_.core_out_buf ? 1 : 0) + (config_.nc_enable ? 1 : 0); + if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, rsp_delay)) { DT(3, simobject_->name() << " core-rsp: " << core_rsp); bank_rsp.pop(); } diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 40d7890501..6371f75d09 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -47,6 +47,7 @@ class Cache : public SimObject { bool bank_hash = false; // hashed bank select bool shared_llc = false; // the LLC serves every memory client alone bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) + bool core_out_buf = false; // registered core-response port }; struct PerfStats { diff --git a/sim/simx/mem/lsu_mem_adapter.cpp b/sim/simx/mem/lsu_mem_adapter.cpp index 7e4460c1c5..a34aacfe6c 100644 --- a/sim/simx/mem/lsu_mem_adapter.cpp +++ b/sim/simx/mem/lsu_mem_adapter.cpp @@ -19,13 +19,15 @@ LsuMemAdapter::LsuMemAdapter( const SimContext& ctx, const char* name, uint32_t num_inputs, - uint32_t delay + uint32_t delay, + uint32_t rsp_delay ) : SimObject(ctx, name) , ReqIn(this) , RspOut(this) , ReqOut(num_inputs, this) , RspIn(num_inputs, this) , delay_(delay) + , rsp_delay_(rsp_delay) , port_delay_(num_inputs, 0) , pending_mask_(num_inputs) { @@ -106,7 +108,7 @@ void LsuMemAdapter::on_tick() { } // send memory response - RspOut.send(out_rsp, 1); + RspOut.send(out_rsp, rsp_delay_); // remove input DT(4, this->name() << "-rsp" << i << ": " << rsp_in); diff --git a/sim/simx/mem/lsu_mem_adapter.h b/sim/simx/mem/lsu_mem_adapter.h index 0d0e05ebe6..6f6499716f 100644 --- a/sim/simx/mem/lsu_mem_adapter.h +++ b/sim/simx/mem/lsu_mem_adapter.h @@ -31,7 +31,8 @@ class LsuMemAdapter : public SimObject { const SimContext& ctx, const char* name, uint32_t num_inputs, - uint32_t delay + uint32_t delay, + uint32_t rsp_delay = 1 ); LsuMemAdapter( @@ -50,6 +51,7 @@ class LsuMemAdapter : public SimObject { private: uint32_t delay_; + uint32_t rsp_delay_; std::vector port_delay_; BitVector pending_mask_; diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index 533e236b13..51a0b95a12 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -108,8 +108,8 @@ void MemCoalescer::on_tick() { } } - // send memory response - RspOut.send(out_rsp, 1); + // the response path is combinational; the cache registers its output + RspOut.send(out_rsp, 0); DT(4, this->name() << " mem-rsp: " << out_rsp); // track remaining responses diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index b7d31c1705..77163fe4df 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -108,6 +108,7 @@ ProcessorImpl::ProcessorImpl() false, // bank_hash L3_SHARED_LLC, // shared_llc VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) } ); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index c919ce378f..064eaba4e2 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -95,6 +95,7 @@ class Socket::Impl { (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash DCACHE_SHARED_LLC, // shared_llc VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register }); #ifdef VX_CFG_EXT_TEX_ENABLE From 6d7df03f8baf09b63c5103500961fee0477e5c1f Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:15 -0700 Subject: [PATCH 041/336] simx: never launch two warps of one CTA into the same slot The CTA dispatcher picked any warp slot that was not active. A short warp of a CTA could exit before the CTA finished launching, and the dispatcher then launched a later warp of the same CTA into that slot. The RTL never reuses a slot within a CTA. The dispatcher tracks the slots the current CTA has launched into and skips them until the next CTA starts. Co-Authored-By: Claude Opus 5.5 --- sim/simx/cta_dispatcher.cpp | 5 ++++- sim/simx/cta_dispatcher.h | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index 1d0fb6883a..67c82eca72 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -61,6 +61,7 @@ CtaDispatcher::~CtaDispatcher() {} void CtaDispatcher::on_reset() { has_cta_ = false; cta_done_ = false; + cta_wids_.reset(); tail_slot_ = 0; init_wait_ = 0; for (uint32_t i = 0; i < num_warps_; ++i) { @@ -176,6 +177,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w cta_size_ = (cta_.block_size + num_threads_ - 1) / num_threads_; rank_ = 0; block_size_rem_ = cta_.block_size; + cta_wids_.reset(); thread_idx_[0] = thread_idx_[1] = thread_idx_[2] = 0; cur_slot_ = base; @@ -216,7 +218,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w uint32_t lane = (lane_rr_ + o) % VX_CFG_FETCH_WIDTH; int free_wid = -1; for (uint32_t wid = lane; wid < num_warps_; wid += VX_CFG_FETCH_WIDTH) { - if (!active_warps.test(wid)) { + if (!active_warps.test(wid) && !cta_wids_.test(wid)) { free_wid = int(wid); break; } @@ -241,6 +243,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w inited.set(t); } wid_to_slot_[free_wid] = cur_slot_; + cta_wids_.set(free_wid); wids[count] = uint32_t(free_wid); ++count; } diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 7c74930292..314fd1d4ce 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -123,6 +123,9 @@ class CtaDispatcher : public SimObject { uint32_t block_size_rem_; uint32_t thread_idx_[3]; uint64_t lmem_addr_; + // Slots this CTA has launched into: a CTA never reuses one, even after the + // warp it launched there has exited. + WarpMask cta_wids_; // Launch lanes are taken round-robin from lane_rr_. Several lanes need the // per-lane thread-index window rebuilt when the block shape changes, which From 64c2039a835834a78aa5ac15795f62438040ed62 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:16 -0700 Subject: [PATCH 042/336] simx: model the depth of the cache networks SimX treated every cache network as one registered crossbar. The RTL builds them from smaller pieces, and each added level is another registered stage: - an arbiter over more than 1.5x the fan-out limit (8) is a buffered tree, one cycle deeper, so the core-side request arbiter gains a stage ahead of each bank when it has more than 12 inputs; - the bank-to-memory-port network is a radix-2 omega network, one registered stage per level, and its arbiter registers once when banks and ports differ in number and either side exceeds two; - a registered core-response port is a network registered at every level, applied as the response leaves the cache. These delays are charged on the request and response paths of each cache. Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/cache.cpp | 92 +++++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index ecd6b16448..efd5979a7e 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -29,6 +29,24 @@ using namespace vortex; +namespace { + +// An arbiter over more than 1.5x the fan-out limit is split into a buffered +// tree, one cycle deeper. +uint32_t arb_tree_delay(uint32_t requesters) { + constexpr uint32_t kMaxFanout = 8; + return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; +} + +// Registered stages of a radix-2 omega network between a and b ports; up to +// two ports on each side it is a single crossbar. +uint32_t omega_stages(uint32_t a, uint32_t b) { + uint32_t n = std::max(a, b); + return (n <= 2) ? 1 : log2ceil(n); +} + +} + struct params_t { uint32_t sets_per_bank; uint32_t lines_per_set; @@ -2066,6 +2084,7 @@ class CachePortArb : public SimObject { // the request crossbar registers its output when it has more than two inputs , delay_((num_inputs > 2) ? 1 : 0) , rr_(num_banks, num_inputs - 1) + , tree_stage_((arb_tree_delay(num_inputs) != 0) ? num_banks : 0, this) { assert(num_inputs <= 64); } @@ -2086,6 +2105,14 @@ class CachePortArb : public SimObject { --init_cycles_; return; } + // the arbiter tree's buffered level ahead of each bank + for (uint32_t b = 0, nb = tree_stage_.size(); b < nb; ++b) { + auto& stage = tree_stage_.at(b); + if (!stage.empty() && !ReqOut.at(b).full()) { + ReqOut.at(b).send(stage.peek(), delay_); + stage.pop(); + } + } uint32_t n = ReqIn.size(); // requesting ports per bank; non-cacheable requests leave on their own port std::vector want(ReqOut.size(), 0); @@ -2104,7 +2131,8 @@ class CachePortArb : public SimObject { want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; } for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { - if (want[b] == 0 || ReqOut.at(b).full()) { + auto& out = tree_stage_.empty() ? ReqOut.at(b) : tree_stage_.at(b); + if (want[b] == 0 || out.full()) { continue; } uint32_t i = rr_[b]; @@ -2114,7 +2142,7 @@ class CachePortArb : public SimObject { collisions_ += __builtin_popcountll(want[b]) - 1; auto req = ReqIn.at(i).peek(); req.tag = (req.tag << lg2_inputs_) | i; - ReqOut.at(b).send(req, delay_); + out.send(req, tree_stage_.empty() ? delay_ : 1); ReqIn.at(i).pop(); rr_[b] = i; } @@ -2126,6 +2154,7 @@ class CachePortArb : public SimObject { uint32_t delay_; uint32_t init_cycles_ = 0; std::vector rr_; + std::vector> tree_stage_; uint64_t collisions_ = 0; friend class SimObject; @@ -2165,13 +2194,30 @@ class Cache::Impl { simobject->mem_rsp_in.at(i).bind(&nc_mem_arbs_.at(i)->RspIn.at(0)); } - // Create bank's memory arbiter + // Create bank's memory arbiter. Each level of the memory-side networks + // beyond the first is a registered stage; the request path buffers the + // levels the arbiter's own register does not cover in mem_req_net_, the + // response path in the bank's memory-response queue. snprintf(sname, 100, "%s-mem_arb", simobject->name().c_str()); - auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports); + uint32_t mem_arb_delay = ((num_banks != config_.mem_ports) && (num_banks > 2 || config_.mem_ports > 2)) ? 1 : 0; + uint32_t mem_net_delay = omega_stages(num_banks, config_.mem_ports) - 1; + uint32_t mem_net_extra = (mem_net_delay > mem_arb_delay) ? (mem_net_delay - mem_arb_delay) : 0; + auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports, + mem_arb_delay + mem_net_extra, mem_arb_delay + mem_net_extra); // Connect bank's memory arbiter to non-cacheable arbiter's input port 0 + if (mem_net_extra != 0) { + mem_req_net_.reserve(config_.mem_ports); + for (uint32_t i = 0; i < config_.mem_ports; ++i) { + mem_req_net_.emplace_back(simobject, mem_arb_delay + mem_net_extra + 1); + } + } for (uint32_t i = 0; i < config_.mem_ports; ++i) { - bank_mem_arb->ReqOut.at(i).bind(&nc_mem_arbs_.at(i)->ReqIn.at(0)); + if (mem_net_extra != 0) { + bank_mem_arb->ReqOut.at(i).bind(&mem_req_net_.at(i)); + } else { + bank_mem_arb->ReqOut.at(i).bind(&nc_mem_arbs_.at(i)->ReqIn.at(0)); + } nc_mem_arbs_.at(i)->RspOut.at(0).bind(&bank_mem_arb->RspIn.at(i)); } @@ -2183,12 +2229,21 @@ class Cache::Impl { } // Create bank's core crossbar; it carries the bank responses back to the - // core ports + // core ports. A registered core-response port is a network registered at + // every level, applied as the response leaves the cache. snprintf(sname, 100, "%s-core_xbar", simobject->name().c_str()); - bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, - [&](const MemCrossBar::ReqType &req) { - return params_.addr_bank_id(req.addr); - }); + auto bank_sel = [&](const MemCrossBar::ReqType &req) { + return params_.addr_bank_id(req.addr); + }; + if (config_.core_out_buf) { + bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel, 0, 0); + core_rsp_delay_ = omega_stages(num_banks, config_.num_inputs); + } else { + bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel); + } + if (config_.nc_enable) { + ++core_rsp_delay_; + } // Create cache banks for (uint32_t i = 0, n = num_banks; i < n; ++i) { @@ -2237,16 +2292,21 @@ class Cache::Impl { } } + // bank memory requests through the memory-side network's deeper levels + for (uint32_t i = 0, n = mem_req_net_.size(); i < n; ++i) { + auto &net = mem_req_net_.at(i); + if (!net.empty() && nc_mem_arbs_.at(i)->ReqIn.at(0).try_send(net.peek(), 0)) { + net.pop(); + } + } + // schedule core responses for (uint32_t req_id = 0, n = config_.num_inputs; req_id < n; ++req_id) { auto &bank_rsp = bank_core_xbar_->RspOut.at(req_id); if (bank_rsp.empty()) continue; auto &core_rsp = bank_rsp.peek(); - // the core-response port's register, then the non-cacheable bypass - // layer's response arbiter - uint32_t rsp_delay = (config_.core_out_buf ? 1 : 0) + (config_.nc_enable ? 1 : 0); - if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, rsp_delay)) { + if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, core_rsp_delay_)) { DT(3, simobject_->name() << " core-rsp: " << core_rsp); bank_rsp.pop(); } @@ -2325,6 +2385,10 @@ class Cache::Impl { std::vector nc_mem_arbs_; CachePortArb::Ptr port_arb_; MemCrossBar::Ptr bank_core_xbar_; + // the core-response network's levels, then the non-cacheable bypass + // layer's response arbiter + uint32_t core_rsp_delay_ = 0; + std::vector> mem_req_net_; uint32_t init_cycles_; bool keep_lines_ = false; }; From f367237417e464ff6d1f50ebd495bc4046373f6b Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:17 -0700 Subject: [PATCH 043/336] simx: register the icache core-response port as the RTL does The icache config now sets the core-response register, as the RTL's icache does, and the fetch latch it fed holds one instruction per slice instead of two. The extra entry stood in for the register and let a fetch slice run one instruction further ahead than the RTL. Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 2 +- sim/simx/socket.cpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 558bf54a6e..4fdf2f54ce 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -83,7 +83,7 @@ class Core::Impl { // icache tags of the slice's in-flight fetches for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { snprintf(sname, 100, "fetch_latch%d", f); - fetch_latch_.emplace_back(std::make_unique>(ctx, sname, 2, 2)); + fetch_latch_.emplace_back(std::make_unique>(ctx, sname, 1, 2)); snprintf(sname, 100, "decode_latch%d", f); decode_latch_.emplace_back(std::make_unique>(ctx, sname, 1, 2)); pending_icache_.emplace_back(PER_FETCH_WARPS); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 064eaba4e2..7c8a76942f 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -70,6 +70,11 @@ class Socket::Impl { VX_CFG_ICACHE_LATENCY, // pipeline latency (capacity-scaled, matches hardware) VX_CFG_ICACHE_REPL_POLICY, // replacement policy false, // is_llc (icache never carries AMO state) + false, // non-cacheable bypass + false, // bank_hash + false, // shared_llc + false, // lazy_fetch + true, // core-response register }); snprintf(sname, 100, "%s-dcache", name.c_str()); From 6f3ea117ec57114c467c2cddb05346859d748a71 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:18 -0700 Subject: [PATCH 044/336] ci: add a model_parity case for SGEMM on two frontend slices parity-sgemm-fw2 runs sgemm -n128 with 16 threads x 16 warps, ISSUE_WIDTH = 4 and FETCH_WIDTH = 2: per-slice fetch on a two-port icache, CTAs as wide as the core, and a 16-port dcache. It covers the per-partition front end on both models. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 4f3f4c657e..6f9187911d 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -176,6 +176,14 @@ tests: args: -n128 configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 shape: {cores: 2, l2cache: true} +# SGEMM on two frontend slices feeding four issue slices: per-slice fetch on a +# two-port icache, CTAs as wide as the core, a 16-port dcache. +- id: parity-sgemm-fw2 + check: model_parity + via: blackbox + app: sgemm + args: -n128 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity From 5f1ef4fad9487b304dcf83bbcdfac487d954b5fe Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:53:11 -0700 Subject: [PATCH 045/336] Count global barriers across the whole device, not per cluster vx_num_cores() returns the device total (cores x clusters), and vortex::gbarrier passes it as the barrier count, but every cluster ran its own global-barrier unit that only ever saw its own cores: - SimX's per-cluster count could never reach the total, so any kernel using a global barrier hung on more than one cluster. - In the RTL the count and core id were NC_WIDTH (cores per cluster) wide. At 2 clusters x 1 core the global path was not even built and the scheduler timed out. At 2 clusters x 2 cores the truncated count released each cluster after its own two arrivals, so the barrier gave no ordering between clusters. One global-barrier unit now serves the device. Each cluster forwards its cores' arrivals through an arbiter in Vortex.sv (OUT_BUF 3 with several clusters, none with one) to a single VX_gbar_unit. The unit keeps one arrival counter per barrier, compared at full width, in place of the per-core mask and popcount; a core arrives once per barrier generation, so the count identifies the release. The request carries the device-global core id, and the global path is built whenever the device has more than one core. SimX adds the matching device-level GbarUnit; with one cluster both models keep their existing path and timing. rtlsim and SimX: async_gbarrier and gbar_phase pass at 2 clusters x 1 core, 2 clusters x 2 cores with L2 and L3, 2 clusters with L2 only, and 4 clusters with L3. Before the fix, gbar_phase at 2 clusters x 1 core hits the scheduler timeout on rtlsim, and every case above hangs on SimX. sgemm, raycast and gbar_phase at one cluster are cycle-identical on both models. CI adds the four 2-cluster cases on simx and rtlsim. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 43 +++++++++++++++++++++++ hw/rtl/VX_cluster.sv | 12 ++----- hw/rtl/VX_gpu_pkg.sv | 6 ++-- hw/rtl/Vortex.sv | 26 ++++++++++++++ hw/rtl/core/VX_bar_unit.sv | 24 +++++++------ hw/rtl/mem/VX_gbar_unit.sv | 25 +++++++------ hw/unittest/core/VX_core_top.sv | 4 +-- sim/simx/Makefile | 2 +- sim/simx/cluster.cpp | 58 +++++++++++++++++++++++-------- sim/simx/cluster.h | 6 ++++ sim/simx/gbar_unit.cpp | 50 ++++++++++++++++++++++++++ sim/simx/gbar_unit.h | 45 ++++++++++++++++++++++++ sim/simx/processor.cpp | 8 +++++ sim/simx/processor_impl.h | 2 ++ sw/kernel/include/vx_intrinsics.h | 2 +- 15 files changed, 259 insertions(+), 54 deletions(-) create mode 100644 sim/simx/gbar_unit.cpp create mode 100644 sim/simx/gbar_unit.h diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index af35816d28..4c16b29ba2 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -98,6 +98,49 @@ tests: app: gbar_phase shape: cores: 4 +# A global barrier spans clusters: the device-level unit counts every core, so +# one core per cluster still meets the count and two clusters of two cores do +# not release on a cluster's own arrivals. +- id: gbar-phase-cl2-1 + via: blackbox + drivers: + - simx + - rtlsim + app: gbar_phase + shape: + clusters: 2 + cores: 1 +- id: gbar-phase-cl2-2 + via: blackbox + drivers: + - simx + - rtlsim + app: gbar_phase + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +- id: async-gbarrier-cl2-1 + via: blackbox + drivers: + - simx + - rtlsim + app: async_gbarrier + shape: + clusters: 2 + cores: 1 +- id: async-gbarrier-cl2-2 + via: blackbox + drivers: + - simx + - rtlsim + app: async_gbarrier + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true # SimX<->rtlsim cycle parity on the multi-core global barrier. Until this case # existed the gate could not observe the gbar path at all: every other parity # case is single-core, so USE_GBAR is 0 and the request queue is not built. diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 79f625eda4..05fed84daa 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -38,6 +38,9 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; // KMU bus VX_kmu_bus_if.slave kmu_bus_if[1], + // Global barrier bus to the device-level barrier unit + VX_gbar_bus_if.master gbar_bus_if, + `ifdef VX_CFG_EXT_RASTER_ENABLE // Delegated draw launch (device KMU → raster engines) VX_raster_launch_if.slave raster_launch_if[1], @@ -161,7 +164,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif VX_gbar_bus_if per_socket_gbar_bus_if[NUM_SOCKETS](); - VX_gbar_bus_if gbar_bus_if(); VX_gbar_arb #( .NUM_REQS (NUM_SOCKETS), @@ -174,14 +176,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .bus_out_if (gbar_bus_if) ); - VX_gbar_unit #( - .INSTANCE_ID (`SFORMATF(("%s-gbar", INSTANCE_ID))) - ) gbar_unit ( - .clk (clk), - .reset (reset), - .gbar_bus_if (gbar_bus_if) - ); - // L2 input buses (socket ports + cluster-resident gfx cache ports) VX_mem_bus_if #( .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index ec92eca60a..db202a7779 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -88,7 +88,7 @@ package VX_gpu_pkg; localparam BAR_ADDR_W = `UP(BAR_ADDR_BITS); localparam BAR_ID_SHIFT = 8; - localparam BAR_SIZE_W = `MAX(`MAX(NW_WIDTH, NC_WIDTH), `CLOG2(`VX_CFG_MAX_BAR_EVENTS)); + localparam BAR_SIZE_W = `MAX(`MAX(NW_WIDTH, CORE_ID_WIDTH), `CLOG2(`VX_CFG_MAX_BAR_EVENTS)); localparam UOP_PACKLD = 0; localparam UOP_TCU = UOP_PACKLD + 1; @@ -802,8 +802,8 @@ package VX_gpu_pkg; typedef struct packed { logic [NB_WIDTH-1:0] id; - logic [NC_WIDTH-1:0] size_m1; - logic [NC_WIDTH-1:0] core_id; + logic [CORE_ID_WIDTH-1:0] size_m1; + logic [CORE_ID_WIDTH-1:0] core_id; } gbar_req_t; typedef struct packed { diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index e536ba88b8..b3bb5010d4 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -331,6 +331,30 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .bus_out_if (per_cluster_dcr_bus_if) ); + // A global barrier's participants can span clusters, so its arrivals are + // counted once for the whole device. + VX_gbar_bus_if per_cluster_gbar_bus_if[`VX_CFG_NUM_CLUSTERS](); + VX_gbar_bus_if gbar_bus_if(); + + VX_gbar_arb #( + .NUM_REQS (`VX_CFG_NUM_CLUSTERS), + .REQ_OUT_BUF ((`VX_CFG_NUM_CLUSTERS > 1) ? 3 : 0), + .RSP_OUT_BUF ((`VX_CFG_NUM_CLUSTERS > 1) ? 3 : 0) + ) gbar_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (per_cluster_gbar_bus_if), + .bus_out_if (gbar_bus_if) + ); + + VX_gbar_unit #( + .INSTANCE_ID ("gbar") + ) gbar_unit ( + .clk (clk), + .reset (reset), + .gbar_bus_if (gbar_bus_if) + ); + // Generate all clusters for (genvar cluster_id = 0; cluster_id < `VX_CFG_NUM_CLUSTERS; ++cluster_id) begin : g_clusters @@ -353,6 +377,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .kmu_bus_if (per_cluster_kmu_bus_if[cluster_id +: 1]), + .gbar_bus_if (per_cluster_gbar_bus_if[cluster_id]), + `ifdef VX_CFG_EXT_RASTER_ENABLE .raster_launch_if (per_cluster_raster_launch_if[cluster_id +: 1]), `endif diff --git a/hw/rtl/core/VX_bar_unit.sv b/hw/rtl/core/VX_bar_unit.sv index f608b71e79..1ab924331a 100644 --- a/hw/rtl/core/VX_bar_unit.sv +++ b/hw/rtl/core/VX_bar_unit.sv @@ -40,12 +40,14 @@ module VX_bar_unit import VX_gpu_pkg::*; #( // warp mask + warp count + event count localparam EVENT_WIDTH = `CLOG2(`VX_CFG_MAX_BAR_EVENTS + 1); - localparam BAR_STATEW = `VX_CFG_NUM_WARPS + NW_WIDTH + EVENT_WIDTH; - localparam GBAR_REQW = NB_WIDTH + NC_WIDTH; - localparam USE_GBAR = (`VX_CFG_NUM_CORES > 1); + // the count holds warps for a local barrier, cores for a global one + localparam COUNT_W = `MAX(NW_WIDTH, CORE_ID_WIDTH); + localparam BAR_STATEW = `VX_CFG_NUM_WARPS + COUNT_W + EVENT_WIDTH; + localparam GBAR_REQW = NB_WIDTH + CORE_ID_WIDTH; + localparam USE_GBAR = ((`VX_CFG_NUM_CORES * `VX_CFG_NUM_CLUSTERS) > 1); logic [`VX_CFG_NUM_WARPS-1:0] mask_r, mask_n; - logic [NW_WIDTH-1:0] count_r, count_n; + logic [COUNT_W-1:0] count_r, count_n; logic [EVENT_WIDTH-1:0] events_r, events_n; logic phase_r, phase_n; @@ -59,14 +61,14 @@ module VX_bar_unit import VX_gpu_pkg::*; #( logic gbar_enqueue; logic [NB_WIDTH-1:0] gbar_enqueue_id; - logic [NC_WIDTH-1:0] gbar_enqueue_size_m1; + logic [CORE_ID_WIDTH-1:0] gbar_enqueue_size_m1; wire gbar_rsp_ready = ~req_valid; wire gbar_rsp_fire = gbar_bus_if.rsp_valid && gbar_rsp_ready; wire gbar_rsp_apply = USE_GBAR && gbar_rsp_fire && gbar_pending_r[gbar_bus_if.rsp_data.id]; wire [`VX_CFG_NUM_WARPS-1:0] wait_mask = ((`VX_CFG_NUM_WARPS)'(1) << req_wid) | mask_r; - wire [NW_WIDTH-1:0] next_count = count_r + NW_WIDTH'(1); + wire [COUNT_W-1:0] next_count = count_r + COUNT_W'(1); wire next_phase = ~phase_r; always @(*) begin @@ -104,7 +106,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( end end else if (req_data.is_arrive) begin // barrier arrival - if (count_r == NW_WIDTH'(req_data.size_m1)) begin + if (count_r == COUNT_W'(NW_WIDTH'(req_data.size_m1))) begin count_n = '0; if (events_r == 0) begin mask_n = '0; @@ -148,11 +150,11 @@ module VX_bar_unit import VX_gpu_pkg::*; #( mask_n = '0; gbar_enqueue = 1; // notify global barrier gbar_enqueue_id = req_data.id; - gbar_enqueue_size_m1 = NC_WIDTH'(count_r); // was saved in barrier_arrive + gbar_enqueue_size_m1 = CORE_ID_WIDTH'(count_r); // was saved in barrier_arrive end end else if (req_data.is_arrive) begin // barrier arrival - count_n = NW_WIDTH'(req_data.size_m1); // store participating number of cores + count_n = COUNT_W'(req_data.size_m1); // store participating number of cores if (req_data.is_sync) begin gbar_waiters_n[req_data.id][req_wid] = 1; end @@ -160,7 +162,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( mask_n = '0; gbar_enqueue = 1; // notify global barrier gbar_enqueue_id = req_data.id; - gbar_enqueue_size_m1 = NC_WIDTH'(req_data.size_m1); + gbar_enqueue_size_m1 = CORE_ID_WIDTH'(req_data.size_m1); end else begin // Add arriving warp to arrival mask mask_n = wait_mask; @@ -329,7 +331,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( assign gbar_bus_if.req_valid = gbar_req_valid_r; assign {gbar_bus_if.req_data.id, gbar_bus_if.req_data.size_m1} = req_queue_data; - assign gbar_bus_if.req_data.core_id = NC_WIDTH'(32'(core_id) % `VX_CFG_NUM_CORES); + assign gbar_bus_if.req_data.core_id = core_id; assign gbar_bus_if.rsp_ready = gbar_rsp_ready; `RUNTIME_ASSERT(~gbar_enqueue || ~gbar_pending_r[gbar_enqueue_id], ("%s duplicate global barrier generation: id=%0d", INSTANCE_ID, gbar_enqueue_id)) diff --git a/hw/rtl/mem/VX_gbar_unit.sv b/hw/rtl/mem/VX_gbar_unit.sv index 9af6345ef9..072105d63a 100644 --- a/hw/rtl/mem/VX_gbar_unit.sv +++ b/hw/rtl/mem/VX_gbar_unit.sv @@ -23,36 +23,35 @@ module VX_gbar_unit import VX_gpu_pkg::*; #( ); `UNUSED_SPARAM (INSTANCE_ID) - reg [`VX_CFG_NUM_BARRIERS-1:0][`VX_CFG_NUM_CORES-1:0] barrier_masks; - wire [`CLOG2(`VX_CFG_NUM_CORES+1)-1:0] active_barrier_count; + // A core sends one request per barrier generation, so counting requests + // tracks the participants without a per-core mask over the whole device. + reg [`VX_CFG_NUM_BARRIERS-1:0][CORE_ID_WIDTH-1:0] arrive_counts; wire [NB_WIDTH-1:0] req_id = gbar_bus_if.req_data.id; - wire [`VX_CFG_NUM_CORES-1:0] curr_barrier_mask = barrier_masks[req_id]; - - `POP_COUNT(active_barrier_count, curr_barrier_mask); - `UNUSED_VAR (active_barrier_count) + wire [CORE_ID_WIDTH-1:0] curr_arrive_count = arrive_counts[req_id]; + `UNUSED_VAR (gbar_bus_if.req_data.core_id) reg rsp_valid; reg [NB_WIDTH-1:0] rsp_bar_id; - wire req_file = gbar_bus_if.req_valid && gbar_bus_if.req_ready; + wire req_fire = gbar_bus_if.req_valid && gbar_bus_if.req_ready; wire rsp_fire = gbar_bus_if.rsp_valid && gbar_bus_if.rsp_ready; wire rsp_stall = gbar_bus_if.rsp_valid && ~gbar_bus_if.rsp_ready; always @(posedge clk) begin if (reset) begin - barrier_masks <= '0; + arrive_counts <= '0; rsp_valid <= 0; end else begin if (rsp_fire) begin rsp_valid <= 0; end - if (req_file) begin - if (active_barrier_count[NC_WIDTH-1:0] == gbar_bus_if.req_data.size_m1) begin - barrier_masks[req_id] <= '0; + if (req_fire) begin + if (curr_arrive_count == gbar_bus_if.req_data.size_m1) begin + arrive_counts[req_id] <= '0; rsp_valid <= 1; rsp_bar_id <= req_id; end else begin - barrier_masks[req_id][gbar_bus_if.req_data.core_id] <= 1; + arrive_counts[req_id] <= curr_arrive_count + CORE_ID_WIDTH'(1); end end end @@ -67,7 +66,7 @@ module VX_gbar_unit import VX_gpu_pkg::*; #( `ifdef DBG_TRACE_GBAR always @(posedge clk) begin - if (req_file) begin + if (req_fire) begin `TRACE(2, ("%t: %s arrive: bar_id=%0d, size=%0d, core_id=%0d\n", $time, INSTANCE_ID, gbar_bus_if.req_data.id, gbar_bus_if.req_data.size_m1, gbar_bus_if.req_data.core_id)) end diff --git a/hw/unittest/core/VX_core_top.sv b/hw/unittest/core/VX_core_top.sv index b0dbb9b44a..d47bfad744 100644 --- a/hw/unittest/core/VX_core_top.sv +++ b/hw/unittest/core/VX_core_top.sv @@ -57,8 +57,8 @@ module VX_core_top import VX_gpu_pkg::*; output wire gbar_req_valid, output wire [NB_WIDTH-1:0] gbar_req_id, - output wire [NC_WIDTH-1:0] gbar_req_size_m1, - output wire [NC_WIDTH-1:0] gbar_req_core_id, + output wire [CORE_ID_WIDTH-1:0] gbar_req_size_m1, + output wire [CORE_ID_WIDTH-1:0] gbar_req_core_id, input wire gbar_req_ready, input wire gbar_rsp_valid, input wire [NB_WIDTH-1:0] gbar_rsp_id, diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 746bd6f7f8..75a9caa738 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -33,7 +33,7 @@ XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME) # Source files definition SRCS = $(SW_COMMON_DIR)/util.cpp $(SIM_COMMON_DIR)/mem.cpp $(SW_COMMON_DIR)/softfloat_ext.cpp $(SW_COMMON_DIR)/rvfloats.cpp $(SIM_COMMON_DIR)/dram_sim.cpp $(SIM_COMMON_DIR)/cmd_processor.cpp $(SIM_COMMON_DIR)/elf_loader.cpp $(SIM_COMMON_DIR)/host_monitor.cpp $(SIM_COMMON_DIR)/cout_drainer.cpp -SRCS += $(SRC_DIR)/processor.cpp $(SRC_DIR)/cluster.cpp $(SRC_DIR)/socket.cpp $(SRC_DIR)/core.cpp $(SRC_DIR)/scheduler.cpp $(SRC_DIR)/cta_dispatcher.cpp $(SRC_DIR)/barrier_unit.cpp +SRCS += $(SRC_DIR)/processor.cpp $(SRC_DIR)/cluster.cpp $(SRC_DIR)/socket.cpp $(SRC_DIR)/core.cpp $(SRC_DIR)/scheduler.cpp $(SRC_DIR)/cta_dispatcher.cpp $(SRC_DIR)/barrier_unit.cpp $(SRC_DIR)/gbar_unit.cpp SRCS += $(SRC_DIR)/kmu/kmu.cpp SRCS += $(SRC_DIR)/decode.cpp $(SRC_DIR)/decompressor.cpp $(SRC_DIR)/scoreboard.cpp $(SRC_DIR)/sequencer.cpp $(SRC_DIR)/opc_unit.cpp $(SRC_DIR)/dispatcher.cpp SRCS += $(SRC_DIR)/alu_unit.cpp $(SRC_DIR)/fpu_unit.cpp $(SRC_DIR)/lsu_unit.cpp $(SRC_DIR)/sfu_unit.cpp $(SRC_DIR)/wctl_unit.cpp $(SRC_DIR)/csr_unit.cpp diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 88ab5dbb20..18fa87ad96 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -12,6 +12,7 @@ // limitations under the License. #include "cluster.h" +#include #include "socket.h" #include "core.h" #include "local_mem.h" @@ -31,6 +32,11 @@ using namespace vortex; +// Output-buffer stages of the socket and cluster gbar arbiters (OUT_BUF 3, one +// cycle each), present when the arbiter has more than one input. +static constexpr uint32_t kSocketArbLag = (VX_CFG_SOCKET_SIZE > 1) ? 1 : 0; +static constexpr uint32_t kClusterArbLag = (NUM_SOCKETS > 1) ? 1 : 0; + class Cluster::Impl { public: Impl(Cluster* simobject) @@ -304,6 +310,7 @@ class Cluster::Impl { for (auto& gbar : gbarriers_) { gbar.reset(); } + gbar_fwd_next_ = 0; // Sockets are SimObjects; reset by SimPlatform. } @@ -334,10 +341,7 @@ class Cluster::Impl { auto bar_index = bar_id % gbarriers_.size(); auto& gbar = gbarriers_.at(bar_index); - auto sockets_per_cluster = sockets_.size(); - auto cores_per_socket = cores_per_socket_; - - uint32_t cores_per_cluster = sockets_per_cluster * cores_per_socket; + uint32_t cores_per_cluster = sockets_.size() * cores_per_socket_; uint32_t local_core_id = core_id % cores_per_cluster; // set core arrival bit @@ -345,19 +349,37 @@ class Cluster::Impl { DT(4, "*** Global barrier arrive: cluster #" << simobject_->id() << ", core #" << core_id << " at barrier #" << bar_id << ", arrived=" << gbar.mask.count()); - if (gbar.mask.count() == (size_t)count) { - // resume all suspended cores - for (uint32_t s = 0; s < sockets_per_cluster; ++s) { - for (uint32_t c = 0; c < cores_per_socket; ++c) { - uint32_t i = s * cores_per_socket + c; - if (gbar.mask.test(i)) { - simobject_->gbar_resume_out.at(i).send({bar_id}); - } + if constexpr (VX_CFG_NUM_CLUSTERS > 1) { + // Up to the device unit: the socket arbiter's output buffer, the cluster + // arbiter (one arrival per cycle) and its output buffer, then the device + // arbiter's output buffer. + uint64_t now = SimPlatform::instance().cycles(); + uint64_t out = std::max(now + kSocketArbLag, gbar_fwd_next_); + gbar_fwd_next_ = out + 1; + simobject_->gbar_fwd_out.send({bar_id, count, core_id}, out + kClusterArbLag + 1 - now); + } else if (gbar.mask.count() == (size_t)count) { + this->global_barrier_resume(bar_id, 1); + } + } + + // The device unit's release crosses the cluster's and the socket's response + // forks on its way down. + void global_barrier_fwd_resume(uint32_t bar_id) { + this->global_barrier_resume(bar_id, 1 + kClusterArbLag + kSocketArbLag); + } + + // Resume the cores of this cluster that arrived, then clear the barrier. + void global_barrier_resume(uint32_t bar_id, uint32_t delay) { + auto& gbar = gbarriers_.at(bar_id % gbarriers_.size()); + for (uint32_t s = 0; s < sockets_.size(); ++s) { + for (uint32_t c = 0; c < cores_per_socket_; ++c) { + uint32_t i = s * cores_per_socket_ + c; + if (gbar.mask.test(i)) { + simobject_->gbar_resume_out.at(i).send({bar_id}, delay); } } - // reset mask and advance phase - gbar.mask.reset(); } + gbar.mask.reset(); } Cluster::PerfStats perf_stats() const { @@ -568,6 +590,7 @@ class Cluster::Impl { Cluster* simobject_; std::vector sockets_; std::vector gbarriers_; + uint64_t gbar_fwd_next_ = 0; // next free cycle at the cluster's gbar arbiter Cache::Ptr l2cache_; uint32_t cores_per_socket_; #ifdef VX_CFG_VM_ENABLE @@ -598,11 +621,14 @@ Cluster::Cluster(const SimContext& ctx, , mem_rsp_in(VX_CFG_L2_MEM_PORTS, this) , gbar_arrive_in(this) , gbar_resume_out(NUM_SOCKETS * VX_CFG_SOCKET_SIZE, this) + , gbar_fwd_out(this) + , gbar_fwd_resume_in(this) , cluster_id_(cluster_id) , processor_(processor) , impl_(new Impl(this)) { gbar_arrive_in.bind(this, &Cluster::on_gbar_arrive); + gbar_fwd_resume_in.bind(this, &Cluster::on_gbar_fwd_resume); } Cluster::~Cluster() { @@ -625,6 +651,10 @@ void Cluster::on_gbar_arrive(const GbarArrive& msg) { impl_->global_barrier_arrive(msg.bar_id, msg.count, msg.core_id); } +void Cluster::on_gbar_fwd_resume(const GbarResume& msg) { + impl_->global_barrier_fwd_resume(msg.bar_id); +} + Cluster::PerfStats Cluster::perf_stats() const { return impl_->perf_stats(); } diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index bc7fa210a4..1bcc518427 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -77,6 +77,11 @@ class Cluster : public SimObject { SimEventLink gbar_arrive_in; std::vector> gbar_resume_out; + // With more than one cluster the barrier spans clusters: arrivals go up to + // the device-level GbarUnit and its resume comes back down here. + SimEventLink gbar_fwd_out; + SimEventLink gbar_fwd_resume_in; + Cluster(const SimContext& ctx, const char* name, uint32_t cluster_id, @@ -151,6 +156,7 @@ class Cluster : public SimObject { private: void on_gbar_arrive(const GbarArrive& msg); + void on_gbar_fwd_resume(const GbarResume& msg); uint32_t cluster_id_; ProcessorImpl* processor_; diff --git a/sim/simx/gbar_unit.cpp b/sim/simx/gbar_unit.cpp new file mode 100644 index 0000000000..f331c6272c --- /dev/null +++ b/sim/simx/gbar_unit.cpp @@ -0,0 +1,50 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gbar_unit.h" +#include +#include "debug.h" + +using namespace vortex; + +GbarUnit::GbarUnit(const SimContext& ctx, const char* name, uint32_t num_clusters) + : SimObject(ctx, name) + , arrive_in(this) + , resume_out(num_clusters, this) + , arrived_(VX_CFG_NUM_BARRIERS, 0) + , accept_next_(0) +{ + arrive_in.bind(this, &GbarUnit::on_arrive); +} + +void GbarUnit::on_reset() { + std::fill(arrived_.begin(), arrived_.end(), 0); + accept_next_ = 0; +} + +// A core arrives once per generation, so a count of arrivals tracks the +// participants without a mask over every core of the device. +void GbarUnit::on_arrive(const GbarArrive& msg) { + uint64_t now = SimPlatform::instance().cycles(); + uint64_t accept = std::max(now, accept_next_); + accept_next_ = accept + 1; + auto& arrived = arrived_.at(msg.bar_id % arrived_.size()); + ++arrived; + DT(4, "*** Device barrier arrive: core #" << msg.core_id << " at barrier #" << msg.bar_id << ", arrived=" << arrived); + if (arrived == msg.count) { + for (auto& link : resume_out) { + link.send({msg.bar_id}, accept - now + 1); + } + arrived = 0; + } +} diff --git a/sim/simx/gbar_unit.h b/sim/simx/gbar_unit.h new file mode 100644 index 0000000000..fc06fe8fe8 --- /dev/null +++ b/sim/simx/gbar_unit.h @@ -0,0 +1,45 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "types.h" + +namespace vortex { + +// Device-level global barrier: with more than one cluster, a barrier's +// participants span clusters, so each cluster forwards its cores' arrivals +// here and every cluster is resumed once the count is met. Arrivals enter +// through the device arbiter's output buffer, one per cycle; the release is +// registered in the unit and again in the response fork to the clusters. +class GbarUnit : public SimObject { +public: + SimEventLink arrive_in; + std::vector> resume_out; + + GbarUnit(const SimContext& ctx, const char* name, uint32_t num_clusters); + +protected: + void on_reset(); + +private: + void on_arrive(const GbarArrive& msg); + + std::vector arrived_; + uint64_t accept_next_; // first cycle the unit can take another arrival + + friend class SimObject; +}; + +} // namespace vortex diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 77163fe4df..268773e2d8 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -63,6 +63,14 @@ ProcessorImpl::ProcessorImpl() clusters_.at(i) = Cluster::Create(sname, i, this); } + if constexpr (VX_CFG_NUM_CLUSTERS > 1) { + gbar_unit_ = GbarUnit::Create("gbar", VX_CFG_NUM_CLUSTERS); + for (uint32_t i = 0; i < VX_CFG_NUM_CLUSTERS; ++i) { + clusters_.at(i)->gbar_fwd_out.bind(&gbar_unit_->arrive_in); + gbar_unit_->resume_out.at(i).bind(&clusters_.at(i)->gbar_fwd_resume_in); + } + } + // Launch bus: a registered, credit-gated lane from the KMU to each core's // dispatcher; CTAs cross into the core's domain through the slice, and // admission credits return to the KMU through a slice of their own. diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index 3e2f697b87..50ecc05106 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -18,6 +18,7 @@ #include "constants.h" #include "cluster.h" #include "kmu.h" +#include "gbar_unit.h" namespace vortex { @@ -85,6 +86,7 @@ class ProcessorImpl { Kmu::Ptr kmu_; std::vector clusters_; + GbarUnit::Ptr gbar_unit_; // device-level global barrier (multi-cluster) Memory::Ptr memsim_; #ifdef VX_CFG_VM_ENABLE uint64_t mmu_satp_ = 0; // assembled from the two DCR halves diff --git a/sw/kernel/include/vx_intrinsics.h b/sw/kernel/include/vx_intrinsics.h index 78468f7744..b4ed86b144 100644 --- a/sw/kernel/include/vx_intrinsics.h +++ b/sw/kernel/include/vx_intrinsics.h @@ -204,7 +204,7 @@ inline __attribute__((const)) int vx_num_warps() { return (int)csr_read_nv(VX_CSR_NUM_WARPS); } -// Return the number of cores per cluster +// Return the number of cores across all clusters inline __attribute__((const)) int vx_num_cores() { return (int)csr_read_nv(VX_CSR_NUM_CORES); } From 3f27ba402b36c1b59786e47d82591ab28488f670 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 17:13:31 -0700 Subject: [PATCH 046/336] Stop fences at the point of coherence and skip the walk for IO fences Every RISC-V fence went to the data cache as a full flush token: each cache walked all its lines and forwarded the token, so a fence walked the L1, the L2 and the L3, and locked each cache's inputs for the walk. On printf-style kernels that fence after every device write, the walks dominated run time. Fences now take one of two lighter forms: - A memory fence (any R or W in its predecessor or successor set) sends an L1-only flush token. Each private level walks and forwards it, and the shared LLC, the point of coherence, answers without a walk. A replicated LLC still walks, since data crosses it between cores. - A fence ordering only device accesses (no R or W) decodes as a new LSU op and sends an IO token (is_flush with the io attribute). No level walks. Each cache waits until nothing is held between its inputs and its memory ports, on the cacheable and the non-cacheable path, then sends the token down every memory port, so its answer proves the earlier IO writes reached memory. The non-cacheable bypass always routes a flush token to the cache, whatever its io bit. Tokens of different kinds that meet in one cache combine to the weakest kind that serves them all. fence_mp is a new message-passing test across cores, with one case per topology: 1 core, 2 cores + L2, 2 clusters x 1 core + L2, and 2 clusters x 2 cores + L2 + L3 (rtlsim). It fails when the L2 is wrongly treated as the shared LLC on 2 clusters. rtlsim before -> after: default 2 cores + L2 printf 437,106 -> 249,404 4,006,822 -> 295,165 (13.6x) fence 6,888 -> 6,263 15,163 -> 6,798 async_gbar 67,262 -> 67,262 77,779 -> 67,402 sgemm, raycast and dogfood are cycle-identical. At 2 clusters x 2 cores + L2 + L3 the four apps above do not change. SimX does not model the tokens yet: there a fence is an ordering barrier in the LSU. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 36 ++++++ docs/designs/cache_subsystem.md | 8 +- docs/designs/lsu_pipeline_design.md | 4 +- docs/designs/multicache_amo_coherence.md | 35 +++--- hw/rtl/VX_gpu_pkg.sv | 7 +- hw/rtl/VX_trace_pkg.sv | 1 + hw/rtl/cache/VX_cache.sv | 15 ++- hw/rtl/cache/VX_cache_bank.sv | 2 + hw/rtl/cache/VX_cache_bypass.sv | 4 +- hw/rtl/cache/VX_cache_flush.sv | 7 +- hw/rtl/cache/VX_cache_init.sv | 53 +++++++-- hw/rtl/cache/VX_cache_wrap.sv | 3 +- hw/rtl/core/VX_decode.sv | 5 +- hw/rtl/core/VX_lsu_block.sv | 13 ++- tests/regression/Makefile | 2 +- tests/regression/fence_mp/Makefile | 16 +++ tests/regression/fence_mp/common.h | 18 +++ tests/regression/fence_mp/kernel.cpp | 45 ++++++++ tests/regression/fence_mp/main.cpp | 135 +++++++++++++++++++++++ 19 files changed, 369 insertions(+), 40 deletions(-) create mode 100644 tests/regression/fence_mp/Makefile create mode 100644 tests/regression/fence_mp/common.h create mode 100644 tests/regression/fence_mp/kernel.cpp create mode 100644 tests/regression/fence_mp/main.cpp diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 4c16b29ba2..b59f66d3a7 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -238,6 +238,42 @@ tests: cores: 2 l2cache: true l3cache: true +# A memory fence carries data from one core to another across the caches they +# do not share: the producer's token reaches the point of coherence, the +# consumer's invalidates its copies above it. Two clusters with an L2 each +# replicate the last level, so the fence must walk both L2s there. +- id: fence_mp-1 + via: blackbox + drivers: + - rtlsim + app: fence_mp +- id: fence_mp-2 + via: blackbox + drivers: + - rtlsim + app: fence_mp + shape: + cores: 2 + l2cache: true +- id: fence_mp-3 + via: blackbox + drivers: + - rtlsim + app: fence_mp + shape: + clusters: 2 + cores: 1 + l2cache: true +- id: fence_mp-4 + via: blackbox + drivers: + - rtlsim + app: fence_mp + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true # Kernel to kernel with no host transfer between (one command batch): the # launch flushes are L1-only, so the last-level cache alone carries each # round's writes to the next round, and the full flush runs before readback. diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 7f7c26e390..80c5038263 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -615,8 +615,12 @@ write's `VX_CACHE_FLUSH_L1_ONLY` bit, and the dcache and icache injectors carry it. The ocache, whose tokens reach the L2, sends an L1-only token only when every requester of its flush round is L1-only: the cluster flush (kind latched from the same broadcast write) and the dcache tokens held at the OM steers. -The other graphics caches do not forward and send full tokens. A fence -carries no byte enables, so a fence token is always a full one. +The other graphics caches do not forward and send full tokens. A memory fence +sends an L1-only token. A fence ordering only device accesses sends an **IO** +token (the `io` attribute): no level walks, each quiesces its banks, waits +until nothing is held on its uncached path either, and forwards an IO token +down every memory port, so the answer proves the earlier IO writes reached +memory. Kinds visible together combine to the weakest that serves them all. **Core traffic sits on the priority input, not the flush.** The flush waits until upstream is quiescent — which it reliably is, because the host only issues diff --git a/docs/designs/lsu_pipeline_design.md b/docs/designs/lsu_pipeline_design.md index 025c834d9f..1e3531a12d 100644 --- a/docs/designs/lsu_pipeline_design.md +++ b/docs/designs/lsu_pipeline_design.md @@ -148,7 +148,9 @@ A fence is a **total barrier on its slice**. The slice sets `fence_lock` when a fence's last packet is accepted and clears it when that fence's response returns. While locked, the slice accepts nothing. -The fence itself travels to memory as a request with `is_flush`, behind every +The fence itself travels to memory as a request with `is_flush` (an IO token, +the `io` attribute set, for a fence ordering only device accesses; otherwise an +L1-only token, byte enables set), behind every request the slice issued before it, so its response proves they have drained. Only the last packet of a multi-packet fence is sent; the earlier packets complete immediately. diff --git a/docs/designs/multicache_amo_coherence.md b/docs/designs/multicache_amo_coherence.md index e6ad6901ea..b0233a0252 100644 --- a/docs/designs/multicache_amo_coherence.md +++ b/docs/designs/multicache_amo_coherence.md @@ -240,19 +240,28 @@ updates it, core 0's copy is stale. The LLC never reaches up. The weak model resolves this the way a GPU does: the **consumer** discards its own copies at an acquire point and refetches. -There is no dedicated acquire-invalidate. What exists today: - -| model | what a `fence` does to the issuing core's data cache | -|---|---| -| RTL | the LSU sends the fence as a request with `is_flush` set, which starts the **whole-cache flush**; on a write-through cache the walk clears every valid bit | -| SimX | the fence is an ordering barrier in the LSU only; a flush of a write-through cache is a no-op and no line is invalidated | - -So an RTL `fence` happens to give the consumer-side invalidate, at the cost -of a full flush walk that locks the cache's inputs for its duration, and -SimX does not model it. **The two models disagree, and no regression case -exercises fence-managed cross-core visibility** — `atomic_critical`, which -runs plain loads and stores inside a lock, is skipped on multi-core -configurations for exactly this reason. +There is no dedicated acquire-invalidate. In the RTL the LSU sends a memory +`fence` (any R or W in its predecessor or successor set) to the data cache as +an **L1-only** flush token (`is_flush` with byte enables set). Each cache above +the point of coherence holds its inputs, drains, walks every set (every way of +a write-back cache), writing back dirty lines and clearing every valid bit, and +forwards the token. The shared LLC, the point of coherence, answers it without +a walk; a replicated LLC still walks. The fence holds its LSU block until the +answer. + +So a memory `fence` gives the consumer-side invalidate above the point of +coherence, at the cost of a flush walk of each private level. `fence_mp` +passes data between cores this way on shared and replicated last levels. +`atomic_critical`, which runs plain loads and stores inside a lock, is still +skipped on multi-core configurations. + +A fence ordering only device accesses (`fence ow, ow`, no R or W) sends an +**IO** token (`is_flush` with `io`): no level walks; each quiesces and waits +until its uncached path is empty, then forwards, so the answer proves the +earlier IO writes reached memory. + +SimX does not model the tokens yet: there the fence is an ordering barrier in +the LSU only. Programs that share data across cores through plain memory are outside what is validated. Share through atomics, or through the single shared level. diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index db202a7779..c18e3050cc 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -488,6 +488,7 @@ package VX_gpu_pkg; localparam INST_LSU_SH = 4'b1001; localparam INST_LSU_SW = 4'b1010; localparam INST_LSU_SD = 4'b1011; // new for RV64I SD + localparam INST_LSU_FENCE_IO = 4'b1110; // a fence ordering only device (IO) accesses localparam INST_LSU_FENCE = 4'b1111; localparam INST_LSU_BITS = 4; @@ -504,7 +505,11 @@ package VX_gpu_pkg; endfunction function automatic logic inst_lsu_is_fence(input logic [INST_LSU_BITS-1:0] op); - return (op == INST_LSU_FENCE); + return (op[3:1] == 3'b111); + endfunction + + function automatic logic inst_lsu_is_fence_io(input logic [INST_LSU_BITS-1:0] op); + return (op == INST_LSU_FENCE_IO); endfunction /////////////////////////////////////////////////////////////////////////// diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index dd16cd3dc7..68139de7cc 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -202,6 +202,7 @@ package VX_trace_pkg; INST_LSU_SW: `TRACE(level, ("SW")) INST_LSU_SD: `TRACE(level, ("SD")) INST_LSU_FENCE:`TRACE(level,("FENCE")) + INST_LSU_FENCE_IO:`TRACE(level,("FENCE.IO")) default: `TRACE(level, ("?")) endcase end diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 3ee2bae7aa..dce6f187a7 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -64,7 +64,9 @@ module VX_cache import VX_gpu_pkg::*; #( input wire clk, input wire reset, VX_mem_bus_if.slave core_bus_if [NUM_REQS], - VX_mem_bus_if.master mem_bus_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_if [MEM_PORTS], + // nothing held between this level's inputs and its memory side, on either path + input wire nc_req_empty ); `STATIC_ASSERT(NUM_BANKS == (1 << `CLOG2(NUM_BANKS)), ("invalid parameter: number of banks must be power of 2")) `STATIC_ASSERT(WRITE_ENABLE || !WRITEBACK, ("invalid parameter: writeback requires write enable")) @@ -104,12 +106,14 @@ module VX_cache import VX_gpu_pkg::*; #( .TAG_WIDTH (TAG_WIDTH) ) core_bus2_if[NUM_REQS](); wire [NUM_BANKS-1:0] per_bank_flush_begin; + wire flush_skip; wire [`UP(UUID_WIDTH)-1:0] flush_uuid; wire [NUM_BANKS-1:0] per_bank_flush_end; wire [NUM_BANKS-1:0] per_bank_core_req_fire; wire mem_req_empty; wire [MEM_PORTS-1:0] mem_flush_valid; wire mem_flush_l1_only; + wire mem_flush_io; wire [MEM_PORTS-1:0] mem_flush_ready; wire mem_flush_rsp_phase; wire [MEM_PORTS-1:0] mem_flush_rsp_valid; @@ -133,11 +137,14 @@ module VX_cache import VX_gpu_pkg::*; #( .core_bus_out_if (core_bus2_if), .bank_req_fire (per_bank_core_req_fire), .flush_begin (per_bank_flush_begin), + .flush_skip (flush_skip), .flush_uuid (flush_uuid), .flush_end (per_bank_flush_end), .mem_req_empty (mem_req_empty), + .nc_req_empty (nc_req_empty), .mem_flush_valid (mem_flush_valid), .mem_flush_l1_only (mem_flush_l1_only), + .mem_flush_io (mem_flush_io), .mem_flush_ready (mem_flush_ready), .mem_flush_rsp_phase (mem_flush_rsp_phase), .mem_flush_rsp_valid (mem_flush_rsp_valid), @@ -417,6 +424,7 @@ module VX_cache import VX_gpu_pkg::*; #( .mem_rsp_tag (per_bank_mem_rsp_tag[bank_id]), .mem_rsp_ready (per_bank_mem_rsp_ready[bank_id]), .flush_begin (per_bank_flush_begin[bank_id]), + .flush_skip (flush_skip), .flush_uuid (flush_uuid), .flush_end (per_bank_flush_end[bank_id]) ); @@ -507,7 +515,8 @@ module VX_cache import VX_gpu_pkg::*; #( wire [MEM_PORTS-1:0][`UP(`LOG2UP(NUM_BANKS))-1:0] mem_req_tmp_idx; wire [MEM_PORTS-1:0] mem_req_tmp_ready; // The flush token for port i is a read of the port's first line with the - // flush attribute; byteen bit 0 carries the L1-only kind. It takes bank + // flush attribute; byteen bit 0 carries the L1-only kind and the io + // attribute the IO kind. It takes bank // i's lane into the request crossbar (bank i maps to port i), so the // crossbar forms its address and it leaves through the same registered // outputs as the write-backs ahead of it. @@ -524,7 +533,7 @@ module VX_cache import VX_gpu_pkg::*; #( SECTOR_SIZE'(mem_flush_l1_only), `CS_SECTOR_WIDTH'(0), BANK_MEM_TAG_WIDTH'(0), - `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) + `UP(MEM_ATTR_WIDTH)'((1 << MEM_ATTR_FLUSH_OFFS) | (32'(mem_flush_io) << MEM_ATTR_IO_OFFS)) }; assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 90d8f1ddcb..6137bff9fc 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -91,6 +91,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Flush input wire flush_begin, + input wire flush_skip, input wire [`UP(UUID_WIDTH)-1:0] flush_uuid, output wire flush_end ); @@ -308,6 +309,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .flush_begin (flush_begin), + .flush_skip (flush_skip), .flush_end (flush_end), .flush_init (init_valid), .flush_valid (flush_valid), diff --git a/hw/rtl/cache/VX_cache_bypass.sv b/hw/rtl/cache/VX_cache_bypass.sv index 29232bc2fa..950f434392 100644 --- a/hw/rtl/cache/VX_cache_bypass.sv +++ b/hw/rtl/cache/VX_cache_bypass.sv @@ -61,7 +61,9 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] core_req_nc_sel; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_req_is_nc if (CACHE_ENABLE) begin : g_cache - assign core_req_nc_sel[i] = ~core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; + // a flush token always reaches the cache, whatever its io kind bit + assign core_req_nc_sel[i] = ~core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS] + | core_bus_in_if[i].req_data.attr[MEM_ATTR_FLUSH_OFFS]; end else begin : g_no_cache assign core_req_nc_sel[i] = 1'b0; end diff --git a/hw/rtl/cache/VX_cache_flush.sv b/hw/rtl/cache/VX_cache_flush.sv index c51a5aa2cc..d8227664e6 100644 --- a/hw/rtl/cache/VX_cache_flush.sv +++ b/hw/rtl/cache/VX_cache_flush.sv @@ -23,6 +23,7 @@ module VX_cache_flush import VX_gpu_pkg::*; #( input wire clk, input wire reset, input wire flush_begin, + input wire flush_skip, // the flush only quiesces the bank: an IO token output wire flush_end, output wire flush_init, output wire flush_valid, @@ -48,6 +49,7 @@ module VX_cache_flush import VX_gpu_pkg::*; #( // latch flush_begin that arrives while init is in progress reg flush_pending_r, flush_pending_n; + reg skip_r; always @(*) begin state_n = state; @@ -74,7 +76,7 @@ module VX_cache_flush import VX_gpu_pkg::*; #( // Wait for the bank to fully quiesce before evicting: // both MSHR must drain, and the bank pipeline as well to ensure no inflight misses. if (mshr_empty && bank_empty) begin - state_n = STATE_FLUSH; + state_n = skip_r ? STATE_WAIT2 : STATE_FLUSH; end end STATE_FLUSH: begin @@ -104,6 +106,9 @@ module VX_cache_flush import VX_gpu_pkg::*; #( end else begin state <= state_n; flush_pending_r <= flush_pending_n; + if (flush_begin) begin + skip_r <= flush_skip; + end if (state != STATE_IDLE) begin if ((state == STATE_INIT) || ((state == STATE_FLUSH) && flush_ready)) begin diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index 9dc988ff3a..ce44844a53 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -31,6 +31,11 @@ // the token without a walk. Any other last-level cache still walks, since a // replicated instance or a client reaching memory beside it may hold or need // lines it has not written back. +// An IO token (the io attribute: a fence ordering only device accesses) walks +// nowhere. Once nothing is held between the cache's inputs and its memory +// ports, on the cacheable and the non-cacheable path alike, it goes down every +// memory port, so its answer proves the earlier IO writes reached memory. +// Kinds visible together combine to the weakest one that serves them all. module VX_cache_init import VX_gpu_pkg::*; #( parameter NUM_REQS = 4, // Number of Word requests per cycle parameter NUM_BANKS = 1, // Number of banks @@ -47,12 +52,15 @@ module VX_cache_init import VX_gpu_pkg::*; #( VX_mem_bus_if.master core_bus_out_if [NUM_REQS], input wire [NUM_BANKS-1:0] bank_req_fire, output wire [NUM_BANKS-1:0] flush_begin, + output wire flush_skip, // the banks only quiesce: no walk is owed output wire [`UP(UUID_WIDTH)-1:0] flush_uuid, input wire [NUM_BANKS-1:0] flush_end, // flush tokens to the next level input wire mem_req_empty, // no write-back left between the banks and the ports + input wire nc_req_empty, // no request held between the inputs and the memory side output wire [MEM_PORTS-1:0] mem_flush_valid, output wire mem_flush_l1_only, + output wire mem_flush_io, input wire [MEM_PORTS-1:0] mem_flush_ready, output wire mem_flush_rsp_phase, input wire [MEM_PORTS-1:0] mem_flush_rsp_valid, @@ -119,12 +127,18 @@ module VX_cache_init import VX_gpu_pkg::*; #( end wire flush_req_enable = (| flush_req_mask); - wire [NUM_REQS-1:0] flush_l1_mask; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_l1_mask - assign flush_l1_mask[i] = flush_req_mask[i] && (| core_bus_in_if[i].req_data.byteen); + wire [NUM_REQS-1:0] flush_l1_mask, flush_io_mask; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_kind_mask + assign flush_io_mask[i] = flush_req_mask[i] && core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; + assign flush_l1_mask[i] = flush_req_mask[i] && ~flush_io_mask[i] && (| core_bus_in_if[i].req_data.byteen); end + wire any_full = (| (flush_req_mask & ~flush_l1_mask & ~flush_io_mask)); + wire any_l1 = (| flush_l1_mask); + wire any_io = (| flush_io_mask); // every visible token is L1-only - wire flush_l1_only = (flush_l1_mask == flush_req_mask); + wire flush_l1_only = any_l1 && ~any_full && ~any_io; + // a walk is owed to a full token, and to an L1-only one above the point of coherence + wire flush_walk = any_full || (any_l1 && ~SHARED_LLC); reg [NUM_BANKS-1:0] flush_done, flush_done_n; reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now @@ -133,6 +147,8 @@ module VX_cache_init import VX_gpu_pkg::*; #( reg [NUM_REQS-1:0] rsp_pending, rsp_pending_n; // token still to be answered reg [`UP(UUID_WIDTH)-1:0] flush_uuid_r, flush_uuid_n; reg l1_only_r, l1_only_n; + reg io_r, io_n; + reg skip_r, skip_n; // answer the lowest pending input; its token stays at the input until then wire [NUM_REQS-1:0] rsp_sel_onehot; @@ -213,6 +229,10 @@ module VX_cache_init import VX_gpu_pkg::*; #( end end + // an IO token also waits for the non-cacheable path, which carries the IO + // writes it orders + wire fwd_ready = mem_req_empty && (~io_r || nc_req_empty); + always @(*) begin state_n = state; flush_done_n = flush_done; @@ -222,16 +242,19 @@ module VX_cache_init import VX_gpu_pkg::*; #( rsp_pending_n = rsp_pending & ~(rsp_sel_onehot & {NUM_REQS{flush_rsp_fire}}); flush_uuid_n = flush_uuid_r; l1_only_n = l1_only_r; + io_n = io_r; + skip_n = skip_r; case (state) // STATE_IDLE: default: begin if (flush_req_enable) begin if (BANK_SEL_LATENCY != 0) begin state_n = STATE_WAIT1; - end else if (SHARED_LLC && flush_l1_only) begin + end else if (~flush_walk && ~any_io) begin rsp_pending_n = flush_req_mask; state_n = STATE_RESPOND; end else begin + skip_n = ~flush_walk; state_n = STATE_FLUSH; end for (integer i = NUM_REQS-1; i >= 0; --i) begin @@ -243,10 +266,11 @@ module VX_cache_init import VX_gpu_pkg::*; #( end STATE_WAIT1: begin if (no_inflight_reqs) begin - if (SHARED_LLC && flush_l1_only) begin + if (~flush_walk && ~any_io) begin rsp_pending_n = flush_req_mask; state_n = STATE_RESPOND; end else begin + skip_n = ~flush_walk; state_n = STATE_FLUSH; end end @@ -260,14 +284,17 @@ module VX_cache_init import VX_gpu_pkg::*; #( flush_done_n = flush_done | flush_end; if (flush_done_n == {NUM_BANKS{1'b1}}) begin flush_done_n = '0; - // answer the tokens visible now; later ones start a new walk - rsp_pending_n = flush_req_mask; - l1_only_n = flush_l1_only; + // answer the tokens visible now; later ones start a new walk. + // A quiesce without a walk serves only the tokens owing none. + rsp_pending_n = skip_r ? (flush_io_mask | (SHARED_LLC ? flush_l1_mask : '0)) + : flush_req_mask; + l1_only_n = flush_l1_only && ~skip_r; + io_n = skip_r; state_n = FLUSH_FORWARD ? STATE_FORWARD : STATE_RESPOND; end end STATE_FORWARD: begin - fwd_go_n = fwd_go | mem_req_empty; + fwd_go_n = fwd_go | fwd_ready; fwd_sent_n = fwd_sent | (mem_flush_valid & mem_flush_ready); fwd_done_n = fwd_done | mem_flush_rsp_valid; if (fwd_done_n == {MEM_PORTS{1'b1}}) begin @@ -303,15 +330,19 @@ module VX_cache_init import VX_gpu_pkg::*; #( end flush_uuid_r <= flush_uuid_n; l1_only_r <= l1_only_n; + io_r <= io_n; + skip_r <= skip_n; end assign flush_begin = {NUM_BANKS{state == STATE_FLUSH}}; + assign flush_skip = skip_r; assign flush_uuid = flush_uuid_r; // The drain counter counts the tokens too, so the first one sent must not // take the others' valid away. - assign mem_flush_valid = {MEM_PORTS{(state == STATE_FORWARD) && (fwd_go || mem_req_empty)}} & ~fwd_sent; + assign mem_flush_valid = {MEM_PORTS{(state == STATE_FORWARD) && (fwd_go || fwd_ready)}} & ~fwd_sent; assign mem_flush_rsp_phase = (state == STATE_FORWARD); assign mem_flush_l1_only = l1_only_r; + assign mem_flush_io = io_r; endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 0f87a80e45..2471155263 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -169,7 +169,8 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .cache_perf (cache_perf), `endif .core_bus_if (core_bus_cache_if), - .mem_bus_if (mem_bus_cache_if) + .mem_bus_if (mem_bus_cache_if), + .nc_req_empty (req_empty) ); end else begin : g_passthru for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_cache_if diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 9c2a3fa44d..538164a70a 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -349,7 +349,10 @@ module VX_decode import end INST_FENCE: begin ex_type = EX_LSU; - op_type = INST_LSU_FENCE; + // neither the predecessor nor the successor set holds a memory + // read or write (pred = instr[27:24], succ = instr[23:20], IORW) + op_type = ((funct3 == 3'b000) && (instr[25:24] == 2'b00) && (instr[21:20] == 2'b00)) + ? INST_LSU_FENCE_IO : INST_LSU_FENCE; op_args.lsu.is_store = 0; op_args.lsu.is_float = 0; op_args.lsu.pack = 0; diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 3d9406851a..56aeda0455 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -79,6 +79,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // VX_lsu_agu; this slice contains no address arithmetic). wire req_is_fence, rsp_is_fence; + wire req_fence_io = inst_lsu_is_fence_io(execute_if.data.op_type); wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr @@ -121,8 +122,11 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire lane_is_om = 1'b0; assign mem_req_attr_struct[i].is_addr_om = 1'b0; `endif - assign mem_req_attr_struct[i].is_addr_io = ((block_addr >= io_addr_start) && (block_addr < io_addr_end)) - || lane_is_om; + // a fence's token carries its kind: io for a fence ordering only device + // accesses + assign mem_req_attr_struct[i].is_addr_io = req_is_fence ? req_fence_io + : (((block_addr >= io_addr_start) && (block_addr < io_addr_end)) + || lane_is_om); `ifdef VX_CFG_LMEM_ENABLE // is local memory address assign mem_req_attr_struct[i].is_addr_local = @@ -264,8 +268,9 @@ module VX_lsu_block import VX_gpu_pkg::*; #( default : mem_req_byteen_w = {LSU_WORD_SIZE{1'b1}}; endcase end - // a fence is a flush token: it carries no data, so no byte enables - assign mem_req_byteen[i] = req_is_fence ? '0 : mem_req_byteen_w; + // A fence is a flush token and carries no data. A memory fence is an + // L1-only token (byte enables set): it stops at the shared LLC. + assign mem_req_byteen[i] = req_is_fence ? (req_fence_io ? '0 : '1) : mem_req_byteen_w; end // memory misalignment not supported! diff --git a/tests/regression/Makefile b/tests/regression/Makefile index bb82626739..15db9427dd 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -5,7 +5,7 @@ include $(ROOT_DIR)/config.mk TESTS := \ minimal \ basic demo dogfood dropout dotproduct dotproduct2 mstress io_addr \ - arith printf diverge diverge_large sort fence ldst_order \ + arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ softmax \ diff --git a/tests/regression/fence_mp/Makefile b/tests/regression/fence_mp/Makefile new file mode 100644 index 0000000000..84b07d28fe --- /dev/null +++ b/tests/regression/fence_mp/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := fence_mp + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n64 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/fence_mp/common.h b/tests/regression/fence_mp/common.h new file mode 100644 index 0000000000..634f577e13 --- /dev/null +++ b/tests/regression/fence_mp/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define FENCE_MP_MAGIC 0x5a5a0000u + +typedef struct { + uint32_t num_tasks; // threads launched + uint32_t count; // data words passed from producer to consumer + uint32_t spin_limit; // consumer polls before giving up + uint32_t pad; + uint64_t data_addr; // count words, zero on entry + uint64_t flag_addr; // one word, zero on entry + uint64_t out_addr; // [0] mismatches, [1] 1 if the consumer timed out, [2] pre-read sum +} kernel_arg_t; + +#endif diff --git a/tests/regression/fence_mp/kernel.cpp b/tests/regression/fence_mp/kernel.cpp new file mode 100644 index 0000000000..80b47e7909 --- /dev/null +++ b/tests/regression/fence_mp/kernel.cpp @@ -0,0 +1,45 @@ +#include +#include +#include "common.h" + +// Message passing across cores through plain memory: the producer (the first +// thread) writes data, fences, then raises a flag; the consumer (the last +// thread, on the last core) reads the data once beforehand so stale copies sit +// in its caches, polls the flag behind a fence, fences again and checks the +// data. A memory fence must make the data visible across every cache the two +// cores do not share. +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + auto data = reinterpret_cast(arg->data_addr); + auto flag = reinterpret_cast(arg->flag_addr); + auto out = reinterpret_cast(arg->out_addr); + if (gid == 0) { + for (uint32_t k = 0; k < arg->count; ++k) { + data[k] = FENCE_MP_MAGIC + k; + } + vx_fence(); + *flag = 1; + vx_fence(); + } else if (gid == arg->num_tasks - 1) { + uint32_t sum = 0; + for (uint32_t k = 0; k < arg->count; ++k) { + sum += data[k]; + } + out[2] = sum; + uint32_t timeout = 1; + for (uint32_t s = 0; s < arg->spin_limit; ++s) { + vx_fence(); + if (*flag != 0) { + timeout = 0; + break; + } + } + vx_fence(); + uint32_t errors = 0; + for (uint32_t k = 0; k < arg->count; ++k) { + errors += (data[k] != FENCE_MP_MAGIC + k); + } + out[0] = errors; + out[1] = timeout; + } +} diff --git a/tests/regression/fence_mp/main.cpp b/tests/regression/fence_mp/main.cpp new file mode 100644 index 0000000000..a8b2c05393 --- /dev/null +++ b/tests/regression/fence_mp/main.cpp @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +const char* kernel_file = "kernel.vxbin"; +uint32_t count = 64; +uint32_t spin_limit = 4096; + +vx_device_h device = nullptr; +vx_buffer_h data_buffer = nullptr; +vx_buffer_h flag_buffer = nullptr; +vx_buffer_h out_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-s spin limit] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:s:k:h")) != -1) { + switch (c) { + case 'n': count = atoi(optarg); break; + case 's': spin_limit = atoi(optarg); break; + case 'k': kernel_file = optarg; break; + case 'h': show_usage(); exit(0); break; + default: show_usage(); exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (data_buffer) vx_buffer_release(data_buffer); + if (flag_buffer) vx_buffer_release(flag_buffer); + if (out_buffer) vx_buffer_release(out_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_release(device); + } +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + + RT_CHECK(vx_device_open(0, &device)); + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_cores, num_warps, num_threads; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + // one thread per hardware slot, so producer and consumer are resident together + uint32_t total_threads = num_cores * num_warps * num_threads; + + kernel_arg.num_tasks = total_threads; + kernel_arg.count = count; + kernel_arg.spin_limit = spin_limit; + + std::vector zeros(count, 0); + std::vector out(4, 0); + uint32_t data_size = count * sizeof(uint32_t); + uint32_t out_size = out.size() * sizeof(uint32_t); + RT_CHECK(vx_buffer_create(device, data_size, VX_MEM_READ_WRITE, &data_buffer)); + RT_CHECK(vx_buffer_address(data_buffer, &kernel_arg.data_addr)); + RT_CHECK(vx_buffer_create(device, sizeof(uint32_t), VX_MEM_READ_WRITE, &flag_buffer)); + RT_CHECK(vx_buffer_address(flag_buffer, &kernel_arg.flag_addr)); + RT_CHECK(vx_buffer_create(device, out_size, VX_MEM_READ_WRITE, &out_buffer)); + RT_CHECK(vx_buffer_address(out_buffer, &kernel_arg.out_addr)); + RT_CHECK(vx_enqueue_write(queue, data_buffer, 0, zeros.data(), data_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, flag_buffer, 0, zeros.data(), sizeof(uint32_t), 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, out_buffer, 0, out.data(), out_size, 0, nullptr, nullptr)); + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + uint32_t n = total_threads; + uint32_t grid_dim[1], block_dim[1]; + RT_CHECK(vx_device_max_occupancy_grid(device, 1, &n, grid_dim, block_dim)); + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid_dim[0]; + li.block_dim[0] = block_dim[0]; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + RT_CHECK(vx_enqueue_read(queue, out.data(), out_buffer, 0, out_size, 1, &launch_ev, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + int errors = 0; + if (out[1] != 0) { + std::cout << "consumer timed out waiting for the flag" << std::endl; + ++errors; + } + if (out[0] != 0) { + std::cout << "consumer saw " << std::dec << out[0] << " stale data words" << std::endl; + errors += out[0]; + } + + cleanup(); + + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} From 9473f26e94864e0e3d61fe51200a6d9e86d29b7a Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 20:52:08 -0700 Subject: [PATCH 047/336] Let a shared L3 answer L1-only flush tokens without a walk The walk condition negated SHARED_LLC bitwise. SHARED_LLC is an untyped parameter that takes the width of the value its parent passes: the L2 gets a 1-bit expression, but the L3 gets VX_CFG_L3_ENABLED, a 32-bit 1, whose bitwise negation is nonzero. A shared L3 therefore walked on every L1-only token, from fences and from L1-only launch flushes alike, and the fence gain stopped at the L3. Compare the parameter with zero instead. rtlsim at 2 clusters x 2 cores + L2 + L3, before -> after: printf 8,566,931 -> 724,024 (11.8x) fence 34,264 -> 9,078 (3.8x) async_gbarrier 89,997 -> 69,231 (-23%) dogfood is unchanged. At 2 cores + L2 and at the default config every app is cycle-identical, as are sgemm and raycast. fence_mp passes at all four topologies. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_init.sv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index ce44844a53..75c0aae526 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -138,7 +138,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( // every visible token is L1-only wire flush_l1_only = any_l1 && ~any_full && ~any_io; // a walk is owed to a full token, and to an L1-only one above the point of coherence - wire flush_walk = any_full || (any_l1 && ~SHARED_LLC); + wire flush_walk = any_full || (any_l1 && (SHARED_LLC == 0)); reg [NUM_BANKS-1:0] flush_done, flush_done_n; reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now From 1d3db49b626210df9c170371c36c2aa7b74950e5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 02:02:57 -0700 Subject: [PATCH 048/336] Let the command processor's DMA use the shared last-level cache Host transfers went straight to memory, so the runtime fully flushed the last-level cache before every upload and read-back: a walk of the whole LLC that grows with its size, and kernels that started cold after each transfer. On kernels launched between transfers, those flushes dominated run time. With VX_CFG_CP_LLC_CLIENT set, the CP's DMA becomes a client of the LLC when one LLC alone serves every core (an L2 with one cluster, or an L3). The LLC is then the point of coherence for host transfers, which need only the L1-only flush. VX_CFG_CP_ON_LLC records where the option took effect. The default is off. - VX_cp_llc_port takes the CP's slot on the LLC input, beside the cores, without changing the bank count. It marks IO-aperture addresses with the io attribute, so console output still bypasses the cache. A launch that follows DMA writes is held until an L1-only flush token sent on the slot is answered, so the kernel sees the data. - Write-around: transfers of at least half the LLC carry a new no-alloc attribute. A write-back bank sends a no-alloc write miss with no fill pending straight to memory and updates a hit in place, so a large upload does not fill the LLC with dirty lines that write back during the kernel. Only the rtlsim CP sets it; the xrt bridge passes 0. - The runtime sees a new CP capability bit (DMA_LLC_CLIENT) and then issues only L1-only flushes around transfers. - On the multi-bank xrt AFU, the CP's memory bridge feeds the new Vortex port, which frees one fabric port. The single-bank AFU and OPAE tie it off. rtlsim total cycles, off -> on (the off path's transfers cost no device cycles on rtlsim; the on path's are timed and included): 2 clusters + L2 + L3 dogfood 267,131 -> 186,870 (-30.0%) multikernel 33,443 -> 19,152 (-42.7%) sgemm -n32 35,657 -> 30,634 (-14.1%) 4 cores + L2 dogfood 192,764 -> 175,218 (-9.1%) multikernel 21,835 -> 16,715 (-23.4%) With an 8 MB L3, dogfood drops 64%. Off, every config is cycle-identical. flush_coherence (chain-2/3, stress-1/2 under DBG_FLUSH_STALL) and fence_mp pass with the option on, including with every DMA write forced around the cache. The coherence test fails when CP traffic bypasses the LLC while the runtime skips the full flush, and when write-around data is dropped. The launch fence protects the xrt CP; rtlsim's CP model already orders its writes, so removing the fence is not caught there. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 7 ++ hw/rtl/VX_cluster.sv | 15 ++- hw/rtl/VX_gpu_pkg.sv | 36 ++++++- hw/rtl/Vortex.sv | 91 +++++++++++++++- hw/rtl/Vortex_axi.sv | 13 +++ hw/rtl/afu/common/VX_afu_wrap.sv | 117 ++++++++++++++++---- hw/rtl/afu/opae/vortex_afu.sv | 13 +++ hw/rtl/cache/VX_cache.sv | 4 + hw/rtl/cache/VX_cache_bank.sv | 53 ++++++--- hw/rtl/cache/VX_cache_wrap.sv | 3 + hw/rtl/core/VX_lsu_block.sv | 1 + hw/rtl/cp/VX_cp_axil_regfile.sv | 10 +- hw/rtl/cp/VX_cp_core.sv | 9 +- hw/rtl/mem/VX_cp_llc_port.sv | 154 +++++++++++++++++++++++++++ sim/common/cmd_processor.cpp | 14 ++- sim/common/cmd_processor.h | 11 ++ sim/rtlsim/processor.cpp | 143 +++++++++++++++++++++++++ sim/rtlsim/processor.h | 7 ++ sim/rtlsim/rtlsim_shim.sv | 28 +++++ sw/runtime/common/device.cpp | 17 +-- sw/runtime/common/vortex2_internal.h | 9 +- sw/runtime/rtlsim/vortex.cpp | 32 ++++++ 22 files changed, 725 insertions(+), 62 deletions(-) create mode 100644 hw/rtl/mem/VX_cp_llc_port.sv diff --git a/VX_config.toml b/VX_config.toml index 2646ef048a..51115ae941 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -199,6 +199,13 @@ l3_is_llc = "expr: $VX_CFG_L3_ENABLED == 1" single_core = "expr: $VX_CFG_NUM_CORES == 1 and $VX_CFG_NUM_CLUSTERS == 1" single_cluster = "expr: $VX_CFG_NUM_CLUSTERS == 1" +# The command processor's DMA becomes a client of the last-level cache, which +# is then the point of coherence for host transfers: they need no cache flush. +# Only a last-level cache that alone serves every core can take it (an L2 with +# one cluster, or an L3); VX_CFG_CP_ON_LLC is set where it took effect. +VX_CFG_CP_LLC_CLIENT = 0 +VX_CFG_CP_ON_LLC = "expr: int($VX_CFG_CP_LLC_CLIENT != 0 and (($l2_is_llc and $single_cluster) or $l3_is_llc))" + [l1cache] VX_CFG_ICACHE_SIZE = 16384 VX_CFG_ICACHE_NUM_WAYS = 4 diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 05fed84daa..7e42f65be8 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -56,6 +56,10 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; output wire mmu_flush_done, `endif + // Command-processor DMA, entering as an L2 client when this cluster's L2 + // is the LLC (CP_ON_L2); tied off otherwise. + VX_mem_bus_if.slave cp_mem_if, + // Status output wire busy ); @@ -180,7 +184,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_mem_bus_if #( .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (L2_TAG_WIDTH) - ) per_socket_mem_bus_if[L2_NUM_REQS](); + ) per_socket_mem_bus_if[L2_CACHE_REQS](); // Socket memory output buses (socket arb output tag width) VX_mem_bus_if #( @@ -216,7 +220,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_BANKS (L2_NUM_BANKS), .NUM_WAYS (`VX_CFG_L2_NUM_WAYS), .WORD_SIZE (L2_WORD_SIZE), - .NUM_REQS (L2_NUM_REQS), + .NUM_REQS (L2_CACHE_REQS), .MEM_PORTS (L2_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L2_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L2_MSHR_SIZE), @@ -228,6 +232,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), + .WRITE_AROUND (CP_ON_L2), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), @@ -291,6 +296,12 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ASSIGN_VX_MEM_BUS_IF (mem_bus_if[i], l2_mem_bus_if[i]); end + if (CP_ON_L2 != 0) begin : g_cp_l2_client + `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_CP_IDX], cp_mem_if); + end else begin : g_no_cp_l2_client + `UNUSED_VX_MEM_BUS_IF (cp_mem_if) + end + /////////////////////////////////////////////////////////////////////////// wire [NUM_SOCKETS-1:0] per_socket_busy; diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index c18e3050cc..7acc4b6daf 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -209,8 +209,11 @@ package VX_gpu_pkg; // is_flush at LSB so dcr_flush can drive it deterministically. // is_addr_om is appended ABOVE amo, deliberately: the amo field is cast by // offset (VX_cache_bank), so inserting a bit below it would silently shift - // MEM_ATTR_AMO_OFFS and reinterpret the AMO sideband. + // MEM_ATTR_AMO_OFFS and reinterpret the AMO sideband. is_noalloc sits above + // amo for the same reason: a write that misses a write-around cache goes to + // memory instead of allocating. typedef struct packed { + logic is_noalloc; // MEM_ATTR_NOALLOC_OFFS amo_req_t amo; // MEM_ATTR_AMO_OFFS = 4 logic is_addr_om; // MEM_ATTR_OM_OFFS = 3 logic is_addr_local; // MEM_ATTR_LOCAL_OFFS = 2 @@ -223,6 +226,7 @@ package VX_gpu_pkg; localparam MEM_ATTR_LOCAL_OFFS = 2; localparam MEM_ATTR_OM_OFFS = 3; localparam MEM_ATTR_AMO_OFFS = 4; + localparam MEM_ATTR_NOALLOC_OFFS = MEM_ATTR_AMO_OFFS + AMO_REQ_BITS; // Total width of the mem-bus attr field. Use this as the parameter // default for VX_mem_bus_if's ATTR_WIDTH parameter and as the @@ -1952,6 +1956,19 @@ package VX_gpu_pkg; localparam L2_PTW_IDX = L2_SOCKET_REQS + L2_GFX_REQS; localparam L2_NUM_REQS = L2_SOCKET_REQS + L2_GFX_REQS + L2_PTW_REQS; + + // The command processor's DMA enters the shared LLC on one more client + // slot (VX_CFG_CP_ON_LLC): the cluster L2 when it is the LLC of a single + // cluster, the L3 otherwise. The slot does not size the banks, so + // L2_NUM_REQS / L3_NUM_REQS exclude it and *_CACHE_REQS count it. + localparam CP_ON_L2 = ((`VX_CFG_CP_ON_LLC != 0) + && (`VX_CFG_NUM_CLUSTERS == 1) + && (`VX_CFG_L2_ENABLED != 0) + && (`VX_CFG_L3_ENABLED == 0)) ? 1 : 0; + localparam CP_ON_L3 = ((`VX_CFG_CP_ON_LLC != 0) && (`VX_CFG_L3_ENABLED != 0)) ? 1 : 0; + localparam L2_CP_IDX = L2_NUM_REQS; + localparam L2_CACHE_REQS = L2_NUM_REQS + CP_ON_L2; + localparam L2_NUM_BANKS = `VX_CFG_L2_NUM_BANKS; localparam L2_MEM_PORTS = `VX_CFG_L2_MEM_PORTS; @@ -1974,9 +1991,9 @@ package VX_gpu_pkg; // Memory request tag bits `ifdef VX_CFG_L2_ENABLE - localparam L2_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L2_MSHR_SIZE, L2_NUM_BANKS, L2_NUM_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH, UUID_WIDTH); + localparam L2_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L2_MSHR_SIZE, L2_NUM_BANKS, L2_CACHE_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH, UUID_WIDTH); `else - localparam L2_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L2_NUM_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH); + localparam L2_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L2_CACHE_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH); `endif // L2 is the LLC iff L2 is enabled and L3 is not. @@ -1999,6 +2016,8 @@ package VX_gpu_pkg; localparam L3_PTW_IDX = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS; localparam L3_NUM_REQS = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS + ((`VX_CFG_VM_ENABLED != 0) && (PTW_ON_L2 == 0) ? 1 : 0); + localparam L3_CP_IDX = L3_NUM_REQS; + localparam L3_CACHE_REQS = L3_NUM_REQS + CP_ON_L3; localparam L3_NUM_BANKS = `VX_CFG_L3_NUM_BANKS; localparam L3_MEM_PORTS = `VX_CFG_L3_MEM_PORTS; @@ -2010,14 +2029,21 @@ package VX_gpu_pkg; // Memory request tag bits `ifdef VX_CFG_L3_ENABLE - localparam L3_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L3_MSHR_SIZE, L3_NUM_BANKS, L3_NUM_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH, UUID_WIDTH); + localparam L3_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L3_MSHR_SIZE, L3_NUM_BANKS, L3_CACHE_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH, UUID_WIDTH); `else - localparam L3_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L3_NUM_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH); + localparam L3_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L3_CACHE_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH); `endif // L3 is the LLC whenever it is enabled. localparam L3_IS_LLC = `VX_CFG_L3_ENABLED; localparam L3_SHARED_LLC = L3_IS_LLC; + // CP DMA port into the LLC client slot: one LLC word per request. The + // slot tag carries the port's tag plus a fence marker. + localparam CP_MEM_DATA_SIZE = (CP_ON_L2 != 0) ? L2_WORD_SIZE : L3_WORD_SIZE; + localparam CP_MEM_ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(CP_MEM_DATA_SIZE); + localparam CP_SLOT_TAG_WIDTH = (CP_ON_L2 != 0) ? L2_TAG_WIDTH : L3_TAG_WIDTH; + localparam CP_MEM_TAG_WIDTH = CP_SLOT_TAG_WIDTH - UUID_WIDTH - 1; + /////////////////////////////////////////////////////////////////////////// localparam VX_MEM_PORTS = L3_MEM_PORTS; diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index b3bb5010d4..434d9ab4ab 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -45,6 +45,21 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( output wire dcr_rsp_valid, output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + // Command-processor DMA into the last-level cache (VX_CFG_CP_ON_LLC) + input wire cp_mem_req_valid, + input wire cp_mem_req_rw, + input wire cp_mem_req_noalloc, + input wire [CP_MEM_ADDR_WIDTH-1:0] cp_mem_req_addr, + input wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_req_data, + input wire [CP_MEM_DATA_SIZE-1:0] cp_mem_req_byteen, + input wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_req_tag, + output wire cp_mem_req_ready, + + output wire cp_mem_rsp_valid, + output wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_rsp_data, + output wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_rsp_tag, + input wire cp_mem_rsp_ready, + // ctrl/status input wire start, output wire busy @@ -82,6 +97,9 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( assign dcr_rsp_valid = dcr_bus_if.rsp_valid; assign dcr_rsp_data = dcr_bus_if.rsp_data; + wire kmu_start; + wire cp_mem_busy; + // Kernel Management Unit VX_kmu_bus_if kmu_bus_in[1](); wire kmu_busy; @@ -93,7 +111,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( ) kmu ( .clk (clk), .reset (reset), - .start (start), + .start (kmu_start), .busy (kmu_busy), .dcr_req_valid (dcr_req_valid), .dcr_req_rw (dcr_req_rw), @@ -124,7 +142,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( VX_mem_bus_if #( .DATA_SIZE (L2_SECTOR_SIZE), .TAG_WIDTH (L3_TAG_WIDTH) - ) per_cluster_mem_bus_if[L3_NUM_REQS](); + ) per_cluster_mem_bus_if[L3_CACHE_REQS](); VX_mem_bus_if #( .DATA_SIZE (L3_SECTOR_SIZE), @@ -141,7 +159,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .NUM_BANKS (L3_NUM_BANKS), .NUM_WAYS (`VX_CFG_L3_NUM_WAYS), .WORD_SIZE (L3_WORD_SIZE), - .NUM_REQS (L3_NUM_REQS), + .NUM_REQS (L3_CACHE_REQS), .MEM_PORTS (L3_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L3_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L3_MSHR_SIZE), @@ -153,6 +171,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .WRITEBACK (`VX_CFG_L3_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L3_DIRTYBYTES), .LAZY_FETCH (`VX_CFG_L3_LAZY_FETCH), + .WRITE_AROUND (CP_ON_L3), .REPL_POLICY (`VX_CFG_L3_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), @@ -190,6 +209,68 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( assign mem_rsp_ready[i] = mem_bus_if[i].rsp_ready; end + // Command-processor DMA: one leg per cluster so every instance owns its + // port's drivers; only cluster 0's leg carries traffic, and only under + // CP_ON_L2. Under CP_ON_L3 the DMA is the L3's last client. + VX_mem_bus_if #( + .DATA_SIZE (CP_MEM_DATA_SIZE), + .TAG_WIDTH (CP_SLOT_TAG_WIDTH) + ) per_cluster_cp_mem_if [`VX_CFG_NUM_CLUSTERS] (); + + if ((CP_ON_L2 != 0) || (CP_ON_L3 != 0)) begin : g_cp_mem + VX_mem_bus_if #( + .DATA_SIZE (CP_MEM_DATA_SIZE), + .TAG_WIDTH (CP_SLOT_TAG_WIDTH) + ) cp_mem_if (); + + VX_cp_llc_port #( + .DATA_SIZE (CP_MEM_DATA_SIZE), + .SLOT_TAG_WIDTH (CP_SLOT_TAG_WIDTH), + .TAG_WIDTH (CP_MEM_TAG_WIDTH), + .ADDR_WIDTH (CP_MEM_ADDR_WIDTH) + ) cp_llc_port ( + .clk (clk), + .reset (reset), + .req_valid (cp_mem_req_valid), + .req_rw (cp_mem_req_rw), + .req_noalloc (cp_mem_req_noalloc), + .req_addr (cp_mem_req_addr), + .req_data (cp_mem_req_data), + .req_byteen (cp_mem_req_byteen), + .req_tag (cp_mem_req_tag), + .req_ready (cp_mem_req_ready), + .rsp_valid (cp_mem_rsp_valid), + .rsp_data (cp_mem_rsp_data), + .rsp_tag (cp_mem_rsp_tag), + .rsp_ready (cp_mem_rsp_ready), + .start_in (start), + .start_out (kmu_start), + .busy (cp_mem_busy), + .mem_bus_if (cp_mem_if) + ); + + if (CP_ON_L2 != 0) begin : g_l2 + `ASSIGN_VX_MEM_BUS_IF (per_cluster_cp_mem_if[0], cp_mem_if); + end else begin : g_l3 + `ASSIGN_VX_MEM_BUS_IF (per_cluster_mem_bus_if[L3_CP_IDX], cp_mem_if); + `INIT_VX_MEM_BUS_IF (per_cluster_cp_mem_if[0]) + end + for (genvar c = 1; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_cp_mem_legs + `INIT_VX_MEM_BUS_IF (per_cluster_cp_mem_if[c]) + end + end else begin : g_no_cp_mem + assign cp_mem_req_ready = 1'b0; + assign cp_mem_rsp_valid = 1'b0; + assign cp_mem_rsp_data = '0; + assign cp_mem_rsp_tag = '0; + `UNUSED_VAR ({cp_mem_req_valid, cp_mem_req_rw, cp_mem_req_noalloc, cp_mem_req_addr, cp_mem_req_data, cp_mem_req_byteen, cp_mem_req_tag, cp_mem_rsp_ready}) + assign kmu_start = start; + assign cp_mem_busy = 1'b0; + for (genvar c = 0; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_cp_mem_legs + `INIT_VX_MEM_BUS_IF (per_cluster_cp_mem_if[c]) + end + end + wire [`VX_CFG_NUM_CLUSTERS-1:0] per_cluster_busy; VX_kmu_bus_if per_cluster_kmu_bus_if[`VX_CFG_NUM_CLUSTERS](); @@ -390,6 +471,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .mmu_flush_done (cl_mmu_flush_done[cluster_id]), `endif + .cp_mem_if (per_cluster_cp_mem_if[cluster_id]), + .busy (per_cluster_busy[cluster_id]) ); end @@ -403,7 +486,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( end wire busy_r; `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy) | dev_mmu_busy | ~l3_req_empty, 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); - assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid | (|per_cluster_kmu_valid); + assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid | (|per_cluster_kmu_valid) | cp_mem_busy; `ifdef PERF_ENABLE diff --git a/hw/rtl/Vortex_axi.sv b/hw/rtl/Vortex_axi.sv index 5a159c9d2a..109f2b4898 100644 --- a/hw/rtl/Vortex_axi.sv +++ b/hw/rtl/Vortex_axi.sv @@ -138,6 +138,19 @@ module Vortex_axi import VX_gpu_pkg::*; #( .dcr_rsp_valid (dcr_rsp_valid), .dcr_rsp_data (dcr_rsp_data), + .cp_mem_req_valid (1'b0), + .cp_mem_req_rw (1'b0), + .cp_mem_req_noalloc (1'b0), + .cp_mem_req_addr ('0), + .cp_mem_req_data ('0), + .cp_mem_req_byteen ('0), + .cp_mem_req_tag ('0), + `UNUSED_PIN (cp_mem_req_ready), + `UNUSED_PIN (cp_mem_rsp_valid), + `UNUSED_PIN (cp_mem_rsp_data), + `UNUSED_PIN (cp_mem_rsp_tag), + .cp_mem_rsp_ready (1'b1), + .start (start), .busy (vx_busy) ); diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index 5f21e4842d..bea895dfe0 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -338,7 +338,14 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire cp_interrupt; - VX_cp_core cp_core ( + // With several memory ports the CP's device master enters the + // last-level cache's client slot when the configuration has one; with a + // single port it keeps sharing that port below the caches. + localparam CP_DMA_LLC = ((C_M_AXI_MEM_NUM_BANKS > 1) && ((CP_ON_L2 != 0) || (CP_ON_L3 != 0))) ? 1 : 0; + + VX_cp_core #( + .DMA_LLC_CLIENT (CP_DMA_LLC) + ) cp_core ( .clk (clk), .reset (subsys_reset), .axil_s (cp_axil), @@ -714,7 +721,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( localparam VX_MEM_TAG_A_WIDTH = VX_MEM_TAG_WIDTH + `MAX(SUB_LDATAW, 0); localparam VX_MEM_ADDR_A_WIDTH = VX_MEM_ADDR_WIDTH - SUB_LDATAW; - localparam FAB_NUM_PORTS = VX_MEM_PORTS + 1; + localparam FAB_NUM_PORTS = VX_MEM_PORTS + (CP_DMA_LLC ? 0 : 1); localparam FAB_CP_PORT = VX_MEM_PORTS; localparam FAB_TAG_WIDTH = `MAX(VX_MEM_TAG_A_WIDTH, `VX_CP_AXI_TID_WIDTH); localparam FAB_DATA_SIZE = C_M_AXI_MEM_DATA_WIDTH / 8; @@ -733,6 +740,18 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire [VX_MEM_TAG_WIDTH-1:0] vx_mem_rsp_tag [VX_MEM_PORTS]; wire vx_mem_rsp_ready [VX_MEM_PORTS]; + wire cp_llc_req_valid; + wire cp_llc_req_rw; + wire [CP_MEM_ADDR_WIDTH-1:0] cp_llc_req_addr; + wire [CP_MEM_DATA_SIZE*8-1:0] cp_llc_req_data; + wire [CP_MEM_DATA_SIZE-1:0] cp_llc_req_byteen; + wire [CP_MEM_TAG_WIDTH-1:0] cp_llc_req_tag; + wire cp_llc_req_ready; + wire cp_llc_rsp_valid; + wire [CP_MEM_DATA_SIZE*8-1:0] cp_llc_rsp_data; + wire [CP_MEM_TAG_WIDTH-1:0] cp_llc_rsp_tag; + wire cp_llc_rsp_ready; + Vortex vortex ( `SCOPE_IO_BIND (1) @@ -760,6 +779,19 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .dcr_rsp_valid (dcr_rsp_valid), .dcr_rsp_data (dcr_rsp_data), + .cp_mem_req_valid (cp_llc_req_valid), + .cp_mem_req_rw (cp_llc_req_rw), + .cp_mem_req_noalloc (1'b0), + .cp_mem_req_addr (cp_llc_req_addr), + .cp_mem_req_data (cp_llc_req_data), + .cp_mem_req_byteen (cp_llc_req_byteen), + .cp_mem_req_tag (cp_llc_req_tag), + .cp_mem_req_ready (cp_llc_req_ready), + .cp_mem_rsp_valid (cp_llc_rsp_valid), + .cp_mem_rsp_data (cp_llc_rsp_data), + .cp_mem_rsp_tag (cp_llc_rsp_tag), + .cp_mem_rsp_ready (cp_llc_rsp_ready), + .start (vx_start), .busy (vx_busy) ); @@ -822,11 +854,20 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // The CP's device addresses come from the same host-side allocator as the // pointers handed to the cores -- Device::global_mem_, based at // VX_MEM_USER_BASE_ADDR -- so they are offset-relative already, exactly - // like Vortex's own. They enter the fabric unchanged and - // PLATFORM_MEMORY_OFFSET is applied once, at the bank port, for both - // masters. - wire [CP_LINE_ADDRW-1:0] cp_mem_req_addr; - wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_req_tag; + // like Vortex's own. They enter the fabric (or the last-level cache) + // unchanged and PLATFORM_MEMORY_OFFSET is applied once, at the bank + // port, for both masters. + wire cp_mem_req_valid; + wire cp_mem_req_rw; + wire [CP_LINE_ADDRW-1:0] cp_mem_req_addr; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] cp_mem_req_data; + wire [FAB_DATA_SIZE-1:0] cp_mem_req_byteen; + wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_req_tag; + wire cp_mem_req_ready; + wire cp_mem_rsp_valid; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] cp_mem_rsp_data; + wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_rsp_tag; + wire cp_mem_rsp_ready; VX_membus_from_axi #( .ADDR_W (64), @@ -836,23 +877,59 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .clk (clk), .reset (subsys_reset), .axi_s (cp_axi_dev), - .mem_req_valid (fab_req_valid[FAB_CP_PORT]), - .mem_req_rw (fab_req_rw[FAB_CP_PORT]), + .mem_req_valid (cp_mem_req_valid), + .mem_req_rw (cp_mem_req_rw), .mem_req_addr (cp_mem_req_addr), - .mem_req_data (fab_req_data[FAB_CP_PORT]), - .mem_req_byteen (fab_req_byteen[FAB_CP_PORT]), + .mem_req_data (cp_mem_req_data), + .mem_req_byteen (cp_mem_req_byteen), .mem_req_tag (cp_mem_req_tag), - .mem_req_ready (fab_req_ready[FAB_CP_PORT]), - .mem_rsp_valid (fab_rsp_valid[FAB_CP_PORT]), - .mem_rsp_data (fab_rsp_data[FAB_CP_PORT]), - .mem_rsp_tag (fab_rsp_tag[FAB_CP_PORT][`VX_CP_AXI_TID_WIDTH-1:0]), - .mem_rsp_ready (fab_rsp_ready[FAB_CP_PORT]) + .mem_req_ready (cp_mem_req_ready), + .mem_rsp_valid (cp_mem_rsp_valid), + .mem_rsp_data (cp_mem_rsp_data), + .mem_rsp_tag (cp_mem_rsp_tag), + .mem_rsp_ready (cp_mem_rsp_ready) ); - assign fab_req_addr[FAB_CP_PORT] = cp_mem_req_addr[VX_MEM_ADDR_A_WIDTH-1:0]; - assign fab_req_tag[FAB_CP_PORT] = FAB_TAG_WIDTH'(cp_mem_req_tag); - `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:VX_MEM_ADDR_A_WIDTH]) - `UNUSED_VAR (fab_rsp_tag[FAB_CP_PORT]) + if (CP_DMA_LLC != 0) begin : g_cp_llc + `STATIC_ASSERT(CP_MEM_DATA_SIZE == FAB_DATA_SIZE, ("CP DMA beat must match the last-level cache word")) + `STATIC_ASSERT(CP_MEM_TAG_WIDTH >= `VX_CP_AXI_TID_WIDTH, ("CP DMA id does not fit the last-level cache slot tag")) + assign cp_llc_req_valid = cp_mem_req_valid; + assign cp_llc_req_rw = cp_mem_req_rw; + assign cp_llc_req_addr = cp_mem_req_addr[CP_MEM_ADDR_WIDTH-1:0]; + assign cp_llc_req_data = cp_mem_req_data; + assign cp_llc_req_byteen = cp_mem_req_byteen; + assign cp_llc_req_tag = CP_MEM_TAG_WIDTH'(cp_mem_req_tag); + assign cp_mem_req_ready = cp_llc_req_ready; + assign cp_mem_rsp_valid = cp_llc_rsp_valid; + assign cp_mem_rsp_data = cp_llc_rsp_data; + assign cp_mem_rsp_tag = cp_llc_rsp_tag[`VX_CP_AXI_TID_WIDTH-1:0]; + assign cp_llc_rsp_ready = cp_mem_rsp_ready; + `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:CP_MEM_ADDR_WIDTH]) + `UNUSED_VAR (cp_llc_rsp_tag) + end else begin : g_cp_fabric + assign fab_req_valid[FAB_CP_PORT] = cp_mem_req_valid; + assign fab_req_rw[FAB_CP_PORT] = cp_mem_req_rw; + assign fab_req_addr[FAB_CP_PORT] = cp_mem_req_addr[VX_MEM_ADDR_A_WIDTH-1:0]; + assign fab_req_data[FAB_CP_PORT] = cp_mem_req_data; + assign fab_req_byteen[FAB_CP_PORT] = cp_mem_req_byteen; + assign fab_req_tag[FAB_CP_PORT] = FAB_TAG_WIDTH'(cp_mem_req_tag); + assign cp_mem_req_ready = fab_req_ready[FAB_CP_PORT]; + assign cp_mem_rsp_valid = fab_rsp_valid[FAB_CP_PORT]; + assign cp_mem_rsp_data = fab_rsp_data[FAB_CP_PORT]; + assign cp_mem_rsp_tag = fab_rsp_tag[FAB_CP_PORT][`VX_CP_AXI_TID_WIDTH-1:0]; + assign fab_rsp_ready[FAB_CP_PORT] = cp_mem_rsp_ready; + `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:VX_MEM_ADDR_A_WIDTH]) + `UNUSED_VAR (fab_rsp_tag[FAB_CP_PORT]) + + assign cp_llc_req_valid = 1'b0; + assign cp_llc_req_rw = 1'b0; + assign cp_llc_req_addr = '0; + assign cp_llc_req_data = '0; + assign cp_llc_req_byteen = '0; + assign cp_llc_req_tag = '0; + assign cp_llc_rsp_ready = 1'b1; + `UNUSED_VAR ({cp_llc_req_ready, cp_llc_rsp_valid, cp_llc_rsp_data, cp_llc_rsp_tag}) + end // The optional AXI4 sideband signals (size/burst) are unused by the // bridge's reduced view — pin them sink-side so lint stays clean. diff --git a/hw/rtl/afu/opae/vortex_afu.sv b/hw/rtl/afu/opae/vortex_afu.sv index 61ad02a8e4..5e7ca4cecb 100644 --- a/hw/rtl/afu/opae/vortex_afu.sv +++ b/hw/rtl/afu/opae/vortex_afu.sv @@ -740,6 +740,19 @@ module vortex_afu import ccip_if_pkg::*; import local_mem_cfg_pkg::*; import VX_ .dcr_rsp_valid (vx_dcr_rsp_valid), .dcr_rsp_data (vx_dcr_rsp_data), + .cp_mem_req_valid (1'b0), + .cp_mem_req_rw (1'b0), + .cp_mem_req_noalloc (1'b0), + .cp_mem_req_addr ('0), + .cp_mem_req_data ('0), + .cp_mem_req_byteen ('0), + .cp_mem_req_tag ('0), + `UNUSED_PIN (cp_mem_req_ready), + `UNUSED_PIN (cp_mem_rsp_valid), + `UNUSED_PIN (cp_mem_rsp_data), + `UNUSED_PIN (cp_mem_rsp_tag), + .cp_mem_rsp_ready (1'b1), + .start (vx_start), .busy (vx_busy) ); diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index dce6f187a7..c5e7989c83 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -42,6 +42,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter WRITE_AROUND = 0, // Write misses flagged no-allocate go to memory (write-back only) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -72,6 +74,7 @@ module VX_cache import VX_gpu_pkg::*; #( `STATIC_ASSERT(WRITE_ENABLE || !WRITEBACK, ("invalid parameter: writeback requires write enable")) `STATIC_ASSERT(WRITEBACK || !DIRTY_BYTES, ("invalid parameter: dirty bytes require writeback")) `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) + `STATIC_ASSERT(!WRITE_AROUND || WRITEBACK, ("invalid parameter: write-around requires a write-back cache")) `STATIC_ASSERT(NUM_BANKS >= MEM_PORTS, ("invalid parameter: number of banks must be greater or equal to number of memory ports")) `STATIC_ASSERT(!BANK_HASH || NUM_BANKS <= 256, ("invalid parameter: bank hash supports up to 256 banks")) localparam REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS); @@ -376,6 +379,7 @@ module VX_cache import VX_gpu_pkg::*; #( .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), .LAZY_FETCH (LAZY_FETCH), + .WRITE_AROUND (WRITE_AROUND), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 6137bff9fc..070cdadcdf 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -32,6 +32,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter WRITEBACK = 0, parameter DIRTY_BYTES = 0, parameter LAZY_FETCH = 0, // write miss allocates without a fill; a load to a partial sector refetches + parameter WRITE_AROUND = 0, // a write miss flagged no-allocate is sent to memory instead parameter REPL_POLICY = `CS_REPL_FIFO, parameter TAG_WIDTH = UUID_WIDTH + 1, parameter CORE_OUT_BUF = 0, @@ -105,6 +106,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( localparam CRSQ_QUEUE_SIZE = 1 << `CLOG2(2 + CRSQ_SIZE); `STATIC_ASSERT(LATENCY >= 2, ("invalid parameter: cache bank LATENCY must be >= 2")) `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) + `STATIC_ASSERT(!WRITE_AROUND || (WRITEBACK && WRITE_ENABLE), ("invalid parameter: write-around requires a write-back cache")) `UNUSED_PARAM (MRSQ_SIZE) // ======================================================================== @@ -940,10 +942,10 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); // stD write word delayed to stC for its consumers there (the AMO RMW - // operand and the write-through memory payload); elided when no stC - // consumer exists (write-through banks, or writeback banks with LLC AMO). + // operand and the write-through or write-around memory payload); elided + // when no stC consumer exists. wire [`CS_WORD_WIDTH-1:0] word_stc; - if ((WRITE_ENABLE != 0) && (!WRITEBACK || (AMO_ENABLE != 0 && IS_LLC != 0))) begin : g_word_stc + if ((WRITE_ENABLE != 0) && (!WRITEBACK || (AMO_ENABLE != 0 && IS_LLC != 0) || (WRITE_AROUND != 0))) begin : g_word_stc VX_pipe_register #( .DATAW (`CS_WORD_WIDTH) ) reg_word_stc ( @@ -977,7 +979,10 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // release the entry on a hit. wire mshr_release_st1; if (WRITEBACK) begin : g_mshr_release - assign mshr_release_st1 = st1.lk.is_hit && ~is_amo_fwd_rel; + // a write-around miss keeps nothing: it leaves for memory at stC. + wire write_around_st1 = (WRITE_AROUND != 0) && st1.req.rw && st1.req.attr[MEM_ATTR_NOALLOC_OFFS] + && ~st1.lk.is_hit && ~st1.lk.mshr_pending; + assign mshr_release_st1 = (st1.lk.is_hit || write_around_st1) && ~is_amo_fwd_rel; end else begin : g_mshr_release_ro // keep missed writes in MSHR if a pending entry exists for the line, so a // pending fill arriving without the write content replays them locally. @@ -1331,8 +1336,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mreq_queue_rw; wire is_fill_or_flush_stc = stC.req.is_fill || (stC.req.is_flush && WRITEBACK); + // a no-allocate write miss with no fill pending on its line is sent to + // memory as is: nothing is allocated or fetched. + wire write_around_stc = (WRITE_AROUND != 0) && do_write_stc && stC.req.attr[MEM_ATTR_NOALLOC_OFFS] + && ~stC.lk.is_hit && ~stC.lk.mshr_pending; // a write-miss root under lazy fetch loops its fill back instead. - wire lazy_alloc_stc = (LAZY_FETCH != 0) && do_write_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending; + wire lazy_alloc_stc = (LAZY_FETCH != 0) && do_write_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending + && ~write_around_stc; wire do_fill_or_flush_stc = stC.req.valid && is_fill_or_flush_stc; // a sector refill keeps the resident line (no eviction), so never writes back. wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill && ~refetch_first_stc; @@ -1421,13 +1431,30 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // on a dirty eviction. The two are mutually exclusive (a writeback is a // fill/flush commit; a fill request is a creq commit). wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending - && ~lazy_alloc_stc && ~pipe_stall; - assign mreq_queue_push = fill_req_push || wb_beat_accept; + && ~lazy_alloc_stc && ~write_around_stc && ~pipe_stall; + wire wa_req_push = write_around_stc && ~pipe_stall; + assign mreq_queue_push = fill_req_push || wa_req_push || wb_beat_accept; assign mreq_queue_addr = is_wb_beat ? wb_mreq_addr : rd_mreq_addr; - assign mreq_queue_rw = is_wb_beat; - assign mreq_queue_data = wb_data_sec; // read fill request: data unused - assign mreq_queue_byteen = is_wb_beat ? wb_byteen_sec : {SECTOR_SIZE{1'b1}}; - `UNUSED_VAR ({stC.req.byteen, stC.req.is_replay}) + assign mreq_queue_rw = is_wb_beat || write_around_stc; + if (WRITE_AROUND != 0) begin : g_wa_data + wire [LINE_SIZE-1:0] full_byteen; + VX_demux #( + .DATAW (WORD_SIZE), + .N (`CS_WORDS_PER_LINE) + ) byteen_demux ( + .sel_in (stC.req.word_idx), + .data_in (stC.req.byteen), + .data_out (full_byteen) + ); + wire [SECTOR_SIZE-1:0] sec_byteen = full_byteen[sector_idx_stc*SECTOR_SIZE +: SECTOR_SIZE]; + assign mreq_queue_data = is_wb_beat ? wb_data_sec : {`CS_WORDS_PER_SECTOR{word_stc}}; + assign mreq_queue_byteen = is_wb_beat ? wb_byteen_sec : (write_around_stc ? sec_byteen : {SECTOR_SIZE{1'b1}}); + end else begin : g_no_wa_data + assign mreq_queue_data = wb_data_sec; // read fill request: data unused + assign mreq_queue_byteen = is_wb_beat ? wb_byteen_sec : {SECTOR_SIZE{1'b1}}; + `UNUSED_VAR (stC.req.byteen) + end + `UNUSED_VAR (stC.req.is_replay) end else begin : g_wt // word byte-enable demuxed over the line, then sliced to the word's sector. wire [LINE_SIZE-1:0] full_byteen; @@ -1449,7 +1476,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign mreq_queue_rw = stC.req.rw; assign mreq_queue_data = {`CS_WORDS_PER_SECTOR{word_stc}}; assign mreq_queue_byteen = (stC.req.rw || is_amo_fwd_st1) ? sec_byteen : {SECTOR_SIZE{1'b1}}; - `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, evict_addr_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask}) + `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, evict_addr_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask, write_around_stc}) end end else begin : g_mreq_queue_ro assign mreq_queue_push = (do_read_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending) && ~pipe_stall; @@ -1457,7 +1484,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign mreq_queue_rw = 0; assign mreq_queue_data = '0; assign mreq_queue_byteen = {SECTOR_SIZE{1'b1}}; - `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, do_writeback_stc, evict_addr_stc, evict_byteen_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask, stC.req.byteen, stC.req.word_idx, stC.req.is_replay, do_write_stc}) + `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, do_writeback_stc, evict_addr_stc, evict_byteen_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask, stC.req.byteen, stC.req.word_idx, stC.req.is_replay, do_write_stc, write_around_stc}) end if (UUID_WIDTH != 0) begin : g_mreq_queue_tag_uuid diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 2471155263..2f04c76436 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -44,6 +44,8 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter WRITE_AROUND = 0, // Write misses flagged no-allocate go to memory (write-back only) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -148,6 +150,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), .LAZY_FETCH (LAZY_FETCH), + .WRITE_AROUND (WRITE_AROUND), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 56aeda0455..eebfad60a4 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -104,6 +104,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire [MEM_ADDRW-1:0] io_addr_start = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >> MEM_ASHIFT); wire [MEM_ADDRW-1:0] io_addr_end = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) >> MEM_ASHIFT); assign mem_req_attr_struct[i].is_flush = req_is_fence; + assign mem_req_attr_struct[i].is_noalloc = 1'b0; `ifdef VX_CFG_EXT_OM_ENABLE // OM fragment-export aperture. // diff --git a/hw/rtl/cp/VX_cp_axil_regfile.sv b/hw/rtl/cp/VX_cp_axil_regfile.sv index 806a996da9..531e5e064d 100644 --- a/hw/rtl/cp/VX_cp_axil_regfile.sv +++ b/hw/rtl/cp/VX_cp_axil_regfile.sv @@ -15,7 +15,7 @@ // 0x000 CP_CTRL RW bit0=enable_global, bit1=reset_all // 0x004 CP_STATUS RO bit0=busy, bit1=error // 0x008 CP_DEV_CAPS RO [7:0]NUM_QUEUES | [15:8]RING_SIZE_LOG2_MAX -// [23:16]AXI_TID_WIDTH +// [23:16]AXI_TID_WIDTH | [28]DMA_LLC_CLIENT // 0x010 CP_CYCLE_LO RO free-running cycle counter low 32 bits // 0x014 CP_CYCLE_HI RO high 32 bits // 0x018 GPU_DEV_CAPS_LO RO device-config caps, low 32 bits @@ -51,7 +51,9 @@ module VX_cp_axil_regfile parameter int ADDR_W = 16, // Static device-caps fields (set at synthesis time from VX_cp_pkg). parameter int RING_SIZE_LOG2_MAX = VX_CP_RING_SIZE_LOG2_C, - parameter int AXI_TID_W = VX_CP_AXI_TID_WIDTH_C + parameter int AXI_TID_W = VX_CP_AXI_TID_WIDTH_C, + // The device-side DMA reaches memory through the last-level cache. + parameter int DMA_LLC_CLIENT = 0 )( input wire clk, input wire reset, @@ -204,7 +206,9 @@ module VX_cp_axil_regfile case (addr[5:0]) 6'h00: return r_cp_ctrl; 6'h04: return {30'd0, cp_error, cp_busy}; - 6'h08: return {8'd0, + 6'h08: return {3'd0, + 1'(DMA_LLC_CLIENT), + 4'd0, 8'(AXI_TID_W), 8'(RING_SIZE_LOG2_MAX), 8'(NUM_QUEUES)}; diff --git a/hw/rtl/cp/VX_cp_core.sv b/hw/rtl/cp/VX_cp_core.sv index ee0849b4e7..417a3584f9 100644 --- a/hw/rtl/cp/VX_cp_core.sv +++ b/hw/rtl/cp/VX_cp_core.sv @@ -81,7 +81,9 @@ module VX_cp_core parameter int ADDR_W = 64, parameter int DATA_W = 512, parameter int ID_W = VX_CP_AXI_TID_WIDTH_C, - parameter int AXIL_AW = 16 + parameter int AXIL_AW = 16, + // axi_dev enters the last-level cache's client slot instead of memory. + parameter int DMA_LLC_CLIENT = 0 )( input wire clk, input wire reset, @@ -148,8 +150,9 @@ module VX_cp_core logic engine_idle [NUM_QUEUES]; VX_cp_axil_regfile #( - .NUM_QUEUES (NUM_QUEUES), - .ADDR_W (AXIL_AW) + .NUM_QUEUES (NUM_QUEUES), + .ADDR_W (AXIL_AW), + .DMA_LLC_CLIENT (DMA_LLC_CLIENT) ) u_regfile ( .clk (clk), .reset (reset), diff --git a/hw/rtl/mem/VX_cp_llc_port.sv b/hw/rtl/mem/VX_cp_llc_port.sv new file mode 100644 index 0000000000..5542758f62 --- /dev/null +++ b/hw/rtl/mem/VX_cp_llc_port.sv @@ -0,0 +1,154 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Command-processor DMA port into the last-level cache's client slot. +// Requests in the IO aperture carry the IO attribute and bypass the cache; +// a write flagged no-allocate writes around it on a miss. +// A launch that follows DMA writes is held until those writes are inside the +// cache banks: the port sends an L1-only flush token behind them, which the +// last-level cache answers once its input crossbar has drained, and only then +// passes the start pulse on. Reads need no such fence, since the DMA waits for +// their data. +module VX_cp_llc_port import VX_gpu_pkg::*; #( + parameter DATA_SIZE = 64, + parameter SLOT_TAG_WIDTH = UUID_WIDTH + 2, + parameter TAG_WIDTH = SLOT_TAG_WIDTH - UUID_WIDTH - 1, + parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE) +) ( + input wire clk, + input wire reset, + + input wire req_valid, + input wire req_rw, + input wire req_noalloc, + input wire [ADDR_WIDTH-1:0] req_addr, + input wire [DATA_SIZE*8-1:0] req_data, + input wire [DATA_SIZE-1:0] req_byteen, + input wire [TAG_WIDTH-1:0] req_tag, + output wire req_ready, + + output wire rsp_valid, + output wire [DATA_SIZE*8-1:0] rsp_data, + output wire [TAG_WIDTH-1:0] rsp_tag, + input wire rsp_ready, + + input wire start_in, + output wire start_out, + output wire busy, + + VX_mem_bus_if.master mem_bus_if +); + `STATIC_ASSERT(TAG_WIDTH >= 1, ("invalid parameter: TAG_WIDTH=%0d", TAG_WIDTH)) + `STATIC_ASSERT(0 == (`VX_MEM_IO_BASE_ADDR % DATA_SIZE), ("invalid parameter")) + + localparam ASHIFT = `CLOG2(DATA_SIZE); + localparam REQ_DATAW = 1 + ADDR_WIDTH + DATA_SIZE * 8 + DATA_SIZE + `UP(MEM_ATTR_WIDTH) + SLOT_TAG_WIDTH; + + localparam STATE_IDLE = 0; + localparam STATE_FENCE = 1; + localparam STATE_WAIT = 2; + + reg [1:0] state; + reg wr_pending; + + wire fence_rsp = mem_bus_if.rsp_valid && mem_bus_if.rsp_data.tag.value[0]; + + wire [ADDR_WIDTH-1:0] io_addr_start = ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >> ASHIFT); + wire [ADDR_WIDTH-1:0] io_addr_end = ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) >> ASHIFT); + + mem_bus_attr_t req_attr; + always @(*) begin + req_attr = '0; + req_attr.is_addr_io = (req_addr >= io_addr_start) && (req_addr < io_addr_end); + req_attr.is_noalloc = req_rw && req_noalloc; + end + + wire [`UP(MEM_ATTR_WIDTH)-1:0] fence_attr = `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS); + + wire [SLOT_TAG_WIDTH-1:0] req_slot_tag = SLOT_TAG_WIDTH'({req_tag, 1'b0}); + wire [SLOT_TAG_WIDTH-1:0] fence_slot_tag = SLOT_TAG_WIDTH'(1); + + wire fence_req = (state == STATE_FENCE); + + wire buf_valid_in = fence_req || ((state == STATE_IDLE) && req_valid); + wire buf_ready_in; + wire [REQ_DATAW-1:0] buf_data_in = fence_req ? + {1'b0, ADDR_WIDTH'(0), (DATA_SIZE*8)'(0), DATA_SIZE'(1), fence_attr, fence_slot_tag} : + {req_rw, req_addr, req_data, req_byteen, `UP(MEM_ATTR_WIDTH)'(req_attr), req_slot_tag}; + + assign req_ready = (state == STATE_IDLE) && buf_ready_in; + + wire req_fire = req_valid && req_ready; + wire fence_fire = fence_req && buf_ready_in; + + VX_elastic_buffer #( + .DATAW (REQ_DATAW), + .SIZE (1) + ) req_buf ( + .clk (clk), + .reset (reset), + .valid_in (buf_valid_in), + .ready_in (buf_ready_in), + .data_in (buf_data_in), + .data_out ({mem_bus_if.req_data.rw, + mem_bus_if.req_data.addr, + mem_bus_if.req_data.data, + mem_bus_if.req_data.byteen, + mem_bus_if.req_data.attr, + mem_bus_if.req_data.tag}), + .ready_out (mem_bus_if.req_ready), + .valid_out (mem_bus_if.req_valid) + ); + + always @(posedge clk) begin + if (reset) begin + state <= STATE_IDLE; + wr_pending <= 1'b0; + end else begin + case (state) + STATE_FENCE: begin + if (fence_fire) begin + state <= STATE_WAIT; + end + end + STATE_WAIT: begin + if (fence_rsp) begin + state <= STATE_IDLE; + wr_pending <= 1'b0; + end + end + default: begin + if (start_in && wr_pending) begin + state <= STATE_FENCE; + end else if (req_fire && req_rw) begin + wr_pending <= 1'b1; + end + end + endcase + end + end + + assign start_out = ((state == STATE_IDLE) && start_in && ~wr_pending) + || ((state == STATE_WAIT) && fence_rsp); + assign busy = (state != STATE_IDLE); + + assign rsp_valid = mem_bus_if.rsp_valid && ~fence_rsp; + assign rsp_data = mem_bus_if.rsp_data.data; + assign rsp_tag = mem_bus_if.rsp_data.tag.value[1 +: TAG_WIDTH]; + assign mem_bus_if.rsp_ready = fence_rsp || rsp_ready; + + `RUNTIME_ASSERT (~(start_in && (state != STATE_IDLE)), ("%t: launch while a DMA fence is pending", $time)) + +endmodule diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 2ace286517..0f5d3a80f2 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -110,7 +110,8 @@ uint32_t CommandProcessor::mmio_read(uint32_t off) const { case 0x000: return cp_ctrl_; case 0x004: return uint32_t(busy() ? 1 : 0); // CP_STATUS bit0 case 0x008: { - // CP_DEV_CAPS: {SUPPORTS_QMD:1 @bit26 | SUPPORTS_DRAW:1 @bit25 | + // CP_DEV_CAPS: {DMA_LLC_CLIENT:1 @bit28 | MMU_FAULT_REPORT:1 @bit27 | + // SUPPORTS_QMD:1 @bit26 | SUPPORTS_DRAW:1 @bit25 | // VM_ENABLED:1 @bit24 | AXI_TID_W:8 | RING_LOG2:8 | NUM_QUEUES:8}. // Defaults: TID=6, RING_LOG2=16, NUM_QUEUES=1. VM_ENABLED reflects // the build-config so the config-agnostic libvortex.so can @@ -125,7 +126,8 @@ uint32_t CommandProcessor::mmio_read(uint32_t off) const { const uint32_t supports_draw = 1u << 25; const uint32_t supports_qmd = 1u << 26; const uint32_t fault_report = hooks_.mmu_fault_report ? (1u << 27) : 0; - return supports_qmd | supports_draw | vm_enabled | fault_report + const uint32_t llc_client = hooks_.dma_llc_client ? (1u << 28) : 0; + return llc_client | supports_qmd | supports_draw | vm_enabled | fault_report | (uint32_t(6) << 16) | (uint32_t(16) << 8) | uint32_t(1); } case 0x010: return uint32_t(cycle_counter_ & 0xFFFFFFFF); @@ -547,6 +549,8 @@ void CommandProcessor::tick_engine() { src = cp_translate(src, physical); } const uint64_t total = cur_cmd_.arg2; + const bool no_alloc = hooks_.dma_no_alloc_min != 0 + && total >= hooks_.dma_no_alloc_min; constexpr uint64_t CHUNK = 64 * 1024; std::vector buf( std::size_t(total < CHUNK ? total : CHUNK)); @@ -554,7 +558,11 @@ void CommandProcessor::tick_engine() { uint64_t n = total - done; if (n > CHUNK) n = CHUNK; hooks_.dram_read (src + done, buf.data(), n); - hooks_.dram_write(dst + done, buf.data(), n); + if (hooks_.dma_write) { + hooks_.dma_write(dst + done, buf.data(), n, no_alloc); + } else { + hooks_.dram_write(dst + done, buf.data(), n); + } done += n; } } diff --git a/sim/common/cmd_processor.h b/sim/common/cmd_processor.h index 956d786e03..f249b89ca5 100644 --- a/sim/common/cmd_processor.h +++ b/sim/common/cmd_processor.h @@ -107,6 +107,17 @@ class CommandProcessor { // so the runtime does not read them on a target that would leave // the DCR bus waiting for a response. bool mmu_fault_report = false; + + // Write `bytes` bytes from `src` to `addr` for a CMD_MEM_* transfer. + // `no_alloc` marks a transfer of at least dma_no_alloc_min bytes, + // whose writes should not allocate in the device caches. Unset: + // dram_write. + std::function dma_write; + uint64_t dma_no_alloc_min = 0; + + // Device-memory accesses go through the last-level cache, so host + // transfers need no cache flush. Published in DEV_CAPS. + bool dma_llc_client = false; }; explicit CommandProcessor(const Hooks& hooks); diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index eaef68cc0e..92724c8923 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -29,7 +29,9 @@ #endif +#include #include +#include #include #include #include @@ -39,6 +41,7 @@ #include #include +#include #include #include #include @@ -72,6 +75,15 @@ typedef uint64_t Word; #define VL_WDATA_GETW(lwp, i, n, w) \ VL_SEL_IWII(0, n * w, 0, 0, lwp, i * w, w) +// Command-processor DMA port into the last-level cache: one cache word per +// request, and as many reads in flight as one 4 KB DMA burst holds. +#if VX_CFG_L3_ENABLED +#define CP_MEM_DATA_SIZE VX_CFG_L2_SECTOR_SIZE +#else +#define CP_MEM_DATA_SIZE VX_CFG_L1_LINE_SIZE +#endif +#define CP_MEM_RD_SLOTS (4096 / CP_MEM_DATA_SIZE) + using namespace vortex; static uint64_t g_mem_done_seq = 0; @@ -253,11 +265,66 @@ class Processor::Impl { } } + void cp_mem_write(uint64_t addr, const uint8_t* src, uint64_t size, bool noalloc) { + while (size != 0) { + uint64_t offset = addr % CP_MEM_DATA_SIZE; + uint64_t n = std::min(size, CP_MEM_DATA_SIZE - offset); + cp_req_t req; + req.rw = true; + req.noalloc = noalloc; + req.addr = addr / CP_MEM_DATA_SIZE; + req.byteen = 0; + req.data.fill(0); + for (uint64_t i = 0; i < n; ++i) { + req.data[offset + i] = src[i]; + req.byteen |= uint64_t(1) << (offset + i); + } + req.tag = 0; + cp_reqs_.push_back(req); + addr += n; + src += n; + size -= n; + } + while (!cp_reqs_.empty()) { + this->tick(); + } + } + + void cp_mem_read(uint64_t addr, uint8_t* dst, uint64_t size) { + uint32_t inflight = 0; + while (size != 0 || inflight != 0) { + // two queued requests keep one read entering the port every cycle + while (size != 0 && cp_reqs_.size() < 2 && !cp_free_slots_.empty()) { + uint64_t offset = addr % CP_MEM_DATA_SIZE; + uint64_t n = std::min(size, CP_MEM_DATA_SIZE - offset); + uint32_t slot = cp_free_slots_.back(); + cp_free_slots_.pop_back(); + cp_rd_slots_[slot] = {dst, uint32_t(offset), uint32_t(n)}; + cp_req_t req; + req.rw = false; + req.noalloc = false; + req.addr = addr / CP_MEM_DATA_SIZE; + req.byteen = ~uint64_t(0); + req.data.fill(0); + req.tag = slot; + cp_reqs_.push_back(req); + ++inflight; + addr += n; + dst += n; + size -= n; + } + this->tick(); + inflight -= cp_rd_completed_; + cp_rd_completed_ = 0; + } + } + private: void reset() { this->mem_bus_reset(); this->dcr_bus_reset(); + this->cp_mem_reset(); print_bufs_.clear(); @@ -301,11 +368,13 @@ class Processor::Impl { this->eval(); this->mem_bus_eval(0); + this->cp_mem_eval(0); device_->clk = 1; this->eval(); this->mem_bus_eval(1); + this->cp_mem_eval(1); dram_sim_.tick(); @@ -447,6 +516,51 @@ class Processor::Impl { device_->dcr_req_rw = 0; } + void cp_mem_reset() { + cp_reqs_.clear(); + cp_free_slots_.clear(); + for (uint32_t i = 0; i < CP_MEM_RD_SLOTS; ++i) { + cp_free_slots_.push_back(CP_MEM_RD_SLOTS - 1 - i); + } + cp_req_fire_ = false; + cp_rd_completed_ = 0; + device_->cp_mem_req_valid = 0; + device_->cp_mem_rsp_ready = 1; + } + + // Inputs change after the rising edge; the handshakes they take part in + // are sampled after the falling-edge evaluation, ahead of the next edge. + void cp_mem_eval(bool clk) { + if (!clk) { + cp_req_fire_ = device_->cp_mem_req_valid && device_->cp_mem_req_ready; + if (device_->cp_mem_rsp_valid) { + uint32_t slot = device_->cp_mem_rsp_tag; + auto& rd = cp_rd_slots_.at(slot); + auto data = VDataCast::get(device_->cp_mem_rsp_data); + std::memcpy(rd.dst, data + rd.offset, rd.size); + cp_free_slots_.push_back(slot); + ++cp_rd_completed_; + } + return; + } + if (cp_req_fire_) { + cp_reqs_.pop_front(); + cp_req_fire_ = false; + } + if (cp_reqs_.empty()) { + device_->cp_mem_req_valid = 0; + return; + } + auto& req = cp_reqs_.front(); + device_->cp_mem_req_valid = 1; + device_->cp_mem_req_rw = req.rw; + device_->cp_mem_req_noalloc = req.noalloc; + device_->cp_mem_req_addr = req.addr; + device_->cp_mem_req_byteen = req.byteen; + device_->cp_mem_req_tag = req.tag; + memcpy(VDataCast::get(device_->cp_mem_req_data), req.data.data(), CP_MEM_DATA_SIZE); + } + void wait(uint32_t cycles) { for (int i = 0; i < cycles; ++i) { this->tick(); @@ -468,6 +582,27 @@ class Processor::Impl { std::unordered_map print_bufs_; + typedef struct { + std::array data; + uint64_t addr; + uint64_t byteen; + uint32_t tag; + bool rw; + bool noalloc; + } cp_req_t; + + typedef struct { + uint8_t* dst; + uint32_t offset; + uint32_t size; + } cp_rd_t; + + std::deque cp_reqs_; + std::array cp_rd_slots_; + std::vector cp_free_slots_; + uint32_t cp_rd_completed_; + bool cp_req_fire_; + std::list pending_mem_reqs_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; std::queue dram_queue_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; @@ -515,6 +650,14 @@ int Processor::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { return impl_->dcr_read(addr, tag, value); } +void Processor::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { + impl_->cp_mem_write(addr, static_cast(src), size, noalloc); +} + +void Processor::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { + impl_->cp_mem_read(addr, static_cast(dst), size); +} + void Processor::drain() { impl_->drain(); } \ No newline at end of file diff --git a/sim/rtlsim/processor.h b/sim/rtlsim/processor.h index c6ae7ae464..9fe3c2f68e 100644 --- a/sim/rtlsim/processor.h +++ b/sim/rtlsim/processor.h @@ -44,6 +44,13 @@ class Processor { // write it issued. void drain(); + // Device-memory access through the command processor's port into the + // last-level cache. A write returns once every request entered the port; + // the device orders it ahead of the next launch. A read returns with data. + void cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc); + + void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + private: class Impl; diff --git a/sim/rtlsim/rtlsim_shim.sv b/sim/rtlsim/rtlsim_shim.sv index bde1d6501f..9e214e1c90 100644 --- a/sim/rtlsim/rtlsim_shim.sv +++ b/sim/rtlsim/rtlsim_shim.sv @@ -50,6 +50,21 @@ module rtlsim_shim import VX_gpu_pkg::*; #( output wire dcr_rsp_valid, output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + // Command-processor DMA into the last-level cache + input wire cp_mem_req_valid, + input wire cp_mem_req_rw, + input wire cp_mem_req_noalloc, + input wire [CP_MEM_ADDR_WIDTH-1:0] cp_mem_req_addr, + input wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_req_data, + input wire [CP_MEM_DATA_SIZE-1:0] cp_mem_req_byteen, + input wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_req_tag, + output wire cp_mem_req_ready, + + output wire cp_mem_rsp_valid, + output wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_rsp_data, + output wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_rsp_tag, + input wire cp_mem_rsp_ready, + // ctrl/status input wire start, output wire busy @@ -104,6 +119,19 @@ module rtlsim_shim import VX_gpu_pkg::*; #( .dcr_rsp_valid (dcr_rsp_valid), .dcr_rsp_data (dcr_rsp_data), + .cp_mem_req_valid (cp_mem_req_valid), + .cp_mem_req_rw (cp_mem_req_rw), + .cp_mem_req_noalloc (cp_mem_req_noalloc), + .cp_mem_req_addr (cp_mem_req_addr), + .cp_mem_req_data (cp_mem_req_data), + .cp_mem_req_byteen (cp_mem_req_byteen), + .cp_mem_req_tag (cp_mem_req_tag), + .cp_mem_req_ready (cp_mem_req_ready), + .cp_mem_rsp_valid (cp_mem_rsp_valid), + .cp_mem_rsp_data (cp_mem_rsp_data), + .cp_mem_rsp_tag (cp_mem_rsp_tag), + .cp_mem_rsp_ready (cp_mem_rsp_ready), + .start (start), .busy (vx_busy) ); diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index c8d25999af..a32f18e9c2 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -363,6 +363,9 @@ vx_result_t Device::cp_init() { // Reading them on a target that does not decode them stalls the DCR // bus waiting for a response that never arrives. mmu_fault_report_ = (dev_caps & (1u << 27)) != 0; + // DMA_LLC_CLIENT (bit 28): the CP DMA is a client of the last-level + // cache, which is then the point of coherence for host transfers. + cp_dma_llc_client_ = (dev_caps & (1u << 28)) != 0; } // Resolve the core count now, while nothing holds cp_mu_. Deferring it to @@ -552,9 +555,10 @@ vx_result_t Device::cp_ring_append_(const void* cl) { // Caller holds cp_mu_. A kernel's results are flushed lazily, by the // next command that needs them: the next launch gets an L1-only flush // (the last-level cache is the point of coherence between kernels), and - // the next transfer to or from device memory a full one, since the CP - // DMA reaches device memory beneath the last-level cache. Appending the - // flush under the same lock keeps any other command from slipping between. + // the next transfer to or from device memory a full one when the CP DMA + // reaches device memory beneath the last-level cache, an L1-only one when + // it is a client of that cache. Appending the flush under the same lock + // keeps any other command from slipping between. const uint8_t* b = static_cast(cl); const bool dma = cp_dma_touches_device_mem(b); const bool launch = (b[0] == CP_OPCODE_LAUNCH) @@ -562,11 +566,12 @@ vx_result_t Device::cp_ring_append_(const void* cl) { || (b[0] == CP_OPCODE_DRAW); if ((dma && (l1_dirty_ || llc_dirty_)) || (launch && l1_dirty_)) { uint8_t flush[CP_CL_BYTES]; - build_cache_flush(flush, cp_num_cores_, /*l1_only=*/!dma); + const bool full = dma && llc_dirty_; + build_cache_flush(flush, cp_num_cores_, /*l1_only=*/!full); auto r = cp_ring_write_(flush); if (r != VX_SUCCESS) return r; l1_dirty_ = false; - llc_dirty_ = llc_dirty_ && !dma; + llc_dirty_ = llc_dirty_ && !full; } assert(!(dma && (l1_dirty_ || llc_dirty_))); auto r = cp_ring_write_(cl); @@ -574,7 +579,7 @@ vx_result_t Device::cp_ring_append_(const void* cl) { if (launch) { // a draw descriptor ends on its own L1-only flush step l1_dirty_ = (b[0] != CP_OPCODE_DRAW); - llc_dirty_ = true; + llc_dirty_ = !cp_dma_llc_client_; } else if (b[0] == CP_OPCODE_CACHE_FLUSH) { l1_dirty_ = false; llc_dirty_ = llc_dirty_ && (b[1] & CP_FLUSH_FLAG_L1_ONLY); diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index 23c688ad63..724449d825 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -387,8 +387,8 @@ class Device : public RefCounted { // the non-batch path, or via cp_batch_begin). No doorbell, no poll. // Inserts the lazy kernel-boundary CMD_CACHE_FLUSH (AMD ACQUIRE_MEM // model) under the same lock: L1-only ahead of a launch while - // l1_dirty_, full ahead of a CMD_MEM_* touching device memory while - // either flag is set. + // l1_dirty_, and ahead of a CMD_MEM_* touching device memory while + // either flag is set -- full while llc_dirty_. vx_result_t cp_ring_append_(const void* cl); vx_result_t cp_ring_write_(const void* cl); @@ -460,7 +460,7 @@ class Device : public RefCounted { // stale lines. bool l1_dirty_ = false; // A kernel ran since the last full cache flush, so the last-level cache - // may hold lines the CP DMA (which reaches device memory beneath it) + // may hold lines the CP DMA (when it reaches device memory beneath it) // would miss or later overwrite. Both flags are guarded by cp_mu_. bool llc_dirty_ = false; std::mutex cp_mu_; // serialize ring writes @@ -492,6 +492,9 @@ class Device : public RefCounted { // the OP_DRAW mirror). Discovered at open. bool cp_supports_draw_ = false; bool cp_supports_qmd_ = false; + // The CP DMA reaches device memory through the last-level cache (CP + // DEV_CAPS bit 28), so transfers never need that cache flushed. + bool cp_dma_llc_client_ = false; class CpMemIO; std::unique_ptr vm_io_; std::unique_ptr vm_mgr_; diff --git a/sw/runtime/rtlsim/vortex.cpp b/sw/runtime/rtlsim/vortex.cpp index 8d7f24e96f..0376f03e77 100644 --- a/sw/runtime/rtlsim/vortex.cpp +++ b/sw/runtime/rtlsim/vortex.cpp @@ -23,6 +23,7 @@ // dram hooks dereference it directly (the sim runs in-process). // ============================================================================ +#include #include #include @@ -143,15 +144,46 @@ class vx_device { std::memcpy(dst, hp, bytes); return; } + #if VX_CFG_CP_ON_LLC + if (future_.valid()) future_.wait(); + processor_.cp_mem_read(addr, dst, bytes); + #else ram_.read(static_cast(dst), addr, bytes); + #endif }; h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { if (void* hp = host_region_ptr(addr)) { std::memcpy(hp, src, bytes); return; } + #if VX_CFG_CP_ON_LLC + if (future_.valid()) future_.wait(); + processor_.cp_mem_write(addr, src, bytes, false); + #else ram_.write(static_cast(src), addr, bytes); + #endif }; + #if VX_CFG_CP_ON_LLC + // The CP DMA is a client of the last-level cache: transfers stay coherent + // with it, and those of at least half its size write around it. + h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + if (future_.valid()) future_.wait(); + processor_.cp_mem_write(addr, src, bytes, no_alloc); + }; + h.dma_llc_client = true; + #if VX_CFG_L3_ENABLED + h.dma_no_alloc_min = VX_CFG_L3_SIZE / 2; + #else + h.dma_no_alloc_min = VX_CFG_L2_SIZE / 2; + #endif + if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { + h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + } + #endif h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { // Wait for any background processor_.run() to finish so dcr_write // does not race the Verilator state. From 1de9126e6ba1dbe40f19b78a3ca4e4a9538df5b7 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 02:47:48 -0700 Subject: [PATCH 049/336] Check a victim's dirty bytes only when the victim way is valid With lazy fetch on, the dirty-byte mask is one RAM indexed by way and line. The reset walk leaves it alone and every install clears its line's mask, so an invalid way's mask is undefined until its first install. In simulation the banks can commit random operations for a few cycles while reset propagates, which may write the RAM. The commit-stage check that a clean victim carries no dirty bytes then fired on an invalid victim, at 2 clusters + L3 with L3 lazy fetch on every app, and at 4 cores + L2 with L2 lazy fetch once the CP joined the LLC. Hardware never uses those bytes: write-backs take bytes only from dirty sectors, and merges only read lines installed since reset. The tags now report whether the victim way holds a valid sector, and the check applies only to valid victims. The flag reaches the commit stage through a simulation-only register; synthesis drops it. The check that a dirty sector carries bytes is unchanged. rtlsim with lazy fetch on, at 2 clusters + L3 and at 4 cores + L2: sgemm, raycast, dogfood, multikernel, flush_coherence and mstress pass, as do the lazy_fetch, mstress and flush_coherence CI cases. The unmodified bank fails multikernel and dogfood at 2 clusters + L3 on the check. Lazy fetch off is cycle-identical. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_bank.sv | 22 ++++++++++++++++++++-- hw/rtl/cache/VX_cache_tags.sv | 4 ++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 070cdadcdf..4187d85858 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -557,7 +557,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0] tag_matches_st0; wire [NUM_WAYS-1:0] line_present_st0; wire [`CS_WAY_SEL_WIDTH-1:0] hit_idx_st0; - wire evict_dirty_st0; + wire evict_dirty_st0, evict_valid_st0; + `UNUSED_VAR (evict_valid_st0) wire [`CS_SECTORS_PER_LINE-1:0] evict_dirty_mask_st0; wire [`CS_TAG_SEL_BITS-1:0] evict_tag_st0; @@ -660,6 +661,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .tag_matches (tag_matches_st0), .line_present (line_present_st0), .evict_dirty (evict_dirty_st0), + .evict_valid (evict_valid_st0), .evict_dirty_mask (evict_dirty_mask_st0), .evict_tag (evict_tag_st0), .partial_match (partial_match_st0) @@ -1419,7 +1421,23 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire has_dirty_bytes = (| evict_byteen_stc); // a refetch evicts nothing: the mask it reads is its resident way's wire check_evict_bytes = do_fill_or_flush_stc && ~refetch_first_stc; - `RUNTIME_ASSERT (~check_evict_bytes || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + `ifndef SYNTHESIS + // An invalid way's mask is undefined: the reset walk leaves the + // mask alone and every install clears its line's mask, so only a + // valid victim's bytes must agree with its dirty flag. + wire evict_valid_stc; + VX_pipe_register #( + .DATAW (1), + .DEPTH (1 + PIPE_EX) + ) reg_evict_valid ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (evict_valid_st0), + .data_out (evict_valid_stc) + ); + `RUNTIME_ASSERT (~(check_evict_bytes && evict_valid_stc) || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + `endif // per-sector: a dirty sector must carry dirty bytes, or its writeback beat is dropped. wire [SEC-1:0] sec_has_dirty_bytes; for (genvar s = 0; s < SEC; ++s) begin : g_sec_has_dirty_bytes diff --git a/hw/rtl/cache/VX_cache_tags.sv b/hw/rtl/cache/VX_cache_tags.sv index 5ecfc5c0be..62433812f8 100644 --- a/hw/rtl/cache/VX_cache_tags.sv +++ b/hw/rtl/cache/VX_cache_tags.sv @@ -76,6 +76,8 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // writeback); a single bit == evict_dirty when 1 sector/line. output wire [`CS_SECTORS_PER_LINE-1:0] evict_dirty_mask, output wire [`CS_TAG_SEL_BITS-1:0] evict_tag, + // the evict way holds a valid sector + output wire evict_valid, // the requested sector is resident but partial (a load/AMO to it refetches) output wire partial_match ); @@ -92,6 +94,8 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][SEC-1:0] read_partial; wire [NUM_WAYS-1:0] partial_hit; + assign evict_valid = (| read_valid[evict_way]); + if (WRITEBACK) begin : g_evict_tag_wb assign evict_dirty = (| read_dirty[evict_way]); // dirty if any sector dirty assign evict_dirty_mask = read_dirty[evict_way]; From 4e1d2a7ec81452365d50cabada2a9f53281446a0 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 03:05:26 -0700 Subject: [PATCH 050/336] Serve a warp's identical loads with one dcache bank access The LSU coalescer merges lanes only inside each output channel's lane group, so a load whose lanes in different groups read the same word, such as every uniform load (kernel arguments, the shared operand of SGEMM, x[k] in SGEMV), sent one identical request per channel, all to the same bank. Each was served separately, and the bank was the bottleneck. With VX_CFG_DCACHE_READ_MERGE (default on), the dcache serves them with one bank access: - Plain reads (no write, flush, io, local, OM or AMO attribute) from ports of one LSU block, with the same word address and tag in the same cycle, form a group. The lowest port leads; the others stay out of the request crossbar and are accepted in the cycle their leader is. Each port's order into the bank is therefore unchanged: a thread's earlier store to the word is already in that bank's queue ahead of the load, and a later one cannot overtake it. Fences and flush tokens are consumed before the merge and block every input while visible. - The merged ports' mask rides below the tag through the bank, MSHR and replay. - The response network becomes a flat multicast crossbar: each bank lane keeps a sent mask, delivers a merged response to all of its ports, normally in one cycle, and releases it once every port has taken it. - The dcache reqs counter counts a merged read once. Merging is tied to an unregistered request crossbar by a static assert, since the flush unit counts in-flight requests by port handshakes. uniform_ld is a new test. It checks that a warp sees its own lane's store to a shared word, does not see a store issued after the load, and gets its own word when lanes in groups of 1 to 16 share one line. rtlsim sgemm, off -> on: 16 threads, 8-byte dcache word (8 banks) -n128 2,464,685 -> 2,041,896 (-17.2%) 16 threads, 2 dcache banks -n128 1,844,005 -> 1,662,951 (-9.8%) 32 threads -n128 874,395 -> 805,401 (-7.9%) 16 threads -n128 1,780,951 -> 1,665,973 (-6.5%) 4 cores + L2, 16 threads -n128 439,572 -> 411,124 (-6.5%) default config 801,222 -> 799,610 (-0.2%) At 16 threads, dcache reads fall 25% and bank stalls 55%. sgemv 512 at 4 cores + L2: -6.3%. raycast: 0 to -1%. ldst_order, fence, fence_mp, uniform_ld and amo (with A) pass, including at 2 clusters + L3. Comparing lines instead of words, releasing a follower before its leader is accepted, and merging AMOs each fail a test. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 4 + hw/rtl/VX_socket.sv | 1 + hw/rtl/cache/VX_cache.sv | 174 +++++++++++++++++++++---- hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_wrap.sv | 2 + tests/regression/Makefile | 2 +- tests/regression/uniform_ld/Makefile | 16 +++ tests/regression/uniform_ld/common.h | 27 ++++ tests/regression/uniform_ld/kernel.cpp | 60 +++++++++ tests/regression/uniform_ld/main.cpp | 164 +++++++++++++++++++++++ 10 files changed, 423 insertions(+), 29 deletions(-) create mode 100644 tests/regression/uniform_ld/Makefile create mode 100644 tests/regression/uniform_ld/common.h create mode 100644 tests/regression/uniform_ld/kernel.cpp create mode 100644 tests/regression/uniform_ld/main.cpp diff --git a/VX_config.toml b/VX_config.toml index 51115ae941..2b6d21a4fe 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -255,6 +255,10 @@ VX_CFG_DCACHE_NUM_BANKS = "expr: pow(2, clog2(min($VX_CFG_DCACHE_NUM_REQS, 16))) # With two banks it can only swap a line on parity, and the extra parallel # misses cost more in memory queuing than the spreading saves. VX_CFG_DCACHE_BANK_HASH = "expr: int($VX_CFG_DCACHE_NUM_BANKS >= 4)" +# One bank access serves a warp's same-word reads arriving together on its +# coalescer channels (uniform loads), and the response returns to all of +# them in the same cycle. +VX_CFG_DCACHE_READ_MERGE = 1 VX_CFG_L1_MEM_PORTS = "expr: min($VX_CFG_DCACHE_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS) if ($VX_CFG_ICACHE_ENABLE or $VX_CFG_DCACHE_ENABLE) else min($VX_CFG_DCACHE_NUM_REQS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS)" diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 08aa35fc86..cf04039049 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -234,6 +234,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .IS_LLC (DCACHE_IS_LLC), .SHARED_LLC (DCACHE_SHARED_LLC), .BANK_HASH (`VX_CFG_DCACHE_BANK_HASH), + .READ_MERGE (`VX_CFG_DCACHE_READ_MERGE ? DCACHE_CHANNELS : 0), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) dcache ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index c5e7989c83..446b7355e1 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -58,6 +58,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) + parameter READ_MERGE = 0, // merge same-word reads within groups of this many adjacent ports (0: off) + parameter AMO_ENABLE = 0 // synthesize atomic-op logic in the banks ) ( `ifdef PERF_ENABLE @@ -87,7 +89,12 @@ module VX_cache import VX_gpu_pkg::*; #( localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS); localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); localparam LINE_ADDR_WIDTH = (`CS_WORD_ADDR_WIDTH - BANK_SEL_BITS - WORD_SEL_BITS); - localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + TAG_WIDTH + `UP(MEM_ATTR_WIDTH); + // Read merging carries the merged ports' mask (one bit per port of the + // leader's group) below the request tag through the bank. + localparam MERGE_GROUP = (READ_MERGE != 0) ? `MIN(READ_MERGE, NUM_REQS) : 1; + localparam DO_READ_MERGE = (MERGE_GROUP > 1); + localparam BANK_TAG_WIDTH = TAG_WIDTH + (DO_READ_MERGE ? MERGE_GROUP : 0); + localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + BANK_TAG_WIDTH + `UP(MEM_ATTR_WIDTH); localparam CORE_RSP_DATAW = WORD_WIDTH + TAG_WIDTH; localparam BANK_MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH; localparam MEM_REQ_DATAW = (`UP(BANK_SEL_BITS) + `CS_LINE_SECTOR_ADDR_WIDTH + 1 + SECTOR_SIZE + `CS_SECTOR_WIDTH + BANK_MEM_TAG_WIDTH + `UP(MEM_ATTR_WIDTH)); @@ -98,6 +105,10 @@ module VX_cache import VX_gpu_pkg::*; #( localparam REQ_XBAR_BUF = (NUM_REQS > 2) ? 2 : 0; localparam CORE_RSP_BUF_ENABLE = (NUM_BANKS != 1) || (NUM_REQS != 1); localparam MEM_REQ_BUF_ENABLE = (NUM_BANKS != 1); + `STATIC_ASSERT(!DO_READ_MERGE || (`IS_POW2(MERGE_GROUP) && `IS_DIVISBLE(NUM_REQS, MERGE_GROUP)), ("invalid parameter: read merge group")) + // a merged request is one bank access for several port handshakes, which + // the in-flight request count of a registered request crossbar would miss + `STATIC_ASSERT(!DO_READ_MERGE || (`TO_OUT_BUF_REG(REQ_XBAR_BUF) == 0), ("invalid parameter: read merge with a registered request crossbar")) `ifdef PERF_ENABLE wire [NUM_BANKS-1:0] perf_read_miss_per_bank; wire [NUM_BANKS-1:0] perf_write_miss_per_bank; @@ -251,13 +262,13 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][WORD_SEL_WIDTH-1:0] per_bank_core_req_wsel; wire [NUM_BANKS-1:0][WORD_SIZE-1:0] per_bank_core_req_byteen; wire [NUM_BANKS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_req_data; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_core_req_tag; + wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_core_req_tag; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_core_req_idx; wire [NUM_BANKS-1:0][`UP(MEM_ATTR_WIDTH)-1:0] per_bank_core_req_attr; wire [NUM_BANKS-1:0] per_bank_core_req_ready; wire [NUM_BANKS-1:0] per_bank_core_rsp_valid; wire [NUM_BANKS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_rsp_data; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_core_rsp_tag; + wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_core_rsp_tag; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_core_rsp_idx; wire [NUM_BANKS-1:0] per_bank_core_rsp_ready; wire [NUM_BANKS-1:0] per_bank_mem_req_valid; @@ -313,6 +324,57 @@ module VX_cache import VX_gpu_pkg::*; #( assign core_req_bid[i] = '0; end end + // Read merging: plain reads of the same word with the same tag from ports + // of one group in the same cycle are served by one bank access. The lowest + // such port leads; the others stay out of the request crossbar and are + // accepted in the cycle their leader is, so every port keeps its own + // request order into the bank. + wire [NUM_REQS-1:0] core_req_follower; + wire [NUM_REQS-1:0] core_req_xbar_ready; + wire [NUM_REQS-1:0][BANK_TAG_WIDTH-1:0] core_req_bank_tag; + if (DO_READ_MERGE) begin : g_read_merge + // a plain read has no flush, I/O, local, OM or atomic attribute + localparam [`UP(MEM_ATTR_WIDTH)-1:0] SPECIAL_ATTRS = `UP(MEM_ATTR_WIDTH)'((1 << MEM_ATTR_FLUSH_OFFS) + | (1 << MEM_ATTR_IO_OFFS) + | (1 << MEM_ATTR_LOCAL_OFFS) + | (1 << MEM_ATTR_OM_OFFS) + | (1 << MEM_ATTR_AMO_OFFS)); + wire [NUM_REQS-1:0] plain_read; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_plain_read + assign plain_read[i] = core_req_valid[i] && ~core_req_rw[i] && ((core_req_attr[i] & SPECIAL_ATTRS) == '0); + end + // same_word[i][j]: port i reads what a lower port j of its group reads + wire [NUM_REQS-1:0][NUM_REQS-1:0] same_word; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_same_i + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_same_j + if (j < i && (j / MERGE_GROUP) == (i / MERGE_GROUP)) begin : g_cmp + assign same_word[i][j] = plain_read[i] && plain_read[j] + && (core_req_addr[i] == core_req_addr[j]) + && (core_req_tag[i] == core_req_tag[j]); + end else begin : g_none + assign same_word[i][j] = 1'b0; + end + end + end + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_merge + localparam GROUP_BASE = (i / MERGE_GROUP) * MERGE_GROUP; + wire [MERGE_GROUP-1:0] merged_mask; + wire [NUM_REQS-1:0] leader_fire; + for (genvar k = 0; k < MERGE_GROUP; ++k) begin : g_merged_mask + assign merged_mask[k] = same_word[GROUP_BASE + k][i]; + end + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_leader_fire + assign leader_fire[j] = same_word[i][j] && ~core_req_follower[j] && core_req_xbar_ready[j]; + end + assign core_req_follower[i] = (| same_word[i]); + assign core_req_bank_tag[i] = {core_req_tag[i], merged_mask}; + assign core_req_ready[i] = core_req_follower[i] ? (| leader_fire) : core_req_xbar_ready[i]; + end + end else begin : g_no_read_merge + assign core_req_follower = '0; + assign core_req_bank_tag = core_req_tag; + assign core_req_ready = core_req_xbar_ready; + end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_req_data_in assign core_req_data_in[i] = { core_req_line_addr[i], @@ -320,7 +382,7 @@ module VX_cache import VX_gpu_pkg::*; #( core_req_wsel[i], core_req_byteen[i], core_req_data[i], - core_req_tag[i], + core_req_bank_tag[i], core_req_attr[i] }; end @@ -343,10 +405,10 @@ module VX_cache import VX_gpu_pkg::*; #( `else `UNUSED_PIN(collisions), `endif - .valid_in (core_req_valid), + .valid_in (core_req_valid & ~core_req_follower), .data_in (core_req_data_in), .sel_in (core_req_bid), - .ready_in (core_req_ready), + .ready_in (core_req_xbar_ready), .valid_out (per_bank_core_req_valid), .data_out (core_req_data_out), .sel_out (per_bank_core_req_idx), @@ -386,7 +448,7 @@ module VX_cache import VX_gpu_pkg::*; #( .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), - .TAG_WIDTH (TAG_WIDTH), + .TAG_WIDTH (BANK_TAG_WIDTH), .CORE_OUT_BUF (CORE_RSP_BUF_ENABLE ? 3 : 0), .MEM_OUT_BUF (MEM_REQ_BUF_ENABLE ? 3 : 0), .IS_LLC (IS_LLC), @@ -438,7 +500,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] core_rsp_ready; wire [NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] per_bank_core_rsp_pdata; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_core_rsp_pdata - assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i], per_bank_core_rsp_tag[i]}; + assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i], per_bank_core_rsp_tag[i][BANK_TAG_WIDTH-1 -: TAG_WIDTH]}; end // A flush token's answer takes bank 0's lane into the response crossbar // when the bank has no response of its own, so it leaves through the same @@ -462,25 +524,80 @@ module VX_cache import VX_gpu_pkg::*; #( assign per_bank_core_rsp_ready[i] = core_rsp_xbar_ready[i]; end end - VX_stream_omega #( - .NUM_INPUTS (NUM_BANKS), - .NUM_OUTPUTS (NUM_REQS), - .DATAW (CORE_RSP_DATAW), - .ARBITER ("R"), - .OUT_BUF (CORE_OUT_BUF) - ) core_rsp_xbar ( - .clk (clk), - .reset (reset), - .valid_in (core_rsp_xbar_valid), - .data_in (core_rsp_xbar_pdata), - .sel_in (core_rsp_xbar_idx), - .ready_in (core_rsp_xbar_ready), - .valid_out (core_rsp_valid), - .data_out (core_rsp_pdata), - `UNUSED_PIN (sel_out), - .ready_out (core_rsp_ready), - `UNUSED_PIN (collisions) - ); + if (DO_READ_MERGE) begin : g_core_rsp_mcast + // A merged read's response goes to its leader's port and to every + // merged port: each output takes it when its arbiter grants it, and the + // bank's response is released once all its targets have taken it. + localparam GROUP_BITS = `CLOG2(MERGE_GROUP); + wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_pending; + wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_taken; + wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_valid; + wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_ready; + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_lanes + wire [MERGE_GROUP-1:0] merged_mask; + if (i == 0) begin : g_flush_mask + assign merged_mask = per_bank_core_rsp_valid[0] ? per_bank_core_rsp_tag[0][MERGE_GROUP-1:0] : '0; + end else begin : g_bank_mask + assign merged_mask = per_bank_core_rsp_tag[i][MERGE_GROUP-1:0]; + end + wire [REQ_SEL_WIDTH-1:0] group_base = (core_rsp_xbar_idx[i] >> GROUP_BITS) << GROUP_BITS; + wire [NUM_REQS-1:0] targets = (NUM_REQS'(1) << core_rsp_xbar_idx[i]) + | (NUM_REQS'(merged_mask) << group_base); + reg [NUM_REQS-1:0] sent_r; + assign lane_pending[i] = targets & ~sent_r & {NUM_REQS{core_rsp_xbar_valid[i]}}; + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_taken + assign per_output_valid[j][i] = lane_pending[i][j]; + assign lane_taken[i][j] = lane_pending[i][j] && per_output_ready[j][i]; + end + assign core_rsp_xbar_ready[i] = core_rsp_xbar_valid[i] && ((lane_pending[i] & ~lane_taken[i]) == '0); + always @(posedge clk) begin + if (reset) begin + sent_r <= '0; + end else if (core_rsp_xbar_valid[i]) begin + sent_r <= core_rsp_xbar_ready[i] ? '0 : (sent_r | lane_taken[i]); + end + end + end + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_outputs + VX_stream_arb #( + .NUM_INPUTS (NUM_BANKS), + .NUM_OUTPUTS (1), + .DATAW (CORE_RSP_DATAW), + .ARBITER ("R"), + .OUT_BUF (CORE_OUT_BUF) + ) core_rsp_arb ( + .clk (clk), + .reset (reset), + .valid_in (per_output_valid[j]), + .data_in (core_rsp_xbar_pdata), + .ready_in (per_output_ready[j]), + .valid_out (core_rsp_valid[j]), + .data_out (core_rsp_pdata[j]), + `UNUSED_PIN (sel_out), + .ready_out (core_rsp_ready[j]) + ); + end + end else begin : g_core_rsp_xbar + VX_stream_omega #( + .NUM_INPUTS (NUM_BANKS), + .NUM_OUTPUTS (NUM_REQS), + .DATAW (CORE_RSP_DATAW), + .ARBITER ("R"), + .OUT_BUF (CORE_OUT_BUF) + ) core_rsp_xbar ( + .clk (clk), + .reset (reset), + .valid_in (core_rsp_xbar_valid), + .data_in (core_rsp_xbar_pdata), + .sel_in (core_rsp_xbar_idx), + .ready_in (core_rsp_xbar_ready), + .valid_out (core_rsp_valid), + .data_out (core_rsp_pdata), + `UNUSED_PIN (sel_out), + .ready_out (core_rsp_ready), + `UNUSED_PIN (collisions) + ); + end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_rsp assign core_bus2_if[i].rsp_valid = core_rsp_valid[i]; assign {core_bus2_if[i].rsp_data.data, core_bus2_if[i].rsp_data.tag} = core_rsp_pdata[i]; @@ -661,7 +778,8 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] perf_core_writes_per_req; wire [NUM_REQS-1:0] perf_crsp_stall_per_req; wire [MEM_PORTS-1:0] perf_mem_stall_per_port; - `BUFFER(perf_core_reads_per_req, core_req_valid & core_req_ready & ~core_req_rw); + // a merged read counts once, as the bank access that serves it + `BUFFER(perf_core_reads_per_req, core_req_valid & core_req_ready & ~core_req_rw & ~core_req_follower); `BUFFER(perf_core_writes_per_req, core_req_valid & core_req_ready & core_req_rw); for (genvar i = 0; i < NUM_REQS; ++i) begin : g_perf_crsp_stall_per_req assign perf_crsp_stall_per_req[i] = core_bus_if[i].rsp_valid && ~core_bus_if[i].rsp_ready; diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 88dc19d40d..94a9ebdb58 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -60,6 +60,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). + parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). parameter AMO_ENABLE = 0 // Enable atomic memory operations. ) ( @@ -183,6 +184,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .SHARED_LLC (SHARED_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), + .READ_MERGE (READ_MERGE), .AMO_ENABLE (AMO_ENABLE) ) cache_wrap ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 2f04c76436..f0d21ecb98 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -62,6 +62,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). + parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). parameter AMO_ENABLE = 0 // Enable atomic operations in cache banks. ) ( @@ -164,6 +165,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .SHARED_LLC (SHARED_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), + .READ_MERGE (READ_MERGE), .AMO_ENABLE (AMO_ENABLE) ) cache ( .clk (clk), diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 15db9427dd..5b2f655a92 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax \ + softmax uniform_ld \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/uniform_ld/Makefile b/tests/regression/uniform_ld/Makefile new file mode 100644 index 0000000000..fb9ed28371 --- /dev/null +++ b/tests/regression/uniform_ld/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := uniform_ld + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n64 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/uniform_ld/common.h b/tests/regression/uniform_ld/common.h new file mode 100644 index 0000000000..010b9dc91b --- /dev/null +++ b/tests/regression/uniform_ld/common.h @@ -0,0 +1,27 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define UNIFORM_MARK 0x80000000u + +// Per warp and iteration: one RAW word and one WAR word (four words apart, +// so four iterations share a line), and one 16-word line of read-only words. +#define RAW_STRIDE 4 +#define WAR_STRIDE 4 +#define RO_LINE 16 + +typedef struct { + uint32_t num_iters; // iterations per warp + uint32_t num_threads; // total threads launched + uint32_t warp_size; // threads per warp + uint32_t scatter_words; + uint64_t raw_addr; // in: initial words; out: RAW words overwritten with UNIFORM_MARK + index + uint64_t war_addr; // in: initial words; out: WAR words overwritten with UNIFORM_MARK + index + uint64_t ro_addr; // in: read-only words + uint64_t scatter_addr;// in: read-only words scattered across lines + uint64_t err_addr; // out: per-thread error count + uint64_t sink_addr; // out: per-thread sum of the scattered loads (keeps them alive) +} kernel_arg_t; + +#endif diff --git a/tests/regression/uniform_ld/kernel.cpp b/tests/regression/uniform_ld/kernel.cpp new file mode 100644 index 0000000000..7673a99508 --- /dev/null +++ b/tests/regression/uniform_ld/kernel.cpp @@ -0,0 +1,60 @@ +#include +#include "common.h" + +// Warp-uniform loads around single-lane stores. Per iteration k, lane +// s = k % W (so every lane, and with it every dcache channel, takes a turn): +// +// scattered loads - every lane a different line, so the channels drain +// unevenly before the accesses below +// RAW: lane s stores its word, then the whole warp loads it (one address): +// lane s must see its own store +// WAR: the whole warp loads a word, then lane s stores it: lane s's load +// must not see its own later store +// read-only loads - lanes in groups of 1, 2, 4, 8 or 16 share a word of +// one line: every lane must get its own word +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto raw = reinterpret_cast(arg->raw_addr); + auto war = reinterpret_cast(arg->war_addr); + auto ro = reinterpret_cast(arg->ro_addr); + auto scatter = reinterpret_cast(arg->scatter_addr); + auto err = reinterpret_cast(arg->err_addr); + auto sink = reinterpret_cast(arg->sink_addr); + const uint32_t W = arg->warp_size; + const uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const uint32_t lane = tid % W; + const uint32_t warp = tid / W; + uint32_t errors = 0; + uint32_t acc = 0; + for (uint32_t k = 0; k < arg->num_iters; ++k) { + uint32_t s = k % W; + uint32_t slot = warp * arg->num_iters + k; + + acc += scatter[(lane * RO_LINE + slot * 5) % arg->scatter_words]; + + uint32_t ri = slot * RAW_STRIDE; + if (lane == s) { + raw[ri] = UNIFORM_MARK + ri; + } + uint32_t rv = raw[ri]; + if (lane == s && rv != UNIFORM_MARK + ri) { + ++errors; + } + + uint32_t wi = slot * WAR_STRIDE; + uint32_t wv = war[wi]; + if (lane == s) { + war[wi] = UNIFORM_MARK + wi; + } + if (lane == s && wv != wi * 7 + 1) { + ++errors; + } + + uint32_t shift = k % 5; + uint32_t oi = slot * RO_LINE + ((lane >> shift) & (RO_LINE - 1)); + if (ro[oi] != oi * 3 + 2) { + ++errors; + } + } + err[tid] = errors; + sink[tid] = acc; +} diff --git a/tests/regression/uniform_ld/main.cpp b/tests/regression/uniform_ld/main.cpp new file mode 100644 index 0000000000..40d0943689 --- /dev/null +++ b/tests/regression/uniform_ld/main.cpp @@ -0,0 +1,164 @@ +// uniform_ld — warp-uniform loads must keep each thread's order with its own +// stores, and every lane must receive its own word. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t num_iters = 64; +uint32_t scatter_words = 65536; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:s:k:h")) != -1) { + switch (c) { + case 'n': num_iters = std::atoi(optarg); break; + case 's': scatter_words = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-n iterations] [-s scattered words] [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + if (num_iters == 0) num_iters = 1; + if (scatter_words == 0) scatter_words = 1; + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // Fill every warp of every core. + uint64_t nt = 0, nw = 0, nc = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + const uint32_t W = static_cast(nt); + const uint32_t num_threads = static_cast(nt * nw * nc); + const uint32_t num_slots = (num_threads / W) * num_iters; + std::cout << "uniform_ld: iters=" << num_iters << " threads=" << num_threads + << " scatter=" << scatter_words << std::endl; + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + const uint32_t raw_words = num_slots * RAW_STRIDE; + const uint32_t war_words = num_slots * WAR_STRIDE; + const uint32_t ro_words = num_slots * RO_LINE; + vx_buffer_h raw_buf = nullptr, war_buf = nullptr, ro_buf = nullptr; + vx_buffer_h scatter_buf = nullptr, err_buf = nullptr, sink_buf = nullptr; + CHECK(vx_buffer_create(dev, raw_words * sizeof(uint32_t), VX_MEM_READ_WRITE, &raw_buf)); + CHECK(vx_buffer_create(dev, war_words * sizeof(uint32_t), VX_MEM_READ_WRITE, &war_buf)); + CHECK(vx_buffer_create(dev, ro_words * sizeof(uint32_t), VX_MEM_READ, &ro_buf)); + CHECK(vx_buffer_create(dev, scatter_words * sizeof(uint32_t), VX_MEM_READ, &scatter_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &sink_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.num_iters = num_iters; + kernel_arg.num_threads = num_threads; + kernel_arg.warp_size = W; + kernel_arg.scatter_words = scatter_words; + CHECK(vx_buffer_address(raw_buf, &kernel_arg.raw_addr)); + CHECK(vx_buffer_address(war_buf, &kernel_arg.war_addr)); + CHECK(vx_buffer_address(ro_buf, &kernel_arg.ro_addr)); + CHECK(vx_buffer_address(scatter_buf, &kernel_arg.scatter_addr)); + CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); + CHECK(vx_buffer_address(sink_buf, &kernel_arg.sink_addr)); + + std::vector h_raw(raw_words), h_war(war_words), h_ro(ro_words), h_scatter(scatter_words); + for (uint32_t i = 0; i < raw_words; ++i) h_raw[i] = i * 7 + 1; + for (uint32_t i = 0; i < war_words; ++i) h_war[i] = i * 7 + 1; + for (uint32_t i = 0; i < ro_words; ++i) h_ro[i] = i * 3 + 2; + for (uint32_t i = 0; i < scatter_words; ++i) h_scatter[i] = i; + CHECK(vx_enqueue_write(q, raw_buf, 0, h_raw.data(), raw_words * sizeof(uint32_t), 0, nullptr, nullptr)); + CHECK(vx_enqueue_write(q, war_buf, 0, h_war.data(), war_words * sizeof(uint32_t), 0, nullptr, nullptr)); + CHECK(vx_enqueue_write(q, ro_buf, 0, h_ro.data(), ro_words * sizeof(uint32_t), 0, nullptr, nullptr)); + CHECK(vx_enqueue_write(q, scatter_buf, 0, h_scatter.data(), scatter_words * sizeof(uint32_t), 0, nullptr, nullptr)); + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &num_threads, grid, block)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0]= block[0]; + + std::vector h_err(num_threads); + vx_event_h launch_ev = nullptr, read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_raw.data(), raw_buf, 0, raw_words * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_war.data(), war_buf, 0, war_words * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] == 0) continue; + if (errors < 16) + std::printf("*** thread %u: %u wrong loads\n", t, h_err[t]); + errors += h_err[t]; + } + for (uint32_t slot = 0; slot < num_slots; ++slot) { + uint32_t ri = slot * RAW_STRIDE, wi = slot * WAR_STRIDE; + if (h_raw[ri] != UNIFORM_MARK + ri) { + if (errors < 16) std::printf("*** raw[%u] expected=0x%x actual=0x%x\n", ri, UNIFORM_MARK + ri, h_raw[ri]); + ++errors; + } + if (h_war[wi] != UNIFORM_MARK + wi) { + if (errors < 16) std::printf("*** war[%u] expected=0x%x actual=0x%x\n", wi, UNIFORM_MARK + wi, h_war[wi]); + ++errors; + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(sink_buf); + vx_buffer_release(err_buf); + vx_buffer_release(scatter_buf); + vx_buffer_release(ro_buf); + vx_buffer_release(war_buf); + vx_buffer_release(raw_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} From 5e8e5c234dbabe2354434ff427436ded036606de Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 03:18:19 -0700 Subject: [PATCH 051/336] Size the local-memory reservation hold budget by the core's harts A local-memory reservation refuses LRs from other harts until its own SC lands, spending one credit per refused LR, and yields to the next LR once the credits run out. The budget was fixed at 3 bits (7 refusals), sized as if credits were spent per cycle of the local round trip. A warp's lr.w reaches the bank as one request per lane, back to back, so at 16 threads one warp's own burst spent the budget and its 9th lane took the reservation from its first. Most SCs then found their reservation gone, and a contended LR/SC loop stopped making progress: amo's lrsc_lmem case failed at 16 threads with the A extension. The budget now covers two refusals per hart of the core: CLOG2(NUM_WARPS * NUM_THREADS) + 1 bits. rtlsim, amo with A: lrsc_lmem -n4, 16 threads FAILED (1,825,530 cycles) -> PASSED (35,313) full amo, 16 threads 14/15 -> 15/15 full amo, 8 threads and default config: 15/15 lrsc_counter on global memory at 4 cores + L2, 16 threads (256 contending harts) passes with the cache bank's existing budget. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/mem/VX_local_mem.sv | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hw/rtl/mem/VX_local_mem.sv b/hw/rtl/mem/VX_local_mem.sv index 6d780960e5..5614595503 100644 --- a/hw/rtl/mem/VX_local_mem.sv +++ b/hw/rtl/mem/VX_local_mem.sv @@ -479,11 +479,16 @@ module VX_local_mem import VX_gpu_pkg::*; #( // holder-credit protocol a hot word maps to exactly one station, so // depth only reduces false eviction between distinct reserved words // aliasing in one bank -- and a spurious SC failure is architecturally - // legal. Clamp the depth and shorten the credit budget to the local - // round trip instead of inheriting the L1-miss sizing. + // legal. Clamp the depth. + // A credit is spent per refused LR, not per cycle: a warp's LR reaches + // the bank as one request per lane, back to back, so between a holder's + // LR and its SC every other hart of the core can be refused, and a + // refused hart can retry once more. The budget covers two refusals per + // hart; a smaller one lets one warp's own LR burst displace its holder + // and a contended retry loop stops making progress. VX_amo_unit #( .NUM_RES_ENTRIES (`MIN(`VX_CFG_AMO_RS_SIZE, 4)), - .HOLD_CREDIT_BITS (3), + .HOLD_CREDIT_BITS (`CLOG2(`VX_CFG_NUM_WARPS * `VX_CFG_NUM_THREADS) + 1), .LINE_ADDR_BITS (BANK_ADDR_WIDTH), .DATA_WIDTH (WORD_WIDTH) ) amo_unit ( From 75984cccd24f1440424d7c496678a393c238fc75 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 03:41:10 -0700 Subject: [PATCH 052/336] Track MSHR misses in a line table so a cache can hold more in flight At realistic DRAM latency the dcache MSHR becomes the limit: with 16 entries and 400-cycle memory, sgemm -n128 at 16 threads finds a bank's MSHR full 20% of its cycles. The full MSHR holds only one or two distinct missing lines; the entries are requests merging onto them. A bigger flat MSHR helps, but every entry stores a full line address and takes part in the wide address compares and read muxes, so its area grows with the request count. With VX_CFG_{DCACHE,L2,L3}_MSHR_LINES set, the line address and sector move into a table of MSHR_LINES entries keyed by {line, sector}. Each entry is reference-counted by the request entries pointing at it and frees when the count reaches zero. Request entries (still MSHR_SIZE) keep only a small line index, so the wide compare covers the line table, and per-request matching, the AMO probe, fill, dequeue and the debug checks go through the index. Allocation, finalize, tail-find, the same-cycle late-joiner splice, word hazards, lazy-fetch parking and the payload RAM are unchanged, so replay and AMO order are too. The MSHR admits a request only while two line entries are free, folded into the existing almost-full stall, so the mshr stall counter covers it. Simulation checks that each count equals its live requests and that no line is needed when none is free. The default, 0, keeps the per-request address. sector_merge is a new test: alternate warps read opposite halves of one 128-byte line, so both sectors of a sectored L2 line miss together in one bank, and every word is checked against a host reference. It fails when line matching ignores the sector. rtlsim, MSHR 16 -> 64 requests over 16 lines (dcache, L2, L3), memory latency 400, 16 threads x 16 warps: sgemm -n128 2,733,260 -> 2,315,677 (-15.3%) raycast 539,032 -> 495,457 (-8.1%) Without B2's read merge the same sgemm gains 24.8%. At the stock latency the MSHR does not bind and the gain is under 0.5%, so the default stays off. 64/16 gives a flat 64-entry MSHR's cycles at about half its area (estimated). Off, every config is cycle-identical. mstress-6, amo (with A) at 1 core and at 4 cores + L2, uniform_ld, lazy_fetch-1 and flush_coherence chain-3 with the CP on the LLC pass with the line table on. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 6 + ci/testcases/cache.yaml | 22 +++ hw/rtl/VX_cluster.sv | 1 + hw/rtl/VX_socket.sv | 1 + hw/rtl/Vortex.sv | 1 + hw/rtl/cache/VX_cache.sv | 2 + hw/rtl/cache/VX_cache_bank.sv | 8 +- hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_mshr.sv | 192 ++++++++++++++++++++--- hw/rtl/cache/VX_cache_wrap.sv | 2 + hw/unittest/cache/VX_cache_top.sv | 2 + tests/regression/Makefile | 2 +- tests/regression/sector_merge/Makefile | 16 ++ tests/regression/sector_merge/common.h | 13 ++ tests/regression/sector_merge/kernel.cpp | 23 +++ tests/regression/sector_merge/main.cpp | 180 +++++++++++++++++++++ 16 files changed, 448 insertions(+), 25 deletions(-) create mode 100644 tests/regression/sector_merge/Makefile create mode 100644 tests/regression/sector_merge/common.h create mode 100644 tests/regression/sector_merge/kernel.cpp create mode 100644 tests/regression/sector_merge/main.cpp diff --git a/VX_config.toml b/VX_config.toml index 2b6d21a4fe..ab8a07a5dc 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -235,6 +235,10 @@ VX_CFG_DCACHE_DIRTYBYTES = "expr: $VX_CFG_DCACHE_LAZY_FETCH" VX_CFG_DCACHE_LAZY_FETCH = 0 VX_CFG_DCACHE_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_DCACHE_MSHR_SIZE = 16 +# MSHR line entries: distinct {line, sector} misses a bank tracks, each naming +# the chain of requests merged on it; MSHR_SIZE then sizes the request pool. +# 0 keeps the address in every request entry (lines = requests). +VX_CFG_DCACHE_MSHR_LINES = 0 # dcache is unsectored: line = sector = mem transaction granule = L1_LINE (which # defaults to MEM_BLOCK but can be configured smaller). Banking comes from the # reduced WORD above (>= sector is clamped), not the line. Deriving from L1_LINE @@ -270,6 +274,7 @@ VX_CFG_L2_DIRTYBYTES = "expr: $VX_CFG_L2_LAZY_FETCH" VX_CFG_L2_LAZY_FETCH = 0 VX_CFG_L2_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L2_MSHR_SIZE = 16 +VX_CFG_L2_MSHR_LINES = 0 VX_CFG_L2_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L2_SIZE) - clog2(1048576))" VX_CFG_L2_MRSQ_SIZE = 4 VX_CFG_L2_MREQ_SIZE = 0 @@ -289,6 +294,7 @@ VX_CFG_L3_DIRTYBYTES = "expr: $VX_CFG_L3_LAZY_FETCH" VX_CFG_L3_LAZY_FETCH = 0 VX_CFG_L3_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L3_MSHR_SIZE = 16 +VX_CFG_L3_MSHR_LINES = 0 VX_CFG_L3_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L3_SIZE) - clog2(2097152))" VX_CFG_L3_MRSQ_SIZE = 4 VX_CFG_L3_MREQ_SIZE = 0 diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index ceb79a2788..361ec300e2 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -337,3 +337,25 @@ tests: shape: cores: 4 l2cache: true +# Both sectors of one sectored L2 line miss together from different warps and +# cores: each must get its own fill, never chain behind the other sector's. +# Run with the address held per request and with the MSHR line table. +- id: sector_merge-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sector_merge + configs: -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true +- id: sector_merge-2 + via: blackbox + drivers: + - rtlsim + app: sector_merge + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_DCACHE_MSHR_SIZE=32 -DVX_CFG_DCACHE_MSHR_LINES=8 -DVX_CFG_L2_MSHR_SIZE=32 -DVX_CFG_L2_MSHR_LINES=8 + shape: + cores: 4 + l2cache: true diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 7e42f65be8..1b2ff700c5 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -224,6 +224,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .MEM_PORTS (L2_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L2_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L2_MSHR_SIZE), + .MSHR_LINES (`VX_CFG_L2_MSHR_LINES), .MRSQ_SIZE (`VX_CFG_L2_MRSQ_SIZE), .MREQ_SIZE (`VX_CFG_L2_MREQ_SIZE), .LATENCY (`VX_CFG_L2_LATENCY), diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index cf04039049..69c111f121 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -219,6 +219,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .MEM_PORTS (L1_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_DCACHE_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_DCACHE_MSHR_SIZE), + .MSHR_LINES (`VX_CFG_DCACHE_MSHR_LINES), .MRSQ_SIZE (`VX_CFG_DCACHE_MRSQ_SIZE), .MREQ_SIZE (`VX_CFG_DCACHE_MREQ_SIZE), .LATENCY (`VX_CFG_DCACHE_LATENCY), diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 434d9ab4ab..59bebc3865 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -163,6 +163,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .MEM_PORTS (L3_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L3_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L3_MSHR_SIZE), + .MSHR_LINES (`VX_CFG_L3_MSHR_LINES), .MRSQ_SIZE (`VX_CFG_L3_MRSQ_SIZE), .MREQ_SIZE (`VX_CFG_L3_MREQ_SIZE), .LATENCY (`VX_CFG_L3_LATENCY), diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 446b7355e1..d51f0c3e28 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -29,6 +29,7 @@ module VX_cache import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Core Response Queue Size parameter MSHR_SIZE = 16, // Miss Reserv Queue Knob + parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 4, // Memory Response Queue Size parameter MREQ_SIZE = 0, // Memory Request Queue Size (0 = derived minimum) @@ -445,6 +446,7 @@ module VX_cache import VX_gpu_pkg::*; #( .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 4187d85858..ae219feccb 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -26,6 +26,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter SECTOR_SIZE = LINE_SIZE,// sector (fill/eviction granule); = LINE_SIZE => 1 sector parameter CRSQ_SIZE = 0, // extra core-response queue slots over minimum parameter MSHR_SIZE = 1, // miss reservation queue size + parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 1, // memory response queue size (sized at wrapper) parameter MREQ_SIZE = 0, // memory request queue size (0 = derived minimum) parameter WRITE_ENABLE = 1, @@ -173,6 +174,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Shared signals // ------------------------------------------------------------------------ wire crsp_queue_stall, mshr_alm_full; + wire mshr_req_full, mshr_line_ready; wire mshr_allocate_st0; (* max_fanout = 64 *) wire pipe_stall; // Authoritative "no entry held", straight off the MSHR's valid mask. The @@ -1005,11 +1007,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .decr (mshr_dequeue), `UNUSED_PIN (empty), `UNUSED_PIN (alm_empty), - .full (mshr_alm_full), + .full (mshr_req_full), `UNUSED_PIN (alm_full), `UNUSED_PIN (size) ); + assign mshr_alm_full = mshr_req_full || ~mshr_line_ready; + VX_cache_mshr #( .INSTANCE_ID (`SFORMATF(("%s-mshr", INSTANCE_ID))), .BANK_ID (BANK_ID), @@ -1018,6 +1022,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .SECTOR_SIZE (SECTOR_SIZE), .NUM_BANKS (NUM_BANKS), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .WRITEBACK (WRITEBACK), .AMO_ENABLE (AMO_ENABLE != 0), .AMO_PASSTHRU ((AMO_ENABLE != 0) && (IS_LLC == 0)), @@ -1069,6 +1074,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .allocate_pending_ovl (mshr_pending_ovl_raw), .allocate_previd (mshr_previd), `UNUSED_PIN (allocate_ready), + .allocate_line_ready (mshr_line_ready), .finalize_valid (mshr_finalize_st1 && ~pipe_stall), .finalize_is_release (mshr_release_st1), // Only link an entry into the pending chain if it is KEPT (a miss). A diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 94a9ebdb58..a8d0f99a44 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -33,6 +33,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Core Response Queue Size parameter MSHR_SIZE = 16, // Miss Reserv Queue Knob + parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 4, // Memory Response Queue Size parameter MREQ_SIZE = 0, // Memory Request Queue Size (0 = derived minimum) @@ -171,6 +172,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index c8e66e0c10..c314df553c 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -38,6 +38,15 @@ // The finalize request is used to persit or release the currently allocated MSHR entry // if we had a cache miss or a hit, respectively. // +// With MSHR_LINES != 0 the line address lives in a separate table of +// MSHR_LINES entries, one per distinct {line, sector} held, and each request +// entry stores only an index into it. The address CAM then spans the lines +// rather than the requests, so the request pool can grow (many requests merge +// on few lines) without growing the wide compare. A line entry is reference +// counted by the request entries naming it and frees when the count drains. +// The bank admits a core request only while two line entries are free: the +// request already past admission may claim one before this one reaches S0. +// // Warning: This MSHR implementation is strongly coupled with the bank pipeline // and as such changes to either module requires careful evaluation. // @@ -50,6 +59,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (coalescing/fill granule); = LINE_SIZE => 1 sector parameter NUM_BANKS = 1, // Number of banks parameter MSHR_SIZE = 4, // Miss Reserv Queue Knob + parameter MSHR_LINES = 0, // distinct line entries (0 = address held per request) parameter DATA_WIDTH = 1, // MSHR parameters parameter WRITEBACK = 0, // Enable cache writeback parameter AMO_ENABLE = 0, // Per-entry AMO tracking (probe outputs) @@ -127,6 +137,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire allocate_pending_ovl, output wire [MSHR_ADDR_WIDTH-1:0] allocate_previd, output wire allocate_ready, + // a line entry is free for every request the bank can still admit + output wire allocate_line_ready, // finalize input wire finalize_valid, @@ -144,11 +156,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( `UNUSED_PARAM (BANK_ID) `UNUSED_PARAM (BANK_HASH) - reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; - // Per-entry sector. Coalescing matches on {line, sector} so same-line - // different-sector misses get independent fills (each replay then hits its - // own filled sector). Zero-width-equivalent (1 bit, all 0) when 1 sector/line. - reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_table [0:MSHR_SIZE-1]; + `STATIC_ASSERT((MSHR_LINES == 0) || ((MSHR_LINES >= 2) && (MSHR_LINES <= MSHR_SIZE)), ("invalid parameter: MSHR_LINES")) + // Per-entry word-in-line select, kept outside the payload RAM so the // allocate probe can compare it: the hit-demotion order hazard is // word-granular (a chained entry constrains only requests that touch @@ -182,6 +191,13 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( wire dequeue_park_fire = (LAZY_FETCH != 0) && dequeue_fire && dequeue_park; wire dequeue_free = dequeue_fire && ~dequeue_park_fire; + // Entry holds the allocate request's {line, sector} / the probe's line. + // Coalescing matches on {line, sector} so same-line different-sector misses + // get independent fills (each replay then hits its own filled sector). + wire [MSHR_SIZE-1:0] entry_match; + wire [MSHR_SIZE-1:0] entry_probe; + wire [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_n; + wire [MSHR_SIZE-1:0] addr_matches; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_addr_matches // Exclude the entry being consumed this cycle: an allocate that links @@ -193,8 +209,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // cycle: a released head never fills, so a request that linked behind // it would wait forever. Exclude it too — the requester re-looks-up // and hits the line the releasing entry just installed. - assign addr_matches[i] = valid_table[i] && (addr_table[i] == allocate_addr) - && (sector_table[i] == allocate_sector) && ~amo_mask[i] + assign addr_matches[i] = valid_table[i] && entry_match[i] && ~amo_mask[i] && ~(dequeue_free && (dequeue_id == MSHR_ADDR_WIDTH'(i))) && ~(finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i))); end @@ -288,8 +303,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( end if (allocate_fire) begin - addr_table[allocate_id] <= allocate_addr; - sector_table[allocate_id] <= allocate_sector; word_table[allocate_id] <= allocate_word_idx; write_table[allocate_id] <= allocate_rw; end @@ -303,14 +316,150 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( next_table <= next_table_n; end + // Per-entry address, for checks and traces only. + wire [`CS_LINE_ADDR_WIDTH-1:0] entry_addr [MSHR_SIZE]; + + if (MSHR_LINES != 0) begin : g_line_table + localparam LINE_ID_W = `LOG2UP(MSHR_LINES); + localparam LCNT_W = `CLOG2(MSHR_SIZE + 1); + + reg [`CS_LINE_ADDR_WIDTH-1:0] line_addr [0:MSHR_LINES-1]; + reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] line_sector [0:MSHR_LINES-1]; + reg [LCNT_W-1:0] line_cnt [0:MSHR_LINES-1]; + reg [LINE_ID_W-1:0] entry_line [0:MSHR_SIZE-1]; + + reg [LINE_ID_W-1:0] line_free_id_r; + reg line_free_r, line_ready_r; + + wire [MSHR_LINES-1:0] line_valid, line_hits, line_probes, line_free_n; + for (genvar l = 0; l < MSHR_LINES; ++l) begin : g_line_match + assign line_valid[l] = (line_cnt[l] != '0); + assign line_hits[l] = line_valid[l] && (line_addr[l] == allocate_addr) && (line_sector[l] == allocate_sector); + assign line_probes[l] = line_valid[l] && (line_addr[l] == probe_addr); + end + + // a {line, sector} occupies at most one line entry + wire line_hit; + wire [LINE_ID_W-1:0] line_hit_id; + VX_priority_encoder #( + .N (MSHR_LINES) + ) line_hit_sel ( + .data_in (line_hits), + .index_out (line_hit_id), + .valid_out (line_hit), + `UNUSED_PIN (onehot_out) + ); + + wire [LINE_ID_W-1:0] alloc_line = line_hit ? line_hit_id : line_free_id_r; + + for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match + assign entry_match[i] = line_hit && (entry_line[i] == line_hit_id); + assign entry_probe[i] = line_probes[entry_line[i]]; + assign entry_addr[i] = line_addr[entry_line[i]]; + end + + wire [LINE_ID_W-1:0] dequeue_line = entry_line[dequeue_id]; + wire [LINE_ID_W-1:0] finalize_line = entry_line[finalize_id]; + wire finalize_free = finalize_valid && finalize_is_release; + + for (genvar l = 0; l < MSHR_LINES; ++l) begin : g_line_cnt + wire inc = allocate_fire && (alloc_line == LINE_ID_W'(l)); + wire dec_d = dequeue_free && (dequeue_line == LINE_ID_W'(l)); + wire dec_f = finalize_free && (finalize_line == LINE_ID_W'(l)); + wire [LCNT_W-1:0] cnt_n = line_cnt[l] + LCNT_W'(inc) - LCNT_W'(dec_d) - LCNT_W'(dec_f); + assign line_free_n[l] = (cnt_n == '0); + always @(posedge clk) begin + if (reset) begin + line_cnt[l] <= '0; + end else begin + line_cnt[l] <= cnt_n; + end + end + end + + wire [MSHR_LINES-1:0] line_free_oh; + wire [LINE_ID_W-1:0] line_free_id_n; + wire line_free_any_n; + VX_priority_encoder #( + .N (MSHR_LINES) + ) line_free_sel ( + .data_in (line_free_n), + .onehot_out (line_free_oh), + .index_out (line_free_id_n), + .valid_out (line_free_any_n) + ); + + always @(posedge clk) begin + if (reset) begin + line_free_r <= 1'b0; + line_ready_r <= 1'b0; + end else begin + line_free_r <= line_free_any_n; + line_ready_r <= line_free_any_n && (| (line_free_n & ~line_free_oh)); + end + line_free_id_r <= line_free_id_n; + if (allocate_fire) begin + entry_line[allocate_id] <= alloc_line; + if (~line_hit) begin + line_addr[line_free_id_r] <= allocate_addr; + line_sector[line_free_id_r] <= allocate_sector; + end + end + end + + assign fill_addr = line_addr[entry_line[fill_id]]; + assign fill_sector = line_sector[entry_line[fill_id]]; + assign dequeue_addr_n = line_addr[entry_line[dequeue_id_n]]; + assign allocate_line_ready = line_ready_r; + + `RUNTIME_ASSERT(~(allocate_fire && ~line_hit && ~line_free_r), ("*** %s line table overflow: addr=0x%0h (#%0d)", INSTANCE_ID, + `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID), alc_req_uuid)) + + `ifdef SIMULATION + // Each line count equals the live entries naming the line. + for (genvar l = 0; l < MSHR_LINES; ++l) begin : g_line_check + integer refs; + always @(*) begin + refs = 0; + for (integer i = 0; i < MSHR_SIZE; ++i) begin + refs += 32'(valid_table[i] && (entry_line[i] == LINE_ID_W'(l))); + end + end + `RUNTIME_ASSERT(refs == 32'(line_cnt[l]), ("*** %s line count mismatch: line=%0d, count=%0d, refs=%0d", INSTANCE_ID, + l, line_cnt[l], refs)) + end + `endif + end else begin : g_entry_table + reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; + reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_table [0:MSHR_SIZE-1]; + + always @(posedge clk) begin + if (allocate_fire) begin + addr_table[allocate_id] <= allocate_addr; + sector_table[allocate_id] <= allocate_sector; + end + end + + for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match + assign entry_match[i] = (addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector); + assign entry_probe[i] = (addr_table[i] == probe_addr); + assign entry_addr[i] = addr_table[i]; + end + + assign fill_addr = addr_table[fill_id]; + assign fill_sector = sector_table[fill_id]; + assign dequeue_addr_n = addr_table[dequeue_id_n]; + assign allocate_line_ready = 1'b1; + end + `RUNTIME_ASSERT(~(allocate_fire && valid_table[allocate_id_r]), ("*** %s inuse allocation: addr=0x%0h, id=%0d (#%0d)", INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID), allocate_id_r, alc_req_uuid)) `RUNTIME_ASSERT(~(finalize_valid && ~valid_table[finalize_id]), ("*** %s invalid release: addr=0x%0h, id=%0d (#%0d)", INSTANCE_ID, - `CS_BANK_TO_FULL_ADDR(addr_table[finalize_id], BANK_ID), finalize_id, fin_req_uuid)) + `CS_BANK_TO_FULL_ADDR(entry_addr[finalize_id], BANK_ID), finalize_id, fin_req_uuid)) `RUNTIME_ASSERT(~(fill_valid && ~valid_table[fill_id]), ("*** %s invalid fill: addr=0x%0h, id=%0d", INSTANCE_ID, - `CS_BANK_TO_FULL_ADDR(addr_table[fill_id], BANK_ID), fill_id)) + `CS_BANK_TO_FULL_ADDR(entry_addr[fill_id], BANK_ID), fill_id)) // A walk confined to its own live chain never lands on an entry that is // already invalid. next_table and next_index are cleared only at allocate, @@ -328,15 +477,15 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( reg [`CS_LINE_ADDR_WIDTH-1:0] chain_addr_r; always @(posedge clk) begin if (fill_valid) begin - chain_addr_r <= addr_table[fill_id]; + chain_addr_r <= entry_addr[fill_id]; end end - `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (addr_table[dequeue_id] != chain_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, - dequeue_id, `CS_BANK_TO_FULL_ADDR(addr_table[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(chain_addr_r, BANK_ID))) + `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (entry_addr[dequeue_id] != chain_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, + dequeue_id, `CS_BANK_TO_FULL_ADDR(entry_addr[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(chain_addr_r, BANK_ID))) // The slot the dequeue pointer names must not be handed to another line - // while the pointer still rests on it: `dequeue_addr` is combinational on - // `addr_table[dequeue_id_r]`, so the rewrite retargets an in-flight replay. + // while the pointer still rests on it: a combinational `dequeue_addr` on + // the entry at `dequeue_id_r` would retarget an in-flight replay. `RUNTIME_ASSERT(~(allocate_fire && dequeue_val && (allocate_id == dequeue_id)), ("*** %s allocate over live dequeue pointer: id=%0d, addr=0x%0h", INSTANCE_ID, allocate_id, `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID))) `endif @@ -358,9 +507,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( .rdata (dequeue_data) ); - assign fill_addr = addr_table[fill_id]; - assign fill_sector = sector_table[fill_id]; - if (LAZY_FETCH != 0) begin : g_refetch reg [MSHR_SIZE-1:0] refetch_table, retry_table; always @(posedge clk) begin @@ -430,7 +576,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( wire [MSHR_SIZE-1:0] probe_ld, probe_amo; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_probe_matches - wire addr_match = valid_table[i] && persisted_table[i] && (addr_table[i] == probe_addr); + wire addr_match = valid_table[i] && persisted_table[i] && entry_probe[i]; assign probe_ld[i] = addr_match && ~amo_table[i]; assign probe_amo[i] = addr_match && amo_table[i]; end @@ -466,7 +612,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign probe_pending_amo = 1'b0; assign {probe_any_ld, probe_any_amo, persist_ld, persist_amo} = '0; `UNUSED_PARAM (AMO_PASSTHRU) - `UNUSED_VAR ({allocate_is_amo, probe_addr}) + `UNUSED_VAR ({allocate_is_amo, probe_addr, entry_probe}) end assign allocate_ready = allocate_rdy; @@ -518,7 +664,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( reg [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_r; reg dequeue_rw_r; always @(posedge clk) begin - dequeue_addr_r <= addr_table[dequeue_id_n]; + dequeue_addr_r <= dequeue_addr_n; dequeue_rw_r <= write_table[dequeue_id_n]; end @@ -556,7 +702,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( `TRACE(3, ("%t: %s table: ", $time, INSTANCE_ID)) for (integer i = 0; i < MSHR_SIZE; ++i) begin if (valid_table[i]) begin - `TRACE(3, (" %0d=0x%0h", i, `CS_BANK_TO_FULL_ADDR(addr_table[i], BANK_ID))) + `TRACE(3, (" %0d=0x%0h", i, `CS_BANK_TO_FULL_ADDR(entry_addr[i], BANK_ID))) if (write_table[i]) begin `TRACE(3, ("(w)")) end else begin diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index f0d21ecb98..07b7e2b310 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -31,6 +31,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Core Response Queue Size parameter MSHR_SIZE = 16, // Miss Reserv Queue Knob + parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 4, // Memory Response Queue Size parameter MREQ_SIZE = 0, // Memory Request Queue Size (0 = derived minimum) @@ -155,6 +156,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index 5d797998c9..68e16685af 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -39,6 +39,7 @@ module VX_cache_top import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Miss Reserv Queue Knob parameter MSHR_SIZE = 16, + parameter MSHR_LINES = 0, // Memory Response Queue Size parameter MRSQ_SIZE = 8, parameter MREQ_SIZE = 0, @@ -179,6 +180,7 @@ module VX_cache_top import VX_gpu_pkg::*; #( .MEM_PORTS (MEM_PORTS), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 5b2f655a92..a11c131c2f 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -15,7 +15,7 @@ TESTS := \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ async_barrier async_gbarrier gbar_phase bar_slot_phase packld wgather wsync quad_lod \ occupancy multikernel reopen module_reload single_cta \ - flush_coherence lazy_fetch \ + flush_coherence lazy_fetch sector_merge \ vm_test vm_fault vm_stress # --- common exclude list --------------------------------------------- diff --git a/tests/regression/sector_merge/Makefile b/tests/regression/sector_merge/Makefile new file mode 100644 index 0000000000..62863e5aee --- /dev/null +++ b/tests/regression/sector_merge/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := sector_merge + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n256 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/sector_merge/common.h b/tests/regression/sector_merge/common.h new file mode 100644 index 0000000000..7b3a54079a --- /dev/null +++ b/tests/regression/sector_merge/common.h @@ -0,0 +1,13 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +typedef struct { + uint32_t num_lines; + uint32_t line_words; + uint32_t group; + uint32_t reserved; + uint64_t src_addr; + uint64_t dst_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/sector_merge/kernel.cpp b/tests/regression/sector_merge/kernel.cpp new file mode 100644 index 0000000000..389b7cbb55 --- /dev/null +++ b/tests/regression/sector_merge/kernel.cpp @@ -0,0 +1,23 @@ +#include +#include "common.h" + +// Consecutive thread groups read opposite halves of the same lines, so the +// misses for both sectors of a sectored line are in flight together. +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + const uint32_t* src = (const uint32_t*)arg->src_addr; + uint32_t* dst = (uint32_t*)arg->dst_addr; + uint32_t num_lines = arg->num_lines; + uint32_t line_words = arg->line_words; + uint32_t half_words = line_words / 2; + + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t half = (gid / arg->group) & 1; + uint32_t word = half * half_words + (gid % arg->group) % half_words; + + uint32_t sum = 0; + #pragma unroll 4 + for (uint32_t k = 0; k < num_lines; ++k) { + sum += src[k * line_words + word] * (k + 1); + } + dst[gid] = sum; +} diff --git a/tests/regression/sector_merge/main.cpp b/tests/regression/sector_merge/main.cpp new file mode 100644 index 0000000000..54e9134872 --- /dev/null +++ b/tests/regression/sector_merge/main.cpp @@ -0,0 +1,180 @@ +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.vxbin"; +uint32_t num_lines = 256; +uint32_t line_bytes = 128; + +vx_device_h device = nullptr; +vx_buffer_h src_buffer = nullptr; +vx_buffer_h dst_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n lines] [-l line bytes] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:l:k:h")) != -1) { + switch (c) { + case 'n': + num_lines = atoi(optarg); + break; + case 'l': + line_bytes = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + show_usage(); + exit(0); + break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (src_buffer) vx_buffer_release(src_buffer); + if (dst_buffer) vx_buffer_release(dst_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); + vx_device_release(device); + } +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_device_open(0, &device)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_cores, num_warps, num_threads; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + + uint32_t line_words = line_bytes / sizeof(uint32_t); + uint32_t num_tasks = num_cores * num_warps * num_threads; + uint32_t num_words = num_lines * line_words; + + kernel_arg.num_lines = num_lines; + kernel_arg.line_words = line_words; + kernel_arg.group = num_threads; + + std::cout << "number of tasks: " << num_tasks << ", lines: " << num_lines + << ", line bytes: " << line_bytes << std::endl; + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_buffer_create(device, num_words * sizeof(uint32_t), VX_MEM_READ, &src_buffer)); + RT_CHECK(vx_buffer_address(src_buffer, &kernel_arg.src_addr)); + RT_CHECK(vx_buffer_create(device, num_tasks * sizeof(uint32_t), VX_MEM_WRITE, &dst_buffer)); + RT_CHECK(vx_buffer_address(dst_buffer, &kernel_arg.dst_addr)); + + // upload source buffer + std::cout << "upload source buffer" << std::endl; + std::vector h_src(num_words); + for (uint32_t i = 0; i < num_words; ++i) { + h_src[i] = i * 2654435761u + 12345u; + } + RT_CHECK(vx_enqueue_write(queue, src_buffer, 0, h_src.data(), num_words * sizeof(uint32_t), 0, nullptr, nullptr)); + + // load kernel module + std::cout << "load kernel module" << std::endl; + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + // start device + std::cout << "start device" << std::endl; + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + uint32_t n = num_tasks; + uint32_t grid_dim[1], block_dim[1]; + RT_CHECK(vx_device_max_occupancy_grid(device, 1, &n, grid_dim, block_dim)); + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid_dim[0]; + li.block_dim[0] = block_dim[0]; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + std::vector h_dst(num_tasks); + RT_CHECK(vx_enqueue_read(queue, h_dst.data(), dst_buffer, 0, num_tasks * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + // verify result + std::cout << "verify result" << std::endl; + uint32_t half_words = line_words / 2; + int errors = 0; + for (uint32_t gid = 0; gid < num_tasks; ++gid) { + uint32_t half = (gid / num_threads) & 1; + uint32_t word = half * half_words + (gid % num_threads) % half_words; + uint32_t ref = 0; + for (uint32_t k = 0; k < num_lines; ++k) { + ref += h_src[k * line_words + word] * (k + 1); + } + if (h_dst[gid] != ref) { + if (errors < 100) { + std::cout << "error at result #" << gid << ": actual 0x" << std::hex << h_dst[gid] + << ", expected 0x" << ref << std::dec << std::endl; + } + ++errors; + } + } + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + if (errors != 0) { + std::cout << "Found " << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + + std::cout << "PASSED!" << std::endl; + + return 0; +} From 227c49e35e1f85d602e137922ce1830812e04e63 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:17:40 -0700 Subject: [PATCH 053/336] tests: make mstress check every loaded value mstress's host reference was `ref *= value` starting from 0, so every expected result was 0 and the float compare allowed a few ULPs: the test never checked loaded data. A store with a flipped bit, or a fill that returned the wrong sector, still passed. The stress pattern is unchanged: an index table drives eight dependent indirect gathers per point, with the same strides and addresses. The data is now uint32, each point folds its eight loads as value * 31 + src[i], and the host computes the same fold and compares exactly, so every loaded value reaches the result and no rounding can hide or fake an error. All mstress CI cases (cache mstress-1..6, config2 mstress-4/13/14) pass on rtlsim and simx. A cache bank that flips bit 0 of store data now fails with 512 wrong results; the old test passed it. Co-Authored-By: Claude Opus 5.5 --- tests/regression/mstress/kernel.cpp | 8 ++-- tests/regression/mstress/main.cpp | 66 ++++------------------------- 2 files changed, 13 insertions(+), 61 deletions(-) diff --git a/tests/regression/mstress/kernel.cpp b/tests/regression/mstress/kernel.cpp index 8a66f2047c..2ae7f15ac3 100644 --- a/tests/regression/mstress/kernel.cpp +++ b/tests/regression/mstress/kernel.cpp @@ -4,18 +4,18 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t stride = arg->stride; uint32_t* addr_ptr = (uint32_t*)arg->src0_addr; - float* src_ptr = (float*)arg->src1_addr; - float* dst_ptr = (float*)arg->dst_addr; + uint32_t* src_ptr = (uint32_t*)arg->src1_addr; + uint32_t* dst_ptr = (uint32_t*)arg->dst_addr; size_t gid = blockIdx.x * blockDim.x + threadIdx.x; size_t offset = gid * stride; for (size_t i = 0; i < stride; ++i) { - float value = 0.0f; + uint32_t value = 0; for (uint32_t j = 0; j < NUM_LOADS; ++j) { size_t addr = offset + i + j; uint32_t index = addr_ptr[addr]; - value *= src_ptr[index]; + value = value * 31 + src_ptr[index]; } dst_ptr[offset+i] = value; } diff --git a/tests/regression/mstress/main.cpp b/tests/regression/mstress/main.cpp index 9754c79856..b5a6d63bd7 100644 --- a/tests/regression/mstress/main.cpp +++ b/tests/regression/mstress/main.cpp @@ -4,8 +4,6 @@ #include #include "common.h" #include -#include -#include #include #define RT_CHECK(_expr) \ @@ -20,51 +18,6 @@ /////////////////////////////////////////////////////////////////////////////// -union Float_t { - float f; - int i; - struct { - uint32_t man : 23; - uint32_t exp : 8; - uint32_t sign : 1; - } parts; -}; - -inline float fround(float x, int32_t precision = 8) { - auto power_of_10 = std::pow(10, precision); - return std::round(x * power_of_10) / power_of_10; -} - -inline bool almost_equal_eps(float a, float b, int ulp = 128) { - auto eps = std::numeric_limits::epsilon() * (std::max(fabs(a), fabs(b)) * ulp); - auto d = fabs(a - b); - if (d > eps) { - std::cout << "*** almost_equal_eps: d=" << d << ", eps=" << eps << std::endl; - return false; - } - return true; -} - -inline bool almost_equal_ulp(float a, float b, int32_t ulp = 6) { - Float_t fa{a}, fb{b}; - auto d = std::abs(fa.i - fb.i); - if (d > ulp) { - std::cout << "*** almost_equal_ulp: a=" << a << ", b=" << b << ", ulp=" << d << ", ia=" << std::hex << fa.i << ", ib=" << fb.i << std::endl; - return false; - } - return true; -} - -inline bool almost_equal(float a, float b) { - if (a == b) - return true; - /*if (almost_equal_eps(a, b)) - return true;*/ - return almost_equal_ulp(a, b); -} - -/////////////////////////////////////////////////////////////////////////////// - const char* kernel_file = "kernel.vxbin"; uint32_t count = 0; @@ -116,7 +69,7 @@ void cleanup() { } } -void gen_src_data(std::vector& test_data, +void gen_src_data(std::vector& test_data, std::vector& addr_table, uint32_t num_points, uint32_t num_addrs) { @@ -124,8 +77,7 @@ void gen_src_data(std::vector& test_data, addr_table.resize(num_addrs); for (uint32_t i = 0; i < num_points; ++i) { - float r = static_cast(std::rand()) / RAND_MAX; - test_data[i] = r; + test_data[i] = static_cast(std::rand()); } for (uint32_t i = 0; i < num_addrs; ++i) { @@ -190,8 +142,8 @@ int main(int argc, char *argv[]) { // allocate host buffers std::cout << "allocate host buffers" << std::endl; std::vector h_addr; - std::vector h_src; - std::vector h_dst(num_points); + std::vector h_src; + std::vector h_dst(num_points); gen_src_data(h_src, h_addr, num_points, num_addrs); // upload source buffer0 @@ -239,17 +191,17 @@ int main(int argc, char *argv[]) { std::cout << "verify result" << std::endl; int errors = 0; for (uint32_t i = 0; i < num_points; ++i) { - float ref = 0.0f; + uint32_t ref = 0; for (uint32_t j = 0; j < NUM_LOADS; ++j) { uint32_t addr = i + j; uint32_t index = h_addr[addr]; - float value = h_src[index]; + uint32_t value = h_src[index]; //printf("*** [%d] addr=%d, index=%d, value=%f\n", i, addr, index, value); - ref *= value; + ref = ref * 31 + value; } - float cur = h_dst[i]; - if (!almost_equal(cur, ref)) { + uint32_t cur = h_dst[i]; + if (cur != ref) { std::cout << "error at result #" << std::dec << i << ": actual " << cur << ", expected " << ref << std::endl; ++errors; From f0b75042815cde63cc0f3430ddbc8c12b188a81b Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:04 -0700 Subject: [PATCH 054/336] simx: give each warp's instruction buffer its staging slot In the RTL a warp's instructions wait in an IBUF_SIZE elastic buffer, pass the uop sequencer, and enter the scoreboard's per-warp staging register (a one-entry pipe buffer) before they issue. The buffer frees a slot when its head moves on to staging, and the uop sequencer takes a macro-op out of the buffer only with its last micro-op. SimX released the head's slot early as the RTL does, but its buffer FIFO held only IBUF_SIZE entries, the staged head included. Fetch then ran one instruction ahead into a buffer with no room for it, and decode blocked. A macro-op also gave up its slot with its first micro-op. The FIFO now holds IBUF_SIZE entries plus the head standing in for the staging register, and a macro-op releases its slot with its last micro-op. SimX vs rtlsim cycles (sgemm -n128 unless noted), tip -> this change: 16 threads x 16 warps, IW4, FW2 579,368 -> 573,905 raycast, 16 threads x 16 warps 376,910 -> 359,059 8 threads x 8 warps, 2 cores + L2 1,162,128 -> 1,133,705 sgemm default 797,701 -> 793,868 Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 13 +++++++++---- sim/simx/sequencer.h | 3 +++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 4fdf2f54ce..57b7a1c555 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -125,7 +125,9 @@ class Core::Impl { // create ibuffers for (uint32_t i = 0; i < ibuffers_.size(); ++i) { snprintf(sname, 100, "%s-ibuffer%d", name.c_str(), i); - ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE); + // IBUF_SIZE entries plus the head, which stands for the warp's staging + // register once it has moved there + ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE + 1); } // create the memory coalescer, or the lane adapter that replaces it. @@ -651,16 +653,19 @@ class Core::Impl { // check scoreboard against sequencer micro-op (handles uop WAW/RAW) auto trace = ibuffer->peek(); + auto seq = sequencers_.at(wid); + auto uop_trace = seq->get(trace); // returns cached uop or generates next + // The head moves into the warp's staging register, freeing its buffer // slot, once it has arrived and the warp's previous instruction has - // issued; the scheduler sees the freed slot the cycle after. + // issued; a macro-op leaves the buffer with its last micro-op. The + // scheduler sees the freed slot the cycle after. if (!head_staged_.at(wid) + && (!trace->instr_ptr->is_macro_op() || seq->last()) && SimPlatform::instance().cycles() > std::max(last_issue_.at(wid), trace->ibuf_time)) { head_staged_.at(wid) = true; this->ibuf_release(wid); } - auto seq = sequencers_.at(wid); - auto uop_trace = seq->get(trace); // returns cached uop or generates next // The uop sequencer starts a macro-op once it reaches the front of the // buffer behind the staged instruction, and offers its first uop the diff --git a/sim/simx/sequencer.h b/sim/simx/sequencer.h index 9f7222f270..31f525ec21 100644 --- a/sim/simx/sequencer.h +++ b/sim/simx/sequencer.h @@ -51,6 +51,9 @@ class Sequencer : public SimObject { // A macro-op's first micro-op is the current one. bool starting() const { return state_.active && state_.uop_index == 0; } + // A macro-op's last micro-op is the current one. + bool last() const { return state_.active && (state_.uop_index + 1) == state_.uop_count; } + // Drop any cached uop / macro-op state for this sequencer. Used by // Core::flush_warp_pipeline at async-trap entry: when the ibuffer is // flushed, the trace pointer this sequencer cached in state_.current_uop From 62dc764f4cfa7e78491bf1c8f54212da0afbe403 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:05 -0700 Subject: [PATCH 055/336] simx: send a fence to the data cache as a flush token In the RTL the LSU sends a fence to the data cache as a flush token on the lanes of its last packet, behind every request the block issued before it, and holds the block until the token is answered. A cache that sees a token holds its cacheable inputs, drains, walks its banks (writing back dirty lines and invalidating), sends one token down each memory port behind that port's write-backs, and answers the tokens visible at the end of the walk once every port has answered, one per cycle. SimX treated a fence as an ordering barrier in the LSU only. SimX now models the token end to end: - The LSU sends the token and releases the fence on its response. - The cache's port arbiter holds cacheable requests while a token is visible. The banks walk one entry per cycle. The token for port i takes bank i's lane into the bank-to-port arbiter, and a tap on each port's response path consumes the token's answer in the same cycle, so other responses are not delayed. - Bank i drives memory port i mod mem_ports, as in the RTL. SimX grouped contiguous banks per port, which sent tokens down the wrong port and deadlocked a fence on any cache with more banks than ports (at least 2); it also changes timing on such caches (IW=2 dcache, the 2-frontend shape, multi-bank L2/L3 with several ports) to match the RTL. - Flush tokens skip address translation. Fence-free runs at the default shape are cycle-identical. CI adds printf parity cases at the default shape, at IW=2 and on the 2-frontend shape. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 18 ++ docs/designs/multicache_amo_coherence.md | 3 +- sim/simx/lsu_unit.cpp | 52 +++- sim/simx/lsu_unit.h | 6 + sim/simx/mem/cache.cpp | 288 ++++++++++++++++++++++- sim/simx/mem/mmu.cpp | 5 +- sim/simx/types.h | 5 +- 7 files changed, 365 insertions(+), 12 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 6f9187911d..d7ab85e2bd 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -184,6 +184,24 @@ tests: app: sgemm args: -n128 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 +# Device printf: per-character IO ring stores, each published behind a fence +# that drains, flushes and invalidates the data cache. +- id: parity-printf + check: model_parity + via: blackbox + app: printf +# The same fences on data caches with more banks than memory ports: every +# port's flush token must be answered on that port. +- id: parity-printf-iw2 + check: model_parity + via: blackbox + app: printf + configs: -DVX_CFG_ISSUE_WIDTH=2 +- id: parity-printf-fw2 + check: model_parity + via: blackbox + app: printf + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity diff --git a/docs/designs/multicache_amo_coherence.md b/docs/designs/multicache_amo_coherence.md index b0233a0252..d553e9ce57 100644 --- a/docs/designs/multicache_amo_coherence.md +++ b/docs/designs/multicache_amo_coherence.md @@ -260,8 +260,7 @@ A fence ordering only device accesses (`fence ow, ow`, no R or W) sends an until its uncached path is empty, then forwards, so the answer proves the earlier IO writes reached memory. -SimX does not model the tokens yet: there the fence is an ordering barrier in -the LSU only. +SimX sends every fence as a full flush token, walking every level. Programs that share data across cores through plain memory are outside what is validated. Share through atomics, or through the single shared level. diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index c81c3458aa..40c2b6db80 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -236,6 +236,16 @@ void LsuUnit::process_response_step(uint32_t b) { } #endif + if (entry.is_flush) { + DT(3, this->name() << " fence-token-rsp: " << lsu_rsp); + entry.count -= lsu_rsp.mask.count(); + if (entry.count == 0) { + state.pending_reqs.release(lsu_rsp.tag); + } + lsu_rsp_in.pop(); + return; + } + auto trace = entry.trace; auto& output = Outputs.at(b); // Every response fragment retires through its own writeback slot, one @@ -351,6 +361,42 @@ void LsuUnit::ingest_inputs(uint32_t b) { input.pop(); } +// The fence travels to the data cache as a flush token on the lanes of its +// last packet, behind every request the block issued before it. The cache +// answers once it has drained, flushed and invalidated, so the fence holds +// the block until the token's response returns. +void LsuUnit::send_fence_token(uint32_t b) { + auto& state = states_.at(b); + if (state.pending_reqs.full() || core_->lmem_switch(b)->ReqIn.full()) + return; + auto trace = state.fence.trace(); + uint32_t base = ((VX_CFG_NUM_THREADS - 1) / VX_CFG_NUM_LSU_LANES) * VX_CFG_NUM_LSU_LANES; + LsuReq lsu_req(VX_CFG_NUM_LSU_LANES); + lsu_req.op = MemOp::LD; + lsu_req.flags.flush = 1; + std::vector lane_entries(VX_CFG_NUM_LSU_LANES); + for (uint32_t t = base; t < VX_CFG_NUM_THREADS; ++t) { + if (trace->tmask.test(t)) { + lsu_req.mask.set(t - base); + lsu_req.tids.at(t - base) = t; + } + } + if (lsu_req.mask.none()) { + lsu_req.mask.set(0); + lsu_req.tids.at(0) = base; + } + uint32_t count = lsu_req.mask.count(); + pending_req_t entry{nullptr, count, true, std::move(lane_entries), IntrLsuArgs{}, false}; + entry.is_flush = true; + lsu_req.tag = state.pending_reqs.allocate(std::move(entry)); + lsu_req.cid = trace->cid; + lsu_req.wid = trace->wid; + lsu_req.uuid = trace->uuid; + core_->lmem_switch(b)->ReqIn.send(lsu_req); + DT(3, this->name() << " fence-token: " << lsu_req); + state.fence_token = false; +} + // Per-cycle request handler. Dispatches at most one batch (one // memory-side mem_req) per call. The head trace stays at the queue // front until all its batches are issued (req_sent_all-driven). @@ -360,7 +406,10 @@ void LsuUnit::process_request_step(uint32_t b) { // Attempt fence release first so a steady stream of TCU client forwards // (which consume the dispatch slot and return) cannot starve it. const bool fence_locked = state.fence.locked(); - if (fence_locked) { + if (fence_locked && state.fence_token) { + this->send_fence_token(b); + } + if (fence_locked && !state.fence_token) { bool released = state.fence.try_release(Outputs.at(b), state.pending_reqs.empty()); if (released) { DT(3, this->name() << " fence-unlock: " << state.fence.trace()); @@ -438,6 +487,7 @@ void LsuUnit::process_request_step(uint32_t b) { // Engage fence lock; the next call will try to release it once // the pending-reqs table drains. state.fence.engage(trace); + state.fence_token = true; DT(3, this->name() << " fence-lock: " << *trace); state.req_queue.pop(); return; diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index 02a693998a..8ab00012dc 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -100,6 +100,7 @@ class LsuUnit : public FuncUnit { // Per-cycle handlers. void process_response_step(uint32_t b); void process_request_step(uint32_t b); + void send_fence_token(uint32_t b); // Drain Inputs[b] into req_queue. void ingest_inputs(uint32_t b); @@ -123,6 +124,8 @@ class LsuUnit : public FuncUnit { // TcuRspOut with client_tag restored instead of written to registers. bool is_tcu = false; uint32_t client_tag = 0; + // A fence's flush token: its response only releases the fence. + bool is_flush = false; }; // Per-block LSU state. Each member is a named hardware sub-block. @@ -140,6 +143,8 @@ class LsuUnit : public FuncUnit { uint32_t staged_reads = 0; bool head_staged = false; FenceController fence; + // the engaged fence's flush token is still to be sent + bool fence_token = false; std::vector addr_list; uint32_t remain_addrs = 0; @@ -149,6 +154,7 @@ class LsuUnit : public FuncUnit { this->staged_reads = 0; this->head_staged = false; this->fence.reset(); + this->fence_token = false; this->addr_list.clear(); this->remain_addrs = 0; } diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index efd5979a7e..a61cbc8ebd 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -779,6 +779,8 @@ class CacheBank : public SimObject { flush_sector_idx_ = 0; return; } + // the two walks share the sector cursor + __assert(!token_flushing_, "flush walk started during a flush-token walk"); flushing_ = true; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -790,6 +792,20 @@ class CacheBank : public SimObject { return !flushing_; } + // A flush token's walk: one entry per cycle, every set (every way of a + // write-back cache), writing back dirty data and invalidating the lines. + void token_flush_begin() { + __assert(!flushing_, "flush-token walk started during a flush walk"); + token_flushing_ = true; + token_flush_idx_ = 0; + flush_sector_idx_ = 0; + this->tick_wake(); + } + + bool token_flush_done() const { + return !token_flushing_; + } + protected: void on_reset() { perf_stats_ = Cache::PerfStats(); @@ -824,6 +840,8 @@ class CacheBank : public SimObject { flush_set_idx_ = 0; flush_way_idx_ = 0; flush_sector_idx_ = 0; + token_flushing_ = false; + token_flush_idx_ = 0; #if VX_CFG_EXT_A_ENABLED amo_unit_.reset(); for (auto &e : amo_passthru_) { @@ -859,6 +877,9 @@ class CacheBank : public SimObject { if (flushing_) { this->processFlush(); } + if (token_flushing_) { + this->processTokenFlush(); + } // calculate memory latency perf_stats_.mem_latency += pending_fill_reqs_; @@ -869,7 +890,7 @@ class CacheBank : public SimObject { // any packet reserved toward core_req_in/mem_rsp_in re-arms on arrival. if (this->core_req_in.size() == 0 && this->mem_rsp_in.size() == 0 && pipe_req_->size() == 0 && lazy_q_->size() == 0 && !mshr_.has_ready_reqs() - && !fwd_active_ && !flushing_ && pending_fill_reqs_ == 0) { + && !fwd_active_ && !flushing_ && !token_flushing_ && pending_fill_reqs_ == 0) { this->tick_sleep(); } } @@ -1973,6 +1994,47 @@ class CacheBank : public SimObject { DT(3, this->name() << " flush-done"); } + void processTokenFlush() { + if (pending_fill_reqs_ != 0 + || pipe_req_->size() != 0 + || !mshr_.empty()) { + return; + } + uint32_t ways = config_.write_back ? params_.lines_per_set : 1; + uint32_t set_id = token_flush_idx_ % params_.sets_per_bank; + uint32_t way_id = token_flush_idx_ / params_.sets_per_bank; + auto &set = sets_.at(set_id); + if (config_.write_back) { + auto &line = set.lines.at(way_id); + while (flush_sector_idx_ < line.sectors.size()) { + auto &sec = line.sectors.at(flush_sector_idx_); + if (sec.valid && sec.dirty) { + if (this->mem_req_out.full()) + return; + MemReq mem_req; + mem_req.addr = params_.mem_addr_sector(bank_id_, set_id, line.tag, flush_sector_idx_); + mem_req.op = MemOp::ST; + mem_req.data = sec.data; + mem_req.byteen = sec.dirty_mask; + this->mem_req_out.send(mem_req, MEM_REQ_DELAY); + DT(3, this->name() << " flush-wb: " << mem_req); + ++perf_stats_.evictions; + } + ++flush_sector_idx_; + } + flush_sector_idx_ = 0; + line.reset(); + } else { + for (auto &line : set.lines) { + line.reset(); + } + } + if (++token_flush_idx_ == params_.sets_per_bank * ways) { + token_flushing_ = false; + DT(3, this->name() << " token-flush-done"); + } + } + Cache::Config config_; params_t params_; uint32_t bank_id_; @@ -2037,6 +2099,8 @@ class CacheBank : public SimObject { uint32_t flush_set_idx_; uint32_t flush_way_idx_; uint32_t flush_sector_idx_; + bool token_flushing_; + uint32_t token_flush_idx_; #if VX_CFG_EXT_A_ENABLED AmoUnit amo_unit_; @@ -2091,6 +2155,39 @@ class CachePortArb : public SimObject { uint64_t collisions() const { return collisions_; } + // Ports whose head is a flush token. While any is visible, cacheable + // requests are held at every port. + uint64_t tokens() const { + uint64_t mask = 0; + for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { + if (!ReqIn.at(i).empty() && ReqIn.at(i).peek().flags.flush) { + mask |= 1ull << i; + } + } + return mask; + } + + // No cacheable request is left between the ports and the banks. + bool drained() const { + for (auto &stage : tree_stage_) { + if (stage.size() != 0) { + return false; + } + } + for (auto &out : ReqOut) { + if (out.size() != 0) { + return false; + } + } + return true; + } + + MemReq take_token(uint32_t i) { + auto req = ReqIn.at(i).peek(); + ReqIn.at(i).pop(); + return req; + } + // The cache's line initialization holds the core ports for these cycles. void set_init_cycles(uint32_t cycles) { init_cycles_ = cycles; } @@ -2114,6 +2211,7 @@ class CachePortArb : public SimObject { } } uint32_t n = ReqIn.size(); + const bool hold = (this->tokens() != 0); // requesting ports per bank; non-cacheable requests leave on their own port std::vector want(ReqOut.size(), 0); for (uint32_t i = 0; i < n; ++i) { @@ -2122,13 +2220,18 @@ class CachePortArb : public SimObject { continue; } auto& req = in.peek(); + if (req.flags.flush) { + continue; + } if (req.flags.io) { if (IoOut.at(i).try_send(req, 0)) { in.pop(); } continue; } - want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; + if (!hold) { + want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; + } } for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { auto& out = tree_stage_.empty() ? ReqOut.at(b) : tree_stage_.at(b); @@ -2160,6 +2263,64 @@ class CachePortArb : public SimObject { friend class SimObject; }; +// Sits on one memory port's cache-side response path, ahead of the response +// network. While the port's flush token is out the cache is quiescent, so the +// port's next response is the token's answer and stops here; every other +// response passes in the same cycle. +class FlushTap : public SimObject { +public: + using Ptr = std::shared_ptr; + + SimChannel RspIn; + SimChannel RspOut; + + FlushTap(const SimContext& ctx, const char* name) + : SimObject(ctx, name) + , RspIn(this) + , RspOut(this) + {} + + // the port's flush token has been sent + void arm() { + armed_ = true; + answered_ = false; + } + + // the token's answer has arrived; reading it disarms the tap + bool take_answer() { + bool answered = answered_; + answered_ = false; + return answered; + } + +protected: + void on_reset() { + armed_ = false; + answered_ = false; + } + + void on_tick() { + if (RspIn.empty()) { + this->tick_sleep(); + return; + } + if (armed_) { + DT(3, this->name() << " flush-fwd-rsp"); + armed_ = false; + answered_ = true; + RspIn.pop(); + } else if (RspOut.try_send(RspIn.peek(), 0)) { + RspIn.pop(); + } + } + +private: + bool armed_ = false; + bool answered_ = false; + + friend class SimObject; +}; + class Cache::Impl { public: Impl(Cache *simobject, const Config &config) @@ -2188,6 +2349,15 @@ class Cache::Impl { nc_mem_arbs_.at(i) = MemArbiter::Create(sname, ArbiterType::Priority, 2, 1); } + // flush-token taps, ticking after the arbiters that feed them and before + // the response network they feed + flush_taps_.resize(config_.mem_ports); + for (uint32_t i = 0; i < config_.mem_ports; ++i) { + snprintf(sname, 100, "%s-flush_tap%d", simobject->name().c_str(), i); + flush_taps_.at(i) = FlushTap::Create(sname); + nc_mem_arbs_.at(i)->RspOut.at(0).bind(&flush_taps_.at(i)->RspIn); + } + // Connect non-cacheable arbiter output port 0 to outgoing memory ports for (uint32_t i = 0; i < config_.mem_ports; ++i) { nc_mem_arbs_.at(i)->ReqOut.at(0).bind(&simobject->mem_req_out.at(i)); @@ -2202,6 +2372,7 @@ class Cache::Impl { uint32_t mem_arb_delay = ((num_banks != config_.mem_ports) && (num_banks > 2 || config_.mem_ports > 2)) ? 1 : 0; uint32_t mem_net_delay = omega_stages(num_banks, config_.mem_ports) - 1; uint32_t mem_net_extra = (mem_net_delay > mem_arb_delay) ? (mem_net_delay - mem_arb_delay) : 0; + __assert(num_banks >= config_.mem_ports, "a cache needs a bank per memory port"); auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports, mem_arb_delay + mem_net_extra, mem_arb_delay + mem_net_extra); @@ -2218,8 +2389,11 @@ class Cache::Impl { } else { bank_mem_arb->ReqOut.at(i).bind(&nc_mem_arbs_.at(i)->ReqIn.at(0)); } - nc_mem_arbs_.at(i)->RspOut.at(0).bind(&bank_mem_arb->RspIn.at(i)); } + for (uint32_t i = 0; i < config_.mem_ports; ++i) { + flush_taps_.at(i)->RspOut.bind(&bank_mem_arb->RspIn.at(i)); + } + bank_arb_ = bank_mem_arb; // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); @@ -2255,8 +2429,8 @@ class Cache::Impl { banks_.at(i)->core_rsp_out.bind(&bank_core_xbar_->RspIn.at(i)); // bind memory ports - banks_.at(i)->mem_req_out.bind(&bank_mem_arb->ReqIn.at(i)); - bank_mem_arb->RspOut.at(i).bind(&banks_.at(i)->mem_rsp_in); + banks_.at(i)->mem_req_out.bind(&bank_mem_arb->ReqIn.at(this->bank_lane(i))); + bank_mem_arb->RspOut.at(this->bank_lane(i)).bind(&banks_.at(i)->mem_rsp_in); } } @@ -2292,6 +2466,8 @@ class Cache::Impl { } } + this->flush_step(); + // bank memory requests through the memory-side network's deeper levels for (uint32_t i = 0, n = mem_req_net_.size(); i < n; ++i) { auto &net = mem_req_net_.at(i); @@ -2321,6 +2497,94 @@ class Cache::Impl { } } + // Flush tokens (fences) at any port start one walk: the ports hold their + // cacheable requests, in-flight ones drain, every bank writes back and + // invalidates its lines, a token goes down each memory port behind that + // port's write-backs, and once every port has answered, each token + // visible at the end of the walk is answered through the response path, + // one per cycle. + void flush_step() { + switch (flush_state_) { + case FlushState::Idle: + if (port_arb_->tokens() != 0) { + flush_state_ = FlushState::Drain; + } + break; + case FlushState::Drain: { + if (!port_arb_->drained()) + break; + for (auto &bank : banks_) { + if (bank->core_req_in.size() != 0) + return; + } + for (auto &bank : banks_) { + bank->token_flush_begin(); + } + flush_state_ = FlushState::Walk; + break; + } + case FlushState::Walk: + for (auto &bank : banks_) { + if (!bank->token_flush_done()) + return; + } + rsp_pending_ = port_arb_->tokens(); + fwd_sent_ = 0; + fwd_done_ = 0; + flush_state_ = FlushState::Forward; + break; + case FlushState::Forward: { + bool mem_req_empty = true; + for (auto &lane : bank_arb_->ReqIn) { + mem_req_empty &= (lane.size() == 0); + } + for (auto &net : mem_req_net_) { + mem_req_empty &= (net.size() == 0); + } + for (uint32_t i = 0, n = config_.mem_ports; i < n && mem_req_empty; ++i) { + if (fwd_sent_ & (1ull << i)) + continue; + // the token for port i is a read of the port's first line, sent down + // bank i's lane (bank i maps to port i) behind its write-backs + MemReq token; + token.addr = uint64_t(i) << config_.L; + token.flags.flush = 1; + token.tag = 0; + if (bank_arb_->ReqIn.at(this->bank_lane(i)).try_send(token, 1)) { + DT(3, simobject_->name() << " flush-fwd: " << token); + flush_taps_.at(i)->arm(); + fwd_sent_ |= 1ull << i; + } + } + for (uint32_t i = 0, n = config_.mem_ports; i < n; ++i) { + if (flush_taps_.at(i)->take_answer()) { + fwd_done_ |= 1ull << i; + } + } + uint64_t all = (config_.mem_ports < 64) ? ((1ull << config_.mem_ports) - 1) : ~0ull; + if (fwd_done_ == all) { + flush_state_ = FlushState::Respond; + } + break; + } + case FlushState::Respond: { + if (rsp_pending_ == 0) { + flush_state_ = FlushState::Idle; + break; + } + uint32_t i = __builtin_ctzll(rsp_pending_); + auto &token = port_arb_->ReqIn.at(i).peek(); + MemRsp rsp{token.tag, token.hart_id, token.uuid}; + if (simobject_->core_rsp_out.at(i).try_send(rsp, core_rsp_delay_)) { + DT(3, simobject_->name() << " flush-rsp: " << rsp); + port_arb_->take_token(i); + rsp_pending_ &= ~(1ull << i); + } + break; + } + } + } + PerfStats perf_stats() const { PerfStats perf_stats; if (!config_.bypass) { @@ -2388,7 +2652,21 @@ class Cache::Impl { // the core-response network's levels, then the non-cacheable bypass // layer's response arbiter uint32_t core_rsp_delay_ = 0; + enum class FlushState { Idle, Drain, Walk, Forward, Respond }; + FlushState flush_state_ = FlushState::Idle; + uint64_t rsp_pending_ = 0; + uint64_t fwd_sent_ = 0; + uint64_t fwd_done_ = 0; std::vector> mem_req_net_; + std::vector flush_taps_; + + // Bank i drives memory port i mod mem_ports: its lane into the bank-to-port + // arbiter, which serves each port from a contiguous range of lanes. + uint32_t bank_lane(uint32_t bank_id) const { + uint32_t num_banks = 1 << config_.B; + uint32_t banks_per_port = num_banks / config_.mem_ports; + return (bank_id % config_.mem_ports) * banks_per_port + bank_id / config_.mem_ports; + } uint32_t init_cycles_; bool keep_lines_ = false; }; diff --git a/sim/simx/mem/mmu.cpp b/sim/simx/mem/mmu.cpp index f2c440a3fe..e2d578ba9e 100644 --- a/sim/simx/mem/mmu.cpp +++ b/sim/simx/mem/mmu.cpp @@ -91,11 +91,12 @@ bool Mmu::needs_translation(const MemReq& req) const { // needed. BARE mode (SATP unprogrammed) skips translation, covering // the few instruction fetches between reset and the kernel's csrw // satp. IO-flagged requests skip it too: the IO/OM apertures carry - // device registers or encoded coordinates, not virtual addresses. + // device registers or encoded coordinates, not virtual addresses. So do + // flush tokens, which carry no address. if (!satp_ || satp_->get_mode() == BARE) { return false; } - if (req.flags.io) { + if (req.flags.io || req.flags.flush) { return false; } return true; diff --git a/sim/simx/types.h b/sim/simx/types.h index 85b248ade7..d85c1186db 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -444,6 +444,7 @@ struct MemFlags { uint32_t io : 1; // bit 1: uncacheable I/O uint32_t local : 1; // bit 2: LMEM port uint32_t amo_unsigned : 1; // bit 3: MIN/MAX signedness (1=unsigned) + uint32_t flush : 1; // bit 4: flush token (a fence) #ifdef VX_CFG_EXT_DXA_ENABLE // DXA completion sideband — populated only on DXA's LMEM-DMA writes. // notify_bar_id carries the RAW (kernel-encoded) bar handle: @@ -453,8 +454,8 @@ struct MemFlags { // be wide enough to hold (bar_no << 8) | (MAX_CTAS_per_core - 1). // For NUM_BARRIERS = 8 and NUM_WARPS ≤ 16, that's at most 11 bits; // 16 bits leaves headroom for larger configs. - uint32_t dxa_notify_done : 1; // bit 4 - uint32_t dxa_notify_bar_id : 16; // bits 5..20 + uint32_t dxa_notify_done : 1; // bit 5 + uint32_t dxa_notify_bar_id : 16; // bits 6..21 #endif }; }; From 18b6fe5219cbd69796788d4cd5c1df57df496eb8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:05 -0700 Subject: [PATCH 056/336] simx: stop fences at the point of coherence and skip the walk for IO fences SimX side of the targeted fences in the RTL (VX_decode, VX_lsu_block, VX_cache_init, VX_cache_flush, VX_cache_bypass): - A memory fence (any R or W in its predecessor or successor set) sends an L1-only token. Caches above the shared LLC walk and forward it with the same kind; the shared LLC answers it without a walk; a replicated LLC still walks. - A fence ordering only device accesses decodes as an IO fence and sends an IO token. No level walks: each quiesces its banks, waits until nothing is held between its inputs and its memory ports on the cacheable and the non-cacheable path, and forwards an IO token on every port. - Tokens of different kinds visible together combine to the weakest kind that serves them all; a quiesce without a walk answers only the tokens that owe none. - The LSU adapters keep the token's io bit instead of deriving it from the address. fence_mp now also runs on simx. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 4 + docs/designs/multicache_amo_coherence.md | 2 - sim/simx/decode.cpp | 5 +- sim/simx/lsu_unit.cpp | 7 +- sim/simx/mem/cache.cpp | 118 ++++++++++++++++++++--- sim/simx/mem/lsu_mem_adapter.cpp | 4 +- sim/simx/mem/mem_coalescer.cpp | 2 +- sim/simx/types.h | 9 +- 8 files changed, 130 insertions(+), 21 deletions(-) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index b59f66d3a7..ccaf86aa66 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -245,11 +245,13 @@ tests: - id: fence_mp-1 via: blackbox drivers: + - simx - rtlsim app: fence_mp - id: fence_mp-2 via: blackbox drivers: + - simx - rtlsim app: fence_mp shape: @@ -258,6 +260,7 @@ tests: - id: fence_mp-3 via: blackbox drivers: + - simx - rtlsim app: fence_mp shape: @@ -267,6 +270,7 @@ tests: - id: fence_mp-4 via: blackbox drivers: + - simx - rtlsim app: fence_mp shape: diff --git a/docs/designs/multicache_amo_coherence.md b/docs/designs/multicache_amo_coherence.md index d553e9ce57..9525593bc1 100644 --- a/docs/designs/multicache_amo_coherence.md +++ b/docs/designs/multicache_amo_coherence.md @@ -260,8 +260,6 @@ A fence ordering only device accesses (`fence ow, ow`, no R or W) sends an until its uncached path is empty, then forwards, so the answer proves the earlier IO writes reached memory. -SimX sends every fence as a full flush token, walking every level. - Programs that share data across cores through plain memory are outside what is validated. Share through atomics, or through the single shared level. diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 9fb835db56..e0a3d8593e 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -631,7 +631,10 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { case Opcode::FENCE: { instr->set_fu_type(FUType::LSU); instr->set_op_type(LsuType::FENCE); - instr->set_args(IntrLsuArgs{0, 0, 0}); + // neither the predecessor nor the successor set holds a memory read or + // write (pred = code[27:24], succ = code[23:20], IORW) + uint32_t fence_io = (funct3 == 0) && ((code >> 24) & 0x3) == 0 && ((code >> 20) & 0x3) == 0; + instr->set_args(IntrLsuArgs{fence_io, 0, 0}); } break; case Opcode::AMO: { instr->set_fu_type(FUType::LSU); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 40c2b6db80..e17a131bee 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -367,13 +367,18 @@ void LsuUnit::ingest_inputs(uint32_t b) { // the block until the token's response returns. void LsuUnit::send_fence_token(uint32_t b) { auto& state = states_.at(b); - if (state.pending_reqs.full() || core_->lmem_switch(b)->ReqIn.full()) + if (state.pending_reqs.full() || core_->lmem_switch(b)->ReqIn.full()) { return; + } auto trace = state.fence.trace(); uint32_t base = ((VX_CFG_NUM_THREADS - 1) / VX_CFG_NUM_LSU_LANES) * VX_CFG_NUM_LSU_LANES; LsuReq lsu_req(VX_CFG_NUM_LSU_LANES); lsu_req.op = MemOp::LD; lsu_req.flags.flush = 1; + // an IO fence's token walks nowhere; a memory fence's stops at the shared LLC + auto fence_args = std::get(trace->instr_ptr->get_args()); + lsu_req.flags.io = lsu_fence_is_io(fence_args); + lsu_req.flags.flush_l1 = !lsu_fence_is_io(fence_args); std::vector lane_entries(VX_CFG_NUM_LSU_LANES); for (uint32_t t = base; t < VX_CFG_NUM_THREADS; ++t) { if (trace->tmask.test(t)) { diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index a61cbc8ebd..b74b4bce1a 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -794,9 +794,10 @@ class CacheBank : public SimObject { // A flush token's walk: one entry per cycle, every set (every way of a // write-back cache), writing back dirty data and invalidating the lines. - void token_flush_begin() { + void token_flush_begin(bool skip) { __assert(!flushing_, "flush-token walk started during a flush walk"); token_flushing_ = true; + token_flush_skip_ = skip; token_flush_idx_ = 0; flush_sector_idx_ = 0; this->tick_wake(); @@ -2000,6 +2001,11 @@ class CacheBank : public SimObject { || !mshr_.empty()) { return; } + if (token_flush_skip_) { + token_flushing_ = false; + DT(3, this->name() << " token-quiesce-done"); + return; + } uint32_t ways = config_.write_back ? params_.lines_per_set : 1; uint32_t set_id = token_flush_idx_ % params_.sets_per_bank; uint32_t way_id = token_flush_idx_ / params_.sets_per_bank; @@ -2009,8 +2015,9 @@ class CacheBank : public SimObject { while (flush_sector_idx_ < line.sectors.size()) { auto &sec = line.sectors.at(flush_sector_idx_); if (sec.valid && sec.dirty) { - if (this->mem_req_out.full()) + if (this->mem_req_out.full()) { return; + } MemReq mem_req; mem_req.addr = params_.mem_addr_sector(bank_id_, set_id, line.tag, flush_sector_idx_); mem_req.op = MemOp::ST; @@ -2100,6 +2107,7 @@ class CacheBank : public SimObject { uint32_t flush_way_idx_; uint32_t flush_sector_idx_; bool token_flushing_; + bool token_flush_skip_ = false; uint32_t token_flush_idx_; #if VX_CFG_EXT_A_ENABLED @@ -2167,6 +2175,22 @@ class CachePortArb : public SimObject { return mask; } + // Tokens owing no walk: IO tokens, and L1-only ones unless they still need + // a walk here. + uint64_t tokens(bool l1_needs_walk) const { + uint64_t mask = 0; + for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { + if (ReqIn.at(i).empty()) { + continue; + } + auto &flags = ReqIn.at(i).peek().flags; + if (flags.flush && (flags.io || (flags.flush_l1 && !l1_needs_walk))) { + mask |= 1ull << i; + } + } + return mask; + } + // No cacheable request is left between the ports and the banks. bool drained() const { for (auto &stage : tree_stage_) { @@ -2511,28 +2535,43 @@ class Cache::Impl { } break; case FlushState::Drain: { - if (!port_arb_->drained()) + if (!port_arb_->drained()) { break; + } for (auto &bank : banks_) { - if (bank->core_req_in.size() != 0) + if (bank->core_req_in.size() != 0) { return; + } + } + auto kinds = this->token_kinds(); + bool walk = kinds.walk(config_.shared_llc); + if (!walk && !kinds.any_io) { + rsp_pending_ = port_arb_->tokens(); + flush_state_ = FlushState::Respond; + break; } + // with no walk owed, an IO token still quiesces the banks and goes down + // to memory + walk_skip_ = !walk; for (auto &bank : banks_) { - bank->token_flush_begin(); + bank->token_flush_begin(walk_skip_); } flush_state_ = FlushState::Walk; break; } - case FlushState::Walk: + case FlushState::Walk: { for (auto &bank : banks_) { - if (!bank->token_flush_done()) + if (!bank->token_flush_done()) { return; + } } - rsp_pending_ = port_arb_->tokens(); - fwd_sent_ = 0; - fwd_done_ = 0; + auto kinds = this->token_kinds(); + // a quiesce without a walk serves only the tokens owing none + rsp_pending_ = walk_skip_ ? port_arb_->tokens(!config_.shared_llc) : port_arb_->tokens(); + this->forward_begin(walk_skip_, !walk_skip_ && kinds.any_l1 && !kinds.any_full && !kinds.any_io); flush_state_ = FlushState::Forward; break; + } case FlushState::Forward: { bool mem_req_empty = true; for (auto &lane : bank_arb_->ReqIn) { @@ -2541,14 +2580,28 @@ class Cache::Impl { for (auto &net : mem_req_net_) { mem_req_empty &= (net.size() == 0); } - for (uint32_t i = 0, n = config_.mem_ports; i < n && mem_req_empty; ++i) { - if (fwd_sent_ & (1ull << i)) + // an IO token also waits for the non-cacheable path, which carries the + // IO writes it orders + if (fwd_io_ && mem_req_empty) { + for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { + mem_req_empty &= (port_arb_->IoOut.at(i).size() == 0); + } + for (auto &arb : nc_mem_arbs_) { + mem_req_empty &= (arb->ReqIn.at(0).size() == 0) && (arb->ReqIn.at(1).size() == 0); + } + } + fwd_go_ |= mem_req_empty; + for (uint32_t i = 0, n = config_.mem_ports; i < n && fwd_go_; ++i) { + if (fwd_sent_ & (1ull << i)) { continue; + } // the token for port i is a read of the port's first line, sent down // bank i's lane (bank i maps to port i) behind its write-backs MemReq token; token.addr = uint64_t(i) << config_.L; token.flags.flush = 1; + token.flags.io = fwd_io_; + token.flags.flush_l1 = fwd_l1_; token.tag = 0; if (bank_arb_->ReqIn.at(this->bank_lane(i)).try_send(token, 1)) { DT(3, simobject_->name() << " flush-fwd: " << token); @@ -2585,6 +2638,43 @@ class Cache::Impl { } } + struct token_kinds_t { + bool any_full = false; + bool any_l1 = false; + bool any_io = false; + // a walk is owed to a full token, and to an L1-only one above the point of coherence + bool walk(bool shared_llc) const { return any_full || (any_l1 && !shared_llc); } + }; + + // Kinds of the visible tokens. Several kinds together combine to the + // weakest one that serves them all. + token_kinds_t token_kinds() const { + token_kinds_t kinds; + uint64_t tokens = port_arb_->tokens(); + for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { + if (!(tokens & (1ull << i))) { + continue; + } + auto &flags = port_arb_->ReqIn.at(i).peek().flags; + if (flags.io) { + kinds.any_io = true; + } else if (flags.flush_l1) { + kinds.any_l1 = true; + } else { + kinds.any_full = true; + } + } + return kinds; + } + + void forward_begin(bool io, bool l1_only) { + fwd_io_ = io; + fwd_l1_ = l1_only; + fwd_go_ = false; + fwd_sent_ = 0; + fwd_done_ = 0; + } + PerfStats perf_stats() const { PerfStats perf_stats; if (!config_.bypass) { @@ -2657,6 +2747,10 @@ class Cache::Impl { uint64_t rsp_pending_ = 0; uint64_t fwd_sent_ = 0; uint64_t fwd_done_ = 0; + bool fwd_go_ = false; // nothing is left ahead of the forwarded tokens + bool fwd_io_ = false; // the forwarded tokens' kind + bool fwd_l1_ = false; + bool walk_skip_ = false; // the banks only quiesce: no walk is owed std::vector> mem_req_net_; std::vector flush_taps_; diff --git a/sim/simx/mem/lsu_mem_adapter.cpp b/sim/simx/mem/lsu_mem_adapter.cpp index a34aacfe6c..60076bb902 100644 --- a/sim/simx/mem/lsu_mem_adapter.cpp +++ b/sim/simx/mem/lsu_mem_adapter.cpp @@ -47,7 +47,7 @@ LsuMemAdapter::LsuMemAdapter( mr.flags = req.flags; mr.amo_cmp = req.amo_cmp.at(0); auto t = get_addr_type(mr.addr); - mr.flags.io = (t == AddrType::IO); + mr.flags.io = mr.flags.flush ? mr.flags.io : (t == AddrType::IO); mr.flags.local = (t == AddrType::Shared); return mr; }); @@ -142,7 +142,7 @@ void LsuMemAdapter::on_tick() { out_req.flags = in_req.flags; { auto t = get_addr_type(out_req.addr); - out_req.flags.io = (t == AddrType::IO); + out_req.flags.io = out_req.flags.flush ? out_req.flags.io : (t == AddrType::IO); out_req.flags.local = (t == AddrType::Shared); } diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index 51a0b95a12..239acc1b50 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -273,7 +273,7 @@ void MemCoalescer::on_tick() { mr.uuid = in_req.uuid; mr.flags = in_req.flags; auto t = get_addr_type(mr.addr); - mr.flags.io = (t == AddrType::IO); + mr.flags.io = mr.flags.flush ? mr.flags.io : (t == AddrType::IO); mr.flags.local = (t == AddrType::Shared); } diff --git a/sim/simx/types.h b/sim/simx/types.h index d85c1186db..f3f9b4cf84 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -386,6 +386,10 @@ struct IntrLsuArgs { int32_t offset; // immediate addr offset (signed); 0 for packLD uops }; +// A fence carries no access width; its width field says it orders only +// device (IO) accesses. +inline bool lsu_fence_is_io(const IntrLsuArgs& args) { return args.width != 0; } + inline std::ostream &operator<<(std::ostream &os, const LsuType& type) { switch (type) { case LsuType::LOAD: os << "LOAD"; break; @@ -445,6 +449,7 @@ struct MemFlags { uint32_t local : 1; // bit 2: LMEM port uint32_t amo_unsigned : 1; // bit 3: MIN/MAX signedness (1=unsigned) uint32_t flush : 1; // bit 4: flush token (a fence) + uint32_t flush_l1 : 1; // bit 5: L1-only flush token: stops at the shared LLC #ifdef VX_CFG_EXT_DXA_ENABLE // DXA completion sideband — populated only on DXA's LMEM-DMA writes. // notify_bar_id carries the RAW (kernel-encoded) bar handle: @@ -454,8 +459,8 @@ struct MemFlags { // be wide enough to hold (bar_no << 8) | (MAX_CTAS_per_core - 1). // For NUM_BARRIERS = 8 and NUM_WARPS ≤ 16, that's at most 11 bits; // 16 bits leaves headroom for larger configs. - uint32_t dxa_notify_done : 1; // bit 5 - uint32_t dxa_notify_bar_id : 16; // bits 6..21 + uint32_t dxa_notify_done : 1; // bit 6 + uint32_t dxa_notify_bar_id : 16; // bits 7..22 #endif }; }; From 2342ffdcd323e764c8f544348104d81c7ecd3af8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:05 -0700 Subject: [PATCH 057/336] simx: serve a warp's identical loads with one dcache bank access SimX side of the dcache read merge in the RTL (VX_cache with READ_MERGE, VX_CFG_DCACHE_READ_MERGE, default on): - Port arbiter: plain reads (no write, flush, IO, local or atomic attribute) of the same word with the same tag, from ports of one LSU block in the same cycle, are served by one bank access. The lowest such port leads; the others stay out of arbitration and are accepted in the cycle their leader is granted. Nothing merges while a flush token holds the inputs. - The merged ports' mask rides below the request tag through the bank. - Response network: with merging, the RTL replaces the omega network with a flat crossbar of one buffered round-robin arbiter per core port over the bank lanes. A lane delivers a merged response to every target port, each when that port's arbiter grants it, and is released once all have taken it. Its depth is the output buffer plus the arbiter tree's buffered level above 12 banks, in place of the omega's levels. - The read counter counts a merged read once, since it reaches the bank once. A new parity case runs uniform_ld at 16 threads, where up to four channels of one warp merge. SimX, merge off -> on: sgemm -n128 on the 2-frontend shape (16 dcache ports in groups of 4) 571,848 -> 556,176 with dcache requests 1,069,056 -> 741,224 (rtlsim: 719,541, 528,299 cycles). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 7 ++ sim/simx/mem/cache.cpp | 159 +++++++++++++++++++++++++++++++++++++++-- sim/simx/mem/cache.h | 1 + sim/simx/socket.cpp | 1 + 4 files changed, 161 insertions(+), 7 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index d7ab85e2bd..1ae0064d19 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -202,6 +202,13 @@ tests: via: blackbox app: printf configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 +# Same-word reads from several coalescer channels of one warp, served by one +# dcache bank access and returned to every channel. +- id: parity-uniform-ld + check: model_parity + via: blackbox + app: uniform_ld + configs: -DVX_CFG_NUM_THREADS=16 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index b74b4bce1a..f813cdec29 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2139,6 +2139,10 @@ class CacheBank : public SimObject { // while the bank's input buffer has room; non-cacheable requests are split off // for the bypass path. It ticks ahead of the banks, so a grant reaches its bank // in the same cycle unless the crossbar registers its output. +// With read merging, plain reads of the same word with the same tag from ports +// of one group in the same cycle are served by one bank access: the lowest +// such port leads, the others stay out of arbitration and are accepted with +// it. The merged ports' mask rides below the tag through the bank. class CachePortArb : public SimObject { public: std::vector> ReqIn; // core ports @@ -2146,7 +2150,7 @@ class CachePortArb : public SimObject { std::vector> IoOut; // non-cacheable requests, per port CachePortArb(const SimContext& ctx, const char* name, const params_t& params, - uint32_t num_inputs, uint32_t num_banks) + uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group) : SimObject(ctx, name) , ReqIn(num_inputs, this) , ReqOut(num_banks, this) @@ -2157,10 +2161,16 @@ class CachePortArb : public SimObject { , delay_((num_inputs > 2) ? 1 : 0) , rr_(num_banks, num_inputs - 1) , tree_stage_((arb_tree_delay(num_inputs) != 0) ? num_banks : 0, this) + , merge_group_(merge_group) { assert(num_inputs <= 64); + __assert(merge_group_ <= 1 || (ispow2(merge_group_) && (num_inputs % merge_group_) == 0), + "a read-merge group must be a power of two dividing the ports"); } + // Bits of the merged-port mask carried below the request tag. + uint32_t merge_bits() const { return (merge_group_ > 1) ? merge_group_ : 0; } + uint64_t collisions() const { return collisions_; } // Ports whose head is a flush token. While any is visible, cacheable @@ -2236,11 +2246,34 @@ class CachePortArb : public SimObject { } uint32_t n = ReqIn.size(); const bool hold = (this->tokens() != 0); + // merged ports riding each leader's bank access + uint64_t followers = 0; + std::vector merged(n, 0); + if (merge_group_ > 1 && !hold) { + for (uint32_t i = 0; i < n; ++i) { + if (!this->is_plain_read(i)) { + continue; + } + auto& req_i = ReqIn.at(i).peek(); + for (uint32_t j = (i / merge_group_) * merge_group_; j < i; ++j) { + if ((followers & (1ull << j)) || !this->is_plain_read(j)) { + continue; + } + auto& req_j = ReqIn.at(j).peek(); + if ((req_j.addr >> params_.word_select_addr_start) == (req_i.addr >> params_.word_select_addr_start) + && req_j.tag == req_i.tag) { + merged.at(j) |= 1ull << i; + followers |= 1ull << i; + break; + } + } + } + } // requesting ports per bank; non-cacheable requests leave on their own port std::vector want(ReqOut.size(), 0); for (uint32_t i = 0; i < n; ++i) { auto& in = ReqIn.at(i); - if (in.empty()) { + if (in.empty() || (followers & (1ull << i))) { continue; } auto& req = in.peek(); @@ -2268,13 +2301,32 @@ class CachePortArb : public SimObject { } while (!(want[b] & (1ull << i))); collisions_ += __builtin_popcountll(want[b]) - 1; auto req = ReqIn.at(i).peek(); - req.tag = (req.tag << lg2_inputs_) | i; + uint64_t mask = 0; + if (merge_group_ > 1) { + mask = merged.at(i) >> ((i / merge_group_) * merge_group_); + for (uint64_t f = merged.at(i); f != 0; f &= f - 1) { + DT(3, this->name() << " read-merge: port=" << __builtin_ctzll(f) << ", leader=" << i << ", " << req); + ReqIn.at(__builtin_ctzll(f)).pop(); + } + } + uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; + __assert((tag >> 32) == 0, "cache bank tag overflow"); + req.tag = uint32_t(tag); out.send(req, tree_stage_.empty() ? delay_ : 1); ReqIn.at(i).pop(); rr_[b] = i; } } + // A plain read: no write, flush, IO, local or atomic attribute. + bool is_plain_read(uint32_t i) const { + if (ReqIn.at(i).empty()) { + return false; + } + auto& req = ReqIn.at(i).peek(); + return req.op == MemOp::LD && !req.flags.flush && !req.flags.io && !req.flags.local; + } + private: params_t params_; uint32_t lg2_inputs_; @@ -2282,11 +2334,97 @@ class CachePortArb : public SimObject { uint32_t init_cycles_ = 0; std::vector rr_; std::vector> tree_stage_; + uint32_t merge_group_; uint64_t collisions_ = 0; friend class SimObject; }; +// The core-response network under read merging: a flat crossbar of one +// round-robin arbiter per core port over the bank lanes. A merged read's +// response goes to the port that sent it and to every merged port; each port +// takes it when its arbiter grants that lane, and the lane is released once +// all of them have. +class CacheRspMcast : public SimObject { +public: + using Ptr = std::shared_ptr; + + std::vector> RspIn; // banks + std::vector> RspOut; // core ports + + CacheRspMcast(const SimContext& ctx, const char* name, uint32_t num_banks, + uint32_t num_outputs, uint32_t merge_group) + : SimObject(ctx, name) + , RspIn(num_banks, this) + , RspOut(num_outputs, this) + , lg2_outputs_(log2ceil(num_outputs)) + , merge_group_(merge_group) + , sent_(num_banks, 0) + , last_grant_(num_outputs, num_banks - 1) + { + assert(num_outputs <= 64); + assert(merge_group < 64); + } + +protected: + void on_reset() { + std::fill(sent_.begin(), sent_.end(), 0); + std::fill(last_grant_.begin(), last_grant_.end(), RspIn.size() - 1); + } + + void on_tick() { + uint32_t nb = RspIn.size(); + uint32_t no = RspOut.size(); + std::vector pending(nb, 0); + bool idle = true; + for (uint32_t b = 0; b < nb; ++b) { + if (RspIn.at(b).empty()) { + continue; + } + idle = false; + auto tag = RspIn.at(b).peek().tag; + uint32_t port = tag & ((1u << lg2_outputs_) - 1); + uint64_t mask = (tag >> lg2_outputs_) & ((1ull << merge_group_) - 1); + uint64_t targets = (1ull << port) | (mask << ((port / merge_group_) * merge_group_)); + pending.at(b) = targets & ~sent_.at(b); + } + if (idle) { + this->tick_sleep(); + return; + } + for (uint32_t o = 0; o < no; ++o) { + for (uint32_t k = 1; k <= nb; ++k) { + uint32_t b = (last_grant_.at(o) + k) % nb; + if (!(pending.at(b) & (1ull << o))) { + continue; + } + auto rsp = RspIn.at(b).peek(); + rsp.tag >>= (lg2_outputs_ + merge_group_); + if (RspOut.at(o).try_send(rsp, 0)) { + DT(4, this->name() << " rsp" << b << "_" << o << ": " << rsp); + sent_.at(b) |= 1ull << o; + last_grant_.at(o) = b; + } + break; + } + } + for (uint32_t b = 0; b < nb; ++b) { + if (pending.at(b) != 0 && (pending.at(b) & ~sent_.at(b)) == 0) { + RspIn.at(b).pop(); + sent_.at(b) = 0; + } + } + } + +private: + uint32_t lg2_outputs_; + uint32_t merge_group_; + std::vector sent_; // [bank] ports that took the lane's response + std::vector last_grant_; // [port] round-robin pointer + + friend class SimObject; +}; + // Sits on one memory port's cache-side response path, ahead of the response // network. While the port's flush token is out the cache is quiescent, so the // port's next response is the token's answer and stops here; every other @@ -2421,7 +2559,8 @@ class Cache::Impl { // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); - port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks); + uint32_t merge_group = std::min(config_.read_merge, config_.num_inputs); + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group); for (uint32_t i = 0; i < config_.num_inputs; ++i) { simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); } @@ -2433,7 +2572,12 @@ class Cache::Impl { auto bank_sel = [&](const MemCrossBar::ReqType &req) { return params_.addr_bank_id(req.addr); }; - if (config_.core_out_buf) { + if (port_arb_->merge_bits() != 0) { + // one buffered arbiter per core port over the bank lanes + __assert(config_.core_out_buf, "read merging needs a registered core-response port"); + rsp_mcast_ = CacheRspMcast::Create(sname, num_banks, config_.num_inputs, port_arb_->merge_bits()); + core_rsp_delay_ = 1 + arb_tree_delay(num_banks); + } else if (config_.core_out_buf) { bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel, 0, 0); core_rsp_delay_ = omega_stages(num_banks, config_.num_inputs); } else { @@ -2450,7 +2594,7 @@ class Cache::Impl { // bind core ports port_arb_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); - banks_.at(i)->core_rsp_out.bind(&bank_core_xbar_->RspIn.at(i)); + banks_.at(i)->core_rsp_out.bind(rsp_mcast_ ? &rsp_mcast_->RspIn.at(i) : &bank_core_xbar_->RspIn.at(i)); // bind memory ports banks_.at(i)->mem_req_out.bind(&bank_mem_arb->ReqIn.at(this->bank_lane(i))); @@ -2502,7 +2646,7 @@ class Cache::Impl { // schedule core responses for (uint32_t req_id = 0, n = config_.num_inputs; req_id < n; ++req_id) { - auto &bank_rsp = bank_core_xbar_->RspOut.at(req_id); + auto &bank_rsp = rsp_mcast_ ? rsp_mcast_->RspOut.at(req_id) : bank_core_xbar_->RspOut.at(req_id); if (bank_rsp.empty()) continue; auto &core_rsp = bank_rsp.peek(); @@ -2739,6 +2883,7 @@ class Cache::Impl { std::vector nc_mem_arbs_; CachePortArb::Ptr port_arb_; MemCrossBar::Ptr bank_core_xbar_; + CacheRspMcast::Ptr rsp_mcast_; // the core-response network's levels, then the non-cacheable bypass // layer's response arbiter uint32_t core_rsp_delay_ = 0; diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 6371f75d09..7c61c03bdd 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -48,6 +48,7 @@ class Cache : public SimObject { bool shared_llc = false; // the LLC serves every memory client alone bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) bool core_out_buf = false; // registered core-response port + uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) }; struct PerfStats { diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 7c8a76942f..e1dbe03d09 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -101,6 +101,7 @@ class Socket::Impl { DCACHE_SHARED_LLC, // shared_llc VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch true, // core-response register + (VX_CFG_DCACHE_READ_MERGE != 0) ? DCACHE_CHANNELS : 0, // read_merge }); #ifdef VX_CFG_EXT_TEX_ENABLE From c1516d3e1d9d00e8d238e4f4a84c4ad42e0087a4 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:05 -0700 Subject: [PATCH 058/336] simx: route a single cluster's global barrier through its arbiters With one cluster, SimX counted a global-barrier arrival as it left the core and released the cores one cycle after the last one. In the RTL the arrival still crosses the socket's gbar arbiter and the cluster's (each with an output buffer when it has several inputs, the cluster's taking one arrival per cycle) to the device unit, whose registered release then crosses the cluster's and the socket's response forks. Only the device arbiter is a wire with one cluster. The single-cluster path now takes the same arbiter lags and serialization as the multi-cluster path, without the device arbiter's buffer. Co-Authored-By: Claude Opus 5.5 --- sim/simx/cluster.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 18fa87ad96..481fbe601a 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -349,16 +349,24 @@ class Cluster::Impl { DT(4, "*** Global barrier arrive: cluster #" << simobject_->id() << ", core #" << core_id << " at barrier #" << bar_id << ", arrived=" << gbar.mask.count()); - if constexpr (VX_CFG_NUM_CLUSTERS > 1) { - // Up to the device unit: the socket arbiter's output buffer, the cluster - // arbiter (one arrival per cycle) and its output buffer, then the device - // arbiter's output buffer. + // Up to the device unit: the socket arbiter's output buffer, then the + // cluster arbiter (one arrival per cycle) and its output buffer. A + // single-core device has no global-barrier path. + uint32_t lag = 0; + if constexpr (VX_CFG_NUM_CLUSTERS * VX_CFG_NUM_CORES > 1) { uint64_t now = SimPlatform::instance().cycles(); uint64_t out = std::max(now + kSocketArbLag, gbar_fwd_next_); gbar_fwd_next_ = out + 1; - simobject_->gbar_fwd_out.send({bar_id, count, core_id}, out + kClusterArbLag + 1 - now); + lag = out + kClusterArbLag - now; + } + if constexpr (VX_CFG_NUM_CLUSTERS > 1) { + // the device arbiter's output buffer + simobject_->gbar_fwd_out.send({bar_id, count, core_id}, lag + 1); } else if (gbar.mask.count() == (size_t)count) { - this->global_barrier_resume(bar_id, 1); + // With one cluster the device arbiter is a wire. The unit registers its + // release, which then crosses the cluster's and the socket's response + // forks. + this->global_barrier_resume(bar_id, lag + 1 + kClusterArbLag + kSocketArbLag); } } From 46080c037ec27def01d5f0263047c9a27bda1c9d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:06 -0700 Subject: [PATCH 059/336] simx: track MSHR misses in a line table when MSHR_LINES is set SimX side of the MSHR line table in the RTL (VX_cache_mshr with MSHR_LINES, VX_CFG_{DCACHE,L2,L3}_MSHR_LINES, default 0 = off): - The cache config carries mshr_lines for the dcache, L2 and L3. - The MSHR keeps a table of the distinct {line, sector} misses its request entries hold, each reference-counted by the entries chained on it and freed when the count drains. With a cap, overflowing it is a modelling error. - The bank admits a core request only while a line entry is free for it after every request ahead of it has claimed its own: the lines in the MSHR plus each distinct line a request still inside the pipe may allocate, since the RTL allocates at the pipe's first stage. This is the RTL's two-free-entries rule (the request admitted the cycle before plus this one) in SimX's later-allocating pipe. The stall counts as an MSHR stall, as the RTL folds it into its almost-full stall. - The pipe mirror of in-flight lines keeps the sector, which the line table keys on. With the cap at 0 every config is cycle-identical. sector_merge-2 (MSHR 32 requests over 8 lines on the dcache and L2) now also runs on simx. sgemm -n128, 16 threads x 16 warps, memory latency 400: dcache MSHR 16 (no table): SimX 2,628,803 rtlsim 2,733,260 dcache MSHR 64 over 16 lines: SimX 2,296,987 rtlsim 2,315,677 (0.8%) Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 1 + sim/simx/cluster.cpp | 2 + sim/simx/mem/cache.cpp | 111 +++++++++++++++++++++++++++++++++++----- sim/simx/mem/cache.h | 1 + sim/simx/processor.cpp | 2 + sim/simx/socket.cpp | 1 + 6 files changed, 106 insertions(+), 12 deletions(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 361ec300e2..92148b00ed 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -353,6 +353,7 @@ tests: - id: sector_merge-2 via: blackbox drivers: + - simx - rtlsim app: sector_merge configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_DCACHE_MSHR_SIZE=32 -DVX_CFG_DCACHE_MSHR_LINES=8 -DVX_CFG_L2_MSHR_SIZE=32 -DVX_CFG_L2_MSHR_LINES=8 diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 481fbe601a..adf1f221fc 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -95,6 +95,8 @@ class Cluster::Impl { L2_SHARED_LLC, // shared_llc VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch true, // core-response register (behind the bypass layer) + 0, // read_merge + VX_CFG_L2_MSHR_LINES, // mshr_lines }); diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index f813cdec29..cb54186e6b 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -506,10 +506,21 @@ struct mshr_entry_t { } }; +// With a line cap, the {line, sector} misses live in a table of that many +// entries, each reference-counted by the request entries chained on it. class MSHR { public: - MSHR(uint32_t size) - : entries_(size), ready_reqs_(0), size_(0) {} + MSHR(uint32_t size, uint32_t line_cap) + : entries_(size), ready_reqs_(0), size_(0), line_cap_(line_cap) {} + + // Distinct {line, sector} misses held. + uint32_t lines() const { + return lines_.size(); + } + + bool holds_line(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) const { + return this->find_line(set_id, addr_tag, sector_id) != lines_.end(); + } uint32_t capacity() const { return entries_.size(); @@ -604,6 +615,7 @@ class MSHR { entry.refetch = refetch; entry.refetch_retry = false; ++size_; + this->line_ref(set_id, addr_tag, sector_id); return i; } } @@ -706,6 +718,7 @@ class MSHR { picked->bank_req.type = bank_req_t::None; --ready_reqs_; --size_; + this->line_unref(picked->set_id, picked->addr_tag, picked->sector_id); } void reset() { @@ -714,12 +727,51 @@ class MSHR { } ready_reqs_ = 0; size_ = 0; + lines_.clear(); } private: + struct line_ref_t { + uint32_t set_id; + uint64_t addr_tag; + uint32_t sector_id; + uint32_t count; + }; + + std::vector::const_iterator find_line(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) const { + return std::find_if(lines_.begin(), lines_.end(), [&](const line_ref_t& l) { + return l.set_id == set_id && l.addr_tag == addr_tag && l.sector_id == sector_id; + }); + } + + void line_ref(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) { + for (auto &l : lines_) { + if (l.set_id == set_id && l.addr_tag == addr_tag && l.sector_id == sector_id) { + ++l.count; + return; + } + } + __assert(line_cap_ == 0 || lines_.size() < line_cap_, "MSHR line table overflow"); + lines_.push_back({set_id, addr_tag, sector_id, 1}); + } + + void line_unref(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) { + for (auto it = lines_.begin(); it != lines_.end(); ++it) { + if (it->set_id == set_id && it->addr_tag == addr_tag && it->sector_id == sector_id) { + if (--it->count == 0) { + lines_.erase(it); + } + return; + } + } + __assert(false, "MSHR line count underflow"); + } + std::vector entries_; uint32_t ready_reqs_; uint32_t size_; + uint32_t line_cap_; + std::vector lines_; }; class CacheBank : public SimObject { @@ -750,7 +802,7 @@ class CacheBank : public SimObject { const Cache::Config &config, const params_t ¶ms, uint32_t bank_id) - : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), lazy_q_(TFifo::Create("", 1, lazyq_size(config))), rand_ctr_(0), adm_seq_ctr_(0) + : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size, config.mshr_lines), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), lazy_q_(TFifo::Create("", 1, lazyq_size(config))), rand_ctr_(0), adm_seq_ctr_(0) #if VX_CFG_EXT_A_ENABLED , amo_unit_(__MAX(2u, (uint32_t)VX_CFG_AMO_RS_SIZE)) #endif @@ -1112,6 +1164,13 @@ class CacheBank : public SimObject { ++perf_stats_.mshr_stalls; return; } + // With a line table, a line entry must stay free for this request once + // every request ahead of it has claimed its own. + if (needs_mshr && config_.mshr_lines != 0 + && this->mshrLinesClaimed() >= config_.mshr_lines) { + ++perf_stats_.mshr_stalls; + return; + } // Any admitted request may become a write-miss root: the lazy queue // keeps room for every request already in the pipe. if (config_.lazy_fetch && lazy_q_->size() >= lazyq_size(config_) - config_.latency) { @@ -1209,8 +1268,9 @@ class CacheBank : public SimObject { // stays out of this counter. if (!is_amo_passthru) { ++pending_mshr_size_; - pipe_core_lines_.emplace_back(params_.addr_set_id(core_req.addr), - params_.addr_tag(core_req.addr)); + pipe_core_lines_.push_back({params_.addr_set_id(core_req.addr), + params_.addr_tag(core_req.addr), + params_.addr_sector_id(core_req.addr)}); } #if VX_CFG_EXT_A_ENABLED if (is_amo_passthru) ++pending_amo_probes_; @@ -2050,29 +2110,56 @@ class CacheBank : public SimObject { MSHR mshr_; uint32_t pending_mshr_size_; uint32_t pending_amo_probes_; // AmoProbe requests in the pipe, each holding a table reservation - // (set, tag) of Core requests inside the pipe: their MSHR entries exist - // only from pipe exit, so ordering checks against in-flight lines (the - // AMO admission defer) must consult this mirror as well. - std::vector> pipe_core_lines_; + // {set, tag, sector} of Core requests inside the pipe: their MSHR entries + // exist only from pipe exit, so ordering checks against in-flight lines (the + // AMO admission defer) and the line-table admission must consult this + // mirror as well. + struct pipe_line_t { + uint32_t set_id; + uint64_t addr_tag; + uint32_t sector_id; + }; + std::vector pipe_core_lines_; bool pipeLineBusy(uint32_t set_id, uint64_t addr_tag) const { - for (const auto &kv : pipe_core_lines_) { - if (kv.first == set_id && kv.second == addr_tag) { + for (const auto &l : pipe_core_lines_) { + if (l.set_id == set_id && l.addr_tag == addr_tag) { return true; } } return false; } + // The pipe is in order, so the oldest match is the request leaving it. void pipeLineErase(uint32_t set_id, uint64_t addr_tag) { for (auto it = pipe_core_lines_.begin(); it != pipe_core_lines_.end(); ++it) { - if (it->first == set_id && it->second == addr_tag) { + if (it->set_id == set_id && it->addr_tag == addr_tag) { pipe_core_lines_.erase(it); return; } } assert(false && "in-pipe line mirror underflow"); } + + // MSHR line entries held or claimed: the lines in the MSHR, plus a line for + // each distinct one a request inside the pipe may still allocate. The RTL + // allocates at the pipe's first stage, so these already hold their entries. + uint32_t mshrLinesClaimed() const { + uint32_t n = mshr_.lines(); + for (uint32_t i = 0, k = pipe_core_lines_.size(); i < k; ++i) { + auto &l = pipe_core_lines_.at(i); + if (mshr_.holds_line(l.set_id, l.addr_tag, l.sector_id)) { + continue; + } + bool seen = false; + for (uint32_t j = 0; j < i && !seen; ++j) { + auto &o = pipe_core_lines_.at(j); + seen = (o.set_id == l.set_id && o.addr_tag == l.addr_tag && o.sector_id == l.sector_id); + } + n += !seen; + } + return n; + } TFifo::Ptr pipe_req_; TFifo::Ptr lazy_q_; // write-miss roots awaiting their data-less fill diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 7c61c03bdd..1bd4e925f3 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -49,6 +49,7 @@ class Cache : public SimObject { bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) bool core_out_buf = false; // registered core-response port uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) + uint16_t mshr_lines = 0; // MSHR distinct {line, sector} entries (0: one per request) }; struct PerfStats { diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 268773e2d8..aa318ab00a 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -117,6 +117,8 @@ ProcessorImpl::ProcessorImpl() L3_SHARED_LLC, // shared_llc VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch true, // core-response register (behind the bypass layer) + 0, // read_merge + VX_CFG_L3_MSHR_LINES, // mshr_lines } ); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index e1dbe03d09..b44fb21845 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -102,6 +102,7 @@ class Socket::Impl { VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch true, // core-response register (VX_CFG_DCACHE_READ_MERGE != 0) ? DCACHE_CHANNELS : 0, // read_merge + VX_CFG_DCACHE_MSHR_LINES, // mshr_lines }); #ifdef VX_CFG_EXT_TEX_ENABLE From a2b16ad26af4dc90c5d229d88b177e347f9d4c36 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 16:06:10 -0700 Subject: [PATCH 060/336] simx: let each slice of a cache arbiter tree grant on its own Over 12 requesters the RTL's arbiters (VX_stream_arb with MAX_FANOUT 8) become a tree: each slice of up to 8 inputs has its own round-robin arbiter and a 2-entry output buffer, and a join arbiter picks one slice buffer per cycle. Two slices can therefore each accept a request for the same output in one cycle. SimX modeled the tree as one arbiter plus one buffer per output, so it accepted only one request per output per cycle and over-counted bank conflicts. This affected the cache's request crossbar (one arbiter per bank over the core ports) and, under read merging, the response crossbar (one arbiter per core port over the bank lanes). Both now have a buffer and a round-robin pointer per slice, and a join arbiter per output. The response tree's buffered level is now the slice buffers rather than an added delay. sgemm on the 2-frontend shape (16 dcache ports, 16 banks), SimX vs rtlsim: -n32 11,029 -> 10,549 (rtlsim 9,885) -n64 72,572 -> 66,039 (rtlsim 65,949) -n128 556,176 -> 553,250 (rtlsim 528,299) Configurations with 12 or fewer ports per cache are unchanged. Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/cache.cpp | 173 +++++++++++++++++++++++++++++------------ 1 file changed, 124 insertions(+), 49 deletions(-) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index cb54186e6b..cdfe079bdc 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -31,13 +31,20 @@ using namespace vortex; namespace { +constexpr uint32_t kMaxFanout = 8; + // An arbiter over more than 1.5x the fan-out limit is split into a buffered // tree, one cycle deeper. uint32_t arb_tree_delay(uint32_t requesters) { - constexpr uint32_t kMaxFanout = 8; return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; } +// Slices of the tree's first level: each arbitrates up to the fan-out limit +// into its own buffer, and a join arbiter picks among the slice buffers. +uint32_t arb_tree_slices(uint32_t requesters) { + return (arb_tree_delay(requesters) != 0) ? (requesters + kMaxFanout - 1) / kMaxFanout : 1; +} + // Registered stages of a radix-2 omega network between a and b ports; up to // two ports on each side it is a single crossbar. uint32_t omega_stages(uint32_t a, uint32_t b) { @@ -2246,8 +2253,10 @@ class CachePortArb : public SimObject { , lg2_inputs_(log2ceil(num_inputs)) // the request crossbar registers its output when it has more than two inputs , delay_((num_inputs > 2) ? 1 : 0) - , rr_(num_banks, num_inputs - 1) - , tree_stage_((arb_tree_delay(num_inputs) != 0) ? num_banks : 0, this) + , slices_(arb_tree_slices(num_inputs)) + , rr_(num_banks * slices_, num_inputs - 1) + , join_rr_(num_banks, slices_ - 1) + , tree_stage_((slices_ > 1) ? num_banks * slices_ : 0, this) , merge_group_(merge_group) { assert(num_inputs <= 64); @@ -2315,6 +2324,7 @@ class CachePortArb : public SimObject { protected: void on_reset() { std::fill(rr_.begin(), rr_.end(), ReqIn.size() - 1); + std::fill(join_rr_.begin(), join_rr_.end(), slices_ - 1); collisions_ = 0; } @@ -2323,12 +2333,20 @@ class CachePortArb : public SimObject { --init_cycles_; return; } - // the arbiter tree's buffered level ahead of each bank - for (uint32_t b = 0, nb = tree_stage_.size(); b < nb; ++b) { - auto& stage = tree_stage_.at(b); - if (!stage.empty() && !ReqOut.at(b).full()) { - ReqOut.at(b).send(stage.peek(), delay_); - stage.pop(); + // the arbiter tree's join level: one slice buffer per bank per cycle + for (uint32_t b = 0, nb = (slices_ > 1) ? ReqOut.size() : 0; b < nb; ++b) { + if (ReqOut.at(b).full()) { + continue; + } + for (uint32_t k = 1; k <= slices_; ++k) { + uint32_t sl = (join_rr_[b] + k) % slices_; + auto& stage = tree_stage_.at(b * slices_ + sl); + if (!stage.empty()) { + ReqOut.at(b).send(stage.peek(), delay_); + stage.pop(); + join_rr_[b] = sl; + break; + } } } uint32_t n = ReqIn.size(); @@ -2377,32 +2395,55 @@ class CachePortArb : public SimObject { want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; } } + // each bank's arbiter, or each slice of its tree, grants one port per cycle for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { - auto& out = tree_stage_.empty() ? ReqOut.at(b) : tree_stage_.at(b); - if (want[b] == 0 || out.full()) { + if (want[b] == 0) { continue; } - uint32_t i = rr_[b]; - do { - i = (i + 1) % n; - } while (!(want[b] & (1ull << i))); - collisions_ += __builtin_popcountll(want[b]) - 1; - auto req = ReqIn.at(i).peek(); - uint64_t mask = 0; - if (merge_group_ > 1) { - mask = merged.at(i) >> ((i / merge_group_) * merge_group_); - for (uint64_t f = merged.at(i); f != 0; f &= f - 1) { - DT(3, this->name() << " read-merge: port=" << __builtin_ctzll(f) << ", leader=" << i << ", " << req); - ReqIn.at(__builtin_ctzll(f)).pop(); + bool granted = false; + for (uint32_t sl = 0; sl < slices_; ++sl) { + uint64_t slice_want = want[b]; + if (slices_ > 1) { + uint32_t lo = sl * kMaxFanout; + uint32_t hi = std::min(lo + kMaxFanout, n); + uint64_t range = ((hi < 64) ? (1ull << hi) : 0) - (1ull << lo); + slice_want &= range; + } + auto& out = (slices_ > 1) ? tree_stage_.at(b * slices_ + sl) : ReqOut.at(b); + if (slice_want == 0 || out.full()) { + continue; } + auto& rr = rr_[b * slices_ + sl]; + uint32_t i = rr; + do { + i = (i + 1) % n; + } while (!(slice_want & (1ull << i))); + this->grant(i, out, merged); + rr = i; + granted = true; + } + if (granted) { + collisions_ += __builtin_popcountll(want[b]) - 1; + } + } + } + + // Send port i's request to its bank, with the ports merged into it. + void grant(uint32_t i, SimChannel& out, const std::vector& merged) { + auto req = ReqIn.at(i).peek(); + uint64_t mask = 0; + if (merge_group_ > 1) { + mask = merged.at(i) >> ((i / merge_group_) * merge_group_); + for (uint64_t f = merged.at(i); f != 0; f &= f - 1) { + DT(3, this->name() << " read-merge: port=" << __builtin_ctzll(f) << ", leader=" << i << ", " << req); + ReqIn.at(__builtin_ctzll(f)).pop(); } - uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; - __assert((tag >> 32) == 0, "cache bank tag overflow"); - req.tag = uint32_t(tag); - out.send(req, tree_stage_.empty() ? delay_ : 1); - ReqIn.at(i).pop(); - rr_[b] = i; } + uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; + __assert((tag >> 32) == 0, "cache bank tag overflow"); + req.tag = uint32_t(tag); + out.send(req, tree_stage_.empty() ? delay_ : 1); + ReqIn.at(i).pop(); } // A plain read: no write, flush, IO, local or atomic attribute. @@ -2419,8 +2460,10 @@ class CachePortArb : public SimObject { uint32_t lg2_inputs_; uint32_t delay_; uint32_t init_cycles_ = 0; - std::vector rr_; - std::vector> tree_stage_; + uint32_t slices_; + std::vector rr_; // [bank][slice] + std::vector join_rr_; // [bank] + std::vector> tree_stage_; // [bank][slice] uint32_t merge_group_; uint64_t collisions_ = 0; @@ -2431,7 +2474,8 @@ class CachePortArb : public SimObject { // round-robin arbiter per core port over the bank lanes. A merged read's // response goes to the port that sent it and to every merged port; each port // takes it when its arbiter grants that lane, and the lane is released once -// all of them have. +// all of them have. Over many banks a port's arbiter is a tree: each slice of +// lanes grants into its own buffer, and a join picks among the slices. class CacheRspMcast : public SimObject { public: using Ptr = std::shared_ptr; @@ -2446,8 +2490,11 @@ class CacheRspMcast : public SimObject { , RspOut(num_outputs, this) , lg2_outputs_(log2ceil(num_outputs)) , merge_group_(merge_group) + , slices_(arb_tree_slices(num_banks)) , sent_(num_banks, 0) - , last_grant_(num_outputs, num_banks - 1) + , last_grant_(num_outputs * slices_, num_banks - 1) + , join_rr_(num_outputs, slices_ - 1) + , tree_stage_((slices_ > 1) ? num_outputs * slices_ : 0, this) { assert(num_outputs <= 64); assert(merge_group < 64); @@ -2457,13 +2504,32 @@ class CacheRspMcast : public SimObject { void on_reset() { std::fill(sent_.begin(), sent_.end(), 0); std::fill(last_grant_.begin(), last_grant_.end(), RspIn.size() - 1); + std::fill(join_rr_.begin(), join_rr_.end(), slices_ - 1); } void on_tick() { uint32_t nb = RspIn.size(); uint32_t no = RspOut.size(); - std::vector pending(nb, 0); bool idle = true; + // the tree's join level: one slice buffer per port per cycle + for (uint32_t o = 0; o < no && slices_ > 1; ++o) { + for (uint32_t k = 1; k <= slices_; ++k) { + uint32_t sl = (join_rr_.at(o) + k) % slices_; + auto& stage = tree_stage_.at(o * slices_ + sl); + if (stage.empty()) { + continue; + } + if (RspOut.at(o).try_send(stage.peek(), 0)) { + stage.pop(); + join_rr_.at(o) = sl; + } + break; + } + } + for (auto& stage : tree_stage_) { + idle &= (stage.size() == 0); + } + std::vector pending(nb, 0); for (uint32_t b = 0; b < nb; ++b) { if (RspIn.at(b).empty()) { continue; @@ -2480,19 +2546,25 @@ class CacheRspMcast : public SimObject { return; } for (uint32_t o = 0; o < no; ++o) { - for (uint32_t k = 1; k <= nb; ++k) { - uint32_t b = (last_grant_.at(o) + k) % nb; - if (!(pending.at(b) & (1ull << o))) { - continue; - } - auto rsp = RspIn.at(b).peek(); - rsp.tag >>= (lg2_outputs_ + merge_group_); - if (RspOut.at(o).try_send(rsp, 0)) { - DT(4, this->name() << " rsp" << b << "_" << o << ": " << rsp); - sent_.at(b) |= 1ull << o; - last_grant_.at(o) = b; + for (uint32_t sl = 0; sl < slices_; ++sl) { + uint32_t lo = (slices_ > 1) ? sl * kMaxFanout : 0; + uint32_t hi = (slices_ > 1) ? std::min(lo + kMaxFanout, nb) : nb; + auto& out = (slices_ > 1) ? tree_stage_.at(o * slices_ + sl) : RspOut.at(o); + auto& rr = last_grant_.at(o * slices_ + sl); + for (uint32_t k = 1; k <= nb; ++k) { + uint32_t b = (rr + k) % nb; + if (b < lo || b >= hi || !(pending.at(b) & (1ull << o))) { + continue; + } + auto rsp = RspIn.at(b).peek(); + rsp.tag >>= (lg2_outputs_ + merge_group_); + if (out.try_send(rsp, (slices_ > 1) ? 1 : 0)) { + DT(4, this->name() << " rsp" << b << "_" << o << ": " << rsp); + sent_.at(b) |= 1ull << o; + rr = b; + } + break; } - break; } } for (uint32_t b = 0; b < nb; ++b) { @@ -2506,8 +2578,11 @@ class CacheRspMcast : public SimObject { private: uint32_t lg2_outputs_; uint32_t merge_group_; - std::vector sent_; // [bank] ports that took the lane's response - std::vector last_grant_; // [port] round-robin pointer + uint32_t slices_; + std::vector sent_; // [bank] ports that took the lane's response + std::vector last_grant_; // [port][slice] round-robin pointer + std::vector join_rr_; // [port] + std::vector> tree_stage_; // [port][slice] friend class SimObject; }; @@ -2663,7 +2738,7 @@ class Cache::Impl { // one buffered arbiter per core port over the bank lanes __assert(config_.core_out_buf, "read merging needs a registered core-response port"); rsp_mcast_ = CacheRspMcast::Create(sname, num_banks, config_.num_inputs, port_arb_->merge_bits()); - core_rsp_delay_ = 1 + arb_tree_delay(num_banks); + core_rsp_delay_ = 1; } else if (config_.core_out_buf) { bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel, 0, 0); core_rsp_delay_ = omega_stages(num_banks, config_.num_inputs); From a9bd77bc12c3a562ffedeb4ac8e843e6e86f0e25 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 16:06:10 -0700 Subject: [PATCH 061/336] simx: register a shared cache's input arbiter as the RTL does When the cores of a socket share an L1 cache (SOCKET_SIZE larger than the cache count), VX_cache_cluster puts a VX_mem_bus_arb in front of each cache port, with a registered request buffer (REQ_OUT_BUF 3) and, when the cache registers its core-response port, a registered response buffer (RSP_OUT_BUF = CORE_OUT_BUF). SimX's arbiter for two cores onto one cache had no delay in either direction, so instruction fetch and data accesses on a shared icache and dcache were two cycles short. The arbiter now takes one cycle on the request and, with a registered core-response port, one on the response, whenever cores share a cache. async_gbarrier at 2 cores in one socket with an L2: SimX 69,852 -> 78,230, rtlsim 78,237. A new parity case covers this shape. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 10 ++++++++++ sim/simx/mem/cache_cluster.cpp | 10 ++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index ccaf86aa66..9238aa4c88 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -156,6 +156,16 @@ tests: app: async_gbarrier shape: cores: 2 +# Two cores sharing one socket's instruction and data caches: their requests +# and responses cross each cache's input arbiter. +- id: model_parity-async-gbarrier-ss2 + check: model_parity + via: blackbox + app: async_gbarrier + configs: -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 2 + l2cache: true - id: bar-slot-phase-1 via: blackbox diff --git a/sim/simx/mem/cache_cluster.cpp b/sim/simx/mem/cache_cluster.cpp index ee88f524d9..5b8bddc2b8 100644 --- a/sim/simx/mem/cache_cluster.cpp +++ b/sim/simx/mem/cache_cluster.cpp @@ -35,11 +35,17 @@ CacheCluster::CacheCluster(const SimContext& ctx, char sname[100]; - // Arbitrate incoming core interfaces + // Arbitrate incoming core interfaces. Arbitrating cores onto fewer caches + // registers the request, and the response once more when the cache + // registers its core-response port. + const bool shared = (num_inputs != num_units); + const uint32_t arb_req_delay = shared ? 1 : 0; + const uint32_t arb_rsp_delay = (shared && cache_config.core_out_buf) ? 1 : 0; std::vector input_arbs(cache_config.num_inputs); for (uint32_t i = 0; i < cache_config.num_inputs; ++i) { snprintf(sname, 100, "%s-input-arb%d", name, i); - input_arbs.at(i) = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_inputs, num_units); + input_arbs.at(i) = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_inputs, num_units, + arb_req_delay, arb_rsp_delay); for (uint32_t j = 0; j < num_inputs; ++j) { this->core_req_in.at(j).at(i).bind(&input_arbs.at(i)->ReqIn.at(j)); input_arbs.at(i)->RspOut.at(j).bind(&this->core_rsp_out.at(j).at(i)); From bbad636e32e8f7a7ab727105f8174d8f46d25c81 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 16:06:11 -0700 Subject: [PATCH 062/336] simx: let the command processor's DMA use the shared last-level cache SimX side of VX_CFG_CP_LLC_CLIENT (VX_cp_llc_port, the LLC client slot and write-around in the RTL). The default is off; off, every configuration is unchanged. With VX_CFG_CP_ON_LLC: - The shared LLC gets one more client slot beside the cores: the cluster L2 when it is the LLC of one cluster, the L3 otherwise. It does not change the bank count. - The CP port sends host transfers into that slot, one LLC word per request. IO-aperture addresses carry the io attribute and bypass the cache. Reads keep up to one 4 KB burst in flight. - Writes of transfers of at least half the LLC carry a no-allocate attribute. A write-back bank sends a no-allocate write miss with no fill pending straight to memory and updates a hit in place. - A launch that follows DMA writes first sends an L1-only flush token on the slot behind them and starts once the LLC answers it. - The SimX runtime asks the simulator whether its CP is an LLC client and then routes CP transfers through the port, so they are timed and counted, and publishes the DMA_LLC_CLIENT capability, so the runtime issues only L1-only flushes around transfers. It does not read the hardware build configuration. The model resets its state at each launch. The LLC keeps the lines that transfers left in it into the launch, and the launch first waits for the transfers' fills, which in the RTL would simply keep going. dogfood, flush_coherence and fence_mp pass with the option on at 4 cores + L2 and 2 clusters x 2 cores + L2 + L3. A parity case runs dogfood at 4 cores + L2. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 11 ++ sim/simx/cluster.cpp | 27 ++++- sim/simx/cluster.h | 7 ++ sim/simx/constants.h | 15 +++ sim/simx/cp_llc_port.h | 191 +++++++++++++++++++++++++++++++++++ sim/simx/mem/cache.cpp | 24 ++++- sim/simx/mem/cache.h | 4 + sim/simx/processor.cpp | 88 +++++++++++++++- sim/simx/processor.h | 13 +++ sim/simx/processor_impl.h | 18 ++++ sim/simx/types.h | 5 +- sw/runtime/simx/vortex.cpp | 34 +++++++ 12 files changed, 432 insertions(+), 5 deletions(-) create mode 100644 sim/simx/cp_llc_port.h diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 9238aa4c88..a6042df1ef 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -355,6 +355,17 @@ tests: app: flush_coherence args: -n2048 -r16 -c -x configs: -DVX_CFG_EXT_DXA_ENABLE +# The command processor's DMA as a client of the shared L2: host transfers +# are timed through the L2, and each launch waits for the DMA's fence. +- id: model_parity-dogfood-cp-llc + check: model_parity + via: blackbox + app: dogfood + args: -n16 -xbar -xgbar + configs: -DVX_CFG_CP_LLC_CLIENT=1 + shape: + cores: 4 + l2cache: true # The shared L2 keeps its lines across the L1-only launch flushes. - id: model_parity-flush-chain check: model_parity diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index adf1f221fc..a2c83774b9 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -82,7 +82,7 @@ class Cluster::Impl { log2ceil(VX_CFG_L2_NUM_WAYS), // A log2ceil(VX_CFG_L2_NUM_BANKS), // B VX_CFG_XLEN, // address bits - VX_CFG_L2_NUM_REQS, // request size + L2_CACHE_REQS, // request size VX_CFG_L2_MEM_PORTS, // memory ports VX_CFG_L2_WRITEBACK, // write-back false, // write response @@ -97,6 +97,7 @@ class Cluster::Impl { true, // core-response register (behind the bypass layer) 0, // read_merge VX_CFG_L2_MSHR_LINES, // mshr_lines + CP_ON_L2 != 0, // write_around }); @@ -580,6 +581,18 @@ class Cluster::Impl { l2cache_->flush_begin(l1_only); } + void l2_keep_lines() { + l2cache_->keep_lines(); + } + + SimChannel& cp_mem_req_in() { + return l2cache_->core_req_in.at(L2_CP_IDX); + } + + SimChannel& cp_mem_rsp_out() { + return l2cache_->core_rsp_out.at(L2_CP_IDX); + } + bool l2_flush_done() const { return l2cache_->flush_done(); } @@ -743,6 +756,18 @@ void Cluster::l2_flush_begin(bool l1_only) { impl_->l2_flush_begin(l1_only); } +void Cluster::l2_keep_lines() { + impl_->l2_keep_lines(); +} + +SimChannel& Cluster::cp_mem_req_in() { + return impl_->cp_mem_req_in(); +} + +SimChannel& Cluster::cp_mem_rsp_out() { + return impl_->cp_mem_rsp_out(); +} + bool Cluster::l2_flush_done() const { return impl_->l2_flush_done(); } diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index 1bcc518427..e47453af1d 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -143,6 +143,13 @@ class Cluster : public SimObject { bool rtcache_flush_done() const; #endif void l2_flush_begin(bool l1_only); + // The L2, when it is the shared last-level cache, keeps its lines into the + // next launch. + void l2_keep_lines(); + // The command processor DMA's client slot on this cluster's L2, valid only + // under CP_ON_L2 (the L2 is the LLC). + SimChannel& cp_mem_req_in(); + SimChannel& cp_mem_rsp_out(); bool l2_flush_done() const; #ifdef VX_CFG_EXT_RASTER_ENABLE diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 64a61c14d2..fabb7e3925 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -102,6 +102,21 @@ inline constexpr uint32_t VX_CFG_L2_NUM_REQS = VX_CFG_L2_DEMAND_REQS; inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_L2_MEM_PORTS; #endif +// With VX_CFG_CP_ON_LLC the command processor's DMA is one more client of the +// shared last-level cache: the cluster L2 when it is the LLC of one cluster, +// the L3 otherwise. The slot does not size the banks. +inline constexpr uint32_t CP_ON_L2 = ((VX_CFG_CP_ON_LLC != 0) + && (VX_CFG_NUM_CLUSTERS == 1) + && (VX_CFG_L2_ENABLED != 0) + && (VX_CFG_L3_ENABLED == 0)) ? 1 : 0; +inline constexpr uint32_t CP_ON_L3 = ((VX_CFG_CP_ON_LLC != 0) && (VX_CFG_L3_ENABLED != 0)) ? 1 : 0; +inline constexpr uint32_t L2_CP_IDX = VX_CFG_L2_NUM_REQS; +inline constexpr uint32_t L3_CP_IDX = VX_CFG_L3_NUM_REQS; +inline constexpr uint32_t L2_CACHE_REQS = VX_CFG_L2_NUM_REQS + CP_ON_L2; +inline constexpr uint32_t L3_CACHE_REQS = VX_CFG_L3_NUM_REQS + CP_ON_L3; +// one last-level cache word per DMA request +inline constexpr uint32_t CP_MEM_DATA_SIZE = (CP_ON_L2 != 0) ? VX_CFG_L1_LINE_SIZE : VX_CFG_L2_SECTOR_SIZE; + inline constexpr uint32_t PER_ISSUE_WARPS = VX_CFG_NUM_WARPS / VX_CFG_ISSUE_WIDTH; inline constexpr uint32_t ISSUE_WIS_BITS = log2ceil(PER_ISSUE_WARPS); diff --git a/sim/simx/cp_llc_port.h b/sim/simx/cp_llc_port.h new file mode 100644 index 0000000000..a298216046 --- /dev/null +++ b/sim/simx/cp_llc_port.h @@ -0,0 +1,191 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include "types.h" +#include "constants.h" +#include "mem_block_pool.h" +#include "debug.h" + +namespace vortex { + +// The command processor's DMA port into the last-level cache's client slot. +// Requests in the IO aperture carry the IO attribute and bypass the cache; a +// write flagged no-allocate writes around it on a miss. A launch that follows +// DMA writes first sends an L1-only flush token behind them, which the +// last-level cache answers once its inputs have drained. +class CpLlcPort : public SimObject { +public: + using Ptr = std::shared_ptr; + + SimChannel MemReqOut; + SimChannel MemRspIn; + + // one 4 KB DMA burst of reads in flight + static constexpr uint32_t kReadSlots = 4096 / CP_MEM_DATA_SIZE; + + CpLlcPort(const SimContext& ctx, const char* name) + : SimObject(ctx, name) + , MemReqOut(this) + , MemRspIn(this) + , slots_(kReadSlots) + {} + + // Queue one LLC word of a transfer; `src` is null for a read into `dst`. + void push(uint64_t addr, const uint8_t* src, uint8_t* dst, uint32_t offset, + uint32_t size, bool noalloc) { + MemReq req; + req.addr = addr; + req.op = src ? MemOp::ST : MemOp::LD; + req.flags.io = (get_addr_type(addr) == AddrType::IO); + req.flags.noalloc = src && noalloc; + // the word's bytes sit at its offset in the memory block + uint32_t boff = uint32_t(addr % VX_CFG_MEM_BLOCK_SIZE) + offset; + if (src) { + req.data = make_mem_block(); + std::memset(req.data->data(), 0, req.data->size()); + std::memcpy(req.data->data() + boff, src, size); + req.byteen = ((size < 64) ? ((1ull << size) - 1) : ~0ull) << boff; + } else { + uint32_t woff = uint32_t(addr % VX_CFG_MEM_BLOCK_SIZE); + req.byteen = ((CP_MEM_DATA_SIZE < 64) ? ((1ull << CP_MEM_DATA_SIZE) - 1) : ~0ull) << woff; + } + queue_.push_back({req, dst, boff, size}); + this->tick_wake(); + } + + // Every queued request has entered the port, and every read has returned. + bool idle() const { + return queue_.empty() && reads_ == 0 && !fencing_; + } + + // DMA writes entered the port since the last fence. + bool writes_pending() const { + return wr_pending_; + } + + // Send the launch fence behind the pending writes. + void fence() { + fencing_ = true; + fence_sent_ = false; + this->tick_wake(); + } + +protected: + void on_reset() { + queue_.clear(); + free_slots_.clear(); + for (uint32_t i = 0; i < kReadSlots; ++i) { + free_slots_.push_back(kReadSlots - 1 - i); + } + reads_ = 0; + wr_pending_ = false; + fencing_ = false; + fence_sent_ = false; + } + + void on_tick() { + if (!MemRspIn.empty()) { + auto& rsp = MemRspIn.peek(); + DT(3, this->name() << " rsp: " << rsp); + if (rsp.tag == kFenceTag) { + fencing_ = false; + wr_pending_ = false; + } else { + uint32_t slot = rsp.tag >> 1; + auto& rd = slots_.at(slot); + std::memcpy(rd.dst, rsp.data->data() + rd.offset, rd.size); + free_slots_.push_back(slot); + --reads_; + } + MemRspIn.pop(); + } + if (fencing_) { + if (!fence_sent_) { + MemReq token; + token.flags.flush = 1; + token.flags.flush_l1 = 1; + token.tag = kFenceTag; + if (MemReqOut.try_send(token, 1)) { + DT(3, this->name() << " fence: " << token); + fence_sent_ = true; + } + } + return; + } + if (queue_.empty()) { + if (reads_ == 0 && MemRspIn.empty()) { + this->tick_sleep(); + } + return; + } + auto& entry = queue_.front(); + bool is_read = !entry.req.is_write(); + if (is_read && free_slots_.empty()) { + return; + } + MemReq req = entry.req; + uint32_t slot = 0; + if (is_read) { + slot = free_slots_.back(); + req.tag = slot << 1; + } else { + req.tag = 0; + } + if (!MemReqOut.try_send(req, 1)) { + return; + } + DT(3, this->name() << " req: " << req); + if (is_read) { + free_slots_.pop_back(); + slots_.at(slot) = {entry.dst, entry.offset, entry.size}; + ++reads_; + } else { + wr_pending_ = true; + } + queue_.pop_front(); + } + +private: + static constexpr uint32_t kFenceTag = 1; + + struct entry_t { + MemReq req; + uint8_t* dst; + uint32_t offset; + uint32_t size; + }; + + struct read_slot_t { + uint8_t* dst = nullptr; + uint32_t offset = 0; + uint32_t size = 0; + }; + + std::deque queue_; + std::vector slots_; + std::vector free_slots_; + uint32_t reads_ = 0; + bool wr_pending_ = false; + bool fencing_ = false; + bool fence_sent_ = false; + + friend class SimObject; +}; + +} diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index cdfe079bdc..14fee5673c 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -1941,7 +1941,17 @@ class CacheBank : public SimObject { } } else { // Cache miss. - if (bank_req.write && !config_.write_back) { + if (config_.write_around && bank_req.write && bank_req.flags.noalloc + && !mshr_.lookup(set_id, addr_tag, sector_id)) { + // A no-allocate write miss with no fill pending on its line goes to + // memory as is: nothing is allocated or fetched. + if (!this->emitWritethrough(set_id, addr_tag, sector_id, + bank_req.addr, bank_req.hart_id, bank_req.uuid, + bank_req.data, bank_req.byteen, + need_core_rsp(bank_req), bank_req.req_tag)) { + return; + } + } else if (bank_req.write && !config_.write_back) { // Write-through write miss: forward store to memory and respond // immediately. No fill / no MSHR allocation (this store doesn't // wait for a line). If a fill for this line is already in flight @@ -3002,6 +3012,14 @@ class Cache::Impl { } } + void keep_lines() { + if (config_.bypass || !config_.shared_llc) return; + keep_lines_ = true; + for (auto &bank : banks_) { + bank->flush_begin(true); + } + } + bool flush_done() const { if (config_.bypass) return true; for (const auto &bank : banks_) { @@ -3100,3 +3118,7 @@ void Cache::flush_begin(bool l1_only) { bool Cache::flush_done() const { return impl_->flush_done(); } + +void Cache::keep_lines() { + impl_->keep_lines(); +} diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 1bd4e925f3..ae8b783dd9 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -50,6 +50,7 @@ class Cache : public SimObject { bool core_out_buf = false; // registered core-response port uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) uint16_t mshr_lines = 0; // MSHR distinct {line, sector} entries (0: one per request) + bool write_around = false; // a no-allocate write miss goes to memory (write-back only) }; struct PerfStats { @@ -93,6 +94,9 @@ class Cache : public SimObject { void flush_begin(bool l1_only = false); bool flush_done() const; + // A shared last-level cache keeps its lines into the next launch. + void keep_lines(); + protected: void on_reset(); void on_tick(); diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index aa318ab00a..d0a8a8c6e8 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -104,7 +104,7 @@ ProcessorImpl::ProcessorImpl() log2ceil(VX_CFG_L3_NUM_WAYS), // A log2ceil(VX_CFG_L3_NUM_BANKS), // B VX_CFG_XLEN, // address bits - VX_CFG_L3_NUM_REQS, // request size + L3_CACHE_REQS, // request size VX_CFG_L3_MEM_PORTS, // memory ports VX_CFG_L3_WRITEBACK, // write-back false, // write response @@ -119,6 +119,7 @@ ProcessorImpl::ProcessorImpl() true, // core-response register (behind the bypass layer) 0, // read_merge VX_CFG_L3_MSHR_LINES, // mshr_lines + CP_ON_L3 != 0, // write_around } ); @@ -171,6 +172,18 @@ ProcessorImpl::ProcessorImpl() } #endif + // The CP DMA's client slot on the shared last-level cache. + if constexpr (CP_ON_L2 != 0 || CP_ON_L3 != 0) { + cp_llc_port_ = CpLlcPort::Create("cp-llc-port"); + if constexpr (CP_ON_L2 != 0) { + cp_llc_port_->MemReqOut.bind(&clusters_.at(0)->cp_mem_req_in()); + clusters_.at(0)->cp_mem_rsp_out().bind(&cp_llc_port_->MemRspIn); + } else { + cp_llc_port_->MemReqOut.bind(&l3cache_->core_req_in.at(L3_CP_IDX)); + l3cache_->core_rsp_out.at(L3_CP_IDX).bind(&cp_llc_port_->MemRspIn); + } + } + // connect L3 memory interfaces for (uint32_t i = 0; i < VX_CFG_L3_MEM_PORTS; ++i) { l3cache_->mem_req_out.at(i).bind(&memsim_->mem_req_in.at(i)); @@ -287,6 +300,7 @@ void ProcessorImpl::flush_caches(bool l1_only) { } int ProcessorImpl::run() { + this->cp_mem_fence(); this->reset(); kmu_->start(); this->forward_delegated_launch(); @@ -326,6 +340,61 @@ int ProcessorImpl::run() { return exitcode; } +void ProcessorImpl::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { + this->cp_mem_transfer(addr, static_cast(src), nullptr, size, noalloc); +} + +void ProcessorImpl::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { + this->cp_mem_transfer(addr, nullptr, static_cast(dst), size, false); +} + +void ProcessorImpl::cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* dst, uint64_t size, bool noalloc) { + __assert(cp_llc_port_, "no CP port on the last-level cache"); + if (!platform_reset_) { + this->reset(); + } + while (size != 0) { + uint32_t offset = uint32_t(addr % CP_MEM_DATA_SIZE); + uint32_t n = uint32_t(std::min(size, CP_MEM_DATA_SIZE - offset)); + cp_llc_port_->push(addr - offset, src, dst, offset, n, noalloc); + addr += n; + size -= n; + if (src) { + src += n; + } + if (dst) { + dst += n; + } + } + while (!cp_llc_port_->idle()) { + SimPlatform::instance().tick(); + } + this->llc_keep_lines(); +} + +void ProcessorImpl::cp_mem_fence() { + if (!cp_llc_port_) { + return; + } + if (cp_llc_port_->writes_pending()) { + cp_llc_port_->fence(); + } + // The launch resets the model, so a transfer's fills land first: memory + // holds a read in flight without a channel packet. + while (!cp_llc_port_->idle() || !SimPlatform::instance().idle() || perf_mem_pending_reads_ != 0) { + SimPlatform::instance().tick(); + } + this->llc_keep_lines(); +} + +void ProcessorImpl::llc_keep_lines() { + if constexpr (CP_ON_L2 != 0) { + clusters_.at(0)->l2_keep_lines(); + } else { + l3cache_->keep_lines(); + } +} + bool ProcessorImpl::mmu_fault_pending() const { #ifdef VX_CFG_VM_ENABLE return dev_ptw_->fault_info().valid; @@ -346,6 +415,7 @@ void ProcessorImpl::forward_delegated_launch() { void ProcessorImpl::reset() { SimPlatform::instance().reset(); + platform_reset_ = true; perf_mem_reads_ = 0; perf_mem_writes_ = 0; perf_mem_latency_ = 0; @@ -493,6 +563,22 @@ void Processor::reset() { impl_->reset(); } +void Processor::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { + impl_->cp_mem_write(addr, src, size, noalloc); +} + +void Processor::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { + impl_->cp_mem_read(addr, dst, size); +} + +bool Processor::cp_llc_client() const { + return (CP_ON_L2 != 0) || (CP_ON_L3 != 0); +} + +uint64_t Processor::cp_llc_size() const { + return (CP_ON_L3 != 0) ? VX_CFG_L3_SIZE : VX_CFG_L2_SIZE; +} + void Processor::start_kmu() { impl_->kmu().start(); } diff --git a/sim/simx/processor.h b/sim/simx/processor.h index 66081ebbd2..13bd99aa40 100644 --- a/sim/simx/processor.h +++ b/sim/simx/processor.h @@ -59,6 +59,19 @@ class Processor { int dcr_read(uint32_t addr, uint32_t tag, uint32_t* value); + // Device-memory access through the command processor's port into the + // last-level cache, when the configuration has one (VX_CFG_CP_ON_LLC). A + // write returns once every request entered the port; the next launch is + // ordered behind it. A read returns with its data. + void cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc); + + void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + + // The command processor's DMA is a client of the last-level cache, of + // this size in bytes. + bool cp_llc_client() const; + uint64_t cp_llc_size() const; + private: ProcessorImpl* impl_; }; diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index 50ecc05106..052d06f8f4 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -19,6 +19,7 @@ #include "cluster.h" #include "kmu.h" #include "gbar_unit.h" +#include "cp_llc_port.h" namespace vortex { @@ -75,12 +76,27 @@ class ProcessorImpl { // each level's evictions reach the next before that level itself flushes. void flush_caches(bool l1_only); + // Device-memory access through the command processor's port into the + // last-level cache (CP_ON_L2 / CP_ON_L3). A write returns once every + // request entered the port, a read once its data returned. + void cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc); + void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + private: // A grid-less launch is a delegated draw launch: the KMU walks no CTAs and // the frame kick is forwarded to every cluster's raster engine instead. void forward_delegated_launch(); + // Queue a transfer on the CP port and tick until it has entered, or for a + // read returned; the last-level cache keeps what it took into the next launch. + void cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* dst, uint64_t size, bool noalloc); + + // A launch that follows DMA writes waits for the port's fence. + void cp_mem_fence(); + + void llc_keep_lines(); + // True once the device walker has latched a page fault. bool mmu_fault_pending() const; @@ -97,6 +113,8 @@ class ProcessorImpl { #endif RAM* ram_ = nullptr; // functional backing store (set by attach_ram) Cache::Ptr l3cache_; + CpLlcPort::Ptr cp_llc_port_; // CP DMA client of the last-level cache + bool platform_reset_ = false; uint64_t perf_mem_reads_; uint64_t perf_mem_writes_; uint64_t perf_mem_latency_; diff --git a/sim/simx/types.h b/sim/simx/types.h index f3f9b4cf84..595891d401 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -450,6 +450,7 @@ struct MemFlags { uint32_t amo_unsigned : 1; // bit 3: MIN/MAX signedness (1=unsigned) uint32_t flush : 1; // bit 4: flush token (a fence) uint32_t flush_l1 : 1; // bit 5: L1-only flush token: stops at the shared LLC + uint32_t noalloc : 1; // bit 6: a write that misses a write-around cache goes to memory #ifdef VX_CFG_EXT_DXA_ENABLE // DXA completion sideband — populated only on DXA's LMEM-DMA writes. // notify_bar_id carries the RAW (kernel-encoded) bar handle: @@ -459,8 +460,8 @@ struct MemFlags { // be wide enough to hold (bar_no << 8) | (MAX_CTAS_per_core - 1). // For NUM_BARRIERS = 8 and NUM_WARPS ≤ 16, that's at most 11 bits; // 16 bits leaves headroom for larger configs. - uint32_t dxa_notify_done : 1; // bit 6 - uint32_t dxa_notify_bar_id : 16; // bits 7..22 + uint32_t dxa_notify_done : 1; // bit 7 + uint32_t dxa_notify_bar_id : 16; // bits 8..23 #endif }; }; diff --git a/sw/runtime/simx/vortex.cpp b/sw/runtime/simx/vortex.cpp index d72ba8d60d..b2120aaf0b 100644 --- a/sw/runtime/simx/vortex.cpp +++ b/sw/runtime/simx/vortex.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -150,6 +151,13 @@ class vx_device { std::memcpy(dst, hp, bytes); return; } + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_read(addr, dst, bytes); + return; + } ram_.read(static_cast(dst), addr, bytes); }; h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { @@ -157,8 +165,34 @@ class vx_device { std::memcpy(hp, src, bytes); return; } + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, false); + return; + } ram_.write(static_cast(src), addr, bytes); }; + if (processor_.cp_llc_client()) { + // The CP DMA is a client of the last-level cache: transfers stay + // coherent with it, and those of at least half its size write around it. + h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, no_alloc); + }; + h.dma_llc_client = true; + h.dma_no_alloc_min = processor_.cp_llc_size() / 2; + if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { + h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + } + } h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { // Wait for any background processor_.run() to finish before writing DCR. if (future_.valid()) { From fd151efaa9fb735c9897828178f91d37fcdc5e3d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 18:29:25 -0700 Subject: [PATCH 063/336] Build VX_csr_fpu only when the F extension is enabled VX_csr_fpu included VX_fpu_define.vh unconditionally. The DUT source lists preprocess every file under hw/rtl/core, and the tcu DUT is built with VX_CFG_EXT_F_DISABLE and no FPU include path, so both synthesis gates failed on tcu before synthesis. Both instantiation sites already sit under VX_CFG_EXT_F_ENABLE; the module now does too, like the FPU include in VX_csr_data and VX_sfu_unit. fpga_gate tcu: PASS (300 MHz, LUT unchanged). asic_gate tcu: PASS (839 MHz, area unchanged). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_csr_fpu.sv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hw/rtl/core/VX_csr_fpu.sv b/hw/rtl/core/VX_csr_fpu.sv index 7e3889c40f..d19d308279 100644 --- a/hw/rtl/core/VX_csr_fpu.sv +++ b/hw/rtl/core/VX_csr_fpu.sv @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +`ifdef VX_CFG_EXT_F_ENABLE + `include "VX_define.vh" `include "VX_fpu_define.vh" @@ -101,3 +103,5 @@ module VX_csr_fpu import VX_gpu_pkg::*, VX_fpu_pkg::*; #( end endmodule + +`endif From cc7bb17ccf8305a3220e9ad31903424dda529943 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 18:30:54 -0700 Subject: [PATCH 064/336] rtlsim: ask the processor whether the CP DMA is an LLC client The rtlsim runtime included VX_config.h to decide whether the command processor's DMA goes through the last-level cache and to size the write-around threshold, which ci/check_config_boundary.sh forbids: the hardware build configuration is private to RTL and the simulators. The rtlsim Processor now answers both: cp_llc_client() (VX_CFG_CP_ON_LLC) and cp_llc_size() (the L3 size with an L3, else the L2 size). The runtime routes its DMA hooks and sets DMA_LLC_CLIENT and the no-alloc threshold from them, as the SimX runtime does. rtlsim dogfood -n16 at 4 cores + L2 with VX_CFG_CP_LLC_CLIENT=1 is cycle-identical (353,584) and passes all 22 tests. All ci/check_*.sh pass. Co-Authored-By: Claude Opus 5.5 --- sim/rtlsim/processor.cpp | 8 +++++ sim/rtlsim/processor.h | 5 +++ sw/runtime/rtlsim/vortex.cpp | 61 ++++++++++++++++++------------------ 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index 92724c8923..327a131bb3 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -658,6 +658,14 @@ void Processor::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { impl_->cp_mem_read(addr, static_cast(dst), size); } +bool Processor::cp_llc_client() const { + return VX_CFG_CP_ON_LLC != 0; +} + +uint64_t Processor::cp_llc_size() const { + return VX_CFG_L3_ENABLED ? VX_CFG_L3_SIZE : VX_CFG_L2_SIZE; +} + void Processor::drain() { impl_->drain(); } \ No newline at end of file diff --git a/sim/rtlsim/processor.h b/sim/rtlsim/processor.h index 9fe3c2f68e..05e0b4605c 100644 --- a/sim/rtlsim/processor.h +++ b/sim/rtlsim/processor.h @@ -51,6 +51,11 @@ class Processor { void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + // The command processor's DMA is a client of the last-level cache, of + // this size in bytes. + bool cp_llc_client() const; + uint64_t cp_llc_size() const; + private: class Impl; diff --git a/sw/runtime/rtlsim/vortex.cpp b/sw/runtime/rtlsim/vortex.cpp index 0376f03e77..7f32b98eb9 100644 --- a/sw/runtime/rtlsim/vortex.cpp +++ b/sw/runtime/rtlsim/vortex.cpp @@ -23,7 +23,6 @@ // dram hooks dereference it directly (the sim runs in-process). // ============================================================================ -#include #include #include @@ -144,46 +143,48 @@ class vx_device { std::memcpy(dst, hp, bytes); return; } - #if VX_CFG_CP_ON_LLC - if (future_.valid()) future_.wait(); - processor_.cp_mem_read(addr, dst, bytes); - #else + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_read(addr, dst, bytes); + return; + } ram_.read(static_cast(dst), addr, bytes); - #endif }; h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { if (void* hp = host_region_ptr(addr)) { std::memcpy(hp, src, bytes); return; } - #if VX_CFG_CP_ON_LLC - if (future_.valid()) future_.wait(); - processor_.cp_mem_write(addr, src, bytes, false); - #else - ram_.write(static_cast(src), addr, bytes); - #endif - }; - #if VX_CFG_CP_ON_LLC - // The CP DMA is a client of the last-level cache: transfers stay coherent - // with it, and those of at least half its size write around it. - h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { - if (void* hp = host_region_ptr(addr)) { - std::memcpy(hp, src, bytes); + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, false); return; } - if (future_.valid()) future_.wait(); - processor_.cp_mem_write(addr, src, bytes, no_alloc); + ram_.write(static_cast(src), addr, bytes); }; - h.dma_llc_client = true; - #if VX_CFG_L3_ENABLED - h.dma_no_alloc_min = VX_CFG_L3_SIZE / 2; - #else - h.dma_no_alloc_min = VX_CFG_L2_SIZE / 2; - #endif - if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { - h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + if (processor_.cp_llc_client()) { + // The CP DMA is a client of the last-level cache: transfers stay + // coherent with it, and those of at least half its size write around it. + h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, no_alloc); + }; + h.dma_llc_client = true; + h.dma_no_alloc_min = processor_.cp_llc_size() / 2; + if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { + h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + } } - #endif h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { // Wait for any background processor_.run() to finish so dcr_write // does not race the Verilator state. From 54754389c2fd51fd6ccf166c9725f10557c0315c Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 18:30:54 -0700 Subject: [PATCH 065/336] simx: drop the extra register on the LSU's path to commit In the RTL a load response crosses three registers between the LSU block and the commit arbiter: the block's response buffer (rsp_buf), its result arbiter (rsp_arb, OUT_BUF 3) and the unit's lane gather (OUT_BUF 3). SimX counted the first two in the LSU's output channel and then added the two-cycle register that the commit stage applies to every unit, which for the ALU stands for its result buffer and its PE switch. Loads therefore reached commit one cycle late: uncontended, the response-to-commit distance was 4 cycles in SimX and 3 in rtlsim, at the default and the 8-thread shapes. The commit stage now adds only the lane gather for the LSU. SimX vs rtlsim: parity-sgemm-mc 1,220,795 -> 1,152,088 (rtlsim 1,102,494; 10.73% -> 4.50%) parity-sgemm-fw2 553,250 -> 543,888 (rtlsim 528,299; 4.72% -> 2.95%) parity-sgemm 794,010 -> 793,955 (rtlsim 799,610; 0.71%) parity-raycast 1,497,605 -> 1,494,492 (rtlsim 1,498,691; 0.28%) parity-vecadd 367,367 -> 365,304 (rtlsim 368,717; 0.93%) parity-sgemv 893,874 -> 901,575 (rtlsim 883,260; 2.07%) Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 57b7a1c555..8b5682742e 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -877,8 +877,11 @@ class Core::Impl { auto trace = fu_out.peek(); uint32_t iw = trace->wid % VX_CFG_ISSUE_WIDTH; auto& arb_in = *commit_queues_.at(iw).at(fu); - // A unit registers its result once more before the commit arbiter. - if (arb_in.try_send(trace, 2)) { + // The unit's output registers ahead of the commit arbiter that its + // output channel does not already carry: the LSU's response buffer and + // result arbiter are in its channel, leaving its lane gather. + uint32_t commit_delay = (fu == (uint32_t)FUType::LSU) ? 1 : 2; + if (arb_in.try_send(trace, commit_delay)) { // Release the warp as soon as its stalling instruction's result leaves // the functional unit — the branch target / fence / warp-control is // resolved at that point. The release lands in stalled_warps so the From 14f944c668c6a8f43bbd9a544e27e46ec50d3d1c Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 18:57:26 -0700 Subject: [PATCH 066/336] dxa: stream GMEM responses straight to smem_wr again VX_dxa_gmem_req buffered every GMEM response in a MAX_OUTSTANDING-deep line FIFO with a bypass, so a busy smem_wr would not hold the socket memory response path it shares with the L1 fills. The FIFO costs eight cache lines of storage plus a line-wide bypass mux, and it only engages when smem_wr's own line registers (fill buffer, one captured line, the deferred last line) are full. rtlsim, buffered -> direct (all PASS): dxa_copy -d1 (NT4) 6827 -> 6827 dxa_copy -d3, 2 cores 17649 -> 17649 dxa_copy_mcast 3381 -> 3381 sgemm2_dxa 40390 -> 40390 sgemm2_dxa, 2 cores 20731 -> 20731 sgemm2_dxa_mcast 37578 -> 37578 sgemm_tcu_wg_dxa_mcast 128x128x64 NT16 78882 -> 78882 sgemm_tcu_wg_dxa 128x128x64 (IW4 NT32) 32312 -> 32823 (+1.6%) dxa_kmajor_check PASS -> PASS Only the tiled-scatter WGMMA producer ever fills smem_wr's line registers, and there the buffer saves 1.6%. It does not prevent a deadlock: smem_wr only waits on LMEM writes, and they never depend on an L1 fill. Synthesis, dxa DUT: fpga_gate: LUT 11157 -> 10294 (baseline 10102), LUTRAM 938 -> 638, Fmax 291 -> 301 MHz asic_gate: cell area 14919 -> 12449 um^2 (baseline 12449) The responses go back to smem_wr directly. SimX now models this. smem_wr takes one line per cycle, and a GMEM response goes ahead of a pending OOB line. A line is taken only into a free line register. A response that smem_wr cannot take stays on the memory response path, so the responses queued behind it wait too. Before, SimX accepted every response at once. On the wgmma-dxa parity config the SimX/rtlsim gap drops from 5.2% (30638 vs 32312) to 4.0% (31519 vs 32823). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/dxa/VX_dxa_gmem_req.sv | 50 ++------------------ sim/simx/dxa/dxa_core.cpp | 88 +++++++++++++++++++++++++---------- 2 files changed, 68 insertions(+), 70 deletions(-) diff --git a/hw/rtl/dxa/VX_dxa_gmem_req.sv b/hw/rtl/dxa/VX_dxa_gmem_req.sv index 929b02c3e7..43e467ab46 100644 --- a/hw/rtl/dxa/VX_dxa_gmem_req.sv +++ b/hw/rtl/dxa/VX_dxa_gmem_req.sv @@ -15,9 +15,9 @@ // // Manages a slot-table-indexed pool of MAX_OUTSTANDING in-flight tags. On // each accept, allocates a free slot and writes per-tag metadata. GMEM -// responses (out-of-order from the L1) queue in a response buffer with one -// entry per slot and drain to smem_wr via the `sw_*` channel. OOB entries -// skip the bus and are presented to smem_wr via an oob_pending[] bitvector. +// responses (out-of-order from the L1) are streamed directly to smem_wr +// via the `sw_*` channel — no rsp_buf BRAM. OOB entries skip the bus and +// are presented to smem_wr via an oob_pending[] bitvector. // // Tag reuse is per-slot via a busy[] bitvector and outstanding counter; // release order is independent of issue order (OOO drain in smem_wr). @@ -242,48 +242,6 @@ module VX_dxa_gmem_req import VX_gpu_pkg::*, VX_dxa_pkg::*; #( end end - VX_mem_bus_if #( - .DATA_SIZE (GMEM_BYTES), - .TAG_WIDTH (GMEM_TAG_WIDTH) - ) mem_bus_raw (); - - assign mem_bus_raw.req_valid = mem_bus_w.req_valid; - assign mem_bus_raw.req_data = mem_bus_w.req_data; - assign mem_bus_w.req_ready = mem_bus_raw.req_ready; - - // smem_wr takes a scattered line one element per cycle. The memory - // response path is shared with the L1 fills, so a response must never - // wait on it: one entry per slot means the buffer is never full. A - // response passes straight through when the buffer is empty and smem_wr - // is ready. - wire rsp_buf_empty, rsp_buf_full; - wire [GMEM_DATAW + GMEM_TAG_WIDTH-1:0] rsp_buf_data; - wire rsp_bypass = rsp_buf_empty && mem_bus_w.rsp_ready; - - VX_fifo_queue #( - .DATAW (GMEM_DATAW + GMEM_TAG_WIDTH), - .DEPTH (MAX_OUTSTANDING) - ) rsp_buf ( - .clk (clk), - .reset (reset), - .push (mem_bus_raw.rsp_valid && ~rsp_bypass), - .pop (~rsp_buf_empty && mem_bus_w.rsp_ready), - .data_in (mem_bus_raw.rsp_data), - .data_out (rsp_buf_data), - .empty (rsp_buf_empty), - .full (rsp_buf_full), - `UNUSED_PIN (alm_empty), - `UNUSED_PIN (alm_full), - `UNUSED_PIN (size) - ); - - `RUNTIME_ASSERT(~(mem_bus_raw.rsp_valid && ~rsp_bypass && rsp_buf_full), - ("%t: GMEM response buffer overflow", $time)) - - assign mem_bus_raw.rsp_ready = 1'b1; - assign mem_bus_w.rsp_valid = mem_bus_raw.rsp_valid || ~rsp_buf_empty; - assign mem_bus_w.rsp_data = rsp_buf_empty ? mem_bus_raw.rsp_data : rsp_buf_data; - VX_mem_bus_slice #( .DATA_SIZE (GMEM_BYTES), .TAG_WIDTH (GMEM_TAG_WIDTH), @@ -292,7 +250,7 @@ module VX_dxa_gmem_req import VX_gpu_pkg::*, VX_dxa_pkg::*; #( ) gmem_out_slice ( .clk (clk), .reset (reset), - .bus_in_if (mem_bus_raw), + .bus_in_if (mem_bus_w), .bus_out_if (gmem_bus_if) ); diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 3ad602dd27..7a227118a7 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -97,6 +97,8 @@ class DxaCore::Impl { struct InflightSlot { bool allocated = false; bool rsp_arrived = false; + bool oob_pending = false; // OOB line waiting for its turn into smem_wr + bool deferred = false; // last line parked until the others drain uint64_t ready_cycle = 0; // first cycle the line can be written to SMEM uint64_t free_cycle = 0; // first cycle the slot can carry a new read LineWork work; // captured at allocation @@ -126,7 +128,7 @@ class DxaCore::Impl { std::vector work_list; uint32_t ag_idx = 0; // next entry to issue (gmem_req) - // gmem_req → rsp_buf → smem_wr inflight bookkeeping. + // gmem_req → smem_wr inflight bookkeeping. std::array inflight; std::deque issued_order; // outstanding tags in issue order uint32_t drain_slot = UINT32_MAX; // slot currently draining @@ -217,27 +219,15 @@ class DxaCore::Impl { void tick() { ++cycle_; - // Drain in reverse-pipeline order (smem_wr → rsp_buf → gmem_req → - // addr_gen → setup → req-arb) so each stage's tick reads what its - // upstream produced earlier this cycle, mimicking forward stage flow. + // Drain in reverse-pipeline order (smem_wr → gmem_req → addr_gen → + // setup → req-arb) so each stage's tick reads what its upstream produced + // earlier this cycle, mimicking forward stage flow. - // 1) Drain GMEM responses → mark inflight slots arrived. - // gmem_arb_ already routes responses back to RspOut[worker_id] (its - // input index) and restores the user's tag (the per-worker slot id). + // 1) Hand one line per worker to smem_wr. gmem_arb_ already routes + // responses back to RspOut[worker_id] (its input index) and restores the + // user's tag (the per-worker slot id). for (uint32_t worker_id = 0; worker_id < workers_.size(); ++worker_id) { - auto& ch = gmem_arb_->RspOut.at(worker_id); - // One response per port per cycle (each worker has its own RspOut port). - if (!ch.empty()) { - auto& rsp = ch.peek(); - uint32_t slot_id = uint32_t(rsp.tag) & (VX_CFG_DXA_MAX_INFLIGHT - 1); - auto& w = workers_[worker_id]; - if (slot_id < w.inflight.size() && w.inflight[slot_id].allocated) { - w.inflight[slot_id].rsp_arrived = true; - w.inflight[slot_id].ready_cycle = cycle_ + kSmemWrLatency; - w.inflight[slot_id].rsp_data = rsp.data; - } - ch.pop(); - } + capture_line(workers_[worker_id], gmem_arb_->RspOut.at(worker_id)); } // 2) Tick workers (smem_wr drain → gmem_req issue). @@ -564,6 +554,52 @@ class DxaCore::Impl { } } + // ── smem_wr capture: one line per cycle into its line registers ─────── + // A GMEM response has priority over a pending OOB line. smem_wr holds one + // line being written, one captured behind it, and the transfer's last line + // while others are outstanding. A response it cannot take stays on the + // memory response path and holds the responses queued behind it. + void capture_line(Worker& w, SimChannel& ch) { + bool pend_busy = false; + bool defer_busy = false; + uint32_t oob_slot = UINT32_MAX; + for (uint32_t i = 0; i < w.inflight.size(); ++i) { + const auto& s = w.inflight[i]; + if (s.rsp_arrived && i != w.drain_slot) { + (s.deferred ? defer_busy : pend_busy) = true; + } + if (s.oob_pending && oob_slot == UINT32_MAX) { + oob_slot = i; + } + } + + uint32_t slot = oob_slot; + if (!ch.empty()) { + slot = uint32_t(ch.peek().tag) & (VX_CFG_DXA_MAX_INFLIGHT - 1); + if (!w.inflight[slot].allocated) { + ch.pop(); + return; + } + } + if (slot == UINT32_MAX) { + return; + } + + auto& s = w.inflight[slot]; + bool defer = s.work.last && (w.issued_order.size() > 1); + if (defer ? defer_busy : pend_busy) { + return; + } + if (!ch.empty()) { + s.rsp_data = ch.peek().data; + ch.pop(); + } + s.oob_pending = false; + s.rsp_arrived = true; + s.deferred = defer; + s.ready_cycle = cycle_ + kSmemWrLatency; + } + // ── gmem_req: issue MemReqs with available inflight slots ──────────── void tick_worker_gmem_req(Worker& w) { // One AGU step per cycle: advance ag_idx by at most one, issuing at most @@ -592,12 +628,13 @@ class DxaCore::Impl { return; } - // OOB lines skip the GMEM request entirely; we synthesize an immediate - // arrival (rsp data left null — smem_wr will use cfill). + // OOB lines skip the GMEM request entirely; smem_wr takes them in a cycle + // with no GMEM response (rsp data left null — smem_wr will use cfill). if (lw.oob) { w.inflight[slot].allocated = true; - w.inflight[slot].rsp_arrived = true; - w.inflight[slot].ready_cycle = cycle_ + kSmemWrLatency; + w.inflight[slot].rsp_arrived = false; + w.inflight[slot].oob_pending = true; + w.inflight[slot].deferred = false; w.inflight[slot].rsp_data.reset(); w.inflight[slot].work = lw; w.issued_order.push_back(slot); @@ -619,6 +656,8 @@ class DxaCore::Impl { w.inflight[slot].allocated = true; w.inflight[slot].rsp_arrived = false; + w.inflight[slot].oob_pending = false; + w.inflight[slot].deferred = false; w.inflight[slot].rsp_data.reset(); w.inflight[slot].work = lw; w.issued_order.push_back(slot); @@ -816,6 +855,7 @@ class DxaCore::Impl { // Slot done — release. s.allocated = false; s.rsp_arrived = false; + s.deferred = false; s.free_cycle = cycle_ + 1; s.rsp_data.reset(); auto it = std::find(w.issued_order.begin(), w.issued_order.end(), slot); From 9e9500226b3c86e41da5e7bf8f2045e0b7e8d6ae Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 20 Oct 2026 02:50:30 -0700 Subject: [PATCH 067/336] Include VX_define.vh before VX_csr_fpu's F-extension guard fd151efaa put the VX_CFG_EXT_F_ENABLE guard above the include of VX_define.vh, which is where the macro gets defined. Verilator compiles all files as one unit and saw the macro from earlier files, so rtlsim and xrtsim never noticed. Vivado project mode reads the file on its own, dropped the module, and the U55C xclbin build failed at elaboration (module 'VX_csr_fpu' not found). Include the config first, as VX_csr_data does. With this, the default U55C xclbin builds, and sgemm, raycast, dogfood and printf pass on the board. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_csr_fpu.sv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hw/rtl/core/VX_csr_fpu.sv b/hw/rtl/core/VX_csr_fpu.sv index d19d308279..1aefbf8642 100644 --- a/hw/rtl/core/VX_csr_fpu.sv +++ b/hw/rtl/core/VX_csr_fpu.sv @@ -11,9 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +`include "VX_define.vh" + `ifdef VX_CFG_EXT_F_ENABLE -`include "VX_define.vh" `include "VX_fpu_define.vh" // Per-warp FP CSRs (frm + fflags) for one residue class of warps modulo WID_STRIDE, From 360cd99e2ba9e2e97d973d8c3199d961ee419a2f Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 20 Oct 2026 07:12:20 -0700 Subject: [PATCH 068/336] hw/unittest: test the CP DMA port's launch fence VX_cp_llc_port holds a launch that follows DMA writes until an L1-only flush token sent behind them is answered by the last-level cache. No system test sees this: removing the fence passed flush_coherence and dogfood on rtlsim and on xrtsim at 4 cores + L2 with VX_CFG_CP_LLC_CLIENT=1, also with DBG_FLUSH_STALL. The runtime submits every DMA (vx_enqueue_write, the launch args, the QMD) as its own ring command and polls for its retire, and command batches carry only DCR writes and launches. So a host round trip always separates the last DMA write from the next launch, and the write has long drained by then. This unit test plays the cache on the port's slot. Requests enter an in-order input queue that the test can stall, and a request takes effect only when it leaves that queue. In every cycle that start_out is high, every DMA write the port accepted must already have taken effect. The test covers: - a launch with no write passes straight through; - reads do not arm the fence; - a launch behind a write held at the cache input waits for that write and for the token's answer, and the port takes no DMA request in the meantime; - the token is a read with the flush attribute and slot tag bit 0, and its answer does not reach the DMA; - the fence clears after the launch; - bursts of writes into a cache that accepts and drains intermittently; - the IO and no-allocate attributes. Known-bad, both caught by the start_out check: fence removed (start_out = start_in) FAIL at cycle 31 start released when the token is issued FAIL at cycle 96 tip PASSED (XLEN 32 and 64, NDEBUG and DEBUG builds) The hw unittest CI lane only builds the tests, so a new lane runs this one when VX_cp_llc_port.sv or the test changes. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/unittest.yaml | 7 + hw/unittest/Makefile | 3 + hw/unittest/cp_llc_port/Makefile | 27 ++ hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv | 102 +++++ hw/unittest/cp_llc_port/main.cpp | 393 ++++++++++++++++++ 5 files changed, 532 insertions(+) create mode 100644 hw/unittest/cp_llc_port/Makefile create mode 100644 hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv create mode 100644 hw/unittest/cp_llc_port/main.cpp diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 89c8ba08e8..12619a17cd 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -31,6 +31,13 @@ tests: via: script run: "make -C hw/unittest run-tcu-dsp" touches: [hw/rtl/tcu, hw/unittest/tcu_fedp] + # The command processor's DMA port into the last-level cache: a launch behind + # DMA writes must wait for them to reach the cache. No runtime path launches + # right behind a DMA write, so no system test observes this ordering. + - id: hw-cp-llc-port + via: script + run: "make -C hw/unittest/cp_llc_port run" + touches: [hw/rtl/mem/VX_cp_llc_port.sv, hw/unittest/cp_llc_port] # Texel blend (VX_tex_lerp, both forms the sampler uses) against the C model # the SW sampler and SimX share, and the exact Vulkan f/256 blend, over every # input. diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 0387f2a267..fb703fade9 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -21,6 +21,7 @@ all: $(MAKE) -C afu_reset_seq $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma + $(MAKE) -C cp_llc_port $(MAKE) -C cp_core $(MAKE) -C fma_unit $(MAKE) -C fdiv_unit @@ -52,6 +53,7 @@ run: $(MAKE) -C afu_reset_seq run $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run + $(MAKE) -C cp_llc_port run $(MAKE) -C cp_core run $(MAKE) -C fma_unit run $(MAKE) -C fdiv_unit run @@ -147,6 +149,7 @@ clean: $(MAKE) -C cp_axil_regfile clean $(MAKE) -C cp_axi_path clean $(MAKE) -C cp_dma clean + $(MAKE) -C cp_llc_port clean $(MAKE) -C cp_core clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean diff --git a/hw/unittest/cp_llc_port/Makefile b/hw/unittest/cp_llc_port/Makefile new file mode 100644 index 0000000000..f37bc1700d --- /dev/null +++ b/hw/unittest/cp_llc_port/Makefile @@ -0,0 +1,27 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := cp_llc_port + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) +CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/VX_trace_pkg.sv + +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(DPI_DIR) \ + -I$(RTL_DIR)/libs -I$(VORTEX_HOME)/hw/unittest/$(PROJECT) +RTL_INCLUDE += -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/fpu \ + -I$(RTL_DIR)/core -I$(RTL_DIR)/cp + +TOP := VX_cp_llc_port_top + +include ../common.mk diff --git a/hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv b/hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv new file mode 100644 index 0000000000..ad339b8342 --- /dev/null +++ b/hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv @@ -0,0 +1,102 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_define.vh" + +// Test wrapper around VX_cp_llc_port: the last-level cache side of the +// port's VX_mem_bus_if is flattened so the C++ harness can play the cache. +module VX_cp_llc_port_top import VX_gpu_pkg::*; #( + parameter DATA_SIZE = 64, + parameter TAG_WIDTH = 4, + parameter SLOT_TAG_WIDTH = UUID_WIDTH + 1 + TAG_WIDTH, + parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE) +) ( + input wire clk, + input wire reset, + + input wire req_valid, + input wire req_rw, + input wire req_noalloc, + input wire [ADDR_WIDTH-1:0] req_addr, + input wire [DATA_SIZE*8-1:0] req_data, + input wire [DATA_SIZE-1:0] req_byteen, + input wire [TAG_WIDTH-1:0] req_tag, + output wire req_ready, + + output wire rsp_valid, + output wire [DATA_SIZE*8-1:0] rsp_data, + output wire [TAG_WIDTH-1:0] rsp_tag, + input wire rsp_ready, + + input wire start_in, + output wire start_out, + output wire busy, + + output wire mem_req_valid, + output wire mem_req_rw, + output wire [ADDR_WIDTH-1:0] mem_req_addr, + output wire [DATA_SIZE*8-1:0] mem_req_data, + output wire [DATA_SIZE-1:0] mem_req_byteen, + output wire mem_req_flush, + output wire mem_req_io, + output wire mem_req_noalloc, + output wire [SLOT_TAG_WIDTH-1:0] mem_req_tag, + input wire mem_req_ready, + + input wire mem_rsp_valid, + input wire [DATA_SIZE*8-1:0] mem_rsp_data, + input wire [SLOT_TAG_WIDTH-1:0] mem_rsp_tag, + output wire mem_rsp_ready +); + VX_mem_bus_if #( + .DATA_SIZE (DATA_SIZE), + .TAG_WIDTH (SLOT_TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH) + ) mem_bus_if (); + + VX_cp_llc_port #( + .DATA_SIZE (DATA_SIZE), + .SLOT_TAG_WIDTH (SLOT_TAG_WIDTH), + .TAG_WIDTH (TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH) + ) dut ( + .clk (clk), + .reset (reset), + .req_valid (req_valid), + .req_rw (req_rw), + .req_noalloc (req_noalloc), + .req_addr (req_addr), + .req_data (req_data), + .req_byteen (req_byteen), + .req_tag (req_tag), + .req_ready (req_ready), + .rsp_valid (rsp_valid), + .rsp_data (rsp_data), + .rsp_tag (rsp_tag), + .rsp_ready (rsp_ready), + .start_in (start_in), + .start_out (start_out), + .busy (busy), + .mem_bus_if (mem_bus_if) + ); + + assign mem_req_valid = mem_bus_if.req_valid; + assign mem_req_rw = mem_bus_if.req_data.rw; + assign mem_req_addr = mem_bus_if.req_data.addr; + assign mem_req_data = mem_bus_if.req_data.data; + assign mem_req_byteen = mem_bus_if.req_data.byteen; + mem_bus_attr_t mem_req_attr; + assign mem_req_attr = mem_bus_attr_t'(mem_bus_if.req_data.attr); + assign mem_req_flush = mem_req_attr.is_flush; + assign mem_req_io = mem_req_attr.is_addr_io; + assign mem_req_noalloc = mem_req_attr.is_noalloc; + `UNUSED_VAR (mem_req_attr) + assign mem_req_tag = mem_bus_if.req_data.tag; + assign mem_bus_if.req_ready = mem_req_ready; + + assign mem_bus_if.rsp_valid = mem_rsp_valid; + assign mem_bus_if.rsp_data.data = mem_rsp_data; + assign mem_bus_if.rsp_data.tag = mem_rsp_tag; + assign mem_rsp_ready = mem_bus_if.rsp_ready; + +endmodule diff --git a/hw/unittest/cp_llc_port/main.cpp b/hw/unittest/cp_llc_port/main.cpp new file mode 100644 index 0000000000..5975b84561 --- /dev/null +++ b/hw/unittest/cp_llc_port/main.cpp @@ -0,0 +1,393 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +// Verilator unit test for VX_cp_llc_port, the command processor's DMA port +// into the last-level cache. +// +// The harness plays the cache: requests enter an in-order input queue that the +// test can stall, and a request takes effect only when it leaves that queue, so +// an accepted DMA write is not yet visible while it sits in the port's buffer +// or in the queue. Every cycle the launch leaves the port (start_out), all DMA +// writes the port accepted must already have taken effect. +// +// Coverage: +// 1. A launch with no DMA write behind it passes straight through. +// 2. DMA reads return data and tag, and do not arm the launch fence. +// 3. A launch behind a DMA write held at the cache input waits for that +// write, then for the fence token's answer; the token is a read with the +// flush attribute and slot tag bit 0 set, and its answer is not forwarded +// to the DMA. The port takes no DMA request while the fence is pending. +// 4. The fence clears: a following launch with no new write passes through. +// 5. Several writes into a cache that accepts and drains in bursts. +// 6. Request attributes: IO aperture, and no-allocate on writes only. + +#include "vl_simulator.h" +#include "VVX_cp_llc_port_top.h" +#include +#include +#include +#include +#include + +#ifndef TRACE_START_TIME +#define TRACE_START_TIME 0ull +#endif +#ifndef TRACE_STOP_TIME +#define TRACE_STOP_TIME -1ull +#endif + +static uint64_t timestamp = 0; +static bool trace_en = false; +double sc_time_stamp() { return timestamp; } +bool sim_trace_enabled() { return trace_en; } +void sim_trace_enable(bool e) { trace_en = e; } + +#define EXPECT(cond, msg) do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d (cycle %llu): %s\n", __FILE__, __LINE__, \ + (unsigned long long)cycles, msg); \ + std::exit(1); \ + } \ +} while (0) + +using Top = VVX_cp_llc_port_top; + +static constexpr uint32_t LINE_SIZE = 64; +static constexpr uint32_t TAG_BITS = 4; +static constexpr uint32_t DATA_LINE = 0x100000; // above the IO aperture +static constexpr uint32_t IO_LINE = VX_MEM_IO_BASE_ADDR / LINE_SIZE; + +static_assert(VX_MEM_IO_BASE_ADDR % LINE_SIZE == 0, "IO base must be line aligned"); +static_assert(DATA_LINE >= VX_MEM_IO_END_ADDR / LINE_SIZE, "data line must be outside the IO aperture"); + +struct CacheReq { + bool rw; + bool flush; + uint32_t addr; + uint32_t data; + uint64_t tag; +}; + +struct CacheRsp { + uint32_t data; + uint64_t tag; +}; + +static uint64_t cycles = 0; + +// The last-level cache seen through the port's slot. +struct Cache { + bool accept = true; // input ready + uint32_t period = 0; // with duty: ready duty of every period cycles + uint32_t duty = 0; + uint32_t service = 1; // cycles per request + std::deque input; + std::deque rsps; + std::unordered_map mem; + uint32_t busy_cycles = 0; + uint64_t writes_applied = 0; + + // Retire at most one request from the head of the input queue. + void step() { + if (input.empty()) { + return; + } + if (++busy_cycles < service) { + return; + } + busy_cycles = 0; + CacheReq r = input.front(); + input.pop_front(); + if (r.flush) { + rsps.push_back({0, r.tag}); + } else if (r.rw) { + mem[r.addr] = r.data; + ++writes_applied; + } else { + auto it = mem.find(r.addr); + rsps.push_back({it != mem.end() ? it->second : 0, r.tag}); + } + } +}; + +struct Bench { + vl_simulator sim; + uint64_t tick = 0; + Cache cache; + + uint64_t dma_writes = 0; // DMA writes the port accepted + uint64_t fence_tokens = 0; + uint64_t starts = 0; + std::deque dma_rsps; + + Bench() { + idle_inputs(); + tick = sim.reset(tick); + } + + void idle_inputs() { + sim->req_valid = 0; + sim->req_rw = 0; + sim->req_noalloc = 0; + sim->req_addr = 0; + sim->req_byteen = 0; + sim->req_tag = 0; + sim->rsp_ready = 1; + sim->start_in = 0; + sim->mem_req_ready = 0; + sim->mem_rsp_valid = 0; + sim->mem_rsp_tag = 0; + for (uint32_t i = 0; i < LINE_SIZE / 4; ++i) { + sim->req_data[i] = 0; + sim->mem_rsp_data[i] = 0; + } + } + + // One clock: present the cache's outputs, sample every handshake on the + // settled inputs, then advance the edge and the cache model. + void cycle() { + sim->mem_req_ready = cache.accept && (cache.period == 0 || (cycles % cache.period) < cache.duty); + sim->mem_rsp_valid = !cache.rsps.empty(); + if (!cache.rsps.empty()) { + sim->mem_rsp_tag = cache.rsps.front().tag; + sim->mem_rsp_data[0] = cache.rsps.front().data; + } + sim->eval(); + + if (sim->start_out) { + ++starts; + EXPECT(cache.writes_applied == dma_writes, + "launch left the port before its DMA writes reached the cache"); + } + if (sim->req_valid && sim->req_ready && sim->req_rw) { + ++dma_writes; + } + if (sim->mem_req_valid && sim->mem_req_ready) { + CacheReq r; + r.rw = sim->mem_req_rw; + r.flush = sim->mem_req_flush; + r.addr = sim->mem_req_addr; + r.data = sim->mem_req_data[0]; + r.tag = sim->mem_req_tag; + if (r.flush) { + ++fence_tokens; + EXPECT(!r.rw, "fence token must be a read"); + EXPECT((r.tag & 1) == 1, "fence token must carry slot tag bit 0"); + } else { + EXPECT((r.tag & 1) == 0, "DMA request must clear slot tag bit 0"); + } + cache.input.push_back(r); + } + if (sim->mem_rsp_valid && sim->mem_rsp_ready) { + CacheRsp rsp = cache.rsps.front(); + cache.rsps.pop_front(); + if ((rsp.tag & 1) == 0) { + dma_rsps.push_back({rsp.data, (rsp.tag >> 1) & ((1u << TAG_BITS) - 1)}); + } + } + if (sim->rsp_valid && sim->rsp_ready) { + EXPECT(!dma_rsps.empty(), "the port forwarded a response the DMA never asked for"); + EXPECT(sim->rsp_tag == dma_rsps.front().tag, "DMA response tag mismatch"); + EXPECT(sim->rsp_data[0] == dma_rsps.front().data, "DMA response data mismatch"); + dma_rsps.pop_front(); + } + + tick = sim.step(tick, 2); + ++cycles; + cache.step(); + } + + // Hold one DMA request until the port takes it. + void dma_req(bool rw, uint32_t addr, uint32_t data, uint32_t tag, bool noalloc = false) { + sim->req_valid = 1; + sim->req_rw = rw; + sim->req_noalloc = noalloc; + sim->req_addr = addr; + sim->req_data[0] = data; + sim->req_byteen = ~0ull; + sim->req_tag = tag; + for (int i = 0;; ++i) { + EXPECT(i < 1000, "DMA request never accepted"); + sim->eval(); + bool fire = sim->req_ready; + cycle(); + if (fire) { + break; + } + } + sim->req_valid = 0; + } + + // Pulse start_in for one cycle; returns whether start_out fired with it. + bool launch() { + sim->start_in = 1; + uint64_t before = starts; + cycle(); + sim->start_in = 0; + return starts != before; + } + + void run(int n) { + for (int i = 0; i < n; ++i) { + cycle(); + } + } + + // Wait for the launch to leave the port; returns the cycles waited. + int wait_start(int limit) { + uint64_t before = starts; + for (int i = 0; i < limit; ++i) { + cycle(); + if (starts != before) { + return i + 1; + } + } + EXPECT(false, "launch never left the port"); + return -1; + } +}; + +static void test_launch_without_writes(Bench& b) { + b.cache.accept = true; + b.run(4); + EXPECT(!b.sim->busy, "port busy at idle"); + EXPECT(b.launch(), "launch with no DMA write must pass through in the same cycle"); + b.run(8); + EXPECT(b.fence_tokens == 0, "no fence token expected without a DMA write"); + EXPECT(!b.sim->busy, "port busy after a pass-through launch"); +} + +static void test_reads(Bench& b) { + b.cache.accept = true; + b.cache.mem[DATA_LINE + 7] = 0xcafe0007; + b.cache.mem[DATA_LINE + 9] = 0xcafe0009; + b.dma_req(false, DATA_LINE + 7, 0, 3); + b.dma_req(false, DATA_LINE + 9, 0, 5); + b.run(10); + EXPECT(b.dma_rsps.empty(), "DMA read responses missing"); + uint64_t tokens = b.fence_tokens; + EXPECT(b.launch(), "DMA reads must not hold the launch"); + b.run(4); + EXPECT(b.fence_tokens == tokens, "DMA reads must not arm the fence"); +} + +static void test_fence_holds_launch(Bench& b) { + constexpr int STALL = 64; + b.cache.accept = false; + b.dma_req(true, DATA_LINE + 1, 0x11111111, 1); + EXPECT(b.cache.writes_applied < b.dma_writes, "write should still be held at the cache input"); + + uint64_t starts = b.starts; + EXPECT(!b.launch(), "launch passed the port with a DMA write still in front of the cache"); + for (int i = 0; i < STALL; ++i) { + b.sim->eval(); + EXPECT(b.sim->busy, "port must be busy while the fence is pending"); + EXPECT(!b.sim->req_ready, "port must take no DMA request while the fence is pending"); + b.cycle(); + } + EXPECT(b.starts == starts, "launch left the port while the cache input was stalled"); + + // Release the cache slowly: the write, then the token, each take 6 cycles. + uint64_t tokens = b.fence_tokens; + b.cache.accept = true; + b.cache.service = 6; + int waited = b.wait_start(200); + EXPECT(waited >= 2 * 6, "launch left before the cache answered the fence token"); + EXPECT(b.fence_tokens == tokens + 1, "exactly one fence token expected"); + EXPECT(b.dma_rsps.empty(), "fence answer must not reach the DMA"); + b.run(2); + EXPECT(!b.sim->busy, "port still busy after the fence answered"); + b.cache.service = 1; +} + +static void test_fence_clears(Bench& b) { + uint64_t tokens = b.fence_tokens; + EXPECT(b.launch(), "a launch with no new DMA write must pass through after a fence"); + b.run(8); + EXPECT(b.fence_tokens == tokens, "fence re-armed without a new DMA write"); +} + +static void test_write_bursts(Bench& b) { + for (int round = 0; round < 8; ++round) { + b.cache.service = 1 + (round % 4); + b.cache.accept = true; + b.cache.period = 5 + round; + b.cache.duty = 1 + (round % 3); + for (int i = 0; i < 6; ++i) { + b.dma_req(true, DATA_LINE + 32 * round + i, 0x20000000 + 16 * round + i, i & 0xf); + } + b.cache.period = 0; + b.cache.accept = (round % 2) == 0; + uint64_t starts = b.starts; + bool passed = b.launch(); + EXPECT(!passed || b.cache.writes_applied == b.dma_writes, "launch passed undrained writes"); + if (!passed) { + b.run(round * 5); + b.cache.accept = true; + for (int i = 0; b.starts == starts; ++i) { + EXPECT(i < 500, "launch never left the port"); + b.cycle(); + } + } + b.run(4); + EXPECT(!b.sim->busy, "port busy after a burst launch"); + } + // Every written line reads back through the port. + b.cache.accept = true; + b.cache.service = 1; + b.dma_req(false, DATA_LINE + 32 * 7 + 5, 0, 2); + b.run(10); + EXPECT(b.cache.mem[DATA_LINE + 32 * 7 + 5] == 0x20000000 + 16 * 7 + 5, "burst write lost"); +} + +static void test_attributes(Bench& b) { + b.cache.accept = false; + struct Case { bool rw; uint32_t addr; bool noalloc; bool io; bool exp_noalloc; }; + const Case cases[] = { + {true, IO_LINE, false, true, false}, + {false, IO_LINE, true, true, false}, + {true, DATA_LINE + 3, true, false, true }, + {false, DATA_LINE + 3, true, false, false}, + }; + for (const auto& c : cases) { + b.sim->req_valid = 1; + b.sim->req_rw = c.rw; + b.sim->req_noalloc = c.noalloc; + b.sim->req_addr = c.addr; + b.sim->req_tag = 1; + b.sim->eval(); + EXPECT(b.sim->req_ready, "port buffer should be empty"); + b.cycle(); + b.sim->req_valid = 0; + b.sim->eval(); + EXPECT(b.sim->mem_req_valid, "request missing at the cache input"); + EXPECT(b.sim->mem_req_io == c.io, "IO attribute mismatch"); + EXPECT(b.sim->mem_req_noalloc == c.exp_noalloc, "no-allocate attribute mismatch"); + EXPECT(!b.sim->mem_req_flush, "DMA request carries the flush attribute"); + b.cache.accept = true; + b.cycle(); + b.cache.accept = false; + } + b.cache.accept = true; + b.run(10); + // Leave the fence armed by these writes discharged. + b.launch(); + b.wait_start(100); + b.run(4); +} + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + Bench b; + + test_launch_without_writes(b); + test_reads(b); + test_fence_holds_launch(b); + test_fence_clears(b); + test_write_bursts(b); + test_attributes(b); + + EXPECT(b.cache.input.empty() && b.cache.rsps.empty(), "cache traffic left over"); + std::printf("PASSED\n"); + return 0; +} From 64f4268018454cd4b3ab5323f8f314a10c0f779b Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 20 Oct 2026 07:12:20 -0700 Subject: [PATCH 069/336] docs: select the FPGA target with blackbox.sh --target blackbox.sh sets TARGET to empty before it parses its options and passes TARGET to make only from --target. The FPGA docs exported TARGET=hw instead, so the run fell back to the xrt driver's default xrtsim target and failed with "No such file 'vortex_afu.xclbin'". This changes the xrt runs in docs/fpga_setup.md, hw/syn/xilinx/README and the apptainer README, and the FireSim run in docs/simulation.md, to use --target. It also drops the PLATFORM variable from those runs, since nothing on the run path reads it. The blackbox.sh help no longer says --target is for aved only. It lists the targets for xrt, firesim and aved. With --target=hw, sgemm, raycast, dogfood and printf ran on the U55C. Co-Authored-By: Claude Opus 5.5 --- ci/blackbox.sh | 2 +- docs/fpga_setup.md | 12 +++++++----- docs/simulation.md | 3 +-- hw/syn/xilinx/README | 6 +++--- miscs/apptainer/README.md | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ci/blackbox.sh b/ci/blackbox.sh index 21215167a0..c541d04c14 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -27,7 +27,7 @@ show_help() show_usage echo " where" echo "--driver: gpu, simx, rtlsim, opae, xrt, aved" - echo "--target: aved driver only -- avedsim (default), sim, hw" + echo "--target: xrt -- xrtsim (default), hw_emu, hw; firesim -- hw, hw_emu; aved -- avedsim (default), sim, hw" echo "--app: any subfolder test under regression, graphics, mpi, opencl, or hip" echo "--perf: 0=disable, 1=core, 3=icache, 4=dcache, 5=l2cache, 6=l3cache, 7=mem," echo " 11=tcu, 12=raster, 13=tex, 14=om, 15=rtu, 16=dxa" diff --git a/docs/fpga_setup.md b/docs/fpga_setup.md index 0419cfd18d..7701c11383 100644 --- a/docs/fpga_setup.md +++ b/docs/fpga_setup.md @@ -148,20 +148,22 @@ sudo bash -c 'source /opt/xilinx/xrt/setup.sh && \ The [blackbox.sh](./simulation.md) script within the build directory can be used to run a test with Vortex’s xrt driver using the following command: -`FPGA_BIN_DIR= TARGET=hw|hw_emu PLATFORM= ./ci/blackbox.sh --driver=xrt --app=` +`FPGA_BIN_DIR= ./ci/blackbox.sh --driver=xrt --target=hw|hw_emu --app=` + +The target is a `blackbox.sh` option, not an environment variable: `blackbox.sh` resets `TARGET`, so an exported `TARGET=hw` is ignored and the run falls back to the `xrtsim` simulator. For example: -```FPGA_BIN_DIR= hw/syn/xilinx/xrt/build_4c_u280_xilinx_u280_gen3x16_xdma_1_202211_1_hw/bin TARGET=hw PLATFORM=xilinx_u280_gen3x16_xdma_1_202211_1 ./ci/blackbox.sh --driver=xrt --app=demo``` +```FPGA_BIN_DIR= hw/syn/xilinx/xrt/build_4c_u280_xilinx_u280_gen3x16_xdma_1_202211_1_hw/bin ./ci/blackbox.sh --driver=xrt --target=hw --app=demo``` Run sgemm on a U55C `hw` bitstream (host memory enabled per the section above), `-n` sets the matrix size: ```bash -FPGA_BIN_DIR=$(realpath hw/syn/xilinx/xrt//bin) TARGET=hw \ - ./ci/blackbox.sh --driver=xrt --app=sgemm --args="-n128" +FPGA_BIN_DIR=$(realpath hw/syn/xilinx/xrt//bin) \ + ./ci/blackbox.sh --driver=xrt --target=hw --app=sgemm --args="-n128" ``` -The same command with `TARGET=hw_emu` against a `hw_emu` build runs it in emulation +The same command with `--target=hw_emu` against a `hw_emu` build runs it in emulation (use a small `-n`, e.g. `-n16`, since emulation is slow). Synthesis for Intel (Altera) Boards diff --git a/docs/simulation.md b/docs/simulation.md index 8fbc55f94d..7803c072b5 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -29,9 +29,8 @@ For scale, `sgemm -n32` takes ~1228 ms on rtlsim and ~8 ms on an Alveo U55C, and `sgemm -n1024` — over a billion instructions — completes in ~85 s. ```bash -export TARGET=hw export FPGA_BIN_DIR= -./ci/blackbox.sh --driver=firesim --app=sgemm --args="-n1024" +./ci/blackbox.sh --driver=firesim --target=hw --app=sgemm --args="-n1024" ``` The architecture, build flow and design invariants are documented in diff --git a/hw/syn/xilinx/README b/hw/syn/xilinx/README index 7f0c43d38c..0ae222a173 100644 --- a/hw/syn/xilinx/README +++ b/hw/syn/xilinx/README @@ -70,9 +70,9 @@ TARGET=hw PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 VPP_FLAGS="--from_step vp TARGET=hw PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 VPP_FLAGS="--from_step vpl.impl.route_design" make > build.log 2>&1 & # running test -FPGA_BIN_DIR= TARGET=hw_emu ./ci/blackbox.sh --driver=xrt --app=demo -FPGA_BIN_DIR= TARGET=hw ./ci/blackbox.sh --driver=xrt --app=sgemm --args="-n1024" -FPGA_BIN_DIR= XRT_DEVICE_INDEX=1 TARGET=hw ./ci/blackbox.sh --driver=xrt --app=sgemm --args="-n1024" +FPGA_BIN_DIR= ./ci/blackbox.sh --driver=xrt --target=hw_emu --app=demo +FPGA_BIN_DIR= ./ci/blackbox.sh --driver=xrt --target=hw --app=sgemm --args="-n1024" +FPGA_BIN_DIR= XRT_DEVICE_INDEX=1 ./ci/blackbox.sh --driver=xrt --target=hw --app=sgemm --args="-n1024" # build report logs /bin/vortex_afu.xclbin.info diff --git a/miscs/apptainer/README.md b/miscs/apptainer/README.md index 06e6a4e3f2..541db951f7 100644 --- a/miscs/apptainer/README.md +++ b/miscs/apptainer/README.md @@ -105,7 +105,7 @@ Apptainer> pwd Apptainer> make -C runtime/ clean -Apptainer> FPGA_BIN_DIR=/home/vortex/build/hw/syn/xilinx/xrt/test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin TARGET=hw PLATFORM=xilinx_u50_gen3x16_xdma_5_202210_1 ./ci/blackbox.sh --driver=xrt --app=demo +Apptainer> FPGA_BIN_DIR=/home/vortex/build/hw/syn/xilinx/xrt/test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin ./ci/blackbox.sh --driver=xrt --target=hw --app=demo Verify following line being printed: info: device name=xilinx_u50_gen3x16_xdma_base_5, memory_capacity=0x200000000 bytes, memory_banks=32. From c961402531c10f46ca68ee5b635903eec9276437 Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 20 Oct 2026 09:29:46 -0700 Subject: [PATCH 070/336] Round-robin the DXA and TCU on the LMEM DMA port The LMEM DMA port is shared by two clients: DXA writes into SMEM and the TCU's WGMMA operand reads out of it. The port gave the DXA fixed priority. A tiled-scatter DXA transfer writes one bank row almost every cycle for about 540 cycles, and during that time the TCU's B-tile reads got no grant. In sgemm_tcu_wg_dxa, a WGMMA batch that had not finished when such a transfer started stalled for about 510 cycles. The stall held up the warpgroup's barrier, so the producer's next DXA request arrived late and the DXA engine sat idle. Whether a batch lost this race came down to a few cycles of timing. The race started to go the wrong way once rtlsim began completing DRAM requests in per-bank arrival order. With that testbench change reverted, the same RTL runs in 31,037 cycles instead of 32,206. Grant the port round-robin so both clients keep moving. This adds one bit of arbiter state and no storage. A TCU read now waits at most one DXA write. Configurations with a single DMA client are unchanged, because the arbiter there is a pass-through. SimX serves its DMA clients round-robin in the same way. It also lets the DXA capture a GMEM line in the same cycle that its smem_wr pend register loads the fill buffer. VX_dxa_smem_wr's pend is free again that cycle, so rtlsim takes a line every two cycles. SimX had captured before draining, which held pend busy one extra cycle and admitted a line only every three cycles. rtlsim cycles, before -> after: wgmma-dxa (IW4 NT32, 128x128x64) 32,823 -> 29,487 (-10.2%) wgmma-sp-dxa (IW4 NT32, 128x128x128) 54,973 -> 48,573 (-11.6%) wgmma-dxa-mcast NT4 358,969 -> 358,897 wgmma-dxa-mcast NT16 78,882 -> 78,840 copy NT4 / NT16 / mcast, wgmma-fp16-ss, wgmma-fedp2k-rs, wgmma-sparse unchanged SimX vs rtlsim: wgmma-dxa 2.29% (was 5.24%), wgmma-sp-dxa 2.52% (was 4.97%), wgmma-dxa-mcast 0.60%. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_mem_unit.sv | 3 ++- sim/simx/core.cpp | 4 ++-- sim/simx/dxa/dxa_core.cpp | 17 +++++++++-------- sim/simx/mem/local_mem.cpp | 12 ++++++++---- sim/simx/mem/local_mem.h | 2 +- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index a766858777..17b5d7c665 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -166,6 +166,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( `ASSIGN_VX_MEM_BUS_IF_EX (dma_arb_in_if[LMEM_DMA_TCU_IDX], tcu_lmem_if, LMEM_DMA_IN_TAG_W, TCU_LMEM_TAG_W, UUID_WIDTH); `endif + // Round-robin, so a DXA write stream cannot starve the TCU's operand reads. VX_mem_bus_arb #( .NUM_INPUTS (LMEM_DMA_INPUTS), .NUM_OUTPUTS (1), @@ -173,7 +174,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .TAG_WIDTH (LMEM_DMA_IN_TAG_W), .ATTR_WIDTH (LMEM_DMA_ATTR_W), .ADDR_WIDTH (LMEM_DMA_ADDR_WIDTH), - .ARBITER ("P") + .ARBITER ("R") ) lmem_dma_arb ( .clk (clk), .reset (reset), diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 8b5682742e..2455e026b1 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -146,7 +146,7 @@ class Core::Impl { // create local memory. // The DXA writes and the TCU tile-buffer reads share the row-wide DMA - // port, DXA first. + // port, round-robin. snprintf(sname, 100, "%s-lmem", name.c_str()); local_mem_ = LocalMem::Create(sname, LocalMem::Config{ (1 << VX_CFG_LMEM_LOG_SIZE), @@ -274,7 +274,7 @@ class Core::Impl { #ifdef VX_CFG_TCU_WGMMA_ENABLE // Bind the TCU tile-buffer subsystem (TcuTbuf) to the LMEM DMA port, - // after the DXA. + // as the client after the DXA. { auto& tbuf = tcu_unit_->tbuf(); uint32_t base = VX_CFG_EXT_DXA_ENABLED * LocalMem::DMA_PORTS; diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 7a227118a7..55ef8786d3 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -223,17 +223,18 @@ class DxaCore::Impl { // setup → req-arb) so each stage's tick reads what its upstream produced // earlier this cycle, mimicking forward stage flow. - // 1) Hand one line per worker to smem_wr. gmem_arb_ already routes - // responses back to RspOut[worker_id] (its input index) and restores the - // user's tag (the per-worker slot id). + // 1) smem_wr drain, then 2) hand it one line per worker, then 3) issue + // the next GMEM read. A line leaving pend for the fill buffer frees pend + // for a capture in the same cycle. gmem_arb_ already routes responses + // back to RspOut[worker_id] (its input index) and restores the user's tag + // (the per-worker slot id). for (uint32_t worker_id = 0; worker_id < workers_.size(); ++worker_id) { - capture_line(workers_[worker_id], gmem_arb_->RspOut.at(worker_id)); - } - - // 2) Tick workers (smem_wr drain → gmem_req issue). - for (auto& w : workers_) { + auto& w = workers_[worker_id]; if (w.state == WState::RUNNING) { tick_worker_smem_wr(w); + } + capture_line(w, gmem_arb_->RspOut.at(worker_id)); + if (w.state == WState::RUNNING) { tick_worker_gmem_req(w); } } diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 9e3dc16884..925bcc3b92 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -54,6 +54,7 @@ class LocalMem::Impl { #endif MemCrossBar::Ptr mem_xbar_; mutable PerfStats perf_stats_; + uint32_t dma_last_grant_ = 0; uint64_t to_local_addr(uint64_t addr) { return bit_getw(addr, 0, addr_bits_-1); @@ -110,6 +111,7 @@ class LocalMem::Impl { void reset() { perf_stats_ = PerfStats(); + dma_last_grant_ = config_.dma_clients ? (config_.dma_clients - 1) : 0; #if VX_CFG_EXT_A_ENABLED amo_unit_.reset(); std::fill(amo_busy_.begin(), amo_busy_.end(), 0); @@ -119,13 +121,14 @@ class LocalMem::Impl { } // Serve one DMA client's row access this cycle and return the banks it - // occupies. Clients are served at fixed priority, and DMA wins every bank - // it touches over the LSU: a read touches all banks, a write those its - // byte enables cover. + // occupies. Clients are served round-robin, starting after the last one + // served, and DMA wins every bank it touches over the LSU: a read touches + // all banks, a write those its byte enables cover. uint64_t dma_tick() { uint32_t num_banks = (1 << config_.B); uint32_t lg2_line_size = log2ceil(config_.line_size); - for (uint32_t c = 0; c < config_.dma_clients; ++c) { + for (uint32_t i = 0; i < config_.dma_clients; ++i) { + uint32_t c = (dma_last_grant_ + 1 + i) % config_.dma_clients; bool pending = false; bool rsp_ready = true; for (uint32_t p = 0; p < DMA_PORTS; ++p) { @@ -178,6 +181,7 @@ class LocalMem::Impl { DT(4, simobject_->name() << "-dma" << c << " req : " << req); in.pop(); } + dma_last_grant_ = c; return banks; } return 0; diff --git a/sim/simx/mem/local_mem.h b/sim/simx/mem/local_mem.h index 2a879974bd..6772c8b04f 100644 --- a/sim/simx/mem/local_mem.h +++ b/sim/simx/mem/local_mem.h @@ -31,7 +31,7 @@ class LocalMem : public SimObject { uint32_t num_reqs; uint32_t B; // log2 number of banks bool write_reponse; - uint32_t dma_clients; // row-wide DMA masters, in priority order + uint32_t dma_clients; // row-wide DMA masters, served round-robin }; struct PerfStats { From 4c0f266ead4a923026cda861122160242ff938a1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 07:47:54 -0700 Subject: [PATCH 071/336] Size each slice of a stream arbiter tree by its own requests VX_stream_arb turns an arbiter with more than 1.5x MAX_FANOUT requests into a tree. On the input side, slices of MAX_FANOUT rows feed a join arbiter. On the output side, a fork arbiter feeds the slices. Every slice was wired as if it were full: its select was declared CLOG2(MAX_FANOUT) bits wide, and it got all NUM_OUTPUTS lanes (NUM_INPUTS on the output side). A last slice with fewer rows has a narrower select, so the slice's packed sel_out did not line up with the wire it drove. Verilator rejects that width mismatch. With more than one output, the selects of different outputs also landed at the wrong offsets. A last slice with fewer requests than the other side's width was elaborated as the opposite kind of arbiter, and it delivered requests to the wrong outputs. This happens whenever the requests are not a whole number of slices. One example is VX_CFG_CP_LLC_CLIENT: the command processor's DMA slot sits beside 2^k L1 ports on the shared L2. At 4 cores x 16 L1 ports that gives 65 inputs per bank arbiter, the last slice has one input, and the rtlsim build fails lint (WIDTHTRUNC at VX_stream_arb.sv:85). Each slice is now sized by the rows it actually has. Its select is that wide and is zero-extended into the join's data or the fork's select table. A last slice with fewer requests than the other side keeps only the lanes it can serve, and the join (or fork) gets that many lanes in its last row. Nothing is buffered on a lane that cannot drain. The flat output-side arbiter also stops indexing ready_out past NUM_OUTPUTS when NUM_OUTPUTS is not a multiple of NUM_INPUTS. The missing outputs of the last row never request. Full slices produce the same logic as before. For every parameter set in a sweep where the old module is Verilator-lint clean, Yosys proves the new module sequentially equivalent to it (equiv_make/equiv_simple/equiv_induct after sv2v). The sweep has 748 shapes: NUM_INPUTS 1-40 x NUM_OUTPUTS 1-8 at MAX_FANOUT 4 and 8; 48-130 requests on one side with 1, 2 or 4 on the other, in both directions; and OUT_BUF 0-3 and ARBITER R/P/C/M on the slice shapes. The old module lints clean on 584 of them, and all 584 are proven. The new module also lints clean on the other 164, which the old one could not build. A mutant with a corrupted slice select fails the same check. The synthesis-gate DUTs elaborate only full slices (16x1 in tensor at both gates, and in the 16-thread core at the ASIC gate), so their netlists do not change. hw/unittest/stream_arb drives random valids and output stalls through 21 shapes. They include last slices of one lane, last slices narrower than the other side, last slices that are not a whole number of rows, two-level joins, and both arbiter directions. The test checks that every packet leaves exactly once, on an output its input maps to, with its row as select. Against the old module the test does not build (14 lint warnings). Built anyway with -Wno-fatal, it misroutes packets. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/unittest.yaml | 8 + hw/rtl/libs/VX_stream_arb.sv | 79 ++++++--- hw/unittest/Makefile | 3 + hw/unittest/stream_arb/Makefile | 23 +++ hw/unittest/stream_arb/VX_stream_arb_tb.sv | 179 ++++++++++++++++++++ hw/unittest/stream_arb/VX_stream_arb_top.sv | 63 +++++++ hw/unittest/stream_arb/main.cpp | 62 +++++++ 7 files changed, 391 insertions(+), 26 deletions(-) create mode 100644 hw/unittest/stream_arb/Makefile create mode 100644 hw/unittest/stream_arb/VX_stream_arb_tb.sv create mode 100644 hw/unittest/stream_arb/VX_stream_arb_top.sv create mode 100644 hw/unittest/stream_arb/main.cpp diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 12619a17cd..2f3f33d952 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -38,6 +38,14 @@ tests: via: script run: "make -C hw/unittest/cp_llc_port run" touches: [hw/rtl/mem/VX_cp_llc_port.sv, hw/unittest/cp_llc_port] + # Stream arbiters over more than 1.5x MAX_FANOUT requests become trees of + # slices; a last slice narrower than the arbiter's other side must keep every + # request on its own output and select. The system shapes reach only a few of + # these slice shapes. + - id: hw-stream-arb + via: script + run: "make -C hw/unittest/stream_arb run" + touches: [hw/rtl/libs/VX_stream_arb.sv, hw/unittest/stream_arb] # Texel blend (VX_tex_lerp, both forms the sampler uses) against the C model # the SW sampler and SimX share, and the exact Vulkan f/256 blend, over every # input. diff --git a/hw/rtl/libs/VX_stream_arb.sv b/hw/rtl/libs/VX_stream_arb.sv index 9ef71a6cfd..64e4eba814 100644 --- a/hw/rtl/libs/VX_stream_arb.sv +++ b/hw/rtl/libs/VX_stream_arb.sv @@ -50,24 +50,29 @@ module VX_stream_arb #( localparam LOG_NUM_REQS2 = `CLOG2(MAX_FANOUT); localparam LOG_NUM_REQS3 = `CLOG2(NUM_SLICES); localparam DATAW2 = DATAW + LOG_NUM_REQS2; + localparam SLICE_STRIDE = MAX_FANOUT * NUM_OUTPUTS; + // a last slice with fewer inputs than outputs feeds only the outputs of its inputs + localparam LAST_OUTPUTS = `MIN(NUM_INPUTS - (NUM_SLICES - 1) * SLICE_STRIDE, NUM_OUTPUTS); + localparam JOIN_INPUTS = (NUM_SLICES - 1) * NUM_OUTPUTS + LAST_OUTPUTS; - wire [NUM_SLICES-1:0][NUM_OUTPUTS-1:0] valid_tmp; - wire [NUM_SLICES-1:0][NUM_OUTPUTS-1:0][DATAW2-1:0] data_tmp; - wire [NUM_SLICES-1:0][NUM_OUTPUTS-1:0] ready_tmp; + wire [JOIN_INPUTS-1:0] valid_tmp; + wire [JOIN_INPUTS-1:0][DATAW2-1:0] data_tmp; + wire [JOIN_INPUTS-1:0] ready_tmp; for (genvar s = 0; s < NUM_SLICES; ++s) begin : g_slice_arbs - localparam SLICE_STRIDE= MAX_FANOUT * NUM_OUTPUTS; localparam SLICE_BEGIN = s * SLICE_STRIDE; localparam SLICE_END = `MIN(SLICE_BEGIN + SLICE_STRIDE, NUM_INPUTS); localparam SLICE_SIZE = SLICE_END - SLICE_BEGIN; + localparam SLICE_OUTPUTS = `MIN(SLICE_SIZE, NUM_OUTPUTS); + localparam SLICE_SEL_W = `UP(`CLOG2(`CDIV(SLICE_SIZE, SLICE_OUTPUTS))); - wire [NUM_OUTPUTS-1:0][DATAW-1:0] data_tmp_u; - wire [NUM_OUTPUTS-1:0][LOG_NUM_REQS2-1:0] sel_tmp_u; + wire [SLICE_OUTPUTS-1:0][DATAW-1:0] data_tmp_u; + wire [SLICE_OUTPUTS-1:0][SLICE_SEL_W-1:0] sel_tmp_u; VX_stream_arb #( .NUM_INPUTS (SLICE_SIZE), - .NUM_OUTPUTS (NUM_OUTPUTS), + .NUM_OUTPUTS (SLICE_OUTPUTS), .DATAW (DATAW), .ARBITER (ARBITER), .STICKY (STICKY), @@ -79,14 +84,14 @@ module VX_stream_arb #( .valid_in (valid_in[SLICE_END-1: SLICE_BEGIN]), .data_in (data_in[SLICE_END-1: SLICE_BEGIN]), .ready_in (ready_in[SLICE_END-1: SLICE_BEGIN]), - .valid_out (valid_tmp[s]), + .valid_out (valid_tmp[s * NUM_OUTPUTS +: SLICE_OUTPUTS]), .data_out (data_tmp_u), - .ready_out (ready_tmp[s]), + .ready_out (ready_tmp[s * NUM_OUTPUTS +: SLICE_OUTPUTS]), .sel_out (sel_tmp_u) ); - for (genvar o = 0; o < NUM_OUTPUTS; ++o) begin : g_data_tmp - assign data_tmp[s][o] = {data_tmp_u[o], sel_tmp_u[o]}; + for (genvar o = 0; o < SLICE_OUTPUTS; ++o) begin : g_data_tmp + assign data_tmp[s * NUM_OUTPUTS + o] = {data_tmp_u[o], LOG_NUM_REQS2'(sel_tmp_u[o])}; end end @@ -94,7 +99,7 @@ module VX_stream_arb #( wire [NUM_OUTPUTS-1:0][LOG_NUM_REQS3-1:0] sel_out_u; VX_stream_arb #( - .NUM_INPUTS (NUM_SLICES * NUM_OUTPUTS), + .NUM_INPUTS (JOIN_INPUTS), .NUM_OUTPUTS (NUM_OUTPUTS), .DATAW (DATAW2), .ARBITER (ARBITER), @@ -210,15 +215,19 @@ module VX_stream_arb #( localparam NUM_SLICES = `CDIV(NUM_REQS, MAX_FANOUT); localparam LOG_NUM_REQS2 = `CLOG2(MAX_FANOUT); localparam LOG_NUM_REQS3 = `CLOG2(NUM_SLICES); - - wire [NUM_SLICES-1:0][NUM_INPUTS-1:0] valid_tmp; - wire [NUM_SLICES-1:0][NUM_INPUTS-1:0][DATAW-1:0] data_tmp; - wire [NUM_SLICES-1:0][NUM_INPUTS-1:0] ready_tmp; + localparam SLICE_STRIDE = MAX_FANOUT * NUM_INPUTS; + // a last slice with fewer outputs than inputs takes only the inputs of its outputs + localparam LAST_INPUTS = `MIN(NUM_OUTPUTS - (NUM_SLICES - 1) * SLICE_STRIDE, NUM_INPUTS); + localparam FORK_OUTPUTS = (NUM_SLICES - 1) * NUM_INPUTS + LAST_INPUTS; + + wire [FORK_OUTPUTS-1:0] valid_tmp; + wire [FORK_OUTPUTS-1:0][DATAW-1:0] data_tmp; + wire [FORK_OUTPUTS-1:0] ready_tmp; wire [NUM_INPUTS-1:0][LOG_NUM_REQS3-1:0] sel_tmp; VX_stream_arb #( .NUM_INPUTS (NUM_INPUTS), - .NUM_OUTPUTS (NUM_SLICES * NUM_INPUTS), + .NUM_OUTPUTS (FORK_OUTPUTS), .DATAW (DATAW), .ARBITER (ARBITER), .STICKY (STICKY), @@ -240,15 +249,16 @@ module VX_stream_arb #( for (genvar s = 0; s < NUM_SLICES; ++s) begin : g_slice_arbs - localparam SLICE_STRIDE= MAX_FANOUT * NUM_INPUTS; localparam SLICE_BEGIN = s * SLICE_STRIDE; localparam SLICE_END = `MIN(SLICE_BEGIN + SLICE_STRIDE, NUM_OUTPUTS); localparam SLICE_SIZE = SLICE_END - SLICE_BEGIN; + localparam SLICE_INPUTS= `MIN(SLICE_SIZE, NUM_INPUTS); + localparam SLICE_SEL_W = `UP(`CLOG2(`CDIV(SLICE_SIZE, SLICE_INPUTS))); - wire [NUM_INPUTS-1:0][LOG_NUM_REQS2-1:0] sel_out_u; + wire [SLICE_INPUTS-1:0][SLICE_SEL_W-1:0] sel_out_u; VX_stream_arb #( - .NUM_INPUTS (NUM_INPUTS), + .NUM_INPUTS (SLICE_INPUTS), .NUM_OUTPUTS (SLICE_SIZE), .DATAW (DATAW), .ARBITER (ARBITER), @@ -258,14 +268,23 @@ module VX_stream_arb #( ) fanout_slice_arb ( .clk (clk), .reset (reset), - .valid_in (valid_tmp[s]), - .ready_in (ready_tmp[s]), - .data_in (data_tmp[s]), + .valid_in (valid_tmp[s * NUM_INPUTS +: SLICE_INPUTS]), + .ready_in (ready_tmp[s * NUM_INPUTS +: SLICE_INPUTS]), + .data_in (data_tmp[s * NUM_INPUTS +: SLICE_INPUTS]), .data_out (data_out[SLICE_END-1: SLICE_BEGIN]), .valid_out (valid_out[SLICE_END-1: SLICE_BEGIN]), .ready_out (ready_out[SLICE_END-1: SLICE_BEGIN]), - .sel_out (sel_out_w[s]) + .sel_out (sel_out_u) ); + + // inputs past a narrow last slice are never forked to it + for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_sel_out_w + if (i < SLICE_INPUTS) begin : g_sel + assign sel_out_w[s][i] = LOG_NUM_REQS2'(sel_out_u[i]); + end else begin : g_pad + assign sel_out_w[s][i] = '0; + end + end end for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_sel_out @@ -284,7 +303,11 @@ module VX_stream_arb #( wire [NUM_INPUTS-1:0] requests; for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_i localparam o = r * NUM_INPUTS + i; - assign requests[i] = ready_out[o]; + if (o < NUM_OUTPUTS) begin : g_req_ready + assign requests[i] = ready_out[o]; + end else begin : g_req_pad + assign requests[i] = 1'b0; + end end assign arb_requests[r] = (| requests); end @@ -318,7 +341,11 @@ module VX_stream_arb #( wire [NUM_REQS-1:0] ready_out_s; for (genvar r = 0; r < NUM_REQS; ++r) begin : g_r localparam o = r * NUM_INPUTS + i; - assign ready_out_s[r] = ready_out_w[o]; + if (o < NUM_OUTPUTS) begin : g_ready + assign ready_out_s[r] = ready_out_w[o]; + end else begin : g_padding + assign ready_out_s[r] = 1'b0; + end end assign ready_in[i] = (NUM_INPUTS == 1) ? arb_valid : (| (ready_out_s & arb_onehot)); end diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index fb703fade9..87c8e63f85 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -22,6 +22,7 @@ all: $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma $(MAKE) -C cp_llc_port + $(MAKE) -C stream_arb $(MAKE) -C cp_core $(MAKE) -C fma_unit $(MAKE) -C fdiv_unit @@ -54,6 +55,7 @@ run: $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run $(MAKE) -C cp_llc_port run + $(MAKE) -C stream_arb run $(MAKE) -C cp_core run $(MAKE) -C fma_unit run $(MAKE) -C fdiv_unit run @@ -150,6 +152,7 @@ clean: $(MAKE) -C cp_axi_path clean $(MAKE) -C cp_dma clean $(MAKE) -C cp_llc_port clean + $(MAKE) -C stream_arb clean $(MAKE) -C cp_core clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean diff --git a/hw/unittest/stream_arb/Makefile b/hw/unittest/stream_arb/Makefile new file mode 100644 index 0000000000..ee2f11ccfd --- /dev/null +++ b/hw/unittest/stream_arb/Makefile @@ -0,0 +1,23 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := stream_arb + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := + +RTL_INCLUDE := -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs -I$(SRC_DIR) + +TOP := VX_stream_arb_top + +include ../common.mk diff --git a/hw/unittest/stream_arb/VX_stream_arb_tb.sv b/hw/unittest/stream_arb/VX_stream_arb_tb.sv new file mode 100644 index 0000000000..d94197bf9e --- /dev/null +++ b/hw/unittest/stream_arb/VX_stream_arb_tb.sv @@ -0,0 +1,179 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_platform.vh" + +// Self-checking harness around one VX_stream_arb: every input sends numbered +// packets under random valid and output backpressure, and every packet must +// leave once, on an output its input maps to, with the select of its row. +module VX_stream_arb_tb #( + parameter NUM_INPUTS = 1, + parameter NUM_OUTPUTS = 1, + parameter MAX_FANOUT = 8, + parameter SEED = 1 +) ( + input wire clk, + input wire reset, + input wire drain, + output wire idle, + output wire [31:0] delivered, + output wire [31:0] errors +); + localparam SRC_W = 8; + localparam SEQ_W = 16; + localparam DATAW = SRC_W + SEQ_W; + localparam NUM_REQS = (NUM_INPUTS > NUM_OUTPUTS) ? `CDIV(NUM_INPUTS, NUM_OUTPUTS) : `CDIV(NUM_OUTPUTS, NUM_INPUTS); + localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(NUM_REQS)); + + `STATIC_ASSERT (NUM_INPUTS < (1 << SRC_W), ("invalid parameter: NUM_INPUTS=%0d", NUM_INPUTS)) + + function automatic logic [31:0] hash32(input logic [31:0] x); + logic [31:0] h; + h = x ^ (x >> 16); + h = h * 32'h7feb352d; + h = h ^ (h >> 15); + h = h * 32'h846ca68b; + return h ^ (h >> 16); + endfunction + + reg [31:0] cycle; + + reg [NUM_INPUTS-1:0] valid_in; + reg [NUM_INPUTS-1:0][SEQ_W-1:0] seq_in; + wire [NUM_INPUTS-1:0][DATAW-1:0] data_in; + wire [NUM_INPUTS-1:0] ready_in; + + wire [NUM_OUTPUTS-1:0] valid_out; + wire [NUM_OUTPUTS-1:0][DATAW-1:0] data_out; + wire [NUM_OUTPUTS-1:0] ready_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] sel_out; + + for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_data_in + assign data_in[i] = {SRC_W'(i), seq_in[i]}; + end + + // outputs stall at random, with a longer stall now and then so the + // arbiter buffers fill up + for (genvar o = 0; o < NUM_OUTPUTS; ++o) begin : g_ready_out + wire [1:0] rnd = 2'(hash32(cycle * 32'd977 + 32'd7919 * 32'(o + 1024) + 32'(SEED))); + assign ready_out[o] = (rnd != 0) && (cycle[9:6] != 4'(SEED)); + end + + VX_stream_arb #( + .NUM_INPUTS (NUM_INPUTS), + .NUM_OUTPUTS (NUM_OUTPUTS), + .DATAW (DATAW), + .ARBITER ("R"), + .MAX_FANOUT (MAX_FANOUT), + .OUT_BUF (3) + ) dut ( + .clk (clk), + .reset (reset), + .valid_in (valid_in), + .data_in (data_in), + .ready_in (ready_in), + .valid_out (valid_out), + .data_out (data_out), + .ready_out (ready_out), + .sel_out (sel_out) + ); + + wire [NUM_OUTPUTS-1:0][SRC_W-1:0] src_out; + wire [NUM_OUTPUTS-1:0][SEQ_W-1:0] seq_out; + + for (genvar o = 0; o < NUM_OUTPUTS; ++o) begin : g_data_out + assign {src_out[o], seq_out[o]} = data_out[o]; + end + + reg [NUM_INPUTS-1:0][31:0] sent_cnt, recv_cnt; + reg [NUM_INPUTS-1:0][31:0] sent_sum, recv_sum; + reg [31:0] errors_r; + + logic [NUM_INPUTS-1:0][31:0] recv_inc, recv_sum_inc; + logic [31:0] errors_inc; + + always_comb begin + recv_inc = '0; + recv_sum_inc = '0; + errors_inc = '0; + for (integer o = 0; o < NUM_OUTPUTS; ++o) begin + if (valid_out[o] && ready_out[o]) begin + if (32'(src_out[o]) >= NUM_INPUTS) begin + errors_inc = errors_inc + 1; + end else begin + recv_inc[src_out[o]] = recv_inc[src_out[o]] + 1; + recv_sum_inc[src_out[o]] = recv_sum_inc[src_out[o]] + 32'(seq_out[o]); + if (NUM_INPUTS >= NUM_OUTPUTS) begin + // an input reaches one output, in order, with its row as select + if ((32'(src_out[o]) % NUM_OUTPUTS) != o + || seq_out[o] != recv_cnt[src_out[o]][SEQ_W-1:0] + || 32'(sel_out[o % SEL_COUNT]) != (32'(src_out[o]) / NUM_OUTPUTS)) begin + errors_inc = errors_inc + 1; + end + end else begin + // an input reaches any output of its column + if ((o % NUM_INPUTS) != 32'(src_out[o])) begin + errors_inc = errors_inc + 1; + end + end + end + end + end + end + + always @(posedge clk) begin + if (reset) begin + cycle <= '0; + valid_in <= '0; + seq_in <= '0; + sent_cnt <= '0; + sent_sum <= '0; + recv_cnt <= '0; + recv_sum <= '0; + errors_r <= '0; + end else begin + cycle <= cycle + 1; + for (integer i = 0; i < NUM_INPUTS; ++i) begin + logic [2:0] rnd; + rnd = 3'(hash32(cycle * 32'd977 + 32'd7919 * 32'(i) + 32'(SEED))); + if (valid_in[i] && ready_in[i]) begin + sent_cnt[i] <= sent_cnt[i] + 1; + sent_sum[i] <= sent_sum[i] + 32'(seq_in[i]); + seq_in[i] <= seq_in[i] + 1; + valid_in[i] <= ~drain && (rnd != 0); + end else if (~valid_in[i]) begin + valid_in[i] <= ~drain && (rnd != 0); + end + end + for (integer i = 0; i < NUM_INPUTS; ++i) begin + recv_cnt[i] <= recv_cnt[i] + recv_inc[i]; + recv_sum[i] <= recv_sum[i] + recv_sum_inc[i]; + end + errors_r <= errors_r + errors_inc; + if (errors_inc != 0) begin + $display("cycle %0d: stream_arb %0dx%0d (fanout %0d): misrouted packet", cycle, NUM_INPUTS, NUM_OUTPUTS, MAX_FANOUT); + end + end + end + + logic [31:0] delivered_w; + logic idle_w; + + always_comb begin + delivered_w = '0; + idle_w = (valid_in == '0); + for (integer i = 0; i < NUM_INPUTS; ++i) begin + delivered_w = delivered_w + recv_cnt[i]; + // every input must have been served, and nothing lost + idle_w = idle_w && (recv_cnt[i] != 0) + && (recv_cnt[i] == sent_cnt[i]) + && (recv_sum[i] == sent_sum[i]); + end + end + + assign idle = idle_w; + assign delivered = delivered_w; + assign errors = errors_r; + +endmodule diff --git a/hw/unittest/stream_arb/VX_stream_arb_top.sv b/hw/unittest/stream_arb/VX_stream_arb_top.sv new file mode 100644 index 0000000000..7698928767 --- /dev/null +++ b/hw/unittest/stream_arb/VX_stream_arb_top.sv @@ -0,0 +1,63 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_platform.vh" + +// VX_stream_arb shapes under test, each in its own self-checking harness. +// Over 1.5x MAX_FANOUT requests an arbiter becomes a tree of slices; the +// shapes cover full slices, a last slice of one input or output, one smaller +// than the other side, one not a whole number of rows, and a join that is a +// tree itself. +module VX_stream_arb_top #( + parameter NUM_CFGS = 21 +) ( + input wire clk, + input wire reset, + input wire drain, + output wire idle, + output wire [NUM_CFGS-1:0] busy, + output wire [31:0] delivered, + output wire [31:0] errors +); + // input select | output select | flat + localparam int CFG_NI [NUM_CFGS] = '{64, 17, 65, 13, 19, 34, 36, 40, 26, 27, 129, 200, 1, 4, 2, 1, 4, 5, 8, 3, 1}; + localparam int CFG_NO [NUM_CFGS] = '{ 1, 1, 1, 1, 1, 4, 4, 4, 2, 2, 1, 1, 17, 34, 27, 64, 32, 2, 8, 5, 1}; + localparam int CFG_MF [NUM_CFGS] = '{ 8, 8, 8, 8, 8, 4, 4, 4, 4, 4, 8, 4, 8, 4, 4, 8, 8, 8, 8, 8, 8}; + + wire [NUM_CFGS-1:0] idle_w; + wire [NUM_CFGS-1:0][31:0] delivered_w; + wire [NUM_CFGS-1:0][31:0] errors_w; + + for (genvar c = 0; c < NUM_CFGS; ++c) begin : g_cfgs + VX_stream_arb_tb #( + .NUM_INPUTS (CFG_NI[c]), + .NUM_OUTPUTS (CFG_NO[c]), + .MAX_FANOUT (CFG_MF[c]), + .SEED (c + 1) + ) tb ( + .clk (clk), + .reset (reset), + .drain (drain), + .idle (idle_w[c]), + .delivered (delivered_w[c]), + .errors (errors_w[c]) + ); + end + + logic [31:0] delivered_s, errors_s; + + always_comb begin + delivered_s = '0; + errors_s = '0; + for (integer c = 0; c < NUM_CFGS; ++c) begin + delivered_s = delivered_s + delivered_w[c]; + errors_s = errors_s + errors_w[c]; + end + end + + assign idle = (& idle_w); + assign busy = ~idle_w; + assign delivered = delivered_s; + assign errors = errors_s; + +endmodule diff --git a/hw/unittest/stream_arb/main.cpp b/hw/unittest/stream_arb/main.cpp new file mode 100644 index 0000000000..740795a1ca --- /dev/null +++ b/hw/unittest/stream_arb/main.cpp @@ -0,0 +1,62 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +// Verilator unit test for VX_stream_arb over the shapes in VX_stream_arb_top. +// Each shape's harness checks its own traffic: random input valids and output +// stalls for a while, then the inputs stop and every packet must have left +// exactly once, on the right output, with the right select. + +#include "vl_simulator.h" +#include "VVX_stream_arb_top.h" +#include +#include + +#ifndef TRACE_START_TIME +#define TRACE_START_TIME 0ull +#endif +#ifndef TRACE_STOP_TIME +#define TRACE_STOP_TIME -1ull +#endif + +static uint64_t timestamp = 0; +static bool trace_en = false; + +double sc_time_stamp() { return timestamp; } +bool sim_trace_enabled() { return trace_en; } +void sim_trace_enable(bool e) { trace_en = e; } + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + vl_simulator sim; + uint64_t tick = 0; + + constexpr uint32_t kTrafficCycles = 20000; + constexpr uint32_t kDrainCycles = 20000; + + sim->drain = 0; + sim->reset = 1; + tick = sim.step(tick, 8); + sim->reset = 0; + + tick = sim.step(tick, 2 * kTrafficCycles); + sim->drain = 1; + uint32_t cycles = 0; + while (!sim->idle && sim->errors == 0 && cycles < kDrainCycles) { + tick = sim.step(tick, 2); + ++cycles; + } + timestamp = tick; + + std::printf("delivered=%u, errors=%u, idle=%d\n", sim->delivered, sim->errors, sim->idle); + for (uint32_t c = 0; c < 8 * sizeof(sim->busy); ++c) { + if ((sim->busy >> c) & 1) { + std::printf("shape %u did not drain\n", c); + } + } + if (sim->errors != 0 || !sim->idle) { + std::printf("FAILED!\n"); + return 1; + } + std::printf("PASSED!\n"); + return 0; +} From f321887e6f48c72c7635eeed7d7dfd11df044730 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 07:47:54 -0700 Subject: [PATCH 072/336] simx: let a cache's port arbiter take more than 64 ports The cache's core-port arbiter kept its requesting ports, read-merge followers and flush tokens in 64-bit masks. With VX_CFG_CP_LLC_CLIENT, a shared L2 with 64 L1 ports plus the command processor's DMA slot has 65 ports. On x86 the shift for port 64 wraps around to port 0, so port 0 was granted while its input channel was empty, and SimX aborted with "channel is empty" (simobject.h peek). Release builds compile out the assert(num_inputs <= 64) that should have caught this. Requests are now collected per bank and per first-level slice of the arbiter tree; one slice is at most 12 ports wide. The flush tokens are kept per port. Read-merge masks are taken relative to the merge group. The tree now follows the RTL's recursion. When a level has more than 1.5x the fan-out limit of slice buffers, those buffers are sliced again behind another buffered level before the join. Each slice, including a partial last one, round-robins over its own inputs into a 2-entry buffer. Configurations of up to 96 ports keep the single level they had, and their cycle counts are unchanged. The read-merge response network keeps its 64-port, one-level limit, now as an always-on check. SimX cycles, c96140253 -> this change (sgemm -n32 / raycast): default 115,067 / 1,494,492 unchanged 4 cores + L2 30,086 / 478,511 unchanged 4c, 16 L1 ports (64-port L2) 29,996 / 478,186 unchanged 8c IW4, CP_LLC_CLIENT (65) abort -> PASS rtx5080 4c x 32t, CP (65) abort -> PASS (sgemm -n32 10,009) CI adds model_parity-sgemm-cp-llc-17 (2 cores x 8 L1 ports + DMA = 17 L2 ports, a one-port last slice) and a SimX sgemm case at 65 ports. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 22 ++++ sim/simx/mem/cache.cpp | 238 +++++++++++++++++++++-------------- 2 files changed, 169 insertions(+), 91 deletions(-) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index a6042df1ef..e6475f33c4 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -366,6 +366,28 @@ tests: shape: cores: 4 l2cache: true +# Eight L1 ports per core and the DMA's slot give the shared L2 2^k + 1 +# ports, so each bank's port arbiter is a tree whose last slice has one port: +# 17 ports on two cores, and 65 on eight, past a 64-bit port mask. +- id: model_parity-sgemm-cp-llc-17 + check: model_parity + via: blackbox + app: sgemm + args: -n32 + configs: -DVX_CFG_CP_LLC_CLIENT=1 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 + shape: + cores: 2 + l2cache: true +- id: sgemm-cp-llc-65 + via: blackbox + drivers: + - simx + app: sgemm + args: -n32 + configs: -DVX_CFG_CP_LLC_CLIENT=1 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 + shape: + cores: 8 + l2cache: true # The shared L2 keeps its lines across the L1-only launch flushes. - id: model_parity-flush-chain check: model_parity diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 14fee5673c..67a8e88131 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -34,15 +34,23 @@ namespace { constexpr uint32_t kMaxFanout = 8; // An arbiter over more than 1.5x the fan-out limit is split into a buffered -// tree, one cycle deeper. -uint32_t arb_tree_delay(uint32_t requesters) { - return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; +// tree: slices of up to the fan-out limit each grant into their own buffer, +// and those buffers are arbitrated the same way, one level per cycle, until +// few enough remain for a single join arbiter. Returns each buffered level's +// slice count, none for a single arbiter. +std::vector arb_tree_levels(uint32_t requesters) { + std::vector levels; + for (uint32_t n = requesters; n > kMaxFanout + kMaxFanout / 2;) { + n = (n + kMaxFanout - 1) / kMaxFanout; + levels.push_back(n); + } + return levels; } -// Slices of the tree's first level: each arbitrates up to the fan-out limit -// into its own buffer, and a join arbiter picks among the slice buffers. +// Slices of the tree's first level, 1 for a single arbiter. uint32_t arb_tree_slices(uint32_t requesters) { - return (arb_tree_delay(requesters) != 0) ? (requesters + kMaxFanout - 1) / kMaxFanout : 1; + auto levels = arb_tree_levels(requesters); + return levels.empty() ? 1 : levels.front(); } // Registered stages of a radix-2 omega network between a and b ports; up to @@ -2263,15 +2271,19 @@ class CachePortArb : public SimObject { , lg2_inputs_(log2ceil(num_inputs)) // the request crossbar registers its output when it has more than two inputs , delay_((num_inputs > 2) ? 1 : 0) - , slices_(arb_tree_slices(num_inputs)) - , rr_(num_banks * slices_, num_inputs - 1) - , join_rr_(num_banks, slices_ - 1) - , tree_stage_((slices_ > 1) ? num_banks * slices_ : 0, this) + , levels_(arb_tree_levels(num_inputs)) , merge_group_(merge_group) { - assert(num_inputs <= 64); - __assert(merge_group_ <= 1 || (ispow2(merge_group_) && (num_inputs % merge_group_) == 0), - "a read-merge group must be a power of two dividing the ports"); + __assert(merge_group_ <= 1 || (merge_group_ <= 64 && ispow2(merge_group_) && (num_inputs % merge_group_) == 0), + "a read-merge group must be a power of two of at most 64 dividing the ports"); + tree_stage_.reserve(levels_.size()); + for (auto slices : levels_) { + tree_stage_.emplace_back(num_banks * slices, this); + } + rr_.resize(levels_.size() + 1); + for (uint32_t s = 0; s <= levels_.size(); ++s) { + rr_.at(s).resize(num_banks * this->arb_slices(s)); + } } // Bits of the merged-port mask carried below the request tag. @@ -2279,39 +2291,51 @@ class CachePortArb : public SimObject { uint64_t collisions() const { return collisions_; } - // Ports whose head is a flush token. While any is visible, cacheable - // requests are held at every port. - uint64_t tokens() const { - uint64_t mask = 0; + bool is_token(uint32_t i) const { + return !ReqIn.at(i).empty() && ReqIn.at(i).peek().flags.flush; + } + + // While any port's head is a flush token, cacheable requests are held at + // every port. + bool any_token() const { for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { - if (!ReqIn.at(i).empty() && ReqIn.at(i).peek().flags.flush) { - mask |= 1ull << i; + if (this->is_token(i)) { + return true; } } + return false; + } + + // Ports whose head is a flush token. + std::vector tokens() const { + std::vector mask(ReqIn.size(), false); + for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { + mask.at(i) = this->is_token(i); + } return mask; } // Tokens owing no walk: IO tokens, and L1-only ones unless they still need // a walk here. - uint64_t tokens(bool l1_needs_walk) const { - uint64_t mask = 0; + std::vector tokens(bool l1_needs_walk) const { + std::vector mask(ReqIn.size(), false); for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { - if (ReqIn.at(i).empty()) { + if (!this->is_token(i)) { continue; } auto &flags = ReqIn.at(i).peek().flags; - if (flags.flush && (flags.io || (flags.flush_l1 && !l1_needs_walk))) { - mask |= 1ull << i; - } + mask.at(i) = flags.io || (flags.flush_l1 && !l1_needs_walk); } return mask; } // No cacheable request is left between the ports and the banks. bool drained() const { - for (auto &stage : tree_stage_) { - if (stage.size() != 0) { - return false; + for (auto &level : tree_stage_) { + for (auto &stage : level) { + if (stage.size() != 0) { + return false; + } } } for (auto &out : ReqOut) { @@ -2333,36 +2357,65 @@ class CachePortArb : public SimObject { protected: void on_reset() { - std::fill(rr_.begin(), rr_.end(), ReqIn.size() - 1); - std::fill(join_rr_.begin(), join_rr_.end(), slices_ - 1); + for (auto &rr : rr_) { + std::fill(rr.begin(), rr.end(), 0); + } collisions_ = 0; } + // Slices of the arbiter level s: level 0 grants the ports, each level above + // the buffers of the level below, and the last level is the join arbiter. + uint32_t arb_slices(uint32_t s) const { + return (s < levels_.size()) ? levels_.at(s) : 1; + } + + // Inputs of slice sl of the arbiter level s. + uint32_t arb_slice_size(uint32_t s, uint32_t sl) const { + uint32_t inputs = (s == 0) ? ReqIn.size() : levels_.at(s - 1); + if (s == levels_.size()) { + return inputs; + } + return std::min(kMaxFanout, inputs - sl * kMaxFanout); + } + void on_tick() { if (init_cycles_ != 0) { --init_cycles_; return; } - // the arbiter tree's join level: one slice buffer per bank per cycle - for (uint32_t b = 0, nb = (slices_ > 1) ? ReqOut.size() : 0; b < nb; ++b) { - if (ReqOut.at(b).full()) { - continue; - } - for (uint32_t k = 1; k <= slices_; ++k) { - uint32_t sl = (join_rr_[b] + k) % slices_; - auto& stage = tree_stage_.at(b * slices_ + sl); - if (!stage.empty()) { - ReqOut.at(b).send(stage.peek(), delay_); - stage.pop(); - join_rr_[b] = sl; - break; + uint32_t nb = ReqOut.size(); + uint32_t nl = levels_.size(); + // the tree's buffered levels, join first: each slice passes one buffer of + // the level below per cycle + for (uint32_t s = nl; s > 0; --s) { + uint32_t slices = this->arb_slices(s); + uint32_t inputs = levels_.at(s - 1); + for (uint32_t b = 0; b < nb; ++b) { + for (uint32_t sl = 0; sl < slices; ++sl) { + auto& out = (s < nl) ? tree_stage_.at(s).at(b * slices + sl) : ReqOut.at(b); + if (out.full()) { + continue; + } + uint32_t lo = (s < nl) ? sl * kMaxFanout : 0; + uint32_t size = this->arb_slice_size(s, sl); + auto& rr = rr_.at(s).at(b * slices + sl); + for (uint32_t k = 0; k < size; ++k) { + uint32_t j = (rr + k) % size; + auto& in = tree_stage_.at(s - 1).at(b * inputs + lo + j); + if (!in.empty()) { + out.send(in.peek(), (s < nl) ? 1 : delay_); + in.pop(); + rr = (j + 1) % size; + break; + } + } } } } uint32_t n = ReqIn.size(); - const bool hold = (this->tokens() != 0); - // merged ports riding each leader's bank access - uint64_t followers = 0; + const bool hold = this->any_token(); + // merged ports riding each leader's bank access, by offset in its group + std::vector followers(n, false); std::vector merged(n, 0); if (merge_group_ > 1 && !hold) { for (uint32_t i = 0; i < n; ++i) { @@ -2370,25 +2423,30 @@ class CachePortArb : public SimObject { continue; } auto& req_i = ReqIn.at(i).peek(); - for (uint32_t j = (i / merge_group_) * merge_group_; j < i; ++j) { - if ((followers & (1ull << j)) || !this->is_plain_read(j)) { + uint32_t base = (i / merge_group_) * merge_group_; + for (uint32_t j = base; j < i; ++j) { + if (followers.at(j) || !this->is_plain_read(j)) { continue; } auto& req_j = ReqIn.at(j).peek(); if ((req_j.addr >> params_.word_select_addr_start) == (req_i.addr >> params_.word_select_addr_start) && req_j.tag == req_i.tag) { - merged.at(j) |= 1ull << i; - followers |= 1ull << i; + merged.at(j) |= 1ull << (i - base); + followers.at(i) = true; break; } } } } - // requesting ports per bank; non-cacheable requests leave on their own port - std::vector want(ReqOut.size(), 0); + // requesting ports per bank and first-level slice; non-cacheable requests + // leave on their own port + uint32_t slices = this->arb_slices(0); + uint32_t slice_ports = (nl != 0) ? kMaxFanout : n; + std::vector want(nb * slices, 0); + std::vector wants(nb, 0); for (uint32_t i = 0; i < n; ++i) { auto& in = ReqIn.at(i); - if (in.empty() || (followers & (1ull << i))) { + if (in.empty() || followers.at(i)) { continue; } auto& req = in.peek(); @@ -2402,38 +2460,35 @@ class CachePortArb : public SimObject { continue; } if (!hold) { - want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; + uint32_t b = params_.addr_bank_id(req.addr); + want.at(b * slices + i / slice_ports) |= 1u << (i % slice_ports); + ++wants.at(b); } } // each bank's arbiter, or each slice of its tree, grants one port per cycle - for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { - if (want[b] == 0) { + for (uint32_t b = 0; b < nb; ++b) { + if (wants.at(b) == 0) { continue; } bool granted = false; - for (uint32_t sl = 0; sl < slices_; ++sl) { - uint64_t slice_want = want[b]; - if (slices_ > 1) { - uint32_t lo = sl * kMaxFanout; - uint32_t hi = std::min(lo + kMaxFanout, n); - uint64_t range = ((hi < 64) ? (1ull << hi) : 0) - (1ull << lo); - slice_want &= range; - } - auto& out = (slices_ > 1) ? tree_stage_.at(b * slices_ + sl) : ReqOut.at(b); + for (uint32_t sl = 0; sl < slices; ++sl) { + uint32_t slice_want = want.at(b * slices + sl); + auto& out = (nl != 0) ? tree_stage_.at(0).at(b * slices + sl) : ReqOut.at(b); if (slice_want == 0 || out.full()) { continue; } - auto& rr = rr_[b * slices_ + sl]; - uint32_t i = rr; - do { - i = (i + 1) % n; - } while (!(slice_want & (1ull << i))); - this->grant(i, out, merged); - rr = i; + uint32_t size = this->arb_slice_size(0, sl); + auto& rr = rr_.at(0).at(b * slices + sl); + uint32_t j = rr; + while (!(slice_want & (1u << j))) { + j = (j + 1) % size; + } + this->grant(sl * slice_ports + j, out, merged); + rr = (j + 1) % size; granted = true; } if (granted) { - collisions_ += __builtin_popcountll(want[b]) - 1; + collisions_ += wants.at(b) - 1; } } } @@ -2443,16 +2498,18 @@ class CachePortArb : public SimObject { auto req = ReqIn.at(i).peek(); uint64_t mask = 0; if (merge_group_ > 1) { - mask = merged.at(i) >> ((i / merge_group_) * merge_group_); - for (uint64_t f = merged.at(i); f != 0; f &= f - 1) { - DT(3, this->name() << " read-merge: port=" << __builtin_ctzll(f) << ", leader=" << i << ", " << req); - ReqIn.at(__builtin_ctzll(f)).pop(); + uint32_t base = (i / merge_group_) * merge_group_; + mask = merged.at(i); + for (uint64_t f = mask; f != 0; f &= f - 1) { + uint32_t port = base + __builtin_ctzll(f); + DT(3, this->name() << " read-merge: port=" << port << ", leader=" << i << ", " << req); + ReqIn.at(port).pop(); } } uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; __assert((tag >> 32) == 0, "cache bank tag overflow"); req.tag = uint32_t(tag); - out.send(req, tree_stage_.empty() ? delay_ : 1); + out.send(req, levels_.empty() ? delay_ : 1); ReqIn.at(i).pop(); } @@ -2470,10 +2527,9 @@ class CachePortArb : public SimObject { uint32_t lg2_inputs_; uint32_t delay_; uint32_t init_cycles_ = 0; - uint32_t slices_; - std::vector rr_; // [bank][slice] - std::vector join_rr_; // [bank] - std::vector> tree_stage_; // [bank][slice] + std::vector levels_; // [level] slices per bank + std::vector>> tree_stage_; // [level][bank][slice] + std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; uint64_t collisions_ = 0; @@ -2506,8 +2562,8 @@ class CacheRspMcast : public SimObject { , join_rr_(num_outputs, slices_ - 1) , tree_stage_((slices_ > 1) ? num_outputs * slices_ : 0, this) { - assert(num_outputs <= 64); - assert(merge_group < 64); + __assert(num_outputs <= 64 && merge_group < 64, "the read-merge response network takes at most 64 ports"); + __assert(arb_tree_levels(num_banks).size() <= 1, "the read-merge response network takes one tree level"); } protected: @@ -2846,7 +2902,7 @@ class Cache::Impl { void flush_step() { switch (flush_state_) { case FlushState::Idle: - if (port_arb_->tokens() != 0) { + if (port_arb_->any_token()) { flush_state_ = FlushState::Drain; } break; @@ -2937,17 +2993,18 @@ class Cache::Impl { break; } case FlushState::Respond: { - if (rsp_pending_ == 0) { + auto next = std::find(rsp_pending_.begin(), rsp_pending_.end(), true); + if (next == rsp_pending_.end()) { flush_state_ = FlushState::Idle; break; } - uint32_t i = __builtin_ctzll(rsp_pending_); + uint32_t i = std::distance(rsp_pending_.begin(), next); auto &token = port_arb_->ReqIn.at(i).peek(); MemRsp rsp{token.tag, token.hart_id, token.uuid}; if (simobject_->core_rsp_out.at(i).try_send(rsp, core_rsp_delay_)) { DT(3, simobject_->name() << " flush-rsp: " << rsp); port_arb_->take_token(i); - rsp_pending_ &= ~(1ull << i); + rsp_pending_.at(i) = false; } break; } @@ -2966,9 +3023,8 @@ class Cache::Impl { // weakest one that serves them all. token_kinds_t token_kinds() const { token_kinds_t kinds; - uint64_t tokens = port_arb_->tokens(); for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { - if (!(tokens & (1ull << i))) { + if (!port_arb_->is_token(i)) { continue; } auto &flags = port_arb_->ReqIn.at(i).peek().flags; @@ -3069,7 +3125,7 @@ class Cache::Impl { uint32_t core_rsp_delay_ = 0; enum class FlushState { Idle, Drain, Walk, Forward, Respond }; FlushState flush_state_ = FlushState::Idle; - uint64_t rsp_pending_ = 0; + std::vector rsp_pending_; uint64_t fwd_sent_ = 0; uint64_t fwd_done_ = 0; bool fwd_go_ = false; // nothing is left ahead of the forwarded tokens From 9681b608b7c18c42a6d88708bc00cd23218ce8c6 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 08:10:06 -0700 Subject: [PATCH 073/336] Keep each uncached request on its own memory port when a cache has fewer core ports A cache with more memory ports than core ports does not build. One example is VX_CFG_DCACHE_NUM_BANKS=16 at 4 threads: 2 dcache requests and 16 memory ports. Two separate faults cause it. First, VX_cache_bypass sends uncached core requests to the memory ports through a VX_mem_bus_arb. With fewer inputs than outputs, that is a VX_stream_arb that picks a ready output, so its valid depends on ready_out. With REQ_OUT_BUF(0) this valid feeds mem_bus_out_arb, which grants on valid and returns ready. That is a real combinational loop: valid -> grant -> ready -> grant -> valid (Verilator UNOPTFLAT). Beside a cache, each uncached core request i now keeps memory port i, as SimX has always modelled the bypass (req_id % mem_ports). The memory ports past the last core port carry cached traffic only. Without a cache, the memory side is a pass-through, there is no loop, and the bypass still spreads requests over the ports as before. Second, VX_mem_bus_arb and VX_lsu_mem_arb sized the arbiter's select from ARB_SEL_BITS(I, O), which is 0 when I < O. VX_stream_arb's output-side select is CLOG2(CDIV(O, I)) bits per input. Once O > 2I the widths differ (WIDTHEXPAND), and VX_lsu_mem_arb also declared it per output instead of per input. Both now take the arbiter's own select shape, as VX_tex_bus_arb already does. Both faults are already present at the merge-base with origin/master (7fe193695). Configurations that build today are unchanged. When there are at least as many core ports as memory ports, or no cache, the bypass elaborates exactly as before. The select width equals UP(ARB_SEL_BITS) whenever the old code was lint-clean. Yosys proves the new VX_mem_bus_arb and VX_cache_bypass sequentially equivalent to the old ones for every parameter set where the old ones are lint-clean: 388 shapes (VX_mem_bus_arb 1-12 x 1-12 inputs and outputs at buffer depths 0 and 2; VX_cache_bypass with 1-8 core ports, 1-16 memory ports, with and without a cache, at 4- and 16-byte lines). The old modules lint clean on 234 of them. A mutant fails the same check. None of the synthesis-gate DUTs elaborates a bypass or bus arbiter with fewer inputs than outputs. With DCACHE_NUM_BANKS=16 and 16 memory banks at 4 threads, the rtlsim build is now lint clean. sgemm and raycast pass on rtlsim and SimX: sgemm -n32 115,253 rtlsim / 115,177 SimX (-0.07%), raycast 1,501,399 / 1,494,417 (-0.47%). CI adds model_parity-sgemm-dcache-16-banks. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 8 ++++++++ hw/rtl/cache/VX_cache_bypass.sv | 13 ++++++++++--- hw/rtl/mem/VX_lsu_mem_arb.sv | 4 +++- hw/rtl/mem/VX_mem_bus_arb.sv | 3 ++- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index e6475f33c4..d2d6c695d6 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -388,6 +388,14 @@ tests: shape: cores: 8 l2cache: true +# More dcache banks and memory ports than core requests: the uncached path keeps +# each core request on its own memory port. +- id: model_parity-sgemm-dcache-16-banks + check: model_parity + via: blackbox + app: sgemm + args: -n32 + configs: -DVX_CFG_DCACHE_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 # The shared L2 keeps its lines across the L1-only launch flushes. - id: model_parity-flush-chain check: model_parity diff --git a/hw/rtl/cache/VX_cache_bypass.sv b/hw/rtl/cache/VX_cache_bypass.sv index 950f434392..f3e6d36ac4 100644 --- a/hw/rtl/cache/VX_cache_bypass.sv +++ b/hw/rtl/cache/VX_cache_bypass.sv @@ -53,6 +53,10 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( localparam MEM_TAG_NC1_WIDTH = UUID_WIDTH + MEM_TAG_ID_WIDTH; localparam MEM_TAG_NC2_WIDTH = MEM_TAG_NC1_WIDTH + WSEL_BITS; localparam MEM_TAG_OUT_WIDTH = CACHE_ENABLE ? `MAX(MEM_TAG_IN_WIDTH, MEM_TAG_NC2_WIDTH) : MEM_TAG_NC2_WIDTH; + // Beside a cache, the memory-side arbiter grants on valid, so spreading + // uncached requests over more memory ports by their ready would close a + // valid-ready loop: with fewer core ports, core port i keeps memory port i. + localparam NC_PORTS = CACHE_ENABLE ? `MIN(NUM_REQS, MEM_PORTS) : MEM_PORTS; `STATIC_ASSERT(0 == (`VX_MEM_IO_BASE_ADDR % `VX_CFG_MEM_BLOCK_SIZE), ("invalid parameter")) VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), @@ -108,10 +112,10 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (MEM_TAG_NC1_WIDTH) - ) core_bus_nc_arb_if[MEM_PORTS](); + ) core_bus_nc_arb_if[NC_PORTS](); VX_mem_bus_arb #( .NUM_INPUTS (NUM_REQS), - .NUM_OUTPUTS(MEM_PORTS), + .NUM_OUTPUTS(NC_PORTS), .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (CORE_TAG_WIDTH), .TAG_SEL_IDX(TAG_SEL_IDX), @@ -128,7 +132,7 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( .DATA_SIZE (LINE_SIZE), .TAG_WIDTH (MEM_TAG_NC2_WIDTH) ) mem_bus_out_nc_if[MEM_PORTS](); - for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_out_nc + for (genvar i = 0; i < NC_PORTS; ++i) begin : g_mem_bus_out_nc wire core_req_nc_arb_rw; wire [WORD_SIZE-1:0] core_req_nc_arb_byteen; wire [CORE_ADDR_WIDTH-1:0] core_req_nc_arb_addr; @@ -203,6 +207,9 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( }; assign mem_bus_out_nc_if[i].rsp_ready = core_bus_nc_arb_if[i].rsp_ready; end + for (genvar i = NC_PORTS; i < MEM_PORTS; ++i) begin : g_mem_bus_out_nc_idle + `INIT_VX_MEM_BUS_IF (mem_bus_out_nc_if[i]) + end VX_mem_bus_if #( .DATA_SIZE (LINE_SIZE), .TAG_WIDTH (MEM_TAG_OUT_WIDTH) diff --git a/hw/rtl/mem/VX_lsu_mem_arb.sv b/hw/rtl/mem/VX_lsu_mem_arb.sv index 29aa903045..90df8a233a 100644 --- a/hw/rtl/mem/VX_lsu_mem_arb.sv +++ b/hw/rtl/mem/VX_lsu_mem_arb.sv @@ -37,6 +37,8 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( ); localparam DATA_WIDTH = (8 * DATA_SIZE); localparam LOG_NUM_REQS = `ARB_SEL_BITS(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(`CDIV(`MAX(NUM_INPUTS, NUM_OUTPUTS), SEL_COUNT))); localparam REQ_DATAW = 1 + NUM_LANES * (1 + ADDR_WIDTH + DATA_WIDTH + DATA_SIZE + `UP(USER_WIDTH)) + TAG_WIDTH; localparam RSP_DATAW = NUM_LANES * (1 + DATA_WIDTH) + TAG_WIDTH; @@ -45,7 +47,7 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( wire [NUM_OUTPUTS-1:0] req_valid_out; wire [NUM_OUTPUTS-1:0][REQ_DATAW-1:0] req_data_out; wire [NUM_OUTPUTS-1:0] req_ready_out; - wire [NUM_OUTPUTS-1:0][`UP(LOG_NUM_REQS)-1:0] req_sel_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] req_sel_out; wire [NUM_INPUTS-1:0] req_valid_in; wire [NUM_INPUTS-1:0][REQ_DATAW-1:0] req_data_in; diff --git a/hw/rtl/mem/VX_mem_bus_arb.sv b/hw/rtl/mem/VX_mem_bus_arb.sv index 3b9fc8198f..70f16613d7 100644 --- a/hw/rtl/mem/VX_mem_bus_arb.sv +++ b/hw/rtl/mem/VX_mem_bus_arb.sv @@ -37,6 +37,7 @@ module VX_mem_bus_arb import VX_gpu_pkg::*; #( localparam REQ_DATAW = 1 + ADDR_WIDTH + DATA_WIDTH + DATA_SIZE + ATTR_WIDTH + TAG_WIDTH; localparam RSP_DATAW = DATA_WIDTH + TAG_WIDTH; localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(`CDIV(`MAX(NUM_INPUTS, NUM_OUTPUTS), SEL_COUNT))); wire [NUM_INPUTS-1:0] req_valid_in; wire [NUM_INPUTS-1:0][REQ_DATAW-1:0] req_data_in; @@ -44,7 +45,7 @@ module VX_mem_bus_arb import VX_gpu_pkg::*; #( wire [NUM_OUTPUTS-1:0] req_valid_out; wire [NUM_OUTPUTS-1:0][REQ_DATAW-1:0] req_data_out; - wire [SEL_COUNT-1:0][`UP(LOG_NUM_REQS)-1:0] req_sel_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] req_sel_out; wire [NUM_OUTPUTS-1:0] req_ready_out; for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_req_data_in From 18cccfdd5ce2dc6a17c9968688d564cf4327c6db Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:37:58 -0700 Subject: [PATCH 074/336] lsu: select the stack-interleave high bits instead of adding the correction The AGU's last step, addr_hi = lin_hi - off_borrow + addr_carry, was a carry chain waiting on the two group-field corrections, which settle last. Their difference is -1, 0 or +1, so lin_hi - 1 and lin_hi + 1 are now formed alongside them and the corrections only select. The result is exactly the same for every input. This path (lane dispatch -> AGU -> memory scheduler request queue) limits the core DUT. On pblocks: core 276 -> 290.3 MHz, all 49 perf_gate rtlsim cases cycle-identical. rtl-port's core DUT met 285.8 MHz before this change. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_lsu_agu.sv | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hw/rtl/core/VX_lsu_agu.sv b/hw/rtl/core/VX_lsu_agu.sv index c9dde01271..960e9c1b6e 100644 --- a/hw/rtl/core/VX_lsu_agu.sv +++ b/hw/rtl/core/VX_lsu_agu.sv @@ -155,9 +155,12 @@ module VX_lsu_agu import VX_gpu_pkg::*; ( assign addr_carry = sum[CORR_BITS]; end - // The window base cancels on the high bits; only the disagreement - // between the two group-field corrections survives. - wire [HI_BITS-1:0] addr_hi = lin_hi - HI_BITS'(off_borrow) + HI_BITS'(addr_carry); + // Only the two group-field corrections survive on the high bits. They + // settle last and differ by at most one, so they select a precomputed neighbour. + wire [HI_BITS-1:0] lin_hi_dec = lin_hi - HI_BITS'(1); + wire [HI_BITS-1:0] lin_hi_inc = lin_hi + HI_BITS'(1); + wire [HI_BITS-1:0] addr_hi = (off_borrow == addr_carry) ? lin_hi + : (off_borrow ? lin_hi_dec : lin_hi_inc); assign addr = in_stack ? {addr_hi, addr_lo} : lin_addr; end else begin : g_stack_linear From 5a3e84facf639ab11575db8885545d365f3c8a98 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:37:59 -0700 Subject: [PATCH 075/336] core: decide the warp-control unit's ready from the opcode VX_wctl_unit's execute_if.ready depended on whether a tmc/pred exits, which reduces the operand-selected thread mask to zero. That put a deep cone from the SFU dispatch buffer's output register back to its own enable; on pblocks it dropped the core DUT from 288.5 to 250 MHz. Ready now uses opcode-only terms. Every tmc/pred at the end of a packet counts as a possible posted request, so it waits for room for one; the request itself is still sent only for a real exit, and the request state machine is unchanged. With one SFU block the shared unit always accepts, so nothing changes there. With several, a non-exiting tmc/pred waits while the 2-entry request buffer is full. A direct request is answered in the cycle it is sent and never reaches the wait state, so the wait-state term, which carries the shared unit's same-cycle response, is dropped for it. On pblocks the core DUT recovered to 276 MHz (290.3 MHz together with the LSU AGU carry-select), with all 49 perf_gate rtlsim cases cycle-identical. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_wctl_unit.sv | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/hw/rtl/core/VX_wctl_unit.sv b/hw/rtl/core/VX_wctl_unit.sv index 7a2a0117de..60b4cef82a 100644 --- a/hw/rtl/core/VX_wctl_unit.sv +++ b/hw/rtl/core/VX_wctl_unit.sv @@ -186,6 +186,11 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire is_shared = execute_if.data.header.eop && (is_bar || is_wspawn || is_exit); wire sh_posted = is_shared && ~bar_rsp_valid; + // Ready is decided from the opcode: whether a tmc/pred exits depends on its + // operand mask, so every tmc/pred waits for room for a posted request. + wire may_share = execute_if.data.header.eop && (is_bar || is_wspawn || is_tmc || is_pred); + wire may_post = may_share && ~bar_rsp_valid; + // an exit is retired by the shared unit, which deactivates the warp itself assign tmc_valid = wctl_valid && (is_tmc || is_pred) && !is_shared; @@ -235,10 +240,12 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( end wire sh_req_fire = sh_req_valid && sh_req_ready; - wire sh_ok = ~is_shared || (sh_state == SH_DONE) - || ((sh_state == SH_WAIT) && warp_shared_if.rsp_valid) - || (sh_req_fire && (sh_posted || warp_shared_if.rsp_valid)); - wire exec_ok = rsp_buf_ready && !wsync_drain && sh_ok && (is_shared || !bar_drain); + wire sh_can_req = (sh_state == SH_IDLE) && !bar_drain && sh_req_ready; + // a direct request is answered in the cycle it is sent, so it never waits + wire sh_ok = ~may_share || (sh_state == SH_DONE) + || (!SH_DIRECT && (sh_state == SH_WAIT) && warp_shared_if.rsp_valid) + || (sh_can_req && (may_post || SH_DIRECT)); + wire exec_ok = rsp_buf_ready && !wsync_drain && sh_ok && (may_share || !bar_drain); assign execute_if.ready = exec_ok; wire execute_fire = execute_if.valid && execute_if.ready; From f0667ebd8999aedb845f3512874ffc52b75a2d75 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:38:00 -0700 Subject: [PATCH 076/336] simx: hold off a local-memory read after any write and release the coalescer input early Two SimX timing fixes on the LSU's local-memory path, each matching the RTL: - A local-memory read of the word written the previous cycle waits one cycle whichever port wrote it: DMA, an LSU store or an atomic's write-back. The RTL stalls on all three; SimX modeled the atomic only. - The coalescer releases its input as it builds the last batch of a request, as the RTL raises its input ready in that state, not when the batch is accepted downstream. On pblocks, model_parity mean error went from 1.38% to 1.22%, max 4.48%. Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/local_mem.cpp | 73 ++++++++++++++++++++-------------- sim/simx/mem/mem_coalescer.cpp | 15 ++++--- sim/simx/mem/mem_coalescer.h | 1 - 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 925bcc3b92..211f46c5d7 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -40,18 +40,19 @@ class LocalMem::Impl { // else meanwhile. static constexpr uint32_t AMO_BANK_STALL_CYCLES = 2; std::vector amo_busy_; - // A read issued the cycle after a write to the same word returns the word's - // pre-update value, so the bank holds such a read off for one cycle. The - // write-back is the third cycle of the occupancy, so this shadow covers the - // fourth -- the cycle the next contending atomic arrives, which is why - // back-to-back atomics on one word cost four cycles and not three. - std::vector amo_rdw_addr_; - std::vector amo_rdw_valid_; - // Whether the atomic now occupying the bank ends in a write at all. A - // load-reserved never writes and a failed store-conditional must not, so - // neither leaves a shadow behind. + // The word the atomic now occupying the bank writes back, and whether it + // writes at all: a load-reserved never writes and a failed + // store-conditional must not, so neither leaves a shadow behind. + std::vector amo_wb_addr_; std::vector amo_rdw_pending_; #endif + // A read of a word written the previous cycle (by DMA, a store or an atomic's + // write-back) would see the old value, so the bank holds it off one cycle. + // rdw_* records this cycle's writes; rdw_cur_* holds last cycle's. + std::vector rdw_addr_; + std::vector rdw_valid_; + std::vector rdw_cur_addr_; + std::vector rdw_cur_valid_; MemCrossBar::Ptr mem_xbar_; mutable PerfStats perf_stats_; uint32_t dma_last_grant_ = 0; @@ -60,13 +61,11 @@ class LocalMem::Impl { return bit_getw(addr, 0, addr_bits_-1); } -#if VX_CFG_EXT_A_ENABLED // The word a bank is addressed by. Two byte addresses falling in one word // are the same access as far as the bank is concerned. uint64_t bank_word(uint64_t addr) { return to_local_addr(addr) >> log2ceil(config_.line_size); } -#endif public: Impl(LocalMem* simobject, const Config& config) @@ -77,10 +76,13 @@ class LocalMem::Impl { #if VX_CFG_EXT_A_ENABLED , amo_unit_(VX_CFG_AMO_RS_SIZE < 2 ? 2u : (uint32_t)VX_CFG_AMO_RS_SIZE) , amo_busy_(1 << config.B, 0) - , amo_rdw_addr_(1 << config.B, 0) - , amo_rdw_valid_(1 << config.B, false) + , amo_wb_addr_(1 << config.B, 0) , amo_rdw_pending_(1 << config.B, false) #endif + , rdw_addr_(1 << config.B, 0) + , rdw_valid_(1 << config.B, false) + , rdw_cur_addr_(1 << config.B, 0) + , rdw_cur_valid_(1 << config.B, false) { char sname[100]; snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); @@ -115,9 +117,10 @@ class LocalMem::Impl { #if VX_CFG_EXT_A_ENABLED amo_unit_.reset(); std::fill(amo_busy_.begin(), amo_busy_.end(), 0); - std::fill(amo_rdw_valid_.begin(), amo_rdw_valid_.end(), false); std::fill(amo_rdw_pending_.begin(), amo_rdw_pending_.end(), false); #endif + std::fill(rdw_valid_.begin(), rdw_valid_.end(), false); + std::fill(rdw_cur_valid_.begin(), rdw_cur_valid_.end(), false); } // Serve one DMA client's row access this cycle and return the banks it @@ -160,7 +163,10 @@ class LocalMem::Impl { if (req.byteen & (1ull << b)) { uint8_t value = (*req.data)[b]; ram_.write(&value, line_addr + b, 1); - banks |= 1ull << (((line_addr + b) >> lg2_line_size) & (num_banks - 1)); + uint32_t bank = ((line_addr + b) >> lg2_line_size) & (num_banks - 1); + banks |= 1ull << bank; + rdw_addr_[bank] = this->bank_word(line_addr + b); + rdw_valid_[bank] = true; } } #if VX_CFG_EXT_A_ENABLED @@ -188,6 +194,10 @@ class LocalMem::Impl { } void tick() { + rdw_cur_addr_.swap(rdw_addr_); + rdw_cur_valid_.swap(rdw_valid_); + std::fill(rdw_valid_.begin(), rdw_valid_.end(), false); + uint64_t dma_banks = this->dma_tick(); // process bank requets from xbar @@ -196,16 +206,13 @@ class LocalMem::Impl { #if VX_CFG_EXT_A_ENABLED // Cycles an in-flight atomic owns; the bank serves nothing else. if (amo_busy_[i] != 0) { - if (0 == --amo_busy_[i]) { - amo_rdw_valid_[i] = amo_rdw_pending_[i]; // write-back just issued + if (0 == --amo_busy_[i] && amo_rdw_pending_[i]) { + // write-back just issued + rdw_addr_[i] = amo_wb_addr_[i]; + rdw_valid_[i] = true; } continue; } - // The shadow is one cycle wide: consume it here whether or not - // anything is waiting on the bank. - const bool rdw_shadow = amo_rdw_valid_[i]; - const uint64_t rdw_addr = amo_rdw_addr_[i]; - amo_rdw_valid_[i] = false; #endif if ((dma_banks >> i) & 1) { continue; @@ -216,18 +223,22 @@ class LocalMem::Impl { auto& bank_req = xbar_req_out.peek(); + // A read of the word written last cycle waits a cycle. An atomic + // reads the bank for its old value, so it counts as a reader here + // even though the request carries a write flag; that is what makes + // contending atomics on one word cost four cycles. #if VX_CFG_EXT_A_ENABLED - // A read of the word the write-back just stored waits a cycle. An - // atomic reads the bank for its old value, so it counts as a reader - // here even though the request carries a write flag; that is what - // makes contending atomics on one word cost four cycles. const bool reads_bank = !bank_req.is_write() || memop_is_atomic(bank_req.op); - if (rdw_shadow +#else + const bool reads_bank = !bank_req.is_write(); +#endif + if (rdw_cur_valid_[i] && reads_bank - && this->bank_word(bank_req.addr) == rdw_addr) { + && this->bank_word(bank_req.addr) == rdw_cur_addr_[i]) { continue; } +#if VX_CFG_EXT_A_ENABLED // Shared-memory atomics: read-modify-write in place and ALWAYS // return the old word. An AMO has a destination register, so the // LSU reserved a response slot; a missing response hangs the warp @@ -285,7 +296,7 @@ class LocalMem::Impl { perf_stats_.reads += !do_store; perf_stats_.writes += do_store; DT(4, simobject_->name() << "-bank" << i << " amo : " << bank_req); - amo_rdw_addr_[i] = this->bank_word(bank_req.addr); + amo_wb_addr_[i] = this->bank_word(bank_req.addr); amo_rdw_pending_[i] = do_store; xbar_req_out.pop(); amo_busy_[i] = AMO_BANK_STALL_CYCLES; // capture, then write back @@ -302,6 +313,8 @@ class LocalMem::Impl { ram_.write(&value, line_addr + b, 1); } } + rdw_addr_[i] = this->bank_word(bank_req.addr); + rdw_valid_[i] = true; #if VX_CFG_EXT_A_ENABLED // A plain store breaks a reservation on the word it writes, the // same as an atomic's commit does; otherwise a store between a diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index 239acc1b50..bf25a8f05e 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -256,7 +256,6 @@ void MemCoalescer::on_tick() { // build per-channel memory requests out_round_.valid = true; out_round_.lanes = out_mask; - out_round_.cur_mask = cur_mask; out_round_.reqs.assign(output_size_, MemReq{}); for (uint32_t o = 0; o < output_size_; ++o) { if (!out_mask.test(o)) { @@ -285,7 +284,13 @@ void MemCoalescer::on_tick() { // The built round is presented to the downstream on the next tick, not // this one: the coalescer alternates a build tick with a drain tick, so a // batch issues at most every other cycle. Draining is handled at the top - // of on_tick when out_round_ is valid. + // of on_tick when out_round_ is valid. The input is released as its last + // batch is built, without waiting for that batch to be accepted. + sent_mask_ |= cur_mask; + if (sent_mask_ == in_req.mask) { + ReqIn.pop(); + sent_mask_.reset(); + } } // Issue the pending round's per-channel requests; commit the round once @@ -304,12 +309,6 @@ void MemCoalescer::flush_out_round() { } out_round_.valid = false; - auto& in_req = ReqIn.peek(); - sent_mask_ |= out_round_.cur_mask; - if (sent_mask_ == in_req.mask) { - ReqIn.pop(); - sent_mask_.reset(); - } } const MemCoalescer::PerfStats& MemCoalescer::perf_stats() const { diff --git a/sim/simx/mem/mem_coalescer.h b/sim/simx/mem/mem_coalescer.h index e133c4be1a..3cc6d72c8d 100644 --- a/sim/simx/mem/mem_coalescer.h +++ b/sim/simx/mem/mem_coalescer.h @@ -67,7 +67,6 @@ class MemCoalescer : public SimObject { bool valid = false; BitVector<> lanes; // output lanes still to send std::vector reqs; // per output lane - BitVector<> cur_mask; // input lanes covered by this round }; uint32_t input_size_; From b3cad81fb8b0ac00f59f1ac6cf2e81b7fe59650c Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:43:21 -0700 Subject: [PATCH 077/336] simx: launch each cluster on the nearest ready core The RTL launch fan-out (VX_kmu_bus_arb) round-robins compute messages over its ready outputs, starting from its pointer, and puts a two-entry skid in front of each output. SimX sent every cluster to the core the pointer named and stalled while that core was busy. It also let each core hold four CTAs it had not yet admitted. The two differences only matter when cores finish their CTAs at different times. The raycaster's grid is four CTAs wide, so the fixed rotation gave each of four cores one column of the image. The two middle columns hold the teapot and carry most of the work: SimX put 114,408 of 259,276 instructions on core 1 and took 478,511 cycles. The RTL lets the cores that finish early take the next CTAs and took 420,640 cycles, so SimX was 13.8% high. An rtlsim trace of a two-core SGEMM shows the same steering: at the start of the second wave, core 1 accepts three CTAs before core 0 accepts any. The four-deep lanes came in with the admission credits. Strict rotation was then added to undo the skew they caused on a two-core SGEMM. Two credits, which match the RTL's skid depth, together with the RTL's work-conserving choice, fix both. A single-core launch is unchanged, since it has only one lane to choose. The lanes stay flat, one per core. That matches the RTL wherever only one level of the launch tree fans out, which covers every parity shape. Where two levels fan out, as in several clusters of several cores, the upper level's shared skids are not modelled. The design doc now says so, and its description of SimX's steering and credits is corrected. raycast cycles, SimX before -> after vs rtlsim: 4c + L2 478,511 -> 419,820 vs 420,640 (+13.8% -> -0.2%) 2c IW4 8-bank 829,292 -> 767,309 vs 769,586 (+7.8% -> -0.3%) 2c IW4 8-bank CP_LLC=1 841,841 -> 748,610 vs 731,508 (the remaining +2.3% is the cycle counter, fixed separately) default (1 core) 1,494,492, unchanged On two cores the per-core instruction split now matches the RTL exactly: 122,571 / 136,673 with CP_LLC=1 and 128,064 / 131,180 without. sgemm -n32 is unchanged at 115,067 (1 core) and 30,086 (4c + L2). CI adds parity-raycast-mc (raycast, four cores, L2), so CTA placement under uneven work is checked against the RTL. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 9 +++++++ docs/designs/cta_dispatch_architecture.md | 32 +++++++++++++---------- sim/simx/kmu/kmu.cpp | 25 ++++++++++-------- sim/simx/kmu/kmu.h | 9 +++---- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 1ae0064d19..1ccfd9dcd7 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -233,3 +233,12 @@ tests: via: blackbox app: raycast configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 +# The raycaster on four cores sharing an L2: its image columns carry very +# different work, so where the launch places each CTA sets the critical path. +- id: parity-raycast-mc + check: model_parity + via: blackbox + app: raycast + shape: + cores: 4 + l2cache: true diff --git a/docs/designs/cta_dispatch_architecture.md b/docs/designs/cta_dispatch_architecture.md index 7b8af01a77..7906fd124a 100644 --- a/docs/designs/cta_dispatch_architecture.md +++ b/docs/designs/cta_dispatch_architecture.md @@ -462,10 +462,10 @@ Two preconditions make this sound, both already enforced: SimX has no bus, so its `Kmu` carries the routing itself: the destination is fixed **before** a message is sent — a locked cluster's remaining members go -only to the core that took the first, and a new message goes to the core the -round-robin already points at, stalling on a busy core rather than yielding -its turn to a neighbour -([`kmu.cpp:122-153`](../../sim/simx/kmu/kmu.cpp#L122)). +only to the core that took the first, and a new message goes to the nearest +ready core from the round-robin pointer, as the RTL fan-out picks the nearest +ready output +([`kmu.cpp:122-156`](../../sim/simx/kmu/kmu.cpp#L122)). **Why this is load-bearing rather than an optimisation.** The obvious consequence of a split cluster is that LMEM peers resolve to another CTA's @@ -733,16 +733,20 @@ is cycle-for-cycle identical to it on `vecadd` / `sgemm` / `sgemm_tcu_wg` / The KMU→core transport is modelled as per-core **launch lanes**: each lane is a registered boundary stage feeding the dispatcher's endpoint, and the KMU -additionally paces every core with explicit credits — `LAUNCH_CREDITS = 4` -per core ([`kmu.h:59`](../../sim/simx/kmu/kmu.h#L59)) — spent when a CTA is +additionally paces every core with explicit credits — `LAUNCH_CREDITS = 2` +per core ([`kmu.h:58`](../../sim/simx/kmu/kmu.h#L58)) — spent when a CTA is sent and returned by the dispatcher one message per admission -([`cta_dispatcher.cpp:142`](../../sim/simx/cta_dispatcher.cpp#L142)). The -bound is the transport's occupancy made explicit — the boundary stage's -endpoint (2) plus the dispatcher endpoint (1) plus one credit in return -flight — so a busy core never accumulates work beyond what the RTL's skids -would hold, and pacing rests on slot accounting rather than on peeking at -the consumer's state. RTL needs no counter: its registered output skids -apply the same bound as wires. +([`cta_dispatcher.cpp:175`](../../sim/simx/cta_dispatcher.cpp#L175)). The +bound is the two-entry output skid in front of each core made explicit, so a +busy core never holds more unadmitted CTAs than the RTL's skid would, and +pacing rests on slot accounting rather than on peeking at the consumer's +state. The depth matters for placement, not only for pacing: every CTA +committed to a busy core is one a core that frees a slot sooner cannot take. +RTL needs no counter: its registered output skids apply the same bound as +wires. The lanes are flat, one per core, so the model is exact while a single +level of the tree fans out; where two levels fan out (several clusters of +several cores, or several sockets of several cores) the upper level's shared +skids are not modelled. The differences are deliberate, not drift: @@ -754,7 +758,7 @@ The differences are deliberate, not drift: | Bus payload | `cluster_size` (scalar), `aligned_lmem_size` | `cluster_dim[3]`, raw `lmem_size` | | `cta_id` on the bus | absent | present (model-only, unused as identity) | | Stride rounding | in the KMU | in the dispatcher | -| Launch pacing | elastic-skid back-pressure | explicit credits (`LAUNCH_CREDITS = 4`) | +| Launch pacing | elastic-skid back-pressure | explicit credits (`LAUNCH_CREDITS = 2`) | | Fragment path | through `kmu_bus_if` → `VX_cta_dispatch` | injected at the scheduler by the FWD | The last row is the significant one: in RTL a fragment wave is a launch that diff --git a/sim/simx/kmu/kmu.cpp b/sim/simx/kmu/kmu.cpp index ab20f74be8..58734cce0e 100644 --- a/sim/simx/kmu/kmu.cpp +++ b/sim/simx/kmu/kmu.cpp @@ -120,15 +120,12 @@ void Kmu::on_tick() { return; } // Steering: a locked cluster's remaining members stay with the owning core; - // otherwise the next cluster goes to the core the round-robin already points - // at. One CTA per cycle — the launch bus carries a single beat. A lane is - // ready when its transport has room and the core still owes fewer than - // LAUNCH_CREDITS admissions, so a busy core never accumulates work. - // - // The destination is fixed before the message is sent, so a busy core stalls - // the bus instead of yielding its turn to a neighbour. Scanning for any ready - // lane here would let a core that drains faster absorb its neighbour's share - // and leave the placement lopsided. + // otherwise the next cluster goes to the nearest ready lane from the + // round-robin pointer, so a core that drains its work sooner takes the next + // cluster instead of the launch waiting on a busy neighbour. One CTA per + // cycle — the launch bus carries a single beat. A lane is ready when its + // transport has room and the core holds fewer than LAUNCH_CREDITS CTAs it + // has not yet admitted. auto lane_ready = [&](uint32_t c) { return !bus_out.at(c).full() && inflight_[c] < LAUNCH_CREDITS; }; @@ -137,8 +134,14 @@ void Kmu::on_tick() { if (lane_ready(cluster_core_)) { target = cluster_core_; } - } else if (lane_ready(rr_ptr_)) { - target = rr_ptr_; + } else { + for (uint32_t k = 0; k < num_cores; ++k) { + uint32_t c = (rr_ptr_ + k) % num_cores; + if (lane_ready(c)) { + target = c; + break; + } + } } if (target == num_cores) return; // no ready lane this cycle diff --git a/sim/simx/kmu/kmu.h b/sim/simx/kmu/kmu.h index 537703601c..687bd40ec9 100644 --- a/sim/simx/kmu/kmu.h +++ b/sim/simx/kmu/kmu.h @@ -52,11 +52,10 @@ class Kmu : public SimObject { // rests on explicit slot accounting, not on transport back-pressure. std::vector> credit_in; - // In-flight bound per core: the boundary stage's endpoint (2) plus the - // dispatcher endpoint (1) plus one admission credit in return flight, so - // the lane's full() view stays the binding gate whenever the transport - // applies back-pressure. - static constexpr uint32_t LAUNCH_CREDITS = 4; + // Each core's launch buffer holds two CTAs it has not yet admitted. A + // deeper buffer would commit work to a busy core early and keep it from a + // core that frees a slot sooner. + static constexpr uint32_t LAUNCH_CREDITS = 2; Kmu(const SimContext& ctx, const char* name, uint32_t num_cores); From 4470f277f1828129e575d1bd1575cfcadad4e76f Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:43:21 -0700 Subject: [PATCH 078/336] simx: count a core's cycles only while it has work In the RTL, a core's cycle counter advances only while its scheduler is busy. VX_scheduler increments cycles_r under `if (busy)`. busy is the registered OR of three terms: a warp is active (active_warps != 0), a CTA is firing (cta_fire), or an instruction is in flight (the warp scheduler's per-warp issued-minus-committed counter is not empty). That is ORed with the CTA dispatcher's busy, which covers dispatching and accepting a launch. The counter, sched_cycles, is the one VX_csr_mpm returns to the host for VX_CSR_MCYCLE, and the one VX_csr_data returns to a kernel that reads mcycle. SimX advanced its counter on every tick. It therefore also charged a core for the cycles before its first CTA arrived, the cycles after its last instruction committed, and every cycle the host spent moving data while the simulator ticked. The last case is the one that showed. With VX_CFG_CP_LLC_CLIENT, the command processor's uploads and readbacks go through the shared L2, and SimX ticks the whole machine until they drain. On raycast at eight cores, IW4 and 8 memory banks, that added about 19,000 cycles to every core: SimX reported 300,432 against the RTL's 282,021. The RTL's per-core counts show that its counter is gated. They range from 26,749 on the most idle core to 282,021 on the busiest. SimX now reports 26,747 to 281,015, within 0.4% on every core. Count while Scheduler::running() or an instruction is in flight. Those are the RTL's terms: a warp active, a CTA firing, the dispatcher holding a CTA, and an instruction scheduled but not yet committed. SimX's dispatcher also counts the one or two cycles a CTA waits at the core's launch endpoint before it is admitted, which the RTL holds in the fan-out's skid instead. A core that stays busy from launch to completion moves only by those edge cycles: raycast -74 and sgemm -n32 -39 at one core. raycast cycles, SimX before -> after vs rtlsim (on top of the launch fix): 8c IW4 8-bank CP_LLC=1 300,432 -> 281,015 vs 282,021 (+6.5% -> -0.4%) 2c IW4 8-bank CP_LLC=1 748,610 -> 729,206 vs 731,508 (+2.3% -> -0.3%) 4c + L2 419,820 -> 419,477 vs 420,640 (-0.3%) model_parity-dogfood-cp-llc moves from +3.39% to +0.23%. Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 2455e026b1..bd84b24ea5 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -354,8 +354,8 @@ class Core::Impl { std::fill(ibuf_inflight_.begin(), ibuf_inflight_.end(), 0); - // perf_stats_ deliberately survives: these are the MPM counters, which are - // free-running from device reset. A kernel launch re-enters reset() here but + // perf_stats_ deliberately survives: these are the MPM counters, which + // accumulate from device reset. A kernel launch re-enters reset() here but // does not reset the hardware, so clearing them would make each dump report // only the last launch -- an app that dumps between launches would then see // per-launch counts where the device reports a running total. @@ -370,7 +370,13 @@ class Core::Impl { this->fetch(); this->schedule(); - ++perf_stats_.cycles; + // The cycle counter advances only while the core has work -- a CTA + // launching, a warp active, or an instruction in flight -- so cycles it + // spends idle before its first CTA, after its last warp, or while the host + // moves data are not charged to it. + if (scheduler_->running() || !pending_instrs_.empty()) { + ++perf_stats_.cycles; + } DPN(2, std::flush); } From 7039eea2666195fca071dfa1afd0781f4b014622 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:43:22 -0700 Subject: [PATCH 079/336] ci: size the fp16-mc parity case like its perf_gate twin At its default 64^3, model_parity-fp16-mc runs 64 one-warp CTAs on two cores with eight slots each, which is four waves in about 18,000 cycles. A core admits its next CTA into its round-robin tail slot, so one CTA that runs long blocks the core's next wave, and the work-conserving launch then gives that core's share to the other core. The split, and with it the total, moves with the timing of a single CTA. With the launch fix, SimX's per-CTA times match the RTL's to within 1% on average in each of the first two waves (5,074 / 5,077 against 5,103 / 5,050, and 4,160 / 4,155 against 4,191 / 4,255 cycles). But one second-wave CTA on core 0 ran 4,494 cycles in SimX against about 4,000 for the rest. That held core 0's third wave back by 550 cycles, and core 1 ended with 34 CTAs to core 0's 30, where the RTL split them 32 / 32. The case then read +8.15% (19,315 vs 17,859). Before the launch fix, strict rotation forced a 32 / 32 split whatever the timing, so the sensitivity was hidden. At 128^3, the size perf_gate-fp16-mc already runs, the same configuration gives 123,402 against the RTL's 120,867 (+2.1%; +1.3% before the launch fix). The variation of a single CTA no longer decides the result. The tolerance stays at 5%. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/testcases/tensor.yaml b/ci/testcases/tensor.yaml index 246774db01..52b0799215 100644 --- a/ci/testcases/tensor.yaml +++ b/ci/testcases/tensor.yaml @@ -193,6 +193,9 @@ tests: check: model_parity via: blackbox app: sgemm_tcu + # Sized like its perf_gate twin: at the default 64^3 the grid is four waves of + # CTAs, so one CTA running long moves the per-core split and the cycle count. + args: -m 128 -n 128 -k 128 configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 shape: {cores: 2, l2cache: true} From 8d9335a34b4b1f5dd16f356c196feea8ee3c449d Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 16:45:49 -0700 Subject: [PATCH 080/336] CHANGELOG: memory-level parallelism, phase 1 Record the features, design changes and bug fixes on this branch that are not yet on master: per-partition fetch, the hashed bank select, the dcache read merge, write-allocate without fetch, the MSHR line table, the command processor's DMA through the shared LLC, the flush and fence redesign, the LMEM DMA arbitration, the SimX timing model, and the bugs whose faults were also present on master. Optimizations under 10% and fixes to code that is new on this branch are left out. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b384f6a82b..ec2d93d728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,31 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, - **Hardware profiles.** `profiles/.json` names a machine (`tinygpu`, `rtx1080`, `v100`, `a100`, `h100`, `rtx5080`) that every flow builds through `configure --profile=` or `blackbox.sh --profile=`; explicit `CONFIGS` win over the profile. See [docs/simulation.md](docs/simulation.md#hardware-profiles). +- **Per-partition fetch (`VX_CFG_FETCH_WIDTH`, default 1).** The front end splits into FETCH_WIDTH slices, each with its own warp + scheduler, fetch, decode and icache port, serving the warps with `wid % FETCH_WIDTH == f`; CTA dispatch, barriers and warp control + stay core-wide. The core takes one of three structures from the fetch width, issue width and per-unit block counts; with one block + of every unit per issue slice and FETCH_WIDTH equal to ISSUE_WIDTH, each slice is a self-contained compute slice (the NVIDIA + processing-block organization). *Why:* sgemm at 16 warps x 16 threads, ISSUE_WIDTH 4: -31.6% cycles at FETCH_WIDTH 2. +- **Hashed cache bank select (`VX_CFG_{DCACHE,L2}_BANK_HASH`).** The bank index is the raw bank bits XORed with a hash of the line + address above them (IPOLY(5) for 16 banks, IPOLY(37) for 32, an XOR fold otherwise), so power-of-two strides spread over the banks. + On by default for the L2 and for a dcache with 4 or more banks. *Why:* rtlsim sgemv 512x512 at 16 threads -32.8%, fp16 TCU SGEMM + on 2 cores + L2 at 16 threads -13.8%, WGMMA with DXA -13.5%. +- **Dcache read merge (`VX_CFG_DCACHE_READ_MERGE`, on by default).** Plain reads of the same word from the ports of one LSU block in + the same cycle are served by one bank access, and the response is multicast to every port; each port's order into the bank is + unchanged. *Why:* uniform loads (kernel arguments, SGEMM's shared operand) no longer serialize on one bank: rtlsim sgemm -n128 at + 16 threads -6.5%, -17.2% with an 8-byte dcache word. +- **Write-allocate without fetch (`VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH`, off by default).** A store miss in a write-back cache installs + the sector with its dirty-byte mask and a partial bit instead of fetching the line; a later load or AMO merges memory data under the + unwritten bytes, and evictions write back only dirty bytes. *Why:* streaming writes no longer read every line first: 1-core vecadd + DRAM reads -33%. +- **MSHR line table (`VX_CFG_{DCACHE,L2,L3}_MSHR_LINES`, off by default).** Misses are tracked in a reference-counted table of + distinct {line, sector} entries, and request entries keep only an index into it, so the MSHR holds more requests without widening + its address compares. *Why:* at 400-cycle memory latency, 64 requests over 16 lines: sgemm -n128 -15.3%, raycast -8.1%. +- **Command-processor DMA through the shared LLC (`VX_CFG_CP_LLC_CLIENT`, off by default).** When one LLC serves every core, the + CP's transfers enter it on their own client slot, so host transfers need only an L1 flush, not a walk of the whole LLC; transfers + of at least half the LLC write around it. A launch behind DMA writes waits for an L1-only flush token sent on the slot. The runtime + sees the new `DMA_LLC_CLIENT` CP capability. *Why:* rtlsim at 2 clusters + L2 + L3: dogfood -30%, multikernel -42.7%; dogfood + -64% with an 8 MB L3. ### Changed @@ -23,6 +48,48 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, (JEDEC HBM2/HBM3/GDDR6 organization and nanosecond timing, 32 B bursts, permuted open-page bank interleave, per-channel responses), configured from data-sheet values: `VX_CFG_PLATFORM_MEMORY_{TYPE,DATA_RATE,LATENCY,QUEUE_SIZE}`. The default machine keeps its idle read latency (14/26 cycles open-row/conflict, was 13/27) and one-block-per-cycle peak. +- **Cache flush architecture.** The CP starts every core's flush with one broadcast `CACHE_FLUSH` DCR write, then reads each core's + completion, so the flushes overlap; a flush read without a preceding write no longer starts one. A cache walks its banks, sends one + flush token down every memory port and answers once all ports have; every independently buffered write path (each dcache lane, each + OM input of the ocache) carries its own token. Read-only caches (icache, tcache, rtcache, rcache) no longer forward flushes. + `DBG_FLUSH_STALL` adds random input backpressure to the flush path for stress tests. +- **L1-only flush between launches.** When a single LLC serves every memory client it is the point of coherence, so a launch + boundary cleans only the L1s: `CMD_CACHE_FLUSH` gains an L1-only flag, carried as a flush-token kind that a shared LLC answers + without a walk. Replicated LLCs and LLCs with peer clients (TEX, RTU, RASTER, OM, DXA) still walk. The runtime posts flushes + lazily and flushes fully before any device-memory transfer, and device busy stays high until every core write, uncached IO stores + included, is visible in memory. *Why:* per-launch flush cost at 4 cores + L2 drops from 2,750 to 180 cycles. +- **Targeted fences.** A memory fence sends an L1-only flush token that the shared LLC answers without a walk. A fence that orders + only device (I/O) accesses decodes as its own LSU op, and its token makes each level drain to its memory ports without walking. + *Why:* printf, which fences after each device write, runs 13.6x faster on rtlsim at 2 cores + L2. +- **LMEM DMA arbitration.** DXA writes into shared memory and the TCU's WGMMA operand reads now share the local-memory DMA port + round-robin. DXA had fixed priority and starved the TCU for about 510 cycles per tiled-scatter transfer. *Why:* rtlsim WGMMA with + DXA -10.2%, sparse WGMMA with DXA -11.6%. +- **SimX timing model follows the RTL's structure.** Buffer capacities and registered ready at every pipeline boundary, per-slice + cache arbiter trees and network depths, registered cache core-response ports, flush and fence tokens end to end, and CTA launch onto + the nearest ready core with two launch credits per core. SimX's per-core cycle counter (`mcycle`) now advances only while the core + has work, as the RTL's does. rtlsim completes DRAM requests in per-bank arrival order, as the AXI platform does. + +### Fixed + +- **Global barriers across clusters.** Each cluster ran its own global-barrier unit that saw only its own cores, while + `vx_num_cores()`, and so `vortex::gbarrier`, counts the whole device. On SimX every global barrier hung with more than one + cluster; on the RTL, 2 clusters x 1 core timed out and 2 clusters x 2 cores released each cluster on its own arrivals, with no + ordering between clusters. One device-level unit now counts every core. +- **Writes left dirty by a flush.** A cache forwarded its flush token on memory port 0 only, so writes queued on another port could + reach the next level after that level's walk and stay dirty there. +- **Dirty sectors dropped on a sector refill.** With dirty-byte tracking and several sectors per line, installing a sector into a + resident line cleared the whole line's byte mask, so a dirty sibling sector wrote back with no bytes enabled and its data was lost + (mstress at 4 cores with a write-back, sectored L2). +- **Local-memory LR/SC livelock at 16 threads.** One warp's per-lane LR burst used up a reservation's fixed hold budget of 7 refusals, + so contended LR/SC loops on shared memory stopped making progress. The budget now scales with the core's harts. +- **`VX_stream_arb` trees with a partial last slice.** When the request count was not a whole number of slices, the last slice's + select was sized and placed as if it were full: the build failed lint, and with several outputs requests went to the wrong output. +- **`-DSCOPE` builds.** The scope buses of fetch, issue, LSU, TCU, TEX, RASTER and OM were declared only under their `DBG_SCOPE_*` + taps, left unbound, or did not parse, so debug-scope builds failed to elaborate. +- **mstress checked no data.** Its host reference was always 0, so the test passed with corrupted loads; it now checks every loaded + value exactly. +- **FPGA run instructions.** The docs exported `TARGET=hw`, which `blackbox.sh` ignores, so runs fell back to xrtsim; they now pass + `--target=hw`. ## [3.0] — 2026-10-07 From 60e7a2bbca0df5be6320029462e8ece86348348a Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 16:45:49 -0700 Subject: [PATCH 081/336] tests: render raycast at 64x64 by default The 16x12 default gives each core only a handful of CTAs, so multi-core runs measure launch and tail effects more than the raycaster itself. A 64x64 render target gives every configuration enough work to reach a steady state. OPTS still overrides it. Co-Authored-By: Claude Opus 5.5 --- tests/regression/raycast/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/regression/raycast/Makefile b/tests/regression/raycast/Makefile index d736258fb2..b1dedb71a2 100644 --- a/tests/regression/raycast/Makefile +++ b/tests/regression/raycast/Makefile @@ -11,8 +11,8 @@ VX_SRCS := $(SRC_DIR)/kernel.cpp CXXFLAGS += -DASSETS_PATHS='"$(SRC_DIR)"' -# Default workload is small for fast CI sweeps; override OPTS for full-resolution runs. -OPTS ?= -w 16 -h 12 +# Default workload is a 64x64 render target; override OPTS for other resolutions. +OPTS ?= -w 64 -h 64 KERNEL_LIB := vortex2 From 4faf32cb4c0ba94eb108d677a628af73e793a0db Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 18:07:24 -0700 Subject: [PATCH 082/336] cache: size the flush-token attribute and hash input to their fields Two Verilator warnings, fatal in CI, broke every rtlsim, opaesim and xrtsim build that reached them: - The memory-side bank hash read its line address through a full-width wire and used only its low 32 bits. At XLEN 64 the upper bits were unused (UNUSEDSIGNAL) whenever the hash was on: a dcache with 4 or more banks, or the L2/L3. The hash now takes the 32-bit slice directly, as the request side already does; the hash itself only reads the low bits, so the result is unchanged. - The flush token's attribute shifted 32-bit operands into a field that is wider with the A extension (47 bits with Zacas at XLEN 32, 79 at 64), a WIDTHEXPAND. Both operands are now cast to the field width. Lint with -Wall at the failing CI configs (amo with A+Zacas at XLEN 32 and 64, L1_LINE_SIZE=8 at 64, 2 clusters x 2 cores + L2 + L3 at 64) and at the default 32/64: the unfixed file reproduces each CI warning, the fixed one is clean. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache.sv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index d51f0c3e28..926f807452 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -610,8 +610,8 @@ module VX_cache import VX_gpu_pkg::*; #( // the raw bank bits of the memory address, formed while the bank id is a constant wire [`UP(BANK_SEL_BITS)-1:0] raw_bid; if (NUM_BANKS > 1 && BANK_HASH != 0) begin : g_hash - wire [`CS_LINE_ADDR_WIDTH-1:0] line_addr = per_bank_mem_req_addr[i][`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]; - assign raw_bid = BANK_SEL_BITS'(i) ^ BANK_SEL_BITS'(cache_bank_hash(32'(line_addr), BANK_SEL_BITS)); + assign raw_bid = BANK_SEL_BITS'(i) ^ BANK_SEL_BITS'(cache_bank_hash( + 32'(per_bank_mem_req_addr[i][`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]), BANK_SEL_BITS)); end else begin : g_raw assign raw_bid = `UP(BANK_SEL_BITS)'(i); end @@ -656,7 +656,7 @@ module VX_cache import VX_gpu_pkg::*; #( SECTOR_SIZE'(mem_flush_l1_only), `CS_SECTOR_WIDTH'(0), BANK_MEM_TAG_WIDTH'(0), - `UP(MEM_ATTR_WIDTH)'((1 << MEM_ATTR_FLUSH_OFFS) | (32'(mem_flush_io) << MEM_ATTR_IO_OFFS)) + (`UP(MEM_ATTR_WIDTH)'(1) << MEM_ATTR_FLUSH_OFFS) | (`UP(MEM_ATTR_WIDTH)'(mem_flush_io) << MEM_ATTR_IO_OFFS) }; assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; From 3b241d7b208cfa8c70d9b013853bb1e3d9fd7d44 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 18:07:24 -0700 Subject: [PATCH 083/336] simx: let the CP LLC port build in the functional simulator cp_llc_port.h included simobject.h directly, while types.h picks simobject_functional.h under SIMX_FUNCTIONAL. A functional build saw both SimContext classes and failed (debug:simx_functional-1/2 in CI). The port now gets the simobject header through types.h like every other model. simx_functional-1 (sgemm -n128) and -2 (SIMX_MT=4, 4 cores + L2) pass. Co-Authored-By: Claude Opus 5.5 --- sim/simx/cp_llc_port.h | 1 - 1 file changed, 1 deletion(-) diff --git a/sim/simx/cp_llc_port.h b/sim/simx/cp_llc_port.h index a298216046..ab90a85876 100644 --- a/sim/simx/cp_llc_port.h +++ b/sim/simx/cp_llc_port.h @@ -13,7 +13,6 @@ #pragma once -#include #include #include #include From 8e2aa619c6e33a5cd2155e3e447afa6069153457 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 02:15:49 -0700 Subject: [PATCH 084/336] core: give each LSU block its own memory path Each LSU block's memory scheduler and its per-block share of VX_mem_unit (the local-memory switch, the local-memory adapter, the coalescer and the D-cache adapter) were instantiated in VX_core, outside the slice they serve, so every load left the slice and came back. They now live in the block, in a new VX_lsu_mem_path, with their parameters and generate conditions unchanged. VX_mem_unit keeps what the blocks share: the local-memory banks and the D-cache flush injectors. The injectors stay in the core so that every slice stays identical (a private D-cache has an injector on lane 0 only); the flush token is told apart from the lane's responses by a tag bit, and the cache's response reaches the block as wires. No register is added or removed. rtlsim sgemm_tcu_wg_sp_dxa -m 128 -n 128 -k 128 (IW 4, FW 4, 4 LSU and TCU blocks, 32 threads): 144,956 cycles, identical to the parent, PASSED. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 2 +- hw/rtl/core/VX_backend_slice.sv | 32 ++- hw/rtl/core/VX_compute_slice.sv | 32 ++- hw/rtl/core/VX_core.sv | 172 ++++++++------ hw/rtl/core/VX_execute.sv | 36 ++- hw/rtl/core/VX_execute_slice.sv | 42 +++- hw/rtl/core/VX_lsu_block.sv | 47 +++- hw/rtl/core/VX_lsu_mem_path.sv | 287 ++++++++++++++++++++++ hw/rtl/core/VX_lsu_unit.sv | 54 ++++- hw/rtl/core/VX_mem_unit.sv | 304 +++++------------------- hw/unittest/mem_unit/VX_mem_unit_top.sv | 101 ++++---- hw/unittest/tensor/VX_tensor_top.sv | 126 +++++----- 12 files changed, 786 insertions(+), 449 deletions(-) create mode 100644 hw/rtl/core/VX_lsu_mem_path.sv diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 7acc4b6daf..782f1361c9 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1721,7 +1721,7 @@ package VX_gpu_pkg; localparam DCACHE_CORE_TAG_WIDTH = (UUID_WIDTH + DCACHE_TAG_ID_BITS); // Core request tag bits on dcache_bus_if port of VX_core (+1 for flush-arb sel bit) - // Pre-MMU width (what VX_mem_unit produces, including the flush sel bit). + // Pre-MMU width (an LSU block's D-cache port, including the flush sel bit). // When VM_ENABLE is on, the dMMU widens this by DCACHE_TLB_SOURCE_BITS // for its internal bypass/TLB/PTW arbiter, so the cache cluster sees // the wider DCACHE_TAG_WIDTH below. diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv index 93732a84cd..acc5767a67 100644 --- a/hw/rtl/core/VX_backend_slice.sv +++ b/hw/rtl/core/VX_backend_slice.sv @@ -56,7 +56,21 @@ import VX_tcu_pkg::*; `endif // LSU - VX_lsu_sched_if.master lsu_client_if, +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave lsu_tcu_mem_if, +`endif + output wire lsu_sched_empty, + output wire lsu_store_empty, +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], // SFU `ifdef VX_CFG_EXT_DXA_ENABLE @@ -141,7 +155,21 @@ import VX_tcu_pkg::*; .dispatch_if (dispatch_if), .commit_if (commit_if), .branch_ctl_if (branch_ctl_if), - .lsu_client_if (lsu_client_if), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire), + .lsu_coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if), + `endif + .lsu_sched_empty (lsu_sched_empty), + .lsu_store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (dxa_req_bus_if), `endif diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index 0111b36d12..63be670437 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -59,7 +59,21 @@ import VX_tcu_pkg::*; `endif // LSU - VX_lsu_sched_if.master lsu_client_if, +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave lsu_tcu_mem_if, +`endif + output wire lsu_sched_empty, + output wire lsu_store_empty, +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], // SFU `ifdef VX_CFG_EXT_DXA_ENABLE @@ -251,7 +265,21 @@ import VX_tcu_pkg::*; `ifdef VX_CFG_EXT_RTU_ENABLE .sched_unlock_if (sched_unlock_if), `endif - .lsu_client_if (lsu_client_if), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire), + .lsu_coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if), + `endif + .lsu_sched_empty (lsu_sched_empty), + .lsu_store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (dxa_req_bus_if), `endif diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 4b1b9c6e42..aee719c076 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -80,18 +80,38 @@ import VX_tcu_pkg::*; VX_txbar_bus_if dxa_txbar_bus_if(); `endif - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + // the LSU blocks' memory paths + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty; +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if #( .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_mem_if[`VX_CFG_NUM_LSU_BLOCKS](); - - // per-LSU-block memory scheduler clients from execute - VX_lsu_sched_if lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS](); - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; + ) lsu_lmem_bus_if[LSU_NUM_REQS](); +`endif + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) lsu_dcache_bus_if[DCACHE_NUM_REQS](); +`ifdef PERF_ENABLE + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses; +`endif `ifdef TCU_META_ENABLE - VX_lsu_sched_if tcu_mem_if(); + // The TCU metadata AGU is the second client of LSU block 0's scheduler; the + // other blocks' second client is idle. + VX_lsu_sched_if lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); + for (genvar b = 1; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_lsu_tcu_tieoff + assign lsu_tcu_mem_if[b].req_valid = 1'b0; + assign lsu_tcu_mem_if[b].req_data = '0; + assign lsu_tcu_mem_if[b].rsp_ready = 1'b1; + `UNUSED_VAR (lsu_tcu_mem_if[b].req_ready) + `UNUSED_VAR (lsu_tcu_mem_if[b].rsp_valid) + `UNUSED_VAR (lsu_tcu_mem_if[b].rsp_data) + end `endif // frontend-slice icache requests; port 0 also carries the injected flush @@ -136,7 +156,7 @@ import VX_tcu_pkg::*; wire dcr_flush_l1_only; // the in-core global-store path is empty - wire mem_unit_empty; + wire mem_unit_empty = (& lsu_store_empty); // Stores are ack-less and lag warp exit, so the dcache flush waits until every // store of this core has reached L1. @@ -353,7 +373,7 @@ import VX_tcu_pkg::*; .meta_wr_wid (tcu_meta_wr_wid), .meta_wr_idx (tcu_meta_wr_idx), .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (tcu_mem_if), + .tcu_mem_if (lsu_tcu_mem_if[0]), `endif .clk (clk), .reset (reset) @@ -425,7 +445,21 @@ import VX_tcu_pkg::*; .satp_wr_valid (satp_wr_valid[i]), .satp_wr_data (satp_wr_data[i]), `endif - .lsu_client_if (lsu_client_if[i]), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire[i]), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), + .lsu_coalescer_misses (lsu_coalescer_misses[i]), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if[i]), + `endif + .lsu_sched_empty (lsu_sched_empty[i]), + .lsu_store_empty (lsu_store_empty[i]), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + `endif + .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (block_dxa_req_bus_if[i]), `endif @@ -647,7 +681,21 @@ import VX_tcu_pkg::*; `ifdef VX_CFG_EXT_RTU_ENABLE .sched_unlock_if (sched_unlock_if[i]), `endif - .lsu_client_if (lsu_client_if[i]), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire[i]), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), + .lsu_coalescer_misses (lsu_coalescer_misses[i]), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if[i]), + `endif + .lsu_sched_empty (lsu_sched_empty[i]), + .lsu_store_empty (lsu_store_empty[i]), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + `endif + .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (block_dxa_req_bus_if[i]), `endif @@ -727,9 +775,21 @@ import VX_tcu_pkg::*; .reset (reset), .core_id (core_id), - // execute exposes LSU client interfaces; lsu_scheduler sits between - // execute and lsu_mem_if (which connects to mem_unit). - .lsu_client_if (lsu_client_if), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire), + .lsu_coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if), + `endif + .lsu_sched_empty (lsu_sched_empty), + .lsu_store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if), + `endif + .dcache_bus_if (lsu_dcache_bus_if), .dispatch_if (dispatch_if), .commit_if (commit_if), @@ -791,60 +851,6 @@ import VX_tcu_pkg::*; end end - // One memory scheduler per LSU block. With the TCU metadata AGU, every block - // has a second client: block 0 serves the AGU and the others tie it off, which - // keeps the blocks identical at the cost of a few arbiter muxes. -`ifdef TCU_META_ENABLE - localparam LSU_SCHED_NUM_CLIENTS = 2; -`else - localparam LSU_SCHED_NUM_CLIENTS = 1; -`endif - for (genvar block_idx = 0; block_idx < `VX_CFG_NUM_LSU_BLOCKS; ++block_idx) begin : g_lsu_scheduler - VX_lsu_sched_if sched_client_if [LSU_SCHED_NUM_CLIENTS](); - - // client 0: the block's LSU - assign sched_client_if[0].req_valid = lsu_client_if[block_idx].req_valid; - assign sched_client_if[0].req_data = lsu_client_if[block_idx].req_data; - assign lsu_client_if[block_idx].req_ready = sched_client_if[0].req_ready; - assign lsu_client_if[block_idx].rsp_valid = sched_client_if[0].rsp_valid; - assign lsu_client_if[block_idx].rsp_data = sched_client_if[0].rsp_data; - assign sched_client_if[0].rsp_ready = lsu_client_if[block_idx].rsp_ready; - - `ifdef TCU_META_ENABLE - // client 1: the TCU metadata AGU on block 0 - if (block_idx == 0) begin : g_tcu_client - assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; - assign sched_client_if[1].req_data = tcu_mem_if.req_data; - assign tcu_mem_if.req_ready = sched_client_if[1].req_ready; - assign tcu_mem_if.rsp_valid = sched_client_if[1].rsp_valid; - assign tcu_mem_if.rsp_data = sched_client_if[1].rsp_data; - assign sched_client_if[1].rsp_ready = tcu_mem_if.rsp_ready; - end else begin : g_tcu_client_tieoff - assign sched_client_if[1].req_valid = 1'b0; - assign sched_client_if[1].req_data = '0; - assign sched_client_if[1].rsp_ready = 1'b1; - `UNUSED_VAR (sched_client_if[1].req_ready) - `UNUSED_VAR (sched_client_if[1].rsp_valid) - `UNUSED_VAR (sched_client_if[1].rsp_data) - end - `endif - - VX_lsu_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d", INSTANCE_ID, block_idx))), - .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .CORE_QUEUE_SIZE (`VX_CFG_LSU_QUEUE_IN_SIZE), - .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) - ) lsu_scheduler ( - .clk (clk), - .reset (reset), - .client_if (sched_client_if), - .empty (lsu_sched_empty[block_idx]), - .lsu_mem_if (lsu_mem_if[block_idx]) - ); - end - VX_mem_unit #( .INSTANCE_ID (INSTANCE_ID) ) mem_unit ( @@ -852,7 +858,6 @@ import VX_tcu_pkg::*; .reset (reset), `ifdef PERF_ENABLE .lmem_perf (lmem_perf), - .coalescer_perf (coalescer_perf), `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE .tcu_lmem_if (tcu_lmem_if), @@ -861,8 +866,10 @@ import VX_tcu_pkg::*; .dxa_lmem_bus_if (dxa_lmem_bus_if), .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif - .empty (mem_unit_empty), - .lsu_mem_if (lsu_mem_if), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if), + `endif + .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), .flush_l1_only (dcr_flush_l1_only), .dcache_bus_if (dcache_bus_if) @@ -926,13 +933,20 @@ import VX_tcu_pkg::*; wire [LSU_NUM_REQS-1:0] perf_dcache_wr_req_fire, perf_dcache_wr_req_fire_r; wire [LSU_NUM_REQS-1:0] perf_dcache_rsp_fire; - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_perf_dcache - for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_j - assign perf_dcache_rd_req_fire[i * `VX_CFG_NUM_LSU_LANES + j] = lsu_mem_if[i].req_valid && lsu_mem_if[i].req_data.mask[j] && lsu_mem_if[i].req_ready && ~lsu_mem_if[i].req_data.rw; - assign perf_dcache_wr_req_fire[i * `VX_CFG_NUM_LSU_LANES + j] = lsu_mem_if[i].req_valid && lsu_mem_if[i].req_data.mask[j] && lsu_mem_if[i].req_ready && lsu_mem_if[i].req_data.rw; - assign perf_dcache_rsp_fire[i * `VX_CFG_NUM_LSU_LANES + j] = lsu_mem_if[i].rsp_valid && lsu_mem_if[i].rsp_data.mask[j] && lsu_mem_if[i].rsp_ready; - end - end + assign perf_dcache_rd_req_fire = lsu_perf_rd_req_fire; + assign perf_dcache_wr_req_fire = lsu_perf_wr_req_fire; + assign perf_dcache_rsp_fire = lsu_perf_rsp_fire; + + wire [PERF_CTR_BITS-1:0] coalescer_misses; + VX_reduce_tree #( + .IN_W (PERF_CTR_BITS), + .N (`VX_CFG_NUM_LSU_BLOCKS), + .OP ("+") + ) coalescer_reduce ( + .data_in (lsu_coalescer_misses), + .data_out (coalescer_misses) + ); + `BUFFER(coalescer_perf.misses, coalescer_misses); `BUFFER(perf_dcache_rd_req_fire_r, perf_dcache_rd_req_fire); `BUFFER(perf_dcache_wr_req_fire_r, perf_dcache_wr_req_fire); diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 94e33c191a..8258555eaf 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -27,10 +27,22 @@ import VX_tcu_pkg::*; input wire [CORE_ID_WIDTH-1:0] core_id, - // Per-block LSU client connections to VX_lsu_scheduler (at VX_core). - // The dcache port (lsu_mem_if) lives at VX_core; execute exposes - // only the LSU client interfaces. - VX_lsu_sched_if.master lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS], + // LSU memory paths +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], +`endif + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty, +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], // dispatch interface VX_dispatch_if.slave dispatch_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], @@ -115,7 +127,21 @@ import VX_tcu_pkg::*; .core_id (core_id), .dispatch_if (dispatch_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), - .per_block_client_if (lsu_client_if) + `ifdef PERF_ENABLE + .perf_rd_req_fire (lsu_perf_rd_req_fire), + .perf_wr_req_fire (lsu_perf_wr_req_fire), + .perf_rsp_fire (lsu_perf_rsp_fire), + .coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if), + `endif + .sched_empty (lsu_sched_empty), + .store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if) ); `ifdef VX_CFG_EXT_F_ENABLE diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index 541c84efe7..be0a1d001d 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -45,7 +45,21 @@ import VX_tcu_pkg::*; VX_branch_ctl_if.master branch_ctl_if, // LSU - VX_lsu_sched_if.master lsu_client_if, +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave lsu_tcu_mem_if, +`endif + output wire lsu_sched_empty, + output wire lsu_store_empty, +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], // SFU `ifdef VX_CFG_EXT_DXA_ENABLE @@ -172,12 +186,26 @@ import VX_tcu_pkg::*; .INSTANCE_ID (`SFORMATF(("%s-lsu%s", INSTANCE_ID, SLICE_STR))) ) lsu_block ( `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), - .core_id (core_id), - .execute_if (lsu_execute_if[0]), - .result_if (lsu_result_if[0]), - .client_if (lsu_client_if) + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (lsu_execute_if[0]), + .result_if (lsu_result_if[0]), + `ifdef PERF_ENABLE + .perf_rd_req_fire (lsu_perf_rd_req_fire), + .perf_wr_req_fire (lsu_perf_wr_req_fire), + .perf_rsp_fire (lsu_perf_rsp_fire), + .coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if), + `endif + .sched_empty (lsu_sched_empty), + .store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if) ); VX_lane_gather #( diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index eebfad60a4..83c8ed4190 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -29,9 +29,24 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // Outputs VX_result_if.master result_if, - // The slice exposes the memory-client side as a VX_lsu_sched_if master; - // the parent wires it into VX_lsu_scheduler. - VX_lsu_sched_if.master client_if +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] coalescer_misses, +`endif + +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave tcu_mem_if, +`endif + + output wire sched_empty, + output wire store_empty, + +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS] ); localparam NUM_LANES = `VX_CFG_NUM_LSU_LANES; localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); @@ -51,7 +66,6 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `ifndef VX_CFG_EXT_A_ENABLE `UNUSED_VAR (core_id) `endif - `UNUSED_SPARAM (INSTANCE_ID) // tag_width = header + op_type + align + pkt_addr + fence localparam TAG_WIDTH = $bits(lsu_header_t) + INST_LSU_BITS + (NUM_LANES * REQ_ASHIFT) + LSUQ_SIZEW + 1; @@ -385,6 +399,31 @@ module VX_lsu_block import VX_gpu_pkg::*; #( req_is_fence }; + VX_lsu_sched_if client_if(); + + VX_lsu_mem_path #( + .INSTANCE_ID (INSTANCE_ID) + ) mem_path ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .perf_rd_req_fire (perf_rd_req_fire), + .perf_wr_req_fire (perf_wr_req_fire), + .perf_rsp_fire (perf_rsp_fire), + .coalescer_misses (coalescer_misses), + `endif + .client_if (client_if), + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if), + `endif + .sched_empty (sched_empty), + .store_empty (store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if) + ); + // Pack per-lane mem_req signals into the client_if request payload; // unpack the response payload into per-lane mem_rsp signals. assign client_if.req_valid = mem_req_valid; diff --git a/hw/rtl/core/VX_lsu_mem_path.sv b/hw/rtl/core/VX_lsu_mem_path.sv new file mode 100644 index 0000000000..7fd861ca86 --- /dev/null +++ b/hw/rtl/core/VX_lsu_mem_path.sv @@ -0,0 +1,287 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// The memory path of one LSU block: its memory scheduler, the global/local +// switch, the local-memory adapter, the coalescer and the D-cache adapter. +// The D-cache port carries one extra tag LSB: requests leave with it clear, and +// a response with it set is the core's flush token, not this block's. +module VX_lsu_mem_path import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] coalescer_misses, +`endif + + VX_lsu_sched_if.slave client_if, +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave tcu_mem_if, +`endif + + // the scheduler holds no request; the global-store path holds no store + output wire sched_empty, + output wire store_empty, + +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS] +); + `UNUSED_SPARAM (INSTANCE_ID) + + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_mem_if(); + + // scheduler: client 0 is the block's lanes, client 1 the TCU metadata AGU + +`ifdef TCU_META_ENABLE + localparam NUM_CLIENTS = 2; +`else + localparam NUM_CLIENTS = 1; +`endif + + VX_lsu_sched_if sched_client_if [NUM_CLIENTS](); + + assign sched_client_if[0].req_valid = client_if.req_valid; + assign sched_client_if[0].req_data = client_if.req_data; + assign client_if.req_ready = sched_client_if[0].req_ready; + assign client_if.rsp_valid = sched_client_if[0].rsp_valid; + assign client_if.rsp_data = sched_client_if[0].rsp_data; + assign sched_client_if[0].rsp_ready = client_if.rsp_ready; + +`ifdef TCU_META_ENABLE + assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; + assign sched_client_if[1].req_data = tcu_mem_if.req_data; + assign tcu_mem_if.req_ready = sched_client_if[1].req_ready; + assign tcu_mem_if.rsp_valid = sched_client_if[1].rsp_valid; + assign tcu_mem_if.rsp_data = sched_client_if[1].rsp_data; + assign sched_client_if[1].rsp_ready = tcu_mem_if.rsp_ready; +`endif + + VX_lsu_scheduler #( + .INSTANCE_ID (`SFORMATF(("%s-lsusched", INSTANCE_ID))), + .NUM_CLIENTS (NUM_CLIENTS), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .CORE_QUEUE_SIZE (`VX_CFG_LSU_QUEUE_IN_SIZE), + .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), + .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) + ) lsu_scheduler ( + .clk (clk), + .reset (reset), + .client_if (sched_client_if), + .empty (sched_empty), + .lsu_mem_if (lsu_mem_if) + ); + +`ifdef PERF_ENABLE + for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_perf + assign perf_rd_req_fire[j] = lsu_mem_if.req_valid && lsu_mem_if.req_data.mask[j] && lsu_mem_if.req_ready && ~lsu_mem_if.req_data.rw; + assign perf_wr_req_fire[j] = lsu_mem_if.req_valid && lsu_mem_if.req_data.mask[j] && lsu_mem_if.req_ready && lsu_mem_if.req_data.rw; + assign perf_rsp_fire[j] = lsu_mem_if.rsp_valid && lsu_mem_if.rsp_data.mask[j] && lsu_mem_if.rsp_ready; + end +`endif + + // Two terms keep a store visible until it reaches the D-cache port: the + // coalescer (or the adapter input without one), and the adapter's request + // buffer. + wire path_empty; + wire adapter_empty; + assign store_empty = path_empty && adapter_empty; + + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_dcache_if(); + +`ifdef VX_CFG_LMEM_ENABLE + + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_lmem_if(); + + VX_lmem_switch #( + .GLOBAL_OUT_BUF(1), + .LOCAL_OUT_BUF(1), + .RSP_OUT_BUF (1), + .ARBITER ("P") + ) lmem_switch ( + .clk (clk), + .reset (reset), + .lsu_in_if (lsu_mem_if), + .global_out_if(lsu_dcache_if), + .local_out_if (lsu_lmem_if) + ); + + // A dedicated local-memory adapter per block: sharing one adapter's + // unpack/pack buffers between blocks would deadlock. + VX_lsu_adapter #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH), + .TAG_SEL_BITS (LSU_TAG_WIDTH - UUID_WIDTH), + .ARBITER ("P"), + .REQ_OUT_BUF (3), + .RSP_OUT_BUF (0) + ) lmem_adapter ( + .clk (clk), + .reset (reset), + .lsu_mem_if (lsu_lmem_if), + .mem_bus_if (lmem_bus_if) + ); + +`else + + `ASSIGN_VX_MEM_BUS_IF (lsu_dcache_if, lsu_mem_if); + +`endif + + VX_lsu_mem_if #( + .NUM_LANES (DCACHE_CHANNELS), + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) + ) dcache_coalesced_if(); + + if ((`VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE)) begin : g_coalescing + + VX_mem_coalescer #( + .INSTANCE_ID (`SFORMATF(("%s-coalescer", INSTANCE_ID))), + .NUM_REQS (`VX_CFG_NUM_LSU_LANES), + .DATA_IN_SIZE (LSU_WORD_SIZE), + .DATA_OUT_SIZE (DCACHE_WORD_SIZE), + .ADDR_WIDTH (LSU_ADDR_WIDTH), + .USER_WIDTH (MEM_ATTR_WIDTH), + .TAG_WIDTH (LSU_TAG_WIDTH), + .UUID_WIDTH (UUID_WIDTH), + .QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), + .PERF_CTR_BITS (PERF_CTR_BITS) + ) mem_coalescer ( + .clk (clk), + .reset (reset), + + `ifdef PERF_ENABLE + .misses (coalescer_misses), + `else + `UNUSED_PIN (misses), + `endif + + .empty (path_empty), + + // Input request + .in_req_valid (lsu_dcache_if.req_valid), + .in_req_mask (lsu_dcache_if.req_data.mask), + .in_req_rw (lsu_dcache_if.req_data.rw), + .in_req_byteen (lsu_dcache_if.req_data.byteen), + .in_req_addr (lsu_dcache_if.req_data.addr), + .in_req_user (lsu_dcache_if.req_data.user), + .in_req_no_merge(lsu_dcache_if.req_data.user[0][MEM_ATTR_AMO_OFFS]), + .in_req_data (lsu_dcache_if.req_data.data), + .in_req_tag (lsu_dcache_if.req_data.tag), + .in_req_ready (lsu_dcache_if.req_ready), + + // Input response + .in_rsp_valid (lsu_dcache_if.rsp_valid), + .in_rsp_mask (lsu_dcache_if.rsp_data.mask), + .in_rsp_data (lsu_dcache_if.rsp_data.data), + .in_rsp_tag (lsu_dcache_if.rsp_data.tag), + .in_rsp_ready (lsu_dcache_if.rsp_ready), + + // Output request + .out_req_valid (dcache_coalesced_if.req_valid), + .out_req_mask (dcache_coalesced_if.req_data.mask), + .out_req_rw (dcache_coalesced_if.req_data.rw), + .out_req_byteen (dcache_coalesced_if.req_data.byteen), + .out_req_addr (dcache_coalesced_if.req_data.addr), + .out_req_user (dcache_coalesced_if.req_data.user), + .out_req_data (dcache_coalesced_if.req_data.data), + .out_req_tag (dcache_coalesced_if.req_data.tag), + .out_req_ready (dcache_coalesced_if.req_ready), + + // Output response + .out_rsp_valid (dcache_coalesced_if.rsp_valid), + .out_rsp_mask (dcache_coalesced_if.rsp_data.mask), + .out_rsp_data (dcache_coalesced_if.rsp_data.data), + .out_rsp_tag (dcache_coalesced_if.rsp_data.tag), + .out_rsp_ready (dcache_coalesced_if.rsp_ready) + ); + + end else begin : g_no_coalescing + + // The passthrough narrows the tag from LSU_TAG_WIDTH to + // DCACHE_CORE_TAG_WIDTH: the dcache tag-id space must cover the + // mem-scheduler slot id or responses alias. + `STATIC_ASSERT (DCACHE_CORE_TAG_WIDTH >= LSU_TAG_WIDTH, ("dcache tag-id space cannot hold the LSU outstanding slot id")) + + `ASSIGN_VX_MEM_BUS_IF (dcache_coalesced_if, lsu_dcache_if); + `ifdef PERF_ENABLE + assign coalescer_misses = '0; + `endif + // Without a coalescer, a store still upstream of the adapter is one + // presented at its input. + assign path_empty = ~dcache_coalesced_if.req_valid; + + end + + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) + ) dcache_adapter_if[DCACHE_CHANNELS](); + + VX_lsu_adapter #( + .NUM_LANES (DCACHE_CHANNELS), + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), + .TAG_SEL_BITS (DCACHE_CORE_TAG_WIDTH - UUID_WIDTH), + .ARBITER ("P"), + .REQ_OUT_BUF (3), + .RSP_OUT_BUF (0) + ) dcache_adapter ( + .clk (clk), + .reset (reset), + .lsu_mem_if (dcache_coalesced_if), + .mem_bus_if (dcache_adapter_if) + ); + + wire [DCACHE_CHANNELS-1:0] adapter_req_valid; + for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if + assign adapter_req_valid[j] = dcache_adapter_if[j].req_valid; + + assign dcache_bus_if[j].req_valid = dcache_adapter_if[j].req_valid; + assign dcache_bus_if[j].req_data.rw = dcache_adapter_if[j].req_data.rw; + assign dcache_bus_if[j].req_data.addr = dcache_adapter_if[j].req_data.addr; + assign dcache_bus_if[j].req_data.data = dcache_adapter_if[j].req_data.data; + assign dcache_bus_if[j].req_data.byteen = dcache_adapter_if[j].req_data.byteen; + assign dcache_bus_if[j].req_data.attr = dcache_adapter_if[j].req_data.attr; + assign dcache_bus_if[j].req_data.tag = {dcache_adapter_if[j].req_data.tag, 1'b0}; + assign dcache_adapter_if[j].req_ready = dcache_bus_if[j].req_ready; + + assign dcache_adapter_if[j].rsp_valid = dcache_bus_if[j].rsp_valid && ~dcache_bus_if[j].rsp_data.tag[0]; + assign dcache_adapter_if[j].rsp_data.data = dcache_bus_if[j].rsp_data.data; + assign dcache_adapter_if[j].rsp_data.tag = dcache_bus_if[j].rsp_data.tag[DCACHE_TAG_WIDTH-1:1]; + assign dcache_bus_if[j].rsp_ready = dcache_adapter_if[j].rsp_ready; + end + assign adapter_empty = ~(| adapter_req_valid); + +endmodule diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index 2623bc8162..e383dfe5c8 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -29,8 +29,24 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( // Outputs VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH], - // Per-block client connection to VX_lsu_scheduler (at VX_core). - VX_lsu_sched_if.master per_block_client_if [`VX_CFG_NUM_LSU_BLOCKS] +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] perf_rsp_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] coalescer_misses, +`endif + +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], +`endif + + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] sched_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] store_empty, + +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] ); localparam BLOCK_SIZE = `VX_CFG_NUM_LSU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_LSU_LANES; @@ -58,6 +74,24 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( ) per_block_result_if[BLOCK_SIZE](); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks + `ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if #( + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) block_lmem_bus_if[NUM_LANES](); + for (genvar j = 0; j < NUM_LANES; ++j) begin : g_lmem_bus_if + `ASSIGN_VX_MEM_BUS_IF (lmem_bus_if[block_idx * NUM_LANES + j], block_lmem_bus_if[j]); + end + `endif + + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) block_dcache_bus_if[DCACHE_CHANNELS](); + for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if + `ASSIGN_VX_MEM_BUS_IF (dcache_bus_if[block_idx * DCACHE_CHANNELS + j], block_dcache_bus_if[j]); + end + VX_lsu_block #( .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, block_idx))) ) lsu_block( @@ -67,7 +101,21 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( .core_id (core_id), .execute_if (per_block_execute_if[block_idx]), .result_if (per_block_result_if[block_idx]), - .client_if (per_block_client_if[block_idx]) + `ifdef PERF_ENABLE + .perf_rd_req_fire (perf_rd_req_fire[block_idx]), + .perf_wr_req_fire (perf_wr_req_fire[block_idx]), + .perf_rsp_fire (perf_rsp_fire[block_idx]), + .coalescer_misses (coalescer_misses[block_idx]), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if[block_idx]), + `endif + .sched_empty (sched_empty[block_idx]), + .store_empty (store_empty[block_idx]), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (block_lmem_bus_if), + `endif + .dcache_bus_if (block_dcache_bus_if) ); end diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 17b5d7c665..a72e33c8c0 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -21,7 +21,6 @@ module VX_mem_unit import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE output lmem_perf_t lmem_perf, - output coalescer_perf_t coalescer_perf, `endif `ifdef VX_CFG_EXT_DXA_ENABLE @@ -34,28 +33,19 @@ module VX_mem_unit import VX_gpu_pkg::*; #( VX_mem_bus_if.slave tcu_lmem_if, `endif - output wire empty, + // the LSU blocks' memory paths +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.slave lmem_bus_if [LSU_NUM_REQS], +`endif + VX_mem_bus_if.slave lsu_dcache_bus_if [DCACHE_NUM_REQS], - VX_lsu_mem_if.slave lsu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], VX_dcr_flush_if.slave dcr_flush_if, input wire flush_l1_only, VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] ); - // Per-block empty: high when the global-store path holds no in-flight - // write. Folded into core `busy` so the end-of-kernel cache flush is - // ordered strictly behind the last store. Two terms: the coalescer/passthru - // producer, and the dcache adapter's request buffer (which carries state - // when REQ_OUT_BUF > 0 and would otherwise hide a still-settling store). - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] per_block_path_empty; - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] per_block_adapter_empty; - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] per_block_empty; - assign per_block_empty = per_block_path_empty & per_block_adapter_empty; - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_dcache_if[`VX_CFG_NUM_LSU_BLOCKS](); - +`ifndef VX_CFG_LMEM_ENABLE + `UNUSED_SPARAM (INSTANCE_ID) +`endif `ifdef VX_CFG_TCU_WGMMA_ENABLE `STATIC_ASSERT(`VX_CFG_LMEM_ENABLED, ("TCU_WGMMA_ENABLE requires LMEM_ENABLE")) // The WGMMA shared-memory descriptor carries the LMEM byte offset in a 16-bit @@ -71,63 +61,6 @@ module VX_mem_unit import VX_gpu_pkg::*; #( localparam LMEM_ADDR_WIDTH = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE); - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lmem_switches - VX_lmem_switch #( - .GLOBAL_OUT_BUF(1), - .LOCAL_OUT_BUF(1), - .RSP_OUT_BUF (1), - .ARBITER ("P") - ) lmem_switch ( - .clk (clk), - .reset (reset), - .lsu_in_if (lsu_mem_if[i]), - .global_out_if(lsu_dcache_if[i]), - .local_out_if (lsu_lmem_if[i]) - ); - end - - // Per-block local memory adapters to avoid deadlock when NUM_LSU_BLOCKS > 1. - // Each block gets dedicated ports into local memory, eliminating the circular - // dependency that arises from sharing a single adapter's unpack/pack buffers. - - VX_mem_bus_if #( - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lmem_adapt_if[LSU_NUM_REQS](); - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lmem_adapters - - VX_mem_bus_if #( - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lmem_block_if[`VX_CFG_NUM_LSU_LANES](); - - VX_lsu_adapter #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH), - .TAG_SEL_BITS (LSU_TAG_WIDTH - UUID_WIDTH), - .ARBITER ("P"), - .REQ_OUT_BUF (3), - .RSP_OUT_BUF (0) - ) lmem_adapter ( - .clk (clk), - .reset (reset), - .lsu_mem_if (lsu_lmem_if[i]), - .mem_bus_if (lmem_block_if) - ); - - for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_lmem_adapt_if - `ASSIGN_VX_MEM_BUS_IF (lmem_adapt_if[i * `VX_CFG_NUM_LSU_LANES + j], lmem_block_if[j]); - end - end - // DMA arbiter: merge DXA writes and/or TCU reads onto single DMA port. VX_mem_bus_if #( @@ -247,127 +180,17 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .lmem_perf (lmem_perf), `endif .dma_bus_if (lmem_dma_if), - .lsu_bus_if (lmem_adapt_if) + .lsu_bus_if (lmem_bus_if) ); `else - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_dcache_if - `ASSIGN_VX_MEM_BUS_IF (lsu_dcache_if[i], lsu_mem_if[i]); - end - `ifdef PERF_ENABLE assign lmem_perf = '0; `endif `endif - VX_lsu_mem_if #( - .NUM_LANES (DCACHE_CHANNELS), - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) - ) dcache_coalesced_if[`VX_CFG_NUM_LSU_BLOCKS](); - -`ifdef PERF_ENABLE - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] per_block_coalescer_misses; - wire [PERF_CTR_BITS-1:0] coalescer_misses; - VX_reduce_tree #( - .IN_W (PERF_CTR_BITS), - .N (`VX_CFG_NUM_LSU_BLOCKS), - .OP ("+") - ) coalescer_reduce ( - .data_in (per_block_coalescer_misses), - .data_out (coalescer_misses) - ); - `BUFFER(coalescer_perf.misses, coalescer_misses); -`endif - - if ((`VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE)) begin : g_coalescing - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_coalescers - - VX_mem_coalescer #( - .INSTANCE_ID (`SFORMATF(("%s-coalescer%0d", INSTANCE_ID, i))), - .NUM_REQS (`VX_CFG_NUM_LSU_LANES), - .DATA_IN_SIZE (LSU_WORD_SIZE), - .DATA_OUT_SIZE (DCACHE_WORD_SIZE), - .ADDR_WIDTH (LSU_ADDR_WIDTH), - .USER_WIDTH (MEM_ATTR_WIDTH), - .TAG_WIDTH (LSU_TAG_WIDTH), - .UUID_WIDTH (UUID_WIDTH), - .QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PERF_CTR_BITS (PERF_CTR_BITS) - ) mem_coalescer ( - .clk (clk), - .reset (reset), - - `ifdef PERF_ENABLE - .misses (per_block_coalescer_misses[i]), - `else - `UNUSED_PIN (misses), - `endif - - .empty (per_block_path_empty[i]), - - // Input request - .in_req_valid (lsu_dcache_if[i].req_valid), - .in_req_mask (lsu_dcache_if[i].req_data.mask), - .in_req_rw (lsu_dcache_if[i].req_data.rw), - .in_req_byteen (lsu_dcache_if[i].req_data.byteen), - .in_req_addr (lsu_dcache_if[i].req_data.addr), - .in_req_user (lsu_dcache_if[i].req_data.user), - .in_req_no_merge(lsu_dcache_if[i].req_data.user[0][MEM_ATTR_AMO_OFFS]), - .in_req_data (lsu_dcache_if[i].req_data.data), - .in_req_tag (lsu_dcache_if[i].req_data.tag), - .in_req_ready (lsu_dcache_if[i].req_ready), - - // Input response - .in_rsp_valid (lsu_dcache_if[i].rsp_valid), - .in_rsp_mask (lsu_dcache_if[i].rsp_data.mask), - .in_rsp_data (lsu_dcache_if[i].rsp_data.data), - .in_rsp_tag (lsu_dcache_if[i].rsp_data.tag), - .in_rsp_ready (lsu_dcache_if[i].rsp_ready), - - // Output request - .out_req_valid (dcache_coalesced_if[i].req_valid), - .out_req_mask (dcache_coalesced_if[i].req_data.mask), - .out_req_rw (dcache_coalesced_if[i].req_data.rw), - .out_req_byteen (dcache_coalesced_if[i].req_data.byteen), - .out_req_addr (dcache_coalesced_if[i].req_data.addr), - .out_req_user (dcache_coalesced_if[i].req_data.user), - .out_req_data (dcache_coalesced_if[i].req_data.data), - .out_req_tag (dcache_coalesced_if[i].req_data.tag), - .out_req_ready (dcache_coalesced_if[i].req_ready), - - // Output response - .out_rsp_valid (dcache_coalesced_if[i].rsp_valid), - .out_rsp_mask (dcache_coalesced_if[i].rsp_data.mask), - .out_rsp_data (dcache_coalesced_if[i].rsp_data.data), - .out_rsp_tag (dcache_coalesced_if[i].rsp_data.tag), - .out_rsp_ready (dcache_coalesced_if[i].rsp_ready) - ); - end - - end else begin : g_no_coalescing - - // The passthrough narrows the tag from LSU_TAG_WIDTH to - // DCACHE_CORE_TAG_WIDTH: the dcache tag-id space must cover the - // mem-scheduler slot id or responses alias. - `STATIC_ASSERT (DCACHE_CORE_TAG_WIDTH >= LSU_TAG_WIDTH, ("dcache tag-id space cannot hold the LSU outstanding slot id")) - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_dcache_coalesced_if - `ASSIGN_VX_MEM_BUS_IF (dcache_coalesced_if[i], lsu_dcache_if[i]); - `ifdef PERF_ENABLE - assign per_block_coalescer_misses[i] = '0; - `endif - // No coalescer: a store still in flight upstream of the adapter is - // one being presented to the adapter input. The adapter's own buffer - // is accounted for separately by per_block_adapter_empty. - assign per_block_path_empty[i] = ~dcache_coalesced_if[i].req_valid; - end - - end - // A private dcache takes the lanes straight from the adapter, whose drain // store_drained already waits for, so one token suffices. Otherwise (a // shared dcache, or none) the lanes are buffered independently before the @@ -384,61 +207,62 @@ module VX_mem_unit import VX_gpu_pkg::*; #( end assign dcr_flush_if.done = (& lane_flush_done); - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_dcache_adapters - - VX_mem_bus_if #( - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) - ) dcache_bus_tmp_if[DCACHE_CHANNELS](); - - VX_lsu_adapter #( - .NUM_LANES (DCACHE_CHANNELS), - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), - .TAG_SEL_BITS (DCACHE_CORE_TAG_WIDTH - UUID_WIDTH), - .ARBITER ("P"), - .REQ_OUT_BUF (3), // register dcache-request master boundary on all channels (passthru ports 1+) - .RSP_OUT_BUF (0) - ) dcache_adapter ( - .clk (clk), - .reset (reset), - .lsu_mem_if (dcache_coalesced_if[i]), - .mem_bus_if (dcache_bus_tmp_if) - ); - - // A store buffered in the adapter's REQ_OUT_BUF has left dcache_coalesced_if - // but not yet reached the flush-injection point; keep the block non-empty - // until every channel's request has drained past the adapter output. - wire [DCACHE_CHANNELS-1:0] adapter_req_valid; - for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_adapter_req_valid - assign adapter_req_valid[j] = dcache_bus_tmp_if[j].req_valid; - end - assign per_block_adapter_empty[i] = ~(| adapter_req_valid); - - for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if - localparam LANE = i * DCACHE_CHANNELS + j; - if (LANE < FLUSH_LANES) begin : g_flush_lane - VX_dcr_flush #( - .WORD_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), - .REQ_OUT_BUF (3) // register core dcache-request boundary; rsp already registered by L1 CORE_OUT_BUF - ) dcr_flush ( - .clk (clk), - .reset (reset), - .dcr_flush_if (lane_flush_if[LANE]), - .l1_only (flush_l1_only), - .core_bus_if (dcache_bus_tmp_if[j]), - .cache_bus_if (dcache_bus_if[LANE]) - ); - end else begin : g_passthru_lane - // tag is zero-extended from DCACHE_CORE_TAG_WIDTH to - // DCACHE_TAG_WIDTH on request, and MSB-stripped on response - `ASSIGN_VX_MEM_BUS_IF_EX (dcache_bus_if[LANE], dcache_bus_tmp_if[j], DCACHE_TAG_WIDTH_BASE, DCACHE_CORE_TAG_WIDTH, 0); - end + // A block's D-cache request carries a clear tag LSB, the flush arbiter's + // select bit; the block takes the cache response directly and drops the + // flush token's. + for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_bus_if + if (i < FLUSH_LANES) begin : g_flush_lane + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) + ) flush_core_if(); + + assign flush_core_if.req_valid = lsu_dcache_bus_if[i].req_valid; + assign flush_core_if.req_data.rw = lsu_dcache_bus_if[i].req_data.rw; + assign flush_core_if.req_data.addr = lsu_dcache_bus_if[i].req_data.addr; + assign flush_core_if.req_data.data = lsu_dcache_bus_if[i].req_data.data; + assign flush_core_if.req_data.byteen = lsu_dcache_bus_if[i].req_data.byteen; + assign flush_core_if.req_data.attr = lsu_dcache_bus_if[i].req_data.attr; + assign flush_core_if.req_data.tag = lsu_dcache_bus_if[i].req_data.tag[DCACHE_TAG_WIDTH-1:1]; + assign lsu_dcache_bus_if[i].req_ready = flush_core_if.req_ready; + `UNUSED_VAR (lsu_dcache_bus_if[i].req_data.tag[0]) + + assign lsu_dcache_bus_if[i].rsp_valid = dcache_bus_if[i].rsp_valid; + assign lsu_dcache_bus_if[i].rsp_data = dcache_bus_if[i].rsp_data; + assign flush_core_if.rsp_ready = lsu_dcache_bus_if[i].rsp_ready; + `UNUSED_VAR (flush_core_if.rsp_valid) + `UNUSED_VAR (flush_core_if.rsp_data) + + VX_dcr_flush #( + .WORD_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), + .REQ_OUT_BUF (3) // register core dcache-request boundary; rsp already registered by L1 CORE_OUT_BUF + ) dcr_flush ( + .clk (clk), + .reset (reset), + .dcr_flush_if (lane_flush_if[i]), + .l1_only (flush_l1_only), + .core_bus_if (flush_core_if), + .cache_bus_if (dcache_bus_if[i]) + ); + end else begin : g_passthru_lane + // the cache sees the select bit as a clear tag MSB + assign dcache_bus_if[i].req_valid = lsu_dcache_bus_if[i].req_valid; + assign dcache_bus_if[i].req_data.rw = lsu_dcache_bus_if[i].req_data.rw; + assign dcache_bus_if[i].req_data.addr = lsu_dcache_bus_if[i].req_data.addr; + assign dcache_bus_if[i].req_data.data = lsu_dcache_bus_if[i].req_data.data; + assign dcache_bus_if[i].req_data.byteen = lsu_dcache_bus_if[i].req_data.byteen; + assign dcache_bus_if[i].req_data.attr = lsu_dcache_bus_if[i].req_data.attr; + assign dcache_bus_if[i].req_data.tag = {1'b0, lsu_dcache_bus_if[i].req_data.tag[DCACHE_TAG_WIDTH-1:1]}; + assign lsu_dcache_bus_if[i].req_ready = dcache_bus_if[i].req_ready; + `UNUSED_VAR (lsu_dcache_bus_if[i].req_data.tag[0]) + + assign lsu_dcache_bus_if[i].rsp_valid = dcache_bus_if[i].rsp_valid; + assign lsu_dcache_bus_if[i].rsp_data.data = dcache_bus_if[i].rsp_data.data; + assign lsu_dcache_bus_if[i].rsp_data.tag = {dcache_bus_if[i].rsp_data.tag[DCACHE_TAG_WIDTH-2:0], 1'b0}; + assign dcache_bus_if[i].rsp_ready = lsu_dcache_bus_if[i].rsp_ready; + `UNUSED_VAR (dcache_bus_if[i].rsp_data.tag[DCACHE_TAG_WIDTH-1]) end - end - assign empty = (& per_block_empty); - endmodule diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index ad2c75034e..ed556ebf4c 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -14,29 +14,19 @@ `include "VX_define.vh" module VX_mem_unit_top import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter LSU_WORD_WIDTH = LSU_WORD_SIZE * 8 + parameter `STRING INSTANCE_ID = "" ) ( // Clock input wire clk, input wire reset, - // LSU memory request + // LSU block memory clients input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_valid, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_rw, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_req_mask, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_WORD_SIZE-1:0] lsu_req_byteen, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_ADDR_WIDTH-1:0] lsu_req_addr, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][MEM_ATTR_WIDTH-1:0] lsu_req_user, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_WORD_WIDTH-1:0] lsu_req_data, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][LSU_TAG_WIDTH-1:0] lsu_req_tag, + input lsu_req_data_t [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_data, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_ready, - // LSU memory response output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_rsp_valid, - output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_rsp_mask, - output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_WORD_WIDTH-1:0] lsu_rsp_data, - output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][LSU_TAG_WIDTH-1:0] lsu_rsp_tag, + output lsu_rsp_data_t [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_rsp_data, input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_rsp_ready, // Memory request @@ -55,32 +45,60 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( input wire [DCACHE_NUM_REQS-1:0][DCACHE_TAG_WIDTH-1:0] mem_rsp_tag, output wire [DCACHE_NUM_REQS-1:0] mem_rsp_ready ); - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if #( .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_mem_if[`VX_CFG_NUM_LSU_BLOCKS](); - - // LSU memory request - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_mem_req - assign lsu_mem_if[i].req_valid = lsu_req_valid[i]; - assign lsu_mem_if[i].req_data.rw = lsu_req_rw[i]; - assign lsu_mem_if[i].req_data.mask = lsu_req_mask[i]; - assign lsu_mem_if[i].req_data.byteen = lsu_req_byteen[i]; - assign lsu_mem_if[i].req_data.addr = lsu_req_addr[i]; - assign lsu_mem_if[i].req_data.user = lsu_req_user[i]; - assign lsu_mem_if[i].req_data.data = lsu_req_data[i]; - assign lsu_mem_if[i].req_data.tag = lsu_req_tag[i]; - assign lsu_req_ready[i] = lsu_mem_if[i].req_ready; - end + ) lsu_lmem_bus_if[LSU_NUM_REQS](); +`endif + + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) lsu_dcache_bus_if[DCACHE_NUM_REQS](); + + for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_mem_paths + VX_lsu_sched_if client_if(); + + assign client_if.req_valid = lsu_req_valid[i]; + assign client_if.req_data = lsu_req_data[i]; + assign lsu_req_ready[i] = client_if.req_ready; + assign lsu_rsp_valid[i] = client_if.rsp_valid; + assign lsu_rsp_data[i] = client_if.rsp_data; + assign client_if.rsp_ready = lsu_rsp_ready[i]; + + `ifdef TCU_META_ENABLE + VX_lsu_sched_if tcu_mem_if(); + assign tcu_mem_if.req_valid = 1'b0; + assign tcu_mem_if.req_data = '0; + assign tcu_mem_if.rsp_ready = 1'b1; + `UNUSED_VAR (tcu_mem_if.req_ready) + `UNUSED_VAR (tcu_mem_if.rsp_valid) + `UNUSED_VAR (tcu_mem_if.rsp_data) + `endif - // LSU memory response - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_rsp - assign lsu_rsp_valid[i] = lsu_mem_if[i].rsp_valid; - assign lsu_rsp_mask[i] = lsu_mem_if[i].rsp_data.mask; - assign lsu_rsp_data[i] = lsu_mem_if[i].rsp_data.data; - assign lsu_rsp_tag[i] = lsu_mem_if[i].rsp_data.tag; - assign lsu_mem_if[i].rsp_ready = lsu_rsp_ready[i]; + VX_lsu_mem_path #( + .INSTANCE_ID (`SFORMATF(("%s-lsu%0d", INSTANCE_ID, i))) + ) mem_path ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (perf_rd_req_fire), + `UNUSED_PIN (perf_wr_req_fire), + `UNUSED_PIN (perf_rsp_fire), + `UNUSED_PIN (coalescer_misses), + `endif + .client_if (client_if), + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if), + `endif + `UNUSED_PIN (sched_empty), + `UNUSED_PIN (store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + `endif + .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]) + ); end VX_mem_bus_if #( @@ -116,9 +134,6 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( assign dcr_flush_if.req = 1'b0; `UNUSED_VAR (dcr_flush_if.done) - wire mem_unit_empty; - `UNUSED_VAR (mem_unit_empty) - VX_mem_unit #( .INSTANCE_ID (INSTANCE_ID) ) mem_unit ( @@ -127,8 +142,10 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE .lmem_perf (lmem_perf), `endif - .empty (mem_unit_empty), - .lsu_mem_if (lsu_mem_if), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if), + `endif + .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_if), .flush_l1_only (1'b0), .dcache_bus_if (mem_bus_if) diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index e56e5c14b6..05037a0908 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -21,10 +21,11 @@ // per core (VX_CFG_SOCKET_SIZE): // VX_tcu_unit : tensor-core unit (dispatch/commit from top ports) // VX_tcu_shared : WGMMA tile buffers + metadata AGU -// VX_lsu_scheduler : per-LSU-block, 2 clients — the ordinary LSU stream -// (top ports) and the TCU AGU metadata-load port -// VX_mem_unit : LMEM + coalescer + LMEM-DMA arb (TCU WGMMA + DXA -// staging converge here) + dcache adapters +// VX_lsu_mem_path : per-LSU-block scheduler (2 clients — the ordinary LSU +// stream from top ports and the TCU AGU metadata-load +// port), lmem switch, coalescer and adapters +// VX_mem_unit : LMEM + LMEM-DMA arb (TCU WGMMA + DXA staging +// converge here) + dcache flush injectors // per socket: // VX_dxa_core : descriptor-driven tile-transfer engine (DCR-fed) // VX_mem_bus_switch : DXA staging writes back to the owning core's LMEM @@ -36,7 +37,7 @@ // // Memory hierarchy: TCU WGMMA prefetches and DXA staging writes meet at the // real LMEM DMA arbiter inside VX_mem_unit; TCU_LD metadata loads merge with -// the ordinary LSU stream at block-0's VX_lsu_scheduler; global traffic +// the ordinary LSU stream at block-0's scheduler; global traffic // (dcache refills, DXA tile reads) contends at the socket arbiter and the // shared L2, whose memory side exits to top-level ports. // @@ -74,7 +75,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( // ----------------------------------------------------------------------- // Ordinary LSU client streams, flattened [core][lsu-block] - // (client 0 of each VX_lsu_scheduler; the TCU AGU is client 1 on block 0) + // (client 0 of each LSU block's scheduler; the TCU AGU is client 1 on block 0) // ----------------------------------------------------------------------- input wire [LSU_STREAMS-1:0] lsu_req_valid, input lsu_req_data_t [LSU_STREAMS-1:0] lsu_req_data, @@ -121,12 +122,6 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( ); localparam TCU_LMEM_BANK_ADDR_W = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE) - `CLOG2(`VX_CFG_LMEM_NUM_BANKS); -`ifdef TCU_META_ENABLE - localparam LSU_SCHED_NUM_CLIENTS = 2; -`else - localparam LSU_SCHED_NUM_CLIENTS = 1; -`endif - // ------------------------------------------------------------------------ // Per-core DXA staging returns + collected dcache core ports // ------------------------------------------------------------------------ @@ -149,7 +144,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( wire [NC-1:0] lsu_sched_empty; // ======================================================================== - // Per-core tensor slice: TCU + LSU schedulers + mem_unit + // Per-core tensor slice: TCU + LSU memory paths + mem_unit // ======================================================================== for (genvar c = 0; c < NC; ++c) begin : g_cores @@ -173,7 +168,15 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( ) tcu_lmem_if(); `ifdef TCU_META_ENABLE - VX_lsu_sched_if tcu_mem_if(); + VX_lsu_sched_if lsu_tcu_mem_if [NUM_LSU_BLKS](); + for (genvar block_idx = 1; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_tcu_client_tieoff + assign lsu_tcu_mem_if[block_idx].req_valid = 1'b0; + assign lsu_tcu_mem_if[block_idx].req_data = '0; + assign lsu_tcu_mem_if[block_idx].rsp_ready = 1'b1; + `UNUSED_VAR (lsu_tcu_mem_if[block_idx].req_ready) + `UNUSED_VAR (lsu_tcu_mem_if[block_idx].rsp_valid) + `UNUSED_VAR (lsu_tcu_mem_if[block_idx].rsp_data) + end `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE @@ -249,70 +252,65 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .meta_wr_wid (tcu_meta_wr_wid), .meta_wr_idx (tcu_meta_wr_idx), .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (tcu_mem_if), + .tcu_mem_if (lsu_tcu_mem_if[0]), `endif .clk (clk), .reset (reset) ); - // --- LSU schedulers: client 0 from top ports, client 1 = TCU AGU --- - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_mem_if[NUM_LSU_BLKS](); - + // --- LSU memory paths: client 0 from top ports, client 1 = TCU AGU --- wire [NUM_LSU_BLKS-1:0] block_sched_empty; + wire [NUM_LSU_BLKS-1:0] block_store_empty; - for (genvar block_idx = 0; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_lsu_scheduler - localparam s = c * NUM_LSU_BLKS + block_idx; + VX_mem_bus_if #( + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_lmem_bus_if[LSU_NUM_REQS](); - VX_lsu_sched_if sched_client_if [LSU_SCHED_NUM_CLIENTS](); + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) lsu_dcache_bus_if[DCACHE_NUM_REQS](); - assign sched_client_if[0].req_valid = lsu_req_valid[s]; - assign sched_client_if[0].req_data = lsu_req_data[s]; - assign lsu_req_ready[s] = sched_client_if[0].req_ready; - assign lsu_rsp_valid[s] = sched_client_if[0].rsp_valid; - assign lsu_rsp_data[s] = sched_client_if[0].rsp_data; - assign sched_client_if[0].rsp_ready = lsu_rsp_ready[s]; - `ifdef TCU_META_ENABLE - if (block_idx == 0) begin : g_tcu_client - assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; - assign sched_client_if[1].req_data = tcu_mem_if.req_data; - assign tcu_mem_if.req_ready = sched_client_if[1].req_ready; - assign tcu_mem_if.rsp_valid = sched_client_if[1].rsp_valid; - assign tcu_mem_if.rsp_data = sched_client_if[1].rsp_data; - assign sched_client_if[1].rsp_ready = tcu_mem_if.rsp_ready; - end else begin : g_tcu_client_tieoff - assign sched_client_if[1].req_valid = 1'b0; - assign sched_client_if[1].req_data = '0; - assign sched_client_if[1].rsp_ready = 1'b1; - `UNUSED_VAR (sched_client_if[1].req_ready) - `UNUSED_VAR (sched_client_if[1].rsp_valid) - `UNUSED_VAR (sched_client_if[1].rsp_data) - end - `endif + for (genvar block_idx = 0; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_lsu_mem_path + localparam s = c * NUM_LSU_BLKS + block_idx; - VX_lsu_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d-%0d", INSTANCE_ID, c, block_idx))), - .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .CORE_QUEUE_SIZE(`VX_CFG_LSU_QUEUE_IN_SIZE), - .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) - ) lsu_scheduler ( - .clk (clk), - .reset (reset), - .client_if (sched_client_if), - .empty (block_sched_empty[block_idx]), - .lsu_mem_if (lsu_mem_if[block_idx]) + VX_lsu_sched_if client_if(); + + assign client_if.req_valid = lsu_req_valid[s]; + assign client_if.req_data = lsu_req_data[s]; + assign lsu_req_ready[s] = client_if.req_ready; + assign lsu_rsp_valid[s] = client_if.rsp_valid; + assign lsu_rsp_data[s] = client_if.rsp_data; + assign client_if.rsp_ready = lsu_rsp_ready[s]; + + VX_lsu_mem_path #( + .INSTANCE_ID (`SFORMATF(("%s-lsu%0d-%0d", INSTANCE_ID, c, block_idx))) + ) mem_path ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (perf_rd_req_fire), + `UNUSED_PIN (perf_wr_req_fire), + `UNUSED_PIN (perf_rsp_fire), + `UNUSED_PIN (coalescer_misses), + `endif + .client_if (client_if), + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if[block_idx]), + `endif + .sched_empty (block_sched_empty[block_idx]), + .store_empty (block_store_empty[block_idx]), + .lmem_bus_if (lsu_lmem_bus_if[block_idx * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + .dcache_bus_if (lsu_dcache_bus_if[block_idx * DCACHE_CHANNELS +: DCACHE_CHANNELS]) ); end assign lsu_sched_empty[c] = & block_sched_empty; + assign mem_unit_empty[c] = & block_store_empty; - // --- mem_unit: LMEM + DMA arb + dcache adapters --- + // --- mem_unit: LMEM + DMA arb + dcache flush injectors --- VX_txbar_bus_if dxa_txbar_bus_if(); assign txbar_valid[c] = dxa_txbar_bus_if.valid; assign txbar_data[c] = dxa_txbar_bus_if.data; @@ -339,8 +337,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .dxa_lmem_bus_if (per_core_dxa_lmem_bus_if[c]), .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif - .empty (mem_unit_empty[c]), - .lsu_mem_if (lsu_mem_if), + .lmem_bus_if (lsu_lmem_bus_if), + .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), .flush_l1_only (1'b0), .dcache_bus_if (dcache_bus_if) From 304e0db175998cbff826db1d7c9a7e359e48c7f8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 02:15:49 -0700 Subject: [PATCH 085/336] core: export the LSU blocks' status bits instead of combining them in the core With the memory path inside each LSU block, the core would build busy and lsu_drained from other modules' outputs in the same cycle. Each block now exports sched_empty, store_empty and busy. Every slice ANDs the sched_empty vector, with mmu_empty, next to its warp control; the socket ORs the blocks' busy vector into its registered busy, beside the core's own terms; store_drained stays in the core, feeding its flush injectors. Each AND and OR is the same function as before, so no status signal moves in any cycle. rtlsim sgemm_tcu_wg_sp_dxa -m 128 -n 128 -k 128: cycle-identical to the parent (144,956), PASSED. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_socket.sv | 4 +- hw/rtl/core/VX_backend_slice.sv | 2 + hw/rtl/core/VX_compute_slice.sv | 8 +++- hw/rtl/core/VX_core.sv | 53 +++++++++++++------------ hw/rtl/core/VX_execute.sv | 2 + hw/rtl/core/VX_execute_slice.sv | 2 + hw/rtl/core/VX_frontend_xbar.sv | 6 ++- hw/rtl/core/VX_lsu_block.sv | 2 + hw/rtl/core/VX_lsu_mem_path.sv | 2 + hw/rtl/core/VX_lsu_unit.sv | 2 + hw/rtl/core/VX_mem_unit.sv | 11 ++++- hw/rtl/core/VX_wctl_unit.sv | 3 +- hw/rtl/interfaces/VX_warp_ctl_if.sv | 20 ++++++---- hw/unittest/core/VX_core_top.sv | 7 +++- hw/unittest/mem_unit/VX_mem_unit_top.sv | 4 +- hw/unittest/tensor/VX_tensor_top.sv | 4 +- 16 files changed, 88 insertions(+), 44 deletions(-) diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 69c111f121..3df30eaef2 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -842,6 +842,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; /////////////////////////////////////////////////////////////////////////// wire [`VX_CFG_SOCKET_SIZE-1:0] per_core_busy; + wire [`VX_CFG_SOCKET_SIZE-1:0][`VX_CFG_NUM_LSU_BLOCKS-1:0] per_core_lsu_busy; `ifdef EXT_GFX_ANY_ENABLE // Core flush requests fan out to the socket-local gfx caches (tcache, // rtcache) and up to the cluster (rcache, ocache); each leg's done is @@ -936,6 +937,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .gbar_bus_if (per_core_gbar_bus_if[core_id]), + .lsu_busy (per_core_lsu_busy[core_id]), .busy (per_core_busy[core_id]) ); end @@ -960,7 +962,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; wire mem_req_empty = icache_req_empty && dcache_req_empty && tcache_req_empty && rtcache_req_empty && mem_arb_empty; wire busy_r; - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy) | ~mem_req_empty, 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy) | (|per_core_lsu_busy) | ~mem_req_empty, 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); assign busy = busy_r | dcr_bus_if.req_valid | kmu_bus_if[0].valid | (|per_core_kmu_valid); endmodule diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv index acc5767a67..24e380cae0 100644 --- a/hw/rtl/core/VX_backend_slice.sv +++ b/hw/rtl/core/VX_backend_slice.sv @@ -67,6 +67,7 @@ import VX_tcu_pkg::*; `endif output wire lsu_sched_empty, output wire lsu_store_empty, + output wire lsu_busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], `endif @@ -166,6 +167,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), + .lsu_busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index 63be670437..ddf14588e9 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -49,7 +49,8 @@ import VX_tcu_pkg::*; input wire [PERF_CTR_BITS-1:0] cycles, input wire [PERF_CTR_BITS-1:0] instret, input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, - input wire lsu_sched_drained, + input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] all_lsu_sched_empty, + input wire mmu_empty, output wire [NCTA_WIDTH-1:0] csr_rd_cta_id, input cta_ctx_t csr_rd_ctx, `ifdef VX_CFG_VM_ENABLE @@ -70,6 +71,7 @@ import VX_tcu_pkg::*; `endif output wire lsu_sched_empty, output wire lsu_store_empty, + output wire lsu_busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], `endif @@ -201,7 +203,8 @@ import VX_tcu_pkg::*; assign warp_ctl_if.dvstack_ptr = dvstack_ptr[0]; assign warp_ctl_if.mscratch = dvstack_mscratch[0]; assign warp_ctl_if.warp_pending_alm_empty = pending_alm_empty; - assign warp_ctl_if.lsu_sched_drained = lsu_sched_drained; + assign warp_ctl_if.lsu_sched_empty = all_lsu_sched_empty; + assign warp_ctl_if.mmu_empty = mmu_empty; assign csr_rd_wid[0] = sched_csr_if.csr_rd_wid; assign csr_rd_cta_id = sched_csr_if.csr_rd_cta_id; @@ -276,6 +279,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), + .lsu_busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index aee719c076..7ef6230dfa 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -71,7 +71,8 @@ import VX_tcu_pkg::*; // Global barrier VX_gbar_bus_if.master gbar_bus_if, - // Status + // Status: the core's own busy, and each LSU block's memory path busy + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_busy, output wire busy ); VX_warp_shared_if warp_shared_if[`VX_CFG_NUM_SFU_BLOCKS](); @@ -157,6 +158,19 @@ import VX_tcu_pkg::*; // the in-core global-store path is empty wire mem_unit_empty = (& lsu_store_empty); + wire mem_unit_busy; + +`ifdef VX_CFG_VM_ENABLE +`ifdef PERF_ENABLE + // the MMU perf counters are kept at the socket + assign pipeline_perf.mmu = '0; +`endif + // Translation drains at the socket MMU; the socket returns its empty status. + wire mmu_empty = mmu_drained; +`else + wire mmu_empty = 1'b1; +`endif + // Stores are ack-less and lag warp exit, so the dcache flush waits until every // store of this core has reached L1. @@ -189,7 +203,6 @@ import VX_tcu_pkg::*; wire sched_busy; wire [PERF_CTR_BITS-1:0] sched_cycles; wire [PERF_CTR_BITS-1:0] sched_instret; - wire lsu_drained; VX_csr_mpm #( .INSTANCE_ID (`SFORMATF(("%s-mpm", INSTANCE_ID))) @@ -437,7 +450,8 @@ import VX_tcu_pkg::*; .cycles (sched_cycles), .instret (sched_instret), .active_warps (sched_active_warps), - .lsu_sched_drained (lsu_drained), + .all_lsu_sched_empty (lsu_sched_empty), + .mmu_empty (mmu_empty), .csr_rd_cta_id (csr_rd_cta_id[i]), .csr_rd_ctx (csr_rd_ctx[i]), `ifdef VX_CFG_VM_ENABLE @@ -456,6 +470,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty[i]), .lsu_store_empty (lsu_store_empty[i]), + .lsu_busy (lsu_busy[i]), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), `endif @@ -551,7 +566,8 @@ import VX_tcu_pkg::*; .cycles (sched_cycles), .instret (sched_instret), .active_warps (sched_active_warps), - .lsu_sched_drained (lsu_drained), + .lsu_sched_empty (lsu_sched_empty), + .mmu_empty (mmu_empty), .csr_rd_ctx (csr_rd_ctx), `ifdef VX_CFG_VM_ENABLE .satp (sched_satp), @@ -692,6 +708,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty[i]), .lsu_store_empty (lsu_store_empty[i]), + .lsu_busy (lsu_busy[i]), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), `endif @@ -786,6 +803,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), + .lsu_busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lsu_lmem_bus_if), `endif @@ -872,35 +890,18 @@ import VX_tcu_pkg::*; .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), .flush_l1_only (dcr_flush_l1_only), - .dcache_bus_if (dcache_bus_if) + .dcache_bus_if (dcache_bus_if), + .busy (mem_unit_busy) ); -`ifdef VX_CFG_VM_ENABLE -`ifdef PERF_ENABLE - // the MMU perf counters are kept at the socket - assign pipeline_perf.mmu = '0; -`endif - // Translation drains at the socket MMU; the socket returns its empty status. - wire mmu_empty = mmu_drained; -`else - wire mmu_empty = 1'b1; -`endif - // Fragment work drains at the producer (VX_raster_core.busy), not here. // DCR reads (cache flush, MPM readback) order on their own DCR response, so // busy tracks only kernel execution: warps resident or stores not yet at L1. // ~mmu_empty also keeps the core busy while a translation or parked store is // still in flight, so kernel completion cannot race un-drained accesses. - // A request waiting at the dcache port sits past the flush injector's - // buffer, which mem_unit_empty does not cover. - wire [DCACHE_NUM_REQS-1:0] dcache_req_valid; - for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_req_valid - assign dcache_req_valid[i] = dcache_bus_if[i].req_valid; - end - assign busy = sched_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || (| dcache_req_valid) || ~mmu_empty; - - // BAR (vx_barrier / vx_barrier_arrive) drains LSU + MMU before suspending or registering arrival. - assign lsu_drained = (&lsu_sched_empty) && mmu_empty; + // A store still in an LSU block's memory path shows on lsu_busy, which the + // parent ORs in; one past a flush injector's buffer shows on mem_unit_busy. + assign busy = sched_busy || mem_unit_busy || ~mmu_empty; `ifdef PERF_ENABLE diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 8258555eaf..f4fd71f0c3 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -39,6 +39,7 @@ import VX_tcu_pkg::*; `endif output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], `endif @@ -138,6 +139,7 @@ import VX_tcu_pkg::*; `endif .sched_empty (lsu_sched_empty), .store_empty (lsu_store_empty), + .busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index be0a1d001d..9e9bfb34eb 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -56,6 +56,7 @@ import VX_tcu_pkg::*; `endif output wire lsu_sched_empty, output wire lsu_store_empty, + output wire lsu_busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], `endif @@ -202,6 +203,7 @@ import VX_tcu_pkg::*; `endif .sched_empty (lsu_sched_empty), .store_empty (lsu_store_empty), + .busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_frontend_xbar.sv b/hw/rtl/core/VX_frontend_xbar.sv index 8f4d7cf3c7..252d4df822 100644 --- a/hw/rtl/core/VX_frontend_xbar.sv +++ b/hw/rtl/core/VX_frontend_xbar.sv @@ -43,7 +43,8 @@ module VX_frontend_xbar import VX_gpu_pkg::*; ( input wire [PERF_CTR_BITS-1:0] cycles, input wire [PERF_CTR_BITS-1:0] instret, input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, - input wire lsu_sched_drained, + input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, + input wire mmu_empty, input cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, `ifdef VX_CFG_VM_ENABLE input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, @@ -196,7 +197,8 @@ module VX_frontend_xbar import VX_gpu_pkg::*; ( for (genvar k = 0; k < SFU_BLOCK_WARPS; ++k) begin : g_pending assign warp_ctl_if[b].warp_pending_alm_empty[k] = pending_alm_empty[k * BS + b]; end - assign warp_ctl_if[b].lsu_sched_drained = lsu_sched_drained; + assign warp_ctl_if[b].lsu_sched_empty = lsu_sched_empty; + assign warp_ctl_if[b].mmu_empty = mmu_empty; assign sched_csr_if[b].cycles = cycles; assign sched_csr_if[b].instret = instret; diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 83c8ed4190..c43935bc21 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -42,6 +42,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( output wire sched_empty, output wire store_empty, + output wire busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], @@ -418,6 +419,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `endif .sched_empty (sched_empty), .store_empty (store_empty), + .busy (busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_lsu_mem_path.sv b/hw/rtl/core/VX_lsu_mem_path.sv index 7fd861ca86..3b3a255e0c 100644 --- a/hw/rtl/core/VX_lsu_mem_path.sv +++ b/hw/rtl/core/VX_lsu_mem_path.sv @@ -38,6 +38,7 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( // the scheduler holds no request; the global-store path holds no store output wire sched_empty, output wire store_empty, + output wire busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], @@ -107,6 +108,7 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( wire path_empty; wire adapter_empty; assign store_empty = path_empty && adapter_empty; + assign busy = ~sched_empty || ~store_empty; VX_lsu_mem_if #( .NUM_LANES (`VX_CFG_NUM_LSU_LANES), diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index e383dfe5c8..ccd14130a3 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -42,6 +42,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] sched_empty, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] store_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], @@ -112,6 +113,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( `endif .sched_empty (sched_empty[block_idx]), .store_empty (store_empty[block_idx]), + .busy (busy[block_idx]), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (block_lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index a72e33c8c0..11a8228b36 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -41,7 +41,10 @@ module VX_mem_unit import VX_gpu_pkg::*; #( VX_dcr_flush_if.slave dcr_flush_if, input wire flush_l1_only, - VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], + + // a flush lane holds a request at the D-cache port + output wire busy ); `ifndef VX_CFG_LMEM_ENABLE `UNUSED_SPARAM (INSTANCE_ID) @@ -207,6 +210,12 @@ module VX_mem_unit import VX_gpu_pkg::*; #( end assign dcr_flush_if.done = (& lane_flush_done); + wire [FLUSH_LANES-1:0] flush_req_valid; + for (genvar k = 0; k < FLUSH_LANES; ++k) begin : g_flush_req_valid + assign flush_req_valid[k] = dcache_bus_if[k].req_valid; + end + assign busy = (| flush_req_valid); + // A block's D-cache request carries a clear tag LSB, the flush arbiter's // select bit; the block takes the cache response directly and drops the // flush token's. diff --git a/hw/rtl/core/VX_wctl_unit.sv b/hw/rtl/core/VX_wctl_unit.sv index 60b4cef82a..49dbabb063 100644 --- a/hw/rtl/core/VX_wctl_unit.sv +++ b/hw/rtl/core/VX_wctl_unit.sv @@ -175,7 +175,8 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( && !warp_ctl_if.warp_pending_alm_empty[wid_to_sfu_local(execute_if.data.header.wid)]; // BAR (vx_barrier or vx_barrier_arrive) drains LSU before continuing. - wire bar_drain = execute_if.valid && is_bar && !warp_ctl_if.lsu_sched_drained; + wire lsu_sched_drained = (& warp_ctl_if.lsu_sched_empty) && warp_ctl_if.mmu_empty; + wire bar_drain = execute_if.valid && is_bar && !lsu_sched_drained; // Barriers, wspawn and warp retirement are core-wide requests to the shared // unit. Only a barrier arrive writes a result, the barrier phase, so it alone diff --git a/hw/rtl/interfaces/VX_warp_ctl_if.sv b/hw/rtl/interfaces/VX_warp_ctl_if.sv index 0e49757335..26d648a659 100644 --- a/hw/rtl/interfaces/VX_warp_ctl_if.sv +++ b/hw/rtl/interfaces/VX_warp_ctl_if.sv @@ -27,12 +27,14 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); // Pipeline drain status of the scheduler's warps, by SFU-block-local warp id wire [SFU_BLOCK_WARPS-1:0] warp_pending_alm_empty; - // LSU memory-scheduler drained signal (AND-reduced across all LSU blocks). - // Used by VX_wctl_unit to gate BAR (both vx_barrier and vx_barrier_arrive) - // so the barrier acts as a SMEM/GMEM fence — pending LSU memory ops finish - // before the warp suspends or the barrier arrival is registered. This is - // the CUDA __syncthreads / OpenCL barrier(CLK_LOCAL_MEM_FENCE) contract. - wire lsu_sched_drained; + // Every LSU block's memory scheduler is empty, and the translation path + // drained. VX_wctl_unit ANDs them to gate BAR (both vx_barrier and + // vx_barrier_arrive) so the barrier acts as a SMEM/GMEM fence — pending + // LSU memory ops finish before the warp suspends or the barrier arrival is + // registered. This is the CUDA __syncthreads / OpenCL + // barrier(CLK_LOCAL_MEM_FENCE) contract. + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; + wire mmu_empty; // IPDOM stack address and mscratch of the warp at dvstack_wid wire [NW_WIDTH-1:0] dvstack_wid; @@ -49,7 +51,8 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); output split, output sjoin, input warp_pending_alm_empty, - input lsu_sched_drained, + input lsu_sched_empty, + input mmu_empty, output dvstack_wid, input dvstack_ptr, input mscratch @@ -65,7 +68,8 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); input split, input sjoin, output warp_pending_alm_empty, - output lsu_sched_drained, + output lsu_sched_empty, + output mmu_empty, input dvstack_wid, output dvstack_ptr, output mscratch diff --git a/hw/unittest/core/VX_core_top.sv b/hw/unittest/core/VX_core_top.sv index d47bfad744..d5aef6f3d4 100644 --- a/hw/unittest/core/VX_core_top.sv +++ b/hw/unittest/core/VX_core_top.sv @@ -209,6 +209,10 @@ module VX_core_top import VX_gpu_pkg::*; `UNUSED_VAR (scope_bus_out_w) `endif + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] core_lsu_busy; + wire core_busy; + assign busy = core_busy || (| core_lsu_busy); + VX_core #( .INSTANCE_ID (`SFORMATF(("core"))) ) core ( @@ -245,7 +249,8 @@ module VX_core_top import VX_gpu_pkg::*; `endif .kmu_bus_if (kmu_bus_if), - .busy (busy) + .lsu_busy (core_lsu_busy), + .busy (core_busy) ); endmodule diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index ed556ebf4c..1f39567731 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -94,6 +94,7 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( `endif `UNUSED_PIN (sched_empty), `UNUSED_PIN (store_empty), + `UNUSED_PIN (busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), `endif @@ -148,7 +149,8 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_if), .flush_l1_only (1'b0), - .dcache_bus_if (mem_bus_if) + .dcache_bus_if (mem_bus_if), + `UNUSED_PIN (busy) ); endmodule diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 05037a0908..232f6475cb 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -302,6 +302,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( `endif .sched_empty (block_sched_empty[block_idx]), .store_empty (block_store_empty[block_idx]), + `UNUSED_PIN (busy), .lmem_bus_if (lsu_lmem_bus_if[block_idx * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), .dcache_bus_if (lsu_dcache_bus_if[block_idx * DCACHE_CHANNELS +: DCACHE_CHANNELS]) ); @@ -341,7 +342,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), .flush_l1_only (1'b0), - .dcache_bus_if (dcache_bus_if) + .dcache_bus_if (dcache_bus_if), + `UNUSED_PIN (busy) ); for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_passthru From 8410590f44445da0645bee427d7774f35f382b11 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 02:15:50 -0700 Subject: [PATCH 086/336] core: keep each SFU block's CTA context RAM in the block VX_scheduler held one CTA context RAM per SFU block, each read only by that block's CSR unit, so every read address and its data crossed the core. Each RAM now sits in its VX_sfu_block, written by the same broadcast from CTA dispatch, with the same size and registered read. SATP stays in the scheduler: it is one core-wide live CSR whose writes are ordered across SFU blocks. rtlsim sgemm_tcu_wg_sp_dxa -m 128 -n 128 -k 128: cycle-identical to the parent (144,956), PASSED. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_backend_slice.sv | 8 +++++--- hw/rtl/core/VX_compute_slice.sv | 18 ++++-------------- hw/rtl/core/VX_core.sv | 19 +++++++------------ hw/rtl/core/VX_csr_data.sv | 24 +++++++++++++----------- hw/rtl/core/VX_csr_unit.sv | 2 ++ hw/rtl/core/VX_execute.sv | 6 ++++-- hw/rtl/core/VX_execute_slice.sv | 6 ++++-- hw/rtl/core/VX_frontend_xbar.sv | 12 +----------- hw/rtl/core/VX_scheduler.sv | 26 ++------------------------ hw/rtl/core/VX_sfu_block.sv | 27 ++++++++++++++++++++++++++- hw/rtl/core/VX_sfu_unit.sv | 7 +++++-- hw/rtl/interfaces/VX_sched_csr_if.sv | 14 +++++--------- 12 files changed, 78 insertions(+), 91 deletions(-) diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv index 24e380cae0..385db201a5 100644 --- a/hw/rtl/core/VX_backend_slice.sv +++ b/hw/rtl/core/VX_backend_slice.sv @@ -106,8 +106,9 @@ import VX_tcu_pkg::*; input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, `endif - // core-wide warp control - VX_warp_shared_if.master warp_shared_if + // core-wide warp control and CTA contexts + VX_warp_shared_if.master warp_shared_if, + input cta_bcast_t cta_bcast ); `STATIC_ASSERT (BACKEND_SLICES, ("invalid parameter")) @@ -204,7 +205,8 @@ import VX_tcu_pkg::*; `endif .warp_shared_if (warp_shared_if), .warp_ctl_if (warp_ctl_if), - .sched_csr_if (sched_csr_if) + .sched_csr_if (sched_csr_if), + .cta_bcast (cta_bcast) ); // commit diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index ddf14588e9..f241dd9d96 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -51,8 +51,7 @@ import VX_tcu_pkg::*; input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] all_lsu_sched_empty, input wire mmu_empty, - output wire [NCTA_WIDTH-1:0] csr_rd_cta_id, - input cta_ctx_t csr_rd_ctx, + input cta_bcast_t cta_bcast, `ifdef VX_CFG_VM_ENABLE input wire [`VX_CFG_XLEN-1:0] satp, output wire satp_wr_valid, @@ -207,7 +206,6 @@ import VX_tcu_pkg::*; assign warp_ctl_if.mmu_empty = mmu_empty; assign csr_rd_wid[0] = sched_csr_if.csr_rd_wid; - assign csr_rd_cta_id = sched_csr_if.csr_rd_cta_id; assign csr_wr[0].valid = sched_csr_if.csr_wr_valid; assign csr_wr[0].wid = sched_csr_if.csr_wr_wid; assign csr_wr[0].data = sched_csr_if.csr_wr_data; @@ -224,16 +222,7 @@ import VX_tcu_pkg::*; assign sched_csr_if.csr_mepc = csr_rd[0].mepc; assign sched_csr_if.csr_mcause = csr_rd[0].mcause; assign sched_csr_if.csr_mtval = csr_rd[0].mtval; - assign sched_csr_if.cta_csrs.cta_id = sched_csr_if.csr_rd_cta_id; - assign sched_csr_if.cta_csrs.cta_rank = csr_rd[0].cta_rank; - assign sched_csr_if.cta_csrs.cta_size = csr_rd_ctx.cta_size; - assign sched_csr_if.cta_csrs.block_idx = csr_rd_ctx.block_idx; - assign sched_csr_if.cta_csrs.block_dim = csr_rd_ctx.block_dim; - assign sched_csr_if.cta_csrs.grid_dim = csr_rd_ctx.grid_dim; - assign sched_csr_if.cta_csrs.entry = csr_rd_ctx.entry; - assign sched_csr_if.cta_csrs.param = csr_rd_ctx.param; - assign sched_csr_if.cta_csrs.lmem_addr = csr_rd_ctx.lmem_addr; - assign sched_csr_if.cta_csrs.cluster_size = csr_rd_ctx.cluster_size; + assign sched_csr_if.cta_rank = csr_rd[0].cta_rank; assign sched_csr_if.cta_lane = csr_rd[0].cta_lane; `ifdef VX_CFG_VM_ENABLE assign sched_csr_if.csr_satp = satp; @@ -313,7 +302,8 @@ import VX_tcu_pkg::*; .tcu_meta_wr_idx (tcu_meta_wr_idx), .tcu_meta_wr_data (tcu_meta_wr_data), `endif - .warp_shared_if (warp_shared_if) + .warp_shared_if (warp_shared_if), + .cta_bcast (cta_bcast) ); endmodule diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 7ef6230dfa..eedcb21820 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -266,8 +266,7 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data; `endif - wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id; - cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx; + cta_bcast_t cta_bcast; VX_scheduler #( .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))) @@ -284,8 +283,7 @@ import VX_tcu_pkg::*; .txbar_bus_if (dxa_txbar_bus_if), `endif .committed_warps(committed_warps), - .csr_rd_cta_id (csr_rd_cta_id), - .csr_rd_ctx (csr_rd_ctx), + .cta_bcast (cta_bcast), .kmu_bus_if (kmu_bus_if), .gbar_bus_if (gbar_bus_if), .cycles (sched_cycles), @@ -452,8 +450,7 @@ import VX_tcu_pkg::*; .active_warps (sched_active_warps), .all_lsu_sched_empty (lsu_sched_empty), .mmu_empty (mmu_empty), - .csr_rd_cta_id (csr_rd_cta_id[i]), - .csr_rd_ctx (csr_rd_ctx[i]), + .cta_bcast (cta_bcast), `ifdef VX_CFG_VM_ENABLE .satp (sched_satp[i]), .satp_wr_valid (satp_wr_valid[i]), @@ -520,9 +517,6 @@ import VX_tcu_pkg::*; VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); VX_warp_ctl_if warp_ctl_if[`VX_CFG_NUM_SFU_BLOCKS](); VX_sched_csr_if sched_csr_if[`VX_CFG_NUM_SFU_BLOCKS](); - for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_csr_rd_cta_id - assign csr_rd_cta_id[b] = sched_csr_if[b].csr_rd_cta_id; - end `ifdef VX_CFG_EXT_RTU_ENABLE VX_sched_unlock_if sched_unlock_if[`VX_CFG_NUM_SFU_BLOCKS](); `endif @@ -568,7 +562,6 @@ import VX_tcu_pkg::*; .active_warps (sched_active_warps), .lsu_sched_empty (lsu_sched_empty), .mmu_empty (mmu_empty), - .csr_rd_ctx (csr_rd_ctx), `ifdef VX_CFG_VM_ENABLE .satp (sched_satp), .satp_wr_valid (satp_wr_valid), @@ -742,7 +735,8 @@ import VX_tcu_pkg::*; .tcu_meta_wr_idx (tcu_meta_wr_idx), .tcu_meta_wr_data (tcu_meta_wr_data), `endif - .warp_shared_if (warp_shared_if[i]) + .warp_shared_if (warp_shared_if[i]), + .cta_bcast (cta_bcast) ); // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i @@ -848,7 +842,8 @@ import VX_tcu_pkg::*; .warp_shared_if (warp_shared_if), .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if) + .branch_ctl_if (branch_ctl_if), + .cta_bcast (cta_bcast) ); VX_commit #( diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index e7a8799dc6..6a2300fe1a 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -39,6 +39,9 @@ import VX_fpu_pkg::*; VX_sched_csr_if.slave sched_csr_if, + // context of the request's CTA, read by req_cta_id + input cta_ctx_t cta_ctx, + // The request is held for at least one cycle before it fires; that cycle // decodes it, and the fire cycle only merges the values that can still move. input wire req_fire, @@ -54,7 +57,7 @@ import VX_fpu_pkg::*; `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_VAR (reset) `UNUSED_VAR (req_uuid) - wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] __cta_param = sched_csr_if.cta_csrs.param; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] __cta_param = cta_ctx.param; `UNUSED_VAR (__cta_param) // Values that can change between the decode cycle and the fire cycle are @@ -104,7 +107,6 @@ import VX_fpu_pkg::*; // Scheduler CSRs read interface assign sched_csr_if.csr_rd_wid = req_wid; - assign sched_csr_if.csr_rd_cta_id = req_cta_id; reg [`VX_CFG_XLEN-1:0] stable_w; reg [NUM_LIVE-1:0] live_sel_w; @@ -126,7 +128,7 @@ import VX_fpu_pkg::*; `endif `VX_CSR_MSCRATCH : begin stable_w = `VX_CFG_XLEN'(sched_csr_if.mscratch); wr_sel_w[WR_MSCRATCH] = 1; end - `VX_CSR_CTA_ID : stable_w = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cta_id); + `VX_CSR_CTA_ID : stable_w = `VX_CFG_XLEN'(req_cta_id); `VX_CSR_CTA_RANK : live_sel_w[LIVE_CTA_RANK] = 1; `VX_CSR_CTA_SIZE : live_sel_w[LIVE_CTA_SIZE] = 1; `VX_CSR_CTA_BLOCK_ID_X : live_sel_w[LIVE_BLOCK_ID + 0] = 1; @@ -239,16 +241,16 @@ import VX_fpu_pkg::*; `endif // The CTA context RAMs are addressed by the held request and return in the // fire cycle. - assign live_src[LIVE_CTA_RANK] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cta_rank); - assign live_src[LIVE_CTA_SIZE] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cta_size); + assign live_src[LIVE_CTA_RANK] = `VX_CFG_XLEN'(sched_csr_if.cta_rank); + assign live_src[LIVE_CTA_SIZE] = `VX_CFG_XLEN'(cta_ctx.cta_size); for (genvar i = 0; i < 3; ++i) begin : g_cta_dims - assign live_src[LIVE_BLOCK_ID + i] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.block_idx[i]); - assign live_src[LIVE_BLOCK_DIM + i] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.block_dim[i]); - assign live_src[LIVE_GRID_DIM + i] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.grid_dim[i]); + assign live_src[LIVE_BLOCK_ID + i] = `VX_CFG_XLEN'(cta_ctx.block_idx[i]); + assign live_src[LIVE_BLOCK_DIM + i] = `VX_CFG_XLEN'(cta_ctx.block_dim[i]); + assign live_src[LIVE_GRID_DIM + i] = `VX_CFG_XLEN'(cta_ctx.grid_dim[i]); end - assign live_src[LIVE_LMEM_ADDR] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.lmem_addr); - assign live_src[LIVE_CLUSTER] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cluster_size); - assign live_src[LIVE_ENTRY] = `VX_CFG_XLEN'(to_fullPC(sched_csr_if.cta_csrs.entry)); + assign live_src[LIVE_LMEM_ADDR] = `VX_CFG_XLEN'(cta_ctx.lmem_addr); + assign live_src[LIVE_CLUSTER] = `VX_CFG_XLEN'(cta_ctx.cluster_size); + assign live_src[LIVE_ENTRY] = `VX_CFG_XLEN'(to_fullPC(cta_ctx.entry)); `ifdef VX_CFG_VM_ENABLE assign live_src[LIVE_SATP] = sched_csr_if.csr_satp; `else diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index d4f501574b..0a4e859da9 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -34,6 +34,7 @@ module VX_csr_unit import VX_gpu_pkg::*; #( `endif VX_sched_csr_if.slave sched_csr_if, + input cta_ctx_t cta_ctx, VX_execute_if.slave execute_if, VX_result_if.master result_if ); @@ -87,6 +88,7 @@ module VX_csr_unit import VX_gpu_pkg::*; #( .core_id (core_id), .sched_csr_if (sched_csr_if), + .cta_ctx (cta_ctx), `ifdef VX_CFG_EXT_F_ENABLE .fcsr_rd_data (fcsr_rd_data), diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index f4fd71f0c3..84ebe921fb 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -97,7 +97,8 @@ import VX_tcu_pkg::*; VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], - VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS] + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS], + input cta_bcast_t cta_bcast ); `ifdef VX_CFG_EXT_F_ENABLE @@ -213,7 +214,8 @@ import VX_tcu_pkg::*; `endif .sched_csr_if (sched_csr_if), .warp_shared_if (warp_shared_if), - .warp_ctl_if (warp_ctl_if) + .warp_ctl_if (warp_ctl_if), + .cta_bcast (cta_bcast) ); endmodule diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index 9e9bfb34eb..7436dc3573 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -99,7 +99,8 @@ import VX_tcu_pkg::*; // SFU warp control and CSRs VX_warp_shared_if.master warp_shared_if, VX_warp_ctl_if.master warp_ctl_if, - VX_sched_csr_if.slave sched_csr_if + VX_sched_csr_if.slave sched_csr_if, + input cta_bcast_t cta_bcast ); localparam ALU_PARTIAL_BW = (`VX_CFG_NUM_ALU_LANES != `VX_CFG_SIMD_WIDTH); `ifdef VX_CFG_EXT_F_ENABLE @@ -279,7 +280,8 @@ import VX_tcu_pkg::*; `endif .sched_csr_if (sched_csr_if), .warp_shared_if (warp_shared_if), - .warp_ctl_if (warp_ctl_if) + .warp_ctl_if (warp_ctl_if), + .cta_bcast (cta_bcast) ); VX_lane_gather #( diff --git a/hw/rtl/core/VX_frontend_xbar.sv b/hw/rtl/core/VX_frontend_xbar.sv index 252d4df822..0426b3255c 100644 --- a/hw/rtl/core/VX_frontend_xbar.sv +++ b/hw/rtl/core/VX_frontend_xbar.sv @@ -45,7 +45,6 @@ module VX_frontend_xbar import VX_gpu_pkg::*; ( input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, input wire mmu_empty, - input cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, `ifdef VX_CFG_VM_ENABLE input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid, @@ -210,16 +209,7 @@ module VX_frontend_xbar import VX_gpu_pkg::*; ( assign sched_csr_if[b].csr_mepc = csr_rd.mepc; assign sched_csr_if[b].csr_mcause = csr_rd.mcause; assign sched_csr_if[b].csr_mtval = csr_rd.mtval; - assign sched_csr_if[b].cta_csrs.cta_id = sched_csr_if[b].csr_rd_cta_id; - assign sched_csr_if[b].cta_csrs.cta_rank = csr_rd.cta_rank; - assign sched_csr_if[b].cta_csrs.cta_size = csr_rd_ctx[b].cta_size; - assign sched_csr_if[b].cta_csrs.block_idx = csr_rd_ctx[b].block_idx; - assign sched_csr_if[b].cta_csrs.block_dim = csr_rd_ctx[b].block_dim; - assign sched_csr_if[b].cta_csrs.grid_dim = csr_rd_ctx[b].grid_dim; - assign sched_csr_if[b].cta_csrs.entry = csr_rd_ctx[b].entry; - assign sched_csr_if[b].cta_csrs.param = csr_rd_ctx[b].param; - assign sched_csr_if[b].cta_csrs.lmem_addr = csr_rd_ctx[b].lmem_addr; - assign sched_csr_if[b].cta_csrs.cluster_size = csr_rd_ctx[b].cluster_size; + assign sched_csr_if[b].cta_rank = csr_rd.cta_rank; assign sched_csr_if[b].cta_lane = csr_rd.cta_lane; `ifdef VX_CFG_VM_ENABLE assign sched_csr_if[b].csr_satp = satp[b]; diff --git a/hw/rtl/core/VX_scheduler.sv b/hw/rtl/core/VX_scheduler.sv index 246274c7ca..efcc981018 100644 --- a/hw/rtl/core/VX_scheduler.sv +++ b/hw/rtl/core/VX_scheduler.sv @@ -35,9 +35,8 @@ module VX_scheduler import VX_gpu_pkg::*; #( `endif input wire [`VX_CFG_NUM_WARPS-1:0] committed_warps, - // CTA context read by each SFU block's CSR unit, valid a cycle after the address - input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id, - output cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, + // CTA context writes, to every SFU block's copy + output cta_bcast_t cta_bcast, VX_kmu_bus_if.slave kmu_bus_if, @@ -77,7 +76,6 @@ module VX_scheduler import VX_gpu_pkg::*; #( wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] cta_tmask; wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param; wire [FW-1:0] cta_init; - cta_bcast_t cta_bcast; cta_lane_bcast_t [FW-1:0] cta_lane_bcast; wire cta_dispatcher_busy; wire cta_warp_done; @@ -104,26 +102,6 @@ module VX_scheduler import VX_gpu_pkg::*; #( .busy (cta_dispatcher_busy) ); - // CTA contexts, one read port per SFU block - for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_cta_ctx - VX_dp_ram #( - .DATAW ($bits(cta_ctx_t)), - .SIZE (NUM_CTA_MAX), - .RDW_MODE ("R"), - .OUT_REG (1) - ) cta_ctx_ram ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (cta_bcast.ctx_write), - .wren (1'b1), - .waddr (cta_bcast.ctx_waddr), - .wdata (cta_bcast.ctx_wdata), - .raddr (csr_rd_cta_id[b]), - .rdata (csr_rd_ctx[b]) - ); - end - // core-wide warp-control resources wire bar_unlock_valid; wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask; diff --git a/hw/rtl/core/VX_sfu_block.sv b/hw/rtl/core/VX_sfu_block.sv index 15ce7ad91d..92d4fed066 100644 --- a/hw/rtl/core/VX_sfu_block.sv +++ b/hw/rtl/core/VX_sfu_block.sv @@ -52,7 +52,10 @@ module VX_sfu_block import VX_gpu_pkg::*; #( VX_sched_csr_if.slave sched_csr_if, VX_warp_shared_if.master warp_shared_if, - VX_warp_ctl_if.master warp_ctl_if + VX_warp_ctl_if.master warp_ctl_if, + + // CTA dispatch's context writes + input cta_bcast_t cta_bcast ); localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; `ifndef VX_CFG_EXT_RTU_ENABLE @@ -147,6 +150,27 @@ module VX_sfu_block import VX_gpu_pkg::*; #( .result_if (pe_result_if[PE_IDX_WCTL]) ); + // The block's copy of the CTA contexts, read at the CSR request's CTA and + // valid a cycle later. + cta_ctx_t cta_ctx; + + VX_dp_ram #( + .DATAW ($bits(cta_ctx_t)), + .SIZE (NUM_CTA_MAX), + .RDW_MODE ("R"), + .OUT_REG (1) + ) cta_ctx_ram ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (cta_bcast.ctx_write), + .wren (1'b1), + .waddr (cta_bcast.ctx_waddr), + .wdata (cta_bcast.ctx_wdata), + .raddr (pe_execute_if[PE_IDX_CSRS].data.header.cta_id), + .rdata (cta_ctx) + ); + VX_csr_unit #( .INSTANCE_ID (`SFORMATF(("%s-csr", INSTANCE_ID))), .NUM_LANES (NUM_LANES) @@ -162,6 +186,7 @@ module VX_sfu_block import VX_gpu_pkg::*; #( .fcsr_wr_data (fcsr_wr_data), `endif .sched_csr_if (sched_csr_if), + .cta_ctx (cta_ctx), .result_if (pe_result_if[PE_IDX_CSRS]) ); diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index a7f14fe32d..59ba4c8dc3 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -50,7 +50,9 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( // Outputs VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH], VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], - VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS] + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS], + + input cta_bcast_t cta_bcast ); localparam BLOCK_SIZE = `VX_CFG_NUM_SFU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; @@ -110,7 +112,8 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( `endif .sched_csr_if (sched_csr_if[block_idx]), .warp_shared_if (warp_shared_if[block_idx]), - .warp_ctl_if (warp_ctl_if[block_idx]) + .warp_ctl_if (warp_ctl_if[block_idx]), + .cta_bcast (cta_bcast) ); end diff --git a/hw/rtl/interfaces/VX_sched_csr_if.sv b/hw/rtl/interfaces/VX_sched_csr_if.sv index d773df084d..cc88a4d0e2 100644 --- a/hw/rtl/interfaces/VX_sched_csr_if.sv +++ b/hw/rtl/interfaces/VX_sched_csr_if.sv @@ -20,13 +20,11 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); wire [`VX_CFG_NUM_WARPS-1:0] active_warps; wire [`VX_CFG_NUM_THREADS-1:0] thread_mask; // selected by csr_rd_wid - // Read port: slave sends wid + cta_id, master returns selected mscratch - // and cta_csrs. csr_rd_cta_id must carry the executing instruction's - // cta_id so the scheduler can index its per-CTA table directly. + // Read port: slave sends wid, master returns the selected warp's mscratch + // and CTA rank. logic [NW_WIDTH-1:0] csr_rd_wid; - logic [NCTA_WIDTH-1:0] csr_rd_cta_id; logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; - cta_csrs_t cta_csrs; + logic [NW_WIDTH-1:0] cta_rank; // Per-lane launch record; the CSR unit reads the view (thread index or fragment // stamp) the accessed CSR needs. cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_lane; @@ -61,7 +59,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output active_warps, output thread_mask, output mscratch, - output cta_csrs, + output cta_rank, output cta_lane, output csr_mstatus, output csr_mtvec, @@ -74,7 +72,6 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input satp_wr_data, `endif input csr_rd_wid, - input csr_rd_cta_id, input csr_wr_valid, input csr_wr_wid, input csr_wr_data, @@ -88,7 +85,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input active_warps, input thread_mask, input mscratch, - input cta_csrs, + input cta_rank, input cta_lane, input csr_mstatus, input csr_mtvec, @@ -101,7 +98,6 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output satp_wr_data, `endif output csr_rd_wid, - output csr_rd_cta_id, output csr_wr_valid, output csr_wr_wid, output csr_wr_data, From 47844854f4cd405b0098e652d0ef942a5e62399d Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 02:47:51 -0700 Subject: [PATCH 087/336] tcu: give each TCU block its own metadata port into its slice's LSU scheduler When the LSU has one block per TCU block, each TCU block now issues its TCU_LD metadata loads through the scheduler of the LSU block that serves the same warps, instead of all blocks sharing LSU block 0's scheduler. This keeps each slice's memory traffic inside the slice. With a different block count the AGU keeps one shared port on LSU block 0. The AGU keeps its per-block holding registers and shared commit arbiter. Metadata write-after-read safety still comes from TCU_LD being dispatched in order behind the warp's MMA micro-ops in the same TCU block. SimX models the same ports (TCU_AGU_PORTS, same derivation as the RTL). sgemm_tcu_wg_sp_dxa 128x128x128 (IW4, FW4, 4 LSU/4 TCU blocks, 32 threads): rtlsim 144,956 cycles, PASSED, unchanged from the shared port. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_core.sv | 13 +- hw/rtl/tcu/VX_tcu_agu.sv | 309 ++++++++++++++++++++---------------- hw/rtl/tcu/VX_tcu_shared.sv | 32 +++- sim/simx/core.cpp | 8 +- sim/simx/lsu_unit.cpp | 26 +-- sim/simx/lsu_unit.h | 18 ++- sim/simx/tcu/tcu_unit.cpp | 116 ++++++++------ sim/simx/tcu/tcu_unit.h | 17 +- 8 files changed, 313 insertions(+), 226 deletions(-) diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index eedcb21820..a5b31c5e58 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -102,17 +102,8 @@ import VX_tcu_pkg::*; `endif `ifdef TCU_META_ENABLE - // The TCU metadata AGU is the second client of LSU block 0's scheduler; the - // other blocks' second client is idle. + // The TCU metadata AGU is the second client of the LSU schedulers. VX_lsu_sched_if lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); - for (genvar b = 1; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_lsu_tcu_tieoff - assign lsu_tcu_mem_if[b].req_valid = 1'b0; - assign lsu_tcu_mem_if[b].req_data = '0; - assign lsu_tcu_mem_if[b].rsp_ready = 1'b1; - `UNUSED_VAR (lsu_tcu_mem_if[b].req_ready) - `UNUSED_VAR (lsu_tcu_mem_if[b].rsp_valid) - `UNUSED_VAR (lsu_tcu_mem_if[b].rsp_data) - end `endif // frontend-slice icache requests; port 0 also carries the injected flush @@ -384,7 +375,7 @@ import VX_tcu_pkg::*; .meta_wr_wid (tcu_meta_wr_wid), .meta_wr_idx (tcu_meta_wr_idx), .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (lsu_tcu_mem_if[0]), + .tcu_mem_if (lsu_tcu_mem_if), `endif .clk (clk), .reset (reset) diff --git a/hw/rtl/tcu/VX_tcu_agu.sv b/hw/rtl/tcu/VX_tcu_agu.sv index 6690cf17b1..70a77b8126 100644 --- a/hw/rtl/tcu/VX_tcu_agu.sv +++ b/hw/rtl/tcu/VX_tcu_agu.sv @@ -16,12 +16,16 @@ `ifdef TCU_META_ENABLE // Warp-level AGU for TCU_LD instructions. Each TCU block can hold one -// transaction, while the LSU request and metadata write ports are shared. +// transaction. With one memory port the blocks share it; with one port per +// block (NUM_PORTS == BLOCK_SIZE) each block loads through its own LSU +// scheduler, so the loads of a warpgroup's warps proceed in parallel. The +// metadata write port is shared. module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS, - parameter NUM_LANES = `VX_CFG_NUM_TCU_LANES + parameter NUM_LANES = `VX_CFG_NUM_TCU_LANES, + parameter NUM_PORTS = 1 ) ( input wire clk, input wire reset, @@ -33,8 +37,8 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input tcu_execute_t per_block_ld_data [BLOCK_SIZE], output wire [BLOCK_SIZE-1:0] per_block_ld_ready, - // Memory client connection (to VX_lsu_scheduler at VX_core). - VX_lsu_sched_if.master client_if, + // Memory client connections (to the LSU schedulers). + VX_lsu_sched_if.master client_if [NUM_PORTS], // Meta SRAM write port (broadcast — same data lands in every // block's VX_tcu_meta so subsequent wmma_sp reads see it @@ -51,6 +55,7 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire [BLOCK_SIZE-1:0] result_ready ); `UNUSED_SPARAM (INSTANCE_ID) + `STATIC_ASSERT ((NUM_PORTS == 1) || (NUM_PORTS == BLOCK_SIZE), ("invalid parameter: NUM_PORTS=%0d", NUM_PORTS)) localparam BLOCK_IDX_BITS = (BLOCK_SIZE > 1) ? $clog2(BLOCK_SIZE) : 1; logic [BLOCK_SIZE-1:0] busy_r; @@ -64,22 +69,6 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( logic [NUM_LANES-1:0] rsp_lane_done_r [BLOCK_SIZE]; wire [BLOCK_SIZE-1:0] req_pending = busy_r & ~issued_r; - wire [BLOCK_IDX_BITS-1:0] req_block; - wire [BLOCK_SIZE-1:0] req_grant; - wire req_valid; - - VX_generic_arbiter #( - .NUM_REQS (BLOCK_SIZE), - .TYPE ("R") - ) req_arb ( - .clk (clk), - .reset (reset), - .requests (req_pending), - .grant_index (req_block), - .grant_onehot (req_grant), - .grant_valid (req_valid), - .grant_ready (client_if.req_ready) - ); wire [BLOCK_SIZE-1:0] commit_pending = complete_r; wire [BLOCK_IDX_BITS-1:0] commit_block; @@ -99,15 +88,148 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .grant_valid (commit_valid), .grant_ready (commit_ready) ); - - wire [BLOCK_IDX_BITS-1:0] rsp_block = - client_if.rsp_data.tag[BLOCK_IDX_BITS-1:0]; - wire rsp_fire = client_if.rsp_valid && client_if.rsp_ready; - wire req_fire = req_valid && client_if.req_ready; wire commit_fire = commit_valid && commit_ready; - `UNUSED_VAR (req_grant) `UNUSED_VAR (commit_grant) + // ----------------------------------------------------------------------- + // Memory request: address from rs1 (warp-broadcast). NUM_LANES mask + // all-ones — we fetch the metadata tile in one warp-level request. + // Lane T loads the 32-bit word at base + 4*T; the host packs h_meta so + // that word holds the metadata destined for SRAM cell + // (bank = T%PWD, col = T/PWD). One formula serves WMMA-SP and WGMMA-SP. + // ----------------------------------------------------------------------- + localparam PWD = TCU_META_PER_WARP_DEPTH; + `UNUSED_PARAM (PWD) + + // LSU bus carries LSU_WORD_SIZE-byte chunks per lane (= XLEN/8 bytes). + // The metadata layout is contiguous 32-bit words, so when XLEN > 32 + // each LSU response carries XLEN/32 logical 32-bit words and we need + // to pick the right one. LSU_W32_RATIO = bytes/4, so XLEN=64 → 2. + localparam LSU_W32_RATIO = LSU_WORD_SIZE / 4; + localparam LG_W32_RATIO = (LSU_W32_RATIO > 1) ? $clog2(LSU_W32_RATIO) : 0; + + // Per-lane sub-word selector: lane i reads 32-bit word i, so its half of + // the LSU word is fixed by i. + logic [NUM_LANES-1:0][`UP(LG_W32_RATIO)-1:0] half_sel_w; + always_comb begin + half_sel_w = '0; + for (int i = 0; i < NUM_LANES; i++) begin + if (LG_W32_RATIO > 0) begin + half_sel_w[i] = `UP(LG_W32_RATIO)'(i); + end + end + end + if (LG_W32_RATIO == 0) begin : g_half_sel_unused + `UNUSED_VAR (half_sel_w) + end + +`ifdef VX_CFG_LMEM_ENABLE + // Equality on the word-address bits above the window: the base is asserted + // aligned to the capacity, so the prefix compare is exact and needs no end + // address -- which is not representable when the window ends at the top of + // the address space. + localparam LMEM_WORD_LSB = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE); + localparam [LSU_ADDR_WIDTH-1:0] LMEM_WORD_BASE = + LSU_ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> `CLOG2(LSU_WORD_SIZE)); + `STATIC_ASSERT (LSU_ADDR_WIDTH > LMEM_WORD_LSB, ("LMEM window wider than the word address space")) +`endif + + function automatic lsu_req_data_t make_req(input logic [LSU_ADDR_WIDTH-1:0] base_word_addr, + input logic [BLOCK_IDX_BITS-1:0] block); + lsu_req_data_t r; + logic base_is_local; + `ifdef VX_CFG_LMEM_ENABLE + base_is_local = (base_word_addr[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB] + == LMEM_WORD_BASE[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB]); + `else + base_is_local = 1'b0; + `endif + r = '0; + r.rw = 1'b0; // load + r.mask = {NUM_LANES{1'b1}}; + for (int i = 0; i < NUM_LANES; i++) begin + mem_bus_attr_t a; + a = '0; + a.is_addr_local = base_is_local; + // base_addr is pre-advanced per TCU_LD slot by the kernel, so lane + // i reads 32-bit word i relative to it, in LSU-word units. + r.addr[i] = base_word_addr + (LSU_ADDR_WIDTH'(i) >> LG_W32_RATIO); + r.byteen[i] = {LSU_WORD_SIZE{1'b1}}; + r.data[i] = '0; + r.attr[i] = a; + end + r.tag = '0; + r.tag[BLOCK_IDX_BITS-1:0] = block; + return r; + endfunction + + `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) + + // Per-block request and response events, from either port structure. + wire [BLOCK_SIZE-1:0] req_fire_b; + wire [BLOCK_SIZE-1:0] rsp_fire_b; + wire [BLOCK_SIZE-1:0] rsp_eop_b; + wire [NUM_LANES-1:0] rsp_mask_b [BLOCK_SIZE]; + wire [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] rsp_data_b [BLOCK_SIZE]; + wire [`VX_CFG_XLEN-1:0] req_addr_chk [NUM_PORTS]; + wire [NUM_PORTS-1:0] req_fire_chk; + + if (NUM_PORTS == 1) begin : g_shared_port + wire [BLOCK_IDX_BITS-1:0] req_block; + wire [BLOCK_SIZE-1:0] req_grant; + wire req_valid; + + VX_generic_arbiter #( + .NUM_REQS (BLOCK_SIZE), + .TYPE ("R") + ) req_arb ( + .clk (clk), + .reset (reset), + .requests (req_pending), + .grant_index (req_block), + .grant_onehot (req_grant), + .grant_valid (req_valid), + .grant_ready (client_if[0].req_ready) + ); + `UNUSED_VAR (req_grant) + + wire [BLOCK_IDX_BITS-1:0] rsp_block = client_if[0].rsp_data.tag[BLOCK_IDX_BITS-1:0]; + wire req_fire = req_valid && client_if[0].req_ready; + wire rsp_fire = client_if[0].rsp_valid && client_if[0].rsp_ready; + + for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_ev + assign req_fire_b[bi] = req_fire && (req_block == BLOCK_IDX_BITS'(bi)); + assign rsp_fire_b[bi] = rsp_fire && (rsp_block == BLOCK_IDX_BITS'(bi)); + assign rsp_eop_b[bi] = client_if[0].rsp_data.eop; + assign rsp_mask_b[bi] = client_if[0].rsp_data.mask; + assign rsp_data_b[bi] = client_if[0].rsp_data.data; + end + + assign client_if[0].req_valid = req_valid; + assign client_if[0].req_data = make_req(LSU_ADDR_WIDTH'(addr_r[req_block] >> `CLOG2(LSU_WORD_SIZE)), req_block); + assign client_if[0].rsp_ready = busy_r[rsp_block] && issued_r[rsp_block] + && !complete_r[rsp_block]; + assign req_addr_chk[0] = addr_r[req_block]; + assign req_fire_chk[0] = req_fire; + `UNUSED_VAR (client_if[0].rsp_data.sop) + end else begin : g_port_per_block + for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_port + assign client_if[bi].req_valid = req_pending[bi]; + assign client_if[bi].req_data = make_req(LSU_ADDR_WIDTH'(addr_r[bi] >> `CLOG2(LSU_WORD_SIZE)), BLOCK_IDX_BITS'(0)); + assign client_if[bi].rsp_ready = busy_r[bi] && issued_r[bi] && !complete_r[bi]; + + assign req_fire_b[bi] = req_pending[bi] && client_if[bi].req_ready; + assign rsp_fire_b[bi] = client_if[bi].rsp_valid && client_if[bi].rsp_ready; + assign rsp_eop_b[bi] = client_if[bi].rsp_data.eop; + assign rsp_mask_b[bi] = client_if[bi].rsp_data.mask; + assign rsp_data_b[bi] = client_if[bi].rsp_data.data; + assign req_addr_chk[bi] = addr_r[bi]; + assign req_fire_chk[bi] = req_fire_b[bi]; + `UNUSED_VAR (client_if[bi].rsp_data.sop) + `UNUSED_VAR (client_if[bi].rsp_data.tag) + end + end + always @(posedge clk) begin if (reset) begin busy_r <= '0; @@ -132,21 +254,20 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( rsp_data_r[bi] <= '0; rsp_lane_done_r[bi] <= '0; end - end - if (req_fire) begin - issued_r[req_block] <= 1'b1; - end - if (rsp_fire) begin - for (int t = 0; t < NUM_LANES; ++t) begin - if (client_if.rsp_data.mask[t]) begin - rsp_data_r[rsp_block][t] <= client_if.rsp_data.data[t]; - rsp_lane_done_r[rsp_block][t] <= 1'b1; - end + if (req_fire_b[bi]) begin + issued_r[bi] <= 1'b1; end - if (client_if.rsp_data.eop - && ((rsp_lane_done_r[rsp_block] | client_if.rsp_data.mask) - == {NUM_LANES{1'b1}})) begin - complete_r[rsp_block] <= 1'b1; + if (rsp_fire_b[bi]) begin + for (int t = 0; t < NUM_LANES; ++t) begin + if (rsp_mask_b[bi][t]) begin + rsp_data_r[bi][t] <= rsp_data_b[bi][t]; + rsp_lane_done_r[bi][t] <= 1'b1; + end + end + if (rsp_eop_b[bi] + && ((rsp_lane_done_r[bi] | rsp_mask_b[bi]) == {NUM_LANES{1'b1}})) begin + complete_r[bi] <= 1'b1; + end end end if (commit_fire) begin @@ -158,116 +279,24 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( end end - `UNUSED_VAR (client_if.rsp_data.sop) - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_ld_ready assign per_block_ld_ready[bi] = ~busy_r[bi]; end - // ----------------------------------------------------------------------- - // Memory request: address from rs1 (warp-broadcast). NUM_LANES mask - // all-ones — we fetch the metadata tile in one warp-level request. - // - // Per-lane offset matches the host's pack_metadata_wg layout - // (sgemm_tcu_wg_sp/main.cpp): the host writes bank `b` and word `c` - // at index `b * stride + c`. The AGU writes wr_data[T] into - // VX_tcu_meta at (bank=T%PWD, col_in_group=T/PWD), so lane T must - // load `meta_base[(T % PWD) * stride + (T / PWD)]`. - // ----------------------------------------------------------------------- - wire [`VX_CFG_XLEN-1:0] base_addr = addr_r[req_block]; - `UNUSED_VAR (base_addr[`CLOG2(LSU_WORD_SIZE)-1:0]) - - wire [LSU_ADDR_WIDTH-1:0] base_word_addr = - LSU_ADDR_WIDTH'(base_addr[`VX_CFG_XLEN-1:`CLOG2(LSU_WORD_SIZE)]); - - localparam PWD = TCU_META_PER_WARP_DEPTH; - // Per-thread metadata layout: lane T loads h_meta[slot*NT + T]. The - // host packs h_meta so the 32-bit word at offset (slot*NT + T) holds - // the metadata destined for SRAM cell (bank = T%PWD, col = T/PWD). - // One formula serves both WMMA-SP and WGMMA-SP. - - // LSU bus carries LSU_WORD_SIZE-byte chunks per lane (= XLEN/8 bytes). - // The metadata layout is contiguous 32-bit words, so when XLEN > 32 - // each LSU response carries XLEN/32 logical 32-bit words and we need - // to pick the right one. LSU_W32_RATIO = bytes/4, so XLEN=64 → 2. - localparam LSU_W32_RATIO = LSU_WORD_SIZE / 4; - localparam LG_W32_RATIO = (LSU_W32_RATIO > 1) ? $clog2(LSU_W32_RATIO) : 0; - - // is_addr_local route bit — LMEM range check. The metadata buffer is - // per-warp inside shared memory so LMEM detection is uniform across - // lanes; compute once from the base address. -`ifdef VX_CFG_LMEM_ENABLE - // Equality on the word-address bits above the window: the base is asserted - // aligned to the capacity, so the prefix compare is exact and needs no end - // address -- which is not representable when the window ends at the top of - // the address space. - localparam LMEM_WORD_LSB = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE); - localparam [LSU_ADDR_WIDTH-1:0] LMEM_WORD_BASE = - LSU_ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> `CLOG2(LSU_WORD_SIZE)); - `STATIC_ASSERT (LSU_ADDR_WIDTH > LMEM_WORD_LSB, ("LMEM window wider than the word address space")) - wire base_is_local = (base_word_addr[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB] - == LMEM_WORD_BASE[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB]); -`else - wire base_is_local = 1'b0; -`endif - - // Per-lane sub-word selector: which 32-bit slice of the LSU response - // carries the desired metadata word. For XLEN=32 (LSU_W32_RATIO=1) - // every lane reads exactly one 32-bit word, so half_sel is always 0. - // For XLEN=64 (LSU_W32_RATIO=2) we round the 32-bit word offset down - // to an 8-byte LSU address and use the LSB to select low/high half. - logic [NUM_LANES-1:0][`UP(LG_W32_RATIO)-1:0] half_sel_w; - // For XLEN=32, LG_W32_RATIO=0 and half_sel_w is never read downstream; - // tag it unused so Verilator's UNUSEDSIGNAL doesn't fail the build. - if (LG_W32_RATIO == 0) begin : g_half_sel_unused - `UNUSED_VAR (half_sel_w) - end - - lsu_req_data_t req_w; - always_comb begin - req_w = '0; - req_w.rw = 1'b0; // load - req_w.mask = {NUM_LANES{1'b1}}; - half_sel_w = '0; - for (int i = 0; i < NUM_LANES; i++) begin - automatic mem_bus_attr_t a; - automatic logic [LSU_ADDR_WIDTH-1:0] word_off_32; - a = '0; - a.is_addr_local = base_is_local; - // base_addr is pre-advanced per TCU_LD slot by the kernel - // (slot s passes base + s*NT*sizeof(float)), so lane i reads word i - // relative to base_word_addr. - word_off_32 = LSU_ADDR_WIDTH'(i); - // word_off_32 is in 32-bit-word units. The LSU bus addresses - // LSU_WORD_SIZE-byte chunks (= XLEN/32 × 32-bit words), so we - // shift down to bus stride and keep the sub-word half for the - // response extraction below. - req_w.addr[i] = base_word_addr + (word_off_32 >> LG_W32_RATIO); - if (LG_W32_RATIO > 0) begin - half_sel_w[i] = `UP(LG_W32_RATIO)'(word_off_32[`UP(LG_W32_RATIO)-1:0]); - end - req_w.byteen[i] = {LSU_WORD_SIZE{1'b1}}; - req_w.data[i] = '0; - req_w.attr[i] = a; - end - req_w.tag = '0; - req_w.tag[BLOCK_IDX_BITS-1:0] = req_block; - end - `UNUSED_PARAM (PWD) - `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) `ifdef VX_CFG_LSU_STACK_INTERLEAVE_ENABLE // Metadata is read as a linear warp-wide tile, which an interleaved // per-thread stack cannot provide. if (`VX_CFG_NUM_THREADS > 1) begin : g_meta_stack_check - wire [`VX_CFG_XLEN-1:0] meta_end_addr = base_addr + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); - `RUNTIME_ASSERT(~req_fire || (meta_end_addr < STACK_WINDOW_BOTTOM) || (base_addr >= STACK_WINDOW_TOP), - ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, base_addr)) + for (genvar pi = 0; pi < NUM_PORTS; ++pi) begin : g_port + wire [`VX_CFG_XLEN-1:0] meta_end_addr = req_addr_chk[pi] + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); + `RUNTIME_ASSERT(~req_fire_chk[pi] || (meta_end_addr < STACK_WINDOW_BOTTOM) || (req_addr_chk[pi] >= STACK_WINDOW_TOP), + ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, req_addr_chk[pi])) + end end +`else + `UNUSED_VAR (req_addr_chk) + `UNUSED_VAR (req_fire_chk) `endif - assign client_if.req_valid = req_valid; - assign client_if.req_data = req_w; - assign client_if.rsp_ready = busy_r[rsp_block] && issued_r[rsp_block] - && !complete_r[rsp_block]; // ----------------------------------------------------------------------- // VX_tcu_meta write (broadcast to all blocks' meta instances). diff --git a/hw/rtl/tcu/VX_tcu_shared.sv b/hw/rtl/tcu/VX_tcu_shared.sv index 77ff231a72..4fee552c69 100644 --- a/hw/rtl/tcu/VX_tcu_shared.sv +++ b/hw/rtl/tcu/VX_tcu_shared.sv @@ -45,8 +45,8 @@ module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( output wire [NW_WIDTH-1:0] meta_wr_wid, output wire [4:0] meta_wr_idx, output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, - // TCU_LD memory client of the LSU scheduler - VX_lsu_sched_if.master tcu_mem_if, + // TCU_LD memory clients, one per LSU block's scheduler + VX_lsu_sched_if.master tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif input wire clk, @@ -94,17 +94,41 @@ module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // counts must match for the per-lane completion tracking to be correct. `STATIC_ASSERT (`VX_CFG_NUM_TCU_LANES == `VX_CFG_NUM_LSU_LANES, ("VX_tcu_agu requires NUM_TCU_LANES == NUM_LSU_LANES")); + // With one LSU block per TCU block, each block loads its metadata + // through the scheduler of the LSU block that serves the same warps; + // otherwise every block shares LSU block 0's scheduler. + localparam AGU_PORTS = (`VX_CFG_NUM_LSU_BLOCKS == BLOCK_SIZE) ? BLOCK_SIZE : 1; + for (genvar b = AGU_PORTS; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_mem_tieoff + assign tcu_mem_if[b].req_valid = 1'b0; + assign tcu_mem_if[b].req_data = '0; + assign tcu_mem_if[b].rsp_ready = 1'b1; + `UNUSED_VAR (tcu_mem_if[b].req_ready) + `UNUSED_VAR (tcu_mem_if[b].rsp_valid) + `UNUSED_VAR (tcu_mem_if[b].rsp_data) + end + + VX_lsu_sched_if agu_mem_if [AGU_PORTS](); + for (genvar b = 0; b < AGU_PORTS; ++b) begin : g_mem_port + assign tcu_mem_if[b].req_valid = agu_mem_if[b].req_valid; + assign tcu_mem_if[b].req_data = agu_mem_if[b].req_data; + assign agu_mem_if[b].req_ready = tcu_mem_if[b].req_ready; + assign agu_mem_if[b].rsp_valid = tcu_mem_if[b].rsp_valid; + assign agu_mem_if[b].rsp_data = tcu_mem_if[b].rsp_data; + assign tcu_mem_if[b].rsp_ready = agu_mem_if[b].rsp_ready; + end + VX_tcu_agu #( .INSTANCE_ID (`SFORMATF(("%s-agu", INSTANCE_ID))), .BLOCK_SIZE (BLOCK_SIZE), - .NUM_LANES (`VX_CFG_NUM_TCU_LANES) + .NUM_LANES (`VX_CFG_NUM_TCU_LANES), + .NUM_PORTS (AGU_PORTS) ) agu ( .clk (clk), .reset (reset), .per_block_ld_valid (agu_ld_valid), .per_block_ld_data (agu_ld_data), .per_block_ld_ready (agu_ld_ready), - .client_if (tcu_mem_if), + .client_if (agu_mem_if), .meta_wr_en (meta_wr_en), .meta_wr_wid (meta_wr_wid), .meta_wr_idx (meta_wr_idx), diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index bd84b24ea5..bfc91db066 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -285,11 +285,13 @@ class Core::Impl { } #endif #ifdef TCU_META_ENABLE - // Bind the TCU metadata AGU to the LSU block-0 client port. + // Bind the TCU metadata AGU ports to the LSU client ports. { auto lsu = std::static_pointer_cast(func_units_.at((int)FUType::LSU)); - tcu_unit_->agu_req_out.bind(&lsu->TcuReqIn); - lsu->TcuRspOut.bind(&tcu_unit_->agu_rsp_in); + for (uint32_t p = 0; p < TCU_AGU_PORTS; ++p) { + tcu_unit_->agu_req_out.at(p).bind(&lsu->TcuReqIn.at(p)); + lsu->TcuRspOut.at(p).bind(&tcu_unit_->agu_rsp_in.at(p)); + } } #endif #endif diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index e17a131bee..eb0b032917 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -78,8 +78,8 @@ LsuUnit::LsuUnit(const SimContext& ctx, const char* name, Core* core) // instead of throttling the memory response stream. : FuncUnit(ctx, name, core, 6) #ifdef TCU_META_ENABLE - , TcuReqIn(this) - , TcuRspOut(this) + , TcuReqIn(TCU_AGU_PORTS, this) + , TcuRspOut(TCU_AGU_PORTS, this) #endif , pending_loads_(0) {} @@ -94,8 +94,10 @@ bool LsuUnit::drained() const { #ifdef TCU_META_ENABLE // TcuRspOut forwards to the TCU's endpoint, so query via size() (empty() // asserts on forwarding channels). - if (!TcuReqIn.empty() || TcuRspOut.size() != 0) { - return false; + for (uint32_t p = 0; p < TCU_AGU_PORTS; ++p) { + if (!TcuReqIn.at(p).empty() || TcuRspOut.at(p).size() != 0) { + return false; + } } #endif for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { @@ -221,11 +223,11 @@ void LsuUnit::process_response_step(uint32_t b) { // TCU metadata response: forward the fragment with the client's tag // restored; no register writeback, no trace retirement. if (entry.is_tcu) { - if (TcuRspOut.full()) + if (TcuRspOut.at(b).full()) return; // stall LsuRsp fwd = lsu_rsp; fwd.tag = entry.client_tag; - TcuRspOut.send(fwd); + TcuRspOut.at(b).send(fwd); DT(3, this->name() << " tcu-meta-rsp: " << fwd); entry.count -= lsu_rsp.mask.count(); if (entry.count == 0) { @@ -422,15 +424,15 @@ void LsuUnit::process_request_step(uint32_t b) { } #ifdef TCU_META_ENABLE - // TCU metadata client (block 0 only): forward one pending request per + // TCU metadata client (blocks below TCU_AGU_PORTS): forward one pending request per // cycle into lmem_switch, ahead of the block's own dispatch. The wire tag // is re-allocated from pending_reqs so it stays unique on the switch; the // client's own tag is restored on the response. An independent client, so // not gated by the fence lock — its entries still hold fence release via // pending_reqs.empty(). - if (b == 0 && !TcuReqIn.empty()) { - if (!state.pending_reqs.full() && !core_->lmem_switch(0)->ReqIn.full()) { - LsuReq lsu_req = TcuReqIn.peek(); + if (b < TCU_AGU_PORTS && !TcuReqIn.at(b).empty()) { + if (!state.pending_reqs.full() && !core_->lmem_switch(b)->ReqIn.full()) { + LsuReq lsu_req = TcuReqIn.at(b).peek(); uint32_t count = lsu_req.mask.count(); std::vector lane_entries(VX_CFG_NUM_LSU_LANES); for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { @@ -452,9 +454,9 @@ void LsuUnit::process_request_step(uint32_t b) { entry.client_tag = lsu_req.tag; uint32_t tag = state.pending_reqs.allocate(std::move(entry)); lsu_req.tag = tag; - core_->lmem_switch(0)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req); DT(3, this->name() << " tcu-meta-req: " << lsu_req); - TcuReqIn.pop(); + TcuReqIn.at(b).pop(); return; // one request per cycle into the switch } } diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index 8ab00012dc..6030fc0a39 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -27,6 +27,13 @@ #define TCU_META_ENABLE #endif +// TCU_LD memory ports: one per TCU block, through the LSU block that serves +// the same warps, when the LSU has one block per TCU block; otherwise all +// TCU blocks share LSU block 0. Same derivation as the RTL's AGU_PORTS. +#ifndef TCU_AGU_PORTS +#define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) +#endif + namespace vortex { // Micro-op generator for packed-load macro instructions (PACKLB.F / PACKLH.F). @@ -79,11 +86,12 @@ class LsuUnit : public FuncUnit { ~LsuUnit(); #ifdef TCU_META_ENABLE - // TCU metadata client port on block 0: TCU_LD load requests arbitrate - // with the LSU's own traffic into lmem_switch; response fragments return - // here with the client's own tag restored. - SimChannel TcuReqIn; - SimChannel TcuRspOut; + // TCU metadata client ports (TCU_AGU_PORTS, starting at block 0): TCU_LD + // load requests arbitrate with the block's own traffic into its + // lmem_switch; response fragments return here with the client's own tag + // restored. + std::vector> TcuReqIn; + std::vector> TcuRspOut; #endif // Returns true when all LSU blocks have no in-flight requests. diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index 5e2bd89173..0e88015cb3 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -452,58 +452,82 @@ class TcuUnit::Impl { } } - void agu_step() { - if (!simobject_->agu_req_out.full()) { - for (uint32_t offset = 0; offset < VX_CFG_NUM_TCU_BLOCKS; ++offset) { - uint32_t block = (agu_issue_rr_ + offset) % VX_CFG_NUM_TCU_BLOCKS; - auto& agu = agu_.at(block); - if (!agu.busy || agu.next_beat >= kAguBeats) { - continue; - } - LsuReq req(VX_CFG_NUM_LSU_LANES); - req.op = MemOp::LD; - req.tag = block * kAguBeats + agu.next_beat; - req.wid = agu.wid; - req.cid = agu.trace->cid; - req.uuid = agu.trace->uuid; - for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { - uint32_t idx = agu.next_beat * VX_CFG_NUM_LSU_LANES + i; - if (idx >= VX_CFG_NUM_THREADS) { - break; - } - req.mask.set(i); - req.addrs.at(i) = agu.addrs.at(idx); - req.tids.at(i) = idx % VX_CFG_NUM_THREADS; - } - simobject_->agu_req_out.send(req); - ++agu.next_beat; - agu_issue_rr_ = (block + 1) % VX_CFG_NUM_TCU_BLOCKS; + LsuReq agu_make_req(uint32_t block, agu_state_t& agu) { + LsuReq req(VX_CFG_NUM_LSU_LANES); + req.op = MemOp::LD; + req.tag = block * kAguBeats + agu.next_beat; + req.wid = agu.wid; + req.cid = agu.trace->cid; + req.uuid = agu.trace->uuid; + for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { + uint32_t idx = agu.next_beat * VX_CFG_NUM_LSU_LANES + i; + if (idx >= VX_CFG_NUM_THREADS) { break; } + req.mask.set(i); + req.addrs.at(i) = agu.addrs.at(idx); + req.tids.at(i) = idx % VX_CFG_NUM_THREADS; } + ++agu.next_beat; + return req; + } - if (!simobject_->agu_rsp_in.empty()) { - auto& rsp = simobject_->agu_rsp_in.peek(); - uint32_t block = (uint32_t)rsp.tag / kAguBeats; - uint32_t beat = (uint32_t)rsp.tag % kAguBeats; - auto& agu = agu_.at(block); - for (uint32_t lane = 0; lane < rsp.mask.size(); ++lane) { - if (!rsp.mask.test(lane)) { + void agu_receive(const LsuRsp& rsp) { + uint32_t block = (uint32_t)rsp.tag / kAguBeats; + uint32_t beat = (uint32_t)rsp.tag % kAguBeats; + auto& agu = agu_.at(block); + for (uint32_t lane = 0; lane < rsp.mask.size(); ++lane) { + if (!rsp.mask.test(lane)) { + continue; + } + uint32_t idx = beat * VX_CFG_NUM_LSU_LANES + lane; + assert(idx < VX_CFG_NUM_THREADS && rsp.data.at(lane)); + uint32_t off = agu.addrs.at(idx) & (VX_CFG_MEM_BLOCK_SIZE - 1); + uint32_t word = 0; + std::memcpy(&word, rsp.data.at(lane)->data() + off, 4); + agu.words.at(idx) = word; + ++agu.rsp_count; + } + if (agu.busy && !agu.complete && agu.rsp_count == VX_CFG_NUM_THREADS) { + this->agu_deposit(agu); + agu.complete = true; + } + } + + void agu_step() { + if (TCU_AGU_PORTS == 1) { + // every block shares one port: round-robin one request per cycle + if (!simobject_->agu_req_out.at(0).full()) { + for (uint32_t offset = 0; offset < VX_CFG_NUM_TCU_BLOCKS; ++offset) { + uint32_t block = (agu_issue_rr_ + offset) % VX_CFG_NUM_TCU_BLOCKS; + auto& agu = agu_.at(block); + if (!agu.busy || agu.next_beat >= kAguBeats) { + continue; + } + simobject_->agu_req_out.at(0).send(this->agu_make_req(block, agu)); + agu_issue_rr_ = (block + 1) % VX_CFG_NUM_TCU_BLOCKS; + break; + } + } + } else { + // each block issues on its own port + for (uint32_t block = 0; block < VX_CFG_NUM_TCU_BLOCKS; ++block) { + auto& agu = agu_.at(block); + if (!agu.busy || agu.next_beat >= kAguBeats + || simobject_->agu_req_out.at(block).full()) { continue; } - uint32_t idx = beat * VX_CFG_NUM_LSU_LANES + lane; - assert(idx < VX_CFG_NUM_THREADS && rsp.data.at(lane)); - uint32_t off = agu.addrs.at(idx) & (VX_CFG_MEM_BLOCK_SIZE - 1); - uint32_t word = 0; - std::memcpy(&word, rsp.data.at(lane)->data() + off, 4); - agu.words.at(idx) = word; - ++agu.rsp_count; + simobject_->agu_req_out.at(block).send(this->agu_make_req(block, agu)); } - simobject_->agu_rsp_in.pop(); - if (agu.busy && !agu.complete && agu.rsp_count == VX_CFG_NUM_THREADS) { - this->agu_deposit(agu); - agu.complete = true; + } + + for (uint32_t p = 0; p < TCU_AGU_PORTS; ++p) { + auto& rsp_in = simobject_->agu_rsp_in.at(p); + if (rsp_in.empty()) { + continue; } + this->agu_receive(rsp_in.peek()); + rsp_in.pop(); } } @@ -1834,8 +1858,8 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { TcuUnit::TcuUnit(const SimContext &ctx, const char* name, Core* core) : FuncUnit(ctx, name, core, 1u << log2ceil(VX_CFG_TCU_LATENCY + 1)) // landing-queue depth bounds results awaiting the consumer #ifdef TCU_META_ENABLE - , agu_req_out(this) - , agu_rsp_in(this) + , agu_req_out(TCU_AGU_PORTS, this) + , agu_rsp_in(TCU_AGU_PORTS, this) #endif , impl_(new Impl(this, core)) { diff --git a/sim/simx/tcu/tcu_unit.h b/sim/simx/tcu/tcu_unit.h index 93328060f4..681c448b5c 100644 --- a/sim/simx/tcu/tcu_unit.h +++ b/sim/simx/tcu/tcu_unit.h @@ -28,6 +28,13 @@ #define TCU_META_ENABLE #endif +// TCU_LD memory ports: one per TCU block, through the LSU block that serves +// the same warps, when the LSU has one block per TCU block; otherwise all +// TCU blocks share LSU block 0. Same derivation as the RTL's AGU_PORTS. +#ifndef TCU_AGU_PORTS +#define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) +#endif + namespace vortex { class Core; @@ -77,11 +84,11 @@ class TcuUnit : public FuncUnit { virtual ~TcuUnit(); #ifdef TCU_META_ENABLE - // Metadata AGU port: TCU_LD issues load requests through the LSU block-0 - // client port; response fragments return here and accumulate into the - // metadata SRAM. - SimChannel agu_req_out; - SimChannel agu_rsp_in; + // Metadata AGU ports (TCU_AGU_PORTS): TCU_LD issues load requests through + // the LSU client ports; response fragments return here and accumulate into + // the metadata SRAM. + std::vector> agu_req_out; + std::vector> agu_rsp_in; #endif void wmma(uint32_t wid, From fc0a35bb423603e919343fc1b1a67e58aeaaa315 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 04:43:07 -0700 Subject: [PATCH 088/336] tcu: move the TCU_LD AGU into each TCU block and drop its gather buffer The metadata AGU was one core-level unit: a TCU_LD left its slice for the AGU and its metadata came back over a core-wide write bus. Each TCU block now holds its own AGU, wired to the scheduler of the LSU block serving the same warps (inside the execute slice in the sliced core modes) and to the block's own metadata RAM. The AGU no longer buffers a warp's response. Each response fragment writes its lanes straight into the metadata RAM through a new per-lane write mask, and the scheduler's end-of-packet marks completion, so the per-block gather buffer (NUM_LANES x XLEN bits), the lane-done mask, the request and commit arbiters, the cross-block commit mux and the core-wide write bus are gone. A block keeps only busy/issued/done flags, the address, slot and header. Writing early is safe: TCU_LD enters its block in order behind the older MMA micro-ops, which read the metadata RAM when they issue. When the LSU has fewer blocks than the TCU, the blocks share LSU block 0's client through a round-robin arbiter in VX_tcu_unit, as before. SimX now writes each returned lane into the metadata on arrival, matching the RTL. sgemm_tcu_wg_sp_dxa 128x128x128, rtlsim (IW4, FW4, 32 threads), PASSED: stage-sliced 144,956 cycles, compute slices 144,830, both identical to the previous commit; SimX 145,323, identical. Lint clean in 13 configs (XLEN 32/64; stage-sliced, backend and compute slices; shared LSU port; sparse and MX). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_backend_slice.sv | 30 --- hw/rtl/core/VX_compute_slice.sv | 30 --- hw/rtl/core/VX_core.sv | 73 ------ hw/rtl/core/VX_execute.sv | 31 +-- hw/rtl/core/VX_execute_slice.sv | 31 +-- hw/rtl/tcu/VX_tcu_agu.sv | 380 +++++++++----------------------- hw/rtl/tcu/VX_tcu_block.sv | 48 ++-- hw/rtl/tcu/VX_tcu_core.sv | 6 +- hw/rtl/tcu/VX_tcu_meta.sv | 29 ++- hw/rtl/tcu/VX_tcu_shared.sv | 70 +----- hw/rtl/tcu/VX_tcu_unit.sv | 98 +++++--- sim/simx/lsu_unit.h | 2 +- sim/simx/tcu/tcu_unit.cpp | 51 ++--- sim/simx/tcu/tcu_unit.h | 2 +- 14 files changed, 272 insertions(+), 609 deletions(-) diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv index 385db201a5..2d7059d12a 100644 --- a/hw/rtl/core/VX_backend_slice.sv +++ b/hw/rtl/core/VX_backend_slice.sv @@ -61,9 +61,6 @@ import VX_tcu_pkg::*; output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, -`endif -`ifdef TCU_META_ENABLE - VX_lsu_sched_if.slave lsu_tcu_mem_if, `endif output wire lsu_sched_empty, output wire lsu_store_empty, @@ -93,18 +90,6 @@ import VX_tcu_pkg::*; input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, input wire tcu_tbuf_ready, `endif -`ifdef TCU_META_ENABLE - output wire tcu_agu_ld_valid, - output tcu_execute_t tcu_agu_ld_data, - input wire tcu_agu_ld_ready, - input wire tcu_agu_result_valid, - input tcu_result_t tcu_agu_result_data, - output wire tcu_agu_result_ready, - input wire tcu_meta_wr_en, - input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, - input wire [4:0] tcu_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, -`endif // core-wide warp control and CTA contexts VX_warp_shared_if.master warp_shared_if, @@ -162,9 +147,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), .lsu_perf_rsp_fire (lsu_perf_rsp_fire), .lsu_coalescer_misses (lsu_coalescer_misses), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if), `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), @@ -190,18 +172,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), .tcu_tbuf_ready (tcu_tbuf_ready), - `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid), - .tcu_agu_ld_data (tcu_agu_ld_data), - .tcu_agu_ld_ready (tcu_agu_ld_ready), - .tcu_agu_result_valid (tcu_agu_result_valid), - .tcu_agu_result_data (tcu_agu_result_data), - .tcu_agu_result_ready (tcu_agu_result_ready), - .tcu_meta_wr_en (tcu_meta_wr_en), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), `endif .warp_shared_if (warp_shared_if), .warp_ctl_if (warp_ctl_if), diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index f241dd9d96..9eb31133da 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -64,9 +64,6 @@ import VX_tcu_pkg::*; output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, -`endif -`ifdef TCU_META_ENABLE - VX_lsu_sched_if.slave lsu_tcu_mem_if, `endif output wire lsu_sched_empty, output wire lsu_store_empty, @@ -96,18 +93,6 @@ import VX_tcu_pkg::*; input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, input wire tcu_tbuf_ready, `endif -`ifdef TCU_META_ENABLE - output wire tcu_agu_ld_valid, - output tcu_execute_t tcu_agu_ld_data, - input wire tcu_agu_ld_ready, - input wire tcu_agu_result_valid, - input tcu_result_t tcu_agu_result_data, - output wire tcu_agu_result_ready, - input wire tcu_meta_wr_en, - input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, - input wire [4:0] tcu_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, -`endif // core-wide warp control VX_warp_shared_if.master warp_shared_if @@ -262,9 +247,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), .lsu_perf_rsp_fire (lsu_perf_rsp_fire), .lsu_coalescer_misses (lsu_coalescer_misses), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if), `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), @@ -289,18 +271,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), .tcu_tbuf_ready (tcu_tbuf_ready), - `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid), - .tcu_agu_ld_data (tcu_agu_ld_data), - .tcu_agu_ld_ready (tcu_agu_ld_ready), - .tcu_agu_result_valid (tcu_agu_result_valid), - .tcu_agu_result_data (tcu_agu_result_data), - .tcu_agu_result_ready (tcu_agu_result_ready), - .tcu_meta_wr_en (tcu_meta_wr_en), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), `endif .warp_shared_if (warp_shared_if), .cta_bcast (cta_bcast) diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index a5b31c5e58..c018683547 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -101,11 +101,6 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses; `endif -`ifdef TCU_META_ENABLE - // The TCU metadata AGU is the second client of the LSU schedulers. - VX_lsu_sched_if lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); -`endif - // frontend-slice icache requests; port 0 also carries the injected flush VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), @@ -317,16 +312,6 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; `endif -`ifdef TCU_META_ENABLE - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; - tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; - tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire tcu_meta_wr_en; - wire [NW_WIDTH-1:0] tcu_meta_wr_wid; - wire [4:0] tcu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; -`endif VX_sfu_shared #( .INSTANCE_ID (`SFORMATF(("%s-sfu-shared", INSTANCE_ID))) @@ -363,19 +348,6 @@ import VX_tcu_pkg::*; .tbuf_rs2_data (tcu_tbuf_rs2_data), .tbuf_ready_eff (tcu_tbuf_ready), .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (lsu_tcu_mem_if), `endif .clk (clk), .reset (reset) @@ -452,9 +424,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), .lsu_coalescer_misses (lsu_coalescer_misses[i]), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if[i]), `endif .lsu_sched_empty (lsu_sched_empty[i]), .lsu_store_empty (lsu_store_empty[i]), @@ -479,18 +448,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), .tcu_tbuf_ready (tcu_tbuf_ready[i]), - `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid[i]), - .tcu_agu_ld_data (tcu_agu_ld_data[i]), - .tcu_agu_ld_ready (tcu_agu_ld_ready[i]), - .tcu_agu_result_valid (tcu_agu_result_valid[i]), - .tcu_agu_result_data (tcu_agu_result_data[i]), - .tcu_agu_result_ready (tcu_agu_result_ready[i]), - .tcu_meta_wr_en (tcu_meta_wr_en && (NW_WIDTH'(tcu_meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(i))), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), `endif .warp_shared_if (warp_shared_if[i]) ); @@ -686,9 +643,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), .lsu_coalescer_misses (lsu_coalescer_misses[i]), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if[i]), `endif .lsu_sched_empty (lsu_sched_empty[i]), .lsu_store_empty (lsu_store_empty[i]), @@ -713,18 +667,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), .tcu_tbuf_ready (tcu_tbuf_ready[i]), - `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid[i]), - .tcu_agu_ld_data (tcu_agu_ld_data[i]), - .tcu_agu_ld_ready (tcu_agu_ld_ready[i]), - .tcu_agu_result_valid (tcu_agu_result_valid[i]), - .tcu_agu_result_data (tcu_agu_result_data[i]), - .tcu_agu_result_ready (tcu_agu_result_ready[i]), - .tcu_meta_wr_en (tcu_meta_wr_en && (NW_WIDTH'(tcu_meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(i))), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), `endif .warp_shared_if (warp_shared_if[i]), .cta_bcast (cta_bcast) @@ -782,9 +724,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), .lsu_perf_rsp_fire (lsu_perf_rsp_fire), .lsu_coalescer_misses (lsu_coalescer_misses), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if), `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), @@ -807,18 +746,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), .tcu_tbuf_ready (tcu_tbuf_ready), `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid), - .tcu_agu_ld_data (tcu_agu_ld_data), - .tcu_agu_ld_ready (tcu_agu_ld_ready), - .tcu_agu_result_valid (tcu_agu_result_valid), - .tcu_agu_result_data (tcu_agu_result_data), - .tcu_agu_result_ready (tcu_agu_result_ready), - .tcu_meta_wr_en (tcu_meta_wr_en), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), - `endif `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (block_dxa_req_bus_if), `endif diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 84ebe921fb..330e01a254 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -33,9 +33,6 @@ import VX_tcu_pkg::*; output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses, -`endif -`ifdef TCU_META_ENABLE - VX_lsu_sched_if.slave lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty, @@ -80,18 +77,6 @@ import VX_tcu_pkg::*; input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready, `endif -`ifdef TCU_META_ENABLE - output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, - output tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS], - input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_ready, - input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, - input tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS], - output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_ready, - input wire tcu_meta_wr_en, - input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, - input wire [4:0] tcu_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, -`endif // scheduler interfaces VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], @@ -120,6 +105,11 @@ import VX_tcu_pkg::*; `endif `SCOPE_IO_SWITCH (SCOPE_COUNT); +`ifdef TCU_META_ENABLE + // TCU_LD memory clients of the LSU blocks' schedulers + VX_lsu_sched_if lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); +`endif + VX_lsu_unit #( .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))) ) lsu_unit ( @@ -175,16 +165,7 @@ import VX_tcu_pkg::*; .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (lsu_tcu_mem_if), `endif .dispatch_if (dispatch_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index 7436dc3573..5542697bc2 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -50,9 +50,6 @@ import VX_tcu_pkg::*; output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, -`endif -`ifdef TCU_META_ENABLE - VX_lsu_sched_if.slave lsu_tcu_mem_if, `endif output wire lsu_sched_empty, output wire lsu_store_empty, @@ -83,18 +80,6 @@ import VX_tcu_pkg::*; input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, input wire tcu_tbuf_ready, `endif -`ifdef TCU_META_ENABLE - output wire tcu_agu_ld_valid, - output tcu_execute_t tcu_agu_ld_data, - input wire tcu_agu_ld_ready, - input wire tcu_agu_result_valid, - input tcu_result_t tcu_agu_result_data, - output wire tcu_agu_result_ready, - input wire tcu_meta_wr_en, - input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, - input wire [4:0] tcu_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, -`endif // SFU warp control and CSRs VX_warp_shared_if.master warp_shared_if, @@ -184,6 +169,11 @@ import VX_tcu_pkg::*; .execute_if (lsu_execute_if) ); +`ifdef TCU_META_ENABLE + // the TCU block's metadata loads go through this slice's LSU scheduler + VX_lsu_sched_if lsu_tcu_mem_if(); +`endif + VX_lsu_block #( .INSTANCE_ID (`SFORMATF(("%s-lsu%s", INSTANCE_ID, SLICE_STR))) ) lsu_block ( @@ -402,16 +392,7 @@ import VX_tcu_pkg::*; .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (lsu_tcu_mem_if), `endif .execute_if (tcu_execute_if[0]), .result_if (tcu_result_if[0]) diff --git a/hw/rtl/tcu/VX_tcu_agu.sv b/hw/rtl/tcu/VX_tcu_agu.sv index 70a77b8126..315917365f 100644 --- a/hw/rtl/tcu/VX_tcu_agu.sv +++ b/hw/rtl/tcu/VX_tcu_agu.sv @@ -15,113 +15,89 @@ `ifdef TCU_META_ENABLE -// Warp-level AGU for TCU_LD instructions. Each TCU block can hold one -// transaction. With one memory port the blocks share it; with one port per -// block (NUM_PORTS == BLOCK_SIZE) each block loads through its own LSU -// scheduler, so the loads of a warpgroup's warps proceed in parallel. The -// metadata write port is shared. +// A TCU block's TCU_LD unit: one warp-wide load in flight through an LSU +// scheduler client port. Response fragments are written straight into the +// block's metadata RAM, lane by lane, so nothing is buffered here. The load +// cannot overwrite metadata an older MMA still needs: TCU_LD enters the block +// in order behind that MMA, which reads the RAM when it issues. module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS, - parameter NUM_LANES = `VX_CFG_NUM_TCU_LANES, - parameter NUM_PORTS = 1 + parameter NUM_LANES = `VX_CFG_NUM_TCU_LANES ) ( - input wire clk, - input wire reset, + input wire clk, + input wire reset, - // Per-block dispatch observation. The unit feeds INST_TCU_LD - // instructions here (gated by op_type at the wrapper); other - // op_types are masked so the AGU never sees them on .valid. - input wire [BLOCK_SIZE-1:0] per_block_ld_valid, - input tcu_execute_t per_block_ld_data [BLOCK_SIZE], - output wire [BLOCK_SIZE-1:0] per_block_ld_ready, + input wire ld_valid, + input tcu_execute_t ld_data, + output wire ld_ready, - // Memory client connections (to the LSU schedulers). - VX_lsu_sched_if.master client_if [NUM_PORTS], + VX_lsu_sched_if.master client_if, - // Meta SRAM write port (broadcast — same data lands in every - // block's VX_tcu_meta so subsequent wmma_sp reads see it - // regardless of which block the warp resides on). - output wire meta_wr_en, - output wire [NW_WIDTH-1:0] meta_wr_wid, - output wire [4:0] meta_wr_idx, - output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, + output wire meta_wr_en, + output wire [NW_WIDTH-1:0] meta_wr_wid, + output wire [4:0] meta_wr_idx, + output wire [TCU_BLOCK_CAP-1:0] meta_wr_mask, + output wire [TCU_BLOCK_CAP-1:0][31:0] meta_wr_data, - // Result_if hand for the originating block. The wrapper muxes this - // with tcu_core's normal result_if path. - output wire [BLOCK_SIZE-1:0] result_valid, - output tcu_result_t result_data [BLOCK_SIZE], - input wire [BLOCK_SIZE-1:0] result_ready + // releases the TCU_LD's metadata scoreboard bit + output wire result_valid, + output tcu_result_t result_data, + input wire result_ready ); `UNUSED_SPARAM (INSTANCE_ID) - `STATIC_ASSERT ((NUM_PORTS == 1) || (NUM_PORTS == BLOCK_SIZE), ("invalid parameter: NUM_PORTS=%0d", NUM_PORTS)) - localparam BLOCK_IDX_BITS = (BLOCK_SIZE > 1) ? $clog2(BLOCK_SIZE) : 1; + `STATIC_ASSERT (NUM_LANES == `VX_CFG_NUM_LSU_LANES, ("TCU_LD requires NUM_TCU_LANES == NUM_LSU_LANES")) - logic [BLOCK_SIZE-1:0] busy_r; - logic [BLOCK_SIZE-1:0] issued_r; - logic [BLOCK_SIZE-1:0] complete_r; - tcu_header_t header_r [BLOCK_SIZE]; - logic [4:0] slot_r [BLOCK_SIZE]; - logic [`VX_CFG_XLEN-1:0] addr_r [BLOCK_SIZE]; - logic [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] - rsp_data_r [BLOCK_SIZE]; - logic [NUM_LANES-1:0] rsp_lane_done_r [BLOCK_SIZE]; - - wire [BLOCK_SIZE-1:0] req_pending = busy_r & ~issued_r; - - wire [BLOCK_SIZE-1:0] commit_pending = complete_r; - wire [BLOCK_IDX_BITS-1:0] commit_block; - wire [BLOCK_SIZE-1:0] commit_grant; - wire commit_valid; - wire commit_ready = result_ready[commit_block]; - - VX_generic_arbiter #( - .NUM_REQS (BLOCK_SIZE), - .TYPE ("R") - ) commit_arb ( - .clk (clk), - .reset (reset), - .requests (commit_pending), - .grant_index (commit_block), - .grant_onehot (commit_grant), - .grant_valid (commit_valid), - .grant_ready (commit_ready) - ); - wire commit_fire = commit_valid && commit_ready; - `UNUSED_VAR (commit_grant) - - // ----------------------------------------------------------------------- - // Memory request: address from rs1 (warp-broadcast). NUM_LANES mask - // all-ones — we fetch the metadata tile in one warp-level request. - // Lane T loads the 32-bit word at base + 4*T; the host packs h_meta so - // that word holds the metadata destined for SRAM cell - // (bank = T%PWD, col = T/PWD). One formula serves WMMA-SP and WGMMA-SP. - // ----------------------------------------------------------------------- - localparam PWD = TCU_META_PER_WARP_DEPTH; - `UNUSED_PARAM (PWD) - - // LSU bus carries LSU_WORD_SIZE-byte chunks per lane (= XLEN/8 bytes). - // The metadata layout is contiguous 32-bit words, so when XLEN > 32 - // each LSU response carries XLEN/32 logical 32-bit words and we need - // to pick the right one. LSU_W32_RATIO = bytes/4, so XLEN=64 → 2. + // An LSU word holds XLEN/32 metadata words; lane i reads word i. localparam LSU_W32_RATIO = LSU_WORD_SIZE / 4; localparam LG_W32_RATIO = (LSU_W32_RATIO > 1) ? $clog2(LSU_W32_RATIO) : 0; - // Per-lane sub-word selector: lane i reads 32-bit word i, so its half of - // the LSU word is fixed by i. - logic [NUM_LANES-1:0][`UP(LG_W32_RATIO)-1:0] half_sel_w; - always_comb begin - half_sel_w = '0; - for (int i = 0; i < NUM_LANES; i++) begin - if (LG_W32_RATIO > 0) begin - half_sel_w[i] = `UP(LG_W32_RATIO)'(i); + reg busy_r, issued_r, done_r; + tcu_header_t header_r; + reg [4:0] slot_r; + reg [`VX_CFG_XLEN-1:0] addr_r; + + wire ld_fire = ld_valid && ld_ready; + wire req_fire = client_if.req_valid && client_if.req_ready; + wire rsp_fire = client_if.rsp_valid && client_if.rsp_ready; + wire result_fire = result_valid && result_ready; + + always @(posedge clk) begin + if (reset) begin + busy_r <= 1'b0; + issued_r <= 1'b0; + done_r <= 1'b0; + end else begin + if (ld_fire) begin + busy_r <= 1'b1; + end + if (req_fire) begin + issued_r <= 1'b1; + end + if (rsp_fire && client_if.rsp_data.eop) begin + done_r <= 1'b1; + end + if (result_fire) begin + busy_r <= 1'b0; + issued_r <= 1'b0; + done_r <= 1'b0; end end + if (ld_fire) begin + header_r <= ld_data.header; + slot_r <= ld_data.op_args.tcu.fmt_d; + addr_r <= ld_data.rs1_data[0]; + end end - if (LG_W32_RATIO == 0) begin : g_half_sel_unused - `UNUSED_VAR (half_sel_w) - end + + assign ld_ready = ~busy_r; + `UNUSED_VAR (ld_data) + + // ----------------------------------------------------------------------- + // Request: one warp-wide load of NUM_LANES consecutive 32-bit words. + // ----------------------------------------------------------------------- + wire [LSU_ADDR_WIDTH-1:0] base_word_addr = LSU_ADDR_WIDTH'(addr_r >> `CLOG2(LSU_WORD_SIZE)); + `UNUSED_VAR (addr_r[`CLOG2(LSU_WORD_SIZE)-1:0]) `ifdef VX_CFG_LMEM_ENABLE // Equality on the word-address bits above the window: the base is asserted @@ -132,211 +108,75 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam [LSU_ADDR_WIDTH-1:0] LMEM_WORD_BASE = LSU_ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> `CLOG2(LSU_WORD_SIZE)); `STATIC_ASSERT (LSU_ADDR_WIDTH > LMEM_WORD_LSB, ("LMEM window wider than the word address space")) + wire base_is_local = (base_word_addr[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB] + == LMEM_WORD_BASE[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB]); +`else + wire base_is_local = 1'b0; `endif - function automatic lsu_req_data_t make_req(input logic [LSU_ADDR_WIDTH-1:0] base_word_addr, - input logic [BLOCK_IDX_BITS-1:0] block); - lsu_req_data_t r; - logic base_is_local; - `ifdef VX_CFG_LMEM_ENABLE - base_is_local = (base_word_addr[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB] - == LMEM_WORD_BASE[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB]); - `else - base_is_local = 1'b0; - `endif - r = '0; - r.rw = 1'b0; // load - r.mask = {NUM_LANES{1'b1}}; - for (int i = 0; i < NUM_LANES; i++) begin - mem_bus_attr_t a; - a = '0; - a.is_addr_local = base_is_local; - // base_addr is pre-advanced per TCU_LD slot by the kernel, so lane - // i reads 32-bit word i relative to it, in LSU-word units. - r.addr[i] = base_word_addr + (LSU_ADDR_WIDTH'(i) >> LG_W32_RATIO); - r.byteen[i] = {LSU_WORD_SIZE{1'b1}}; - r.data[i] = '0; - r.attr[i] = a; - end - r.tag = '0; - r.tag[BLOCK_IDX_BITS-1:0] = block; - return r; - endfunction - - `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) - - // Per-block request and response events, from either port structure. - wire [BLOCK_SIZE-1:0] req_fire_b; - wire [BLOCK_SIZE-1:0] rsp_fire_b; - wire [BLOCK_SIZE-1:0] rsp_eop_b; - wire [NUM_LANES-1:0] rsp_mask_b [BLOCK_SIZE]; - wire [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] rsp_data_b [BLOCK_SIZE]; - wire [`VX_CFG_XLEN-1:0] req_addr_chk [NUM_PORTS]; - wire [NUM_PORTS-1:0] req_fire_chk; - - if (NUM_PORTS == 1) begin : g_shared_port - wire [BLOCK_IDX_BITS-1:0] req_block; - wire [BLOCK_SIZE-1:0] req_grant; - wire req_valid; - - VX_generic_arbiter #( - .NUM_REQS (BLOCK_SIZE), - .TYPE ("R") - ) req_arb ( - .clk (clk), - .reset (reset), - .requests (req_pending), - .grant_index (req_block), - .grant_onehot (req_grant), - .grant_valid (req_valid), - .grant_ready (client_if[0].req_ready) - ); - `UNUSED_VAR (req_grant) - - wire [BLOCK_IDX_BITS-1:0] rsp_block = client_if[0].rsp_data.tag[BLOCK_IDX_BITS-1:0]; - wire req_fire = req_valid && client_if[0].req_ready; - wire rsp_fire = client_if[0].rsp_valid && client_if[0].rsp_ready; - - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_ev - assign req_fire_b[bi] = req_fire && (req_block == BLOCK_IDX_BITS'(bi)); - assign rsp_fire_b[bi] = rsp_fire && (rsp_block == BLOCK_IDX_BITS'(bi)); - assign rsp_eop_b[bi] = client_if[0].rsp_data.eop; - assign rsp_mask_b[bi] = client_if[0].rsp_data.mask; - assign rsp_data_b[bi] = client_if[0].rsp_data.data; - end - - assign client_if[0].req_valid = req_valid; - assign client_if[0].req_data = make_req(LSU_ADDR_WIDTH'(addr_r[req_block] >> `CLOG2(LSU_WORD_SIZE)), req_block); - assign client_if[0].rsp_ready = busy_r[rsp_block] && issued_r[rsp_block] - && !complete_r[rsp_block]; - assign req_addr_chk[0] = addr_r[req_block]; - assign req_fire_chk[0] = req_fire; - `UNUSED_VAR (client_if[0].rsp_data.sop) - end else begin : g_port_per_block - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_port - assign client_if[bi].req_valid = req_pending[bi]; - assign client_if[bi].req_data = make_req(LSU_ADDR_WIDTH'(addr_r[bi] >> `CLOG2(LSU_WORD_SIZE)), BLOCK_IDX_BITS'(0)); - assign client_if[bi].rsp_ready = busy_r[bi] && issued_r[bi] && !complete_r[bi]; - - assign req_fire_b[bi] = req_pending[bi] && client_if[bi].req_ready; - assign rsp_fire_b[bi] = client_if[bi].rsp_valid && client_if[bi].rsp_ready; - assign rsp_eop_b[bi] = client_if[bi].rsp_data.eop; - assign rsp_mask_b[bi] = client_if[bi].rsp_data.mask; - assign rsp_data_b[bi] = client_if[bi].rsp_data.data; - assign req_addr_chk[bi] = addr_r[bi]; - assign req_fire_chk[bi] = req_fire_b[bi]; - `UNUSED_VAR (client_if[bi].rsp_data.sop) - `UNUSED_VAR (client_if[bi].rsp_data.tag) - end + mem_bus_attr_t req_attr; + always_comb begin + req_attr = '0; + req_attr.is_addr_local = base_is_local; end - always @(posedge clk) begin - if (reset) begin - busy_r <= '0; - issued_r <= '0; - complete_r <= '0; - for (int bi = 0; bi < BLOCK_SIZE; ++bi) begin - header_r[bi] <= '0; - slot_r[bi] <= '0; - addr_r[bi] <= '0; - rsp_data_r[bi] <= '0; - rsp_lane_done_r[bi] <= '0; - end - end else begin - for (int bi = 0; bi < BLOCK_SIZE; ++bi) begin - if (per_block_ld_valid[bi] && per_block_ld_ready[bi]) begin - busy_r[bi] <= 1'b1; - issued_r[bi] <= 1'b0; - complete_r[bi] <= 1'b0; - header_r[bi] <= per_block_ld_data[bi].header; - slot_r[bi] <= per_block_ld_data[bi].op_args.tcu.fmt_d; - addr_r[bi] <= per_block_ld_data[bi].rs1_data[0]; - rsp_data_r[bi] <= '0; - rsp_lane_done_r[bi] <= '0; - end - if (req_fire_b[bi]) begin - issued_r[bi] <= 1'b1; - end - if (rsp_fire_b[bi]) begin - for (int t = 0; t < NUM_LANES; ++t) begin - if (rsp_mask_b[bi][t]) begin - rsp_data_r[bi][t] <= rsp_data_b[bi][t]; - rsp_lane_done_r[bi][t] <= 1'b1; - end - end - if (rsp_eop_b[bi] - && ((rsp_lane_done_r[bi] | rsp_mask_b[bi]) == {NUM_LANES{1'b1}})) begin - complete_r[bi] <= 1'b1; - end - end - end - if (commit_fire) begin - busy_r[commit_block] <= 1'b0; - issued_r[commit_block] <= 1'b0; - complete_r[commit_block] <= 1'b0; - rsp_lane_done_r[commit_block] <= '0; - end + lsu_req_data_t req_data; + always_comb begin + req_data = '0; + req_data.rw = 1'b0; + req_data.mask = {NUM_LANES{1'b1}}; + for (int i = 0; i < NUM_LANES; ++i) begin + req_data.addr[i] = base_word_addr + (LSU_ADDR_WIDTH'(i) >> LG_W32_RATIO); + req_data.byteen[i] = {LSU_WORD_SIZE{1'b1}}; + req_data.attr[i] = req_attr; end end - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_ld_ready - assign per_block_ld_ready[bi] = ~busy_r[bi]; - end + assign client_if.req_valid = busy_r && ~issued_r; + assign client_if.req_data = req_data; + assign client_if.rsp_ready = 1'b1; + `UNUSED_VAR (client_if.rsp_data.sop) + `UNUSED_VAR (client_if.rsp_data.tag) + + `RUNTIME_ASSERT(~client_if.rsp_valid || (issued_r && ~done_r), + ("%t: *** %s unexpected TCU_LD response", $time, INSTANCE_ID)) `ifdef VX_CFG_LSU_STACK_INTERLEAVE_ENABLE // Metadata is read as a linear warp-wide tile, which an interleaved // per-thread stack cannot provide. if (`VX_CFG_NUM_THREADS > 1) begin : g_meta_stack_check - for (genvar pi = 0; pi < NUM_PORTS; ++pi) begin : g_port - wire [`VX_CFG_XLEN-1:0] meta_end_addr = req_addr_chk[pi] + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); - `RUNTIME_ASSERT(~req_fire_chk[pi] || (meta_end_addr < STACK_WINDOW_BOTTOM) || (req_addr_chk[pi] >= STACK_WINDOW_TOP), - ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, req_addr_chk[pi])) - end + wire [`VX_CFG_XLEN-1:0] meta_end_addr = addr_r + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); + `RUNTIME_ASSERT(~req_fire || (meta_end_addr < STACK_WINDOW_BOTTOM) || (addr_r >= STACK_WINDOW_TOP), + ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, addr_r)) end -`else - `UNUSED_VAR (req_addr_chk) - `UNUSED_VAR (req_fire_chk) `endif // ----------------------------------------------------------------------- - // VX_tcu_meta write (broadcast to all blocks' meta instances). + // Metadata RAM write: each response fragment writes its own lanes. // ----------------------------------------------------------------------- - assign meta_wr_en = commit_valid; - assign meta_wr_wid = header_r[commit_block].wid; - assign meta_wr_idx = slot_r[commit_block]; - // Repack response data into TCU_BLOCK_CAP × XLEN. The downstream - // meta SRAM consumes only the low 32 bits, so for XLEN=64 we mux the - // half of the LSU response that holds this lane's logical 32-bit word. - for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_meta_data - if (i < NUM_LANES) begin : g_in - if (LG_W32_RATIO == 0) begin : g_w32_passthru - assign meta_wr_data[i] = `VX_CFG_XLEN'(rsp_data_r[commit_block][i]); - end else begin : g_w32_pick - wire [LSU_W32_RATIO-1:0][31:0] rsp_w32_v = rsp_data_r[commit_block][i]; - assign meta_wr_data[i] = `VX_CFG_XLEN'(rsp_w32_v[half_sel_w[i]]); - end + assign meta_wr_en = rsp_fire; + assign meta_wr_wid = header_r.wid; + assign meta_wr_idx = slot_r; + + for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_meta_wr + if (i < NUM_LANES) begin : g_lane + wire [LSU_W32_RATIO-1:0][31:0] rsp_w32 = client_if.rsp_data.data[i]; + assign meta_wr_mask[i] = client_if.rsp_data.mask[i]; + assign meta_wr_data[i] = rsp_w32[i % LSU_W32_RATIO]; + `UNUSED_VAR (rsp_w32) end else begin : g_pad + assign meta_wr_mask[i] = 1'b0; assign meta_wr_data[i] = '0; end end // ----------------------------------------------------------------------- - // Result drive (commits to scoreboard via writeback, releasing - // wr_xregs[0]). Only the owner block's result_if fires. - // tcu_result_t = { header, data[NUM_LANES][XLEN] }. - // The header is propagated from the latched execute_t header — it - // already carries wr_xregs and wb=0 from the decoder. + // Result: no register writeback; its header releases the scoreboard bit. // ----------------------------------------------------------------------- - tcu_result_t commit_data_w; + assign result_valid = done_r; always_comb begin - commit_data_w = '0; - commit_data_w.header = header_r[commit_block]; - commit_data_w.data = '0; - end - - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_result - assign result_valid[bi] = commit_valid && (commit_block == BLOCK_IDX_BITS'(bi)); - assign result_data[bi] = commit_data_w; + result_data = '0; + result_data.header = header_r; end endmodule diff --git a/hw/rtl/tcu/VX_tcu_block.sv b/hw/rtl/tcu/VX_tcu_block.sv index ee6e19a637..1e493f9d83 100644 --- a/hw/rtl/tcu/VX_tcu_block.sv +++ b/hw/rtl/tcu/VX_tcu_block.sv @@ -13,8 +13,9 @@ `include "VX_define.vh" -// One TCU block: the MMA datapath, with TCU_LD handed to the core-wide AGU and -// WGMMA operands supplied by the core-wide tile buffers (VX_tcu_shared). +// One TCU block: the MMA datapath and the TCU_LD unit that fills its metadata +// RAM, with WGMMA operands supplied by the core-wide tile buffers +// (VX_tcu_shared). module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter `STRING BLOCK_STR = "" @@ -35,17 +36,8 @@ module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `ifdef TCU_META_ENABLE - // TCU_LD to and from the AGU - output wire agu_ld_valid, - output tcu_execute_t agu_ld_data, - input wire agu_ld_ready, - input wire agu_result_valid, - input tcu_result_t agu_result_data, - output wire agu_result_ready, - input wire meta_wr_en, - input wire [NW_WIDTH-1:0] meta_wr_wid, - input wire [4:0] meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, + // TCU_LD memory client of an LSU scheduler + VX_lsu_sched_if.master tcu_mem_if, `endif VX_execute_if.slave execute_if, @@ -63,11 +55,38 @@ module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ) core_result_if(); `ifdef TCU_META_ENABLE + wire agu_ld_valid, agu_ld_ready; + wire agu_result_valid, agu_result_ready; + tcu_result_t agu_result_data; + wire meta_wr_en; + wire [NW_WIDTH-1:0] meta_wr_wid; + wire [4:0] meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0] meta_wr_mask; + wire [TCU_BLOCK_CAP-1:0][31:0] meta_wr_data; + + VX_tcu_agu #( + .INSTANCE_ID (`SFORMATF(("%s-agu%s", INSTANCE_ID, BLOCK_STR))) + ) agu ( + .clk (clk), + .reset (reset), + .ld_valid (agu_ld_valid), + .ld_data (execute_if.data), + .ld_ready (agu_ld_ready), + .client_if (tcu_mem_if), + .meta_wr_en (meta_wr_en), + .meta_wr_wid (meta_wr_wid), + .meta_wr_idx (meta_wr_idx), + .meta_wr_mask (meta_wr_mask), + .meta_wr_data (meta_wr_data), + .result_valid (agu_result_valid), + .result_data (agu_result_data), + .result_ready (agu_result_ready) + ); + // TCU_LD goes to the AGU, every MMA op to the core wire is_tcu_ld = (execute_if.data.op_type == INST_TCU_LD); assign agu_ld_valid = execute_if.valid && is_tcu_ld; - assign agu_ld_data = execute_if.data; assign core_execute_if.valid = execute_if.valid && !is_tcu_ld; assign core_execute_if.data = execute_if.data; assign execute_if.ready = is_tcu_ld ? agu_ld_ready : core_execute_if.ready; @@ -112,6 +131,7 @@ module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .ext_meta_wr_en (meta_wr_en), .ext_meta_wr_wid (meta_wr_wid), .ext_meta_wr_idx (meta_wr_idx), + .ext_meta_wr_mask (meta_wr_mask), .ext_meta_wr_data (meta_wr_data), `endif .execute_if (core_execute_if), diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index e4fc89786f..b70a8e3a50 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -27,12 +27,13 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire tbuf_ready, `endif - // External metadata write port from the shared VX_tcu_agu. + // Metadata write port from the block's VX_tcu_agu. `ifdef TCU_META_ENABLE input wire ext_meta_wr_en, input wire [NW_WIDTH-1:0] ext_meta_wr_wid, input wire [4:0] ext_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] ext_meta_wr_data, + input wire [TCU_BLOCK_CAP-1:0] ext_meta_wr_mask, + input wire [TCU_BLOCK_CAP-1:0][31:0] ext_meta_wr_data, `endif // Inputs @@ -318,6 +319,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .wr_en (ext_meta_wr_en), .wr_wid (ext_meta_wr_wid), .wr_idx (ext_meta_wr_idx), + .wr_mask(ext_meta_wr_mask), .wr_data(ext_meta_wr_data), .rd_wid (execute_if.data.header.wid) `ifdef VX_CFG_TCU_SPARSE_ENABLE diff --git a/hw/rtl/tcu/VX_tcu_meta.sv b/hw/rtl/tcu/VX_tcu_meta.sv index 8a31ca6bad..68bf326390 100644 --- a/hw/rtl/tcu/VX_tcu_meta.sv +++ b/hw/rtl/tcu/VX_tcu_meta.sv @@ -19,7 +19,7 @@ // the MX scale-factor stores (A/B) behind a single warp-indexed block RAM. // // All regions share one warp-indexed address space (SIZE = NUM_WARPS) and the -// one broadcast write port from VX_tcu_agu. wr_idx[4] selects the namespace +// one write port from the block's VX_tcu_agu. wr_idx[4] selects the namespace // (0 = sparse, 1 = MX); within MX wr_idx[0] selects the axis (0 = A, 1 = B); // within sparse wr_idx[3:0] is the column-group index. This matches the // TCU_LD destination-register encoding emitted by vx_tensor.h (x0/x1 = SP, @@ -39,11 +39,12 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire clk, input wire reset, - // Unified broadcast write port (from VX_tcu_agu). + // Write port from the block's VX_tcu_agu; wr_mask selects the lanes written. input wire wr_en, input wire [NW_WIDTH-1:0] wr_wid, input wire [4:0] wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] wr_data, + input wire [TCU_BLOCK_CAP-1:0] wr_mask, + input wire [TCU_BLOCK_CAP-1:0][31:0] wr_data, // Unified read port (warp id from the FEDP path). input wire [NW_WIDTH-1:0] rd_wid @@ -162,15 +163,11 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( for (genvar b = 0; b < PER_WARP_DEPTH; ++b) begin : g_meta_banks for (genvar c = 0; c < NUM_COLS; ++c) begin : g_col localparam int COL = b * NUM_COLS + c; - if (STORES_PER_COL > 1) begin : g_wr - assign merged_wdata[COL * 32 +: 32] = 32'(wr_data[b % BANKS_PER_STORE]); - end else begin : g_wr - // Thread offset for column c within its group: (c % CPL) * PER_WARP_DEPTH + b - localparam int COL_IN_GROUP = c % COLS_PER_LOAD; - localparam int THREAD_OFF = COL_IN_GROUP * PER_WARP_DEPTH + b; - assign merged_wdata[COL * 32 +: 32] = 32'(wr_data[THREAD_OFF]); - end - assign merged_wren[COL] = sp_wr && col_wren[c] && meta_wr_bank_en[b]; + // Thread offset for column c within its group: (c % CPL) * PER_WARP_DEPTH + b + localparam int THREAD_OFF = (STORES_PER_COL > 1) ? (b % BANKS_PER_STORE) + : ((c % COLS_PER_LOAD) * PER_WARP_DEPTH + b); + assign merged_wdata[COL * 32 +: 32] = wr_data[THREAD_OFF]; + assign merged_wren[COL] = sp_wr && col_wren[c] && meta_wr_bank_en[b] && wr_mask[THREAD_OFF]; end end @@ -196,10 +193,10 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire mx_wr_b = wr_en && wr_idx[4] && wr_idx[0]; for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_mx_pack - assign merged_wdata[(MXA_COL_BASE + i) * 32 +: 32] = 32'(wr_data[i]); - assign merged_wdata[(MXB_COL_BASE + i) * 32 +: 32] = 32'(wr_data[i]); - assign merged_wren[MXA_COL_BASE + i] = mx_wr_a; - assign merged_wren[MXB_COL_BASE + i] = mx_wr_b; + assign merged_wdata[(MXA_COL_BASE + i) * 32 +: 32] = wr_data[i]; + assign merged_wdata[(MXB_COL_BASE + i) * 32 +: 32] = wr_data[i]; + assign merged_wren[MXA_COL_BASE + i] = mx_wr_a && wr_mask[i]; + assign merged_wren[MXB_COL_BASE + i] = mx_wr_b && wr_mask[i]; end assign meta_a = merged_rdata[(MXA_COL_BASE * 32) +: (TCU_BLOCK_CAP * 32)]; assign meta_b = merged_rdata[(MXB_COL_BASE * 32) +: (TCU_BLOCK_CAP * 32)]; diff --git a/hw/rtl/tcu/VX_tcu_shared.sv b/hw/rtl/tcu/VX_tcu_shared.sv index 4fee552c69..0b232fa3f4 100644 --- a/hw/rtl/tcu/VX_tcu_shared.sv +++ b/hw/rtl/tcu/VX_tcu_shared.sv @@ -14,7 +14,7 @@ `include "VX_define.vh" // TCU state shared by every TCU block: the WGMMA tile buffers (a warpgroup's -// B operand is common to its blocks) and the warp-level metadata AGU. +// B operand is common to its blocks). module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS @@ -34,31 +34,14 @@ module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( VX_mem_bus_if.master tcu_lmem_if, `endif -`ifdef TCU_META_ENABLE - input wire [BLOCK_SIZE-1:0] agu_ld_valid, - input tcu_execute_t agu_ld_data [BLOCK_SIZE], - output wire [BLOCK_SIZE-1:0] agu_ld_ready, - output wire [BLOCK_SIZE-1:0] agu_result_valid, - output tcu_result_t agu_result_data [BLOCK_SIZE], - input wire [BLOCK_SIZE-1:0] agu_result_ready, - output wire meta_wr_en, - output wire [NW_WIDTH-1:0] meta_wr_wid, - output wire [4:0] meta_wr_idx, - output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, - // TCU_LD memory clients, one per LSU block's scheduler - VX_lsu_sched_if.master tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], -`endif - input wire clk, input wire reset ); `UNUSED_SPARAM (INSTANCE_ID) `ifndef VX_CFG_TCU_WGMMA_ENABLE -`ifndef TCU_META_ENABLE `UNUSED_PARAM (BLOCK_SIZE) `UNUSED_VAR ({clk, reset}) `endif -`endif `ifdef VX_CFG_TCU_WGMMA_ENABLE VX_tcu_wgmma #( @@ -88,55 +71,4 @@ module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `endif -`ifdef TCU_META_ENABLE - // The AGU streams NUM_TCU_LANES operands over the LSU memory client, whose - // request/response mask and data are sized by NUM_LSU_LANES; the two lane - // counts must match for the per-lane completion tracking to be correct. - `STATIC_ASSERT (`VX_CFG_NUM_TCU_LANES == `VX_CFG_NUM_LSU_LANES, ("VX_tcu_agu requires NUM_TCU_LANES == NUM_LSU_LANES")); - - // With one LSU block per TCU block, each block loads its metadata - // through the scheduler of the LSU block that serves the same warps; - // otherwise every block shares LSU block 0's scheduler. - localparam AGU_PORTS = (`VX_CFG_NUM_LSU_BLOCKS == BLOCK_SIZE) ? BLOCK_SIZE : 1; - for (genvar b = AGU_PORTS; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_mem_tieoff - assign tcu_mem_if[b].req_valid = 1'b0; - assign tcu_mem_if[b].req_data = '0; - assign tcu_mem_if[b].rsp_ready = 1'b1; - `UNUSED_VAR (tcu_mem_if[b].req_ready) - `UNUSED_VAR (tcu_mem_if[b].rsp_valid) - `UNUSED_VAR (tcu_mem_if[b].rsp_data) - end - - VX_lsu_sched_if agu_mem_if [AGU_PORTS](); - for (genvar b = 0; b < AGU_PORTS; ++b) begin : g_mem_port - assign tcu_mem_if[b].req_valid = agu_mem_if[b].req_valid; - assign tcu_mem_if[b].req_data = agu_mem_if[b].req_data; - assign agu_mem_if[b].req_ready = tcu_mem_if[b].req_ready; - assign agu_mem_if[b].rsp_valid = tcu_mem_if[b].rsp_valid; - assign agu_mem_if[b].rsp_data = tcu_mem_if[b].rsp_data; - assign tcu_mem_if[b].rsp_ready = agu_mem_if[b].rsp_ready; - end - - VX_tcu_agu #( - .INSTANCE_ID (`SFORMATF(("%s-agu", INSTANCE_ID))), - .BLOCK_SIZE (BLOCK_SIZE), - .NUM_LANES (`VX_CFG_NUM_TCU_LANES), - .NUM_PORTS (AGU_PORTS) - ) agu ( - .clk (clk), - .reset (reset), - .per_block_ld_valid (agu_ld_valid), - .per_block_ld_data (agu_ld_data), - .per_block_ld_ready (agu_ld_ready), - .client_if (agu_mem_if), - .meta_wr_en (meta_wr_en), - .meta_wr_wid (meta_wr_wid), - .meta_wr_idx (meta_wr_idx), - .meta_wr_data (meta_wr_data), - .result_valid (agu_result_valid), - .result_data (agu_result_data), - .result_ready (agu_result_ready) - ); -`endif - endmodule diff --git a/hw/rtl/tcu/VX_tcu_unit.sv b/hw/rtl/tcu/VX_tcu_unit.sv index 91dd5fd292..7d1a5250c2 100644 --- a/hw/rtl/tcu/VX_tcu_unit.sv +++ b/hw/rtl/tcu/VX_tcu_unit.sv @@ -32,17 +32,8 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `ifdef TCU_META_ENABLE - // per-block ports of the core-wide metadata AGU (VX_tcu_shared) - output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_ld_valid, - output tcu_execute_t agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS], - input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_ld_ready, - input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_result_valid, - input tcu_result_t agu_result_data [`VX_CFG_NUM_TCU_BLOCKS], - output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_result_ready, - input wire meta_wr_en, - input wire [NW_WIDTH-1:0] meta_wr_wid, - input wire [4:0] meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, + // TCU_LD memory clients of the LSU blocks' schedulers + VX_lsu_sched_if.master tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif // Inputs @@ -78,11 +69,77 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .data_t (tcu_result_t) ) per_block_result_if[BLOCK_SIZE](); +`ifdef TCU_META_ENABLE + VX_lsu_sched_if block_mem_if [BLOCK_SIZE](); + + if (`VX_CFG_NUM_LSU_BLOCKS == BLOCK_SIZE) begin : g_mem_per_block + // each TCU block loads through the LSU block serving the same warps + for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_port + assign tcu_mem_if[b].req_valid = block_mem_if[b].req_valid; + assign tcu_mem_if[b].req_data = block_mem_if[b].req_data; + assign block_mem_if[b].req_ready = tcu_mem_if[b].req_ready; + assign block_mem_if[b].rsp_valid = tcu_mem_if[b].rsp_valid; + assign block_mem_if[b].rsp_data = tcu_mem_if[b].rsp_data; + assign tcu_mem_if[b].rsp_ready = block_mem_if[b].rsp_ready; + end + end else begin : g_mem_shared + // the TCU blocks share LSU block 0's client, tagged by block + localparam BLOCK_IDX_BITS = `CLOG2(BLOCK_SIZE); + `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) + + wire [BLOCK_SIZE-1:0] req_valids; + wire [`UP(BLOCK_IDX_BITS)-1:0] req_idx; + wire [BLOCK_SIZE-1:0] req_grant; + wire req_valid; + lsu_req_data_t req_datas [BLOCK_SIZE]; + for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_req + assign req_valids[b] = block_mem_if[b].req_valid; + assign req_datas[b] = block_mem_if[b].req_data; + assign block_mem_if[b].req_ready = tcu_mem_if[0].req_ready && req_grant[b]; + end + + VX_generic_arbiter #( + .NUM_REQS (BLOCK_SIZE), + .TYPE ("R") + ) req_arb ( + .clk (clk), + .reset (reset), + .requests (req_valids), + .grant_index (req_idx), + .grant_onehot (req_grant), + .grant_valid (req_valid), + .grant_ready (tcu_mem_if[0].req_ready) + ); + + lsu_req_data_t req_data; + always_comb begin + req_data = req_datas[req_idx]; + req_data.tag[`UP(BLOCK_IDX_BITS)-1:0] = req_idx; + end + assign tcu_mem_if[0].req_valid = req_valid; + assign tcu_mem_if[0].req_data = req_data; + + wire [`UP(BLOCK_IDX_BITS)-1:0] rsp_idx = tcu_mem_if[0].rsp_data.tag[`UP(BLOCK_IDX_BITS)-1:0]; + wire [BLOCK_SIZE-1:0] rsp_readys; + for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_rsp + assign block_mem_if[b].rsp_valid = tcu_mem_if[0].rsp_valid && (rsp_idx == `UP(BLOCK_IDX_BITS)'(b)); + assign block_mem_if[b].rsp_data = tcu_mem_if[0].rsp_data; + assign rsp_readys[b] = block_mem_if[b].rsp_ready; + end + assign tcu_mem_if[0].rsp_ready = rsp_readys[rsp_idx]; + + for (genvar b = 1; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_tieoff + assign tcu_mem_if[b].req_valid = 1'b0; + assign tcu_mem_if[b].req_data = '0; + assign tcu_mem_if[b].rsp_ready = 1'b1; + `UNUSED_VAR (tcu_mem_if[b].req_ready) + `UNUSED_VAR (tcu_mem_if[b].rsp_valid) + `UNUSED_VAR (tcu_mem_if[b].rsp_data) + end + end +`endif + for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - `ifdef TCU_META_ENABLE - wire block_meta_wr_en = meta_wr_en - && (NW_WIDTH'(meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(block_idx)); - `endif VX_tcu_block #( .INSTANCE_ID (INSTANCE_ID), @@ -100,16 +157,7 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .tbuf_ready (tbuf_ready[block_idx]), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (agu_ld_valid[block_idx]), - .agu_ld_data (agu_ld_data[block_idx]), - .agu_ld_ready (agu_ld_ready[block_idx]), - .agu_result_valid (agu_result_valid[block_idx]), - .agu_result_data (agu_result_data[block_idx]), - .agu_result_ready (agu_result_ready[block_idx]), - .meta_wr_en (block_meta_wr_en), - .meta_wr_wid (meta_wr_wid), - .meta_wr_idx (meta_wr_idx), - .meta_wr_data (meta_wr_data), + .tcu_mem_if (block_mem_if[block_idx]), `endif .execute_if (per_block_execute_if[block_idx]), .result_if (per_block_result_if[block_idx]) diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index 6030fc0a39..cb81581e53 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -29,7 +29,7 @@ // TCU_LD memory ports: one per TCU block, through the LSU block that serves // the same warps, when the LSU has one block per TCU block; otherwise all -// TCU blocks share LSU block 0. Same derivation as the RTL's AGU_PORTS. +// TCU blocks share LSU block 0, as in the RTL (VX_tcu_unit). #ifndef TCU_AGU_PORTS #define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) #endif diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index 0e88015cb3..f5e33c6984 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -412,8 +412,8 @@ class TcuUnit::Impl { } #ifdef TCU_META_ENABLE - // Each TCU block tracks one metadata transaction. Requests share the LSU - // client port, while tags allow their memory latency to overlap. + // Each TCU block's AGU tracks one metadata load and writes each returned + // lane straight into the block's metadata SRAM. static constexpr uint32_t kAguBeats = (VX_CFG_NUM_THREADS + VX_CFG_NUM_LSU_LANES - 1) / VX_CFG_NUM_LSU_LANES; @@ -426,7 +426,6 @@ class TcuUnit::Impl { uint32_t next_beat = 0; uint32_t rsp_count = 0; std::array addrs{}; - std::array words{}; }; std::array agu_; uint32_t agu_issue_rr_ = 0; @@ -485,11 +484,10 @@ class TcuUnit::Impl { uint32_t off = agu.addrs.at(idx) & (VX_CFG_MEM_BLOCK_SIZE - 1); uint32_t word = 0; std::memcpy(&word, rsp.data.at(lane)->data() + off, 4); - agu.words.at(idx) = word; + this->agu_write(agu, idx, word); ++agu.rsp_count; } if (agu.busy && !agu.complete && agu.rsp_count == VX_CFG_NUM_THREADS) { - this->agu_deposit(agu); agu.complete = true; } } @@ -531,15 +529,13 @@ class TcuUnit::Impl { } } - // Deposit the fetched words into the selected metadata SRAM. - void agu_deposit(const agu_state_t& agu) { + // Write lane T's fetched word into the selected metadata SRAM. + void agu_write(const agu_state_t& agu, uint32_t T, uint32_t word) { uint32_t wid = agu.wid; #ifdef VX_CFG_TCU_MX_ENABLE if (agu.selector & 0x10) { auto& dst = (agu.selector & 1) ? mx_meta_b_.at(wid) : mx_meta_a_.at(wid); - for (uint32_t lane = 0; lane < VX_CFG_NUM_THREADS; ++lane) { - dst.at(lane) = agu.words.at(lane); - } + dst.at(T) = word; return; } #endif @@ -554,26 +550,25 @@ class TcuUnit::Impl { constexpr uint32_t BPS = cfg::banks_per_store; constexpr uint32_t SPC = cfg::stores_per_col; constexpr uint32_t CPL = cfg::meta_cols_per_load; - for (uint32_t T = 0; T < VX_CFG_NUM_THREADS; ++T) { - uint32_t store_in_load = T / BPS; - uint32_t thread_in_store = T % BPS; - uint32_t flat_store = slot_idx * CPL + store_in_load; - uint32_t col = flat_store / SPC; - uint32_t store_in_col = flat_store % SPC; - uint32_t bank = store_in_col * BPS + thread_in_store; - if (bank >= PWD || col >= kMaxMetaCols) { - continue; - } - uint32_t word = agu.words.at(T); - sparse_meta_.at(wid).at(bank * kMaxMetaCols + col) = word; - // Trace: META_SRAM write (CSV: wid,bank,col,addr,value). - if (const char* p = std::getenv("VORTEX_TCU_TRACE")) { - if (p[0] == '1') { - fprintf(stderr, "META_TRC,%u,%u,%u,0x%lx,0x%08x\n", - wid, bank, col, (unsigned long)agu.addrs.at(T), word); - } + uint32_t store_in_load = T / BPS; + uint32_t thread_in_store = T % BPS; + uint32_t flat_store = slot_idx * CPL + store_in_load; + uint32_t col = flat_store / SPC; + uint32_t store_in_col = flat_store % SPC; + uint32_t bank = store_in_col * BPS + thread_in_store; + if (bank >= PWD || col >= kMaxMetaCols) { + return; + } + sparse_meta_.at(wid).at(bank * kMaxMetaCols + col) = word; + // Trace: META_SRAM write (CSV: wid,bank,col,addr,value). + if (const char* p = std::getenv("VORTEX_TCU_TRACE")) { + if (p[0] == '1') { + fprintf(stderr, "META_TRC,%u,%u,%u,0x%lx,0x%08x\n", + wid, bank, col, (unsigned long)agu.addrs.at(T), word); } } + #else + __unused(T, word); #endif } #endif // TCU_META_ENABLE diff --git a/sim/simx/tcu/tcu_unit.h b/sim/simx/tcu/tcu_unit.h index 681c448b5c..6a4e0e7215 100644 --- a/sim/simx/tcu/tcu_unit.h +++ b/sim/simx/tcu/tcu_unit.h @@ -30,7 +30,7 @@ // TCU_LD memory ports: one per TCU block, through the LSU block that serves // the same warps, when the LSU has one block per TCU block; otherwise all -// TCU blocks share LSU block 0. Same derivation as the RTL's AGU_PORTS. +// TCU blocks share LSU block 0, as in the RTL (VX_tcu_unit). #ifndef TCU_AGU_PORTS #define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) #endif From 365895a619e1faff4bc43779251718c22b710b2e Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 18:09:12 -0700 Subject: [PATCH 089/336] core: drop the live-warps CSR bits above XLEN explicitly The active-warps CSR is XLEN wide, so with more warps than XLEN bits it holds warps 0..XLEN-1 only, as SimX already reports. The implicit truncation left the upper bits of sched_csr_if.active_warps unread, and lint rejected every build with NUM_WARPS > XLEN (e.g. 48 or 64 warps at XLEN=32), so those shapes could not be simulated at all. Lint clean at 32, 48 and 64 warps, XLEN 32 and 64. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_csr_data.sv | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index 6a2300fe1a..0214f50e99 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -229,7 +229,12 @@ import VX_fpu_pkg::*; assign live_src[LIVE_INSTRET_H] = 32'(sched_csr_if.instret[PERF_CTR_BITS-1:32]); `endif assign live_src[LIVE_THREADS] = `VX_CFG_XLEN'(sched_csr_if.thread_mask); - assign live_src[LIVE_WARPS] = `VX_CFG_XLEN'(sched_csr_if.active_warps); + // the CSR is XLEN wide: it holds warps 0..XLEN-1 + wire [`VX_CFG_NUM_WARPS-1:0] active_warps = sched_csr_if.active_warps; + assign live_src[LIVE_WARPS] = `VX_CFG_XLEN'(active_warps); + if (`VX_CFG_NUM_WARPS > `VX_CFG_XLEN) begin : g_active_warps_hi + `UNUSED_VAR (active_warps[`VX_CFG_NUM_WARPS-1:`VX_CFG_XLEN]) + end `ifdef VX_CFG_EXT_F_ENABLE assign live_src[LIVE_FFLAGS] = `VX_CFG_XLEN'(req_fcsr[`FP_FLAGS_BITS-1:0]); assign live_src[LIVE_FRM] = `VX_CFG_XLEN'(req_fcsr[INST_FRM_BITS+`FP_FLAGS_BITS-1:`FP_FLAGS_BITS]); From e311957d9c9d48c64dbda4e7d565fa88f555db81 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 18:09:14 -0700 Subject: [PATCH 090/336] core: support a warp count per issue slice that is not a power of two Three places indexed warps by bit-slicing, which is only right when the count is a power of two. Real GPU shapes are not: an RTX 5080 SM holds 48 warps, 12 per scheduler partition. - Operand collectors: a warp's collector was wis[1:0] and its register slot the upper wis bits. With 12 warps per slice there are 3 collectors, so warps 3, 7 and 11 selected a collector that does not exist, their instructions never left the scoreboard, and the slice deadlocked (scoreboard timeout on sgemm at 4 cores x 48 warps). The collector is now wis % NUM_OPCS and the slot wis / NUM_OPCS, and the per-collector slot count rounds up, as in SimX. - IPDOM stack: entries were addressed {ptr, wid}, which runs past the end of the DEPTH * NUM_WARPS store (up to entry 491 of 372 at 12 warps). The address is now ptr * NUM_WARPS + wid. - mhartid stepped by the next power of two of NUM_WARPS per core, while the stack window holds NUM_WARPS * NUM_THREADS stacks per core, so the last core's stacks fell below the window. It is now dense, (core * NUM_WARPS + wid) * NUM_THREADS + tid, as SimX computes it. For power-of-two counts the constant %, / and * reduce to the same bit selects, so existing configurations are unchanged. Lint clean at 6, 9, 12, 24, 48, 64 and the default warp counts (9 warps failed lint in the IPDOM stack before). rtlsim validation at the 48-warp shape is still running. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 2 +- hw/rtl/core/VX_csr_unit.sv | 5 +++-- hw/rtl/core/VX_ipdom_stack.sv | 6 ++++-- hw/rtl/core/VX_opc_unit.sv | 15 +++++++++++---- hw/rtl/core/VX_operands.sv | 4 ++-- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 782f1361c9..55e2d64049 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -602,7 +602,7 @@ package VX_gpu_pkg; localparam DISPATCH_QSIZE = `VX_CFG_DISPATCH_QUEUE_SIZE; - localparam PER_OPC_WARPS = PER_ISSUE_WARPS / `VX_CFG_NUM_OPCS; + localparam PER_OPC_WARPS = `CDIV(PER_ISSUE_WARPS, `VX_CFG_NUM_OPCS); localparam PER_OPC_NW_BITS = `CLOG2(PER_OPC_WARPS); localparam PER_OPC_NW_W = `UP(PER_OPC_NW_BITS); diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index 0a4e859da9..0a5358ca90 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -114,8 +114,9 @@ module VX_csr_unit import VX_gpu_pkg::*; #( // Thread ids are the lane index on top of a per-request base. wire [`VX_CFG_XLEN-1:0] wtid_base = (PID_BITS != 0) ? `VX_CFG_XLEN'(execute_if.data.header.pid * NUM_LANES) : '0; - wire [`VX_CFG_XLEN-1:0] gtid_base = (`VX_CFG_XLEN'(core_id) << (NW_BITS + NT_BITS)) - + (`VX_CFG_XLEN'(execute_if.data.header.wid) << NT_BITS) + // mhartid is dense, (core * NUM_WARPS + wid) * NUM_THREADS + tid, for any NUM_WARPS + wire [`VX_CFG_XLEN-1:0] gtid_base = ((`VX_CFG_XLEN'(core_id) * `VX_CFG_XLEN'(`VX_CFG_NUM_WARPS) + + `VX_CFG_XLEN'(execute_if.data.header.wid)) << NT_BITS) + wtid_base; wire is_wtid_w = (csr_addr == `VX_CSR_THREAD_ID); diff --git a/hw/rtl/core/VX_ipdom_stack.sv b/hw/rtl/core/VX_ipdom_stack.sv index a5b8a8b3f7..270bfe0f25 100644 --- a/hw/rtl/core/VX_ipdom_stack.sv +++ b/hw/rtl/core/VX_ipdom_stack.sv @@ -78,8 +78,10 @@ module VX_ipdom_stack import VX_gpu_pkg::*; #( wire [BRAW_ADDRW-1:0] raddr, waddr; if (DEPTH > 1 && NUM_WARPS > 1) begin : g_DW - assign waddr = push ? {wr_ptr_w[wid], wid} : {rd_ptr, wid}; - assign raddr = {rd_ptr, wid}; + // entry = ptr * NUM_WARPS + wid keeps the store dense when NUM_WARPS is not a power of two + wire [ADDRW-1:0] waddr_ptr = push ? wr_ptr_w[wid] : rd_ptr; + assign waddr = BRAW_ADDRW'(waddr_ptr * NUM_WARPS + wid); + assign raddr = BRAW_ADDRW'(rd_ptr * NUM_WARPS + wid); end else if (DEPTH > 1) begin : g_D `UNUSED_VAR (wid) assign waddr = push ? wr_ptr_w : rd_ptr; diff --git a/hw/rtl/core/VX_opc_unit.sv b/hw/rtl/core/VX_opc_unit.sv index 203501046a..75b3b4c708 100644 --- a/hw/rtl/core/VX_opc_unit.sv +++ b/hw/rtl/core/VX_opc_unit.sv @@ -236,14 +236,21 @@ module VX_opc_unit import VX_gpu_pkg::*; #( end end + // warps are dealt round-robin across the collectors: slot = wis / NUM_OPCS + wire [PER_OPC_NW_W-1:0] wr_slot = PER_OPC_NW_W'(writeback_if.data.wis / `VX_CFG_NUM_OPCS); + wire [PER_OPC_NW_W-1:0] rd_slot = PER_OPC_NW_W'(pipe_mdata_st1[META_DATAW-UUID_WIDTH-1 -: ISSUE_WIS_W] / `VX_CFG_NUM_OPCS); + if (PER_OPC_NW_BITS == 0) begin : g_slot_unused + `UNUSED_VAR ({wr_slot, rd_slot}) + end + wire [BANK_ADDR_WIDTH-1:0] gpr_wr_addr; if (SIMD_COUNT != 1) begin : g_gpr_wr_addr_sid wire [PER_OPC_NW_BITS + REG_REM_BITS-1:0] tmp; - `CONCAT(tmp, writeback_if.data.wis[ISSUE_WIS_W-1 -: PER_OPC_NW_W], + `CONCAT(tmp, wr_slot, writeback_if.data.rd[NUM_REGS_BITS-1 -: REG_REM_BITS], PER_OPC_NW_BITS, REG_REM_BITS) assign gpr_wr_addr = {writeback_if.data.sid, tmp}; end else begin : g_gpr_wr_addr - `CONCAT(gpr_wr_addr, writeback_if.data.wis[ISSUE_WIS_W-1 -: PER_OPC_NW_W], + `CONCAT(gpr_wr_addr, wr_slot, writeback_if.data.rd[NUM_REGS_BITS-1 -: REG_REM_BITS], PER_OPC_NW_BITS, REG_REM_BITS) end @@ -271,13 +278,13 @@ module VX_opc_unit import VX_gpu_pkg::*; #( wire [BANK_ADDR_WIDTH-1:0] gpr_rd_addr; if (SIMD_COUNT != 1) begin : g_gpr_rd_addr_sid wire [PER_OPC_NW_BITS + REG_REM_BITS-1:0] tmp; - `CONCAT(tmp, pipe_mdata_st1[META_DATAW-UUID_WIDTH-1 -: PER_OPC_NW_W], + `CONCAT(tmp, rd_slot, gpr_rd_reg_st1[b], PER_OPC_NW_BITS, REG_REM_BITS) // pipe_mdata packs {uuid, wis, cta_id, simd_pid, ...}; skip uuid+wis+cta_id // to extract simd_pid, which must match writeback_if.data.sid on the write path. assign gpr_rd_addr = {pipe_mdata_st1[META_DATAW-UUID_WIDTH-ISSUE_WIS_W-NCTA_WIDTH-1 -: SIMD_IDX_W], tmp}; end else begin : g_gpr_rd_addr - `CONCAT(gpr_rd_addr, pipe_mdata_st1[META_DATAW-UUID_WIDTH-1 -: PER_OPC_NW_W], + `CONCAT(gpr_rd_addr, rd_slot, gpr_rd_reg_st1[b], PER_OPC_NW_BITS, REG_REM_BITS) end diff --git a/hw/rtl/core/VX_operands.sv b/hw/rtl/core/VX_operands.sv index 2c1f11b9bc..9a2d417616 100644 --- a/hw/rtl/core/VX_operands.sv +++ b/hw/rtl/core/VX_operands.sv @@ -42,8 +42,8 @@ module VX_operands import VX_gpu_pkg::*; #( wire [NUM_OPCS_W-1:0] sb_opc, wb_opc; if (`VX_CFG_NUM_OPCS != 1) begin : g_wis_opc - assign sb_opc = scoreboard_if.data.wis[NUM_OPCS_W-1:0]; - assign wb_opc = writeback_if.data.wis[NUM_OPCS_W-1:0]; + assign sb_opc = NUM_OPCS_W'(scoreboard_if.data.wis % `VX_CFG_NUM_OPCS); + assign wb_opc = NUM_OPCS_W'(writeback_if.data.wis % `VX_CFG_NUM_OPCS); end else begin : g_wis_opc assign sb_opc = 0; assign wb_opc = 0; From 0c603f49b5d0f884fb0c04e3d9a364664649068e Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 18:09:21 -0700 Subject: [PATCH 091/336] hw/unittest: wire the tcu and tensor wrappers to the per-block TCU memory port fc0a35bb4 moved the TCU_LD AGU into each TCU block and gave VX_tcu_unit a tcu_mem_if port per LSU block, but the two synthesis wrappers still wired the removed core-level AGU and metadata write bus. Both DUTs failed to elaborate in fpga_gate ("unconnected interface port tcu_mem_if"). The wrappers now pass VX_tcu_unit the LSU-block client array and keep VX_tcu_shared for the WGMMA tile buffers only. Both wrappers elaborate under Verilator with the gate configurations; the previous tcu wrapper still fails there (missing tcu_mem_if, 20 removed pins). The fpga_gate and asic_gate tcu/tensor builds have not been rerun. Co-Authored-By: Claude Opus 5.5 --- hw/unittest/tcu_unit/VX_tcu_unit_top.sv | 50 +++++-------------------- hw/unittest/tensor/VX_tensor_top.sv | 50 +++---------------------- 2 files changed, 14 insertions(+), 86 deletions(-) diff --git a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv index 637ba9bd9a..7542fe1600 100644 --- a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv +++ b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv @@ -106,13 +106,13 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( `endif `ifdef TCU_META_ENABLE - VX_lsu_sched_if tcu_mem_if(); - assign tcu_mem_req_valid = tcu_mem_if.req_valid; - assign tcu_mem_req_data = tcu_mem_if.req_data; - assign tcu_mem_if.req_ready = tcu_mem_req_ready; - assign tcu_mem_if.rsp_valid = tcu_mem_rsp_valid; - assign tcu_mem_if.rsp_data = tcu_mem_rsp_data; - assign tcu_mem_rsp_ready = tcu_mem_if.rsp_ready; + VX_lsu_sched_if tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); + assign tcu_mem_req_valid = tcu_mem_if[0].req_valid; + assign tcu_mem_req_data = tcu_mem_if[0].req_data; + assign tcu_mem_if[0].req_ready = tcu_mem_req_ready; + assign tcu_mem_if[0].rsp_valid = tcu_mem_rsp_valid; + assign tcu_mem_if[0].rsp_data = tcu_mem_rsp_data; + assign tcu_mem_rsp_ready = tcu_mem_if[0].rsp_ready; `endif VX_dispatch_if dispatch_if[`VX_CFG_ISSUE_WIDTH](); @@ -171,16 +171,6 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; `endif -`ifdef TCU_META_ENABLE - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; - tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; - tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire tcu_meta_wr_en; - wire [NW_WIDTH-1:0] tcu_meta_wr_wid; - wire [4:0] tcu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; -`endif VX_tcu_unit #( .INSTANCE_ID ("tcu_unit_top") @@ -197,22 +187,13 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), `endif .dispatch_if (dispatch_if), .commit_if (commit_if) ); - // the core-wide TCU state the blocks share: WGMMA tile buffers, metadata AGU + // the core-wide WGMMA tile buffers the TCU blocks share VX_tcu_shared #( .INSTANCE_ID ("tcu_unit_top"), .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) @@ -225,19 +206,6 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( .tbuf_rs2_data (tcu_tbuf_rs2_data), .tbuf_ready_eff (tcu_tbuf_ready), .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (tcu_mem_if), `endif .clk (clk), .reset (reset) diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 232f6475cb..b3059bcdec 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -20,7 +20,7 @@ // // per core (VX_CFG_SOCKET_SIZE): // VX_tcu_unit : tensor-core unit (dispatch/commit from top ports) -// VX_tcu_shared : WGMMA tile buffers + metadata AGU +// VX_tcu_shared : WGMMA tile buffers // VX_lsu_mem_path : per-LSU-block scheduler (2 clients — the ordinary LSU // stream from top ports and the TCU AGU metadata-load // port), lmem switch, coalescer and adapters @@ -75,7 +75,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( // ----------------------------------------------------------------------- // Ordinary LSU client streams, flattened [core][lsu-block] - // (client 0 of each LSU block's scheduler; the TCU AGU is client 1 on block 0) + // (client 0 of each LSU block's scheduler; the TCU block AGUs are client 1) // ----------------------------------------------------------------------- input wire [LSU_STREAMS-1:0] lsu_req_valid, input lsu_req_data_t [LSU_STREAMS-1:0] lsu_req_data, @@ -169,14 +169,6 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( `ifdef TCU_META_ENABLE VX_lsu_sched_if lsu_tcu_mem_if [NUM_LSU_BLKS](); - for (genvar block_idx = 1; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_tcu_client_tieoff - assign lsu_tcu_mem_if[block_idx].req_valid = 1'b0; - assign lsu_tcu_mem_if[block_idx].req_data = '0; - assign lsu_tcu_mem_if[block_idx].rsp_ready = 1'b1; - `UNUSED_VAR (lsu_tcu_mem_if[block_idx].req_ready) - `UNUSED_VAR (lsu_tcu_mem_if[block_idx].rsp_valid) - `UNUSED_VAR (lsu_tcu_mem_if[block_idx].rsp_data) - end `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE @@ -186,16 +178,6 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; `endif - `ifdef TCU_META_ENABLE - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; - tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; - tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire tcu_meta_wr_en; - wire [NW_WIDTH-1:0] tcu_meta_wr_wid; - wire [4:0] tcu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; - `endif VX_tcu_unit #( .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))) @@ -212,22 +194,13 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (lsu_tcu_mem_if), `endif .dispatch_if (dispatch_if), .commit_if (commit_if) ); - // the core-wide TCU state the blocks share: WGMMA tile buffers, metadata AGU + // the core-wide WGMMA tile buffers the TCU blocks share VX_tcu_shared #( .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))), .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) @@ -240,25 +213,12 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .tbuf_rs2_data (tcu_tbuf_rs2_data), .tbuf_ready_eff (tcu_tbuf_ready), .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (lsu_tcu_mem_if[0]), `endif .clk (clk), .reset (reset) ); - // --- LSU memory paths: client 0 from top ports, client 1 = TCU AGU --- + // --- LSU memory paths: client 0 from top ports, client 1 = the TCU block AGUs --- wire [NUM_LSU_BLKS-1:0] block_sched_empty; wire [NUM_LSU_BLKS-1:0] block_store_empty; From 45e6a23561677d121ad4445336bbee8ed4930014 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 23 Oct 2026 12:29:57 -0700 Subject: [PATCH 092/336] afu: let the memory ports keep many transactions in flight The XRT AFU held memory-level parallelism to one transaction per direction at the platform AXI boundary, which caps a port at one line per DRAM round trip whatever the caches have in flight: - Multi-port shells (u280, u200): VX_afu_axi_limit kept AXI port 0 to one read and one write in flight. Lines interleave across ports, so every stream was paced by port 0. It only reproduced the old 2:1 arbiter's policy; it is removed and port 0 behaves like the other ports. - Single-port shells (u55c/u50 merged HBM, u250, versal): the arbiter that merges Vortex and the command processor ran single-outstanding, so the whole GPU had one DRAM read and one write in flight. It now runs MULTI_OUT: the source index rides in the top ID bit and responses are routed by it. Vortex keeps the remaining ID bits, and a static assert checks that the CP's ID fits. VX_mm_axi_arb could also switch its AR/AW grant while an offer was still waiting for READY, changing ADDR/ID/LEN under a held VALID (AXI A3.2.1). An offer the slave has not taken now holds the grant; every other cycle arbitrates as before. The memory-port drain counters widen to 16 bits, since one port can now carry more than 1024 transactions. xrtsim cycles, base -> fix (all PASSED): single port, 1 core, latency 100: vecadd -n65536 2346805 -> 1883709 (1.25x), sgemm -n64 1367849 -> 1074041 (1.27x) 4 ports, 1 core, latency 100: 1.06x / 1.08x single port, 4 cores + L2: sgemv 45914 -> 35316 (1.30x) default timing (short round trip): 1.00-1.13x The gain grows with DRAM latency, so it is larger on HBM hardware. SimX and rtlsim do not model the AFU; their cycles are unchanged. Validated: 23/23 xrt ci cases, unit tests cp_llc_port, cp_axi_path, cp_core and afu_reset_seq, AFU lint at the u55c/u200/u280/u250 shapes with no new warnings. Vivado synthesis not run. Co-Authored-By: Claude Opus 5.5 --- docs/designs/fpga_afu_shell.md | 16 ++-- hw/rtl/afu/common/VX_afu_axi_limit.sv | 99 ----------------------- hw/rtl/afu/common/VX_afu_wrap.sv | 108 ++++++++++---------------- hw/rtl/libs/VX_mm_axi_arb.sv | 70 +++++++++++++++-- 4 files changed, 115 insertions(+), 178 deletions(-) delete mode 100644 hw/rtl/afu/common/VX_afu_axi_limit.sv diff --git a/docs/designs/fpga_afu_shell.md b/docs/designs/fpga_afu_shell.md index f866d40b7a..413a108129 100644 --- a/docs/designs/fpga_afu_shell.md +++ b/docs/designs/fpga_afu_shell.md @@ -27,7 +27,6 @@ DMA-command engine have been removed. | [`VX_afu_ctrl.sv`](../../hw/rtl/afu/xrt/VX_afu_ctrl.sv) | Slimmed AXI-Lite slave (~322 LOC): `ap_ctrl` stub at 0x00 + a SCOPE serial register pair + SCOPE watchdog. | | [`VX_afu_axil_demux.sv`](../../hw/rtl/afu/common/VX_afu_axil_demux.sv) | AXI-Lite demux splitting the control space on `addr[12]`, one outstanding transaction per direction. | | [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing. | -| [`VX_afu_axi_limit.sv`](../../hw/rtl/afu/common/VX_afu_axi_limit.sv) | Holds an AXI master to one read and one write in flight; applied to port 0 of the multi-port shell. | | [`VX_afu_reset_seq.sv`](../../hw/rtl/afu/common/VX_afu_reset_seq.sv) | Quiesce-before-reset sequencer for the soft reset; refuses rather than resetting a master that will not drain. | - **Control.** Host AXI-Lite `addr[12]` splits the slave: `addr[12]=0` → @@ -42,12 +41,15 @@ DMA-command engine have been removed. Covered by `hw/unittest/afu_axil_demux`. - **Memory.** Two structures, selected by the number of memory ports. *One port* (the merged platforms): `Vortex_axi` and the CP's `axi_dev` - share the port through a 2:1 `VX_mm_axi_arb`, and CP bursts reach the - platform as bursts. *Several ports*: Vortex's memory ports and `axi_dev` - (bridged by `VX_membus_from_axi`) are the inputs of one bank adapter, - `VX_mem_to_axi`, so the CP is merged **upstream** of bank selection and - reaches every bank. In both, port 0 carries one read and one write in - flight (`VX_afu_axi_limit` in the second case). CP `axi_host` drives a + share the port through a 2:1 `VX_mm_axi_arb` in its ID-routed + multi-outstanding mode (`MULTI_OUT=1`: the top ID bit names the source, + Vortex gets the remaining ID bits), and CP bursts reach the platform as + bursts. *Several ports*: Vortex's memory ports and `axi_dev` (bridged by + `VX_membus_from_axi`) are the inputs of one bank adapter, `VX_mem_to_axi`, + so the CP is merged **upstream** of bank selection and reaches every bank. + In both, every port carries as many reads and writes in flight as its + masters issue; one outstanding transaction per direction would cap a port + at one line per DRAM round trip. CP `axi_host` drives a **dedicated `m_axi_host` AXI master** for the command ring + host DMA. `PLATFORM_MEMORY_OFFSET` is applied per bank, at the port. - **Interrupt.** The AFU `interrupt` pin is driven from `cp_interrupt` diff --git a/hw/rtl/afu/common/VX_afu_axi_limit.sv b/hw/rtl/afu/common/VX_afu_axi_limit.sv deleted file mode 100644 index 37bc3bf736..0000000000 --- a/hw/rtl/afu/common/VX_afu_axi_limit.sv +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_platform.vh" - -// ============================================================================ -// VX_afu_axi_limit — holds an AXI master to one read and one write in flight. -// -// A new AR is withheld until the previous read's last beat, and a new AW until -// the previous write's B. The limits only ever tighten on a handshake, so an -// offer already made to the slave is never withdrawn. -// -// AW and W of one write may fire in either order. W is let through whenever -// its AW has already gone, and AW whenever its W has, whatever the limit says -// by then: withholding half of an accepted write would hang it. -// ============================================================================ - -`TRACING_OFF -module VX_afu_axi_limit ( - input wire clk, - input wire reset, - - input wire in_awvalid, - output wire in_awready, - input wire in_wvalid, - output wire in_wready, - input wire in_wlast, - input wire in_arvalid, - output wire in_arready, - - output wire out_awvalid, - input wire out_awready, - output wire out_wvalid, - input wire out_wready, - output wire out_arvalid, - input wire out_arready, - - input wire b_fire, - input wire r_fire_last -); - reg rd_busy; - reg wr_busy; - reg w_owed; // AW sent, its W still to come - reg aw_owed; // W sent, its AW still to come - - wire aw_allow = aw_owed || ~wr_busy; - wire w_allow = w_owed || (~wr_busy && ~aw_owed); - wire ar_allow = ~rd_busy; - - assign out_awvalid = in_awvalid && aw_allow; - assign in_awready = out_awready && aw_allow; - assign out_wvalid = in_wvalid && w_allow; - assign in_wready = out_wready && w_allow; - assign out_arvalid = in_arvalid && ar_allow; - assign in_arready = out_arready && ar_allow; - - wire aw_fire = out_awvalid && out_awready; - wire w_fire_last = out_wvalid && out_wready && in_wlast; - wire ar_fire = out_arvalid && out_arready; - - always @(posedge clk) begin - if (reset) begin - rd_busy <= 1'b0; - wr_busy <= 1'b0; - w_owed <= 1'b0; - aw_owed <= 1'b0; - end else begin - if (ar_fire) begin - rd_busy <= 1'b1; - end else if (r_fire_last) begin - rd_busy <= 1'b0; - end - if (aw_fire) begin - wr_busy <= 1'b1; - end else if (b_fire) begin - wr_busy <= 1'b0; - end - if (aw_fire && ~w_fire_last) begin - aw_owed <= 1'b0; - w_owed <= ~aw_owed; - end else if (w_fire_last && ~aw_fire) begin - w_owed <= 1'b0; - aw_owed <= ~w_owed; - end - end - end - -endmodule -`TRACING_ON diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index bea895dfe0..f19ed26bb8 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -479,11 +479,17 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( if (C_M_AXI_MEM_NUM_BANKS == 1) begin : g_single_port + // The arbiter carries the source index in the top ID bit, so each + // master gets the rest of the platform ID. + localparam ARB_SRC_BITS = 1; + localparam SRC_ID_WIDTH = C_M_AXI_MEM_ID_WIDTH - ARB_SRC_BITS; + `STATIC_ASSERT(`VX_CP_AXI_TID_WIDTH <= SRC_ID_WIDTH, ("CP device-master ID does not fit the platform ID")) + // ---- Vortex's AXI port, ahead of the arbiter ---- wire vx_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_awready_a [C_M_AXI_MEM_NUM_BANKS]; wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_awid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_awid_a [C_M_AXI_MEM_NUM_BANKS]; wire [7:0] vx_awlen_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; @@ -494,26 +500,26 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire vx_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_bready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_bid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_bid_a [C_M_AXI_MEM_NUM_BANKS]; wire [1:0] vx_bresp_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_arready_a [C_M_AXI_MEM_NUM_BANKS]; wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_araddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_arid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_arid_a [C_M_AXI_MEM_NUM_BANKS]; wire [7:0] vx_arlen_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_rready_a [C_M_AXI_MEM_NUM_BANKS]; wire [C_M_AXI_MEM_DATA_WIDTH-1:0] vx_rdata_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_rlast_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_rid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_rid_a [C_M_AXI_MEM_NUM_BANKS]; wire [1:0] vx_rresp_a [C_M_AXI_MEM_NUM_BANKS]; Vortex_axi #( .AXI_DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), .AXI_ADDR_WIDTH (M_AXI_MEM_ADDR_WIDTH), - .AXI_TID_WIDTH (C_M_AXI_MEM_ID_WIDTH), + .AXI_TID_WIDTH (SRC_ID_WIDTH), .AXI_NUM_BANKS (C_M_AXI_MEM_NUM_BANKS) ) vortex_axi ( `SCOPE_IO_BIND (1) @@ -577,12 +583,12 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( ); // ---- 2:1 arbiter merges Vortex + CP axi_dev ---- - // Pad CP's narrower ID into the platform ID width so the arbiter sees - // identical signal widths from both sources. - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_awid_padded = - {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.awid}; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_arid_padded = - {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.arid}; + // Zero-extend both sources' IDs to the platform width; the arbiter + // overwrites the top bit with the source index. + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_awid_padded = C_M_AXI_MEM_ID_WIDTH'(vx_awid_a[0]); + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_arid_padded = C_M_AXI_MEM_ID_WIDTH'(vx_arid_a[0]); + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_awid_padded = C_M_AXI_MEM_ID_WIDTH'(cp_axi_dev.awid); + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_arid_padded = C_M_AXI_MEM_ID_WIDTH'(cp_axi_dev.arid); // The CP's device addresses come from the same host-side allocator as the // pointers handed to the cores -- Device::global_mem_, based at @@ -599,7 +605,9 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // Packed 2-master AXI arbiter: index 0 = Vortex bank-0 (priority via // ARBITER="P"), index 1 = CP device master. Input channels are packed // {cp, vx}; the arbiter's slave-side outputs land in local packed wires - // and are split back to the two masters below. + // and are split back to the two masters below. Responses are routed by + // the source bit of their ID, so both masters keep as many transactions + // in flight as they issue. localparam BANK0_STRB_W = C_M_AXI_MEM_DATA_WIDTH/8; wire [1:0] b0_awready; @@ -620,6 +628,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), .ID_WIDTH (C_M_AXI_MEM_ID_WIDTH), .ARBITER ("P"), // index 0 (Vortex bank-0) > index 1 (CP) + .MULTI_OUT (1), .STRB_WIDTH (BANK0_STRB_W) ) bank0_arb ( .clk (clk), @@ -628,7 +637,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .s_awvalid ({cp_axi_dev.awvalid, vx_awvalid_a[0]}), .s_awready (b0_awready), .s_awaddr ({cp_awaddr_dev, vx_awaddr_a[0]}), - .s_awid ({cp_awid_padded, vx_awid_a[0]}), + .s_awid ({cp_awid_padded, vx_awid_padded}), .s_awlen ({cp_axi_dev.awlen, vx_awlen_a[0]}), .s_wvalid ({cp_axi_dev.wvalid, vx_wvalid_a[0]}), @@ -645,7 +654,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .s_arvalid ({cp_axi_dev.arvalid, vx_arvalid_a[0]}), .s_arready (b0_arready), .s_araddr ({cp_araddr_dev, vx_araddr_a[0]}), - .s_arid ({cp_arid_padded, vx_arid_a[0]}), + .s_arid ({cp_arid_padded, vx_arid_padded}), .s_arlen ({cp_axi_dev.arlen, vx_arlen_a[0]}), .s_rvalid (b0_rvalid), @@ -675,6 +684,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // index 0 = Vortex bank-0, index 1 = CP device master. // Declared before their first assignment below (the implicit-net footgun: // a pre-declaration use makes Vivado mint 1-bit nets and keep both). + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_bid_full; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_rid_full; wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_bid_full; wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_rid_full; assign vx_awready_a[0] = b0_awready[0]; @@ -683,7 +694,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( assign cp_axi_dev.wready = b0_wready[1]; assign vx_bvalid_a[0] = b0_bvalid[0]; assign cp_axi_dev.bvalid = b0_bvalid[1]; - assign vx_bid_a[0] = b0_bid[0]; + assign vx_bid_full = b0_bid[0]; assign cp_axi_dev_bid_full = b0_bid[1]; assign vx_bresp_a[0] = b0_bresp[0]; assign cp_axi_dev.bresp = b0_bresp[1]; @@ -695,14 +706,18 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( assign cp_axi_dev.rdata = b0_rdata[1]; assign vx_rlast_a[0] = b0_rlast[0]; assign cp_axi_dev.rlast = b0_rlast[1]; - assign vx_rid_a[0] = b0_rid[0]; + assign vx_rid_full = b0_rid[0]; assign cp_axi_dev_rid_full = b0_rid[1]; assign vx_rresp_a[0] = b0_rresp[0]; assign cp_axi_dev.rresp = b0_rresp[1]; - // Truncate the arbiter's wider ID back to CP's narrower native ID width. + // Truncate the arbiter's platform-width IDs back to each master's own. + assign vx_bid_a[0] = vx_bid_full[SRC_ID_WIDTH-1:0]; + assign vx_rid_a[0] = vx_rid_full[SRC_ID_WIDTH-1:0]; assign cp_axi_dev.bid = cp_axi_dev_bid_full[`VX_CP_AXI_TID_WIDTH-1:0]; assign cp_axi_dev.rid = cp_axi_dev_rid_full[`VX_CP_AXI_TID_WIDTH-1:0]; + `UNUSED_VAR (vx_bid_full) + `UNUSED_VAR (vx_rid_full) `UNUSED_VAR (cp_axi_dev_bid_full) `UNUSED_VAR (cp_axi_dev_rid_full) @@ -939,13 +954,6 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `UNUSED_VAR (cp_axi_dev.arburst) - wire ad_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_awready_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_arready_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_wready_a [C_M_AXI_MEM_NUM_BANKS]; - VX_mem_to_axi #( .DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), .ADDR_WIDTH_IN (VX_MEM_ADDR_A_WIDTH), @@ -974,8 +982,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .mem_rsp_tag (fab_rsp_tag), .mem_rsp_ready (fab_rsp_ready), - .m_axi_awvalid (ad_awvalid_a), - .m_axi_awready (ad_awready_a), + .m_axi_awvalid (pre_awvalid_a), + .m_axi_awready (pre_awready_a), .m_axi_awaddr (m_axi_mem_awaddr_u), .m_axi_awid (m_axi_mem_awid_a), .m_axi_awlen (m_axi_mem_awlen_a), @@ -987,8 +995,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `UNUSED_PIN (m_axi_awqos), `UNUSED_PIN (m_axi_awregion), - .m_axi_wvalid (ad_wvalid_a), - .m_axi_wready (ad_wready_a), + .m_axi_wvalid (m_axi_mem_wvalid_a), + .m_axi_wready (m_axi_mem_wready_a), .m_axi_wdata (m_axi_mem_wdata_a), .m_axi_wstrb (m_axi_mem_wstrb_a), .m_axi_wlast (m_axi_mem_wlast_a), @@ -998,8 +1006,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .m_axi_bid (m_axi_mem_bid_a), .m_axi_bresp (m_axi_mem_bresp_a), - .m_axi_arvalid (ad_arvalid_a), - .m_axi_arready (ad_arready_a), + .m_axi_arvalid (pre_arvalid_a), + .m_axi_arready (pre_arready_a), .m_axi_araddr (m_axi_mem_araddr_u), .m_axi_arid (m_axi_mem_arid_a), .m_axi_arlen (m_axi_mem_arlen_a), @@ -1019,40 +1027,6 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .m_axi_rresp (m_axi_mem_rresp_a) ); - // Bank 0 carries one read and one write at a time, the other banks - // as many as the adapter tracks. Reads in flight cost cycles on a - // saturated channel and save them across several, so moving this - // limit changes cycle counts in both directions. - VX_afu_axi_limit bank0_limit ( - .clk (clk), - .reset (subsys_reset), - .in_awvalid (ad_awvalid_a[0]), - .in_awready (ad_awready_a[0]), - .in_wvalid (ad_wvalid_a[0]), - .in_wready (ad_wready_a[0]), - .in_wlast (m_axi_mem_wlast_a[0]), - .in_arvalid (ad_arvalid_a[0]), - .in_arready (ad_arready_a[0]), - .out_awvalid (pre_awvalid_a[0]), - .out_awready (pre_awready_a[0]), - .out_wvalid (m_axi_mem_wvalid_a[0]), - .out_wready (m_axi_mem_wready_a[0]), - .out_arvalid (pre_arvalid_a[0]), - .out_arready (pre_arready_a[0]), - .b_fire (m_axi_mem_bvalid_a[0] && m_axi_mem_bready_a[0]), - .r_fire_last (m_axi_mem_rvalid_a[0] && m_axi_mem_rready_a[0] - && m_axi_mem_rlast_a[0]) - ); - - for (genvar i = 1; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_bank_direct - assign pre_awvalid_a[i] = ad_awvalid_a[i]; - assign ad_awready_a[i] = pre_awready_a[i]; - assign m_axi_mem_wvalid_a[i] = ad_wvalid_a[i]; - assign ad_wready_a[i] = m_axi_mem_wready_a[i]; - assign pre_arvalid_a[i] = ad_arvalid_a[i]; - assign ad_arready_a[i] = pre_arready_a[i]; - end - end // ======================================================================== @@ -1087,7 +1061,11 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .out_ready (m_axi_mem_arready_a[i]) ); - VX_afu_axi_drain mem_drain ( + // A memory port carries every miss the last-level caches hold in + // flight, so its counters are sized well past any MSHR total. + VX_afu_axi_drain #( + .COUNT_WIDTH (16) + ) mem_drain ( .clk (clk), .reset (reset), .aw_fire (m_axi_mem_awvalid_a[i] && m_axi_mem_awready_a[i]), diff --git a/hw/rtl/libs/VX_mm_axi_arb.sv b/hw/rtl/libs/VX_mm_axi_arb.sv index 0c9536d093..7bfddde3c2 100644 --- a/hw/rtl/libs/VX_mm_axi_arb.sv +++ b/hw/rtl/libs/VX_mm_axi_arb.sv @@ -175,6 +175,20 @@ module VX_mm_axi_arb #( assign aw_pick = '0; assign aw_any = s_awvalid[0]; end else begin : g_aw_arb + // Once offered, the AW must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg aw_held; + reg [SEL_WIDTH-1:0] aw_held_idx; + wire [NUM_INPUTS-1:0] aw_reqs = aw_held ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) + : s_awvalid; + always @(posedge clk) begin + if (reset) begin + aw_held <= 1'b0; + end else begin + aw_held <= c_awvalid && ~c_awready; + end + aw_held_idx <= aw_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), @@ -182,7 +196,7 @@ module VX_mm_axi_arb #( ) aw_arb ( .clk (clk), .reset (reset), - .requests (s_awvalid), + .requests (aw_reqs), .grant_index (aw_pick), `UNUSED_PIN (grant_onehot), .grant_valid (aw_any), @@ -245,6 +259,20 @@ module VX_mm_axi_arb #( assign ar_pick = '0; assign ar_any = s_arvalid[0]; end else begin : g_ar_arb + // Once offered, the AR must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg ar_held; + reg [SEL_WIDTH-1:0] ar_held_idx; + wire [NUM_INPUTS-1:0] ar_reqs = ar_held ? (s_arvalid & (NUM_INPUTS'(1) << ar_held_idx)) + : s_arvalid; + always @(posedge clk) begin + if (reset) begin + ar_held <= 1'b0; + end else begin + ar_held <= c_arvalid && ~c_arready; + end + ar_held_idx <= ar_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), @@ -252,7 +280,7 @@ module VX_mm_axi_arb #( ) ar_arb ( .clk (clk), .reset (reset), - .requests (s_arvalid), + .requests (ar_reqs), .grant_index (ar_pick), `UNUSED_PIN (grant_onehot), .grant_valid (ar_any), @@ -298,8 +326,8 @@ module VX_mm_axi_arb #( // The top SEL_WIDTH bits of the AXI ID carry the source index; masters // must leave them free. Reads are fully concurrent; write bursts serialize // on the shared W channel (AW gated until the prior burst's WLAST) while B - // responses stay concurrent. Sticky arbitration holds the granted addr/id - // stable until the handshake completes (AXI stability requirement). + // responses stay concurrent. An offer the slave has not yet taken holds + // the grant, so its addr/id stay stable until the handshake. localparam LOW_ID = ID_WIDTH - SEL_WIDTH; // source's own low ID bits `STATIC_ASSERT((ID_WIDTH > SEL_WIDTH), ("VX_mm_axi_arb MULTI_OUT: ID_WIDTH must exceed LOG2UP(NUM_INPUTS) to carry the source index")) @@ -312,12 +340,26 @@ module VX_mm_axi_arb #( assign ar_any = s_arvalid[0]; end else begin : g_ar_arb wire ar_fire = c_arvalid && c_arready; + // Once offered, the AR must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg ar_held; + reg [SEL_WIDTH-1:0] ar_held_idx; + wire [NUM_INPUTS-1:0] ar_reqs = ar_held ? (s_arvalid & (NUM_INPUTS'(1) << ar_held_idx)) + : s_arvalid; + always @(posedge clk) begin + if (reset) begin + ar_held <= 1'b0; + end else begin + ar_held <= c_arvalid && ~c_arready; + end + ar_held_idx <= ar_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), - .STICKY (1) // hold grant stable until AR fires (AXI stability) + .STICKY (1) ) ar_arb ( - .clk (clk), .reset (reset), .requests (s_arvalid), + .clk (clk), .reset (reset), .requests (ar_reqs), .grant_index (ar_pick), `UNUSED_PIN (grant_onehot), .grant_valid (ar_any), .grant_ready (ar_fire) ); @@ -350,12 +392,26 @@ module VX_mm_axi_arb #( assign aw_pick = '0; assign aw_any = s_awvalid[0]; end else begin : g_aw_arb + // Once offered, the AW must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg aw_held; + reg [SEL_WIDTH-1:0] aw_held_idx; + wire [NUM_INPUTS-1:0] aw_reqs = aw_held ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) + : s_awvalid; + always @(posedge clk) begin + if (reset) begin + aw_held <= 1'b0; + end else begin + aw_held <= c_awvalid && ~c_awready; + end + aw_held_idx <= aw_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), .STICKY (1) ) aw_arb ( - .clk (clk), .reset (reset), .requests (s_awvalid), + .clk (clk), .reset (reset), .requests (aw_reqs), .grant_index (aw_pick), `UNUSED_PIN (grant_onehot), .grant_valid (aw_any), .grant_ready (aw_fire) ); From f93df327f70df5e35d353a1d1a40ff23282f1b75 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 23 Oct 2026 18:17:31 -0700 Subject: [PATCH 093/336] cache: route a crossbar input's ready from its own output, drain slices before a token VX_stream_xbar ORed every output's ready into each input's ready. Flat arbiters only grant valid requesters, so that equalled the selected output's ready. But a bank arbiter with a fan-in wider than 1.5 x MAX_FANOUT splits into registered slices, and a one-input slice passes its buffer's ready through unqualified. With 17 requesters on the L2 (16 L1 ports plus the command processor's DMA port, CP_LLC_CLIENT=1), another bank's slice acknowledged a CP write bound for a full bank, and the write was dropped: - sgemm read stale B (all 16384 outputs wrong, about 86% of expected) at the a100 shape, reduced down to one core with 4 warps x 16 threads, issue width 4, a 32-bank L2 and HBM timing; - at the default L2 geometry the kernel arguments read as 0xbaadf00d and VX_lsu_block's address assertion fired. Each input now takes its selected output's ready. The flush sequencer's in-flight counter, meant to hold a flush token until earlier requests reach the banks, was sized and wired so it never counted. The request crossbar's slices let requests from different inputs pass each other, so VX_cache_init now counts requests entering the crossbar (a merged read once, as its leader) and drains them before it walks or answers a token. Without slices (12 or fewer requesters, every default configuration) the counter is not built. Known-bad: without the change the reduced repro fails (16384 errors) and the default-geometry config asserts at cycle 12897. With it: both pass; sgemm at the a100 and rtx5080 4-core shapes with CP_LLC_CLIENT=1 pass; hw/unittest cp_llc_port and the CP LLC parity cases (dogfood-cp-llc, sgemm-cp-llc-17, sgemm-cp-llc-65) pass; default sgemm and raycast rtlsim cycles are identical (799610, 21106013); lint clean at XLEN 32 and 64. SimX already behaved correctly. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache.sv | 28 +++++++++++++++++++----- hw/rtl/cache/VX_cache_init.sv | 41 +++++++++++++++++------------------ hw/rtl/libs/VX_stream_xbar.sv | 5 ++++- 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 926f807452..fb54750a87 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -104,12 +104,26 @@ module VX_cache import VX_gpu_pkg::*; #( localparam MEM_PORTS_SEL_WIDTH = `UP(MEM_PORTS_SEL_BITS); localparam MEM_ARB_SEL_BITS = `CLOG2(`CDIV(NUM_BANKS, MEM_PORTS)); localparam REQ_XBAR_BUF = (NUM_REQS > 2) ? 2 : 0; + // A bank's request arbiter splits a fan-in wider than 1.5 x MAX_FANOUT + // into registered two-entry slices, where requests from different inputs + // wait apart: entries per bank over all slice levels (0 without slices). + function automatic integer req_xbar_slice_slots(integer n); + integer m, slots; + m = n; + slots = 0; + while (m > (`MAX_FANOUT + `MAX_FANOUT / 2)) begin + m = `CDIV(m, `MAX_FANOUT); + slots = slots + 2 * m; + end + return slots; + endfunction + localparam REQ_XBAR_SLICE_SLOTS = req_xbar_slice_slots(NUM_REQS); + // requests the crossbar can hold when slices let them pass each other; + // the flush sequencer drains them before it walks or answers a token + localparam REQ_XBAR_SIZE = (REQ_XBAR_SLICE_SLOTS != 0) ? (NUM_BANKS * (REQ_XBAR_SLICE_SLOTS + `TO_OUT_BUF_SIZE(REQ_XBAR_BUF))) : 0; localparam CORE_RSP_BUF_ENABLE = (NUM_BANKS != 1) || (NUM_REQS != 1); localparam MEM_REQ_BUF_ENABLE = (NUM_BANKS != 1); `STATIC_ASSERT(!DO_READ_MERGE || (`IS_POW2(MERGE_GROUP) && `IS_DIVISBLE(NUM_REQS, MERGE_GROUP)), ("invalid parameter: read merge group")) - // a merged request is one bank access for several port handshakes, which - // the in-flight request count of a registered request crossbar would miss - `STATIC_ASSERT(!DO_READ_MERGE || (`TO_OUT_BUF_REG(REQ_XBAR_BUF) == 0), ("invalid parameter: read merge with a registered request crossbar")) `ifdef PERF_ENABLE wire [NUM_BANKS-1:0] perf_read_miss_per_bank; wire [NUM_BANKS-1:0] perf_write_miss_per_bank; @@ -124,6 +138,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire flush_skip; wire [`UP(UUID_WIDTH)-1:0] flush_uuid; wire [NUM_BANKS-1:0] per_bank_flush_end; + wire [NUM_REQS-1:0] core_req_xbar_fire; wire [NUM_BANKS-1:0] per_bank_core_req_fire; wire mem_req_empty; wire [MEM_PORTS-1:0] mem_flush_valid; @@ -143,13 +158,14 @@ module VX_cache import VX_gpu_pkg::*; #( .FLUSH_FORWARD (FLUSH_FORWARD), .SHARED_LLC (SHARED_LLC), .TAG_WIDTH (TAG_WIDTH), - .BANK_SEL_LATENCY (`TO_OUT_BUF_REG(REQ_XBAR_BUF)), // request xbar latency + .XBAR_SIZE (REQ_XBAR_SIZE), .REQ_SEL_WIDTH (REQ_SEL_WIDTH) ) cache_init ( .clk (clk), .reset (reset), .core_bus_in_if (core_bus_if), .core_bus_out_if (core_bus2_if), + .xbar_req_fire (core_req_xbar_fire), .bank_req_fire (per_bank_core_req_fire), .flush_begin (per_bank_flush_begin), .flush_skip (flush_skip), @@ -387,7 +403,9 @@ module VX_cache import VX_gpu_pkg::*; #( core_req_attr[i] }; end - assign per_bank_core_req_fire = per_bank_core_req_valid & per_bank_mem_req_ready; + // a merged read enters the crossbar once, as its leader + assign core_req_xbar_fire = core_req_valid & ~core_req_follower & core_req_xbar_ready; + assign per_bank_core_req_fire = per_bank_core_req_valid & per_bank_core_req_ready; `ifdef PERF_ENABLE wire [PERF_CTR_BITS-1:0] perf_collisions; `endif diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index 75c0aae526..28439232d6 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -23,6 +23,9 @@ // the cache's own response path, and the inputs reopen. // A token that arrives during the walk is visible only after all of its // input's earlier requests were accepted, so the walk covered them. +// A request crossbar that can hold requests apart from their input order +// (XBAR_SIZE != 0) drains into the banks first, while the inputs are held, +// so no earlier request of a token's input is passed by a later one. // An L1-only token (any byteen bit set: a next level with wider words sees // the bit at the token's line offset) is forwarded with its kind. A // last-level cache that alone serves every memory client is the point of @@ -43,13 +46,14 @@ module VX_cache_init import VX_gpu_pkg::*; #( parameter FLUSH_FORWARD = 1, // Forward the flush to the next level parameter SHARED_LLC = 0, // The LLC serving every memory client alone: answers L1-only tokens without a walk parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size - parameter BANK_SEL_LATENCY = 1, // Bank select latency + parameter XBAR_SIZE = 0, // Requests the request crossbar can hold apart from input order parameter REQ_SEL_WIDTH = `UP(`CLOG2(NUM_REQS)) ) ( input wire clk, input wire reset, VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], VX_mem_bus_if.master core_bus_out_if [NUM_REQS], + input wire [NUM_REQS-1:0] xbar_req_fire, input wire [NUM_BANKS-1:0] bank_req_fire, output wire [NUM_BANKS-1:0] flush_begin, output wire flush_skip, // the banks only quiesce: no walk is owed @@ -79,35 +83,30 @@ module VX_cache_init import VX_gpu_pkg::*; #( reg [2:0] state, state_n; - // track in-flight core requests + // track requests inside the request crossbar wire no_inflight_reqs; - if (BANK_SEL_LATENCY != 0) begin : g_bank_sel_latency + if (XBAR_SIZE != 0) begin : g_xbar_inflight - localparam NUM_REQS_W = `CLOG2(NUM_REQS+1); - localparam NUM_BANKS_W = `CLOG2(NUM_BANKS+1); + localparam REQS_W = `CLOG2(NUM_REQS+1); + localparam INCRW = `CLOG2(`MIN(NUM_REQS, XBAR_SIZE)+1); + localparam BANKS_W = `CLOG2(NUM_BANKS+1); - wire [NUM_REQS-1:0] core_bus_out_fire; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_fire - assign core_bus_out_fire[i] = core_bus_out_if[i].req_valid && core_bus_out_if[i].req_ready; - end - - wire [NUM_REQS_W-1:0] core_bus_out_cnt; - wire [NUM_BANKS_W-1:0] bank_req_cnt; + wire [REQS_W-1:0] xbar_req_cnt; + wire [BANKS_W-1:0] bank_req_cnt; - `POP_COUNT(core_bus_out_cnt, core_bus_out_fire); + `POP_COUNT(xbar_req_cnt, xbar_req_fire); `POP_COUNT(bank_req_cnt, bank_req_fire); - `UNUSED_VAR (core_bus_out_cnt) VX_pending_size #( - .SIZE (BANK_SEL_LATENCY * NUM_BANKS), - .INCRW (NUM_BANKS_W), - .DECRW (NUM_BANKS_W) + .SIZE (XBAR_SIZE), + .INCRW (INCRW), + .DECRW (BANKS_W) ) pending_size ( .clk (clk), .reset (reset), - .incr (NUM_BANKS_W'(core_bus_out_cnt)), + .incr (INCRW'(xbar_req_cnt)), .decr (bank_req_cnt), .empty (no_inflight_reqs), `UNUSED_PIN (alm_empty), @@ -116,9 +115,9 @@ module VX_cache_init import VX_gpu_pkg::*; #( `UNUSED_PIN (size) ); - end else begin : g_no_bank_sel_latency + end else begin : g_no_xbar_inflight assign no_inflight_reqs = 0; - `UNUSED_VAR (bank_req_fire) + `UNUSED_VAR ({xbar_req_fire, bank_req_fire}) end wire [NUM_REQS-1:0] flush_req_mask; @@ -248,7 +247,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( // STATE_IDLE: default: begin if (flush_req_enable) begin - if (BANK_SEL_LATENCY != 0) begin + if (XBAR_SIZE != 0) begin state_n = STATE_WAIT1; end else if (~flush_walk && ~any_io) begin rsp_pending_n = flush_req_mask; diff --git a/hw/rtl/libs/VX_stream_xbar.sv b/hw/rtl/libs/VX_stream_xbar.sv index e7b18362c2..c8002a0146 100644 --- a/hw/rtl/libs/VX_stream_xbar.sv +++ b/hw/rtl/libs/VX_stream_xbar.sv @@ -63,8 +63,11 @@ module VX_stream_xbar #( .data_out (per_output_ready_in_w) ); + // An output arbiter may assert ready for an input that does not + // target it (a single-input fan-in slice passes its buffer's + // ready through), so take the ready of the selected output only. for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_ready_in - assign ready_in[i] = | per_output_ready_in_w[i]; + assign ready_in[i] = per_output_ready_in_w[i][sel_in[i]]; end for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_sel_in_demux From c5ed76a73201861f38fc339f50748005ad81c730 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 23 Oct 2026 18:17:32 -0700 Subject: [PATCH 094/336] profiles: per-partition fetch and the CP as an LLC client on the GPU shapes All five GPU profiles now fetch per SM processing block (VX_CFG_FETCH_WIDTH=4: four partitions, each with its own instruction buffer and scheduler) and route the command processor's DMA through the last-level cache (VX_CFG_CP_LLC_CLIENT=1), as on the real parts. SimX at 4 cores: FETCH_WIDTH=4 gives sgemm -29.7% (rtx5080 -n192) and -40.7% (a100 -n128); CP_LLC_CLIENT a further -6.5% / -5.8%. Both pass in rtlsim at the a100 and rtx5080 4-core shapes. The rtx5080 DRAM latency is recalibrated from 339 to 675 cycles with the chase app (cold L2, CP_LLC_CLIENT=0): 802 cycles per dependent DRAM access. L2_LAZY_FETCH and the MSHR line table stay off: lazy fetch does not yet promote fully written sectors, so it costs the CP-through-LLC gain. Co-Authored-By: Claude Opus 5.5 --- profiles/a100.json | 10 +++++++--- profiles/h100.json | 10 +++++++--- profiles/rtx1080.json | 10 +++++++--- profiles/rtx5080.json | 12 ++++++++---- profiles/v100.json | 10 +++++++--- 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/profiles/a100.json b/profiles/a100.json index 7a6003fe0b..a9f778afe1 100644 --- a/profiles/a100.json +++ b/profiles/a100.json @@ -5,7 +5,8 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 131072 + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +14,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 33554432, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32 + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -30,6 +32,8 @@ "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM2 stacks x 8 128-bit channels = 40 channels at 2430 MT/s, modelled as 32 channels at 3038 MT/s for a power-of-two interleave: 32 x 16 B x 3038 MT/s = 1555 GB/s, the card's bandwidth; the array timing is in nanoseconds, so latency is unchanged", "PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_QUEUE_SIZE": "Fixed DRAM latency and per-channel request queue of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA A100 Tensor Core GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } diff --git a/profiles/h100.json b/profiles/h100.json index d95d2fabda..5100b09169 100644 --- a/profiles/h100.json +++ b/profiles/h100.json @@ -5,7 +5,8 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 262144 + "VX_CFG_DCACHE_SIZE": 262144, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +14,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 33554432, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32 + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -27,6 +29,8 @@ "DCACHE_SIZE": "256 KB combined L1/shared per SM (whitepaper)", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways and 32 banks as in the a100 profile", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM3 stacks x 16 64-bit channels at 5236 MT/s = 40 128-bit channel pairs, modelled as 32 at 6545 MT/s for a power-of-two interleave: 32 x 16 B x 6545 MT/s = 3.35 TB/s, the card's bandwidth; latency not calibrated", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } diff --git a/profiles/rtx1080.json b/profiles/rtx1080.json index 73908bb3d2..3c95172e2c 100644 --- a/profiles/rtx1080.json +++ b/profiles/rtx1080.json @@ -5,7 +5,8 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 32768 + "VX_CFG_DCACHE_SIZE": 32768, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +14,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 2097152, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 16 + "VX_CFG_L2_NUM_BANKS": 16, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, @@ -27,6 +29,8 @@ "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at 10000 MT/s: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own instruction buffer and warp scheduler (NVIDIA GeForce GTX 1080 (GP104) whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } diff --git a/profiles/rtx5080.json b/profiles/rtx5080.json index d90b047d99..f703f3e82b 100644 --- a/profiles/rtx5080.json +++ b/profiles/rtx5080.json @@ -6,7 +6,8 @@ "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, "VX_CFG_DCACHE_SIZE": 131072, - "VX_CFG_DCACHE_MSHR_SIZE": 24 + "VX_CFG_DCACHE_MSHR_SIZE": 24, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -15,7 +16,8 @@ "VX_CFG_L2_SIZE": 67108864, "VX_CFG_L2_NUM_WAYS": 16, "VX_CFG_L2_NUM_BANKS": 32, - "VX_CFG_L2_MSHR_SIZE": 192 + "VX_CFG_L2_MSHR_SIZE": 192, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -23,7 +25,7 @@ "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000, "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 15000, - "VX_CFG_PLATFORM_MEMORY_LATENCY": 339 + "VX_CFG_PLATFORM_MEMORY_LATENCY": 675 }, "reference": { "clock_mhz": 2295, @@ -58,6 +60,8 @@ "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", "reference": "Nsight Compute on an RTX 5080 at its 2295 MHz base clock with a cold L2, 2026-10-05", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_DATA_RATE": "30 Gbps GDDR7 x8 channels modelled as GDDR6 x16 channels at 15000 MT/s: the same channel count, 32 B burst time and 960 GB/s (32 x 2 B x 15 GT/s)", - "PLATFORM_MEMORY_LATENCY": "339 core cycles of PHY/NoC latency: the 407 calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference', less the 68 cycles JEDEC GDDR6 timing adds to the device's row-conflict latency (derived, not re-run; recalibrate with a pointer chase when the memory model changes)" + "PLATFORM_MEMORY_LATENCY": "675 core cycles of PHY/NoC latency, calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference' with the same chase (random cyclic permutation, 128 B stride, one thread, cold L2 with CP_LLC_CLIENT=0; cycle difference between 128 KiB and 256 KiB footprints over the 1024 extra loads: 802.1). The L2 link latency is not modelled, so an L2 hit is about 79 cycles against the card's 354. Recalibrate when the memory model changes", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA RTX Blackwell GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } diff --git a/profiles/v100.json b/profiles/v100.json index 451551d910..4bccc586bd 100644 --- a/profiles/v100.json +++ b/profiles/v100.json @@ -5,7 +5,8 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 131072 + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +14,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 4194304, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32 + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -27,6 +29,8 @@ "DCACHE_SIZE": "128 KB combined L1/shared per SM (whitepaper)", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "6 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways, 32 banks for the 32 memory channels", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "4 HBM2 stacks x 8 128-bit channels = 32 channels at 1752 MT/s: 32 x 16 B x 1752 MT/s = 897 GB/s (whitepaper)", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA Tesla V100 GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } From 1a3055f04d583a75106f985e68f448034c3cd2f5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 24 Oct 2026 03:05:07 -0700 Subject: [PATCH 095/336] Support core, socket and cluster counts that are not powers of two Real GPUs are not built in powers of two (RTX 5080 84 SMs, A100 108, H100 132), and SimX already ran those shapes, but Vortex.sv asserted power-of-two NUM_CLUSTERS, NUM_CORES and SOCKET_SIZE, so rtlsim and the FPGA could not build them. The requirement is now only that NUM_CORES is a multiple of SOCKET_SIZE, and the sites that sliced core, socket or cluster IDs by bits use constant arithmetic that reduces to the same bits for powers of two: - KMU fragment routing picks the next hop with (dest / DEST_DIV) % N. - The DXA local-memory core select uses % NUM_CORES, then % SOCKET_SIZE. - The AMO hart id dropped the cluster bits, so cores in different clusters aliased in the LR/SC reservation table; it now carries the global core id. - The command processor exposes the exact SOCKET_SIZE in a new read-only capability word (0x00C), so the runtime reports the right core count at socket sizes that are not powers of two. Older CPs answer that offset with garbage, so a word with its reserved bits set is ignored. - SimX rejects RTU configurations whose socket size is not a multiple of the RTU core count, matching what the RTL can build. Known-bad: vecadd at 3 cores + L2 hit the power-of-two assert; sgemm2_dxa at 4 cores with sockets of 2 hung (core 1 scheduler timeout) under an intermediate version of this change. Both pass now. Validation: sgemm, vecadd, async_gbarrier, dogfood and printf at 3 cores, 6 cores (issue width 4) and 3 clusters x 2 cores (L2 + L3) pass on rtlsim and SimX with matching instruction counts; sgemm at 3 cores + L2 is 1951807 cycles on rtlsim and 1972907 on SimX (+1.08%). Power-of-two configurations are unchanged: default sgemm 5943507, raycast 21106013, 4-core + L2 sgemm 1477825 cycles. Lint clean in 22 shapes at XLEN 32 and 64. New CI cases cover 3 and 6 cores, and DXA at 4 cores in sockets of 2 and 6 cores in sockets of 3. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 3 +- ci/testcases/config1.yaml | 39 ++++++++++++++++++++++ ci/testcases/dxa.yaml | 39 ++++++++++++++++++++++ docs/assets/img/cta_dispatch_hierarchy.svg | 6 ++-- docs/designs/command_processor.md | 6 ++++ docs/designs/cta_dispatch_architecture.md | 10 +++--- hw/rtl/VX_cluster.sv | 4 +-- hw/rtl/VX_gpu_pkg.sv | 16 +++++---- hw/rtl/VX_socket.sv | 10 +++--- hw/rtl/Vortex.sv | 7 ++-- hw/rtl/core/VX_kmu_bus_arb.sv | 11 +++--- hw/rtl/cp/VX_cp_axil_regfile.sv | 12 +++++-- hw/rtl/dxa/VX_dxa_unit.sv | 6 ++-- hw/rtl/interfaces/VX_kmu_bus_if.sv | 2 +- hw/unittest/cp_axil_regfile/main.cpp | 14 +++++--- sim/common/cmd_processor.cpp | 10 ++++-- sim/common/cmd_processor.h | 1 + sim/simx/constants.h | 2 ++ sim/simx/rtu/rtu_types.h | 2 ++ sw/runtime/common/caps.h | 33 +++++++++++------- sw/runtime/common/device.cpp | 6 ++-- sw/runtime/common/vortex2_internal.h | 5 +-- 22 files changed, 181 insertions(+), 63 deletions(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 92148b00ed..9b52dab9ee 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -264,7 +264,7 @@ tests: args: -n1 # Non-power-of-two cache sharing: 3 cores over 2 shared icaches exercises the -# SimX arbiter's ceiling-division input grouping. A floor-grouped arbiter +# arbiter's ceiling-division input grouping. A floor-grouped arbiter # (inputs/outputs) never polls the trailing core's fetch port and the run # hangs at boot; VX_stream_arb sizes its groups by ceiling division, so the # SimX model must match. @@ -272,6 +272,7 @@ tests: via: blackbox drivers: - simx + - rtlsim app: demo configs: -DVX_CFG_SOCKET_SIZE=3 -DVX_CFG_NUM_ICACHES=2 shape: diff --git a/ci/testcases/config1.yaml b/ci/testcases/config1.yaml index cb2d49aa71..f8ef17bbcb 100644 --- a/ci/testcases/config1.yaml +++ b/ci/testcases/config1.yaml @@ -174,6 +174,45 @@ tests: shape: cores: 4 args: -n1 +# Core and cluster counts that are not powers of two: the shared L2 serves +# three sockets, the global barrier counts three cores, and three clusters +# share the L3. +- id: async_gbarrier-npot-1 + via: blackbox + drivers: + - rtlsim + app: async_gbarrier + shape: + cores: 3 + l2cache: true +- id: async_gbarrier-npot-2 + via: blackbox + drivers: + - simx + app: async_gbarrier + shape: + cores: 3 + l2cache: true +- id: vecadd-npot-1 + via: blackbox + drivers: + - rtlsim + app: vecadd + shape: + cores: 2 + clusters: 3 + l2cache: true + l3cache: true +- id: vecadd-npot-2 + via: blackbox + drivers: + - simx + app: vecadd + shape: + cores: 2 + clusters: 3 + l2cache: true + l3cache: true - id: diverge-17 via: blackbox drivers: diff --git a/ci/testcases/dxa.yaml b/ci/testcases/dxa.yaml index 913c71cb57..dad905a839 100644 --- a/ci/testcases/dxa.yaml +++ b/ci/testcases/dxa.yaml @@ -353,6 +353,45 @@ tests: configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=2 shape: cores: 4 +# Shared L2 behind multi-core sockets: the socket routes each DXA lmem write +# to the core named by its cluster-local index modulo the socket size, at a +# power-of-two shape and at sockets of three. +- id: sgemm2_dxa-l2-1 + via: blackbox + drivers: + - simx + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 4 + l2cache: true +- id: sgemm2_dxa-l2-2 + via: blackbox + drivers: + - rtlsim + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 4 + l2cache: true +- id: sgemm2_dxa-npot-1 + via: blackbox + drivers: + - simx + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=3 + shape: + cores: 6 + l2cache: true +- id: sgemm2_dxa-npot-2 + via: blackbox + drivers: + - rtlsim + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=3 + shape: + cores: 6 + l2cache: true - id: sgemm2_dxa-9 via: blackbox drivers: diff --git a/docs/assets/img/cta_dispatch_hierarchy.svg b/docs/assets/img/cta_dispatch_hierarchy.svg index 64634eab83..ca0ce66ba3 100644 --- a/docs/assets/img/cta_dispatch_hierarchy.svg +++ b/docs/assets/img/cta_dispatch_hierarchy.svg @@ -56,7 +56,7 @@ kmu_arb — Vortex.sv:186 1 -> NUM_CLUSTERS - DEST_LSB_DEVICE + DEST_DIV_DEVICE @@ -64,7 +64,7 @@ kmu_arb — VX_cluster.sv:105 2 -> NUM_SOCKETS, ARBITER "R" - DEST_LSB_CLUSTER + DEST_DIV_CLUSTER the only level with 2 inputs @@ -83,7 +83,7 @@ kmu_arb — VX_socket.sv:69 1 -> VX_CFG_SOCKET_SIZE - DEST_LSB_SOCKET + DEST_DIV_SOCKET socket 1..N diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index 356ddb000b..9e0dd9c5a2 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -349,6 +349,7 @@ subtracts it first. | `0x000` | `CP_CTRL` | RW | bit0 = enable_global, bit1 = reset_all | | `0x004` | `CP_STATUS` | RO | bit0 = busy, bit1 = error | | `0x008` | `CP_DEV_CAPS` | RO | `[7:0]` NUM_QUEUES, `[15:8]` RING_SIZE_LOG2_MAX, `[23:16]` AXI_TID_WIDTH — plus feature bits 24-26 on the Emulation CP (§6) | +| `0x00C` | `GPU_DEV_CAPS_EXT` | RO | device-config caps extension (§6) | | `0x010` | `CP_CYCLE_LO` | RO | free-running cycle counter, low 32 | | `0x014` | `CP_CYCLE_HI` | RO | high 32 | | `0x018` / `0x01C` | `GPU_DEV_CAPS_LO/HI` | RO | device-config caps (§6) | @@ -434,6 +435,11 @@ The exact counts in `[63:42]` let a machine that is not a power of two falls back to the log2 fields only when they read zero, as on devices built before they existed. +**`GPU_DEV_CAPS_EXT`** (`0x00C`) carries the exact counts that no longer fit +in `GPU_DEV_CAPS`: `[7:0]` socket_size (`VX_CAPS_SOCKET_SIZE`), the rest zero. +It reads zero on devices built before it existed, so the decoder falls back to +the log2 socket_size field `[16:14]`. + `VX_CAPS_NUM_CORES` is derived, not a field: `num_clusters × cluster_size × socket_size`, from the exact counts. `CACHE_LINE_SIZE`, `GLOBAL_MEM_SIZE`, `CLOCK_RATE`, `PEAK_MEM_BW` and the VM ids are not diff --git a/docs/designs/cta_dispatch_architecture.md b/docs/designs/cta_dispatch_architecture.md index 7906fd124a..48c6aa0a85 100644 --- a/docs/designs/cta_dispatch_architecture.md +++ b/docs/designs/cta_dispatch_architecture.md @@ -41,12 +41,12 @@ between them, so the slave-port `ready` would cross both stages combinationally. Where a level needs both, it composes two instances around a registered launch link: -| Level | File | Shape | `dest` slice | +| Level | File | Shape | `dest` digit | |---|---|---|---| -| Device | [`Vortex.sv:185`](../../hw/rtl/Vortex.sv#L185) | 1 → `VX_CFG_NUM_CLUSTERS` | `KMU_DEST_LSB_DEVICE` | +| Device | [`Vortex.sv:185`](../../hw/rtl/Vortex.sv#L185) | 1 → `VX_CFG_NUM_CLUSTERS` | `KMU_DEST_DIV_DEVICE` | | Cluster merge | [`VX_cluster.sv:121`](../../hw/rtl/VX_cluster.sv#L121) | 2 → 1 | — | -| Cluster distribute | [`VX_cluster.sv:133`](../../hw/rtl/VX_cluster.sv#L133) | 1 → `NUM_SOCKETS` | `KMU_DEST_LSB_CLUSTER` | -| Socket | [`VX_socket.sv:72`](../../hw/rtl/VX_socket.sv#L72) | 1 → `VX_CFG_SOCKET_SIZE` | `KMU_DEST_LSB_SOCKET` | +| Cluster distribute | [`VX_cluster.sv:133`](../../hw/rtl/VX_cluster.sv#L133) | 1 → `NUM_SOCKETS` | `KMU_DEST_DIV_CLUSTER` | +| Socket | [`VX_socket.sv:72`](../../hw/rtl/VX_socket.sv#L72) | 1 → `VX_CFG_SOCKET_SIZE` | `KMU_DEST_DIV_SOCKET` | `NUM_SOCKETS = UP(VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE)` ([`VX_gpu_pkg.sv:149`](../../hw/rtl/VX_gpu_pkg.sv#L149)). The cluster merge is @@ -228,7 +228,7 @@ Two things are worth noticing about what is *not* here: placement hint, so the fan-out drops its first member on any ready core and the message lock keeps the rest with it. This is what spreads a grid across the machine. -- **FRAGMENT** → routed by this level's slice of `dest`. Fragment work is +- **FRAGMENT** → routed by this level's digit of `dest`. Fragment work is bin→core affine, and that affinity is what keeps same-pixel blend order correct; load-balancing it would break the blend order outright. diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 1b2ff700c5..5dfcb2f4b9 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -145,7 +145,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (NUM_SOCKETS), - .DEST_LSB (KMU_DEST_LSB_CLUSTER), + .DEST_DIV (KMU_DEST_DIV_CLUSTER), .OUT_BUF ((NUM_SOCKETS > 1) ? 3 : 0) ) kmu_dist ( .clk (clk), @@ -157,7 +157,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (NUM_SOCKETS), - .DEST_LSB (KMU_DEST_LSB_CLUSTER), + .DEST_DIV (KMU_DEST_DIV_CLUSTER), .OUT_BUF ((NUM_SOCKETS > 1) ? 3 : 0) ) kmu_arb ( .clk (clk), diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 55e2d64049..904bad773c 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -155,8 +155,9 @@ package VX_gpu_pkg; // table keys on hart_id = make_hart_id(cid, wid, tid). // MINU/MAXU collapse into MIN/MAX + amo_unsigned. - // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid. - localparam HART_ID_BITS = NC_BITS + NW_BITS + NT_BITS; + // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid; cid is + // device-wide, so harts of different clusters stay distinct at a shared LLC. + localparam HART_ID_BITS = CORE_ID_BITS + NW_BITS + NT_BITS; localparam HART_ID_WIDTH = `UP(HART_ID_BITS); // ─────────────────────────────────────────────────────────────────────── @@ -944,16 +945,17 @@ package VX_gpu_pkg; // Placement hint: the global index of the core a fragment launch must land // on. Fragment work is bin->core affine (that is what makes same-pixel blend // order safe), so a fragment message cannot be load-balanced like a compute - // CTA. Each fan-out level consumes its own slice of this index, LSB first: - // core-in-socket, then socket-in-cluster, then cluster (see KMU_DEST_LSB_*). + // CTA. Each fan-out level picks its output as (dest / DIV) % NUM_OUTPUTS, + // where DIV is the number of cores below each of its outputs: core-in-socket, + // then socket-in-cluster, then cluster (see KMU_DEST_DIV_*). // Device-wide, so the hint can name any core: VX_CFG_NUM_CORES is per-cluster. // `UP` so a single-core build still has a 1-bit (unused) hint rather than a // zero-width vector. localparam KMU_DEST_W = `UP(`CLOG2(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_CORES)); - localparam KMU_DEST_LSB_SOCKET = 0; - localparam KMU_DEST_LSB_CLUSTER = `CLOG2(`VX_CFG_SOCKET_SIZE); - localparam KMU_DEST_LSB_DEVICE = KMU_DEST_LSB_CLUSTER + `CLOG2(NUM_SOCKETS); + localparam KMU_DEST_DIV_SOCKET = 1; + localparam KMU_DEST_DIV_CLUSTER = `VX_CFG_SOCKET_SIZE; + localparam KMU_DEST_DIV_DEVICE = KMU_DEST_DIV_CLUSTER * NUM_SOCKETS; typedef struct packed { logic [NCTA_WIDTH-1:0] cta_id; diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 3df30eaef2..3734c2771d 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -78,7 +78,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (`VX_CFG_SOCKET_SIZE), - .DEST_LSB (KMU_DEST_LSB_SOCKET), + .DEST_DIV (KMU_DEST_DIV_SOCKET), .OUT_BUF ((`VX_CFG_SOCKET_SIZE > 1) ? 3 : 0) ) kmu_arb ( .clk (clk), @@ -727,9 +727,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `UNUSED_PIN (busy) ); - // Route DXA lmem requests to per-core buses using core_local_id from tag. - // Tag value layout: {core_id[NC_BITS-1:0], engine_value[0]} - // core_local_id = core_id[CORE_LOCAL_BITS-1:0] + // Route DXA lmem requests to per-core buses by the core named in the tag. + // Tag value layout: {core_id[NC_BITS-1:0], engine_value[0]}, core_id being + // the cluster-local index, so the core within this socket is core_id % SOCKET_SIZE. localparam DXA_LMEM_CORE_LOCAL_BITS = `CLOG2(`VX_CFG_SOCKET_SIZE); VX_mem_bus_if #( .DATA_SIZE (DXA_LMEM_WORD_SIZE), @@ -740,7 +740,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; wire [`UP(DXA_LMEM_CORE_LOCAL_BITS)-1:0] dxa_lmem_core_sel; if (`VX_CFG_SOCKET_SIZE > 1) begin : g_dxa_lmem_sel - assign dxa_lmem_core_sel = dxa_lmem_bus_if[0].req_data.tag.value[1 +: DXA_LMEM_CORE_LOCAL_BITS]; + assign dxa_lmem_core_sel = DXA_LMEM_CORE_LOCAL_BITS'(dxa_lmem_bus_if[0].req_data.tag.value[1 +: NC_BITS] % `VX_CFG_SOCKET_SIZE); end else begin : g_dxa_lmem_sel assign dxa_lmem_core_sel = '0; end diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 59bebc3865..320693faa3 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -64,10 +64,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( input wire start, output wire busy ); - // Check clustering configuration - `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_CLUSTERS), ("NUM_CLUSTERS must be a power of 2")); - `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_CORES), ("NUM_CORES must be a power of 2")); - `STATIC_ASSERT(`IS_POW2(`VX_CFG_SOCKET_SIZE), ("SOCKET_SIZE must be a power of 2")); + `STATIC_ASSERT((`VX_CFG_NUM_CORES % `VX_CFG_SOCKET_SIZE) == 0, ("NUM_CORES must be a multiple of SOCKET_SIZE")); // A write-back LLC must be the only instance: replicated ones never make // their dirty lines visible to each other. @@ -279,7 +276,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (`VX_CFG_NUM_CLUSTERS), - .DEST_LSB (KMU_DEST_LSB_DEVICE), + .DEST_DIV (KMU_DEST_DIV_DEVICE), .OUT_BUF ((`VX_CFG_NUM_CLUSTERS > 1) ? 3 : 0) // register per-cluster kmu fan-out (SLR-crossing skid) ) kmu_arb ( .clk (clk), diff --git a/hw/rtl/core/VX_kmu_bus_arb.sv b/hw/rtl/core/VX_kmu_bus_arb.sv index 8ce6d4dcbb..55a87eb547 100644 --- a/hw/rtl/core/VX_kmu_bus_arb.sv +++ b/hw/rtl/core/VX_kmu_bus_arb.sv @@ -27,16 +27,16 @@ // lock would never release and would starve every other master forever. // // Fan-out picks the output by kind: -// FRAGMENT — routed by `dest`, this level's slice (DEST_LSB). Fragment work is -// bin->core affine; load-balancing it would break same-pixel blend -// order. +// FRAGMENT — routed by `dest`, this level's digit (dest / DEST_DIV) % NUM_OUTPUTS. +// Fragment work is bin->core affine; load-balancing it would break +// same-pixel blend order. // COMPUTE — round-robin over the ready outputs, which is what the plain // stream arbiter did before and what keeps CTAs spread across cores. module VX_kmu_bus_arb import VX_gpu_pkg::*; #( parameter NUM_INPUTS = 1, parameter NUM_OUTPUTS = 1, - parameter DEST_LSB = 0, // bit offset of this level's slice of `dest` + parameter DEST_DIV = 1, // cores below each output: this level's digit of `dest` parameter OUT_BUF = 0, parameter `STRING ARBITER = "R" ) ( @@ -167,7 +167,7 @@ module VX_kmu_bus_arb import VX_gpu_pkg::*; #( reg [SEL_O-1:0] lock_sel; reg [SEL_O-1:0] rr_ptr; - wire [SEL_O-1:0] dest_sel = SEL_O'(m_data[PW_DEST +: KMU_DEST_W] >> DEST_LSB); + wire [SEL_O-1:0] dest_sel = SEL_O'((32'(m_data[PW_DEST +: KMU_DEST_W]) / DEST_DIV) % NUM_OUTPUTS); // Round-robin over the ready outputs, starting at rr_ptr. Descending // scan so the nearest ready output (smallest step from rr_ptr) is the @@ -213,6 +213,7 @@ module VX_kmu_bus_arb import VX_gpu_pkg::*; #( end else begin : g_fanout_1 + `UNUSED_PARAM (DEST_DIV) assign out_valid[0] = m_valid; assign out_data[0] = m_data; assign m_ready = out_ready[0]; diff --git a/hw/rtl/cp/VX_cp_axil_regfile.sv b/hw/rtl/cp/VX_cp_axil_regfile.sv index 531e5e064d..96a0ca2fbc 100644 --- a/hw/rtl/cp/VX_cp_axil_regfile.sv +++ b/hw/rtl/cp/VX_cp_axil_regfile.sv @@ -16,6 +16,7 @@ // 0x004 CP_STATUS RO bit0=busy, bit1=error // 0x008 CP_DEV_CAPS RO [7:0]NUM_QUEUES | [15:8]RING_SIZE_LOG2_MAX // [23:16]AXI_TID_WIDTH | [28]DMA_LLC_CLIENT +// 0x00C GPU_DEV_CAPS_EXT RO device-config caps extension: [7:0]SOCKET_SIZE // 0x010 CP_CYCLE_LO RO free-running cycle counter low 32 bits // 0x014 CP_CYCLE_HI RO high 32 bits // 0x018 GPU_DEV_CAPS_LO RO device-config caps, low 32 bits @@ -130,7 +131,8 @@ module VX_cp_axil_regfile localparam int GPU_BANK_ADDR_W = `VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - `CLOG2(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS); - `STATIC_ASSERT(`VX_CFG_NUM_WARPS < 128 && `VX_CFG_NUM_CLUSTERS < 128 && GPU_CLUSTER_SIZE < 256, + `STATIC_ASSERT(`VX_CFG_NUM_WARPS < 128 && `VX_CFG_NUM_CLUSTERS < 128 && GPU_CLUSTER_SIZE < 256 + && `VX_CFG_SOCKET_SIZE < 256, ("GPU_DEV_CAPS exact-count fields overflow")) wire [63:0] gpu_dev_caps = { @@ -149,6 +151,11 @@ module VX_cp_axil_regfile 8'(`VX_ISA_IMPL_ID) }; + wire [31:0] gpu_dev_caps_ext = { + 24'd0, + 8'(`VX_CFG_SOCKET_SIZE) + }; + wire [63:0] gpu_isa_caps = { 32'(`VX_CFG_MISA_EXT), 2'(`CLOG2(`VX_CFG_XLEN)-4), @@ -212,6 +219,7 @@ module VX_cp_axil_regfile 8'(AXI_TID_W), 8'(RING_SIZE_LOG2_MAX), 8'(NUM_QUEUES)}; + 6'h0C: return gpu_dev_caps_ext; 6'h10: return r_cycle_count[31:0]; 6'h14: return r_cycle_count[63:32]; 6'h18: return gpu_dev_caps[31:0]; @@ -226,7 +234,7 @@ module VX_cp_axil_regfile // reports VM_ENABLED=0 in CP_DEV_CAPS and the runtime never writes // them. Reading zero is the honest answer; implement them for real if // an MMU is added. - 6'h0C, 6'h28, 6'h2C: return 32'd0; + 6'h28, 6'h2C: return 32'd0; // AFU host-port drain counters (debug, read-only). 6'h30: return dbg_host_w_counts; 6'h34: return dbg_host_r_counts; diff --git a/hw/rtl/dxa/VX_dxa_unit.sv b/hw/rtl/dxa/VX_dxa_unit.sv index cd3d8c8b13..385d56db7f 100644 --- a/hw/rtl/dxa/VX_dxa_unit.sv +++ b/hw/rtl/dxa/VX_dxa_unit.sv @@ -59,10 +59,8 @@ module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Build dxa_req payload dxa_req_data_t dxa_req_data_in; - assign dxa_req_data_in.core_id = NC_WIDTH'(core_id); - if (CORE_ID_WIDTH > NC_WIDTH) begin : g_core_id_hi_unused - `UNUSED_VAR (core_id[CORE_ID_WIDTH-1:NC_WIDTH]) - end + // cluster-local core index + assign dxa_req_data_in.core_id = NC_WIDTH'(core_id % `VX_CFG_NUM_CORES); assign dxa_req_data_in.uuid = execute_if.data.header.uuid; assign dxa_req_data_in.wid = execute_if.data.header.wid; assign dxa_req_data_in.smem_addr = lmem_rel_byte_addr[DXA_SMEM_ADDR_W-1:0]; diff --git a/hw/rtl/interfaces/VX_kmu_bus_if.sv b/hw/rtl/interfaces/VX_kmu_bus_if.sv index a1f1936898..4351a4976c 100644 --- a/hw/rtl/interfaces/VX_kmu_bus_if.sv +++ b/hw/rtl/interfaces/VX_kmu_bus_if.sv @@ -37,7 +37,7 @@ // `dest` is the global core index a FRAGMENT launch must land on. Fragment work // is bin->core affine — that affinity is what keeps same-pixel blend order // correct — so a fragment message cannot be load-balanced the way a compute CTA -// can. Each fan-out level consumes its own slice of `dest` (KMU_DEST_LSB_*); +// can. Each fan-out level consumes its own digit of `dest` (KMU_DEST_DIV_*); // compute launches ignore it and go to any ready output. interface VX_kmu_bus_if import VX_gpu_pkg::*; (); diff --git a/hw/unittest/cp_axil_regfile/main.cpp b/hw/unittest/cp_axil_regfile/main.cpp index 7574b9d6d9..920a9ee3af 100644 --- a/hw/unittest/cp_axil_regfile/main.cpp +++ b/hw/unittest/cp_axil_regfile/main.cpp @@ -9,8 +9,8 @@ // - CP_STATUS reflects the harness-driven cp_busy / cp_error inputs. // - CP_DEV_CAPS returns the configured (NUM_QUEUES, RING_SIZE_LOG2_MAX, // AXI_TID_WIDTH) fields. -// - GPU_DEV_CAPS / GPU_ISA_CAPS read back non-DECERR, decode to sane -// fields, and ignore writes (read-only). +// - GPU_DEV_CAPS / GPU_DEV_CAPS_EXT / GPU_ISA_CAPS read back non-DECERR, +// decode to sane fields, and ignore writes (read-only). // - CP_CYCLE counter actually advances per clock. // - Atomic Q_TAIL commit: writing Q_TAIL_LO alone does NOT advance // q_state.tail; writing Q_TAIL_HI atomically commits both halves. @@ -23,6 +23,7 @@ #include "vl_simulator.h" #include "VVX_cp_axil_regfile_top.h" +#include #include #include #include @@ -152,6 +153,7 @@ template static uint8_t q_profile_en(T* t) { return (uint8_t)read_ static constexpr uint16_t CP_CTRL = 0x000; static constexpr uint16_t CP_STATUS = 0x004; static constexpr uint16_t CP_DEV_CAPS = 0x008; +static constexpr uint16_t GPU_DEV_CAPS_EXT = 0x00C; static constexpr uint16_t CP_CYCLE_LO = 0x010; static constexpr uint16_t CP_CYCLE_HI = 0x014; static constexpr uint16_t GPU_DEV_CAPS_LO = 0x018; @@ -334,8 +336,12 @@ int main(int argc, char** argv) { "T11: caps must not be the DECERR sentinel"); uint64_t dev_caps = ((uint64_t)dhi << 32) | dlo; uint64_t isa_caps = ((uint64_t)ihi << 32) | ilo; - // dev_caps reserved bits [63:42] must be zero. - EXPECT((dev_caps >> 42) == 0, "T11: dev_caps reserved bits set"); + // Exact counts: warps [48:42], clusters [63:57], socket size in the extension word. + EXPECT(((dev_caps >> 42) & 0x7F) == VX_CFG_NUM_WARPS, "T11: dev_caps exact warps"); + EXPECT(((dev_caps >> 57) & 0x7F) == VX_CFG_NUM_CLUSTERS, "T11: dev_caps exact clusters"); + uint64_t de = axil_read(sim, tick, GPU_DEV_CAPS_EXT); + EXPECT((de >> 32) == 0, "T11: GPU_DEV_CAPS_EXT DECERR"); + EXPECT((uint32_t)de == VX_CFG_SOCKET_SIZE, "T11: GPU_DEV_CAPS_EXT socket size"); // isa_caps XLEN field [31:30] = clog2(XLEN)-4, so 1 (RV32) or 2 (RV64). uint32_t xlen_field = (uint32_t)((isa_caps >> 30) & 0x3); EXPECT(xlen_field == 1 || xlen_field == 2, "T11: isa_caps XLEN field"); diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 0f5d3a80f2..6bb250b452 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -15,7 +15,8 @@ namespace vortex { // ============================================================================ // Static GPU capability words // -// GPU_DEV_CAPS / GPU_ISA_CAPS register values exposed as read-only MMIO. +// GPU_DEV_CAPS / GPU_DEV_CAPS_EXT / GPU_ISA_CAPS register values exposed as +// read-only MMIO. // sw/runtime/common/vx_caps.h is the matching decoder. // ============================================================================ namespace { @@ -45,6 +46,10 @@ uint64_t gpu_dev_caps() { | ((uint64_t(cluster_size) & 0xFF) << 49) | ((uint64_t(VX_CFG_NUM_CLUSTERS) & 0x7F) << 57); } +uint32_t gpu_dev_caps_ext() { + static_assert(VX_CFG_SOCKET_SIZE < 256, "GPU_DEV_CAPS_EXT socket size overflows"); + return uint32_t(VX_CFG_SOCKET_SIZE); +} uint64_t gpu_isa_caps() { return (uint64_t(VX_CFG_MISA_EXT) << 32) | ((uint64_t(cp_clog2(VX_CFG_XLEN) - 4) & 0x3) << 30) @@ -78,7 +83,7 @@ void CommandProcessor::mmio_write(uint32_t off, uint32_t value) { case 0x028: satp_ = (satp_ & 0xFFFFFFFF00000000ULL) | uint64_t(value); return; case 0x02C: satp_ = (satp_ & 0x00000000FFFFFFFFULL) | (uint64_t(value) << 32); return; // STATUS / DEV_CAPS / CYCLE / GPU caps are RO; ignore writes. - case 0x004: case 0x008: case 0x010: case 0x014: + case 0x004: case 0x008: case 0x00C: case 0x010: case 0x014: case 0x018: case 0x01C: case 0x020: case 0x024: return; } // Queue 0 (offsets 0x100..0x12F) @@ -130,6 +135,7 @@ uint32_t CommandProcessor::mmio_read(uint32_t off) const { return llc_client | supports_qmd | supports_draw | vm_enabled | fault_report | (uint32_t(6) << 16) | (uint32_t(16) << 8) | uint32_t(1); } + case 0x00C: return gpu_dev_caps_ext(); case 0x010: return uint32_t(cycle_counter_ & 0xFFFFFFFF); case 0x014: return uint32_t(cycle_counter_ >> 32); case 0x018: return uint32_t(gpu_dev_caps() & 0xFFFFFFFF); diff --git a/sim/common/cmd_processor.h b/sim/common/cmd_processor.h index f249b89ca5..de5199979a 100644 --- a/sim/common/cmd_processor.h +++ b/sim/common/cmd_processor.h @@ -26,6 +26,7 @@ // 0x000 CP_CTRL bit0=enable_global, bit1=reset_all // 0x004 CP_STATUS bit0=busy, bit1=error // 0x008 CP_DEV_CAPS {AXI_TID_W:8 | RING_LOG2:8 | NUM_QUEUES:8} +// 0x00C GPU_DEV_CAPS_EXT device-config caps extension (RO) // 0x010 CP_CYCLE_LO // 0x014 CP_CYCLE_HI // 0x018 GPU_DEV_CAPS_LO device-config caps, low 32 bits (RO) diff --git a/sim/simx/constants.h b/sim/simx/constants.h index fabb7e3925..2142e7efd2 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -81,6 +81,8 @@ inline constexpr uint32_t DCACHE_CHANNELS = __UP((VX_CFG_NUM_LSU_LANES * XLENB) inline constexpr uint32_t VX_CFG_DCACHE_NUM_REQS = (VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS); inline constexpr uint32_t NUM_SOCKETS = __UP(VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE); +static_assert((VX_CFG_NUM_CORES % VX_CFG_SOCKET_SIZE) == 0, + "NUM_CORES must be a multiple of SOCKET_SIZE"); // Demand clients of the cluster L2 (sockets and gfx caches), excluding the // walker's PTE-fetch slot. diff --git a/sim/simx/rtu/rtu_types.h b/sim/simx/rtu/rtu_types.h index 8e84ba11cb..8a20d65e83 100644 --- a/sim/simx/rtu/rtu_types.h +++ b/sim/simx/rtu/rtu_types.h @@ -259,6 +259,8 @@ static_assert(VX_CFG_RTU_MERGE_DEPTH == 0, // Cores served by one RTU, and the per-core slot partition. constexpr uint32_t kRtuCoresPerRtu = VX_CFG_SOCKET_SIZE / VX_CFG_NUM_RTU_CORES; +static_assert(!VX_CFG_EXT_RTU_ENABLED || (kRtuCoresPerRtu * VX_CFG_NUM_RTU_CORES == VX_CFG_SOCKET_SIZE), + "SOCKET_SIZE must be a multiple of NUM_RTU_CORES"); constexpr uint32_t kRtuSlotsPerCore = VX_CFG_RTU_NUM_SLOTS / kRtuCoresPerRtu; static_assert(kRtuSlotsPerCore >= 1, "a core with no slot of its own could never issue a trace"); diff --git a/sw/runtime/common/caps.h b/sw/runtime/common/caps.h index ddb1def115..243fddc820 100644 --- a/sw/runtime/common/caps.h +++ b/sw/runtime/common/caps.h @@ -32,25 +32,34 @@ namespace vortex { // CP regfile offsets for the static capability words. // Pass to a backend's cp_mmio_read(), which applies any platform base offset. +static constexpr uint32_t CP_REG_GPU_DEV_CAPS_EXT = 0x00C; static constexpr uint32_t CP_REG_GPU_DEV_CAPS_LO = 0x018; static constexpr uint32_t CP_REG_GPU_DEV_CAPS_HI = 0x01C; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_LO = 0x020; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_HI = 0x024; -// Warps, sockets per cluster and clusters carry an exact count, so a machine -// that is not a power of two reports truly; the field is zero on devices -// built before it existed, which report only the 3-bit log2 field. -inline uint64_t exact_or_log2(uint64_t dev_caps, uint32_t exact_lsb, uint64_t exact_mask, uint32_t log2_lsb) { - uint64_t exact = (dev_caps >> exact_lsb) & exact_mask; +// Warps, sockets per cluster, clusters and the socket size carry an exact +// count, so a machine that is not a power of two reports truly; the field is +// zero on devices built before it existed, which report only the 3-bit log2 +// field of GPU_DEV_CAPS. +inline uint64_t exact_or_log2(uint64_t exact, uint64_t dev_caps, uint32_t log2_lsb) { return exact ? exact : (1ull << ((dev_caps >> log2_lsb) & 0x7)); } +inline uint64_t exact_or_log2(uint64_t dev_caps, uint32_t exact_lsb, uint64_t exact_mask, uint32_t log2_lsb) { + return exact_or_log2((dev_caps >> exact_lsb) & exact_mask, dev_caps, log2_lsb); +} + // Decode a VX_CAPS_* id from the two CP capability words. Returns true // and writes *out when the id is encoded in the words; returns false // for ids the caller must resolve itself (CACHE_LINE_SIZE, // GLOBAL_MEM_SIZE, CLOCK_RATE, PEAK_MEM_BW). -inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, +inline bool decode_caps(uint64_t dev_caps, uint32_t dev_caps_ext, uint64_t isa_caps, uint32_t caps_id, uint64_t* out) { + // A CP without the extension word answers zero, a DECERR sentinel or all + // ones; any reserved bit set means the word is absent. + const uint32_t caps_ext = (dev_caps_ext >> 8) ? 0 : dev_caps_ext; + const uint64_t socket_size = exact_or_log2(caps_ext & 0xFF, dev_caps, 14); switch (caps_id) { case VX_CAPS_VERSION: *out = (dev_caps >> 0) & 0xff; @@ -62,14 +71,13 @@ inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, *out = exact_or_log2(dev_caps, 42, 0x7F, 11); return true; case VX_CAPS_NUM_CORES: { - uint64_t socket_size = 1ull << ((dev_caps >> 14) & 0x7); uint64_t cluster_size = exact_or_log2(dev_caps, 49, 0xFF, 17); uint64_t num_clusters = exact_or_log2(dev_caps, 57, 0x7F, 20); *out = num_clusters * cluster_size * socket_size; return true; } case VX_CAPS_SOCKET_SIZE: - *out = 1ull << ((dev_caps >> 14) & 0x7); + *out = socket_size; return true; case VX_CAPS_NUM_CLUSTERS: *out = exact_or_log2(dev_caps, 57, 0x7F, 20); @@ -94,14 +102,15 @@ inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, } } -// Read GPU_DEV_CAPS / GPU_ISA_CAPS from the CP regfile through a -// backend's cp_mmio_read primitive. `Reader` is any callable matching -// int(uint32_t off, uint32_t* out) — returns non-zero on failure. +// Read GPU_DEV_CAPS / GPU_DEV_CAPS_EXT / GPU_ISA_CAPS from the CP regfile +// through a backend's cp_mmio_read primitive. `Reader` is any callable +// matching int(uint32_t off, uint32_t* out) — returns non-zero on failure. // Returns 0 on success. template inline int load_caps(Reader&& cp_mmio_read, - uint64_t* dev_caps, uint64_t* isa_caps) { + uint64_t* dev_caps, uint32_t* dev_caps_ext, uint64_t* isa_caps) { uint32_t lo = 0, hi = 0; + if (cp_mmio_read(CP_REG_GPU_DEV_CAPS_EXT, dev_caps_ext)) return -1; if (cp_mmio_read(CP_REG_GPU_DEV_CAPS_LO, &lo)) return -1; if (cp_mmio_read(CP_REG_GPU_DEV_CAPS_HI, &hi)) return -1; *dev_caps = uint64_t(lo) | (uint64_t(hi) << 32); diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index a32f18e9c2..39009c1927 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -1237,7 +1237,7 @@ vx_result_t Device::memory_info(uint64_t* out_free, uint64_t* out_used) { vx_result_t Device::query_caps(uint32_t caps_id, uint64_t* out_value) { if (!out_value) return VX_ERR_INVALID_VALUE; - // The two static caps words are read once from the CP regfile caps + // The static caps words are read once from the CP regfile caps // window; serialize the load against concurrent cp_reg_* traffic. if (!caps_loaded_) { std::lock_guard g(cp_mu_); @@ -1245,12 +1245,12 @@ vx_result_t Device::query_caps(uint32_t caps_id, uint64_t* out_value) { auto rd = [this](uint32_t off, uint32_t* v) -> int { return platform_->cp_reg_read(off, v) == VX_SUCCESS ? 0 : -1; }; - if (vortex::load_caps(rd, &dev_caps_, &isa_caps_) != 0) + if (vortex::load_caps(rd, &dev_caps_, &dev_caps_ext_, &isa_caps_) != 0) return VX_ERR_DEVICE_LOST; caps_loaded_ = true; } } - if (vortex::decode_caps(dev_caps_, isa_caps_, caps_id, out_value)) + if (vortex::decode_caps(dev_caps_, dev_caps_ext_, isa_caps_, caps_id, out_value)) return VX_SUCCESS; // Caps not encoded in the CP words — platform/runtime constants. switch (caps_id) { diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index 724449d825..f268d9a58d 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -435,8 +435,9 @@ class Device : public RefCounted { std::map host_mems_; // Lazily-loaded device/ISA caps words (read once from the CP regfile). - uint64_t dev_caps_ = 0; - uint64_t isa_caps_ = 0; + uint64_t dev_caps_ = 0; + uint32_t dev_caps_ext_ = 0; + uint64_t isa_caps_ = 0; bool caps_loaded_ = false; std::mutex mu_; From 2183b6a0677bb499c6f2eee4819d136a11c76049 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 24 Oct 2026 04:18:44 -0700 Subject: [PATCH 096/336] ci: re-record perf baselines for the MLP Phase 1 timing Regenerated with pytest ci -m perf_gate --update-baselines on fc0a35bb4, where the Phase 1 port moved cycle counts. perf_gate passes exactly against them (49/49) on c5ed76a73. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 68 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 40 +++++++++--------- ci/baselines/perf/graphics.json | 34 ++++++++-------- ci/baselines/perf/raytracing.json | 8 ++-- ci/baselines/perf/tensor.json | 20 ++++----- ci/baselines/perf/tensor_mx.json | 4 +- ci/baselines/perf/tensor_sp.json | 4 +- ci/baselines/perf/tensor_wg.json | 12 +++--- 8 files changed, 95 insertions(+), 95 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 01cd65540c..c9c7dc8a7f 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 564053, - "instrs": 137486 + "cycles": 11185799, + "instrs": 2241925 }, "app": "raycast", "args": "", - "config_hash": "7a2c92e634460e90", + "config_hash": "ef7e97261abf3c7e", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 1498642, - "instrs": 259228 + "cycles": 21106013, + "instrs": 5664693 }, "app": "raycast", "args": "", - "config_hash": "2532cbe6c87ca233", + "config_hash": "4a346b970c5d5f4e", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 720350, + "cycles": 632919, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "6106872241e50c60", + "config_hash": "12eb01a0cf320bec", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2227069, + "cycles": 2251795, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "ecbd8e8a804188dd", + "config_hash": "a5cc1a944ac4fc92", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1782470, + "cycles": 1665973, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "6dcefd7e66ed0cc0", + "config_hash": "dcf77cabb080bf39", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 5939731, + "cycles": 5943507, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "74a554f940a83c4a", + "config_hash": "9b7d85b091d9be08", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1226398, + "cycles": 1225395, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "efc73b5e23a33b8f", + "config_hash": "1181ce2dd247a573", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4219466, + "cycles": 4218769, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "0f2e76cd4d21f998", + "config_hash": "b8d3bbe9aaa587a9", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 460344, + "cycles": 408992, "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "bc1d0a1801524d27", + "config_hash": "a2e93b1416c1222b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 872926, + "cycles": 883260, "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "7fbf266181d2ec8b", + "config_hash": "212b79b7b9e0df8d", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1271056, + "cycles": 1270578, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "c53ddc9ba2cd96bd", + "config_hash": "4a980cd3436e0491", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4860309, + "cycles": 4860082, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "6dea102e1b20464c", + "config_hash": "3d84ea77567e034e", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2272549, + "cycles": 2283980, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "542851d41daf9431", + "config_hash": "5b78acb9a50222ff", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4180969, + "cycles": 4185784, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "fa607fa9e1280813", + "config_hash": "dc41c57ce7819a59", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 99953, + "cycles": 95240, "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "0f4e3087ce474e4b", + "config_hash": "652eaf7c4099617d", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 367765, + "cycles": 368717, "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "0169f39a14561223", + "config_hash": "c602c9c37a145297", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index a331465e29..ec5e66786f 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -6,18 +6,18 @@ }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "2be8c5960886f23f", + "config_hash": "28dc0787284f20c8", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 1992, + "cycles": 1953, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "37f76aae77fd1afd", + "config_hash": "912948e13624b9a4", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,95 +28,95 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "c3b3447f163ec6c3", + "config_hash": "277ced0b15aed8d9", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14023305, + "cycles": 14016579, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "62067f11f3ef5af4", + "config_hash": "f247e0e4b7829023", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14063502, + "cycles": 14051140, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "98c58db8047bceb9", + "config_hash": "3660e26ec6c4ba10", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3921184, + "cycles": 3947991, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "845bebf9fbc108b1", + "config_hash": "55af78d744fa9e46", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14352313, + "cycles": 14352123, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "8cf4b0e4e0c7d144", + "config_hash": "8c7ee15d08de3277", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 79131, + "cycles": 78840, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "d3c9c80b16cdf2c2", + "config_hash": "d27df3466a8eed79", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 360010, + "cycles": 358897, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "38993044a82f32e0", + "config_hash": "776a839834d35db8", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 31160, + "cycles": 29487, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "699f26ecbecc6bb2", + "config_hash": "56519748fbd3c39b", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 55512, + "cycles": 48573, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "8e607400e57e51f5", + "config_hash": "f1664e1fcc03f8a6", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 489e16a09b..4d13a12aaf 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,45 +1,45 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 529868, + "cycles": 529816, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "0777067f98f2730f", + "config_hash": "9b91ab9ac3aa5103", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 143542, + "cycles": 141632, "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "550e1e3d5ffe23d0", + "config_hash": "1f2de39a01c16b67", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 529389, + "cycles": 529359, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "5cc3a870f65d86fa", + "config_hash": "73f771f2643f6527", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 189356, + "cycles": 189979, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "f74350c24c97b1f5", + "config_hash": "132840fd850713c9", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,51 +50,51 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "8a9749c1c0d8264d", + "config_hash": "d8e520e04c9979f4", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14626, + "cycles": 14487, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "967474845abde936", + "config_hash": "e2f119110f028280", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55565, + "cycles": 55553, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "40f6d7ca7755b6ca", + "config_hash": "5109e566c9637f1b", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 35297, + "cycles": 35254, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "e44189b53e136920", + "config_hash": "3876028c632f011b", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 136487, + "cycles": 136492, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "f0cdf08354c9ae88", + "config_hash": "fe367b2efd48140d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index e0ee3a7da4..71ef9f1d16 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -6,18 +6,18 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "4f78e65dfd2497cb", + "config_hash": "c0bd39f0f980f01d", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53443, + "cycles": 52906, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "c0af18c64430f778", + "config_hash": "b899d2a47bcc51ae", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "e394fceca77fe6f1", + "config_hash": "47b882de7074c1db", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 975aa8d248..8ce4e9b7b0 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 197465, + "cycles": 197355, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "9cdc8bb95c856a38", + "config_hash": "b80548e5fa6b84ae", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 36723, + "cycles": 36479, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "153ac3d8001d2be6", + "config_hash": "2935e28e509b5699", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221581, + "cycles": 222420, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "21c6d64b10dab7cb", + "config_hash": "d6ccf2e6532a7a88", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 256743, + "cycles": 254124, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "04953ed130a8c1a9", + "config_hash": "9c339e61cd76a65c", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 590798, + "cycles": 593588, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "26d22a59f411cfbe", + "config_hash": "c27f67158e4b632a", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index a49403e311..3b8f55c440 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 76393, + "cycles": 76410, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "7c8765889acb4f32", + "config_hash": "63975cb90eb9477f", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 3ee3f24144..bf7540cbf3 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 183094, + "cycles": 181992, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "6e523b9d0f6fe4ee", + "config_hash": "3bb64bb8ac3c1a95", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index b2af51b8f2..2047a4ef2a 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 158181, + "cycles": 158197, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "c65e3b69950519cb", + "config_hash": "6e007939f3dbec76", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 834873, + "cycles": 834916, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "0d41f1f51f46d61d", + "config_hash": "76bcef5705d497dc", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 400023, + "cycles": 406430, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "b4944cbd79928ac8", + "config_hash": "d983c017c55aa5a7", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From afd43315053e226f91c21c6af98c187a955c8465 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 24 Oct 2026 06:19:42 -0700 Subject: [PATCH 097/336] afu: one AXI port per GPU memory port, W forwarded with its AW The AFU memory port count was a per-board constant (PLATFORM_MERGED_ MEMORY_INTERFACE), so it did not follow the GPU configuration and the U280 left channels unmapped. The port count now follows the GPU's memory ports, rounded up to a power of two; the build reads it with afu_mem_ports.sh and generates the HBM/DDR connectivity from it. VX_mem_to_axi without INTERLEAVE truncated the bank index away, sending every bank to the first 1/N of memory; bank i now serves the i-th contiguous 1/N, and xrtsim aborts on an address outside its port's range. Interleaved mode is unchanged. VX_mm_axi_arb (MULTI_OUT) held a W behind the AW grant register and could deadlock with a slave that takes W before AW. W now follows AW order through a 4-entry queue and moves in the AW cycle, which doubles single-beat write throughput. VX_afu_wr_gate and the drain counter keep reset quiesce from splitting a burst, including one whose W leads its AW. xrtsim xrt CI 23/23; afu_reset_seq, mm_axi_arb (old arbiter fails 5/13), cp_llc_port, cp_axi_path and cp_core pass. rtlsim and SimX do not build the changed arbiter mode, so perf_gate and model_parity are unaffected. Not yet run: Vivado timing of the combinational W select, and the U200 DDR address base on a board. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/unittest.yaml | 8 + docs/designs/fpga_afu_shell.md | 35 +- hw/rtl/afu/aved/vortex_afu.v | 23 +- hw/rtl/afu/common/VX_afu_axi_drain.sv | 23 +- hw/rtl/afu/common/VX_afu_wr_gate.sv | 98 +++++ hw/rtl/afu/common/VX_afu_wrap.sv | 196 +++++---- hw/rtl/afu/common/vortex_afu.vh | 63 +++ hw/rtl/afu/xrt/vortex_afu.v | 23 +- hw/rtl/libs/VX_mem_to_axi.sv | 7 +- hw/rtl/libs/VX_mm_axi_arb.sv | 82 ++-- hw/scripts/afu_mem_ports.sh | 89 ++++ hw/syn/xilinx/aved/Makefile | 23 +- hw/syn/xilinx/aved/platforms.mk | 4 +- hw/syn/xilinx/xrt/Makefile | 31 +- hw/syn/xilinx/xrt/package_kernel.tcl | 22 +- hw/syn/xilinx/xrt/platforms.mk | 29 +- hw/unittest/Makefile | 3 + .../afu_reset_seq/VX_afu_reset_seq_top.sv | 29 ++ hw/unittest/afu_reset_seq/main.cpp | 152 ++++++- hw/unittest/mm_axi_arb/Makefile | 23 + hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv | 159 +++++++ hw/unittest/mm_axi_arb/main.cpp | 415 ++++++++++++++++++ sim/avedsim/Makefile | 11 + sim/avedsim/vortex_afu_shim.sv | 6 +- sim/xrtsim/Makefile | 12 + sim/xrtsim/vortex_afu_shim.sv | 9 +- sim/xrtsim/xrt_sim.cpp | 254 ++++++----- 27 files changed, 1525 insertions(+), 304 deletions(-) create mode 100644 hw/rtl/afu/common/VX_afu_wr_gate.sv create mode 100755 hw/scripts/afu_mem_ports.sh create mode 100644 hw/unittest/mm_axi_arb/Makefile create mode 100644 hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv create mode 100644 hw/unittest/mm_axi_arb/main.cpp diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 2f3f33d952..8765f57fac 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -46,6 +46,14 @@ tests: via: script run: "make -C hw/unittest/stream_arb run" touches: [hw/rtl/libs/VX_stream_arb.sv, hw/unittest/stream_arb] + # The ID-routed AXI arbiter shared by the AFU and the command processor: + # write data must reach the slave in address order whichever channel a + # master offers first, and a slave that takes data before the address must + # not deadlock it. xrtsim's always-ready slave exercises neither. + - id: hw-mm-axi-arb + via: script + run: "make -C hw/unittest/mm_axi_arb run" + touches: [hw/rtl/libs/VX_mm_axi_arb.sv, hw/unittest/mm_axi_arb] # Texel blend (VX_tex_lerp, both forms the sampler uses) against the C model # the SW sampler and SimX share, and the exact Vulkan f/256 blend, over every # input. diff --git a/docs/designs/fpga_afu_shell.md b/docs/designs/fpga_afu_shell.md index 413a108129..c4c9760d71 100644 --- a/docs/designs/fpga_afu_shell.md +++ b/docs/designs/fpga_afu_shell.md @@ -26,7 +26,8 @@ DMA-command engine have been removed. | [`VX_afu_wrap.sv`](../../hw/rtl/afu/common/VX_afu_wrap.sv) | The real shell (~1,200 LOC): AXI-Lite bit-12 demux, `VX_cp_core`, `m_axi_host`, the Vortex instance and the device-memory path shared with the CP. | | [`VX_afu_ctrl.sv`](../../hw/rtl/afu/xrt/VX_afu_ctrl.sv) | Slimmed AXI-Lite slave (~322 LOC): `ap_ctrl` stub at 0x00 + a SCOPE serial register pair + SCOPE watchdog. | | [`VX_afu_axil_demux.sv`](../../hw/rtl/afu/common/VX_afu_axil_demux.sv) | AXI-Lite demux splitting the control space on `addr[12]`, one outstanding transaction per direction. | -| [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing. | +| [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing, including a write burst under way ahead of its AW. | +| [`VX_afu_req_gate.sv`](../../hw/rtl/afu/common/VX_afu_req_gate.sv) / [`VX_afu_wr_gate.sv`](../../hw/rtl/afu/common/VX_afu_wr_gate.sv) | Withhold new reads / whole new writes while the reset sequencer quiesces, never withdrawing an offer. | | [`VX_afu_reset_seq.sv`](../../hw/rtl/afu/common/VX_afu_reset_seq.sv) | Quiesce-before-reset sequencer for the soft reset; refuses rather than resetting a master that will not drain. | - **Control.** Host AXI-Lite `addr[12]` splits the slave: `addr[12]=0` → @@ -39,19 +40,43 @@ DMA-command engine have been removed. with its address, and routing W by a register that only updates at the AW handshake sent AW and W to different slaves and deadlocked the interface. Covered by `hw/unittest/afu_axil_demux`. -- **Memory.** Two structures, selected by the number of memory ports. - *One port* (the merged platforms): `Vortex_axi` and the CP's `axi_dev` +- **Memory.** The AFU exposes one AXI master per GPU memory port: + `AFU_MEM_PORTS_OF(VX_MEM_PORTS)` ([`vortex_afu.vh`](../../hw/rtl/afu/common/vortex_afu.vh)), + the last-level cache's memory ports (already capped by the platform's + channel count, `PLATFORM_MEMORY_NUM_BANKS`) rounded up to a power of two. + The kernel top must spell its ports out, so every flow resolves the count + to the literal `AFU_MEM_PORTS` by elaborating a probe against its own flags + ([`afu_mem_ports.sh`](../../hw/scripts/afu_mem_ports.sh)); `VX_afu_wrap` + asserts the two agree, `package_kernel.tcl` checks the packaged kernel's + master count, and the XRT Makefile generates the `--connectivity.sp` lines + from the same value. Lines interleave across the N ports + (`PLATFORM_MEMORY_INTERLEAVE`; without it each port takes a contiguous range), and port *i* owns the *i*-th contiguous + 1/N of device memory: its AXI address is the line index divided by N + within that share. The platform maps the share to its own contiguous group + of `PLATFORM_MEMORY_CHANNELS / N` channels (`HBM[2i:2i+1]` for 16 ports on + a U55C), so ports never contend for a channel. `PLATFORM_MEMORY_OFFSET` is + applied once, at every port. + Two structures, selected by N. + *One port*: `Vortex_axi` and the CP's `axi_dev` share the port through a 2:1 `VX_mm_axi_arb` in its ID-routed multi-outstanding mode (`MULTI_OUT=1`: the top ID bit names the source, Vortex gets the remaining ID bits), and CP bursts reach the platform as - bursts. *Several ports*: Vortex's memory ports and `axi_dev` (bridged by + bursts. Writes keep AW order on the shared W channel, and a write's data + goes with its address, so single-beat writes issue one per cycle. + *Several ports*: Vortex's memory ports and `axi_dev` (bridged by `VX_membus_from_axi`) are the inputs of one bank adapter, `VX_mem_to_axi`, so the CP is merged **upstream** of bank selection and reaches every bank. In both, every port carries as many reads and writes in flight as its masters issue; one outstanding transaction per direction would cap a port at one line per DRAM round trip. CP `axi_host` drives a **dedicated `m_axi_host` AXI master** for the command ring + host DMA. - `PLATFORM_MEMORY_OFFSET` is applied per bank, at the port. + While the reset sequencer quiesces, `VX_afu_req_gate` holds new reads and + `VX_afu_wr_gate` holds new writes on both channels together: AXI4 lets a + master present write data before or with its address, so gating AW alone + would strand data the slave can never pair. An address or data offer + already on the bus obliges the other channel, since a slave may wait for + one before accepting the other. xrtsim aborts on a request outside the + issuing port's share of memory. - **Interrupt.** The AFU `interrupt` pin is driven from `cp_interrupt` ([`:335`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv#L335)). diff --git a/hw/rtl/afu/aved/vortex_afu.v b/hw/rtl/afu/aved/vortex_afu.v index 5335b376bb..6490e304ea 100644 --- a/hw/rtl/afu/aved/vortex_afu.v +++ b/hw/rtl/afu/aved/vortex_afu.v @@ -23,22 +23,14 @@ module vortex_afu #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - parameter C_M_AXI_MEM_NUM_BANKS = 1 -`else - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS -`endif + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, - // AXI4 master interface -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), -`endif + // AXI4 master interface, one per GPU memory port + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -69,6 +61,9 @@ module vortex_afu #( output wire interrupt ); + `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CONTROL_ADDR_WIDTH), .C_S_AXI_CTRL_DATA_WIDTH (C_S_AXI_CONTROL_DATA_WIDTH), @@ -79,11 +74,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (~ap_rst_n), - `ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, AXI_MEM_ARGS, MP_COMMA), - `else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), - `endif + `AXI_MEM_ARRAY_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_control_awvalid), .s_axi_ctrl_awready (s_axi_control_awready), diff --git a/hw/rtl/afu/common/VX_afu_axi_drain.sv b/hw/rtl/afu/common/VX_afu_axi_drain.sv index 95e4c05459..b023f7fd0e 100644 --- a/hw/rtl/afu/common/VX_afu_axi_drain.sv +++ b/hw/rtl/afu/common/VX_afu_axi_drain.sv @@ -27,6 +27,10 @@ // directions, so it stays right when AXI4 allows write data to be presented // before its address (which would drive an up/down counter negative). // +// A write burst whose first beats have gone out is in flight too, even when +// its AW has not: the counters only see last beats, so the burst in progress +// is tracked separately. +// // COUNT_WIDTH must exceed the port's maximum outstanding transactions; the // comparison is exact only while fewer than 2**COUNT_WIDTH are in flight. // ============================================================================ @@ -39,6 +43,7 @@ module VX_afu_axi_drain #( // Handshakes observed on the shell side of the request gate. input wire aw_fire, // awvalid && awready + input wire w_fire, // wvalid && wready input wire w_fire_last, // wvalid && wready && wlast input wire b_fire, // bvalid && bready input wire ar_fire, // arvalid && arready @@ -60,15 +65,20 @@ module VX_afu_axi_drain #( reg [COUNT_WIDTH-1:0] b_count; reg [COUNT_WIDTH-1:0] ar_count; reg [COUNT_WIDTH-1:0] r_count; + reg w_in_burst; always @(posedge clk) begin if (reset) begin - aw_count <= '0; - w_count <= '0; - b_count <= '0; - ar_count <= '0; - r_count <= '0; + aw_count <= '0; + w_count <= '0; + b_count <= '0; + ar_count <= '0; + r_count <= '0; + w_in_burst <= 1'b0; end else begin + if (w_fire) begin + w_in_burst <= ~w_fire_last; + end if (aw_fire) begin aw_count <= aw_count + 1'b1; end @@ -94,7 +104,8 @@ module VX_afu_axi_drain #( assign dbg_ar_count = ar_count; assign dbg_r_count = r_count; - assign idle = (aw_count == w_count) + assign idle = ~w_in_burst + && (aw_count == w_count) && (aw_count == b_count) && (ar_count == r_count); diff --git a/hw/rtl/afu/common/VX_afu_wr_gate.sv b/hw/rtl/afu/common/VX_afu_wr_gate.sv new file mode 100644 index 0000000000..821f0526f8 --- /dev/null +++ b/hw/rtl/afu/common/VX_afu_wr_gate.sv @@ -0,0 +1,98 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "vortex_afu.vh" + +// ============================================================================ +// VX_afu_wr_gate — withhold new AXI writes without splitting one. +// +// The write-channel counterpart of VX_afu_req_gate. Gating AW alone is not +// enough: AXI4 lets a master present a write's data before or with its +// address, so a master that does would push W beats of a write whose AW the +// gate then withholds. The slave is left holding data no address will ever +// claim, the drain counters never balance, and the reset sequencer refuses. +// +// A write is the k-th AW paired with the k-th W burst. While stopped, the gate +// lets through only what completes a write already begun on the shell side: +// the AW owed to a W burst that has started or is being offered, the W burst +// owed to an AW that has been accepted or is being offered, the rest of a +// burst in progress, and any offer already presented (VALID is never +// withdrawn, AXI4 §A3.2.1). A slave may hold AWREADY until it sees the data, +// or WREADY until it sees the address, so an offer alone obliges the other +// channel. Everything else waits, so the shell sees whole writes only and the +// drain reaches zero. +// +// LEAD_WIDTH must exceed the number of AWs a master may run ahead of its data. +// ============================================================================ + +module VX_afu_wr_gate #( + parameter LEAD_WIDTH = 16 +) ( + input wire clk, + input wire reset, + + input wire stop_req, // withhold new writes + + input wire in_awvalid, // from the AFU-internal master + output wire in_awready, + output wire out_awvalid, // to the shell + input wire out_awready, + + input wire in_wvalid, + input wire in_wlast, + output wire in_wready, + output wire out_wvalid, + input wire out_wready +); + reg blocked; + reg aw_offered; + reg w_offered; + reg w_in_burst; + reg [LEAD_WIDTH-1:0] aw_lead; // AWs accepted minus W bursts started + + wire aw_fire = out_awvalid && out_awready; + wire w_fire = out_wvalid && out_wready; + wire w_start = w_fire && ~w_in_burst; + + always @(posedge clk) begin + if (reset) begin + blocked <= 1'b0; + aw_offered <= 1'b0; + w_offered <= 1'b0; + w_in_burst <= 1'b0; + aw_lead <= '0; + end else begin + blocked <= stop_req; + aw_offered <= out_awvalid && ~out_awready; + w_offered <= out_wvalid && ~out_wready; + if (w_fire) begin + w_in_burst <= ~in_wlast; + end + aw_lead <= aw_lead + LEAD_WIDTH'(aw_fire) - LEAD_WIDTH'(w_start); + end + end + + wire lead_neg = aw_lead[LEAD_WIDTH-1]; + wire lead_zero = (aw_lead == '0); + wire aw_owed = lead_neg || (lead_zero && w_offered && ~w_in_burst); + wire w_owed = (~lead_neg && ~lead_zero) || (lead_zero && aw_offered); + + wire aw_pass = ~blocked || aw_offered || aw_owed; + wire w_pass = ~blocked || w_offered || w_in_burst || w_owed; + + assign out_awvalid = in_awvalid && aw_pass; + assign in_awready = out_awready && aw_pass; + assign out_wvalid = in_wvalid && w_pass; + assign in_wready = out_wready && w_pass; + +endmodule diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index f19ed26bb8..2d3940170a 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -28,7 +28,8 @@ // register at host-offset 0x000. // // Data plane: -// * Memory banks 0..N-1 ride the platform AXI4 master ports. +// * One platform AXI4 master per GPU memory port (AFU_MEM_PORTS_OF); +// port i owns the i-th contiguous 1/N of device memory. // * The CP's device master shares them with Vortex: at the AXI port when // there is one, upstream of bank selection when there are several. // @@ -41,22 +42,43 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = `VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8, parameter C_M_AXI_MEM_ADDR_WIDTH = 64, -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - parameter C_M_AXI_MEM_NUM_BANKS = 1 -`else - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS -`endif + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS_OF(VX_MEM_PORTS) ) ( // System signals input wire clk, input wire reset, - // AXI4 master interface -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), -`endif + // AXI4 memory masters + output wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS], + input wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS], + output wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS], + + output wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS], + input wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS], + output wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS], + + input wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS], + output wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS], + input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS], + input wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS], + + output wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS], + input wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS], + output wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS], + + input wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS], + output wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS], + input wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS], + input wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS], + input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS], + input wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS], + // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, // AXI4-Lite slave interface @@ -86,42 +108,10 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( ); localparam M_AXI_MEM_ADDR_WIDTH = `VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH; - wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS]; - - // convert memory interface to array -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, AXI_MEM_TO_ARRAY, MP_SEMI); -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_TO_ARRAY, MP_SEMI); -`endif + // The platform top declares one AXI master per GPU memory port; a count + // that disagrees with the GPU would leave ports idle or requests unrouted. + `STATIC_ASSERT(C_M_AXI_MEM_NUM_BANKS == `AFU_MEM_PORTS_OF(VX_MEM_PORTS), + ("AXI memory port count %0d does not match the GPU's %0d", C_M_AXI_MEM_NUM_BANKS, `AFU_MEM_PORTS_OF(VX_MEM_PORTS))) reg [`VX_CFG_RESET_DELAY-1:0] vx_reset_shift_r; wire vx_reset; @@ -362,26 +352,28 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // XRT pins m_axi_host to HOST[0]; host addresses pass straight through // (no PLATFORM_MEMORY_OFFSET — that offset is device-memory specific). // Gated like the bank ports: no new host-side request leaves the AFU while - // the reset sequencer is draining. The gate is a VX_afu_req_gate rather - // than a plain AND so an offer already made to the shell is never - // withdrawn -- see the header of that module. - VX_afu_req_gate host_aw_gate ( - .clk (clk), - .reset (reset), - .stop_req (rst_stop_req), - .in_valid (cp_axi_host.awvalid), - .in_ready (cp_axi_host.awready), - .out_valid (m_axi_host_awvalid), - .out_ready (m_axi_host_awready) + // the reset sequencer is draining. The gates never withdraw an offer + // already made to the shell and never split a write -- see their headers. + VX_afu_wr_gate host_wr_gate ( + .clk (clk), + .reset (reset), + .stop_req (rst_stop_req), + .in_awvalid (cp_axi_host.awvalid), + .in_awready (cp_axi_host.awready), + .out_awvalid (m_axi_host_awvalid), + .out_awready (m_axi_host_awready), + .in_wvalid (cp_axi_host.wvalid), + .in_wlast (cp_axi_host.wlast), + .in_wready (cp_axi_host.wready), + .out_wvalid (m_axi_host_wvalid), + .out_wready (m_axi_host_wready) ); assign m_axi_host_awaddr = cp_axi_host.awaddr; assign m_axi_host_awid = {{(C_M_AXI_MEM_ID_WIDTH-`VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_host.awid}; assign m_axi_host_awlen = cp_axi_host.awlen; - assign m_axi_host_wvalid = cp_axi_host.wvalid; assign m_axi_host_wdata = cp_axi_host.wdata; assign m_axi_host_wstrb = cp_axi_host.wstrb; assign m_axi_host_wlast = cp_axi_host.wlast; - assign cp_axi_host.wready = m_axi_host_wready; assign cp_axi_host.bvalid = m_axi_host_bvalid; assign cp_axi_host.bid = m_axi_host_bid[`VX_CP_AXI_TID_WIDTH-1:0]; assign cp_axi_host.bresp = m_axi_host_bresp; @@ -442,18 +434,39 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_u [C_M_AXI_MEM_NUM_BANKS]; wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_u [C_M_AXI_MEM_NUM_BANKS]; - // Per-bank XRT BO base offset. Each m_axi_mem_ port targets a different - // xrt::bo (one per DDR/HBM channel) that XRT places at a different virtual - // base address. PLATFORM_MEMORY_OFFSET applies the same synthesis-time - // offset to every bank. - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] platform_memory_offsets [C_M_AXI_MEM_NUM_BANKS]; - for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_pmo - assign platform_memory_offsets[i] = C_M_AXI_MEM_ADDR_WIDTH'(`PLATFORM_MEMORY_OFFSET); - end + // Port i owns the i-th contiguous 1/N of the device address space, which + // the platform maps to that port's own group of memory channels, so no + // port reaches across another's. Lines interleave across the ports, so + // within the group a line sits at its index divided by N: drop the port + // field from the line address and put the port number on top. + // Without interleaving the bank adapter already sends each port only its + // own share. PLATFORM_MEMORY_OFFSET then rebases every port by the same + // aperture. + `STATIC_ASSERT(`IS_POW2(C_M_AXI_MEM_NUM_BANKS), + ("AXI memory port count %0d is not a power of two", C_M_AXI_MEM_NUM_BANKS)) + localparam MEM_PORT_BITS = `CLOG2(C_M_AXI_MEM_NUM_BANKS); + localparam MEM_LINE_BITS = `CLOG2(C_M_AXI_MEM_DATA_WIDTH / 8); + localparam MEM_GROUP_BITS = M_AXI_MEM_ADDR_WIDTH - MEM_PORT_BITS - MEM_LINE_BITS; + + wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_g [C_M_AXI_MEM_NUM_BANKS]; + wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_g [C_M_AXI_MEM_NUM_BANKS]; for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_addressing - assign m_axi_mem_awaddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_awaddr_u[i]) + platform_memory_offsets[i]; - assign m_axi_mem_araddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_araddr_u[i]) + platform_memory_offsets[i]; + if (C_M_AXI_MEM_NUM_BANKS > 1 && `VX_CFG_PLATFORM_MEMORY_INTERLEAVE != 0) begin : g_group + assign m_axi_mem_awaddr_g[i] = {MEM_PORT_BITS'(i), + m_axi_mem_awaddr_u[i][MEM_LINE_BITS + MEM_PORT_BITS +: MEM_GROUP_BITS], + m_axi_mem_awaddr_u[i][MEM_LINE_BITS-1:0]}; + assign m_axi_mem_araddr_g[i] = {MEM_PORT_BITS'(i), + m_axi_mem_araddr_u[i][MEM_LINE_BITS + MEM_PORT_BITS +: MEM_GROUP_BITS], + m_axi_mem_araddr_u[i][MEM_LINE_BITS-1:0]}; + `UNUSED_VAR (m_axi_mem_awaddr_u[i][MEM_LINE_BITS +: MEM_PORT_BITS]) + `UNUSED_VAR (m_axi_mem_araddr_u[i][MEM_LINE_BITS +: MEM_PORT_BITS]) + end else begin : g_linear + assign m_axi_mem_awaddr_g[i] = m_axi_mem_awaddr_u[i]; + assign m_axi_mem_araddr_g[i] = m_axi_mem_araddr_u[i]; + end + assign m_axi_mem_awaddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_awaddr_g[i]) + C_M_AXI_MEM_ADDR_WIDTH'(`PLATFORM_MEMORY_OFFSET); + assign m_axi_mem_araddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_araddr_g[i]) + C_M_AXI_MEM_ADDR_WIDTH'(`PLATFORM_MEMORY_OFFSET); end // ======================================================================== @@ -471,9 +484,11 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // ======================================================================== `SCOPE_IO_SWITCH (2); - // Address requests ahead of the request gate. + // Address and write-data requests ahead of the request gates. wire pre_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire pre_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire pre_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire pre_wready_a [C_M_AXI_MEM_NUM_BANKS]; wire pre_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire pre_arready_a [C_M_AXI_MEM_NUM_BANKS]; @@ -667,7 +682,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .m_awvalid (pre_awvalid_a[0]), .m_awready (pre_awready_a[0]), .m_awaddr (m_axi_mem_awaddr_u[0]), .m_awid (m_axi_mem_awid_a[0]), .m_awlen (m_axi_mem_awlen_a[0]), - .m_wvalid (m_axi_mem_wvalid_a[0]), .m_wready (m_axi_mem_wready_a[0]), + .m_wvalid (pre_wvalid_a[0]), .m_wready (pre_wready_a[0]), .m_wdata (m_axi_mem_wdata_a[0]), .m_wstrb (m_axi_mem_wstrb_a[0]), .m_wlast (m_axi_mem_wlast_a[0]), .m_bvalid (m_axi_mem_bvalid_a[0]), .m_bready (m_axi_mem_bready_a[0]), @@ -995,8 +1010,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `UNUSED_PIN (m_axi_awqos), `UNUSED_PIN (m_axi_awregion), - .m_axi_wvalid (m_axi_mem_wvalid_a), - .m_axi_wready (m_axi_mem_wready_a), + .m_axi_wvalid (pre_wvalid_a), + .m_axi_wready (pre_wready_a), .m_axi_wdata (m_axi_mem_wdata_a), .m_axi_wstrb (m_axi_mem_wstrb_a), .m_axi_wlast (m_axi_mem_wlast_a), @@ -1033,22 +1048,27 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // Request gate + drain tracking. // // Everything upstream drives pre_* instead of the bank port directly. The - // gate below withholds new AW/AR while the reset sequencer is quiescing, - // so the outstanding counts can reach zero in bounded time. W, B and R are - // never gated: a burst whose address the interconnect has already accepted - // must be allowed to finish. + // gates below withhold new reads and writes while the reset sequencer is + // quiescing, so the outstanding counts can reach zero in bounded time. + // A write already begun on either channel completes on both, and B and R + // are never gated: a transaction the interconnect has seen must finish. // ======================================================================== wire mem_idle_a [C_M_AXI_MEM_NUM_BANKS]; for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_axi_gate - VX_afu_req_gate mem_aw_gate ( - .clk (clk), - .reset (reset), - .stop_req (rst_stop_req), - .in_valid (pre_awvalid_a[i]), - .in_ready (pre_awready_a[i]), - .out_valid (m_axi_mem_awvalid_a[i]), - .out_ready (m_axi_mem_awready_a[i]) + VX_afu_wr_gate mem_wr_gate ( + .clk (clk), + .reset (reset), + .stop_req (rst_stop_req), + .in_awvalid (pre_awvalid_a[i]), + .in_awready (pre_awready_a[i]), + .out_awvalid (m_axi_mem_awvalid_a[i]), + .out_awready (m_axi_mem_awready_a[i]), + .in_wvalid (pre_wvalid_a[i]), + .in_wlast (m_axi_mem_wlast_a[i]), + .in_wready (pre_wready_a[i]), + .out_wvalid (m_axi_mem_wvalid_a[i]), + .out_wready (m_axi_mem_wready_a[i]) ); VX_afu_req_gate mem_ar_gate ( @@ -1069,6 +1089,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .aw_fire (m_axi_mem_awvalid_a[i] && m_axi_mem_awready_a[i]), + .w_fire (m_axi_mem_wvalid_a[i] && m_axi_mem_wready_a[i]), .w_fire_last (m_axi_mem_wvalid_a[i] && m_axi_mem_wready_a[i] && m_axi_mem_wlast_a[i]), .b_fire (m_axi_mem_bvalid_a[i] && m_axi_mem_bready_a[i]), @@ -1090,6 +1111,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .aw_fire (m_axi_host_awvalid && m_axi_host_awready), + .w_fire (m_axi_host_wvalid && m_axi_host_wready), .w_fire_last (m_axi_host_wvalid && m_axi_host_wready && m_axi_host_wlast), .b_fire (m_axi_host_bvalid && m_axi_host_bready), .ar_fire (m_axi_host_arvalid && m_axi_host_arready), diff --git a/hw/rtl/afu/common/vortex_afu.vh b/hw/rtl/afu/common/vortex_afu.vh index 9544345186..4ff28ab33f 100644 --- a/hw/rtl/afu/common/vortex_afu.vh +++ b/hw/rtl/afu/common/vortex_afu.vh @@ -163,6 +163,69 @@ assign m_axi_mem_bresp_a[i] = m_axi_mem_``i``_bresp; \ assign m_axi_mem_bid_a[i] = m_axi_mem_``i``_bid +// AXI memory masters the AFU exposes: one per GPU memory port, rounded up to +// a power of two so that address bits select the port, and at most one per +// platform memory channel. A platform top must spell its ports out, so the +// build resolves this to the literal AFU_MEM_PORTS +// (hw/scripts/afu_mem_ports.sh) and VX_afu_wrap checks that the two agree. +`define AFU_MEM_PORTS_OF(n) `MIN(1 << `CLOG2(n), `VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + +// Per-port arrays connecting a platform top's flat m_axi_mem_ ports to +// VX_afu_wrap (see AXI_MEM_TO_ARRAY). +`define AXI_MEM_ARRAYS(n) \ + wire m_axi_mem_awvalid_a [n]; \ + wire m_axi_mem_awready_a [n]; \ + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [n]; \ + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [n]; \ + wire [7:0] m_axi_mem_awlen_a [n]; \ + wire m_axi_mem_wvalid_a [n]; \ + wire m_axi_mem_wready_a [n]; \ + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [n]; \ + wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [n]; \ + wire m_axi_mem_wlast_a [n]; \ + wire m_axi_mem_arvalid_a [n]; \ + wire m_axi_mem_arready_a [n]; \ + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [n]; \ + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [n]; \ + wire [7:0] m_axi_mem_arlen_a [n]; \ + wire m_axi_mem_rvalid_a [n]; \ + wire m_axi_mem_rready_a [n]; \ + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [n]; \ + wire m_axi_mem_rlast_a [n]; \ + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [n]; \ + wire [1:0] m_axi_mem_rresp_a [n]; \ + wire m_axi_mem_bvalid_a [n]; \ + wire m_axi_mem_bready_a [n]; \ + wire [1:0] m_axi_mem_bresp_a [n]; \ + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [n]; + +`define AXI_MEM_ARRAY_ARGS \ + .m_axi_mem_awvalid_a(m_axi_mem_awvalid_a), \ + .m_axi_mem_awready_a(m_axi_mem_awready_a), \ + .m_axi_mem_awaddr_a(m_axi_mem_awaddr_a), \ + .m_axi_mem_awid_a(m_axi_mem_awid_a), \ + .m_axi_mem_awlen_a(m_axi_mem_awlen_a), \ + .m_axi_mem_wvalid_a(m_axi_mem_wvalid_a), \ + .m_axi_mem_wready_a(m_axi_mem_wready_a), \ + .m_axi_mem_wdata_a(m_axi_mem_wdata_a), \ + .m_axi_mem_wstrb_a(m_axi_mem_wstrb_a), \ + .m_axi_mem_wlast_a(m_axi_mem_wlast_a), \ + .m_axi_mem_arvalid_a(m_axi_mem_arvalid_a), \ + .m_axi_mem_arready_a(m_axi_mem_arready_a), \ + .m_axi_mem_araddr_a(m_axi_mem_araddr_a), \ + .m_axi_mem_arid_a(m_axi_mem_arid_a), \ + .m_axi_mem_arlen_a(m_axi_mem_arlen_a), \ + .m_axi_mem_rvalid_a(m_axi_mem_rvalid_a), \ + .m_axi_mem_rready_a(m_axi_mem_rready_a), \ + .m_axi_mem_rdata_a(m_axi_mem_rdata_a), \ + .m_axi_mem_rlast_a(m_axi_mem_rlast_a), \ + .m_axi_mem_rid_a(m_axi_mem_rid_a), \ + .m_axi_mem_rresp_a(m_axi_mem_rresp_a), \ + .m_axi_mem_bvalid_a(m_axi_mem_bvalid_a), \ + .m_axi_mem_bready_a(m_axi_mem_bready_a), \ + .m_axi_mem_bresp_a(m_axi_mem_bresp_a), \ + .m_axi_mem_bid_a(m_axi_mem_bid_a) + `include "VX_define.vh" `endif // VORTEX_AFU_VH diff --git a/hw/rtl/afu/xrt/vortex_afu.v b/hw/rtl/afu/xrt/vortex_afu.v index 6e3bd65a4d..5e37bd818c 100644 --- a/hw/rtl/afu/xrt/vortex_afu.v +++ b/hw/rtl/afu/xrt/vortex_afu.v @@ -21,22 +21,14 @@ module vortex_afu #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - parameter C_M_AXI_MEM_NUM_BANKS = 1 -`else - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS -`endif + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, - // AXI4 master interface -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), -`endif + // AXI4 master interface, one per GPU memory port + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -75,6 +67,9 @@ module vortex_afu #( reset_r <= ~ap_rst_n; end + `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), .C_S_AXI_CTRL_DATA_WIDTH (C_S_AXI_CTRL_DATA_WIDTH), @@ -85,11 +80,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (reset_r), - `ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, AXI_MEM_ARGS, MP_COMMA), - `else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), - `endif + `AXI_MEM_ARRAY_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_ctrl_awvalid), .s_axi_ctrl_awready (s_axi_ctrl_awready), diff --git a/hw/rtl/libs/VX_mem_to_axi.sv b/hw/rtl/libs/VX_mem_to_axi.sv index 8c928152f4..9d0070eb87 100644 --- a/hw/rtl/libs/VX_mem_to_axi.sv +++ b/hw/rtl/libs/VX_mem_to_axi.sv @@ -19,6 +19,11 @@ // (interface-free) so it can sit directly at a platform pin boundary; the // optional AXI sideband (lock/cache/prot/qos/region) is driven to constants. // The inverse of VX_membus_from_axi. +// +// The M masters share one ADDR_WIDTH_OUT byte address space, and each request +// leaves on its bank's master with its own address unchanged. INTERLEAVE +// selects the bank from the low line-address bits; otherwise from the top +// bits, so bank i serves the i-th contiguous 1/M of the space. // ============================================================================ `TRACING_OFF @@ -108,7 +113,7 @@ module VX_mem_to_axi #( localparam LOG2_DATA_SIZE = `CLOG2(DATA_SIZE); localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS_OUT); localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); - localparam DST_ADDR_WDITH = (ADDR_WIDTH_OUT - LOG2_DATA_SIZE) + BANK_SEL_BITS; // convert byte-addressable output addresss to block-addressable input space + localparam DST_ADDR_WDITH = ADDR_WIDTH_OUT - LOG2_DATA_SIZE; // line address width of the output space localparam BANK_ADDR_WIDTH = DST_ADDR_WDITH - BANK_SEL_BITS; localparam NUM_PORTS_IN_BITS = `CLOG2(NUM_PORTS_IN); localparam NUM_PORTS_IN_WIDTH = `UP(NUM_PORTS_IN_BITS); diff --git a/hw/rtl/libs/VX_mm_axi_arb.sv b/hw/rtl/libs/VX_mm_axi_arb.sv index 7bfddde3c2..a45d96375d 100644 --- a/hw/rtl/libs/VX_mm_axi_arb.sv +++ b/hw/rtl/libs/VX_mm_axi_arb.sv @@ -41,8 +41,9 @@ // until WLAST. B/R route back to the owner. IDs pass through untouched. // * MULTI_OUT == 1: ID-routed, multi-outstanding fan-in. The top // LOG2UP(NUM_INPUTS) bits of the AXI ID carry the source index (masters -// must leave them free); reads are fully concurrent, write bursts serialize -// on the shared W channel (AW gated until WLAST) while B stays concurrent. +// must leave them free); reads are fully concurrent, writes share the W +// channel in AW order (a write's W may go with its AW, and later AWs may +// run ahead of earlier data) while B stays concurrent. // Requires ID_WIDTH > LOG2UP(NUM_INPUTS) and NUM_OUTPUTS == 1. // // OUT_REG (0/1): registered AXI stage (VX_mm_axi_slice) on each slave port. @@ -324,10 +325,10 @@ module VX_mm_axi_arb #( // ===== Multi-outstanding, ID-routed (source index in ID high bits) ===== // The top SEL_WIDTH bits of the AXI ID carry the source index; masters - // must leave them free. Reads are fully concurrent; write bursts serialize - // on the shared W channel (AW gated until the prior burst's WLAST) while B - // responses stay concurrent. An offer the slave has not yet taken holds - // the grant, so its addr/id stay stable until the handshake. + // must leave them free. Reads are fully concurrent; write data shares + // the W channel in AW order while B responses stay concurrent. An offer + // the slave has not yet taken holds the grant, so its addr/id stay + // stable until the handshake. localparam LOW_ID = ID_WIDTH - SEL_WIDTH; // source's own low ID bits `STATIC_ASSERT((ID_WIDTH > SEL_WIDTH), ("VX_mm_axi_arb MULTI_OUT: ID_WIDTH must exceed LOG2UP(NUM_INPUTS) to carry the source index")) @@ -382,9 +383,19 @@ module VX_mm_axi_arb #( end assign c_rready = s_rready[r_src]; - // ---- Write: sticky RR grant gated on W-drain, tag awid, demux B by bid ---- - reg w_active; - reg [SEL_WIDTH-1:0] w_route; + // ---- Write: sticky RR grant, tag awid, W in AW order, demux B by bid ---- + // An AW is committed the first cycle it is offered; the held grant then + // keeps it on the bus until the slave takes it, so AWs reach the slave + // in commit order. W bursts follow that order through a small queue of + // committed sources, and the burst of the AW being committed this cycle + // bypasses an empty queue: a single-beat write moves AW and W in the + // same cycle, and W never waits for AWREADY (a slave may accept the + // data first). A full queue holds new AWs back. + localparam WQ_SIZE = 4; + localparam WQ_ADDRW = `LOG2UP(WQ_SIZE); + localparam WQ_CNTW = WQ_ADDRW + 1; + + reg aw_pend; // offered last cycle, not yet taken wire [SEL_WIDTH-1:0] aw_pick; wire aw_any; wire aw_fire = c_awvalid && c_awready; @@ -394,16 +405,10 @@ module VX_mm_axi_arb #( end else begin : g_aw_arb // Once offered, the AW must stay unchanged until the slave takes it // (AXI stability): only the offering master may compete meanwhile. - reg aw_held; reg [SEL_WIDTH-1:0] aw_held_idx; - wire [NUM_INPUTS-1:0] aw_reqs = aw_held ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) + wire [NUM_INPUTS-1:0] aw_reqs = aw_pend ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) : s_awvalid; always @(posedge clk) begin - if (reset) begin - aw_held <= 1'b0; - end else begin - aw_held <= c_awvalid && ~c_awready; - end aw_held_idx <= aw_pick; end VX_generic_arbiter #( @@ -416,33 +421,54 @@ module VX_mm_axi_arb #( .grant_valid (aw_any), .grant_ready (aw_fire) ); end - // Gate a new AW until the previous write's W burst drains. - assign c_awvalid = aw_any && !w_active; + + reg [WQ_SIZE-1:0][SEL_WIDTH-1:0] wq_src; + reg [WQ_ADDRW-1:0] wq_rd_ptr, wq_wr_ptr; + reg [WQ_CNTW-1:0] wq_count; + wire wq_empty = (wq_count == '0); + wire wq_full = (wq_count == WQ_CNTW'(WQ_SIZE)); + + assign c_awvalid = aw_any && (aw_pend || ~wq_full); assign c_awaddr = s_awaddr[aw_pick]; assign c_awlen = s_awlen [aw_pick]; assign c_awid = {aw_pick, s_awid[aw_pick][LOW_ID-1:0]}; - assign s_awready = {NUM_INPUTS{!w_active && c_awready}} + assign s_awready = {NUM_INPUTS{(aw_pend || ~wq_full) && c_awready}} & ({{(NUM_INPUTS-1){1'b0}}, aw_any} << aw_pick); + wire aw_commit = c_awvalid && ~aw_pend; + + wire w_route_valid = ~wq_empty || aw_commit; + wire [SEL_WIDTH-1:0] w_route = wq_empty ? aw_pick : wq_src[wq_rd_ptr]; + wire w_last_fire = c_wvalid && c_wready && c_wlast; + wire wq_push = aw_commit && ~(wq_empty && w_last_fire); + wire wq_pop = ~wq_empty && w_last_fire; + always @(posedge clk) begin if (reset) begin - w_active <= 1'b0; - w_route <= '0; + aw_pend <= 1'b0; + wq_rd_ptr <= '0; + wq_wr_ptr <= '0; + wq_count <= '0; end else begin - if (aw_fire) begin - w_active <= 1'b1; - w_route <= aw_pick; - end else if (w_active && c_wvalid && c_wready && c_wlast) begin - w_active <= 1'b0; + aw_pend <= c_awvalid && ~c_awready; + if (wq_push) begin + wq_wr_ptr <= wq_wr_ptr + WQ_ADDRW'(1); + end + if (wq_pop) begin + wq_rd_ptr <= wq_rd_ptr + WQ_ADDRW'(1); end + wq_count <= wq_count + WQ_CNTW'(wq_push) - WQ_CNTW'(wq_pop); + end + if (wq_push) begin + wq_src[wq_wr_ptr] <= aw_pick; end end - assign c_wvalid = w_active && s_wvalid[w_route]; + assign c_wvalid = w_route_valid && s_wvalid[w_route]; assign c_wdata = s_wdata[w_route]; assign c_wstrb = s_wstrb[w_route]; assign c_wlast = s_wlast[w_route]; - assign s_wready = {NUM_INPUTS{w_active && c_wready}} + assign s_wready = {NUM_INPUTS{w_route_valid && c_wready}} & ({{(NUM_INPUTS-1){1'b0}}, 1'b1} << w_route); wire [SEL_WIDTH-1:0] b_src = c_bid[ID_WIDTH-1 -: SEL_WIDTH]; diff --git a/hw/scripts/afu_mem_ports.sh b/hw/scripts/afu_mem_ports.sh new file mode 100755 index 0000000000..df9b7031cd --- /dev/null +++ b/hw/scripts/afu_mem_ports.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prints how many AXI memory masters the AFU exposes for a configuration. +# +# Usage: afu_mem_ports.sh [-c ] +# +# The count follows the GPU's memory ports, which only the elaborated RTL +# package knows; a platform top must nevertheless spell its ports out +# literally. Elaborating a one-line probe against the same flags as the build +# hands the build the exact value VX_afu_wrap checks it against. +# +# With -c, the result is cached under , keyed on the flags and on +# the contents of every package and configuration header they name, so a +# Makefile can resolve it on every parse without re-running Verilator. + +set -eo pipefail + +cache_dir="" +if [ "$1" = "-c" ]; then + cache_dir=$2 + shift 2 +fi + +VERILATOR=$1 +shift + +if [ -n "$cache_dir" ]; then + key=$({ + printf '%s\n' "$@" + for arg in "$@"; do + case "$arg" in + -I*) for h in VX_config.vh VX_types.vh vortex_afu.vh; do + if [ -f "${arg#-I}/$h" ]; then cat "${arg#-I}/$h"; fi + done ;; + *) if [ -f "$arg" ]; then cat "$arg"; fi ;; + esac + done + } | sha1sum | cut -d' ' -f1) + cache_file="$cache_dir/afu_mem_ports.$key" + if [ -s "$cache_file" ]; then + cat "$cache_file" + exit 0 + fi +fi + +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT + +cat > "$tmp_dir/VX_afu_mem_ports_probe.sv" <<'EOF_PROBE' +`include "VX_define.vh" +`include "vortex_afu.vh" +module VX_afu_mem_ports_probe import VX_gpu_pkg::*; (); + $info("AFU_MEM_PORTS=%0d", `AFU_MEM_PORTS_OF(VX_MEM_PORTS)); +endmodule +EOF_PROBE + +if ! out=$("$VERILATOR" --lint-only -Wno-fatal "$@" --top-module VX_afu_mem_ports_probe \ + "$tmp_dir/VX_afu_mem_ports_probe.sv" 2>&1); then + echo "$out" >&2 + echo "afu_mem_ports.sh: the probe did not elaborate" >&2 + exit 1 +fi + +count=$(echo "$out" | grep -o 'AFU_MEM_PORTS=[0-9][0-9]*' | head -1 | cut -d= -f2 || true) +if [ -z "$count" ]; then + echo "$out" >&2 + echo "afu_mem_ports.sh: the probe reported no port count" >&2 + exit 1 +fi + +if [ -n "$cache_dir" ]; then + mkdir -p "$cache_dir" + echo "$count" > "$cache_file.$$" + mv "$cache_file.$$" "$cache_file" +fi +echo "$count" diff --git a/hw/syn/xilinx/aved/Makefile b/hw/syn/xilinx/aved/Makefile index f1471e46d8..0787371146 100644 --- a/hw/syn/xilinx/aved/Makefile +++ b/hw/syn/xilinx/aved/Makefile @@ -172,6 +172,18 @@ CFLAGS += $(RTL_INCLUDE) # ast dump flags XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI +# One AXI memory master per GPU memory port. The kernel top must declare its +# ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(BUILD_DIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator $(filter-out -J%,$(XML_CFLAGS))) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +# config.cfg.tmpl binds the single device-memory master to MEM_TAG. +ifneq ($(AFU_MEM_PORTS),1) +$(error the V80 link binds one device-memory master, but this configuration has AFU_MEM_PORTS=$(AFU_MEM_PORTS)) +endif +CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + .PHONY: all clean gen-sources gen-ast scope-json package-ip gen-bin check-axil-blocks # Fail before synthesis if any 16-byte block in the CP AXI-Lite regfile is only @@ -188,9 +200,18 @@ check-axil-blocks: all: check-axil-blocks $(VBIN_CONTAINER) gen-sources: $(BUILD_DIR)/sources.txt -$(BUILD_DIR)/sources.txt: +$(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt +# The preprocessed sources bake in the kernel's port count: regenerate them +# whenever the configuration changes it. +$(BUILD_DIR)/afu_mem_ports.stamp: force-afu-mem-ports + @mkdir -p $(BUILD_DIR) + @if [ "$$(cat $@ 2>/dev/null)" != "$(AFU_MEM_PORTS)" ]; then echo "$(AFU_MEM_PORTS)" > $@; fi + +.PHONY: force-afu-mem-ports +force-afu-mem-ports: + gen-ast: $(BUILD_DIR)/vortex.xml $(BUILD_DIR)/vortex.xml: mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml diff --git a/hw/syn/xilinx/aved/platforms.mk b/hw/syn/xilinx/aved/platforms.mk index 80ced1dd35..64029f51f0 100644 --- a/hw/syn/xilinx/aved/platforms.mk +++ b/hw/syn/xilinx/aved/platforms.mk @@ -8,10 +8,10 @@ override CONFIGS += -DPLATFORM_MEMORY_DATA_WIDTH=512 # The V80 exposes its HBM stacks through the linker's HBM connectivity tags. # A single wide master fanned across the channels avoids the per-bank base # address problem: the linker assigns each memory range independently, and -# those bases are not known at synthesis time. +# those bases are not known at synthesis time. One memory channel caps the +# AFU at one memory master whatever the GPU's memory port count. override CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 override CONFIGS += -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 -override CONFIGS += -DPLATFORM_MERGED_MEMORY_INTERFACE # Device-memory aperture base. The V80 maps the AFU's m_axi_mem_0 port at # 0x40_0000_0000 in both build flavours -- simulation puts the BRAM model there diff --git a/hw/syn/xilinx/xrt/Makefile b/hw/syn/xilinx/xrt/Makefile index 22dc8986d5..a1ec0c00b2 100644 --- a/hw/syn/xilinx/xrt/Makefile +++ b/hw/syn/xilinx/xrt/Makefile @@ -194,6 +194,26 @@ CFLAGS += $(RTL_INCLUDE) # ast dump flags XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI +# One AXI memory master per GPU memory port. The kernel top must declare its +# ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(BUILD_DIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator $(filter-out -J%,$(XML_CFLAGS))) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + +# Port i owns the i-th contiguous 1/N of device memory (VX_afu_wrap). Bind it +# to the matching contiguous group of the platform's memory channels, so each +# port reaches only its own channels and no two ports share one. +ifdef PLATFORM_MEMORY_TAG +VPP_FLAGS += $(shell n=$(AFU_MEM_PORTS); g=$$(($(PLATFORM_MEMORY_CHANNELS) / n)); \ + for i in $$(seq 0 $$((n - 1))); do \ + lo=$$((i * g)); hi=$$((lo + g - 1)); \ + if [ $$g -eq 1 ]; then r=$$lo; else r=$$lo:$$hi; fi; \ + echo "--connectivity.sp vortex_afu_1.m_axi_mem_$$i:$(PLATFORM_MEMORY_TAG)[$$r]"; \ + done) +endif + # RTL Kernel only supports Hardware and Hardware Emulation. ifneq ($(TARGET),$(findstring $(TARGET), hw hw_emu)) $(warning WARNING:Application supports only hw hw_emu TARGET. Please use the target for running the application) @@ -204,9 +224,18 @@ endif all: check-devices emconfig $(XCLBIN_CONTAINER) report gen-sources: $(BUILD_DIR)/sources.txt -$(BUILD_DIR)/sources.txt: +$(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt +# The preprocessed sources bake in the kernel's port count: regenerate them +# whenever the configuration changes it. +$(BUILD_DIR)/afu_mem_ports.stamp: force-afu-mem-ports + @mkdir -p $(BUILD_DIR) + @if [ "$$(cat $@ 2>/dev/null)" != "$(AFU_MEM_PORTS)" ]; then echo "$(AFU_MEM_PORTS)" > $@; fi + +.PHONY: force-afu-mem-ports +force-afu-mem-ports: + gen-ast: $(BUILD_DIR)/vortex.xml $(BUILD_DIR)/vortex.xml: mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml diff --git a/hw/syn/xilinx/xrt/package_kernel.tcl b/hw/syn/xilinx/xrt/package_kernel.tcl index 2f97354835..d9b9777c72 100644 --- a/hw/syn/xilinx/xrt/package_kernel.tcl +++ b/hw/syn/xilinx/xrt/package_kernel.tcl @@ -45,8 +45,7 @@ set vdefines_list [lindex $vlist 2] #puts ${vdefines_list} set chipscope 0 -set num_banks 1 -set merged_mem_if 0 +set afu_mem_ports -1 # parse vdefines_list for configuration parameters foreach def $vdefines_list { @@ -55,16 +54,9 @@ foreach def $vdefines_list { if { $name == "CHIPSCOPE" } { set chipscope 1 } - if { $name == "PLATFORM_MEMORY_NUM_BANKS" } { - set num_banks [lindex $fields 1] + if { $name == "AFU_MEM_PORTS" } { + set afu_mem_ports [lindex $fields 1] } - if { $name == "PLATFORM_MERGED_MEMORY_INTERFACE" } { - set merged_mem_if 1 - } -} - -if { $merged_mem_if == 1 } { - set num_banks 1 } create_project -force kernel_pack $path_to_tmp_project @@ -177,6 +169,14 @@ foreach m $axi_masters { ipx::associate_bus_interfaces -busif $m -clock ap_clk $core } +# The build binds AFU_MEM_PORTS memory masters to device memory; a kernel +# with any other count would leave ports unbound or bind names it lacks. +set num_mem_masters [llength [lsearch -all -glob $axi_masters "m_axi_mem_*"]] +if { $num_mem_masters != $afu_mem_ports } { + puts "ERROR: packaged kernel has $num_mem_masters m_axi_mem masters, but AFU_MEM_PORTS=$afu_mem_ports" + exit 1 +} + set mem_map [::ipx::add_memory_map -quiet "s_axi_ctrl" $core] set addr_block [::ipx::add_address_block -quiet "reg0" $mem_map] diff --git a/hw/syn/xilinx/xrt/platforms.mk b/hw/syn/xilinx/xrt/platforms.mk index 7b4356bfe0..f50397a122 100644 --- a/hw/syn/xilinx/xrt/platforms.mk +++ b/hw/syn/xilinx/xrt/platforms.mk @@ -18,30 +18,35 @@ else # through the platform slave-bridge / Host Memory Access aperture. All Alveo # XDMA shells expose this as the HOST[0] connectivity tag. VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_host:HOST[0] +# PLATFORM_MEMORY_TAG/CHANNELS name the device-memory channels the AFU's +# memory masters bind to; the Makefile gives each master its own group. ifneq ($(findstring xilinx_u55c,$(XSA)),) # 16 GB of HBM2 with 32 channels (512 MB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=32 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 - CONFIGS += -DPLATFORM_MERGED_MEMORY_INTERFACE - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:HBM[0:31] - #VPP_FLAGS += $(foreach i,$(shell seq 0 31), --connectivity.sp vortex_afu_1.m_axi_mem_$(i):HBM[$(i)]) + PLATFORM_MEMORY_TAG := HBM + PLATFORM_MEMORY_CHANNELS := 32 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 else ifneq ($(findstring xilinx_u50,$(XSA)),) # 8 GB of HBM2 with 32 channels (256 MB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=32 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 - CONFIGS += -DPLATFORM_MERGED_MEMORY_INTERFACE - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:HBM[0:31] + PLATFORM_MEMORY_TAG := HBM + PLATFORM_MEMORY_CHANNELS := 32 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 else ifneq ($(findstring xilinx_u280,$(XSA)),) # 8 GB of HBM2 with 32 channels (256 MB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=32 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:HBM[0:31] + PLATFORM_MEMORY_TAG := HBM + PLATFORM_MEMORY_CHANNELS := 32 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 else ifneq ($(findstring xilinx_u250,$(XSA)),) # 16 GB of DDR4 (single channel, bank 0). Multi-bank requires per-bank XRT # VA offsets that aren't known at synthesis time. - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:DDR[0] + PLATFORM_MEMORY_TAG := DDR + PLATFORM_MEMORY_CHANNELS := 1 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 CONFIGS += -DPLATFORM_MEMORY_OFFSET=40\'h4000000000 else ifneq ($(findstring xilinx_u200,$(XSA)),) # 64 GB of DDR4 with 4 channels (16 GB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=4 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=36 + PLATFORM_MEMORY_TAG := DDR + PLATFORM_MEMORY_CHANNELS := 4 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=36 else CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=32 endif diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 87c8e63f85..4f2916de0f 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -19,6 +19,7 @@ all: $(MAKE) -C cp_axil_regfile $(MAKE) -C afu_axil_demux $(MAKE) -C afu_reset_seq + $(MAKE) -C mm_axi_arb $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma $(MAKE) -C cp_llc_port @@ -52,6 +53,7 @@ run: $(MAKE) -C cp_axil_regfile run $(MAKE) -C afu_axil_demux run $(MAKE) -C afu_reset_seq run + $(MAKE) -C mm_axi_arb run $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run $(MAKE) -C cp_llc_port run @@ -153,6 +155,7 @@ clean: $(MAKE) -C cp_dma clean $(MAKE) -C cp_llc_port clean $(MAKE) -C stream_arb clean + $(MAKE) -C mm_axi_arb clean $(MAKE) -C cp_core clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean diff --git a/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv b/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv index a0fbbf4d70..c7ad7530af 100644 --- a/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv +++ b/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv @@ -30,6 +30,7 @@ module VX_afu_reset_seq_top #( // AXI handshakes, driven directly by the testbench. input wire aw_fire, + input wire w_fire, input wire w_fire_last, input wire b_fire, input wire ar_fire, @@ -43,6 +44,18 @@ module VX_afu_reset_seq_top #( output wire gate_in_ready, output wire gate_out_valid, + // A real write gate on the same stop_req, the testbench again playing + // both the AFU-internal master (wg_in_*) and the shell (wg_out_*). + input wire wg_in_awvalid, + input wire wg_out_awready, + output wire wg_in_awready, + output wire wg_out_awvalid, + input wire wg_in_wvalid, + input wire wg_in_wlast, + input wire wg_out_wready, + output wire wg_in_wready, + output wire wg_out_wvalid, + output wire stop_req, output wire rst_assert, output wire busy, @@ -60,10 +73,26 @@ module VX_afu_reset_seq_top #( .out_ready (gate_out_ready) ); + VX_afu_wr_gate wr_gate ( + .clk (clk), + .reset (reset), + .stop_req (stop_req), + .in_awvalid (wg_in_awvalid), + .in_awready (wg_in_awready), + .out_awvalid (wg_out_awvalid), + .out_awready (wg_out_awready), + .in_wvalid (wg_in_wvalid), + .in_wlast (wg_in_wlast), + .in_wready (wg_in_wready), + .out_wvalid (wg_out_wvalid), + .out_wready (wg_out_wready) + ); + VX_afu_axi_drain drain ( .clk (clk), .reset (reset), .aw_fire (aw_fire), + .w_fire (w_fire), .w_fire_last (w_fire_last), .b_fire (b_fire), .ar_fire (ar_fire), diff --git a/hw/unittest/afu_reset_seq/main.cpp b/hw/unittest/afu_reset_seq/main.cpp index 6f303d8f3c..3656915e4c 100644 --- a/hw/unittest/afu_reset_seq/main.cpp +++ b/hw/unittest/afu_reset_seq/main.cpp @@ -35,9 +35,11 @@ void tick() { } void clear_fires() { - dut->aw_fire = 0; dut->w_fire_last = 0; dut->b_fire = 0; + dut->aw_fire = 0; dut->w_fire = 0; dut->w_fire_last = 0; dut->b_fire = 0; dut->ar_fire = 0; dut->r_fire_last = 0; dut->gate_in_valid = 0; dut->gate_out_ready = 0; + dut->wg_in_awvalid = 0; dut->wg_out_awready = 0; + dut->wg_in_wvalid = 0; dut->wg_in_wlast = 0; dut->wg_out_wready = 0; } void do_reset() { @@ -203,6 +205,154 @@ int main(int argc, char** argv) { run(8); check(!dut->stop_req, "gate reopens after the sequence"); + // ------------------------------------------------------------------ + // AXI4 lets write data precede its address. Quiescing must neither strand + // data whose AW it then withholds nor withhold the AW of data the shell + // already took: the shell must see whole writes only. + printf("8. quiescing never splits a write\n"); + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; // keep QUIESCE long enough to observe + + // (a) data first: a single-beat W is taken before its AW is presented. + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; dut->wg_out_wready = 1; + tick(); + dut->wg_in_wvalid = 0; dut->wg_in_wlast = 0; + pulse_request(); + run(4); + check(dut->stop_req, "sequencer is quiescing"); + dut->wg_in_awvalid = 1; dut->wg_out_awready = 0; + dut->eval(); + check(dut->wg_out_awvalid, "the AW owed to accepted data still goes out"); + tick(); + dut->wg_out_awready = 1; + dut->eval(); + check(dut->wg_out_awvalid && dut->wg_in_awready, "and is accepted"); + tick(); + dut->wg_out_awready = 0; + dut->eval(); + check(!dut->wg_out_awvalid, "the next AW is withheld"); + dut->wg_out_awready = 1; + dut->eval(); + check(!dut->wg_in_awready, "and not reported accepted"); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; + dut->eval(); + check(!dut->wg_out_wvalid, "its data is withheld too"); + check(!dut->wg_in_wready, "and not reported accepted"); + run(4); + check(!dut->wg_out_awvalid && !dut->wg_out_wvalid, "and both stay withheld"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + + // (b) address first, then a two-beat burst: the data owed to an accepted AW + // goes out in full, and nothing after it. + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; + dut->wg_in_awvalid = 1; dut->wg_out_awready = 1; + tick(); + dut->wg_in_awvalid = 0; + pulse_request(); + run(4); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 0; dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_wvalid, "the first beat owed to the AW goes out"); + tick(); + dut->wg_in_wlast = 1; + dut->eval(); + check(dut->wg_out_wvalid, "the burst runs to its last beat"); + tick(); + dut->wg_in_wlast = 0; + dut->eval(); + check(!dut->wg_out_wvalid, "data with no accepted AW is withheld"); + dut->wg_in_awvalid = 1; + dut->eval(); + check(!dut->wg_out_awvalid, "a new AW is withheld"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + check(!dut->stop_req, "gate reopens after the sequence"); + dut->wg_in_awvalid = 1; dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; + dut->wg_out_awready = 1; dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_awvalid && dut->wg_out_wvalid, "writes flow again once reopened"); + clear_fires(); + + // (c) a slave may hold AWREADY until it has seen the data: an address + // already offered when quiescing begins obliges its data. + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; + dut->wg_in_awvalid = 1; dut->wg_out_awready = 0; + tick(); + pulse_request(); + run(4); + check(dut->stop_req, "sequencer is quiescing"); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_wvalid, "data owed to an offered AW goes out"); + tick(); + dut->wg_in_wvalid = 0; dut->wg_in_wlast = 0; + dut->wg_out_awready = 1; + dut->eval(); + check(dut->wg_out_awvalid, "and the slave then takes the AW"); + tick(); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; + dut->eval(); + check(!dut->wg_out_wvalid && !dut->wg_out_awvalid, "nothing further goes out"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + + // (d) a slave may hold WREADY until it has seen the address: data already + // offered when quiescing begins obliges its AW. + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; dut->wg_out_wready = 0; + tick(); + pulse_request(); + run(4); + dut->wg_in_awvalid = 1; dut->wg_out_awready = 1; + dut->eval(); + check(dut->wg_out_awvalid, "the AW owed to offered data goes out"); + tick(); + dut->wg_in_awvalid = 0; + dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_wvalid, "and the slave then takes the data"); + tick(); + dut->wg_in_awvalid = 1; + dut->eval(); + check(!dut->wg_out_wvalid && !dut->wg_out_awvalid, "nothing further goes out"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + + // ------------------------------------------------------------------ + // The drain counters see last beats only. A burst whose first beats went + // out ahead of its AW still owes the interconnect the rest of the burst + // and the AW, so the port is not idle and reset must wait. + printf("9. a write burst in progress keeps the port busy\n"); + do_reset(); + dut->w_fire = 1; tick(); dut->w_fire = 0; // beat 1 of 2, no AW yet + tick(); + check(!dut->masters_idle, "not idle with a burst under way"); + pulse_request(); + run(16); + check(!saw_assert, "rst_assert withheld mid-burst"); + dut->w_fire = 1; dut->w_fire_last = 1; tick(); dut->w_fire = 0; dut->w_fire_last = 0; + dut->aw_fire = 1; tick(); dut->aw_fire = 0; + dut->b_fire = 1; tick(); dut->b_fire = 0; + run(8); + check(dut->masters_idle, "idle once the write completes"); + check(saw_assert, "reset proceeds"); + printf("\n%s (%d failure%s)\n", failures ? "FAILED" : "PASSED", failures, failures == 1 ? "" : "s"); delete dut; diff --git a/hw/unittest/mm_axi_arb/Makefile b/hw/unittest/mm_axi_arb/Makefile new file mode 100644 index 0000000000..7ff7e7effb --- /dev/null +++ b/hw/unittest/mm_axi_arb/Makefile @@ -0,0 +1,23 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := mm_axi_arb + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := + +RTL_INCLUDE := -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs -I$(SRC_DIR) + +TOP := VX_mm_axi_arb_top + +include ../common.mk diff --git a/hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv b/hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv new file mode 100644 index 0000000000..73f72b6724 --- /dev/null +++ b/hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv @@ -0,0 +1,159 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_platform.vh" + +// Harness for the ID-routed, multi-outstanding 2:1 VX_mm_axi_arb; the +// testbench plays both masters and the slave. +module VX_mm_axi_arb_top #( + parameter NUM_INPUTS = 2, + parameter ADDR_WIDTH = 32, + parameter DATA_WIDTH = 32, + parameter ID_WIDTH = 8 +) ( + input wire clk, + input wire reset, + + input wire [NUM_INPUTS-1:0] s_awvalid, + output wire [NUM_INPUTS-1:0] s_awready, + input wire [NUM_INPUTS-1:0][ADDR_WIDTH-1:0] s_awaddr, + input wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_awid, + input wire [NUM_INPUTS-1:0][7:0] s_awlen, + + input wire [NUM_INPUTS-1:0] s_wvalid, + output wire [NUM_INPUTS-1:0] s_wready, + input wire [NUM_INPUTS-1:0][DATA_WIDTH-1:0] s_wdata, + input wire [NUM_INPUTS-1:0] s_wlast, + + output wire [NUM_INPUTS-1:0] s_bvalid, + input wire [NUM_INPUTS-1:0] s_bready, + output wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_bid, + + input wire [NUM_INPUTS-1:0] s_arvalid, + output wire [NUM_INPUTS-1:0] s_arready, + input wire [NUM_INPUTS-1:0][ADDR_WIDTH-1:0] s_araddr, + input wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_arid, + input wire [NUM_INPUTS-1:0][7:0] s_arlen, + + output wire [NUM_INPUTS-1:0] s_rvalid, + input wire [NUM_INPUTS-1:0] s_rready, + output wire [NUM_INPUTS-1:0][DATA_WIDTH-1:0] s_rdata, + output wire [NUM_INPUTS-1:0] s_rlast, + output wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_rid, + + output wire m_awvalid, + input wire m_awready, + output wire [ADDR_WIDTH-1:0] m_awaddr, + output wire [ID_WIDTH-1:0] m_awid, + output wire [7:0] m_awlen, + + output wire m_wvalid, + input wire m_wready, + output wire [DATA_WIDTH-1:0] m_wdata, + output wire m_wlast, + + input wire m_bvalid, + output wire m_bready, + input wire [ID_WIDTH-1:0] m_bid, + + output wire m_arvalid, + input wire m_arready, + output wire [ADDR_WIDTH-1:0] m_araddr, + output wire [ID_WIDTH-1:0] m_arid, + output wire [7:0] m_arlen, + + input wire m_rvalid, + output wire m_rready, + input wire [DATA_WIDTH-1:0] m_rdata, + input wire m_rlast, + input wire [ID_WIDTH-1:0] m_rid +); + localparam STRB_WIDTH = DATA_WIDTH / 8; + + wire [NUM_INPUTS-1:0][1:0] s_bresp, s_rresp; + wire [STRB_WIDTH-1:0] m_wstrb; + `UNUSED_VAR ({s_bresp, s_rresp, m_wstrb}) + + VX_mm_axi_arb #( + .NUM_INPUTS (NUM_INPUTS), + .ADDR_WIDTH (ADDR_WIDTH), + .DATA_WIDTH (DATA_WIDTH), + .ID_WIDTH (ID_WIDTH), + .ARBITER ("R"), + .MULTI_OUT (1) + ) arb ( + .clk (clk), + .reset (reset), + + .s_awvalid (s_awvalid), + .s_awready (s_awready), + .s_awaddr (s_awaddr), + .s_awid (s_awid), + .s_awlen (s_awlen), + + .s_wvalid (s_wvalid), + .s_wready (s_wready), + .s_wdata (s_wdata), + .s_wstrb ({NUM_INPUTS{{STRB_WIDTH{1'b1}}}}), + .s_wlast (s_wlast), + + .s_bvalid (s_bvalid), + .s_bready (s_bready), + .s_bid (s_bid), + .s_bresp (s_bresp), + + .s_arvalid (s_arvalid), + .s_arready (s_arready), + .s_araddr (s_araddr), + .s_arid (s_arid), + .s_arlen (s_arlen), + + .s_rvalid (s_rvalid), + .s_rready (s_rready), + .s_rdata (s_rdata), + .s_rlast (s_rlast), + .s_rid (s_rid), + .s_rresp (s_rresp), + + .m_awvalid (m_awvalid), + .m_awready (m_awready), + .m_awaddr (m_awaddr), + .m_awid (m_awid), + .m_awlen (m_awlen), + + .m_wvalid (m_wvalid), + .m_wready (m_wready), + .m_wdata (m_wdata), + .m_wstrb (m_wstrb), + .m_wlast (m_wlast), + + .m_bvalid (m_bvalid), + .m_bready (m_bready), + .m_bid (m_bid), + .m_bresp (2'b00), + + .m_arvalid (m_arvalid), + .m_arready (m_arready), + .m_araddr (m_araddr), + .m_arid (m_arid), + .m_arlen (m_arlen), + + .m_rvalid (m_rvalid), + .m_rready (m_rready), + .m_rdata (m_rdata), + .m_rlast (m_rlast), + .m_rid (m_rid), + .m_rresp (2'b00) + ); + +endmodule diff --git a/hw/unittest/mm_axi_arb/main.cpp b/hw/unittest/mm_axi_arb/main.cpp new file mode 100644 index 0000000000..0dfd3bbc7c --- /dev/null +++ b/hw/unittest/mm_axi_arb/main.cpp @@ -0,0 +1,415 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The properties under test, for the ID-routed (MULTI_OUT) 2:1 arbiter: +// * the slave sees each write's data in the order it saw the addresses, +// whole bursts at a time, whichever channel each master offers first; +// * an offer the slave has not taken stays unchanged (AXI4 A3.2.1); +// * no master waits on AWREADY for its data, so a slave that takes the +// data before the address cannot deadlock it; +// * B and R reach the master that issued them, in order; +// * a write whose address and data are offered together moves in one cycle. + +#include +#include +#include +#include +#include "VVX_mm_axi_arb_top.h" +#include "verilated.h" + +namespace { + +constexpr int NUM_MASTERS = 2; +constexpr int ID_SRC_BIT = 7; // the arbiter's source field (ID_WIDTH-1) + +VVX_mm_axi_arb_top* dut = nullptr; +int failures = 0; +uint32_t rng = 1; + +uint32_t rnd() { + rng = rng * 1103515245u + 12345u; + return rng >> 16; +} + +bool chance(int pct) { return int(rnd() % 100) < pct; } + +void check(bool ok, const char* what) { + printf(" %-62s %s\n", what, ok ? "PASS" : "*** FAIL ***"); + if (!ok) { + ++failures; + } +} + +int errors = 0; +void error(const char* msg, long a = 0, long b = 0) { + if (errors < 10) { + printf(" error: %s (%ld, %ld)\n", msg, a, b); + } + ++errors; +} + +// Packed [N][W] port helpers. +template uint32_t get(T v, int i, int w) { return uint32_t((uint64_t(v) >> (i * w)) & ((1ull << w) - 1)); } +template void put(T& v, int i, int w, uint32_t x) { + uint64_t m = ((1ull << w) - 1) << (i * w); + v = T((uint64_t(v) & ~m) | ((uint64_t(x) << (i * w)) & m)); +} + +uint32_t waddr(int m, int k) { return (uint32_t(m) << 28) | (uint32_t(k) << 8); } +uint32_t wdata(int m, int k, int beat) { return waddr(m, k) | uint32_t(beat); } + +enum Style { + STYLE_FREE, // AW and W offered independently; data may lead its address + STYLE_AWFIRST // data only after its address was taken, one write at a time +}; + +struct Master { + int id; + Style style; + int pct; // offer probability per cycle + int num_writes, num_reads; + std::vector len; // per write, beats - 1 + int aw_next = 0, aw_done = 0; + bool aw_offer = false; + int w_next = 0, w_beat = 0, w_done = 0; + bool w_offer = false; + int b_recv = 0; + int ar_next = 0; + bool ar_offer = false; + std::deque> r_exp; // addr, len + int r_beat = 0, r_done = 0; + bool done() const { return b_recv == num_writes && r_done == num_reads; } +}; + +struct SlaveCfg { + int aw_pct, w_pct, ar_pct, b_pct, r_pct; + bool w_first; // AWREADY only once that write's data has begun +}; + +struct AwRec { uint32_t addr; uint32_t id; int len; }; + +bool run(std::vector& ms, const SlaveCfg& sc, int max_cycles, int* cycles_out, + int* writes_out = nullptr, int* last_write_cycle = nullptr) { + std::deque aw_q; + std::deque> w_bursts; + std::vector w_cur; + int w_started = 0, aw_taken = 0; + std::deque b_q; + std::deque> ar_q; // addr|id, beats + std::deque r_q; + int r_beat = 0; + bool b_hold = false, r_hold = false; + int writes = 0; + // offer-stability tracking on the slave side + bool pend_aw = false, pend_w = false, pend_ar = false; + uint64_t pend_aw_v = 0, pend_w_v = 0, pend_ar_v = 0; + + int cycle = 0; + for (; cycle < max_cycles; ++cycle) { + bool all_done = true; + for (auto& m : ms) { + all_done = all_done && m.done(); + } + if (all_done && aw_q.empty() && w_bursts.empty() && b_q.empty() && r_q.empty()) { + break; + } + + // ---- masters drive ---- + dut->s_awvalid = 0; dut->s_wvalid = 0; dut->s_arvalid = 0; + dut->s_wlast = 0; dut->s_bready = 0; dut->s_rready = 0; + for (int i = 0; i < NUM_MASTERS; ++i) { + auto& m = ms[i]; + if (!m.aw_offer && m.aw_next < m.num_writes) { + bool ok = (m.style == STYLE_FREE) ? (m.aw_next - m.w_done <= 3) + : (m.aw_next == m.w_done); + m.aw_offer = ok && chance(m.pct); + } + if (m.aw_offer) { + dut->s_awvalid |= (1 << i); + put(dut->s_awaddr, i, 32, waddr(i, m.aw_next)); + put(dut->s_awid, i, 8, m.id); + put(dut->s_awlen, i, 8, m.len[m.aw_next]); + } + if (!m.w_offer && m.w_next < m.num_writes) { + bool ok = (m.style == STYLE_FREE) ? (m.w_next <= m.aw_done + 1) + : (m.w_next < m.aw_done); + m.w_offer = ok && (m.w_beat > 0 || chance(m.pct)); + } + if (m.w_offer) { + dut->s_wvalid |= (1 << i); + put(dut->s_wdata, i, 32, wdata(i, m.w_next, m.w_beat)); + if (m.w_beat == m.len[m.w_next]) { + dut->s_wlast |= (1 << i); + } + } + if (!m.ar_offer && m.ar_next < m.num_reads) { + m.ar_offer = chance(m.pct); + } + if (m.ar_offer) { + dut->s_arvalid |= (1 << i); + put(dut->s_araddr, i, 32, (uint32_t(i) << 28) | (uint32_t(m.ar_next) << 8) | 0x80); + put(dut->s_arid, i, 8, m.id); + put(dut->s_arlen, i, 8, m.ar_next % 3); + } + if (chance(70)) { + dut->s_bready |= (1 << i); + } + if (chance(70)) { + dut->s_rready |= (1 << i); + } + } + + // ---- slave drives ---- + dut->m_awready = chance(sc.aw_pct) && (!sc.w_first || w_started > aw_taken); + dut->m_wready = chance(sc.w_pct); + dut->m_arready = chance(sc.ar_pct); + if (!b_hold && !b_q.empty() && chance(sc.b_pct)) { + b_hold = true; + } + dut->m_bvalid = b_hold; + dut->m_bid = b_hold ? b_q.front() : 0; + if (!r_hold && !r_q.empty() && chance(sc.r_pct)) { + r_hold = true; + } + dut->m_rvalid = r_hold; + if (r_hold) { + dut->m_rdata = r_q.front().addr | uint32_t(r_beat); + dut->m_rid = r_q.front().id; + dut->m_rlast = (r_beat == r_q.front().len); + } + + dut->clk = 0; + dut->eval(); + + // ---- offer stability, slave side ---- + uint64_t aw_v = (uint64_t(dut->m_awaddr) << 16) | (uint64_t(dut->m_awid) << 8) | dut->m_awlen; + uint64_t w_v = (uint64_t(dut->m_wdata) << 1) | dut->m_wlast; + uint64_t ar_v = (uint64_t(dut->m_araddr) << 16) | (uint64_t(dut->m_arid) << 8) | dut->m_arlen; + if (pend_aw && (!dut->m_awvalid || aw_v != pend_aw_v)) { + error("AW offer changed before it was taken", cycle); + } + if (pend_w && (!dut->m_wvalid || w_v != pend_w_v)) { + error("W offer changed before it was taken", cycle); + } + if (pend_ar && (!dut->m_arvalid || ar_v != pend_ar_v)) { + error("AR offer changed before it was taken", cycle); + } + pend_aw = dut->m_awvalid && !dut->m_awready; pend_aw_v = aw_v; + pend_w = dut->m_wvalid && !dut->m_wready; pend_w_v = w_v; + pend_ar = dut->m_arvalid && !dut->m_arready; pend_ar_v = ar_v; + + // ---- slave handshakes ---- + if (dut->m_awvalid && dut->m_awready) { + aw_q.push_back({uint32_t(dut->m_awaddr), uint32_t(dut->m_awid), int(dut->m_awlen)}); + ++aw_taken; + } + if (dut->m_wvalid && dut->m_wready) { + if (w_cur.empty()) { + ++w_started; + } + w_cur.push_back(uint32_t(dut->m_wdata)); + if (dut->m_wlast) { + w_bursts.push_back(w_cur); + w_cur.clear(); + } + } + while (!aw_q.empty() && !w_bursts.empty()) { + auto aw = aw_q.front(); auto& bu = w_bursts.front(); + if (int(bu.size()) != aw.len + 1) { + error("burst length differs from its AW", bu.size(), aw.len); + } + for (size_t b = 0; b < bu.size(); ++b) { + if (bu[b] != (aw.addr | uint32_t(b))) { + error("W data does not belong to the AW at its position", bu[b], aw.addr); + } + } + if (((aw.addr >> 28) & 1) != ((aw.id >> ID_SRC_BIT) & 1)) { + error("AW id source field wrong", aw.addr, aw.id); + } + b_q.push_back(aw.id); + aw_q.pop_front(); w_bursts.pop_front(); + ++writes; + if (last_write_cycle) { + *last_write_cycle = cycle; + } + } + if (dut->m_arvalid && dut->m_arready) { + r_q.push_back({uint32_t(dut->m_araddr), uint32_t(dut->m_arid), int(dut->m_arlen)}); + } + bool b_fire = b_hold && dut->m_bready; + bool r_fire = r_hold && dut->m_rready; + + // ---- master handshakes ---- + for (int i = 0; i < NUM_MASTERS; ++i) { + auto& m = ms[i]; + if (m.aw_offer && get(dut->s_awready, i, 1)) { + m.aw_offer = false; + ++m.aw_next; + ++m.aw_done; + } + if (m.w_offer && get(dut->s_wready, i, 1)) { + m.w_offer = false; + if (m.w_beat == m.len[m.w_next]) { + m.w_beat = 0; + ++m.w_next; + ++m.w_done; + } else { + ++m.w_beat; + } + } + if (m.ar_offer && get(dut->s_arready, i, 1)) { + m.ar_offer = false; + m.r_exp.push_back({(uint32_t(i) << 28) | (uint32_t(m.ar_next) << 8) | 0x80, m.ar_next % 3}); + ++m.ar_next; + } + if (get(dut->s_bvalid, i, 1) && get(dut->s_bready, i, 1)) { + if (get(dut->s_bid, i, 8) != uint32_t(m.id)) { + error("B to the wrong master or id", i, get(dut->s_bid, i, 8)); + } + if (m.b_recv >= m.aw_done) { + error("B before its write was issued", i, m.b_recv); + } + ++m.b_recv; + } + if (get(dut->s_rvalid, i, 1) && get(dut->s_rready, i, 1)) { + if (m.r_exp.empty()) { + error("unexpected R", i); + } else { + auto e = m.r_exp.front(); + if (get(dut->s_rid, i, 8) != uint32_t(m.id)) { + error("R to the wrong master or id", i, get(dut->s_rid, i, 8)); + } + if (get(dut->s_rdata, i, 32) != (e.first | uint32_t(m.r_beat))) { + error("R data wrong", i, get(dut->s_rdata, i, 32)); + } + bool last = (m.r_beat == e.second); + if (get(dut->s_rlast, i, 1) != uint32_t(last)) { + error("RLAST wrong", i, m.r_beat); + } + if (last) { + m.r_exp.pop_front(); + m.r_beat = 0; + ++m.r_done; + } else { + ++m.r_beat; + } + } + } + } + + dut->clk = 1; + dut->eval(); + + if (b_fire) { + b_q.pop_front(); + b_hold = false; + } + if (r_fire) { + if (r_beat == r_q.front().len) { + r_q.pop_front(); + r_beat = 0; + } else { + ++r_beat; + } + r_hold = false; + } + } + *cycles_out = cycle; + if (writes_out) { + *writes_out = writes; + } + for (auto& m : ms) { + if (!m.done()) { + return false; + } + } + return true; +} + +void do_reset() { + dut->reset = 1; + for (int i = 0; i < 4; ++i) { + dut->clk = 0; + dut->eval(); + dut->clk = 1; + dut->eval(); + } + dut->reset = 0; + dut->clk = 0; dut->eval(); +} + +Master make_master(int id, Style style, int pct, int writes, int reads, int max_len) { + Master m; + m.id = id; m.style = style; m.pct = pct; + m.num_writes = writes; m.num_reads = reads; + for (int k = 0; k < writes; ++k) { + m.len.push_back(int(rnd() % (max_len + 1))); + } + return m; +} + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + dut = new VVX_mm_axi_arb_top; + int cycles = 0; + + printf("1. two masters, random traffic and back-pressure\n"); + for (uint32_t seed = 1; seed <= 8; ++seed) { + rng = seed; + errors = 0; + do_reset(); + std::vector ms = { make_master(5, STYLE_FREE, 60, 400, 300, 3), + make_master(6, STYLE_AWFIRST, 60, 400, 300, 3) }; + SlaveCfg sc = {50, 50, 50, 60, 60, false}; + bool ok = run(ms, sc, 200000, &cycles); + char what[96]; + snprintf(what, sizeof(what), "seed %u: all writes and reads complete in order (%d cycles)", seed, cycles); + check(ok && errors == 0, what); + } + + printf("2. a slave that takes the data before the address\n"); + for (uint32_t seed = 11; seed <= 14; ++seed) { + rng = seed; + errors = 0; + do_reset(); + std::vector ms = { make_master(5, STYLE_FREE, 70, 400, 100, 3), + make_master(6, STYLE_FREE, 70, 400, 100, 3) }; + SlaveCfg sc = {60, 60, 60, 70, 70, true}; + bool ok = run(ms, sc, 200000, &cycles); + char what[96]; + snprintf(what, sizeof(what), "seed %u: no deadlock, data still in address order", seed); + check(ok && errors == 0, what); + } + + printf("3. single-beat writes with address and data offered together\n"); + { + rng = 21; + errors = 0; + do_reset(); + std::vector ms = { make_master(5, STYLE_FREE, 100, 1000, 0, 0), + make_master(6, STYLE_FREE, 0, 0, 0, 0) }; + SlaveCfg sc = {100, 100, 100, 100, 100, false}; + int writes = 0, last = 0; + bool ok = run(ms, sc, 20000, &cycles, &writes, &last); + char what[96]; + snprintf(what, sizeof(what), "one write per cycle (%d writes in %d cycles)", writes, last + 1); + check(ok && errors == 0 && last + 1 <= writes + 16, what); + } + + printf("\n%s (%d failure%s)\n", failures ? "FAILED" : "PASSED", + failures, failures == 1 ? "" : "s"); + delete dut; + return failures ? 1 : 0; +} diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index dfe8ab9bb4..5d0921b125 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -141,6 +141,16 @@ RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Verilator from the -I path above. RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv +# One AXI memory master per GPU memory port. The platform top must declare +# its ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(DESTDIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator \ + -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) \ + $(CONFIGS) $(RTL_INCLUDE) $(RTL_PKGS)) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CXXFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic @@ -149,6 +159,7 @@ VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) VL_FLAGS += $(CONFIGS) +VL_FLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) VL_FLAGS += $(RTL_INCLUDE) VL_FLAGS += $(RTL_PKGS) diff --git a/sim/avedsim/vortex_afu_shim.sv b/sim/avedsim/vortex_afu_shim.sv index ecad4997bc..9600a70718 100644 --- a/sim/avedsim/vortex_afu_shim.sv +++ b/sim/avedsim/vortex_afu_shim.sv @@ -20,14 +20,14 @@ module vortex_afu_shim #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, // AXI4 master interface - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -65,7 +65,7 @@ module vortex_afu_shim #( .ap_clk (ap_clk), .ap_rst_n (ap_rst_n), - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_ARGS, MP_COMMA), `AXI_HOST_ARGS, diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index 4e51883d19..408ce494cd 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -138,6 +138,17 @@ RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Verilator from the -I path above. RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv +# One AXI memory master per GPU memory port. The platform top must declare +# its ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(DESTDIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator \ + -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) \ + $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) \ + $(RTL_INCLUDE) $(RTL_PKGS)) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CXXFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic @@ -152,6 +163,7 @@ VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) # VX_CFG_TCU_TYPE_DPI) unmatched. Add just the enum companions gen_config # resolves, so a CONFIGS enum override actually selects its RTL implementation. VL_FLAGS += $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) +VL_FLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) VL_FLAGS += $(RTL_INCLUDE) VL_FLAGS += $(RTL_PKGS) diff --git a/sim/xrtsim/vortex_afu_shim.sv b/sim/xrtsim/vortex_afu_shim.sv index 59ed67bba7..73a8b8f071 100644 --- a/sim/xrtsim/vortex_afu_shim.sv +++ b/sim/xrtsim/vortex_afu_shim.sv @@ -20,14 +20,14 @@ module vortex_afu_shim #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, // AXI4 master interface - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -54,6 +54,9 @@ module vortex_afu_shim #( output wire interrupt `IGNORE_WARNINGS_END ); + `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), .C_S_AXI_CTRL_DATA_WIDTH (C_S_AXI_CTRL_DATA_WIDTH), @@ -65,7 +68,7 @@ module vortex_afu_shim #( .clk (ap_clk), .reset (~ap_rst_n), - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), + `AXI_MEM_ARRAY_ARGS, `AXI_HOST_ARGS, diff --git a/sim/xrtsim/xrt_sim.cpp b/sim/xrtsim/xrt_sim.cpp index af948d92dc..d1d0e349fc 100644 --- a/sim/xrtsim/xrt_sim.cpp +++ b/sim/xrtsim/xrt_sim.cpp @@ -161,7 +161,7 @@ class xrt_sim::Impl { "returned ahead of an older ready one\n", rsp_reorder_seed(), (unsigned long)rsp_reordered_); } - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { delete mem_alloc_[b]; } for (auto ram : ram_) { @@ -217,21 +217,21 @@ class xrt_sim::Impl { sfp_->open(saif_file ? saif_file : "trace.saif"); #endif - // calculate memory bank size - mem_bank_size_ = (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / VX_CFG_PLATFORM_MEMORY_NUM_BANKS; + // Each AXI port owns an equal, contiguous share of device memory. + mem_bank_size_ = (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / AFU_MEM_PORTS; // One memory per AXI port, as on the platform, where each kernel master // is wired to its own memory resource. A request issued on the wrong port // then misses its data instead of finding it in a shared image. - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { ram_[b] = new RAM(0, RAM_PAGE_SIZE); } // initialize AXI memory interfaces - MP_M_AXI_MEM(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + MP_M_AXI_MEM(AFU_MEM_PORTS); // initialize memory allocator - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { mem_alloc_[b] = new MemoryAllocator(0, mem_bank_size_, 4096, 64); } @@ -295,13 +295,13 @@ class xrt_sim::Impl { } int mem_alloc(uint64_t size, uint32_t bank_id, uint64_t* addr) { - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; return mem_alloc_[bank_id]->allocate(size, addr); } int mem_free(uint32_t bank_id, uint64_t addr) { - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; return mem_alloc_[bank_id]->release(addr); } @@ -309,7 +309,7 @@ class xrt_sim::Impl { int mem_write(uint32_t bank_id, uint64_t addr, uint64_t size, const void* data) { HostLock guard(*this); - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; uint64_t base_addr = bank_id * mem_bank_size_ + addr; ram_[bank_id]->write(data, base_addr, size); @@ -319,7 +319,7 @@ class xrt_sim::Impl { int mem_read(uint32_t bank_id, uint64_t addr, uint64_t size, void* data) { HostLock guard(*this); - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; uint64_t base_addr = bank_id * mem_bank_size_ + addr; ram_[bank_id]->read(data, base_addr, size); @@ -328,7 +328,7 @@ class xrt_sim::Impl { int mem_copy(uint32_t bank_id_dest , uint32_t bank_id_src, uint64_t dest_addr, uint64_t src_addr, uint64_t size) { HostLock guard(*this); - if( bank_id_dest >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS || bank_id_src >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if( bank_id_dest >= AFU_MEM_PORTS || bank_id_src >= AFU_MEM_PORTS) return -1; uint64_t dest_base_addr = bank_id_dest * mem_bank_size_ + dest_addr; uint64_t src_base_addr = bank_id_src * mem_bank_size_ + src_addr; @@ -480,10 +480,12 @@ class xrt_sim::Impl { } for (auto& st : m_axi_states_) { + st.aw_pending = {}; st.w_staged.clear(); + st.w_bursts = {}; } - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { std::queue empty; std::swap(dram_queues_[b], empty); } @@ -500,7 +502,7 @@ class xrt_sim::Impl { device_->ap_rst_n = 1; // this AXI device is always ready to accept new requests - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { *m_axi_mem_[b].arready = 1; *m_axi_mem_[b].awready = 1; *m_axi_mem_[b].wready = 1; @@ -532,10 +534,11 @@ class xrt_sim::Impl { dram_sim_.tick(); - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { - if (!dram_queues_[b].empty() && dram_sim_.can_accept(b)) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { + if (!dram_queues_[b].empty() + && dram_sim_.can_accept(dram_channel(dram_queues_[b].front()->addr))) { auto mem_req = dram_queues_[b].front(); - dram_sim_.send_request(b, bank_local_addr(mem_req->addr), mem_req->write, [](void* arg)->bool { + dram_sim_.send_request(dram_channel(mem_req->addr), dram_channel_addr(mem_req->addr), mem_req->write, [](void* arg)->bool { auto orig_req = reinterpret_cast(arg); if (orig_req->ready) { delete orig_req; @@ -553,17 +556,33 @@ class xrt_sim::Impl { #endif } - // Byte address within its bank (DRAM channel) of an AXI bank address: - // VX_mem_to_axi places the bank select above the data-size bits when the - // banks are interleaved. - static uint64_t bank_local_addr(uint64_t axi_addr) { - #if VX_CFG_PLATFORM_MEMORY_INTERLEAVE == 1 - constexpr uint32_t lg2_data_size = log2ceil(VX_CFG_PLATFORM_MEMORY_DATA_SIZE); - constexpr uint32_t bank_sel_bits = log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); - return (axi_addr >> (bank_sel_bits + lg2_data_size)) << lg2_data_size; - #else - return axi_addr; - #endif + // Port addresses are physical: the AFU gives each port its own contiguous + // share of device memory, and the platform lays its memory channels out + // linearly across that space, so a port's traffic stays on its own group of + // channels. + static constexpr uint64_t DRAM_CHANNEL_SIZE = + (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / VX_CFG_PLATFORM_MEMORY_NUM_BANKS; + + static uint32_t dram_channel(uint64_t axi_addr) { + return uint32_t((axi_addr / DRAM_CHANNEL_SIZE) % VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + } + + static uint64_t dram_channel_addr(uint64_t axi_addr) { + return axi_addr % DRAM_CHANNEL_SIZE; + } + + // A port bound to its own channel group cannot reach outside it on the + // platform; a request that would is a routing bug, not a slow access. + void check_port_range(int port, const char* what, uint64_t addr, uint32_t len) { + uint64_t last = addr + uint64_t(len) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + uint64_t lo = uint64_t(port) * mem_bank_size_; + uint64_t hi = lo + mem_bank_size_; + if (addr < lo || last >= hi) { + fprintf(stderr, "[XRTSIM] Error: %s at 0x%lx on m_axi_mem_%d, outside its range " + "[0x%lx, 0x%lx)\n", what, (unsigned long)addr, port, + (unsigned long)lo, (unsigned long)hi); + std::abort(); + } } void eval() { @@ -603,7 +622,7 @@ class xrt_sim::Impl { } void axi_mem_bus_reset() { - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { // read request address *m_axi_mem_[b].arready = 0; @@ -619,24 +638,24 @@ class xrt_sim::Impl { // write response *m_axi_mem_[b].bvalid = 0; - // states - m_axi_states_[b].aw_active = false; } } void axi_host_bus_reset() { // Ready signals stay constant-high (like the m_axi_mem model): a slave // that toggles *ready races the master, which only samples it the cycle - // after it enters its issue state. Our masters keep one outstanding - // read/write at a time, so always-ready is correct. + // after it enters its issue state. Reads and both write channels queue + // what they accept, so always-ready is correct. *m_axi_host_.arready = 1; *m_axi_host_.awready = 1; *m_axi_host_.wready = 1; *m_axi_host_.rvalid = 0; *m_axi_host_.bvalid = 0; host_rd_active_ = false; - host_wr_active_ = false; - host_wr_b_pending_ = false; + host_aw_pending_ = {}; + host_w_staged_.clear(); + host_w_bursts_ = {}; + host_b_pending_ = {}; host_rd_rsp_ready_ = false; host_wr_rsp_ready_ = false; std::queue().swap(host_rd_pending_); @@ -675,7 +694,7 @@ class xrt_sim::Impl { // B channel — retire the write response. if (*m_axi_host_.bvalid && host_wr_rsp_ready_) { *m_axi_host_.bvalid = 0; - host_wr_b_pending_ = false; + host_b_pending_.pop(); } // Accept a read burst. arready stays high (so the master's handshake // always completes the cycle it issues — no ready-toggle race); a request @@ -706,49 +725,57 @@ class xrt_sim::Impl { *m_axi_host_.rresp = 0; *m_axi_host_.rlast = (host_rd_beat_ >= host_rd_len_); } - // Accept a write burst (one outstanding at a time). - if (*m_axi_host_.awvalid && *m_axi_host_.awready - && !host_wr_active_ && !host_wr_b_pending_) { - host_wr_active_ = true; - host_wr_addr_ = uint64_t(*m_axi_host_.awaddr); - host_wr_id_ = *m_axi_host_.awid; - host_wr_beat_ = 0; + // Write channels: the k-th AW pairs with the k-th W burst, and either may + // arrive first (awready/wready stay high), so both sides queue. A write + // lands in host memory once its pair is complete, in AW order. + if (*m_axi_host_.awvalid && *m_axi_host_.awready) { + host_aw_pending_.push({uint64_t(*m_axi_host_.awaddr), uint32_t(*m_axi_host_.awid)}); } - // Accept a write data beat. - if (host_wr_active_ && *m_axi_host_.wvalid && *m_axi_host_.wready) { - uint64_t a = host_wr_addr_ - + uint64_t(host_wr_beat_) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; - auto byteen = *m_axi_host_.wstrb; - auto data = (const uint8_t*)m_axi_host_.wdata->data(); - for (int i = 0; i < VX_CFG_PLATFORM_MEMORY_DATA_SIZE; ++i) { - if ((byteen >> i) & 0x1) - reinterpret_cast(a)[i] = data[i]; - } + if (*m_axi_host_.wvalid && *m_axi_host_.wready) { + w_beat_t beat; + beat.addr = 0; + beat.byteen = uint64_t(*m_axi_host_.wstrb); + std::memcpy(beat.data.data(), m_axi_host_.wdata->data(), + VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + host_w_staged_.emplace_back(beat); if (*m_axi_host_.wlast) { - host_wr_active_ = false; - host_wr_b_pending_ = true; - } else { - ++host_wr_beat_; + host_w_bursts_.push(std::move(host_w_staged_)); + host_w_staged_.clear(); + } + } + while (!host_aw_pending_.empty() && !host_w_bursts_.empty()) { + auto aw = host_aw_pending_.front(); + auto& beats = host_w_bursts_.front(); + for (size_t k = 0; k < beats.size(); ++k) { + // Host memory is plain process memory (see the read path above). + auto dst = reinterpret_cast(aw.addr + k * VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + for (int i = 0; i < VX_CFG_PLATFORM_MEMORY_DATA_SIZE; ++i) { + if ((beats[k].byteen >> i) & 0x1) + dst[i] = beats[k].data[i]; + } } + host_b_pending_.push(aw.tag); + host_aw_pending_.pop(); + host_w_bursts_.pop(); } // Present the write response. - if (host_wr_b_pending_ && !*m_axi_host_.bvalid) { + if (!host_b_pending_.empty() && !*m_axi_host_.bvalid) { *m_axi_host_.bvalid = 1; - *m_axi_host_.bid = host_wr_id_; + *m_axi_host_.bid = host_b_pending_.front(); *m_axi_host_.bresp = 0; } } void axi_mem_bus_eval(bool clk) { if (!clk) { - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { m_axi_states_[b].read_rsp_ready = *m_axi_mem_[b].rready; m_axi_states_[b].write_rsp_ready = *m_axi_mem_[b].bready; } return; } - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { // handle read responses if (*m_axi_mem_[b].rvalid && m_axi_states_[b].read_rsp_ready) { *m_axi_mem_[b].rvalid = 0; @@ -810,6 +837,7 @@ class xrt_sim::Impl { if (*m_axi_mem_[b].arvalid && *m_axi_mem_[b].arready) { uint32_t len = *m_axi_mem_[b].arlen; // beats - 1 uint64_t base = uint64_t(*m_axi_mem_[b].araddr); + check_port_range(b, "read", base, len); for (uint32_t beat = 0; beat <= len; ++beat) { auto mem_req = new mem_req_t(); mem_req->tag = *m_axi_mem_[b].arid; @@ -823,48 +851,51 @@ class xrt_sim::Impl { } } - // handle write address — latch the burst. - if (*m_axi_mem_[b].awvalid && *m_axi_mem_[b].awready - && !m_axi_states_[b].aw_active) { - m_axi_states_[b].aw_active = true; - m_axi_states_[b].aw_addr = uint64_t(*m_axi_mem_[b].awaddr); - m_axi_states_[b].aw_tag = *m_axi_mem_[b].awid; - m_axi_states_[b].aw_beat = 0; + // Write channels. AXI4 lets AW run ahead of its data and data run ahead + // of its address; the k-th AW pairs with the k-th W burst, so both sides + // queue and a write issues once its pair is complete. + auto& st = m_axi_states_[b]; + if (*m_axi_mem_[b].awvalid && *m_axi_mem_[b].awready) { + check_port_range(b, "write", uint64_t(*m_axi_mem_[b].awaddr), *m_axi_mem_[b].awlen); + st.aw_pending.push({uint64_t(*m_axi_mem_[b].awaddr), uint32_t(*m_axi_mem_[b].awid)}); } - // handle write data beats — write one cache line per W beat; the - // last beat (WLAST) queues a single B response for the burst. - if (m_axi_states_[b].aw_active - && *m_axi_mem_[b].wvalid && *m_axi_mem_[b].wready) { - uint64_t byte_addr = m_axi_states_[b].aw_addr - + uint64_t(m_axi_states_[b].aw_beat) - * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; - // The beat is accepted but NOT yet visible in memory: AXI makes a - // write observable only once its B response is returned, and a read to - // the same address issued before that may legally return either value. - // Holding the payload here is what lets a master's own store->fill - // ordering bug appear in simulation instead of only on hardware. + // The beat is accepted but NOT yet visible in memory: AXI makes a write + // observable only once its B response is returned, and a read to the + // same address issued before that may legally return either value. + // Holding the payload here is what lets a master's own store->fill + // ordering bug appear in simulation instead of only on hardware. + if (*m_axi_mem_[b].wvalid && *m_axi_mem_[b].wready) { w_beat_t beat; - beat.addr = byte_addr; + beat.addr = 0; beat.byteen = uint64_t(*m_axi_mem_[b].wstrb); std::memcpy(beat.data.data(), m_axi_mem_[b].wdata->data(), VX_CFG_PLATFORM_MEMORY_DATA_SIZE); - m_axi_states_[b].w_staged.emplace_back(beat); + st.w_staged.emplace_back(beat); if (*m_axi_mem_[b].wlast) { - auto mem_req = new mem_req_t(); - mem_req->tag = m_axi_states_[b].aw_tag; - mem_req->addr = m_axi_states_[b].aw_addr; - mem_req->write = true; - mem_req->ready = false; - mem_req->last = true; - mem_req->w_beats = std::move(m_axi_states_[b].w_staged); - m_axi_states_[b].w_staged.clear(); - pending_mem_reqs_[b].emplace_back(mem_req); - dram_queues_[b].push(mem_req); - m_axi_states_[b].aw_active = false; - } else { - ++m_axi_states_[b].aw_beat; + st.w_bursts.push(std::move(st.w_staged)); + st.w_staged.clear(); + } + } + + // One cache line per W beat (INCR addressing); the burst queues a single + // B response. + while (!st.aw_pending.empty() && !st.w_bursts.empty()) { + auto aw = st.aw_pending.front(); + auto mem_req = new mem_req_t(); + mem_req->tag = aw.tag; + mem_req->addr = aw.addr; + mem_req->write = true; + mem_req->ready = false; + mem_req->last = true; + mem_req->w_beats = std::move(st.w_bursts.front()); + for (size_t i = 0; i < mem_req->w_beats.size(); ++i) { + mem_req->w_beats[i].addr = aw.addr + i * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; } + st.aw_pending.pop(); + st.w_bursts.pop(); + pending_mem_reqs_[b].emplace_back(mem_req); + dram_queues_[b].push(mem_req); } } } @@ -876,15 +907,17 @@ class xrt_sim::Impl { uint64_t byteen; } w_beat_t; + typedef struct { + uint64_t addr; + uint32_t tag; + } aw_req_t; + typedef struct { bool read_rsp_ready; bool write_rsp_ready; - // Write-burst state — latched on AW, advanced one cache line per W beat. - bool aw_active; - uint64_t aw_addr; - uint32_t aw_beat; - uint32_t aw_tag; - std::vector w_staged; // beats accepted so far in this burst + std::queue aw_pending; // accepted AWs awaiting their data + std::vector w_staged; // beats of the burst in progress + std::queue> w_bursts; // whole bursts awaiting their AW } m_axi_state_t; typedef struct { @@ -999,7 +1032,7 @@ class xrt_sim::Impl { } m_axi_mem_t; Vvortex_afu_shim* device_; - RAM* ram_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + RAM* ram_[AFU_MEM_PORTS]; RAM* host_ram_; MemoryAllocator* host_alloc_; DramSim dram_sim_; @@ -1028,17 +1061,17 @@ class xrt_sim::Impl { } }; - std::list pending_mem_reqs_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + std::list pending_mem_reqs_[AFU_MEM_PORTS]; uint32_t rsp_rng_ = rsp_reorder_seed(); uint64_t rsp_reordered_ = 0; - m_axi_mem_t m_axi_mem_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + m_axi_mem_t m_axi_mem_[AFU_MEM_PORTS]; - MemoryAllocator* mem_alloc_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + MemoryAllocator* mem_alloc_[AFU_MEM_PORTS]; - m_axi_state_t m_axi_states_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + m_axi_state_t m_axi_states_[AFU_MEM_PORTS]; - std::queue dram_queues_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + std::queue dram_queues_[AFU_MEM_PORTS]; // m_axi_host AXI slave port + burst state. m_axi_mem_t m_axi_host_; @@ -1048,11 +1081,10 @@ class xrt_sim::Impl { uint32_t host_rd_beat_; uint32_t host_rd_id_; bool host_rd_rsp_ready_; - bool host_wr_active_; - uint64_t host_wr_addr_; - uint32_t host_wr_beat_; - uint32_t host_wr_id_; - bool host_wr_b_pending_; + std::queue host_aw_pending_; + std::vector host_w_staged_; + std::queue> host_w_bursts_; + std::queue host_b_pending_; bool host_wr_rsp_ready_; // Reads accepted while one is already in flight. Several CP masters From 78f2d2ccd7e036771699e10a1069ab9aea1097a2 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:46 -0700 Subject: [PATCH 098/336] simx: model the miss path's buffers and registered stages SimX ran the two-core L2 sgemm shape and the multi-slice fetch shapes several percent off the RTL because it folded buffers and registers on the miss path into bare channel delays. Each is now modeled as the RTL builds it: - The first dcache and icache port also carries the flush injection, whose arbiter buffers two requests and registers them. A registered two-entry TxBuffer stands in for it, replacing the per-port delay knobs on the coalescer and the lane adapter. - A memory response reaches its bank through the port's response queue, when there is one, and a response network registered at every level. Caches now say whether they have that queue (mem_rsp_queue) or a registered boundary that stands for it (mem_rsp_slice), and the flush taps carry what the bank arbiter's response delay does not. - The socket's L1 memory arbiter is a priority arbiter, icache first, registered in both directions. - The LSU's result crosses one register into the lane gather, so its commit queue holds two entries, and its commit-side elasticity is the response buffer plus the result arbiter (four entries, not six). - With two or more operand collectors, their round-robin merge registers its output. - The lmem switch's dcache response buffer holds one entry when the coalescer or a lane adapter feeds it combinationally. - A private dcache's arbiter can keep port 0 out of read merging (merge_port0), which is how the RTL tagged that port. CI: core:parity-sgemm-fw4 (four fetch slices on a four-port icache); parity-sgemm-fw2 tightens to 3%. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 11 ++++++++ sim/simx/cluster.cpp | 1 + sim/simx/core.cpp | 44 +++++++++++++++++++---------- sim/simx/lsu_unit.cpp | 6 ++-- sim/simx/mem/cache.cpp | 46 ++++++++++++++++++++----------- sim/simx/mem/cache.h | 3 ++ sim/simx/mem/local_mem_switch.cpp | 5 ++-- sim/simx/mem/local_mem_switch.h | 3 +- sim/simx/mem/lsu_mem_adapter.cpp | 3 +- sim/simx/mem/lsu_mem_adapter.h | 4 --- sim/simx/mem/mem_coalescer.cpp | 3 +- sim/simx/mem/mem_coalescer.h | 4 --- sim/simx/operands.cpp | 3 +- sim/simx/processor.cpp | 1 + sim/simx/socket.cpp | 28 +++++++++++++++---- sim/simx/types.h | 33 ++++++++++++++++++++++ 16 files changed, 143 insertions(+), 55 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 1ccfd9dcd7..f449e79c87 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -180,10 +180,21 @@ tests: # two-port icache, CTAs as wide as the core, a 16-port dcache. - id: parity-sgemm-fw2 check: model_parity + # Tighter than the default: icache port 0, which carries the flush + # injection, holds four requests, and the shape exercises all of them. + tolerance: 0.03 via: blackbox app: sgemm args: -n128 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 +# Four frontend slices on a four-port icache: every slice but the first fetches +# through two request entries, the first through four. +- id: parity-sgemm-fw4 + check: model_parity + via: blackbox + app: sgemm + args: -n128 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 # Device printf: per-character IO ring stores, each published behind a fence # that drains, flushes and invalidates the data cache. - id: parity-printf diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index a2c83774b9..0f409b8f0e 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -98,6 +98,7 @@ class Cluster::Impl { 0, // read_merge VX_CFG_L2_MSHR_LINES, // mshr_lines CP_ON_L2 != 0, // write_around + VX_CFG_L2_MRSQ_SIZE != 0, // memory-response queue }); diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index bfc91db066..b926fd3070 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -157,10 +157,14 @@ class Core::Impl { VX_CFG_EXT_DXA_ENABLED + VX_CFG_TCU_WGMMA_ENABLED }); - // create lmem switch + // create lmem switch. A data-cache response reaches its response + // arbiter combinationally through the coalescer or the lane adapter, so + // the switch holds it in one entry, the least a channel holds; without + // either, the channel is the cache's own two-entry output buffer. + const bool dc_rsp_comb = dcache_coalescing || (DCACHE_CHANNELS > 1); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { snprintf(sname, 100, "%s-lmem_switch%d", name.c_str(), b); - lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1); + lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1, dc_rsp_comb ? 1 : 2); } // create per-block lmem adapters @@ -188,8 +192,6 @@ class Core::Impl { std::vector*> dc_req_out(VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS); std::vector*> dc_rsp_in(VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS); - // The first data-cache port also carries the cache-flush injection, which - // registers its requests once more. if (dcache_coalescing) { // connect memory coalescer; its memory side drives the dcache // channels directly (combinational lane fan-out/fan-in). @@ -201,7 +203,6 @@ class Core::Impl { dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &mem_coalescers_.at(b)->RspIn.at(c); } } - mem_coalescers_.at(0)->set_port_delay(0, 1); } else { // bypass memory coalescer: per-block lane adapter (channel-fused // pass-through when DCACHE_CHANNELS == 1) @@ -213,9 +214,19 @@ class Core::Impl { dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &lsu_dcache_adapter.at(b)->RspIn.at(c); } } - lsu_dcache_adapter.at(0)->set_port_delay(0, 1); } + // The first data-cache port also carries the cache-flush injection, whose + // arbiter buffers and registers its requests once more. + snprintf(sname, 100, "%s-dcache_flush_buf", name.c_str()); + dcache_flush_buf_ = TxBuffer::Create(sname, 2); + dc_req_out.at(0)->bind(&dcache_flush_buf_->In); + dc_req_out.at(0) = &dcache_flush_buf_->Out; + + // The first icache port does the same for the icache. + snprintf(sname, 100, "%s-icache_flush_buf", name.c_str()); + icache_flush_buf_ = TxBuffer::Create(sname, 2); + #ifdef VX_CFG_VM_ENABLE // Per-core L1 TLB stages: hits translate in place, misses park in // the miss station and resolve through the shared cluster TLB/PTW @@ -247,12 +258,14 @@ class Core::Impl { icache_mmu_->ReqOut.at(f).bind(&simobject_->icache_req_out.at(f)); simobject_->icache_rsp_in.at(f).bind(&icache_mmu_->RspIn.at(f)); } + icache_flush_buf_->Out.bind(&icache_mmu_->ReqIn.at(0)); #else // No-VM: direct passthrough. for (uint32_t p = 0; p < VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS; ++p) { dc_req_out.at(p)->bind(&simobject_->dcache_req_out.at(p)); simobject_->dcache_rsp_in.at(p).bind(dc_rsp_in.at(p)); } + icache_flush_buf_->Out.bind(&simobject_->icache_req_out.at(0)); #endif // initialize execute units @@ -326,9 +339,11 @@ class Core::Impl { for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { auto& queues = commit_queues_.emplace_back(); for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { - // Covers the results still crossing into the queue, so a unit can - // retire one result per cycle. - queues.emplace_back(std::make_unique>(simobject_, 3)); + // A unit's lane gather holds two results. The LSU's crosses one + // register into it; the others cross two and take a third slot so + // that they still retire one result per cycle. + uint32_t capacity = (fu == (uint32_t)FUType::LSU) ? 2 : 3; + queues.emplace_back(std::make_unique>(simobject_, capacity)); } } @@ -456,11 +471,12 @@ class Core::Impl { // The MMU translates and forwards to simobject_->icache_req_out // downstream (driven by SimChannel binding in the constructor). auto& icache_rsp = icache_mmu_->RspOut.at(f); - auto& icache_req = icache_mmu_->ReqIn.at(f); + auto& icache_port = icache_mmu_->ReqIn.at(f); #else auto& icache_rsp = simobject_->icache_rsp_in.at(f); - auto& icache_req = simobject_->icache_req_out.at(f); + auto& icache_port = simobject_->icache_req_out.at(f); #endif + auto& icache_req = (f == 0) ? icache_flush_buf_->In : icache_port; // handle icache response — extract instruction word from the TLM line payload if (!icache_rsp.empty()){ @@ -532,9 +548,7 @@ class Core::Impl { mem_req.tag = tag; mem_req.hart_id = trace->cid; mem_req.uuid = trace->uuid; - // The first icache port also carries the cache-flush injection, which - // registers its requests once more. - if (icache_req.try_send(mem_req, (f == 0) ? 2 : 1)) { + if (icache_req.try_send(mem_req, 1)) { DT(3, simobject_->name() << " icache-req: addr=0x" << std::hex << mem_req.addr << ", tag=0x" << mem_req.tag << std::dec << ", " << *trace); #ifdef VX_CFG_EXT_C_ENABLE decompressor->commit_request(from_refetch); @@ -1192,6 +1206,8 @@ class Core::Impl { LocalMem::Ptr local_mem_; std::vector lmem_switch_; std::vector mem_coalescers_; + TxBuffer::Ptr dcache_flush_buf_; + TxBuffer::Ptr icache_flush_buf_; std::vector>> fetch_latch_; std::vector>> decode_latch_; diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index eb0b032917..7a887721f7 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -73,10 +73,8 @@ Instr::Ptr LsuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { } LsuUnit::LsuUnit(const SimContext& ctx, const char* name, Core* core) - // Commit-side elasticity of 6 beats: the writeback path buffers a whole - // coalesced fragment burst, so a denied commit grant queues beats there - // instead of throttling the memory response stream. - : FuncUnit(ctx, name, core, 6) + // the response buffer and the result arbiter, two entries each + : FuncUnit(ctx, name, core, 4) #ifdef TCU_META_ENABLE , TcuReqIn(TCU_AGU_PORTS, this) , TcuRspOut(TCU_AGU_PORTS, this) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 67a8e88131..ebeabee7ef 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2262,7 +2262,7 @@ class CachePortArb : public SimObject { std::vector> IoOut; // non-cacheable requests, per port CachePortArb(const SimContext& ctx, const char* name, const params_t& params, - uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group) + uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group, bool merge_port0) : SimObject(ctx, name) , ReqIn(num_inputs, this) , ReqOut(num_banks, this) @@ -2273,6 +2273,7 @@ class CachePortArb : public SimObject { , delay_((num_inputs > 2) ? 1 : 0) , levels_(arb_tree_levels(num_inputs)) , merge_group_(merge_group) + , merge_port0_(merge_port0) { __assert(merge_group_ <= 1 || (merge_group_ <= 64 && ispow2(merge_group_) && (num_inputs % merge_group_) == 0), "a read-merge group must be a power of two of at most 64 dividing the ports"); @@ -2425,7 +2426,7 @@ class CachePortArb : public SimObject { auto& req_i = ReqIn.at(i).peek(); uint32_t base = (i / merge_group_) * merge_group_; for (uint32_t j = base; j < i; ++j) { - if (followers.at(j) || !this->is_plain_read(j)) { + if (followers.at(j) || !this->is_plain_read(j) || (j == 0 && !merge_port0_)) { continue; } auto& req_j = ReqIn.at(j).peek(); @@ -2531,6 +2532,7 @@ class CachePortArb : public SimObject { std::vector>> tree_stage_; // [level][bank][slice] std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; + bool merge_port0_; uint64_t collisions_ = 0; friend class SimObject; @@ -2656,7 +2658,7 @@ class CacheRspMcast : public SimObject { // Sits on one memory port's cache-side response path, ahead of the response // network. While the port's flush token is out the cache is quiescent, so the // port's next response is the token's answer and stops here; every other -// response passes in the same cycle. +// response passes on, crossing the port's registered response stages. class FlushTap : public SimObject { public: using Ptr = std::shared_ptr; @@ -2664,10 +2666,11 @@ class FlushTap : public SimObject { SimChannel RspIn; SimChannel RspOut; - FlushTap(const SimContext& ctx, const char* name) + FlushTap(const SimContext& ctx, const char* name, uint32_t delay) : SimObject(ctx, name) , RspIn(this) , RspOut(this) + , delay_(delay) {} // the port's flush token has been sent @@ -2699,12 +2702,13 @@ class FlushTap : public SimObject { armed_ = false; answered_ = true; RspIn.pop(); - } else if (RspOut.try_send(RspIn.peek(), 0)) { + } else if (RspOut.try_send(RspIn.peek(), delay_)) { RspIn.pop(); } } private: + uint32_t delay_; bool armed_ = false; bool answered_ = false; @@ -2739,12 +2743,30 @@ class Cache::Impl { nc_mem_arbs_.at(i) = MemArbiter::Create(sname, ArbiterType::Priority, 2, 1); } + // Bank memory arbiter delays. Each level of the memory-side request + // network beyond the first is a registered stage; the request path + // buffers the levels the arbiter's own register does not cover in + // mem_req_net_. + uint32_t mem_arb_delay = ((num_banks != config_.mem_ports) && (num_banks > 2 || config_.mem_ports > 2)) ? 1 : 0; + uint32_t mem_net_delay = omega_stages(num_banks, config_.mem_ports) - 1; + uint32_t mem_net_extra = (mem_net_delay > mem_arb_delay) ? (mem_net_delay - mem_arb_delay) : 0; + uint32_t mem_arb_rsp_delay = (num_banks != config_.mem_ports) ? (mem_arb_delay + mem_net_extra) : 0; + __assert(num_banks >= config_.mem_ports, "a cache needs a bank per memory port"); + + // A memory response reaches its bank through the port's response queue, + // when there is one, and a response network registered at every level. + // The taps carry what the bank arbiter's response delay does not. + uint32_t mem_rsp_regs = omega_stages(config_.mem_ports, num_banks) + (config_.mem_rsp_queue ? 1 : 0); + uint32_t mem_rsp_credit = mem_arb_rsp_delay + (config_.mem_rsp_slice ? 1 : 0); + __assert(mem_rsp_regs >= mem_rsp_credit, "memory response path shorter than its arbiter"); + uint32_t tap_delay = mem_rsp_regs - mem_rsp_credit; + // flush-token taps, ticking after the arbiters that feed them and before // the response network they feed flush_taps_.resize(config_.mem_ports); for (uint32_t i = 0; i < config_.mem_ports; ++i) { snprintf(sname, 100, "%s-flush_tap%d", simobject->name().c_str(), i); - flush_taps_.at(i) = FlushTap::Create(sname); + flush_taps_.at(i) = FlushTap::Create(sname, tap_delay); nc_mem_arbs_.at(i)->RspOut.at(0).bind(&flush_taps_.at(i)->RspIn); } @@ -2754,17 +2776,9 @@ class Cache::Impl { simobject->mem_rsp_in.at(i).bind(&nc_mem_arbs_.at(i)->RspIn.at(0)); } - // Create bank's memory arbiter. Each level of the memory-side networks - // beyond the first is a registered stage; the request path buffers the - // levels the arbiter's own register does not cover in mem_req_net_, the - // response path in the bank's memory-response queue. snprintf(sname, 100, "%s-mem_arb", simobject->name().c_str()); - uint32_t mem_arb_delay = ((num_banks != config_.mem_ports) && (num_banks > 2 || config_.mem_ports > 2)) ? 1 : 0; - uint32_t mem_net_delay = omega_stages(num_banks, config_.mem_ports) - 1; - uint32_t mem_net_extra = (mem_net_delay > mem_arb_delay) ? (mem_net_delay - mem_arb_delay) : 0; - __assert(num_banks >= config_.mem_ports, "a cache needs a bank per memory port"); auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports, - mem_arb_delay + mem_net_extra, mem_arb_delay + mem_net_extra); + mem_arb_delay + mem_net_extra, mem_arb_rsp_delay); // Connect bank's memory arbiter to non-cacheable arbiter's input port 0 if (mem_net_extra != 0) { @@ -2788,7 +2802,7 @@ class Cache::Impl { // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); uint32_t merge_group = std::min(config_.read_merge, config_.num_inputs); - port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group); + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group, config_.merge_port0); for (uint32_t i = 0; i < config_.num_inputs; ++i) { simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); } diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index ae8b783dd9..70adc78ba5 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -51,6 +51,9 @@ class Cache : public SimObject { uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) uint16_t mshr_lines = 0; // MSHR distinct {line, sector} entries (0: one per request) bool write_around = false; // a no-allocate write miss goes to memory (write-back only) + bool mem_rsp_queue = false; // memory responses enter through a response queue + bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register + bool merge_port0 = true; // port 0's reads match other ports' tags for read merging }; struct PerfStats { diff --git a/sim/simx/mem/local_mem_switch.cpp b/sim/simx/mem/local_mem_switch.cpp index 7cc253575e..a1e64b6c10 100644 --- a/sim/simx/mem/local_mem_switch.cpp +++ b/sim/simx/mem/local_mem_switch.cpp @@ -18,7 +18,8 @@ using namespace vortex; LocalMemSwitch::LocalMemSwitch( const SimContext& ctx, const char* name, - uint32_t delay + uint32_t delay, + uint32_t dc_rsp_size ) : SimObject(ctx, name) // Single-entry ingress: the switch accepts a new request only as its // current one drains, so upstream issue feels downstream cadence @@ -28,7 +29,7 @@ LocalMemSwitch::LocalMemSwitch( , ReqOutLmem(this) , RspInLmem(this) , ReqOutDC(this) - , RspInDC(this) + , RspInDC(this, dc_rsp_size) , delay_(delay) {} diff --git a/sim/simx/mem/local_mem_switch.h b/sim/simx/mem/local_mem_switch.h index 6edbc3f1cb..d765adaead 100644 --- a/sim/simx/mem/local_mem_switch.h +++ b/sim/simx/mem/local_mem_switch.h @@ -33,7 +33,8 @@ class LocalMemSwitch : public SimObject { LocalMemSwitch( const SimContext& ctx, const char* name, - uint32_t delay + uint32_t delay, + uint32_t dc_rsp_size = 2 ); protected: diff --git a/sim/simx/mem/lsu_mem_adapter.cpp b/sim/simx/mem/lsu_mem_adapter.cpp index 60076bb902..e9d5592a8e 100644 --- a/sim/simx/mem/lsu_mem_adapter.cpp +++ b/sim/simx/mem/lsu_mem_adapter.cpp @@ -28,7 +28,6 @@ LsuMemAdapter::LsuMemAdapter( , RspIn(num_inputs, this) , delay_(delay) , rsp_delay_(rsp_delay) - , port_delay_(num_inputs, 0) , pending_mask_(num_inputs) { assert(num_inputs > 0); @@ -146,7 +145,7 @@ void LsuMemAdapter::on_tick() { out_req.flags.local = (t == AddrType::Shared); } - if (ReqOut.at(i).try_send(out_req, delay_ + port_delay_.at(i))) { + if (ReqOut.at(i).try_send(out_req, delay_)) { DT(4, this->name() << " req" << i << ": " << out_req); pending_mask_.reset(i); } diff --git a/sim/simx/mem/lsu_mem_adapter.h b/sim/simx/mem/lsu_mem_adapter.h index 6f6499716f..1864926527 100644 --- a/sim/simx/mem/lsu_mem_adapter.h +++ b/sim/simx/mem/lsu_mem_adapter.h @@ -42,9 +42,6 @@ class LsuMemAdapter : public SimObject { ) : LsuMemAdapter(ctx, name, num_inputs, 0) {} - // Extra request-side registers on one output port. - void set_port_delay(uint32_t port, uint32_t delay) { port_delay_.at(port) = delay; } - protected: void on_reset(); void on_tick(); @@ -52,7 +49,6 @@ class LsuMemAdapter : public SimObject { private: uint32_t delay_; uint32_t rsp_delay_; - std::vector port_delay_; BitVector pending_mask_; friend class SimObject; diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index bf25a8f05e..4940c9c5db 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -39,7 +39,6 @@ MemCoalescer::MemCoalescer( , sent_mask_(input_size) , line_size_(line_size) , delay_(delay) - , port_delay_(output_size, 0) {} void MemCoalescer::on_reset() { @@ -300,7 +299,7 @@ void MemCoalescer::flush_out_round() { if (!out_round_.lanes.test(o)) { continue; } - if (ReqOut.at(o).try_send(out_round_.reqs.at(o), delay_ + port_delay_.at(o))) { + if (ReqOut.at(o).try_send(out_round_.reqs.at(o), delay_)) { out_round_.lanes.reset(o); } } diff --git a/sim/simx/mem/mem_coalescer.h b/sim/simx/mem/mem_coalescer.h index 3cc6d72c8d..cd9df21223 100644 --- a/sim/simx/mem/mem_coalescer.h +++ b/sim/simx/mem/mem_coalescer.h @@ -46,9 +46,6 @@ class MemCoalescer : public SimObject { const PerfStats& perf_stats() const; - // Extra request-side registers on one output port. - void set_port_delay(uint32_t port, uint32_t delay) { port_delay_.at(port) = delay; } - protected: void on_reset(); void on_tick(); @@ -78,7 +75,6 @@ class MemCoalescer : public SimObject { out_round_t out_round_; uint32_t line_size_; uint32_t delay_; - std::vector port_delay_; PerfStats perf_stats_; friend class SimObject; diff --git a/sim/simx/operands.cpp b/sim/simx/operands.cpp index aaaa018d71..d7230c31ca 100644 --- a/sim/simx/operands.cpp +++ b/sim/simx/operands.cpp @@ -108,8 +108,9 @@ Operands::Operands(const SimContext &ctx, const char* name, Core* core) } if (VX_CFG_NUM_OPCS >= 2) { + // the collectors' round-robin merge registers its output snprintf(sname, 100, "%s-rsp_arb", name); - rsp_arb_ = TraceArbiter::Create(sname, ArbiterType::RoundRobin, VX_CFG_NUM_OPCS, 1); + rsp_arb_ = TraceArbiter::Create(sname, ArbiterType::RoundRobin, VX_CFG_NUM_OPCS, 1, 1); for (uint32_t i = 0; i < VX_CFG_NUM_OPCS; ++i) { opc_units_.at(i)->Output.bind(&rsp_arb_->Inputs.at(i)); } diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index d0a8a8c6e8..b015d165ca 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -120,6 +120,7 @@ ProcessorImpl::ProcessorImpl() 0, // read_merge VX_CFG_L3_MSHR_LINES, // mshr_lines CP_ON_L3 != 0, // write_around + VX_CFG_L3_MRSQ_SIZE != 0, // memory-response queue } ); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index b44fb21845..d2df85bb0c 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -75,6 +75,11 @@ class Socket::Impl { false, // shared_llc false, // lazy_fetch true, // core-response register + 0, // read_merge + 0, // mshr_lines + false, // write_around + VX_CFG_ICACHE_MRSQ_SIZE != 0, // memory-response queue + true, // the socket's registered memory port }); snprintf(sname, 100, "%s-dcache", name.c_str()); @@ -103,6 +108,12 @@ class Socket::Impl { true, // core-response register (VX_CFG_DCACHE_READ_MERGE != 0) ? DCACHE_CHANNELS : 0, // read_merge VX_CFG_DCACHE_MSHR_LINES, // mshr_lines + false, // write_around + VX_CFG_DCACHE_MRSQ_SIZE != 0, // memory-response queue + true, // the socket's registered memory port + // A private cache takes the flush injection on port 0 alone, and its + // arbiter re-tags that port's requests apart from the others. + VX_CFG_SOCKET_SIZE != VX_CFG_NUM_DCACHES, // port 0 merges reads }); #ifdef VX_CFG_EXT_TEX_ENABLE @@ -115,7 +126,7 @@ class Socket::Impl { constexpr uint32_t kTcacheWordSize = 4; // = TCACHE_WORD_SIZE constexpr uint32_t kTcacheNumReqs = VX_CFG_TCACHE_NUM_BANKS; constexpr uint32_t kTcacheMemPorts = 1; // = TCACHE_MEM_PORTS - auto tcache = Cache::Create(sname, Cache::Config{ + Cache::Config tcache_config{ false, // bypass log2ceil(VX_CFG_TCACHE_SIZE), // C log2ceil(kTcacheLineSize), // L @@ -132,7 +143,10 @@ class Socket::Impl { 2, // pipeline latency uint8_t(VX_CFG_L2_REPL_POLICY), // replacement policy (use L2 policy as default) false, // is_llc (TCACHE is auxiliary, not LLC) - }); + }; + tcache_config.mem_rsp_queue = (VX_CFG_TCACHE_MRSQ_SIZE != 0); + tcache_config.mem_rsp_slice = true; + auto tcache = Cache::Create(sname, tcache_config); tcache_ = tcache; // tex_core ↔ tcache (per-port). @@ -155,7 +169,7 @@ class Socket::Impl { // VX_CFG_RTCACHE_NUM_BANKS banks. constexpr uint32_t kRtcacheNumInputs = VX_CFG_NUM_RTU_BLOCKS; constexpr uint32_t kRtcacheMemPorts = 1; - auto rtcache = Cache::Create(sname, Cache::Config{ + Cache::Config rtcache_config{ false, // bypass log2ceil(VX_CFG_RTCACHE_SIZE), // C log2ceil(kRtcacheLineSize), // L @@ -172,7 +186,10 @@ class Socket::Impl { 2, // pipeline latency uint8_t(VX_CFG_L2_REPL_POLICY), // replacement policy false, // is_llc - }); + }; + rtcache_config.mem_rsp_queue = (VX_CFG_RTCACHE_MRSQ_SIZE != 0); + rtcache_config.mem_rsp_slice = true; + auto rtcache = Cache::Create(sname, rtcache_config); rtcache_ = rtcache; // RtuCore ↔ rtcache (per memory port). @@ -254,7 +271,8 @@ class Socket::Impl { // connect l1 caches to outgoing memory interfaces for (uint32_t i = 0; i < VX_CFG_L1_MEM_PORTS; ++i) { snprintf(sname, 100, "%s-l1_arb%d", name.c_str(), i); - auto l1_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, 2 * overlap, overlap); + // icache first; the arbiter registers both directions + auto l1_arb = MemArbiter::Create(sname, ArbiterType::Priority, 2 * overlap, overlap, 1, 1); if (i < overlap) { icaches_->mem_req_out.at(i).bind(&l1_arb->ReqIn.at(i)); diff --git a/sim/simx/types.h b/sim/simx/types.h index 595891d401..a7eda35ac2 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -1428,6 +1428,39 @@ class TFifo : public SimObject> { /////////////////////////////////////////////////////////////////////////////// +// A registered elastic buffer: In holds its entries, and the head moves on +// through Out one register later, one per cycle. +template +class TxBuffer : public SimObject> { +public: + using Ptr = std::shared_ptr>; + + SimChannel In; + SimChannel Out; + + TxBuffer(const SimContext& ctx, const char* name, uint32_t size) + : SimObject>(ctx, name) + , In(this, size) + , Out(this) + {} + +protected: + void on_reset() {} + + void on_tick() { + if (!In.empty() && Out.try_send(In.peek(), 1)) { + In.pop(); + } + if (In.size() == 0) { + this->tick_sleep(); + } + } + + friend class SimObject>; +}; + +/////////////////////////////////////////////////////////////////////////////// + template class TxArbiter : public SimObject> { public: From 05d1237ff27039bcd77ed6bc69803d5e36827f58 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:46 -0700 Subject: [PATCH 099/336] cache: refetch a fully written lazy-fetch sector without a memory read Under lazy fetch a write miss allocates its sector without reading memory and leaves it partial. A later load refetches the sector from memory and merges the stored bytes over it. When stores have already covered every byte, nothing from that read survives the merge, yet it still cost a DRAM round trip. That cost is why the GPU profiles keep L2_LAZY_FETCH off. A refetch root (a partial-miss read, or a parked chain head) whose sector's dirty mask is full now loops back through the lazy queue as a data-less refetch instead of going to memory. It does so only while the queue has the room a new request is admitted with; otherwise it reads memory as before. The RTL (VX_cache_bank) and SimX change together. In SimX a refetch fill now learns whether its sector is still resident at its tag lookup, the tick after it enters, as in the RTL. lazy_fetch -f stores whole lines, or lines with one word per sector left unwritten, loads them back, and compares the two runs' memory reads: every fully written sector must save exactly one read. A chained mode loads while the stores' allocation is still pending, and a mixed mode interleaves whole and partial lines. It needs the memory counters (--perf). CI: cache:lazy_fetch-3 (lazy-fetch dcache) and cache:lazy_fetch-4 (write-back lazy-fetch L2 shared by four cores), on simx and rtlsim; core:parity-lazy-fetch-full. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 24 ++++ ci/testcases/core.yaml | 12 ++ hw/rtl/cache/VX_cache_bank.sv | 72 +++++++---- sim/simx/mem/cache.cpp | 112 ++++++++++++----- tests/regression/lazy_fetch/common.h | 49 +++++++- tests/regression/lazy_fetch/kernel.cpp | 65 ++++++++++ tests/regression/lazy_fetch/main.cpp | 160 ++++++++++++++++++++++++- 7 files changed, 433 insertions(+), 61 deletions(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 9b52dab9ee..d2e9377c61 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -338,6 +338,30 @@ tests: shape: cores: 4 l2cache: true +# A sector every store has covered reads no memory when a load refetches it. +# The app compares the memory reads of fully written lines against lines with +# a word per sector unwritten, so it runs with the memory counters. +- id: lazy_fetch-3 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + args: -f -n1 + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 + flags: --perf=7 +- id: lazy_fetch-4 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + args: -f -n1 + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_LAZY_FETCH=1 + flags: --perf=7 + shape: + cores: 4 + l2cache: true # Both sectors of one sectored L2 line miss together from different warps and # cores: each must get its own fill, never chain behind the other sector's. # Run with the address held per request and with the MSHR line table. diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index f449e79c87..676531abed 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -147,6 +147,18 @@ tests: via: blackbox app: lazy_fetch configs: -DVX_CFG_DCACHE_LAZY_FETCH=1 +# Fully written sectors on a lazy-fetch write-back L2 shared by 4 cores: +# partial-miss loads and parked chain heads refetch them through the loopback. +- id: parity-lazy-fetch-full + check: model_parity + via: blackbox + app: lazy_fetch + args: -f -n1 + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_LAZY_FETCH=1 + flags: --perf=7 + shape: + cores: 4 + l2cache: true # FPU + LSU locality, multi-warp reuse (~1.29M cycles on the default shape). - id: parity-sgemm check: model_parity diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index ae219feccb..d71663087e 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -164,6 +164,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( logic is_refetch; // fill whose root refetches a partial sector logic is_retry; // refetch re-issued after its sector was evicted logic is_park; // replay parked on a partial sector: issues its refetch + logic is_loop; // fill taken from the loopback queue } lazy_req_t; data_t sel_req, st0, dat_in, stD; // request + fill line: sel -> S0 -> stD @@ -374,6 +375,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // refetches. A refetch merges into its resident sector one stage late, at // stC; one whose sector was evicted installs nothing and retries as a // plain fill (nothing can dirty an absent sector, so its data is current). + // A refetch whose sector is already fully written has nothing to read: it + // loops back as a data-less refetch instead of going to memory. wire lazyq_valid = ~lazyq_empty; wire [MSHR_ADDR_WIDTH-1:0] lazyq_id; wire lazy_sel = ~mem_rsp_valid && lazyq_valid; @@ -523,10 +526,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // tag_matches is computed at S0; left 0 here (overridden at the data bubble). end - assign lz_sel.is_lazy = fill_enable && lazy_sel; - assign lz_sel.is_refetch = fill_enable && ~lazy_sel && mshr_fill_refetch; + assign lz_sel.is_lazy = fill_enable && lazy_sel && ~mshr_fill_refetch; + assign lz_sel.is_refetch = fill_enable && mshr_fill_refetch; assign lz_sel.is_retry = fill_enable && ~lazy_sel && mshr_fill_retry; assign lz_sel.is_park = replay_enable && park_sel; + assign lz_sel.is_loop = fill_enable && lazy_sel; // UUID extraction (debug + MSHR ordering): per stage, from the carried tag. wire [`UP(UUID_WIDTH)-1:0] req_uuid_sel, req_uuid_st0, req_uuid_st1, req_uuid_stc; @@ -780,9 +784,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // refetch fill writes no data at stD. wire fill_merge_st0 = refetch_first_st0 && fill_resident_st0; wire fill_nodata_st0 = lazy_st0 || refetch_first_st0; - wire fill_merge_stc, fill_nodata_std, lazy_std; + wire fill_merge_stc, fill_nodata_std, loop_std, partial_miss_stc; if (LAZY_FETCH != 0) begin : g_lazy_pipe - wire fill_merge_st1; + wire fill_merge_st1, partial_miss_st1; VX_pipe_register #( .DATAW ($bits(lazy_req_t)), .RESETW ($bits(lazy_req_t)) @@ -794,25 +798,25 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_out (lz_st0) ); VX_pipe_register #( - .DATAW ($bits(lazy_req_t) + 1), - .RESETW ($bits(lazy_req_t) + 1) + .DATAW ($bits(lazy_req_t) + 2), + .RESETW ($bits(lazy_req_t) + 2) ) reg_lz_s1 ( .clk (clk), .reset (reset), .enable (~pipe_stall), - .data_in ({lz_st0, fill_merge_st0}), - .data_out ({lz_st1, fill_merge_st1}) + .data_in ({lz_st0, fill_merge_st0, partial_miss_st0}), + .data_out ({lz_st1, fill_merge_st1, partial_miss_st1}) ); VX_pipe_register #( - .DATAW ($bits(lazy_req_t) + 1), - .RESETW ($bits(lazy_req_t) + 1), + .DATAW ($bits(lazy_req_t) + 2), + .RESETW ($bits(lazy_req_t) + 2), .DEPTH (PIPE_EX) ) reg_lz_cmt ( .clk (clk), .reset (reset), .enable (~pipe_stall), - .data_in ({lz_st1, fill_merge_st1}), - .data_out ({lz_stc, fill_merge_stc}) + .data_in ({lz_st1, fill_merge_st1, partial_miss_st1}), + .data_out ({lz_stc, fill_merge_stc, partial_miss_stc}) ); VX_pipe_register #( .DATAW (2), @@ -822,8 +826,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .enable (~pipe_stall), - .data_in ({lazy_st0, fill_nodata_st0}), - .data_out ({lazy_std, fill_nodata_std}) + .data_in ({lz_st0.is_loop, fill_nodata_st0}), + .data_out ({loop_std, fill_nodata_std}) ); end else begin : g_no_lazy_pipe assign lz_st0 = '0; @@ -831,8 +835,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign lz_stc = '0; assign fill_merge_stc = 1'b0; assign fill_nodata_std = 1'b0; - assign lazy_std = 1'b0; - `UNUSED_VAR ({lz_sel, fill_merge_st0, fill_nodata_st0}) + assign loop_std = 1'b0; + assign partial_miss_stc = 1'b0; + `UNUSED_VAR ({lz_sel, fill_merge_st0, fill_nodata_st0, lz_st0.is_loop, lz_stc.is_loop}) end // a passthru-AMO replay carries its result word instead of an installed @@ -884,7 +889,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_in (fbuf_addr_r), .data_out (fill_addr_std) ); - `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill && ~lazy_std) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) + `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill && ~loop_std) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) `endif // The fill sector, replicated across the line so each sector's slices see @@ -1351,6 +1356,14 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // a write-miss root under lazy fetch loops its fill back instead. wire lazy_alloc_stc = (LAZY_FETCH != 0) && do_write_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending && ~write_around_stc; + // the mask of the request's sector, as of every older store + wire [SECTOR_SIZE-1:0] sector_byteen_stc = evict_byteen_stc[sector_idx_stc * SECTOR_SIZE +: SECTOR_SIZE]; + // a refetch root (a partial-miss read or a parked head) whose sector every + // store already covers loops back instead of reading memory, while the + // loopback has the room a new request is admitted with: a parked head is + // not admitted against it, and memory always has room for the refetch. + wire refetch_loop_stc = (LAZY_FETCH != 0) && do_read_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending + && partial_miss_stc && (& sector_byteen_stc) && ~lazyq_alm_full; wire do_fill_or_flush_stc = stC.req.valid && is_fill_or_flush_stc; // a sector refill keeps the resident line (no eviction), so never writes back. wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill && ~refetch_first_stc; @@ -1455,7 +1468,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // on a dirty eviction. The two are mutually exclusive (a writeback is a // fill/flush commit; a fill request is a creq commit). wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending - && ~lazy_alloc_stc && ~write_around_stc && ~pipe_stall; + && ~lazy_alloc_stc && ~refetch_loop_stc && ~write_around_stc && ~pipe_stall; wire wa_req_push = write_around_stc && ~pipe_stall; assign mreq_queue_push = fill_req_push || wa_req_push || wb_beat_accept; assign mreq_queue_addr = is_wb_beat ? wb_mreq_addr : rd_mreq_addr; @@ -1539,7 +1552,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); assign mem_req_valid = ~mreq_queue_empty; - // Lazy-fetch loopback: write-miss roots awaiting their data-less fill. + // Lazy-fetch loopback: write-miss roots awaiting their data-less fill, and + // refetch roots whose sector needs no data. // Sized like the memory request queue's reservation: a core request is // admitted only while every request already in the pipe still fits. if (LAZY_FETCH != 0) begin : g_lazyq @@ -1552,7 +1566,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ) lazy_queue ( .clk (clk), .reset (reset), - .push (lazy_alloc_stc && ~pipe_stall), + .push ((lazy_alloc_stc || refetch_loop_stc) && ~pipe_stall), .pop (lazy_fire), .data_in (stC.req.mshr_id), .data_out (lazyq_id), @@ -1572,7 +1586,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( chain_partial_r <= 1'b0; chain_retry_r <= 1'b0; end else if (fill_fire) begin - chain_partial_r <= lazy_fire; + chain_partial_r <= lazy_fire && ~mshr_fill_refetch; chain_retry_r <= 1'b0; end else if (refetch_hold && ~pipe_stall) begin chain_partial_r <= fill_evicted_st0; @@ -1589,14 +1603,20 @@ module VX_cache_bank import VX_gpu_pkg::*; #( refetch_inflight_r <= 1'b0; end else if (mem_rsp_fire && mshr_fill_refetch) begin refetch_inflight_r <= 1'b1; - end else if (stC.req.valid && stC.req.is_fill && refetch_stc && ~pipe_stall) begin + end else if (stC.req.valid && stC.req.is_fill && refetch_stc && ~lz_stc.is_loop && ~pipe_stall) begin refetch_inflight_r <= 1'b0; end end assign refetch_inflight = refetch_inflight_r; + // A memory refetch keeps its sector staged until its stC merge. + `RUNTIME_ASSERT (~(stC.req.valid && stC.req.is_fill && refetch_stc && ~lz_stc.is_loop) || refetch_inflight_r, + ("%t: %s memory refetch reached its merge unstaged: addr=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) - `RUNTIME_ASSERT (~(lazy_alloc_stc && ~pipe_stall && lazyq_full && ~lazy_fire), + `RUNTIME_ASSERT (~((lazy_alloc_stc || refetch_loop_stc) && ~pipe_stall && lazyq_full && ~lazy_fire), ("%t: %s lazy queue overflow", $time, INSTANCE_ID)) + // A looped-back refetch carries no data: its merge must find every byte written. + `RUNTIME_ASSERT (~(do_merge_stc && lz_stc.is_loop) || (& sector_byteen_stc), + ("%t: %s looped-back refetch of an incompletely written sector: addr=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) `RUNTIME_ASSERT (~(park_fire && fwd_fire), ("%t: %s park and forward in one cycle", $time, INSTANCE_ID)) `RUNTIME_ASSERT (~replay_valid || (mshr_dequeue_reads == (~replay_rw || replay_amo.amo_valid)), ("%t: %s chain head read flag disagrees with its payload", $time, INSTANCE_ID)) @@ -1614,7 +1634,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign chain_partial = 1'b0; assign chain_retry = 1'b0; assign refetch_inflight = 1'b0; - `UNUSED_VAR ({lazy_alloc_stc, lazy_fire, refetch_hold, fill_resident_st0, fill_evicted_st0, partial_match_st0}) + `UNUSED_VAR ({lazy_alloc_stc, refetch_loop_stc, sector_byteen_stc, lazy_fire, refetch_hold, fill_resident_st0, fill_evicted_st0, partial_match_st0}) end `UNUSED_VAR (do_lookup_st0) @@ -1652,8 +1672,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( mem_rsp_full_addr, mem_rsp_id, mshr_fill_refetch, mshr_fill_retry, mem_rsp_data, req_uuid_sel)) end if (lazy_fire) begin - `TRACE(2, ("%t: %s lazy-fill: addr=0x%0h, mshr_id=%0d\n", $time, INSTANCE_ID, - mem_rsp_full_addr, lazyq_id)) + `TRACE(2, ("%t: %s lazy-fill: addr=0x%0h, mshr_id=%0d, refetch=%b\n", $time, INSTANCE_ID, + mem_rsp_full_addr, lazyq_id, mshr_fill_refetch)) end if (park_fire) begin `TRACE(2, ("%t: %s mshr-park: addr=0x%0h, tag=0x%0h, req_idx=%0d (#%0d)\n", $time, INSTANCE_ID, diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index ebeabee7ef..feae96a223 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -220,6 +220,12 @@ static inline bool sector_hit(const sector_t& sec, bool needs_data) { return sec.valid && !(needs_data && sec.partial); } +// Every byte of the sector has been stored: a refetch of it reads nothing. +static inline bool sector_written(const sector_t& sec) { + constexpr uint64_t full_mask = (VX_CFG_MEM_BLOCK_SIZE >= 64) ? ~0ull : ((1ull << VX_CFG_MEM_BLOCK_SIZE) - 1); + return sec.dirty_mask == full_mask; +} + struct line_t { uint64_t tag; uint32_t lru_ctr; @@ -904,6 +910,7 @@ class CacheBank : public SimObject { chain_tag_ = 0; chain_sector_ = 0; refetch_hold_ = false; + refetch_hold_retry_ = false; flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -977,15 +984,26 @@ class CacheBank : public SimObject { // // At most one input fires per tick. All inputs flow through pipe_req_. void processInputs(bool fwd_consumed) { + // A refetch fill learns whether its sector is still resident at its tag + // lookup, the tick after it enters: nothing enters behind it meanwhile, + // and its chain then replays, or parks again if the sector was evicted. + if (refetch_hold_) { + refetch_hold_ = false; + bool evicted = false; + if (!refetch_hold_retry_) { + const auto &set = sets_.at(chain_set_); + int resident_id = set.find_resident(chain_tag_); + evicted = (resident_id == -1) || !set.lines.at(resident_id).sectors.at(chain_sector_).valid; + } + chain_partial_ = evicted; + chain_retry_ = evicted; + return; + } + // 1) replay: one chain entry leaves per tick — when the fill-forward // drain completed the head this tick, this slot goes to new core // requests instead if (mshr_.has_ready_reqs() && !fwd_consumed) { - // A refetch fill learns whether its sector is still resident when it - // installs; its chain waits for that answer. - if (refetch_hold_) { - return; - } // A chain whose sector is partial cannot serve a load or AMO: park at // that head and refetch the sector; the rest of the chain waits. if (chain_partial_) { @@ -1072,7 +1090,8 @@ class CacheBank : public SimObject { mshr_.replay(mshr_id); this->setChain(set_id, addr_tag, sector_id, false); if (bank_req.refetch) { - refetch_hold_ = true; + refetch_hold_ = true; + refetch_hold_retry_ = bank_req.refetch_retry; } else { fwd_active_ = true; fwd_set_ = set_id; @@ -1086,8 +1105,9 @@ class CacheBank : public SimObject { return; } } else if (!lazy_q_->empty() && !fwd_active_) { - // Lazy allocation of a write-miss root: a data-less fill through the - // same path, below memory fills in priority. + // A data-less fill through the same path, below memory fills in + // priority: the allocation of a write-miss root, or the refetch of a + // sector every store has covered. uint32_t mshr_id = lazy_q_->peek(); const auto &root_peek = mshr_.peek(mshr_id); const uint32_t set_id = root_peek.set_id; @@ -1106,9 +1126,17 @@ class CacheBank : public SimObject { bank_req.uuid = root_peek.bank_req.uuid; bank_req.mshr_id = mshr_id; bank_req.nodata = true; + bank_req.refetch = root_peek.refetch; + bank_req.refetch_retry = root_peek.refetch_retry; pipe_req_->push(bank_req); mshr_.replay(mshr_id); - this->setChain(set_id, addr_tag, sector_id, true); + // A looped-back refetch completes a fully written sector: like a + // memory refetch, its chain waits for it to install. + this->setChain(set_id, addr_tag, sector_id, !bank_req.refetch); + if (bank_req.refetch) { + refetch_hold_ = true; + refetch_hold_retry_ = bank_req.refetch_retry; + } DT(3, this->name() << " lazy-fill: " << bank_req); lazy_q_->pop(); return; @@ -1188,7 +1216,7 @@ class CacheBank : public SimObject { } // Any admitted request may become a write-miss root: the lazy queue // keeps room for every request already in the pipe. - if (config_.lazy_fetch && lazy_q_->size() >= lazyq_size(config_) - config_.latency) { + if (config_.lazy_fetch && !this->lazyRoom()) { return; } @@ -1322,16 +1350,17 @@ class CacheBank : public SimObject { const bool refetch_first = bank_req.refetch && !bank_req.refetch_retry; assert(!sector_resident || refetch_first); if (refetch_first) { - chain_partial_ = !sector_resident; - chain_retry_ = !sector_resident; - refetch_hold_ = false; if (sector_resident) { // Memory supplies the bytes no store wrote; the sector stays dirty // in exactly the stored bytes, which are not in memory yet. - auto &sec = set.lines.at(resident_id).sectors.at(sector_id); - auto written = sec.data; - sec.data = bank_req.data; - sector_merge(sec, written, sec.dirty_mask); + auto &sec = set.lines.at(resident_id).sectors.at(sector_id); + if (bank_req.nodata) { + assert(sector_written(sec) && "looped-back refetch of an incompletely written sector"); + } else { + auto written = sec.data; + sec.data = bank_req.data; + sector_merge(sec, written, sec.dirty_mask); + } sec.partial = false; } return true; @@ -1405,11 +1434,6 @@ class CacheBank : public SimObject { sec.dirty_mask = 0; sec.data = bank_req.data; } - if (bank_req.refetch) { - chain_partial_ = false; - chain_retry_ = false; - refetch_hold_ = false; - } return true; } @@ -1427,6 +1451,13 @@ class CacheBank : public SimObject { return 1u << log2ceil(config.latency + 1); } + // The loopback still fits every request in the pipe: a new request may be + // admitted, and a refetch with no data to read may loop back; one that + // cannot reads memory instead. + bool lazyRoom() const { + return lazy_q_->size() < lazyq_size(config_) - config_.latency; + } + // Fill-forward drain: complete the oldest ready replay directly when it is // a plain read on the armed chain — no pipeline traversal, and the input // slot stays free for new requests. The first write/AMO head closes the @@ -1705,12 +1736,24 @@ class CacheBank : public SimObject { } break; case bank_req_t::Park: { - if (this->mem_req_out.full()) { - return; // stall - } uint32_t set_id = params_.addr_set_id(bank_req.addr); uint64_t addr_tag = params_.addr_tag(bank_req.addr); uint32_t sector_id = params_.addr_sector_id(bank_req.addr); + auto &set = sets_.at(set_id); + // A parked head needs memory room whichever way its refetch goes. + if (this->mem_req_out.full()) { + return; // stall + } + int resident_id = set.find_resident(addr_tag); + if (resident_id != -1) { + const auto &sec = set.lines.at(resident_id).sectors.at(sector_id); + if (sec.valid && sec.partial && sector_written(sec) && this->lazyRoom()) { + lazy_q_->push(bank_req.mshr_id); + DT(3, this->name() << " refetch-loop: " << bank_req); + pipe_req_->pop(); + return; + } + } MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); fill.tag = bank_req.mshr_id; @@ -1992,21 +2035,25 @@ class CacheBank : public SimObject { // fill for this fresh miss. bool fill_pending = mshr_.has_pending_fill(set_id, addr_tag, sector_id, -1); // Lazy fetch: a write-miss root allocates through the loopback - // instead of fetching the sector. + // instead of fetching the sector, and so does the refetch of a + // partial sector every store has already covered. const bool lazy_alloc = config_.lazy_fetch && bank_req.write && !fill_pending; - if (lazy_alloc && lazy_q_->full()) { + const bool refetch_loop = partial_resident && !fill_pending + && sector_written(set.lines.at(present_id).sectors.at(sector_id)) + && this->lazyRoom(); + if ((lazy_alloc || refetch_loop) && lazy_q_->full()) { return; } - if (!fill_pending && !lazy_alloc && this->mem_req_out.full()) { + if (!fill_pending && !lazy_alloc && !refetch_loop && this->mem_req_out.full()) { return; } assert(!mshr_.full()); int mshr_id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id, partial_resident); DT(3, this->name() << " mshr-enqueue: " << bank_req); - if (lazy_alloc) { + if (lazy_alloc || refetch_loop) { lazy_q_->push(mshr_id); - DT(3, this->name() << " lazy-alloc: " << bank_req); + DT(3, this->name() << (lazy_alloc ? " lazy-alloc: " : " refetch-loop: ") << bank_req); } else if (!fill_pending) { MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); @@ -2186,7 +2233,7 @@ class CacheBank : public SimObject { return n; } TFifo::Ptr pipe_req_; - TFifo::Ptr lazy_q_; // write-miss roots awaiting their data-less fill + TFifo::Ptr lazy_q_; // roots awaiting a data-less fill: write misses and written refetches Cache::PerfStats perf_stats_; @@ -2210,7 +2257,8 @@ class CacheBank : public SimObject { uint32_t chain_set_; uint64_t chain_tag_; uint32_t chain_sector_; - bool refetch_hold_; // a refetch fill has not installed yet: its chain waits + bool refetch_hold_; // a refetch fill is at its tag lookup: its chain waits + bool refetch_hold_retry_; // Flush walk state. bool flushing_; diff --git a/tests/regression/lazy_fetch/common.h b/tests/regression/lazy_fetch/common.h index 308f768331..5b6758319b 100644 --- a/tests/regression/lazy_fetch/common.h +++ b/tests/regression/lazy_fetch/common.h @@ -12,6 +12,19 @@ // Same-set lines: one per thread and round, a page apart, so they compete for // one cache set and a line being refetched is often evicted before its fill. #define SET_STRIDE_WORDS 1024 +// A whole cache line of the largest configured line size: two 64-byte +// sectors, stored before any of it is loaded. +#define FULL_LINE_WORDS 32 +#define SECTOR_WORDS 16 +// Lines a flood launch reads one word of, each a memory block apart. +#define FLOOD_LINE_BYTES 64 + +// How a phase-2 launch stores and loads its lines. +enum { + FULL_SEPARATE = 0, // each thread stores its lines, then loads them all back + FULL_CHAIN = 1, // a block's threads store a sector together, load it at once, store, load + FULL_MIXED = 2 // as FULL_SEPARATE, with every odd line left partial +}; enum { CASE_HALF = 0, // store half the line, then load all of it (same thread) @@ -31,8 +44,13 @@ typedef struct { uint32_t warp_size; uint32_t cross_warp; // blocks hold an even number of warps uint32_t amo; // the device implements the A extension - uint32_t phase; // 0: exercise the cases, 1: re-check every region + uint32_t phase; // 0: exercise the cases, 1: re-check every region, 2: full lines, 3: flood uint32_t rotate; // phase 1: check the regions of thread (gid + rotate) + uint64_t full_addr; + uint32_t full_lines; // phase 2: lines per thread (per warp in FULL_CHAIN); 3: lines it reads + uint32_t full_mode; // phase 2: FULL_* + uint32_t full_skip; // phase 2: leave each sector's last word unwritten + uint32_t num_blocks; } kernel_arg_t; // Pre-pattern the host writes over the whole buffer. @@ -52,6 +70,35 @@ static inline uint32_t store_value(uint32_t region, uint32_t w) { return 0xC0000000u | (region << 4) | w; } +// Word w of a line that a phase-2 launch leaves unwritten. +static inline bool full_skipped(uint32_t line, uint32_t w, const kernel_arg_t* arg) { + bool partial = (arg->full_mode == FULL_MIXED) ? ((line & 1) != 0) : (arg->full_skip != 0); + return partial && (w % SECTOR_WORDS) == (SECTOR_WORDS - 1); +} + +static inline uint32_t full_value(uint32_t line, uint32_t w) { + return 0x50000000u | (line << 5) | w; +} + +// The second store a FULL_CHAIN thread makes to its word of each sector. +static inline uint32_t full_value2(uint32_t line, uint32_t w) { + return 0x60000000u | (line << 5) | w; +} + +// Word `word_idx` of a phase-2 buffer once its launch has run; the host +// pre-patterns the buffer. +static inline uint32_t full_expected(uint32_t word_idx, const kernel_arg_t* arg) { + uint32_t line = word_idx / FULL_LINE_WORDS; + uint32_t w = word_idx % FULL_LINE_WORDS; + if (full_skipped(line, w, arg)) { + return pattern_word(word_idx); + } + if (arg->full_mode == FULL_CHAIN) { + return full_value2(line, w); + } + return full_value(line, w); +} + static inline uint32_t region_index(uint32_t round, uint32_t num_threads, uint32_t gid, uint32_t c) { return (round * num_threads + gid) * NUM_CASES + c; } diff --git a/tests/regression/lazy_fetch/kernel.cpp b/tests/regression/lazy_fetch/kernel.cpp index 5f0dc5e4cd..a704245ec5 100644 --- a/tests/regression/lazy_fetch/kernel.cpp +++ b/tests/regression/lazy_fetch/kernel.cpp @@ -129,6 +129,71 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t errors = 0; auto err = reinterpret_cast(arg->err_addr); + if (arg->phase == 3) { + // Read fresh lines from memory, one word each. + auto base = reinterpret_cast(arg->full_addr); + uint32_t sum = 0; + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_threads + gid; + sum += base[line * (FLOOD_LINE_BYTES / sizeof(uint32_t))]; + } + asm volatile ("" :: "r"(sum)); + err[gid] = 0; + return; + } + + if (arg->phase == 2 && arg->full_mode == FULL_CHAIN) { + // The first SECTOR_WORDS threads of a block store one word each of a + // sector, every warp at once after a barrier, and load it right away: the + // loads reach the cache while the stores' allocation is still pending. + // The second store lands while that chain is being refetched. + auto base = reinterpret_cast(arg->full_addr); + uint32_t t = threadIdx.x; + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_blocks + blockIdx.x; + for (uint32_t s = 0; s < FULL_LINE_WORDS; s += SECTOR_WORDS) { + uint32_t idx = line * FULL_LINE_WORDS + s + t; + bool skip = full_skipped(line, s + t, arg); + // a skipped word is stored to a scratch word: no branch delays the load + volatile uint32_t* dst = skip ? &err[arg->num_threads + gid] : &base[idx]; + __syncthreads(); + if (t < SECTOR_WORDS) { + *dst = full_value(line, s + t); + uint32_t first = base[idx]; + *dst = full_value2(line, s + t); + uint32_t second = base[idx]; + errors += (first != (skip ? pattern_word(idx) : full_value(line, s + t))) ? 1 : 0; + errors += (second != full_expected(idx, arg)) ? 1 : 0; + } + } + } + err[gid] = errors; + return; + } + + if (arg->phase == 2) { + // Store this thread's lines, all of each sector or all but one word, + // then load them all back. + auto base = reinterpret_cast(arg->full_addr); + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_threads + gid; + for (uint32_t w = 0; w < FULL_LINE_WORDS; ++w) { + if (!full_skipped(line, w, arg)) { + base[line * FULL_LINE_WORDS + w] = full_value(line, w); + } + } + } + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_threads + gid; + for (uint32_t w = 0; w < FULL_LINE_WORDS; ++w) { + uint32_t idx = line * FULL_LINE_WORDS + w; + errors += (base[idx] != full_expected(idx, arg)) ? 1 : 0; + } + } + err[gid] = errors; + return; + } + if (arg->phase != 0) { // Every region again, from another thread (another core when there are // several), after the launch boundary flushed the partial lines. diff --git a/tests/regression/lazy_fetch/main.cpp b/tests/regression/lazy_fetch/main.cpp index 60734b7711..413e17bf48 100644 --- a/tests/regression/lazy_fetch/main.cpp +++ b/tests/regression/lazy_fetch/main.cpp @@ -2,7 +2,9 @@ #include #include #include +#include #include +#include #include "common.h" #define RT_CHECK(_expr) \ @@ -19,23 +21,25 @@ const char* kernel_file = "kernel.vxbin"; uint32_t rounds = 16; +bool full_mode = false; vx_device_h device = nullptr; vx_buffer_h buf_buffer = nullptr; vx_buffer_h err_buffer = nullptr; vx_buffer_h set_buffer = nullptr; +std::vector full_buffers; vx_queue_h queue = nullptr; vx_module_h module_ = nullptr; vx_kernel_h kernel = nullptr; static void show_usage() { std::cout << "Vortex Test." << std::endl; - std::cout << "Usage: [-k: kernel] [-n rounds] [-h: help]" << std::endl; + std::cout << "Usage: [-k: kernel] [-n rounds] [-f: fully written lines read no memory] [-h: help]" << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "n:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:k:fh")) != -1) { switch (c) { case 'n': rounds = atoi(optarg); @@ -43,6 +47,9 @@ static void parse_args(int argc, char **argv) { case 'k': kernel_file = optarg; break; + case 'f': + full_mode = true; + break; case 'h': show_usage(); exit(0); @@ -59,6 +66,9 @@ void cleanup() { if (buf_buffer) vx_buffer_release(buf_buffer); if (err_buffer) vx_buffer_release(err_buffer); if (set_buffer) vx_buffer_release(set_buffer); + for (auto b : full_buffers) { + vx_buffer_release(b); + } if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); @@ -86,6 +96,139 @@ static int launch(const kernel_arg_t& arg, uint32_t grid, uint32_t block) { return ret; } +static int mem_reads(uint64_t* value) { + return vx_device_mpm_query(device, VX_DCR_MPM_CLASS_MEM, VX_CSR_MPM_MEM_READS, 0, value); +} + +static vx_buffer_h new_buffer(uint32_t size, uint64_t* addr) { + vx_buffer_h buffer = nullptr; + RT_CHECK(vx_buffer_create(device, size, VX_MEM_READ_WRITE, &buffer)); + full_buffers.push_back(buffer); + RT_CHECK(vx_buffer_address(buffer, addr)); + return buffer; +} + +static void read_buffer(void* dst, vx_buffer_h buffer, uint32_t size) { + vx_event_h read_ev = nullptr; + RT_CHECK(vx_enqueue_read(queue, dst, buffer, 0, size, 0, nullptr, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); +} + +static int check_errors(uint32_t num_threads) { + std::vector h_err(num_threads); + read_buffer(h_err.data(), err_buffer, num_threads * sizeof(uint32_t)); + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::cout << "error: thread " << t << " read back 0x" << std::hex << h_err[t] << std::dec + << " wrong words of its lines" << std::endl; + } + ++errors; + } + } + return errors; +} + +static const char* full_name(const kernel_arg_t& arg) { + switch (arg.full_mode) { + case FULL_CHAIN: return arg.full_skip ? "chained, partially written" : "chained, fully written"; + case FULL_MIXED: return "mixed"; + default: return arg.full_skip ? "partially written" : "fully written"; + } +} + +// One phase-2 launch over a fresh pre-patterned buffer, after a launch that +// reads far more. Drivers differ on whether the memory read counter restarts +// at each launch; after the flood, a counter that restarted reads below its +// previous value. Returns the launch's memory reads. +static int64_t run_lines(kernel_arg_t arg, uint32_t grid, uint32_t block, uint32_t lines, int* errors) { + const uint32_t flood_lines = 4096; + uint32_t line_threads = (arg.full_mode == FULL_CHAIN) ? arg.num_blocks : arg.num_threads; + uint32_t words = lines * line_threads * FULL_LINE_WORDS; + vx_buffer_h buffer = new_buffer(words * sizeof(uint32_t), &arg.full_addr); + std::vector h_buf(words); + for (uint32_t i = 0; i < words; ++i) { + h_buf[i] = pattern_word(i); + } + RT_CHECK(vx_enqueue_write(queue, buffer, 0, h_buf.data(), words * sizeof(uint32_t), 0, nullptr, nullptr)); + + kernel_arg_t flood = arg; + flood.phase = 3; + flood.full_lines = flood_lines / arg.num_threads; + new_buffer(flood_lines * FLOOD_LINE_BYTES, &flood.full_addr); + RT_CHECK(launch(flood, grid, block)); + *errors += check_errors(arg.num_threads); + + arg.phase = 2; + arg.full_lines = lines; + uint64_t before = 0, after = 0; + RT_CHECK(mem_reads(&before)); + RT_CHECK(launch(arg, grid, block)); + RT_CHECK(mem_reads(&after)); + int64_t reads = (after < before) ? (int64_t)after : (int64_t)(after - before); + *errors += check_errors(arg.num_threads); + + read_buffer(h_buf.data(), buffer, words * sizeof(uint32_t)); + int bad = 0; + for (uint32_t i = 0; i < words; ++i) { + uint32_t ref = full_expected(i, &arg); + if (h_buf[i] != ref) { + if (bad < 16) { + std::cout << "error: " << full_name(arg) << " word " << i << std::hex << ": actual 0x" << h_buf[i] + << ", expected 0x" << ref << std::dec << std::endl; + } + ++bad; + } + } + *errors += bad; + std::cout << full_name(arg) << " lines, memory reads: " << reads << std::endl; + return reads; +} + +// Lines stored whole and then loaded must not be read from memory. Each pair +// of launches stores and loads the same lines, one leaving a word of each +// sector unwritten: every partial sector is read from memory once, a whole +// one never, and the rest of the two launches' traffic is alike. +static int run_full(kernel_arg_t arg, uint32_t grid, uint32_t block) { + // few enough lines that a 16 KB cache keeps them resident + const uint32_t lines = 2; + int errors = 0; + for (uint32_t mode : {FULL_SEPARATE, FULL_CHAIN}) { + // a chained sector takes a whole block, which needs a word per thread + if (mode == FULL_CHAIN && block < SECTOR_WORDS) { + std::cout << "chained lines skipped: " << block << " threads per block" << std::endl; + continue; + } + arg.full_mode = mode; + uint32_t line_threads = (mode == FULL_CHAIN) ? arg.num_blocks : arg.num_threads; + // a chained sector belongs to a block: give the blocks more lines + uint32_t mode_lines = (mode == FULL_CHAIN) ? lines * 16 : lines; + arg.full_skip = 1; + int64_t partial = run_lines(arg, grid, block, mode_lines, &errors); + arg.full_skip = 0; + int64_t full = run_lines(arg, grid, block, mode_lines, &errors); + if (partial == 0) { + std::cout << "error: memory read counter inactive (run with --perf)" << std::endl; + return errors + 1; + } + int64_t sectors = (int64_t)mode_lines * line_threads * (FULL_LINE_WORDS / SECTOR_WORDS); + int64_t saved = partial - full; + // A sector evicted before its load is read back from memory either way. + int64_t slack = std::max(2, sectors / 16); + std::cout << full_name(arg) << " sectors: " << sectors << ", memory reads saved: " << saved << std::endl; + if (saved < sectors - slack || saved > sectors + slack) { + std::cout << "error: fully written sectors were read from memory" << std::endl; + ++errors; + } + } + // Loads of whole sectors loop back while partial ones read memory. + arg.full_mode = FULL_MIXED; + run_lines(arg, grid, block, lines, &errors); + return errors; +} + int main(int argc, char *argv[]) { parse_args(argc, argv); if (rounds == 0) { @@ -121,6 +264,7 @@ int main(int argc, char *argv[]) { arg.warp_size = num_threads; arg.cross_warp = (num_warps % 2) == 0; arg.amo = (isa_flags & VX_ISA_STD_A) != 0; + arg.num_blocks = grid; std::cout << "threads: " << total_threads << ", amo: " << arg.amo << ", regions: " << num_regions << ", buffer size: " << buf_size << " bytes" << std::endl; @@ -150,6 +294,18 @@ int main(int argc, char *argv[]) { RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + if (full_mode) { + int errors = run_full(arg, grid, block); + cleanup(); + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; + } + std::cout << "launch kernel (exercise)" << std::endl; arg.phase = 0; RT_CHECK(launch(arg, grid, block)); From 73db36c78ef81cec89a73f16dff49526b813c813 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:47 -0700 Subject: [PATCH 100/336] simx: model KMU, CTA retirement, barrier and collector timing SimX ran ahead of, or behind, the RTL at several control points. Each now follows the RTL's registers: - The KMU's lane slice registers a CTA into the core, so the KMU sends it without a delay of its own. The dispatcher returns the lane credit as the CTA is admitted, and the lane buffer offers the freed entry the next cycle. - A CTA's last warp exit reaches the slot table registered and updates it by read-modify-write. The slot frees in the fourth cycle, and the write holds admission off for that cycle. Before, it freed at once. - A transaction-barrier completion crosses the completion queue and a register before it applies (two cycles, not one). - Fetch picks warps by an ibuffer-full flag registered at the end of the previous cycle. - FIFO replacement: the pointer names the victim even when another way is free, and it advances on every installed fill, sector refills included. - With several operand collectors, the scoreboard's skid buffer feeds them through a demux. Only its head is offered, to its own collector, and an entry reaches the head the cycle after the one ahead leaves. Co-Authored-By: Claude Opus 5.5 --- sim/simx/barrier_unit.cpp | 7 ++++--- sim/simx/core.cpp | 15 +++++++++++---- sim/simx/cta_dispatcher.cpp | 27 ++++++++++++++++++++++----- sim/simx/cta_dispatcher.h | 4 +++- sim/simx/kmu/kmu.cpp | 3 ++- sim/simx/mem/cache.cpp | 13 ++++++++++--- sim/simx/operands.cpp | 37 +++++++++++++++++++++++++++---------- sim/simx/operands.h | 4 ++++ 8 files changed, 83 insertions(+), 27 deletions(-) diff --git a/sim/simx/barrier_unit.cpp b/sim/simx/barrier_unit.cpp index 6538afc2a7..968541ad16 100644 --- a/sim/simx/barrier_unit.cpp +++ b/sim/simx/barrier_unit.cpp @@ -24,10 +24,11 @@ using namespace vortex; // Cycles from an SFU barrier op to its effect in the barrier store: the store // is read a cycle before an arrival applies, and with several SFU blocks the // request is also registered out of its block. A transaction-barrier -// completion is registered once before it applies; a global-barrier release -// applies as it arrives. A release then reaches the warp scheduler registered. +// completion crosses the completion queue and a register before it applies; a +// global-barrier release applies as it arrives. A release then reaches the +// warp scheduler registered. static constexpr uint32_t kOpApplyLag = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 2 : 1; -static constexpr uint32_t kEventApplyLag = 1; +static constexpr uint32_t kEventApplyLag = 2; BarrierUnit::BarrierUnit(const SimContext& ctx, const char* name, Core* core, Scheduler* scheduler) : SimObject(ctx, name) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index b926fd3070..213e9634e9 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -72,6 +72,7 @@ class Core::Impl { , fu_full_(VX_CFG_ISSUE_WIDTH, BitVector<>((uint32_t)FUType::Count, 0)) , fu_full_seen_(VX_CFG_ISSUE_WIDTH, BitVector<>((uint32_t)FUType::Count, 0)) , ibuf_inflight_(VX_CFG_NUM_WARPS, 0) + , ibuf_full_r_(VX_CFG_NUM_WARPS, false) , last_issue_(VX_CFG_NUM_WARPS, 0) , staged_since_(VX_CFG_NUM_WARPS, 0) , head_staged_(VX_CFG_NUM_WARPS, false) @@ -370,6 +371,7 @@ class Core::Impl { pending_ifetches_ = 0; std::fill(ibuf_inflight_.begin(), ibuf_inflight_.end(), 0); + std::fill(ibuf_full_r_.begin(), ibuf_full_r_.end(), false); // perf_stats_ deliberately survives: these are the MPM counters, which // accumulate from device reset. A kernel launch re-enters reset() here but @@ -407,11 +409,12 @@ class Core::Impl { scheduler_->schedule_begin(); // Each frontend slice picks among its own warps whose ibuffer is not full; a - // buffer is full at exactly IBUF_SIZE entries. With an L1, once every warp's - // buffer is full any ready warp may fetch, its instruction waiting at decode. + // buffer is full at exactly IBUF_SIZE entries, a flag registered at the end + // of the previous cycle. With an L1, once every warp's buffer is full any + // ready warp may fetch, its instruction waiting at decode. bool all_full = (VX_CFG_ICACHE_ENABLED || VX_CFG_DCACHE_ENABLED); for (uint32_t w = 0; w < VX_CFG_NUM_WARPS && all_full; ++w) { - all_full = (ibuf_inflight_.at(w) == VX_CFG_IBUF_SIZE); + all_full = ibuf_full_r_.at(w); } // the cycle is idle when no slice has a warp to fetch, its latch free or not bool any_cand = false; @@ -419,7 +422,7 @@ class Core::Impl { auto& fetch_latch = fetch_latch_.at(f); WarpMask warp_mask; for (uint32_t w = f; w < VX_CFG_NUM_WARPS; w += VX_CFG_FETCH_WIDTH) { - if (all_full || ibuf_inflight_.at(w) != VX_CFG_IBUF_SIZE) { + if (all_full || !ibuf_full_r_.at(w)) { warp_mask.set(w); } } @@ -445,6 +448,9 @@ class Core::Impl { if (!any_cand) { ++perf_stats_.sched_idle; } + for (uint32_t w = 0; w < VX_CFG_NUM_WARPS; ++w) { + ibuf_full_r_.at(w) = (ibuf_inflight_.at(w) == VX_CFG_IBUF_SIZE); + } scheduler_->schedule_end(); } @@ -1232,6 +1238,7 @@ class Core::Impl { std::vector> fu_full_seen_; // [iw] that flag one cycle later, as issue sees it std::vector ibuf_inflight_; + std::vector ibuf_full_r_; std::vector last_issue_; // [wid] cycle of the warp's last issue std::vector staged_since_; // [wid] cycle the last issued candidate was staged std::vector head_staged_; // [wid] the buffer head has left for staging diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index 67c82eca72..39502a56e5 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -64,6 +64,7 @@ void CtaDispatcher::on_reset() { cta_wids_.reset(); tail_slot_ = 0; init_wait_ = 0; + retiring_.clear(); for (uint32_t i = 0; i < num_warps_; ++i) { slot_rem_warps_[i] = 0; wid_to_slot_[i] = num_warps_; // invalid @@ -103,6 +104,22 @@ uint32_t CtaDispatcher::usable_slots(uint32_t stride) const { } uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs) { + // A retirement is registered, reads its slot's count from the table and + // writes it back: it updates the slot in its fourth cycle, and the write + // holds admission off for that cycle. + uint64_t now = SimPlatform::instance().cycles(); + bool retire_write = false; + for (size_t i = 0; i < retiring_.size();) { + if (retiring_[i].second > now) { + ++i; + continue; + } + assert(slot_rem_warps_[retiring_[i].first] > 0); + --slot_rem_warps_[retiring_[i].first]; + retire_write |= (retiring_[i].second == now); + retiring_.erase(retiring_.begin() + i); + } + if (cta_done_) { cta_done_ = false; has_cta_ = false; @@ -113,7 +130,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w // endpoint holds the one CTA committed to this core. It is popped only on // admission — an unadmitted CTA keeps the lane's credit, so the KMU // cannot over-commit work to a busy core. - if (bus_in.empty()) + if (bus_in.empty() || retire_write) return 0; const kmu_req_t& next = bus_in.peek(); @@ -172,7 +189,8 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w // the admission is reported back to the KMU's launch accounting. cta_ = next; bus_in.pop(); - credit_out.send(1, 1); + // the lane's buffer offers the freed entry to the KMU next cycle + credit_out.send(1, 0); cta_size_ = (cta_.block_size + num_threads_ - 1) / num_threads_; rank_ = 0; @@ -255,10 +273,9 @@ void CtaDispatcher::warp_done(uint32_t wid) { uint32_t slot = wid_to_slot_[wid]; if (slot >= num_warps_) return; // not a CTA-dispatcher warp wid_to_slot_[wid] = num_warps_; // clear assignment - assert(slot_rem_warps_[slot] > 0); // When the last warp of a CTA exits, its slot (and the LMEM region at - // slot × stride) frees immediately for reuse — no in-order reclaim. - --slot_rem_warps_[slot]; + // slot × stride) frees for reuse as the retirement lands — no in-order reclaim. + retiring_.emplace_back(slot, SimPlatform::instance().cycles() + 3); } bool CtaDispatcher::slot_needs_init(uint32_t wid, const ThreadMask& tmask) const { diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 314fd1d4ce..cf1d649b82 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -106,8 +106,10 @@ class CtaDispatcher : public SimObject { // within a kernel. The occupancy bound floor(capacity/stride) (capped at // NUM_WARPS) replaces the byte ring's free_size accounting, so no wrap or // padding arithmetic is needed. Slots are allocated round-robin via a tail - // pointer and freed immediately when a CTA's last warp exits. + // pointer and freed when the retirement of a CTA's last warp lands. std::vector slot_rem_warps_; // remaining active warps; slot free when 0 + // Warp retirements on their way to the slot table: {slot, cycle it lands}. + std::vector> retiring_; // Reverse lookup: wid → slot index std::vector wid_to_slot_; uint32_t tail_slot_; // next slot to allocate (round-robin over usable slots) diff --git a/sim/simx/kmu/kmu.cpp b/sim/simx/kmu/kmu.cpp index 58734cce0e..731dcdf3ac 100644 --- a/sim/simx/kmu/kmu.cpp +++ b/sim/simx/kmu/kmu.cpp @@ -148,7 +148,8 @@ void Kmu::on_tick() { kmu_req_t req{}; if (!this->next_cta(&req, target)) return; - bus_out.at(target).send(req, 1); + // the lane slice ticks after the KMU and registers the CTA into the core + bus_out.at(target).send(req, 0); ++inflight_[target]; if (!cluster_locked_) { // Message complete: advance past the granted core. diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index feae96a223..ccf1a1d6ba 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -1372,6 +1372,10 @@ class CacheBank : public SimObject { } else { int32_t free_id = -1, repl_id = 0; line_id = set.select_victim(config_.repl_policy, rand_ctr_, &free_id, &repl_id); + // The FIFO pointer names the victim even when another way is free. + if (config_.repl_policy == Cache::FIFO) { + line_id = repl_id; + } auto &victim_line = set.lines.at(line_id); // Count the victim's dirty sectors; the egress queue must accept all @@ -1388,9 +1392,7 @@ class CacheBank : public SimObject { return false; // stall } - if (config_.repl_policy == Cache::FIFO) { - set.fifo_ptr = (set.fifo_ptr + 1) % set.lines.size(); - } else if (config_.repl_policy == Cache::RANDOM) { + if (config_.repl_policy == Cache::RANDOM) { ++rand_ctr_; } @@ -1416,6 +1418,11 @@ class CacheBank : public SimObject { victim_line.reset(); victim_line.tag = addr_tag; } + // Every installed fill advances the set's FIFO pointer, a sector refill + // into the resident way included. + if (config_.repl_policy == Cache::FIFO) { + set.fifo_ptr = (set.fifo_ptr + 1) % set.lines.size(); + } auto &line = set.lines.at(line_id); line.tag = addr_tag; diff --git a/sim/simx/operands.cpp b/sim/simx/operands.cpp index d7230c31ca..56a9d971dd 100644 --- a/sim/simx/operands.cpp +++ b/sim/simx/operands.cpp @@ -115,6 +115,19 @@ Operands::Operands(const SimContext &ctx, const char* name, Core* core) opc_units_.at(i)->Output.bind(&rsp_arb_->Inputs.at(i)); } rsp_arb_->Outputs.at(0).bind(&this->Output); + // The scoreboard's output skid buffer feeds the collectors through a + // demux: its head waits for its own collector, the entries behind it too. + // The head leaves the buffer at its last operand read, when its collector + // pops it, and the next head is offered from the following cycle. + Input.set_registered_ready(true); + Input.tx_callback([this](instr_trace_t* const& trace, uint64_t) { + if (!head_offered_ && this->Input.empty()) { + this->offer_head(trace, 0); + } + }); + for (uint32_t i = 0; i < VX_CFG_NUM_OPCS; ++i) { + opc_units_.at(i)->Input.pop_callback([this]() { this->on_head_consumed(); }); + } } else { // pass-thru this->Input.bind(&opc_units_.at(0)->Input); @@ -127,19 +140,23 @@ Operands::~Operands() { } void Operands::on_reset() { - //-- + head_offered_ = false; } void Operands::on_tick() { - if (VX_CFG_NUM_OPCS < 2) - return; // pass-thru - - // process requests - if (Input.empty()) - return; - auto trace = this->Input.peek(); - if (opc_units_.at(wid_to_opc_idx(trace->wid))->Input.try_send(trace)) { - Input.pop(); + this->tick_sleep(); // the demux runs on Input's delivery and pop events +} + +void Operands::offer_head(instr_trace_t* trace, uint64_t delay) { + opc_units_.at(wid_to_opc_idx(trace->wid))->Input.send(trace, delay); + head_offered_ = true; +} + +void Operands::on_head_consumed() { + this->Input.pop(); + head_offered_ = false; + if (!this->Input.empty()) { + this->offer_head(this->Input.peek(), 1); } } diff --git a/sim/simx/operands.h b/sim/simx/operands.h index c302f7abc3..077b905400 100644 --- a/sim/simx/operands.h +++ b/sim/simx/operands.h @@ -54,6 +54,10 @@ class Operands : public SimObject { Core* core_; std::vector opc_units_; TraceArbiter::Ptr rsp_arb_; + bool head_offered_ = false; + + void offer_head(instr_trace_t* trace, uint64_t delay); + void on_head_consumed(); friend class SimObject; }; From 347c914a6e5c45f740733467b82381091a2a7b4f Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:48 -0700 Subject: [PATCH 101/336] runtime: stage kernel args in clean slots without a cache flush Every launch writes its argument block into device memory through the CP DMA. While the last-level cache may hold lines the DMA would miss or later overwrite, each such write first appends a full cache flush. A run of short launches therefore paid one full flush per launch for its arguments alone. The args pool now records, per free slot, the flush epoch from which the last-level cache holds none of the slot's lines. A slot a kernel read becomes clean again after the next full flush. Thirty-two slots are allocated when the device opens, before any kernel has run, so they start clean. args_slot_acquire prefers a clean slot, and a MEM_WRITE into one needs no flush ahead of it. Oversized blocks and slots a kernel has read since the last full flush keep the flush. A dcache is also the point of coherence between kernels when it is the only dcache and every core of the one socket shares it. Before, only a single-core device qualified. The RTL (VX_gpu_pkg) and SimX now accept the socket-wide case, so L1-only flush tokens there are answered without a walk. VX_graphics gains a simulation-only assertion that each cluster flush round finds its flush kind latched by a DCR write before it takes the request. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 10 ++-- hw/rtl/VX_graphics.sv | 20 ++++++++ sim/simx/constants.h | 10 ++-- sw/runtime/common/device.cpp | 69 ++++++++++++++++++++-------- sw/runtime/common/queue.cpp | 20 ++++---- sw/runtime/common/vortex2_internal.h | 35 +++++++++++--- 6 files changed, 123 insertions(+), 41 deletions(-) diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 904bad773c..9e874674d1 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1747,10 +1747,12 @@ package VX_gpu_pkg; // L1 dcache is the LLC iff neither L2 nor L3 is enabled. localparam DCACHE_IS_LLC = !`VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; - // It is the point of coherence between kernels only as the one dcache with - // no other memory client beside it (the texture, ray-tracing and DXA ports - // join below it, the raster and OM caches beside it). - localparam DCACHE_SHARED_LLC = DCACHE_IS_LLC && (`VX_CFG_NUM_CORES == 1) && (`VX_CFG_NUM_CLUSTERS == 1) + // It is the point of coherence between kernels only as the one dcache, + // shared by every core of the one socket, with no other memory client + // beside it (the texture, ray-tracing and DXA ports join below it, the + // raster and OM caches beside it). + localparam DCACHE_SHARED_LLC = DCACHE_IS_LLC && (`VX_CFG_NUM_CLUSTERS == 1) + && (`VX_CFG_NUM_CORES == `VX_CFG_SOCKET_SIZE) && (`VX_CFG_NUM_DCACHES == 1) && !`VX_CFG_EXT_TEX_ENABLED && !`VX_CFG_EXT_RTU_ENABLED && !`VX_CFG_EXT_RASTER_ENABLED && !`VX_CFG_EXT_OM_ENABLED && !`VX_CFG_EXT_DXA_ENABLED; diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index 37a035506d..2e0fa10725 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -517,6 +517,26 @@ module VX_graphics import VX_gpu_pkg::* end wire omf_drained = ~(| om_busy_w) && ~(| om_ingress_pending) && ~(| steer_om_pending); +`ifndef SYNTHESIS + // Each cluster flush round must find the latch written for it, strictly + // before the round takes the request. + wire cluster_flush_kind_wr = dcr_bus_if.req_valid && dcr_bus_if.req_data.rw + && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); + wire omf_cluster_take = (omf_state_r == OMF_IDLE) && omf_cluster_req; + reg cluster_flush_kind_new_r; + always @(posedge clk) begin + if (reset) begin + cluster_flush_kind_new_r <= 1'b0; + end else if (cluster_flush_kind_wr) begin + cluster_flush_kind_new_r <= 1'b1; + end else if (omf_cluster_take) begin + cluster_flush_kind_new_r <= 1'b0; + end + end + `RUNTIME_ASSERT (~omf_cluster_take || (cluster_flush_kind_new_r && ~cluster_flush_kind_wr), + ("%t: cluster flush taken before its kind was latched", $time)) +`endif + always @(posedge clk) begin if (reset) begin omf_state_r <= OMF_IDLE; diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 2142e7efd2..a1cf857579 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -40,11 +40,13 @@ namespace vortex { inline constexpr uint32_t XLENB = (VX_CFG_XLEN / 8); // A last-level cache that alone serves every memory client is the point of -// coherence between kernels. The texture, ray-tracing and DXA ports join below -// the dcache and the raster and OM caches beside it, so a dcache with any of -// them is not one. +// coherence between kernels. A dcache is one when it is the only dcache, +// shared by every core of the one socket. The texture, ray-tracing and DXA +// ports join below the dcache and the raster and OM caches beside it, so a +// dcache with any of them is not one. inline constexpr bool DCACHE_SHARED_LLC = (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0) - && (VX_CFG_NUM_CORES == 1) && (VX_CFG_NUM_CLUSTERS == 1) + && (VX_CFG_NUM_CLUSTERS == 1) && (VX_CFG_NUM_CORES == VX_CFG_SOCKET_SIZE) + && (VX_CFG_NUM_DCACHES == 1) && !VX_CFG_EXT_TEX_ENABLED && !VX_CFG_EXT_RTU_ENABLED && !VX_CFG_EXT_RASTER_ENABLED && !VX_CFG_EXT_OM_ENABLED && !VX_CFG_EXT_DXA_ENABLED; diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index 39009c1927..957564a3a7 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -157,8 +157,8 @@ Device::~Device() { // Drain the kernel-args scratch pool. { std::lock_guard g(args_pool_mu_); - for (uint64_t addr : args_pool_free_) - this->mem_free(addr); + for (const auto& slot : args_pool_free_) + this->mem_free(slot.addr); args_pool_free_.clear(); } // Park the CP before releasing the buffers it masters into. Ordering is @@ -182,10 +182,13 @@ Device::~Device() { // ============================================================================ vx_result_t Device::args_slot_acquire(uint64_t size, uint64_t* out_addr, - bool* out_pooled) { + bool* out_pooled, bool* out_llc_clean) { if (!out_addr || !out_pooled) return VX_ERR_INVALID_VALUE; + bool llc_clean = false; + if (out_llc_clean) *out_llc_clean = false; if (size > ARGS_SLOT_SIZE) { - // Oversized args block — one-off allocation, not pooled. + // Oversized args block — one-off allocation, not pooled. The range may + // have been another buffer the cache still holds. *out_pooled = false; return this->mem_alloc(size, /*VX_MEM_READ*/ 0x1, out_addr); } @@ -193,8 +196,18 @@ vx_result_t Device::args_slot_acquire(uint64_t size, uint64_t* out_addr, { std::lock_guard g(args_pool_mu_); if (!args_pool_free_.empty()) { - *out_addr = args_pool_free_.back(); - args_pool_free_.pop_back(); + const uint64_t epoch = llc_epoch_.load(); + size_t pick = args_pool_free_.size() - 1; + for (size_t i = 0; i < args_pool_free_.size(); ++i) { + if (args_pool_free_[i].clean_epoch <= epoch) { + pick = i; + break; + } + } + *out_addr = args_pool_free_[pick].addr; + llc_clean = (args_pool_free_[pick].clean_epoch <= epoch); + args_pool_free_.erase(args_pool_free_.begin() + pick); + if (out_llc_clean) *out_llc_clean = llc_clean; return VX_SUCCESS; } } @@ -208,7 +221,7 @@ void Device::args_slot_release(uint64_t addr, bool pooled) { return; } std::lock_guard g(args_pool_mu_); - args_pool_free_.push_back(addr); + args_pool_free_.push_back({addr, llc_epoch_.load() + 1}); } vx_result_t Device::open(uint32_t index, Device** out) { @@ -475,6 +488,17 @@ vx_result_t Device::cp_init() { } } + // No kernel has run yet, so the last-level cache holds none of these. + { + std::lock_guard g(args_pool_mu_); + for (uint32_t i = 0; i < ARGS_POOL_CLEAN; ++i) { + uint64_t addr = 0; + auto r = this->mem_alloc(ARGS_SLOT_SIZE, /*VX_MEM_READ*/ 0x1, &addr); + if (r != VX_SUCCESS) return r; + args_pool_free_.push_back({addr, 0}); + } + } + return VX_SUCCESS; } @@ -551,7 +575,7 @@ bool cp_dma_touches_device_mem(const uint8_t* cl) { } } // namespace -vx_result_t Device::cp_ring_append_(const void* cl) { +vx_result_t Device::cp_ring_append_(const void* cl, bool llc_clean) { // Caller holds cp_mu_. A kernel's results are flushed lazily, by the // next command that needs them: the next launch gets an L1-only flush // (the last-level cache is the point of coherence between kernels), and @@ -560,7 +584,10 @@ vx_result_t Device::cp_ring_append_(const void* cl) { // it is a client of that cache. Appending the flush under the same lock // keeps any other command from slipping between. const uint8_t* b = static_cast(cl); - const bool dma = cp_dma_touches_device_mem(b); + // A write into memory the last-level cache holds no line of has nothing + // to wait for: the cache cannot serve stale data from it or write a + // dirty line back over it. + const bool dma = cp_dma_touches_device_mem(b) && !(llc_clean && b[0] == CP_OPCODE_MEM_WRITE); const bool launch = (b[0] == CP_OPCODE_LAUNCH) || (b[0] == CP_OPCODE_LAUNCH_QMD) || (b[0] == CP_OPCODE_DRAW); @@ -572,6 +599,9 @@ vx_result_t Device::cp_ring_append_(const void* cl) { if (r != VX_SUCCESS) return r; l1_dirty_ = false; llc_dirty_ = llc_dirty_ && !full; + if (full) { + ++llc_epoch_; + } } assert(!(dma && (l1_dirty_ || llc_dirty_))); auto r = cp_ring_write_(cl); @@ -583,6 +613,9 @@ vx_result_t Device::cp_ring_append_(const void* cl) { } else if (b[0] == CP_OPCODE_CACHE_FLUSH) { l1_dirty_ = false; llc_dirty_ = llc_dirty_ && (b[1] & CP_FLUSH_FLAG_L1_ONLY); + if (!(b[1] & CP_FLUSH_FLAG_L1_ONLY)) { + ++llc_epoch_; + } } return VX_SUCCESS; } @@ -655,14 +688,14 @@ vx_result_t Device::cp_batch_end() { return drain_cout(); } -vx_result_t Device::cp_submit_cl_(const void* cl) { +vx_result_t Device::cp_submit_cl_(const void* cl, bool llc_clean) { auto* p = platform(); // Batch mode: append only — cp_mu_ is already held for the batch, and // the single doorbell + poll happen in cp_batch_end. Only the thread that // owns the batch may take this path; anyone else must block on cp_mu_. if (tls_batch_owner == this) { - auto r = cp_ring_append_(cl); + auto r = cp_ring_append_(cl, llc_clean); if (r == VX_SUCCESS) cp_batch_target_ = cp_expected_seqnum_; return r; } @@ -675,7 +708,7 @@ vx_result_t Device::cp_submit_cl_(const void* cl) { std::lock_guard g(cp_mu_); // 1) Write the CL into the ring and reserve its seqnum. - auto r = cp_ring_append_(cl); + auto r = cp_ring_append_(cl, llc_clean); if (r != VX_SUCCESS) return r; target = cp_expected_seqnum_; @@ -1009,7 +1042,7 @@ vx_result_t Device::cp_submit_dcr_read(uint32_t addr, uint32_t tag, vx_result_t Device::cp_submit_mem_(uint8_t opcode, uint64_t arg0, uint64_t arg1, uint64_t arg2, - bool physical) { + bool physical, bool llc_clean) { // CMD_MEM_* on-wire layout (cmd_size=28): // bytes 0..3 header { opcode, flags, reserved=0 } // bytes 4..11 arg0 dst address @@ -1021,7 +1054,7 @@ vx_result_t Device::cp_submit_mem_(uint8_t opcode, uint64_t arg0, std::memcpy(cl + 4, &arg0, sizeof(arg0)); std::memcpy(cl + 12, &arg1, sizeof(arg1)); std::memcpy(cl + 20, &arg2, sizeof(arg2)); - return cp_submit_cl_(cl); + return cp_submit_cl_(cl, llc_clean); } vx_result_t Device::cp_submit_mem_copy(uint64_t dst, uint64_t src, @@ -1031,7 +1064,7 @@ vx_result_t Device::cp_submit_mem_copy(uint64_t dst, uint64_t src, } vx_result_t Device::cp_submit_mem_write(uint64_t dev_dst, const void* host_src, - uint64_t size, bool physical) { + uint64_t size, bool physical, bool llc_clean) { if (size == 0) return VX_SUCCESS; if (!host_src) return VX_ERR_INVALID_VALUE; // Stage through a bounded CP-visible host buffer, one chunk at a time: @@ -1054,7 +1087,7 @@ vx_result_t Device::cp_submit_mem_write(uint64_t dev_dst, const void* host_src, r = platform()->host_mem_push(staging.cp_addr); if (r == VX_SUCCESS) r = cp_submit_mem_(CP_OPCODE_MEM_WRITE, dev_dst + off, - staging.cp_addr, n, physical); + staging.cp_addr, n, physical, llc_clean); } host_free(staging.cp_addr); return r; @@ -1275,8 +1308,8 @@ vx_result_t Device::query_caps(uint32_t caps_id, uint64_t* out_value) { // ============================================================================ vx_result_t Device::dev_write(uint64_t dev_addr, const void* src, - uint64_t size) { - return cp_submit_mem_write(dev_addr, src, size); + uint64_t size, bool llc_clean) { + return cp_submit_mem_write(dev_addr, src, size, /*physical=*/false, llc_clean); } vx_result_t Device::dev_read(void* dst, uint64_t dev_addr, uint64_t size) { diff --git a/sw/runtime/common/queue.cpp b/sw/runtime/common/queue.cpp index 68543c0f60..84d4c22f0d 100644 --- a/sw/runtime/common/queue.cpp +++ b/sw/runtime/common/queue.cpp @@ -413,16 +413,17 @@ vx_result_t Queue::enqueue_launch(const vx_launch_info_t* info, // Empty blob → caller pre-programmed the ARG DCRs (legacy path). uint64_t args_addr = 0; bool args_pooled = false; + bool args_clean = false; bool args_staged = !args_blob.empty(); if (args_staged) { auto r = device_->args_slot_acquire(args_blob.size(), - &args_addr, &args_pooled); + &args_addr, &args_pooled, &args_clean); if (r != VX_SUCCESS) { if (kernel) kernel->release(); *s = *e = now_ns(); return r; } r = device_->dev_write(args_addr, args_blob.data(), - args_blob.size()); + args_blob.size(), args_clean); if (r != VX_SUCCESS) { device_->args_slot_release(args_addr, args_pooled); if (kernel) kernel->release(); @@ -607,11 +608,12 @@ vx_result_t cmd_stage_qmds(Device* device_, const std::vector& recs, // 1) Args blob. if (!rec.args.empty()) { + bool clean = false; r = device_->args_slot_acquire(rec.args.size(), - &st.args_addr, &st.args_pooled); + &st.args_addr, &st.args_pooled, &clean); if (r != VX_SUCCESS) break; st.args_active = true; - r = device_->dev_write(st.args_addr, rec.args.data(), rec.args.size()); + r = device_->dev_write(st.args_addr, rec.args.data(), rec.args.size(), clean); if (r != VX_SUCCESS) break; } @@ -672,10 +674,11 @@ vx_result_t cmd_stage_qmds(Device* device_, const std::vector& recs, st.qmd_words = std::move(qmd); if (device_->cp_supports_qmd()) { const uint64_t qmd_bytes = st.qmd_words.size() * sizeof(uint32_t); - r = device_->args_slot_acquire(qmd_bytes, &st.qmd_addr, &st.qmd_pooled); + bool clean = false; + r = device_->args_slot_acquire(qmd_bytes, &st.qmd_addr, &st.qmd_pooled, &clean); if (r != VX_SUCCESS) break; st.qmd_active = true; - r = device_->dev_write(st.qmd_addr, st.qmd_words.data(), qmd_bytes); + r = device_->dev_write(st.qmd_addr, st.qmd_words.data(), qmd_bytes, clean); if (r != VX_SUCCESS) break; } } @@ -841,10 +844,11 @@ vx_result_t Queue::enqueue_draw(const vx_command_t* commands, } } std::memcpy(desc.data(), &num_steps, sizeof(num_steps)); - r = device_->args_slot_acquire(desc.size(), &desc_addr, &desc_pooled); + bool desc_clean = false; + r = device_->args_slot_acquire(desc.size(), &desc_addr, &desc_pooled, &desc_clean); if (r == VX_SUCCESS) { desc_active = true; - r = device_->dev_write(desc_addr, desc.data(), desc.size()); + r = device_->dev_write(desc_addr, desc.data(), desc.size(), desc_clean); } } diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index f268d9a58d..f5a7e474ff 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -300,8 +300,11 @@ class Device : public RefCounted { // cp_submit_mem_read posts CMD_MEM_READ (device->host) into a // VX_MEM_HOST buffer and copies it back to `host_dst`. Synchronous. vx_result_t cp_submit_mem_copy (uint64_t dst, uint64_t src, uint64_t size); + // `llc_clean`: the last-level cache holds no line of the destination, so + // the write needs no cache flush ahead of it. vx_result_t cp_submit_mem_write(uint64_t dev_dst, const void* host_src, - uint64_t size, bool physical = false); + uint64_t size, bool physical = false, + bool llc_clean = false); vx_result_t cp_submit_mem_read (void* host_dst, uint64_t dev_src, uint64_t size, bool physical = false); @@ -312,7 +315,8 @@ class Device : public RefCounted { // engine (CMD_MEM_*) — identically on every backend. The runtime never // touches device memory directly: it only appends commands to the host // ring and the CP executes them. - vx_result_t dev_write(uint64_t dev_addr, const void* src, uint64_t size); + vx_result_t dev_write(uint64_t dev_addr, const void* src, uint64_t size, + bool llc_clean = false); vx_result_t dev_read (void* dst, uint64_t dev_addr, uint64_t size); vx_result_t dev_copy (uint64_t dst, uint64_t src, uint64_t size); @@ -329,8 +333,11 @@ class Device : public RefCounted { // args_slot_release frees correctly. Because vx_enqueue_launch holds // a slot only for the duration of one synchronous CP launch, the // free-list naturally settles at the count of concurrent launches. + // `*out_llc_clean` reports a slot no kernel has read since the last full + // cache flush: the last-level cache holds none of its lines, so staging + // into it needs no flush (see args_pool_free_). vx_result_t args_slot_acquire(uint64_t size, uint64_t* out_addr, - bool* out_pooled); + bool* out_pooled, bool* out_llc_clean = nullptr); void args_slot_release(uint64_t addr, bool pooled); // ----- Device-memory allocation (common-core allocator) ----- @@ -366,7 +373,7 @@ class Device : public RefCounted { // cp_submit_dcr_write / cp_submit_launch — they just build the CL. // While this thread has a batch open it appends only; the single // doorbell + poll are deferred to cp_batch_end. - vx_result_t cp_submit_cl_(const void* cl); + vx_result_t cp_submit_cl_(const void* cl, bool llc_clean = false); // Block until the CP has retired seqnum >= target. // @@ -389,14 +396,15 @@ class Device : public RefCounted { // model) under the same lock: L1-only ahead of a launch while // l1_dirty_, and ahead of a CMD_MEM_* touching device memory while // either flag is set -- full while llc_dirty_. - vx_result_t cp_ring_append_(const void* cl); + vx_result_t cp_ring_append_(const void* cl, bool llc_clean = false); vx_result_t cp_ring_write_(const void* cl); // Build + submit a CMD_MEM_* descriptor (opcode, dst, src, size). // `physical` sets the CMD_MEM header flag so the CP DMA skips VM // translation — used for page-table writes / the PT region. vx_result_t cp_submit_mem_(uint8_t opcode, uint64_t arg0, uint64_t arg1, - uint64_t arg2, bool physical = false); + uint64_t arg2, bool physical = false, + bool llc_clean = false); // ----- CP-visible host memory (command ring + DMA staging) ----- // host_alloc wraps platform()->host_mem_alloc and records the region @@ -464,6 +472,8 @@ class Device : public RefCounted { // may hold lines the CP DMA (when it reaches device memory beneath it) // would miss or later overwrite. Both flags are guarded by cp_mu_. bool llc_dirty_ = false; + // Full cache flushes appended to the ring so far. + std::atomic llc_epoch_{0}; std::mutex cp_mu_; // serialize ring writes // Seqnum the last command appended to the open batch will reach; // cp_batch_end polls for it once. Touched only by the batch owner, which @@ -512,9 +522,20 @@ class Device : public RefCounted { // Kernel-args scratch pool. Free-list of recycled fixed-size device slots; // ARGS_SLOT_SIZE covers typical kernel arg blocks. Drained in ~Device. + // Each free slot records the flush epoch from which the last-level cache + // holds none of its lines: a kernel that read the slot may have left them + // there until the next full flush. The pool starts with ARGS_POOL_CLEAN + // slots allocated at open, when no kernel has run, so a run of launches + // stages its arguments without flushing until it has used them all; the + // next transfer that needs a full flush makes them clean again. static constexpr uint64_t ARGS_SLOT_SIZE = 4096; + static constexpr uint32_t ARGS_POOL_CLEAN = 32; + struct ArgsSlot { + uint64_t addr; + uint64_t clean_epoch; + }; std::mutex args_pool_mu_; - std::vector args_pool_free_; + std::vector args_pool_free_; }; // ============================================================================ From 398197627cdc3b433dbd710d0020c7f5c7a3eff1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:48 -0700 Subject: [PATCH 102/336] simx: feed full-bandwidth ALU and FPU straight from the dispatch queue A full-bandwidth ALU or FPU (as many blocks as issue slots and lanes as the SIMD width) has no lane-dispatch buffer: the RTL unit takes the op straight off the dispatch queue, in the cycle the queue releases it. SimX gave it one more register on the way in. Such a unit now holds only the queue's head and is created after its dispatcher, so it ticks after it. It returns the issue credit itself as it accepts the op, and a resolved branch releases its warp through the result register and then the branch register. The SFU, like the LSU, carries its result buffer and response switch in its output channel, so its commit queue holds two entries and its commit hop is one register. A committed result is written back from the commit arbiter's output register, and issue reads a registered readiness, so the scoreboard releases a register two cycles after commit. Co-Authored-By: Claude Opus 5.5 --- sim/simx/alu_unit.cpp | 18 ++++++++++-- sim/simx/alu_unit.h | 3 ++ sim/simx/core.cpp | 61 +++++++++++++++++++++++++++++------------ sim/simx/dispatcher.cpp | 8 ++++-- sim/simx/dispatcher.h | 5 ++++ sim/simx/fpu_unit.cpp | 14 +++++++++- sim/simx/fpu_unit.h | 4 +++ sim/simx/scoreboard.h | 4 ++- 8 files changed, 93 insertions(+), 24 deletions(-) diff --git a/sim/simx/alu_unit.cpp b/sim/simx/alu_unit.cpp index a48fc3ebe5..e30768ac06 100644 --- a/sim/simx/alu_unit.cpp +++ b/sim/simx/alu_unit.cpp @@ -30,7 +30,16 @@ AluUnit::AluUnit(const SimContext& ctx, const char* name, Core* core) // multiplier keeps one result per cycle. : FuncUnit(ctx, name, core, kMulDivLatency + (kGather ? 1 : 0) + 2) , branch_ctl_out(this, VX_CFG_NUM_WARPS) -{} + , ReleaseOut(this, VX_CFG_ISSUE_WIDTH) +{ + if (!kGather) { + // No buffer between the dispatch queue and the unit: the input holds + // only the queue's head. + for (auto& input : Inputs) { + input = SimChannel(this, 1); + } + } +} // Cycles from this unit's execute to its output, beyond the commit-queue hop. // Execute stands for the integer ALU's registered result (the branch_ctl @@ -572,13 +581,16 @@ void AluUnit::on_tick() { auto trace = input.peek(); this->execute(trace); // A branch releases its warp as it resolves, not when its result - // retires. + // retires: the result register, then the branch register. if (std::get_if(&trace->op_type) && trace->eop && trace->resume_warp) { - branch_ctl_out.send(trace->wid, 1); + branch_ctl_out.send(trace->wid, kGather ? 1 : 2); trace->resume_warp = false; } output.send(trace, this->latency_of(trace) + (kGather ? 1 : 0)); input.pop(); + if (!kGather && trace->eop) { + ReleaseOut.send(trace, 0); + } } } idle &= (input.size() == 0); diff --git a/sim/simx/alu_unit.h b/sim/simx/alu_unit.h index 6e2db38121..c7c11e83ad 100644 --- a/sim/simx/alu_unit.h +++ b/sim/simx/alu_unit.h @@ -30,6 +30,9 @@ class AluUnit : public FuncUnit { // Resolved branch of a stalled warp, registered toward the scheduler. SimChannel branch_ctl_out; + // Full bandwidth: the dispatch queue feeds the unit directly, so an op + // returns its issue credit as the unit accepts it. + SimChannel ReleaseOut; protected: void on_tick() override; diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 213e9634e9..8742972a59 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -269,16 +269,28 @@ class Core::Impl { icache_flush_buf_->Out.bind(&simobject_->icache_req_out.at(0)); #endif - // initialize execute units - snprintf(sname, 100, "%s-alu", name.c_str()); - func_units_.at((int)FUType::ALU) = SimPlatform::instance().create_object(sname, simobject_); - snprintf(sname, 100, "%s-fpu", name.c_str()); - func_units_.at((int)FUType::FPU) = SimPlatform::instance().create_object(sname, simobject_); + // initialize execute units. A full-bandwidth ALU or FPU takes its op + // straight off the dispatch queue, in the cycle the queue releases it, so + // it is created (and ticks) after its dispatcher; the others take their + // input before the dispatcher fills it in the same cycle. + auto create_alu = [&]() { + snprintf(sname, 100, "%s-alu", name.c_str()); + func_units_.at((int)FUType::ALU) = SimPlatform::instance().create_object(sname, simobject_); + }; + auto create_fpu = [&]() { + snprintf(sname, 100, "%s-fpu", name.c_str()); + func_units_.at((int)FUType::FPU) = SimPlatform::instance().create_object(sname, simobject_); + }; + if (AluUnit::kGather) { + create_alu(); + } + if (FpuUnit::kGather) { + create_fpu(); + } snprintf(sname, 100, "%s-lsu", name.c_str()); func_units_.at((int)FUType::LSU) = SimPlatform::instance().create_object(sname, simobject_); snprintf(sname, 100, "%s-sfu", name.c_str()); func_units_.at((int)FUType::SFU) = SimPlatform::instance().create_object(sname, simobject_); - std::static_pointer_cast(func_units_.at((int)FUType::ALU))->branch_ctl_out.bind(&simobject_->branch_ctl_in); std::static_pointer_cast(func_units_.at((int)FUType::SFU))->warp_ctl_out.bind(&simobject_->warp_ctl_in); scheduler_->barrier_unit().unlock_out.bind(&simobject_->bar_unlock_in); #ifdef VX_CFG_EXT_TCU_ENABLE @@ -310,12 +322,13 @@ class Core::Impl { #endif #endif - // initialize dispatchers after the units they feed, so a unit takes its - // input before the dispatcher fills it in the same cycle. + // initialize dispatchers after the buffered units they feed, so such a + // unit takes its input before the dispatcher fills it in the same cycle. { - // A reduced-width ALU or FPU registers the op once more on its way in. - uint32_t alu_dly = AluUnit::kGather ? 2 : 1; - uint32_t fpu_dly = FpuUnit::kGather ? 2 : 1; + // A reduced-width ALU or FPU takes the op through its lane-dispatch + // buffer; a full-bandwidth one takes it as the queue releases it. + uint32_t alu_dly = AluUnit::kGather ? 2 : 0; + uint32_t fpu_dly = FpuUnit::kGather ? 2 : 0; dispatchers_.at((int)FUType::ALU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_ALU_BLOCKS, VX_CFG_NUM_ALU_LANES, alu_dly); dispatchers_.at((int)FUType::FPU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_FPU_BLOCKS, VX_CFG_NUM_FPU_LANES, fpu_dly); dispatchers_.at((int)FUType::LSU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_LSU_BLOCKS, VX_CFG_NUM_LSU_LANES, 1); @@ -323,6 +336,19 @@ class Core::Impl { #ifdef VX_CFG_EXT_TCU_ENABLE dispatchers_.at((int)FUType::TCU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_TCU_BLOCKS, VX_CFG_NUM_TCU_LANES, 1); #endif + if (!AluUnit::kGather) { + create_alu(); + auto& dispatch = dispatchers_.at((int)FUType::ALU); + dispatch->set_release_on_accept(); + std::static_pointer_cast(func_units_.at((int)FUType::ALU))->ReleaseOut.bind(&dispatch->ReleaseOut); + } + if (!FpuUnit::kGather) { + create_fpu(); + auto& dispatch = dispatchers_.at((int)FUType::FPU); + dispatch->set_release_on_accept(); + std::static_pointer_cast(func_units_.at((int)FUType::FPU))->ReleaseOut.bind(&dispatch->ReleaseOut); + } + std::static_pointer_cast(func_units_.at((int)FUType::ALU))->branch_ctl_out.bind(&simobject_->branch_ctl_in); for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { auto& dispatch = dispatchers_.at(fu); auto& func_unit = func_units_.at(fu); @@ -340,10 +366,10 @@ class Core::Impl { for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { auto& queues = commit_queues_.emplace_back(); for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { - // A unit's lane gather holds two results. The LSU's crosses one - // register into it; the others cross two and take a third slot so - // that they still retire one result per cycle. - uint32_t capacity = (fu == (uint32_t)FUType::LSU) ? 2 : 3; + // A unit's lane gather holds two results. The LSU's and the SFU's + // cross one register into it; the others cross two and take a third + // slot so that they still retire one result per cycle. + uint32_t capacity = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU) ? 2 : 3; queues.emplace_back(std::make_unique>(simobject_, capacity)); } } @@ -907,8 +933,9 @@ class Core::Impl { auto& arb_in = *commit_queues_.at(iw).at(fu); // The unit's output registers ahead of the commit arbiter that its // output channel does not already carry: the LSU's response buffer and - // result arbiter are in its channel, leaving its lane gather. - uint32_t commit_delay = (fu == (uint32_t)FUType::LSU) ? 1 : 2; + // result arbiter, and the SFU's result buffer and response switch, are + // in their channels, leaving each its lane gather. + uint32_t commit_delay = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU) ? 1 : 2; if (arb_in.try_send(trace, commit_delay)) { // Release the warp as soon as its stalling instruction's result leaves // the functional unit — the branch target / fence / warp-control is diff --git a/sim/simx/dispatcher.cpp b/sim/simx/dispatcher.cpp index f8a45998dd..3071ae1667 100644 --- a/sim/simx/dispatcher.cpp +++ b/sim/simx/dispatcher.cpp @@ -118,7 +118,9 @@ void Dispatcher::on_tick() { } else { block_pids_.at(b) = -1; // mark block as processed input.pop(); - ReleaseOut.send(trace, 0); + if (!release_on_accept_) { + ReleaseOut.send(trace, 0); + } ++block_sent; } ThreadMask tmask(VX_CFG_NUM_THREADS); @@ -132,7 +134,9 @@ void Dispatcher::on_tick() { } else { // issue the trace input.pop(); - ReleaseOut.send(trace, 0); + if (!release_on_accept_) { + ReleaseOut.send(trace, 0); + } ++block_sent; } DT(3, this->name() << "-pipeline dispatch: " << *new_trace); diff --git a/sim/simx/dispatcher.h b/sim/simx/dispatcher.h index 3fe1eb9468..3f3aad0cba 100644 --- a/sim/simx/dispatcher.h +++ b/sim/simx/dispatcher.h @@ -33,6 +33,10 @@ class Dispatcher : public SimObject { virtual ~Dispatcher(); + // The unit takes ops straight off the slot queue, so the credit returns as + // the unit accepts the op, from the unit itself. + void set_release_on_accept() { release_on_accept_ = true; } + protected: virtual void on_reset(); virtual void on_tick(); @@ -46,6 +50,7 @@ class Dispatcher : public SimObject { uint32_t batch_idx_; uint32_t out_delay_; std::vector block_pids_; + bool release_on_accept_ = false; friend class SimObject; }; diff --git a/sim/simx/fpu_unit.cpp b/sim/simx/fpu_unit.cpp index 270f7b8a3f..56752b4b41 100644 --- a/sim/simx/fpu_unit.cpp +++ b/sim/simx/fpu_unit.cpp @@ -44,7 +44,16 @@ FpuUnit::FpuUnit(const SimContext& ctx, const char* name, Core* core) // until its result is taken off the response port, however deep the // pipelines are. : FuncUnit(ctx, name, core, VX_CFG_FPU_QUEUE_SIZE) -{} + , ReleaseOut(this, VX_CFG_ISSUE_WIDTH) +{ + if (!kGather) { + // No buffer between the dispatch queue and the unit: the input holds + // only the queue's head. + for (auto& input : Inputs) { + input = SimChannel(this, 1); + } + } +} uint32_t FpuUnit::latency_of(const instr_trace_t* trace) const { auto fpu_type = std::get(trace->op_type); @@ -390,6 +399,9 @@ void FpuUnit::on_tick() { uint32_t delay = this->latency_of(trace) + (kGather ? 0 : 1); output.send(trace, delay); input.pop(); + if (!kGather && trace->eop) { + ReleaseOut.send(trace, 0); + } } } idle &= (input.size() == 0); diff --git a/sim/simx/fpu_unit.h b/sim/simx/fpu_unit.h index 4de1f44f72..ef0dcd11a7 100644 --- a/sim/simx/fpu_unit.h +++ b/sim/simx/fpu_unit.h @@ -26,6 +26,10 @@ class FpuUnit : public FuncUnit { static constexpr bool kGather = (VX_CFG_NUM_FPU_BLOCKS != VX_CFG_ISSUE_WIDTH) || (VX_CFG_NUM_FPU_LANES != VX_CFG_SIMD_WIDTH); + // Full bandwidth: the dispatch queue feeds the unit directly, so an op + // returns its issue credit as the unit accepts it. + SimChannel ReleaseOut; + protected: void on_tick() override; diff --git a/sim/simx/scoreboard.h b/sim/simx/scoreboard.h index e05ade6479..d5514bc654 100644 --- a/sim/simx/scoreboard.h +++ b/sim/simx/scoreboard.h @@ -69,7 +69,9 @@ class Scoreboard : public SimObject { return (wid << RegOpd::ID_BITS) | reg.id(); } - static constexpr uint32_t kReleaseDelay = 1; + // A committed result is written back from the commit arbiter's output + // register, and the issue stage reads a registered readiness. + static constexpr uint32_t kReleaseDelay = 2; struct pending_release_t { uint32_t wid; From 03e8304a2507eba9cb6f808ced740bfa476f45ac Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:49 -0700 Subject: [PATCH 103/336] lsu: split memory fences into acquire and release, order .aq/.rl AMOs Every memory fence sent a full L1 flush token, which both drains earlier writes and invalidates the cache. The fences the RISC-V mappings emit usually need only half of that. A load-acquire's fence r,rw needs only later reads to see other harts' writes (invalidate). A store-release's fence rw,w needs only earlier writes to be visible (drain). Decode now derives the kind from the pred/succ sets. Acquire applies when the successor set reads, release when the predecessor set writes or the successor set does not read. A device access in the predecessor set, or fence.i, does both. The token carries the kind in its attributes (acquire: io with byte enables set; release: noalloc). VX_cache_init: - An acquire walks above the point of coherence and forwards only when the cache below may also hold stale lines (ACQ_FORWARD: an L2 that is not the shared LLC). - A release only quiesces the banks and forwards a release token behind the earlier writes. - Kinds visible together combine to the weakest one that serves them all. Atomics with .aq/.rl were treated as sequentially consistent, which nothing enforced. A new uop expander (VX_uop_amo; LsuUopGen in SimX) places a release fence before an atomic with .rl and an acquire fence after one with .aq. The RTL and SimX change together. fence_mp -m selects how the flag is published and observed: 0 full fences, 1 acquire/release fences, 2 amoswap.rl / amoor.aq. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 14 ++++-- hw/rtl/VX_socket.sv | 1 + hw/rtl/cache/VX_cache.sv | 5 +++ hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_init.sv | 64 +++++++++++++++++++++------- hw/rtl/cache/VX_cache_wrap.sv | 2 + hw/rtl/core/VX_decode.sv | 7 +++ hw/rtl/core/VX_lsu_block.sv | 11 +++-- hw/rtl/core/VX_uop_amo.sv | 63 +++++++++++++++++++++++++++ hw/rtl/core/VX_uop_sequencer.sv | 19 +++++++++ sim/simx/constants.h | 3 ++ sim/simx/decode.cpp | 26 +++++++++-- sim/simx/lsu_unit.cpp | 38 +++++++++++++++-- sim/simx/mem/cache.cpp | 62 ++++++++++++++++++++------- sim/simx/mem/cache.h | 1 + sim/simx/socket.cpp | 1 + sim/simx/types.h | 11 +++-- tests/regression/fence_mp/common.h | 9 +++- tests/regression/fence_mp/kernel.cpp | 54 ++++++++++++++++++----- tests/regression/fence_mp/main.cpp | 7 ++- 20 files changed, 338 insertions(+), 62 deletions(-) create mode 100644 hw/rtl/core/VX_uop_amo.sv diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 9e874674d1..db8c1a2e6a 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -98,7 +98,9 @@ package VX_gpu_pkg; // owning one: the two are combinational rewrites of ibuf_in with mutually // exclusive selects (EX_LSU + export_mask vs EX_SFU + INST_SFU_RTUW). localparam UOP_GFX = UOP_TCU + `VX_CFG_EXT_TCU_ENABLED; - localparam UOP_MAX = UOP_GFX + `EXT_GFX_ANY_ENABLED; + // atomics with .aq/.rl expand into fences around the atomic + localparam UOP_AMO = UOP_GFX + `EXT_GFX_ANY_ENABLED; + localparam UOP_MAX = UOP_AMO + `VX_CFG_EXT_A_ENABLED; localparam UOP_CTR_W = 8; localparam CTA_TID_WIDTH = `UP(NW_BITS + NT_BITS); @@ -1120,8 +1122,11 @@ package VX_gpu_pkg; // lsu_args_t carries AMO sideband when EXT_A_ENABLE. // amo_valid / amo_op are inspected only when amo_valid==1; for plain - // loads/stores they're zero. aq/rl are decoded but unused (sequentially - // consistent). amo_unsigned distinguishes the U-variants of MIN/MAX. + // loads/stores they're zero. aq/rl order the AMO with the hart's other + // accesses (fence uops around it); on a memory fence they say what it + // orders: aq = later reads see others' writes (acquire: invalidate), + // rl = earlier writes are visible (release: drain). amo_unsigned + // distinguishes the U-variants of MIN/MAX. // export_mask marks a vx_om_export (OM fragment export) and says which // words of the aperture record the shader is writing: // bit 0 = colour (record + 0), bit 1 = depth (record + 4) @@ -2004,6 +2009,9 @@ package VX_gpu_pkg; localparam L2_IS_LLC = `VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; // Every memory client of a cluster reaches memory through its L2. localparam L2_SHARED_LLC = L2_IS_LLC && (`VX_CFG_NUM_CLUSTERS == 1); + // An acquire must also invalidate the cache below the dcache unless that + // one is the point of coherence (an L3 always is; memory always is). + localparam DCACHE_ACQ_FORWARD = `VX_CFG_L2_ENABLED && !L2_SHARED_LLC; /////////////////////////////// L3 Parameters ///////////////////////////// diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 3734c2771d..a8fc37e113 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -234,6 +234,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .MEM_OUT_BUF (3), .IS_LLC (DCACHE_IS_LLC), .SHARED_LLC (DCACHE_SHARED_LLC), + .ACQ_FORWARD (DCACHE_ACQ_FORWARD), .BANK_HASH (`VX_CFG_DCACHE_BANK_HASH), .READ_MERGE (`VX_CFG_DCACHE_READ_MERGE ? DCACHE_CHANNELS : 0), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index fb54750a87..86814ef6eb 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -54,6 +54,7 @@ module VX_cache import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // last-level cache: banks own AMO commit + RVA reservation tracking parameter SHARED_LLC = 0, // the LLC serves every memory client alone (point of coherence) + parameter ACQ_FORWARD = 0, // an acquire flush also invalidates the next level parameter FLUSH_FORWARD = 1, // a flush also flushes the next level @@ -144,6 +145,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [MEM_PORTS-1:0] mem_flush_valid; wire mem_flush_l1_only; wire mem_flush_io; + wire mem_flush_noalloc; wire [MEM_PORTS-1:0] mem_flush_ready; wire mem_flush_rsp_phase; wire [MEM_PORTS-1:0] mem_flush_rsp_valid; @@ -157,6 +159,7 @@ module VX_cache import VX_gpu_pkg::*; #( .MEM_PORTS (MEM_PORTS), .FLUSH_FORWARD (FLUSH_FORWARD), .SHARED_LLC (SHARED_LLC), + .ACQ_FORWARD (ACQ_FORWARD), .TAG_WIDTH (TAG_WIDTH), .XBAR_SIZE (REQ_XBAR_SIZE), .REQ_SEL_WIDTH (REQ_SEL_WIDTH) @@ -176,6 +179,7 @@ module VX_cache import VX_gpu_pkg::*; #( .mem_flush_valid (mem_flush_valid), .mem_flush_l1_only (mem_flush_l1_only), .mem_flush_io (mem_flush_io), + .mem_flush_noalloc (mem_flush_noalloc), .mem_flush_ready (mem_flush_ready), .mem_flush_rsp_phase (mem_flush_rsp_phase), .mem_flush_rsp_valid (mem_flush_rsp_valid), @@ -675,6 +679,7 @@ module VX_cache import VX_gpu_pkg::*; #( `CS_SECTOR_WIDTH'(0), BANK_MEM_TAG_WIDTH'(0), (`UP(MEM_ATTR_WIDTH)'(1) << MEM_ATTR_FLUSH_OFFS) | (`UP(MEM_ATTR_WIDTH)'(mem_flush_io) << MEM_ATTR_IO_OFFS) + | (`UP(MEM_ATTR_WIDTH)'(mem_flush_noalloc) << MEM_ATTR_NOALLOC_OFFS) }; assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index a8d0f99a44..c7553e07e9 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -59,6 +59,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // Indicates this cache cluster is the LLC (handles AMO completion). parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). + parameter ACQ_FORWARD = 0, // An acquire flush also invalidates the next level. parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). @@ -184,6 +185,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .PASSTHRU (PASSTHRU), .IS_LLC (IS_LLC), .SHARED_LLC (SHARED_LLC), + .ACQ_FORWARD (ACQ_FORWARD), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .READ_MERGE (READ_MERGE), diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index 28439232d6..eb265e5ca9 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -38,6 +38,12 @@ // nowhere. Once nothing is held between the cache's inputs and its memory // ports, on the cacheable and the non-cacheable path alike, it goes down every // memory port, so its answer proves the earlier IO writes reached memory. +// The L1-only kind splits by what the fence orders. A release token (noalloc +// attribute) owes no invalidation: the banks only quiesce and the token goes +// down to the point of coherence behind the earlier writes. An acquire token +// (io attribute with byte enables set) owes no drain: the cache walks and +// answers, forwarding only to a cache below that is not the point of +// coherence (ACQ_FORWARD). // Kinds visible together combine to the weakest one that serves them all. module VX_cache_init import VX_gpu_pkg::*; #( parameter NUM_REQS = 4, // Number of Word requests per cycle @@ -45,6 +51,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( parameter MEM_PORTS = 1, // Number of memory ports parameter FLUSH_FORWARD = 1, // Forward the flush to the next level parameter SHARED_LLC = 0, // The LLC serving every memory client alone: answers L1-only tokens without a walk + parameter ACQ_FORWARD = 0, // An acquire token must also invalidate the next level parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size parameter XBAR_SIZE = 0, // Requests the request crossbar can hold apart from input order parameter REQ_SEL_WIDTH = `UP(`CLOG2(NUM_REQS)) @@ -65,6 +72,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( output wire [MEM_PORTS-1:0] mem_flush_valid, output wire mem_flush_l1_only, output wire mem_flush_io, + output wire mem_flush_noalloc, input wire [MEM_PORTS-1:0] mem_flush_ready, output wire mem_flush_rsp_phase, input wire [MEM_PORTS-1:0] mem_flush_rsp_valid, @@ -126,18 +134,27 @@ module VX_cache_init import VX_gpu_pkg::*; #( end wire flush_req_enable = (| flush_req_mask); - wire [NUM_REQS-1:0] flush_l1_mask, flush_io_mask; + wire [NUM_REQS-1:0] flush_l1_mask, flush_io_mask, flush_acq_mask, flush_rel_mask; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_kind_mask - assign flush_io_mask[i] = flush_req_mask[i] && core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; - assign flush_l1_mask[i] = flush_req_mask[i] && ~flush_io_mask[i] && (| core_bus_in_if[i].req_data.byteen); + wire tok_io = core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; + wire tok_rel = core_bus_in_if[i].req_data.attr[MEM_ATTR_NOALLOC_OFFS]; + wire tok_l1 = (| core_bus_in_if[i].req_data.byteen); + assign flush_io_mask[i] = flush_req_mask[i] && tok_io && ~tok_l1; + assign flush_acq_mask[i] = flush_req_mask[i] && tok_io && tok_l1; + assign flush_rel_mask[i] = flush_req_mask[i] && ~tok_io && tok_l1 && tok_rel; + assign flush_l1_mask[i] = flush_req_mask[i] && ~tok_io && tok_l1 && ~tok_rel; end - wire any_full = (| (flush_req_mask & ~flush_l1_mask & ~flush_io_mask)); + wire any_full = (| (flush_req_mask & ~flush_l1_mask & ~flush_io_mask & ~flush_acq_mask & ~flush_rel_mask)); wire any_l1 = (| flush_l1_mask); wire any_io = (| flush_io_mask); - // every visible token is L1-only - wire flush_l1_only = any_l1 && ~any_full && ~any_io; - // a walk is owed to a full token, and to an L1-only one above the point of coherence - wire flush_walk = any_full || (any_l1 && (SHARED_LLC == 0)); + wire any_acq = (| flush_acq_mask); + wire any_rel = (| flush_rel_mask); + // a walk is owed to a full token, and to an L1-only or acquire one above + // the point of coherence + wire flush_walk = any_full || ((any_l1 || any_acq) && (SHARED_LLC == 0)); + // a drain with no walk is owed to an IO token, and to a release one above + // the point of coherence + wire flush_drain = any_io || (any_rel && (SHARED_LLC == 0)); reg [NUM_BANKS-1:0] flush_done, flush_done_n; reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now @@ -147,6 +164,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( reg [`UP(UUID_WIDTH)-1:0] flush_uuid_r, flush_uuid_n; reg l1_only_r, l1_only_n; reg io_r, io_n; + reg noalloc_r, noalloc_n; reg skip_r, skip_n; // answer the lowest pending input; its token stays at the input until then @@ -230,7 +248,13 @@ module VX_cache_init import VX_gpu_pkg::*; #( // an IO token also waits for the non-cacheable path, which carries the IO // writes it orders - wire fwd_ready = mem_req_empty && (~io_r || nc_req_empty); + wire fwd_ready = mem_req_empty && (~io_r || l1_only_r || nc_req_empty); + + // The kind forwarded after a walk: full if a full token or an IO one asks + // for more than the point of coherence, L1-only if any token owes a drain + // to it, acquire if only acquire tokens were visible. + wire walk_fwd_acq = any_acq && ~any_full && ~any_io && ~any_l1 && ~any_rel; + wire walk_fwd_l1 = (any_l1 || any_rel) && ~any_full && ~any_io; always @(*) begin state_n = state; @@ -242,6 +266,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( flush_uuid_n = flush_uuid_r; l1_only_n = l1_only_r; io_n = io_r; + noalloc_n = noalloc_r; skip_n = skip_r; case (state) // STATE_IDLE: @@ -249,7 +274,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( if (flush_req_enable) begin if (XBAR_SIZE != 0) begin state_n = STATE_WAIT1; - end else if (~flush_walk && ~any_io) begin + end else if (~flush_walk && ~flush_drain) begin rsp_pending_n = flush_req_mask; state_n = STATE_RESPOND; end else begin @@ -265,7 +290,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( end STATE_WAIT1: begin if (no_inflight_reqs) begin - if (~flush_walk && ~any_io) begin + if (~flush_walk && ~flush_drain) begin rsp_pending_n = flush_req_mask; state_n = STATE_RESPOND; end else begin @@ -284,12 +309,17 @@ module VX_cache_init import VX_gpu_pkg::*; #( if (flush_done_n == {NUM_BANKS{1'b1}}) begin flush_done_n = '0; // answer the tokens visible now; later ones start a new walk. - // A quiesce without a walk serves only the tokens owing none. - rsp_pending_n = skip_r ? (flush_io_mask | (SHARED_LLC ? flush_l1_mask : '0)) + // A quiesce without a walk serves only the tokens owing none, + // and forwards IO if it serves an IO token, else release. + rsp_pending_n = skip_r ? (flush_io_mask | flush_rel_mask + | (SHARED_LLC ? (flush_l1_mask | flush_acq_mask) : '0)) : flush_req_mask; - l1_only_n = flush_l1_only && ~skip_r; - io_n = skip_r; - state_n = FLUSH_FORWARD ? STATE_FORWARD : STATE_RESPOND; + l1_only_n = skip_r ? ~any_io : (walk_fwd_l1 || walk_fwd_acq); + io_n = skip_r ? any_io : walk_fwd_acq; + noalloc_n = skip_r && ~any_io; + // an acquire walk owes the next level nothing unless it too + // may hold stale lines + state_n = (FLUSH_FORWARD && (skip_r || ~walk_fwd_acq || ACQ_FORWARD)) ? STATE_FORWARD : STATE_RESPOND; end end STATE_FORWARD: begin @@ -330,6 +360,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( flush_uuid_r <= flush_uuid_n; l1_only_r <= l1_only_n; io_r <= io_n; + noalloc_r <= noalloc_n; skip_r <= skip_n; end @@ -343,5 +374,6 @@ module VX_cache_init import VX_gpu_pkg::*; #( assign mem_flush_rsp_phase = (state == STATE_FORWARD); assign mem_flush_l1_only = l1_only_r; assign mem_flush_io = io_r; + assign mem_flush_noalloc = noalloc_r; endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 07b7e2b310..597e63a042 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -61,6 +61,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // Set when this cache is the LLC (enables AMO completion at this level). parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). + parameter ACQ_FORWARD = 0, // An acquire flush also invalidates the next level. parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). @@ -165,6 +166,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .MEM_OUT_BUF (BYPASS_ENABLE ? 1 : MEM_OUT_BUF), .IS_LLC (IS_LLC), .SHARED_LLC (SHARED_LLC), + .ACQ_FORWARD (ACQ_FORWARD), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .READ_MERGE (READ_MERGE), diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 538164a70a..a3c30a5b67 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -357,6 +357,13 @@ module VX_decode import op_args.lsu.is_float = 0; op_args.lsu.pack = 0; op_args.lsu.offset = 0; + // A later read must see other harts' data (acquire) when the + // successor set reads; earlier writes must be visible (release) + // when the predecessor set writes. A fence with neither + // releases; one with a device access in its predecessor set, + // or fence.i, does both. + op_args.lsu.amo_aq = (funct3 != 3'b000) || (instr[27:26] != 2'b00) || instr[21]; + op_args.lsu.amo_rl = (funct3 != 3'b000) || (instr[27:26] != 2'b00) || instr[24] || ~instr[21]; end `ifdef VX_CFG_EXT_A_ENABLE INST_AMO: begin diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index c43935bc21..46412976c2 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -95,6 +95,10 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire req_is_fence, rsp_is_fence; wire req_fence_io = inst_lsu_is_fence_io(execute_if.data.op_type); + // a memory fence that orders only later reads (acquire) or only earlier + // writes (release) + wire req_fence_acq = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_rl; + wire req_fence_rel = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_aq; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr @@ -119,7 +123,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire [MEM_ADDRW-1:0] io_addr_start = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >> MEM_ASHIFT); wire [MEM_ADDRW-1:0] io_addr_end = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) >> MEM_ASHIFT); assign mem_req_attr_struct[i].is_flush = req_is_fence; - assign mem_req_attr_struct[i].is_noalloc = 1'b0; + // a release fence's token: no invalidation owed + assign mem_req_attr_struct[i].is_noalloc = req_is_fence && req_fence_rel; `ifdef VX_CFG_EXT_OM_ENABLE // OM fragment-export aperture. // @@ -139,8 +144,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( assign mem_req_attr_struct[i].is_addr_om = 1'b0; `endif // a fence's token carries its kind: io for a fence ordering only device - // accesses - assign mem_req_attr_struct[i].is_addr_io = req_is_fence ? req_fence_io + // accesses, and for an acquire fence's (byte enables set: no drain owed) + assign mem_req_attr_struct[i].is_addr_io = req_is_fence ? (req_fence_io || req_fence_acq) : (((block_addr >= io_addr_start) && (block_addr < io_addr_end)) || lane_is_om); `ifdef VX_CFG_LMEM_ENABLE diff --git a/hw/rtl/core/VX_uop_amo.sv b/hw/rtl/core/VX_uop_amo.sv new file mode 100644 index 0000000000..c0e54dd216 --- /dev/null +++ b/hw/rtl/core/VX_uop_amo.sv @@ -0,0 +1,63 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// VX_uop_amo: orders an atomic with the hart's other accesses. +// +// An AMO, LR or SC with .rl is preceded by a release fence (earlier writes +// become visible first) and one with .aq is followed by an acquire fence +// (later reads see what the atomic observed): +// [release fence] atomic [acquire fence] +// The fences are plain LSU fence ops whose aq/rl arguments carry their kind; +// they read and write no register. + +module VX_uop_amo import VX_gpu_pkg::*; ( + input clk, + input reset, + + input ibuffer_t ibuf_in, + output ibuffer_t ibuf_out, + + input wire start, + input wire advance, + input wire [UOP_CTR_W-1:0] uop_idx, + output wire [UOP_CTR_W-1:0] uop_count +); + `UNUSED_VAR ({clk, reset, start, advance}) + + wire aq = ibuf_in.op_args.lsu.amo_aq; + wire rl = ibuf_in.op_args.lsu.amo_rl; + assign uop_count = UOP_CTR_W'(1) + UOP_CTR_W'(aq) + UOP_CTR_W'(rl); + + wire [UOP_CTR_W-1:0] amo_idx = UOP_CTR_W'(rl); + wire is_rel = rl && (uop_idx == '0); + wire is_acq = aq && (uop_idx == amo_idx + UOP_CTR_W'(1)); + + ibuffer_t ibuf_r; + always_comb begin + ibuf_r = ibuf_in; + if (is_rel || is_acq) begin + ibuf_r.op_type = INST_OP_BITS'(INST_LSU_FENCE); + ibuf_r.op_args = '0; + ibuf_r.op_args.lsu.amo_aq = is_acq; + ibuf_r.op_args.lsu.amo_rl = is_rel; + ibuf_r.wb = 1'b0; + ibuf_r.rd_xregs = '0; + ibuf_r.wr_xregs = '0; + ibuf_r.used_rs = '0; + end + end + assign ibuf_out = ibuf_r; + +endmodule diff --git a/hw/rtl/core/VX_uop_sequencer.sv b/hw/rtl/core/VX_uop_sequencer.sv index 268450164c..3fb0ffcf49 100644 --- a/hw/rtl/core/VX_uop_sequencer.sv +++ b/hw/rtl/core/VX_uop_sequencer.sv @@ -193,6 +193,25 @@ module VX_uop_sequencer import ); `endif +`ifdef VX_CFG_EXT_A_ENABLE + // ------------------------------------------------------------------ + // Atomic ordering uop expander: fences around an AMO/LR/SC with .aq/.rl + // ------------------------------------------------------------------ + assign uop_in_valid[UOP_AMO] = (uop_in_data.ex_type == EX_LSU) + && uop_in_data.op_args.lsu.amo_valid + && (uop_in_data.op_args.lsu.amo_aq || uop_in_data.op_args.lsu.amo_rl); + VX_uop_amo uop_amo ( + .clk (clk), + .reset (reset), + .ibuf_in (uop_in_data), + .start (uop_in_start[UOP_AMO]), + .advance (uop_in_next[UOP_AMO]), + .uop_idx (uop_ctr), + .ibuf_out (uop_out_data[UOP_AMO]), + .uop_count (uop_out_count[UOP_AMO]) + ); +`endif + wire uop_hold = is_uop_input && ~uop_active; assign output_if.valid = uop_active || (input_if.valid && ~uop_hold); diff --git a/sim/simx/constants.h b/sim/simx/constants.h index a1cf857579..941eeddea2 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -52,6 +52,9 @@ inline constexpr bool DCACHE_SHARED_LLC = (VX_CFG_DCACHE_ENABLED != 0) && (VX_CF && !VX_CFG_EXT_DXA_ENABLED; inline constexpr bool L2_SHARED_LLC = (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0) && (VX_CFG_NUM_CLUSTERS == 1); inline constexpr bool L3_SHARED_LLC = (VX_CFG_L3_ENABLED != 0); +// An acquire must also invalidate the cache below the dcache unless that one +// is the point of coherence (an L3 always is; memory always is). +inline constexpr bool DCACHE_ACQ_FORWARD = (VX_CFG_L2_ENABLED != 0) && !L2_SHARED_LLC; // A write-back LLC must be the only instance: replicated ones never make their // dirty lines visible to each other. diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index e0a3d8593e..5a559d4726 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -631,10 +631,24 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { case Opcode::FENCE: { instr->set_fu_type(FUType::LSU); instr->set_op_type(LsuType::FENCE); - // neither the predecessor nor the successor set holds a memory read or - // write (pred = code[27:24], succ = code[23:20], IORW) - uint32_t fence_io = (funct3 == 0) && ((code >> 24) & 0x3) == 0 && ((code >> 20) & 0x3) == 0; - instr->set_args(IntrLsuArgs{fence_io, 0, 0}); + // pred = code[27:24], succ = code[23:20], IORW. A fence with no memory + // read or write in either set orders only device accesses. Otherwise a + // later read must see other harts' data (acquire) when the successor set + // reads, and earlier writes must be visible (release) when the + // predecessor set writes. A fence with neither releases; one with a + // device access in its predecessor set, or fence.i, does both. + uint32_t pred = (code >> 24) & 0xf; + uint32_t succ = (code >> 20) & 0xf; + FenceKind kind; + if (funct3 == 0 && (pred & 0x3) == 0 && (succ & 0x3) == 0) { + kind = FenceKind::IO; + } else { + bool full = (funct3 != 0) || (pred & 0xc) != 0; + bool aq = full || (succ & 0x2) != 0; + bool rl = full || (pred & 0x1) != 0 || (succ & 0x2) == 0; + kind = (aq && rl) ? FenceKind::FULL : (aq ? FenceKind::ACQUIRE : FenceKind::RELEASE); + } + instr->set_args(IntrLsuArgs{static_cast(kind), 0, 0}); } break; case Opcode::AMO: { instr->set_fu_type(FUType::LSU); @@ -657,6 +671,10 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { std::abort(); } instr->set_args(IntrAmoArgs{funct3, aq, rl}); + // .aq/.rl order the atomic with fences around it (LsuUopGen) + if (aq || rl) { + instr->set_macro_op(); + } instr->set_dest_reg(rd, RegType::Integer); instr->set_src_reg(0, rs1, RegType::Integer); instr->set_src_reg(1, rs2, RegType::Integer); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 7a887721f7..22b4a9a0ab 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -29,6 +29,10 @@ using namespace vortex; uint32_t LsuUopGen::uop_count(const Instr& instr) { + // An atomic with .aq/.rl: [release fence] atomic [acquire fence] + if (auto amo = std::get_if(&instr.get_args())) { + return 1 + amo->aq + amo->rl; + } // PACKLB.F: width=0 (LB), 4 byte-elements // PACKLH.F: width=1 (LH), 2 halfword-elements auto args = std::get(instr.get_args()); @@ -46,6 +50,30 @@ Instr::Ptr LsuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { uint32_t steps_shift = (total > 1) ? (32 - log2ceil(total)) : 0; uint64_t uop_uuid = (static_cast(uuid_hi) << 32) | ((uop_index << steps_shift) | uuid_lo); + if (auto amo = std::get_if(¯o_instr.get_args())) { + auto uop_instr = std::allocate_shared(pool_, uop_uuid, FUType::LSU); + uop_instr->set_parent_uuid(parent_uuid); + uint32_t amo_index = amo->rl; + if (uop_index == amo_index) { + uop_instr->set_op_type(macro_instr.get_op_type()); + uop_instr->set_args(*amo); + // an atomic into x0 writes nothing back + auto rd = macro_instr.get_dest_reg(); + if (!(rd.type == RegType::Integer && rd.idx == 0)) { + uop_instr->set_dest_reg(rd.idx, rd.type); + } + for (uint32_t i = 0; i < Instr::MAX_REG_SOURCES; ++i) { + auto rs = macro_instr.get_src_reg(i); + uop_instr->set_src_reg(i, rs.idx, rs.type); + } + } else { + auto kind = (uop_index < amo_index) ? FenceKind::RELEASE : FenceKind::ACQUIRE; + uop_instr->set_op_type(LsuType::FENCE); + uop_instr->set_args(IntrLsuArgs{static_cast(kind), 0, 0}); + } + return uop_instr; + } + auto args = std::get(macro_instr.get_args()); // Macro encodes elem-size as width: 0=byte (PACKLB), 1=halfword (PACKLH). // The uop is a regular unsigned load (LBU/LHU = width + 4) so the LSU @@ -375,10 +403,12 @@ void LsuUnit::send_fence_token(uint32_t b) { LsuReq lsu_req(VX_CFG_NUM_LSU_LANES); lsu_req.op = MemOp::LD; lsu_req.flags.flush = 1; - // an IO fence's token walks nowhere; a memory fence's stops at the shared LLC - auto fence_args = std::get(trace->instr_ptr->get_args()); - lsu_req.flags.io = lsu_fence_is_io(fence_args); - lsu_req.flags.flush_l1 = !lsu_fence_is_io(fence_args); + // an IO fence's token walks nowhere; a memory fence's stops at the shared + // LLC, an acquire one owing no drain and a release one no invalidation + auto kind = lsu_fence_kind(std::get(trace->instr_ptr->get_args())); + lsu_req.flags.io = (kind == FenceKind::IO) || (kind == FenceKind::ACQUIRE); + lsu_req.flags.flush_l1 = (kind != FenceKind::IO); + lsu_req.flags.noalloc = (kind == FenceKind::RELEASE); std::vector lane_entries(VX_CFG_NUM_LSU_LANES); for (uint32_t t = base; t < VX_CFG_NUM_THREADS; ++t) { if (trace->tmask.test(t)) { diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index ccf1a1d6ba..77b3fc79eb 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2371,8 +2371,8 @@ class CachePortArb : public SimObject { return mask; } - // Tokens owing no walk: IO tokens, and L1-only ones unless they still need - // a walk here. + // Tokens owing no walk: IO and release tokens, and L1-only or acquire ones + // unless they still need a walk here. std::vector tokens(bool l1_needs_walk) const { std::vector mask(ReqIn.size(), false); for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { @@ -2380,7 +2380,9 @@ class CachePortArb : public SimObject { continue; } auto &flags = ReqIn.at(i).peek().flags; - mask.at(i) = flags.io || (flags.flush_l1 && !l1_needs_walk); + bool io = flags.io && !flags.flush_l1; + bool rel = !flags.io && flags.flush_l1 && flags.noalloc; + mask.at(i) = io || rel || (flags.flush_l1 && !l1_needs_walk); } return mask; } @@ -2986,13 +2988,13 @@ class Cache::Impl { } auto kinds = this->token_kinds(); bool walk = kinds.walk(config_.shared_llc); - if (!walk && !kinds.any_io) { + if (!walk && !kinds.drain(config_.shared_llc)) { rsp_pending_ = port_arb_->tokens(); flush_state_ = FlushState::Respond; break; } - // with no walk owed, an IO token still quiesces the banks and goes down - // to memory + // with no walk owed, an IO or release token still quiesces the banks + // and goes down walk_skip_ = !walk; for (auto &bank : banks_) { bank->token_flush_begin(walk_skip_); @@ -3007,10 +3009,23 @@ class Cache::Impl { } } auto kinds = this->token_kinds(); - // a quiesce without a walk serves only the tokens owing none + // a quiesce without a walk serves only the tokens owing none, and + // forwards IO if it serves an IO token, else release rsp_pending_ = walk_skip_ ? port_arb_->tokens(!config_.shared_llc) : port_arb_->tokens(); - this->forward_begin(walk_skip_, !walk_skip_ && kinds.any_l1 && !kinds.any_full && !kinds.any_io); - flush_state_ = FlushState::Forward; + // after a walk: full if a full token or an IO one asks for more than the + // point of coherence, L1-only if any token owes a drain to it, acquire + // if only acquire tokens were visible + bool fwd_acq = !walk_skip_ && kinds.any_acq && !kinds.any_full && !kinds.any_io + && !kinds.any_l1 && !kinds.any_rel; + bool fwd_l1 = !walk_skip_ && (kinds.any_l1 || kinds.any_rel) && !kinds.any_full && !kinds.any_io; + if (walk_skip_) { + this->forward_begin(kinds.any_io, !kinds.any_io, !kinds.any_io); + } else { + this->forward_begin(fwd_acq, fwd_l1 || fwd_acq, false); + } + // an acquire walk owes the next level nothing unless it too may hold + // stale lines + flush_state_ = (fwd_acq && !config_.acq_forward) ? FlushState::Respond : FlushState::Forward; break; } case FlushState::Forward: { @@ -3023,7 +3038,7 @@ class Cache::Impl { } // an IO token also waits for the non-cacheable path, which carries the // IO writes it orders - if (fwd_io_ && mem_req_empty) { + if (fwd_io_ && !fwd_l1_ && mem_req_empty) { for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { mem_req_empty &= (port_arb_->IoOut.at(i).size() == 0); } @@ -3043,6 +3058,7 @@ class Cache::Impl { token.flags.flush = 1; token.flags.io = fwd_io_; token.flags.flush_l1 = fwd_l1_; + token.flags.noalloc = fwd_noalloc_; token.tag = 0; if (bank_arb_->ReqIn.at(this->bank_lane(i)).try_send(token, 1)) { DT(3, simobject_->name() << " flush-fwd: " << token); @@ -3084,8 +3100,14 @@ class Cache::Impl { bool any_full = false; bool any_l1 = false; bool any_io = false; - // a walk is owed to a full token, and to an L1-only one above the point of coherence - bool walk(bool shared_llc) const { return any_full || (any_l1 && !shared_llc); } + bool any_acq = false; + bool any_rel = false; + // a walk is owed to a full token, and to an L1-only or acquire one above + // the point of coherence + bool walk(bool shared_llc) const { return any_full || ((any_l1 || any_acq) && !shared_llc); } + // a drain with no walk is owed to an IO token, and to a release one above + // the point of coherence + bool drain(bool shared_llc) const { return any_io || (any_rel && !shared_llc); } }; // Kinds of the visible tokens. Several kinds together combine to the @@ -3098,9 +3120,17 @@ class Cache::Impl { } auto &flags = port_arb_->ReqIn.at(i).peek().flags; if (flags.io) { - kinds.any_io = true; + if (flags.flush_l1) { + kinds.any_acq = true; + } else { + kinds.any_io = true; + } } else if (flags.flush_l1) { - kinds.any_l1 = true; + if (flags.noalloc) { + kinds.any_rel = true; + } else { + kinds.any_l1 = true; + } } else { kinds.any_full = true; } @@ -3108,9 +3138,10 @@ class Cache::Impl { return kinds; } - void forward_begin(bool io, bool l1_only) { + void forward_begin(bool io, bool l1_only, bool noalloc) { fwd_io_ = io; fwd_l1_ = l1_only; + fwd_noalloc_ = noalloc; fwd_go_ = false; fwd_sent_ = 0; fwd_done_ = 0; @@ -3200,6 +3231,7 @@ class Cache::Impl { bool fwd_go_ = false; // nothing is left ahead of the forwarded tokens bool fwd_io_ = false; // the forwarded tokens' kind bool fwd_l1_ = false; + bool fwd_noalloc_ = false; bool walk_skip_ = false; // the banks only quiesce: no walk is owed std::vector> mem_req_net_; std::vector flush_taps_; diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 70adc78ba5..92bde10407 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -54,6 +54,7 @@ class Cache : public SimObject { bool mem_rsp_queue = false; // memory responses enter through a response queue bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register bool merge_port0 = true; // port 0's reads match other ports' tags for read merging + bool acq_forward = false; // an acquire flush also invalidates the next level }; struct PerfStats { diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index d2df85bb0c..083f880cd8 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -114,6 +114,7 @@ class Socket::Impl { // A private cache takes the flush injection on port 0 alone, and its // arbiter re-tags that port's requests apart from the others. VX_CFG_SOCKET_SIZE != VX_CFG_NUM_DCACHES, // port 0 merges reads + DCACHE_ACQ_FORWARD, // acq_forward }); #ifdef VX_CFG_EXT_TEX_ENABLE diff --git a/sim/simx/types.h b/sim/simx/types.h index a7eda35ac2..0fb7ab61ea 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -386,9 +386,14 @@ struct IntrLsuArgs { int32_t offset; // immediate addr offset (signed); 0 for packLD uops }; -// A fence carries no access width; its width field says it orders only -// device (IO) accesses. -inline bool lsu_fence_is_io(const IntrLsuArgs& args) { return args.width != 0; } +// A fence carries no access width; its width field says what it orders. +enum class FenceKind : uint32_t { + FULL = 0, // earlier writes visible and later reads see others' writes + IO = 1, // only device (IO) accesses + ACQUIRE = 2, // only later reads see others' writes + RELEASE = 3 // only earlier writes visible +}; +inline FenceKind lsu_fence_kind(const IntrLsuArgs& args) { return static_cast(args.width); } inline std::ostream &operator<<(std::ostream &os, const LsuType& type) { switch (type) { diff --git a/tests/regression/fence_mp/common.h b/tests/regression/fence_mp/common.h index 634f577e13..8d2f0a9994 100644 --- a/tests/regression/fence_mp/common.h +++ b/tests/regression/fence_mp/common.h @@ -5,11 +5,18 @@ #define FENCE_MP_MAGIC 0x5a5a0000u +// How the producer publishes the flag and the consumer observes it. +enum { + MP_FENCE = 0, // full fences on both sides + MP_ACQ_REL = 1, // a release fence before the flag store, an acquire fence after each flag load + MP_AMO = 2 // the flag swapped in by an amoswap.rl, read by an amoor.aq +}; + typedef struct { uint32_t num_tasks; // threads launched uint32_t count; // data words passed from producer to consumer uint32_t spin_limit; // consumer polls before giving up - uint32_t pad; + uint32_t mode; // MP_* uint64_t data_addr; // count words, zero on entry uint64_t flag_addr; // one word, zero on entry uint64_t out_addr; // [0] mismatches, [1] 1 if the consumer timed out, [2] pre-read sum diff --git a/tests/regression/fence_mp/kernel.cpp b/tests/regression/fence_mp/kernel.cpp index 80b47e7909..6d716a0aeb 100644 --- a/tests/regression/fence_mp/kernel.cpp +++ b/tests/regression/fence_mp/kernel.cpp @@ -3,11 +3,44 @@ #include "common.h" // Message passing across cores through plain memory: the producer (the first -// thread) writes data, fences, then raises a flag; the consumer (the last -// thread, on the last core) reads the data once beforehand so stale copies sit -// in its caches, polls the flag behind a fence, fences again and checks the -// data. A memory fence must make the data visible across every cache the two -// cores do not share. +// thread) writes data, then raises a flag; the consumer (the last thread, on +// the last core) reads the data once beforehand so stale copies sit in its +// caches, polls the flag, and checks the data. The release must make the data +// visible across every cache the two cores do not share before the flag, and +// the acquire must drop the consumer's stale copies, whether they are full +// fences, acquire/release fences, or the .aq/.rl bits of atomics. +static void publish(volatile uint32_t* flag, uint32_t mode) { + switch (mode) { + case MP_ACQ_REL: + __atomic_thread_fence(__ATOMIC_RELEASE); + *flag = 1; + break; + case MP_AMO: + __atomic_exchange_n(const_cast(flag), 1u, __ATOMIC_RELEASE); + break; + default: + vx_fence(); + *flag = 1; + vx_fence(); + break; + } +} + +static bool observe(volatile uint32_t* flag, uint32_t mode) { + switch (mode) { + case MP_ACQ_REL: { + uint32_t f = *flag; + __atomic_thread_fence(__ATOMIC_ACQUIRE); + return f != 0; + } + case MP_AMO: + return __atomic_fetch_or(const_cast(flag), 0u, __ATOMIC_ACQUIRE) != 0; + default: + vx_fence(); + return *flag != 0; + } +} + __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto data = reinterpret_cast(arg->data_addr); @@ -17,9 +50,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { for (uint32_t k = 0; k < arg->count; ++k) { data[k] = FENCE_MP_MAGIC + k; } - vx_fence(); - *flag = 1; - vx_fence(); + publish(flag, arg->mode); } else if (gid == arg->num_tasks - 1) { uint32_t sum = 0; for (uint32_t k = 0; k < arg->count; ++k) { @@ -28,13 +59,14 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { out[2] = sum; uint32_t timeout = 1; for (uint32_t s = 0; s < arg->spin_limit; ++s) { - vx_fence(); - if (*flag != 0) { + if (observe(flag, arg->mode)) { timeout = 0; break; } } - vx_fence(); + if (arg->mode == MP_FENCE) { + vx_fence(); + } uint32_t errors = 0; for (uint32_t k = 0; k < arg->count; ++k) { errors += (data[k] != FENCE_MP_MAGIC + k); diff --git a/tests/regression/fence_mp/main.cpp b/tests/regression/fence_mp/main.cpp index a8b2c05393..6439256482 100644 --- a/tests/regression/fence_mp/main.cpp +++ b/tests/regression/fence_mp/main.cpp @@ -18,6 +18,7 @@ const char* kernel_file = "kernel.vxbin"; uint32_t count = 64; uint32_t spin_limit = 4096; +uint32_t mode = MP_FENCE; vx_device_h device = nullptr; vx_buffer_h data_buffer = nullptr; @@ -30,15 +31,16 @@ kernel_arg_t kernel_arg = {}; static void show_usage() { std::cout << "Vortex Test." << std::endl; - std::cout << "Usage: [-k: kernel] [-n words] [-s spin limit] [-h: help]" << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-s spin limit] [-m mode: 0 fences, 1 acquire/release fences, 2 .aq/.rl atomics] [-h: help]" << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "n:s:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:s:m:k:h")) != -1) { switch (c) { case 'n': count = atoi(optarg); break; case 's': spin_limit = atoi(optarg); break; + case 'm': mode = atoi(optarg); break; case 'k': kernel_file = optarg; break; case 'h': show_usage(); exit(0); break; default: show_usage(); exit(-1); @@ -75,6 +77,7 @@ int main(int argc, char *argv[]) { kernel_arg.num_tasks = total_threads; kernel_arg.count = count; kernel_arg.spin_limit = spin_limit; + kernel_arg.mode = mode; std::vector zeros(count, 0); std::vector out(4, 0); From d6be497d75fe0e2cd46cd01fd5f3fd85c38be30d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:50 -0700 Subject: [PATCH 104/336] core: keep the dcache tag whole on passthru lanes so port 0 reads merge VX_mem_unit's passthru lanes shifted the LSU tag right by one and cleared its MSB, while lane 0, which also takes the flush injection, kept its select bit at the LSB. One request's tag was therefore encoded differently on port 0 than on the other ports, and the dcache's same-tag read merge never paired port 0 with another port. The tag now passes unchanged; its clear LSB sits where the flush lane's select bit does. SimX drops the merge_port0 knob that modeled the old behavior. SimX's LSU-to-memory path also follows the RTL more closely: - The lmem switch splits combinationally. The LSU ticks first and hands a request over with no delay, and the switch routes it into its output buffers in that cycle. The LSU checks room only in the buffers the beat addresses (req_ready), instead of a one-entry ingress. - The switch's response arbiter has a one-entry output buffer, which the coalescer or a multi-channel lane adapter fills combinationally. - A coalesced batch crosses the coalescer's output register and then the lane adapter's request buffer. The adapter registers its responses. - The dcache's MSHR occupancy is a registered count: a slot freed this cycle admits a request only from the next. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_mem_unit.sv | 10 ++++---- sim/simx/core.cpp | 40 ++++++++++++++++++++----------- sim/simx/lsu_unit.cpp | 37 +++++++++++++++++++++------- sim/simx/mem/cache.cpp | 16 ++++++++----- sim/simx/mem/cache.h | 1 - sim/simx/mem/local_mem_switch.cpp | 13 +++++----- sim/simx/mem/local_mem_switch.h | 11 +++++++-- sim/simx/mem/mem_coalescer.cpp | 4 ++-- sim/simx/socket.cpp | 3 --- 9 files changed, 87 insertions(+), 48 deletions(-) diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 11a8228b36..6ed8f18c83 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -255,22 +255,22 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .cache_bus_if (dcache_bus_if[i]) ); end else begin : g_passthru_lane - // the cache sees the select bit as a clear tag MSB + // The tag passes unchanged: its clear LSB stands where the flush lane's + // select bit does, so every lane encodes one request's tag alike and + // the cache's read merge compares like with like. assign dcache_bus_if[i].req_valid = lsu_dcache_bus_if[i].req_valid; assign dcache_bus_if[i].req_data.rw = lsu_dcache_bus_if[i].req_data.rw; assign dcache_bus_if[i].req_data.addr = lsu_dcache_bus_if[i].req_data.addr; assign dcache_bus_if[i].req_data.data = lsu_dcache_bus_if[i].req_data.data; assign dcache_bus_if[i].req_data.byteen = lsu_dcache_bus_if[i].req_data.byteen; assign dcache_bus_if[i].req_data.attr = lsu_dcache_bus_if[i].req_data.attr; - assign dcache_bus_if[i].req_data.tag = {1'b0, lsu_dcache_bus_if[i].req_data.tag[DCACHE_TAG_WIDTH-1:1]}; + assign dcache_bus_if[i].req_data.tag = lsu_dcache_bus_if[i].req_data.tag; assign lsu_dcache_bus_if[i].req_ready = dcache_bus_if[i].req_ready; - `UNUSED_VAR (lsu_dcache_bus_if[i].req_data.tag[0]) assign lsu_dcache_bus_if[i].rsp_valid = dcache_bus_if[i].rsp_valid; assign lsu_dcache_bus_if[i].rsp_data.data = dcache_bus_if[i].rsp_data.data; - assign lsu_dcache_bus_if[i].rsp_data.tag = {dcache_bus_if[i].rsp_data.tag[DCACHE_TAG_WIDTH-2:0], 1'b0}; + assign lsu_dcache_bus_if[i].rsp_data.tag = dcache_bus_if[i].rsp_data.tag; assign dcache_bus_if[i].rsp_ready = lsu_dcache_bus_if[i].rsp_ready; - `UNUSED_VAR (dcache_bus_if[i].rsp_data.tag[DCACHE_TAG_WIDTH-1]) end end diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 8742972a59..8e89758a9b 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -131,17 +131,26 @@ class Core::Impl { ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE + 1); } + // The LSU ticks ahead of the coalescer, the lane adapters and the lmem + // switch. The switch routes a request into its output buffers in the cycle + // the LSU hands it over, as the RTL's combinational split does. The + // switch's one-entry response buffer is drained by the LSU before the + // coalescer or an adapter refills it, its ready being combinational in the + // RTL. + snprintf(sname, 100, "%s-lsu", name.c_str()); + func_units_.at((int)FUType::LSU) = SimPlatform::instance().create_object(sname, simobject_); + // create the memory coalescer, or the lane adapter that replaces it. - // Both tick ahead of the lmem switch: their response paths are - // combinational, and the dcache registers its output. const bool dcache_coalescing = (VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE); std::vector lsu_dcache_adapter(VX_CFG_NUM_LSU_BLOCKS); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { snprintf(sname, 100, "%s-coalescer%d", name.c_str(), b); - mem_coalescers_.at(b) = MemCoalescer::Create(sname, LSU_CHANNELS, DCACHE_CHANNELS, DCACHE_WORD_SIZE, LSU_QUEUE_OUT_SIZE, 1); + // a batch crosses the coalescer's output register, then the lane + // adapter's request buffer, on its way to the data cache + mem_coalescers_.at(b) = MemCoalescer::Create(sname, LSU_CHANNELS, DCACHE_CHANNELS, DCACHE_WORD_SIZE, LSU_QUEUE_OUT_SIZE, 2); if (!dcache_coalescing) { snprintf(sname, 100, "%s-lsu_dcache_adapter%d", name.c_str(), b); - lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1, 0); + lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1, 1); } } @@ -158,14 +167,10 @@ class Core::Impl { VX_CFG_EXT_DXA_ENABLED + VX_CFG_TCU_WGMMA_ENABLED }); - // create lmem switch. A data-cache response reaches its response - // arbiter combinationally through the coalescer or the lane adapter, so - // the switch holds it in one entry, the least a channel holds; without - // either, the channel is the cache's own two-entry output buffer. - const bool dc_rsp_comb = dcache_coalescing || (DCACHE_CHANNELS > 1); + // create lmem switch for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { snprintf(sname, 100, "%s-lmem_switch%d", name.c_str(), b); - lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1, dc_rsp_comb ? 1 : 2); + lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1); } // create per-block lmem adapters @@ -198,7 +203,9 @@ class Core::Impl { // channels directly (combinational lane fan-out/fan-in). for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { lmem_switch_.at(b)->ReqOutDC.bind(&mem_coalescers_.at(b)->ReqIn); - mem_coalescers_.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); + // the coalescer's response reaches the switch's response buffer + // combinationally, ahead of a local-memory response + mem_coalescers_.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspOut); for (uint32_t c = 0; c < DCACHE_CHANNELS; ++c) { dc_req_out.at(b * DCACHE_CHANNELS + c) = &mem_coalescers_.at(b)->ReqOut.at(c); dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &mem_coalescers_.at(b)->RspIn.at(c); @@ -209,7 +216,14 @@ class Core::Impl { // pass-through when DCACHE_CHANNELS == 1) for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { lmem_switch_.at(b)->ReqOutDC.bind(&lsu_dcache_adapter.at(b)->ReqIn); - lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); + // A multi-channel adapter packs its response combinationally into the + // switch's response buffer; a single-channel one passes the cache's + // own output buffer through, which the switch then drains. + if (DCACHE_CHANNELS > 1) { + lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspOut); + } else { + lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); + } for (uint32_t c = 0; c < DCACHE_CHANNELS; ++c) { dc_req_out.at(b * DCACHE_CHANNELS + c) = &lsu_dcache_adapter.at(b)->ReqOut.at(c); dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &lsu_dcache_adapter.at(b)->RspIn.at(c); @@ -287,8 +301,6 @@ class Core::Impl { if (FpuUnit::kGather) { create_fpu(); } - snprintf(sname, 100, "%s-lsu", name.c_str()); - func_units_.at((int)FUType::LSU) = SimPlatform::instance().create_object(sname, simobject_); snprintf(sname, 100, "%s-sfu", name.c_str()); func_units_.at((int)FUType::SFU) = SimPlatform::instance().create_object(sname, simobject_); std::static_pointer_cast(func_units_.at((int)FUType::SFU))->warp_ctl_out.bind(&simobject_->warp_ctl_in); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 22b4a9a0ab..21240dd7b4 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -395,7 +395,8 @@ void LsuUnit::ingest_inputs(uint32_t b) { // the block until the token's response returns. void LsuUnit::send_fence_token(uint32_t b) { auto& state = states_.at(b); - if (state.pending_reqs.full() || core_->lmem_switch(b)->ReqIn.full()) { + // the token's lanes carry no address: it goes to the data cache + if (state.pending_reqs.full() || !core_->lmem_switch(b)->req_ready(true, false)) { return; } auto trace = state.fence.trace(); @@ -427,7 +428,7 @@ void LsuUnit::send_fence_token(uint32_t b) { lsu_req.cid = trace->cid; lsu_req.wid = trace->wid; lsu_req.uuid = trace->uuid; - core_->lmem_switch(b)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); DT(3, this->name() << " fence-token: " << lsu_req); state.fence_token = false; } @@ -459,8 +460,17 @@ void LsuUnit::process_request_step(uint32_t b) { // not gated by the fence lock — its entries still hold fence release via // pending_reqs.empty(). if (b < TCU_AGU_PORTS && !TcuReqIn.at(b).empty()) { - if (!state.pending_reqs.full() && !core_->lmem_switch(b)->ReqIn.full()) { - LsuReq lsu_req = TcuReqIn.at(b).peek(); + const LsuReq& meta_req = TcuReqIn.at(b).peek(); + bool to_dc = false, to_lmem = false; + for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { + if (meta_req.mask.test(i)) { + bool local = (get_addr_type(meta_req.addrs.at(i)) == AddrType::Shared); + to_lmem |= local; + to_dc |= !local; + } + } + if (!state.pending_reqs.full() && core_->lmem_switch(b)->req_ready(to_dc, to_lmem)) { + LsuReq lsu_req = meta_req; uint32_t count = lsu_req.mask.count(); std::vector lane_entries(VX_CFG_NUM_LSU_LANES); for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { @@ -482,7 +492,7 @@ void LsuUnit::process_request_step(uint32_t b) { entry.client_tag = lsu_req.tag; uint32_t tag = state.pending_reqs.allocate(std::move(entry)); lsu_req.tag = tag; - core_->lmem_switch(b)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); DT(3, this->name() << " tcu-meta-req: " << lsu_req); TcuReqIn.at(b).pop(); return; // one request per cycle into the switch @@ -579,8 +589,19 @@ void LsuUnit::process_request_step(uint32_t b) { } if (state.remain_addrs != 0) { - // check lmem switch backpressure - if (core_->lmem_switch(b)->ReqIn.full()) + // check backpressure at the switch buffers this beat addresses + uint32_t beat_t0 = state.addr_list.size() - state.remain_addrs; + uint32_t beat_len = std::min(VX_CFG_NUM_LSU_LANES, state.remain_addrs); + bool to_dc = false, to_lmem = false; + for (uint32_t i = 0; i < beat_len; ++i) { + auto& entry = state.addr_list.at(beat_t0 + i); + if (entry.size == 0) + continue; + bool local = (get_addr_type(entry.addr) == AddrType::Shared); + to_lmem |= local; + to_dc |= !local; + } + if (!core_->lmem_switch(b)->req_ready(to_dc, to_lmem)) return; // stall // setup memory request @@ -657,7 +678,7 @@ void LsuUnit::process_request_step(uint32_t b) { lsu_req.uuid = trace->uuid; // send memory request - core_->lmem_switch(b)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); DT(3, this->name() << " mem-req: " << lsu_req); // update stats diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 77b3fc79eb..4668fcb78b 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -884,6 +884,7 @@ class CacheBank : public SimObject { void on_reset() { perf_stats_ = Cache::PerfStats(); pending_mshr_size_ = 0; + mshr_occupancy_ = 0; pending_amo_probes_ = 0; pipe_core_lines_.clear(); pending_read_reqs_ = 0; @@ -930,6 +931,10 @@ class CacheBank : public SimObject { // priority and the forward drain yields (crsp_sent_). crsp_sent_ = false; + // MSHR occupancy is a registered count: slots a forward, replay or + // release frees this tick admit a new request only from the next one. + mshr_occupancy_ = mshr_.size() + pending_mshr_size_; + // Process next request at the head of the pipeline. if (!pipe_req_->empty()) { this->processRequests(); @@ -1203,7 +1208,7 @@ class CacheBank : public SimObject { // responses from draining — a deadlock. Reserve at admission. bool needs_mshr = !is_amo_passthru; (void)is_amo; - if (needs_mshr && (mshr_.size() + pending_mshr_size_) >= mshr_.capacity()) { + if (needs_mshr && mshr_occupancy_ >= mshr_.capacity()) { ++perf_stats_.mshr_stalls; return; } @@ -2188,6 +2193,7 @@ class CacheBank : public SimObject { std::vector sets_; MSHR mshr_; uint32_t pending_mshr_size_; + uint32_t mshr_occupancy_; uint32_t pending_amo_probes_; // AmoProbe requests in the pipe, each holding a table reservation // {set, tag, sector} of Core requests inside the pipe: their MSHR entries // exist only from pipe exit, so ordering checks against in-flight lines (the @@ -2317,7 +2323,7 @@ class CachePortArb : public SimObject { std::vector> IoOut; // non-cacheable requests, per port CachePortArb(const SimContext& ctx, const char* name, const params_t& params, - uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group, bool merge_port0) + uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group) : SimObject(ctx, name) , ReqIn(num_inputs, this) , ReqOut(num_banks, this) @@ -2328,7 +2334,6 @@ class CachePortArb : public SimObject { , delay_((num_inputs > 2) ? 1 : 0) , levels_(arb_tree_levels(num_inputs)) , merge_group_(merge_group) - , merge_port0_(merge_port0) { __assert(merge_group_ <= 1 || (merge_group_ <= 64 && ispow2(merge_group_) && (num_inputs % merge_group_) == 0), "a read-merge group must be a power of two of at most 64 dividing the ports"); @@ -2483,7 +2488,7 @@ class CachePortArb : public SimObject { auto& req_i = ReqIn.at(i).peek(); uint32_t base = (i / merge_group_) * merge_group_; for (uint32_t j = base; j < i; ++j) { - if (followers.at(j) || !this->is_plain_read(j) || (j == 0 && !merge_port0_)) { + if (followers.at(j) || !this->is_plain_read(j)) { continue; } auto& req_j = ReqIn.at(j).peek(); @@ -2589,7 +2594,6 @@ class CachePortArb : public SimObject { std::vector>> tree_stage_; // [level][bank][slice] std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; - bool merge_port0_; uint64_t collisions_ = 0; friend class SimObject; @@ -2859,7 +2863,7 @@ class Cache::Impl { // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); uint32_t merge_group = std::min(config_.read_merge, config_.num_inputs); - port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group, config_.merge_port0); + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group); for (uint32_t i = 0; i < config_.num_inputs; ++i) { simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); } diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 92bde10407..de87d79d01 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -53,7 +53,6 @@ class Cache : public SimObject { bool write_around = false; // a no-allocate write miss goes to memory (write-back only) bool mem_rsp_queue = false; // memory responses enter through a response queue bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register - bool merge_port0 = true; // port 0's reads match other ports' tags for read merging bool acq_forward = false; // an acquire flush also invalidates the next level }; diff --git a/sim/simx/mem/local_mem_switch.cpp b/sim/simx/mem/local_mem_switch.cpp index a1e64b6c10..29f131a851 100644 --- a/sim/simx/mem/local_mem_switch.cpp +++ b/sim/simx/mem/local_mem_switch.cpp @@ -18,18 +18,17 @@ using namespace vortex; LocalMemSwitch::LocalMemSwitch( const SimContext& ctx, const char* name, - uint32_t delay, - uint32_t dc_rsp_size + uint32_t delay ) : SimObject(ctx, name) - // Single-entry ingress: the switch accepts a new request only as its - // current one drains, so upstream issue feels downstream cadence - // immediately instead of running ahead into queue slack. + // The split is combinational: a request handed over is routed into the + // output buffers in the same cycle, and ReqIn holds nothing across cycles. , ReqIn(this, 1) - , RspOut(this) + // the response arbiter's one-entry output buffer + , RspOut(this, 1) , ReqOutLmem(this) , RspInLmem(this) , ReqOutDC(this) - , RspInDC(this, dc_rsp_size) + , RspInDC(this) , delay_(delay) {} diff --git a/sim/simx/mem/local_mem_switch.h b/sim/simx/mem/local_mem_switch.h index d765adaead..73158544ac 100644 --- a/sim/simx/mem/local_mem_switch.h +++ b/sim/simx/mem/local_mem_switch.h @@ -33,10 +33,17 @@ class LocalMemSwitch : public SimObject { LocalMemSwitch( const SimContext& ctx, const char* name, - uint32_t delay, - uint32_t dc_rsp_size = 2 + uint32_t delay ); + // The output buffers a request addresses can take it this cycle: to the data + // cache, to local memory, or both. + bool req_ready(bool to_dc, bool to_lmem) const { + return ReqIn.size() == 0 + && !(to_dc && ReqOutDC.full()) + && !(to_lmem && ReqOutLmem.full()); + } + protected: void on_tick(); diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index 4940c9c5db..11bd5af5e3 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -107,8 +107,8 @@ void MemCoalescer::on_tick() { } } - // the response path is combinational; the cache registers its output - RspOut.send(out_rsp, 0); + // packed combinationally into the lmem switch's response buffer + RspOut.send(out_rsp, 1); DT(4, this->name() << " mem-rsp: " << out_rsp); // track remaining responses diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 083f880cd8..22b0f64820 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -111,9 +111,6 @@ class Socket::Impl { false, // write_around VX_CFG_DCACHE_MRSQ_SIZE != 0, // memory-response queue true, // the socket's registered memory port - // A private cache takes the flush injection on port 0 alone, and its - // arbiter re-tags that port's requests apart from the others. - VX_CFG_SOCKET_SIZE != VX_CFG_NUM_DCACHES, // port 0 merges reads DCACHE_ACQ_FORWARD, // acq_forward }); From a010af42d907f2883a9ab6468bf6959bdd072f3d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:51 -0700 Subject: [PATCH 105/336] cache: drop .aq/.rl fences at a coherent dcache, add a bank-0 delay knob A dcache that is itself the point of coherence commits every atomic in the one cache all harts use, so the fences around an .aq/.rl atomic order nothing more there. The uop sequencer and SimX decode now leave such atomics unexpanded. DBG_MEM_BANK0_WDELAY is a debug-only knob. A write-through cache holds each write at the head of bank 0's memory queue for 2048 cycles, so writes leaving through the other banks overtake it unless a release orders them. Without a delay, no shape exposes a release fence that fails to drain. fence_mp places its data in even 64-byte lines and its flag in an odd one, so on a two-bank dcache they leave through different banks, and it checks the data newest first. SimX models the knob as the RTL builds it: the delay holds the head of bank 0's in-order memory queue, so everything queued behind the write waits too. Every bank memory send goes through send_mem_req(), which keeps a request from leaving before the one ahead of it, and a launch reset clears the queue's next free departure cycle. A later read-miss fill of the same line therefore cannot overtake the write and install the old word. Without the knob, timing is unchanged. SimX LLC atomics: a store-bearing atomic writes its result back into the line through the core-request slot two cycles after its commit, and holds new core requests off until then. A token flush takes two more cycles after its last write-back before the bank reports done. Both follow the RTL bank. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache.sv | 30 +++++++- hw/rtl/core/VX_uop_sequencer.sv | 5 +- sim/simx/decode.cpp | 6 +- sim/simx/mem/cache.cpp | 110 ++++++++++++++++++++++----- tests/regression/fence_mp/common.h | 14 ++++ tests/regression/fence_mp/kernel.cpp | 9 ++- tests/regression/fence_mp/main.cpp | 10 ++- 7 files changed, 153 insertions(+), 31 deletions(-) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 86814ef6eb..5184a6f56d 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -668,6 +668,30 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0] mem_req_xbar_valid; wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] mem_req_xbar_pdata; wire [NUM_BANKS-1:0] mem_req_xbar_ready; + + // Debug: a write-through cache holds each write of bank 0 for a fixed + // delay before it leaves, so requests through the other banks overtake it + // unless something orders them. + wire [NUM_BANKS-1:0] dbg_mem_req_hold; +`ifdef DBG_MEM_BANK0_WDELAY + if (WRITEBACK == 0) begin : g_dbg_wdelay + localparam DBG_WDELAY = 2048; + reg [`CLOG2(DBG_WDELAY+1)-1:0] dbg_wdelay_cnt; + wire dbg_wait = per_bank_mem_req_valid[0] && per_bank_mem_req_rw[0]; + always @(posedge clk) begin + if (reset || ~dbg_wait || (mem_req_xbar_valid[0] && mem_req_xbar_ready[0])) begin + dbg_wdelay_cnt <= '0; + end else if (dbg_wdelay_cnt != DBG_WDELAY) begin + dbg_wdelay_cnt <= dbg_wdelay_cnt + 1; + end + end + assign dbg_mem_req_hold = NUM_BANKS'(dbg_wait && (dbg_wdelay_cnt != DBG_WDELAY)); + end else begin : g_no_dbg_wdelay + assign dbg_mem_req_hold = '0; + end +`else + assign dbg_mem_req_hold = '0; +`endif for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_mem_req_xbar_in if (i < MEM_PORTS) begin : g_flush_req // line 0 hashes to 0, so the raw bank bits are the lane's bank id @@ -681,15 +705,15 @@ module VX_cache import VX_gpu_pkg::*; #( (`UP(MEM_ATTR_WIDTH)'(1) << MEM_ATTR_FLUSH_OFFS) | (`UP(MEM_ATTR_WIDTH)'(mem_flush_io) << MEM_ATTR_IO_OFFS) | (`UP(MEM_ATTR_WIDTH)'(mem_flush_noalloc) << MEM_ATTR_NOALLOC_OFFS) }; - assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; + assign mem_req_xbar_valid[i] = (per_bank_mem_req_valid[i] && ~dbg_mem_req_hold[i]) || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; assign mem_flush_ready[i] = mem_req_xbar_ready[i] && ~per_bank_mem_req_valid[i]; `RUNTIME_ASSERT (~(mem_flush_valid[i] && per_bank_mem_req_valid[i]), ("%t: %s flush token collides with a bank %0d request", $time, INSTANCE_ID, i)) end else begin : g_bank_req - assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i]; + assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] && ~dbg_mem_req_hold[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_pdata[i]; end - assign per_bank_mem_req_ready[i] = mem_req_xbar_ready[i]; + assign per_bank_mem_req_ready[i] = mem_req_xbar_ready[i] && ~dbg_mem_req_hold[i]; end // Write-backs from different banks are not ordered with each other inside // the crossbar, so the tokens wait until it has drained. diff --git a/hw/rtl/core/VX_uop_sequencer.sv b/hw/rtl/core/VX_uop_sequencer.sv index 3fb0ffcf49..2b888a8cce 100644 --- a/hw/rtl/core/VX_uop_sequencer.sv +++ b/hw/rtl/core/VX_uop_sequencer.sv @@ -197,7 +197,10 @@ module VX_uop_sequencer import // ------------------------------------------------------------------ // Atomic ordering uop expander: fences around an AMO/LR/SC with .aq/.rl // ------------------------------------------------------------------ - assign uop_in_valid[UOP_AMO] = (uop_in_data.ex_type == EX_LSU) + // A dcache that is itself the point of coherence orders the atomic with + // the hart's other accesses already: no fence is owed. + assign uop_in_valid[UOP_AMO] = !DCACHE_SHARED_LLC + && (uop_in_data.ex_type == EX_LSU) && uop_in_data.op_args.lsu.amo_valid && (uop_in_data.op_args.lsu.amo_aq || uop_in_data.op_args.lsu.amo_rl); VX_uop_amo uop_amo ( diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 5a559d4726..87e1a98244 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -21,6 +21,7 @@ #include #include #include "debug.h" +#include "constants.h" #include "types.h" #include "decode.h" #include "decompressor.h" @@ -671,8 +672,9 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { std::abort(); } instr->set_args(IntrAmoArgs{funct3, aq, rl}); - // .aq/.rl order the atomic with fences around it (LsuUopGen) - if (aq || rl) { + // .aq/.rl order the atomic with fences around it (LsuUopGen), unless the + // dcache is itself the point of coherence + if ((aq || rl) && !DCACHE_SHARED_LLC) { instr->set_macro_op(); } instr->set_dest_reg(rd, RegType::Integer); diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 4668fcb78b..d957a056a6 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -416,7 +416,8 @@ struct bank_req_t { Replay = 2, Core = 3, AmoProbe = 4, // non-LLC AMO passthrough: probe-and-invalidate then forward - Park = 5 // chain head parked on a partial sector: issues its refetch + Park = 5, // chain head parked on a partial sector: issues its refetch + AmoWb = 6 // an LLC atomic's result written back through the pipe }; uint64_t addr; @@ -872,6 +873,7 @@ class CacheBank : public SimObject { token_flushing_ = true; token_flush_skip_ = skip; token_flush_idx_ = 0; + token_flush_settle_ = 0; flush_sector_idx_ = 0; this->tick_wake(); } @@ -892,6 +894,9 @@ class CacheBank : public SimObject { pending_fill_reqs_ = 0; rand_ctr_ = 0; adm_seq_ctr_ = 0; + #ifdef DBG_MEM_BANK0_WDELAY + dbg_mem_req_free_ = 0; + #endif if (!keep_lines_) { for (auto &set : sets_) { set.reset(); @@ -912,6 +917,9 @@ class CacheBank : public SimObject { chain_sector_ = 0; refetch_hold_ = false; refetch_hold_retry_ = false; + amo_busy_ = false; + amo_wb_pending_ = false; + amo_wb_tick_ = 0; flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -970,7 +978,8 @@ class CacheBank : public SimObject { // any packet reserved toward core_req_in/mem_rsp_in re-arms on arrival. if (this->core_req_in.size() == 0 && this->mem_rsp_in.size() == 0 && pipe_req_->size() == 0 && lazy_q_->size() == 0 && !mshr_.has_ready_reqs() - && !fwd_active_ && !flushing_ && !token_flushing_ && pending_fill_reqs_ == 0) { + && !fwd_active_ && !flushing_ && !token_flushing_ && pending_fill_reqs_ == 0 + && !amo_wb_pending_) { this->tick_sleep(); } } @@ -1148,8 +1157,29 @@ class CacheBank : public SimObject { } } +#if VX_CFG_EXT_A_ENABLED + // An LLC atomic's result goes back into the line as a synthetic write + // through the core-request slot, one compute cycle after its commit. + if (amo_wb_pending_ && SimPlatform::instance().cycles() >= amo_wb_tick_) { + bank_req_t wb_req; + wb_req.reset(); + wb_req.type = bank_req_t::AmoWb; + pipe_req_->push(wb_req); + amo_wb_pending_ = false; + amo_busy_ = false; + return; + } +#endif + // 3) core request if (!this->core_req_in.empty()) { +#if VX_CFG_EXT_A_ENABLED + // A store-bearing atomic holds new core requests off from its lookup + // until its result is written back. + if (amo_busy_) { + return; + } +#endif auto &core_req = this->core_req_in.peek(); if (fill_held && params_.addr_set_id(core_req.addr) == held_set_id @@ -1322,6 +1352,9 @@ class CacheBank : public SimObject { } #if VX_CFG_EXT_A_ENABLED if (is_amo_passthru) ++pending_amo_probes_; + if (is_amo && config_.is_llc && core_req.op != MemOp::AMO_LR) { + amo_busy_ = true; + } #endif if (core_req.is_write()) ++perf_stats_.writes; else ++perf_stats_.reads; @@ -1414,7 +1447,7 @@ class CacheBank : public SimObject { wb.uuid = bank_req.uuid; wb.data = sec.data; wb.byteen = sec.dirty_mask; - this->mem_req_out.send(wb, MEM_REQ_DELAY); + this->send_mem_req(wb); DT(3, this->name() << " writeback: " << wb); ++perf_stats_.evictions; } @@ -1586,13 +1619,22 @@ class CacheBank : public SimObject { w.uuid = bank_req.uuid; w.data = store_block; w.byteen = byteen; - this->mem_req_out.send(w, MEM_REQ_DELAY); + this->send_mem_req(w); DT(3, this->name() << " amo-writethrough: " << w); } // Break other harts' reservations on this line. amo_unit_.invalidate(line_addr, /*except=*/hid); } + // A stored result is written back into the line through the pipe; without + // a store the commit frees the bank at once. + if (do_store) { + amo_wb_pending_ = true; + amo_wb_tick_ = SimPlatform::instance().cycles() + 2; + } else { + amo_busy_ = false; + } + // Send AMO response. MemRsp rsp{bank_req.req_tag, bank_req.hart_id, bank_req.uuid}; rsp.data = rsp_block; @@ -1605,6 +1647,25 @@ class CacheBank : public SimObject { // Pipeline tail: process the head of pipe_req_ — at most one bank_req_t per tick. + // Send a request down the bank's in-order memory queue. + // Debug: a write-through cache holds each write at the head of bank 0's + // queue for a fixed delay, the requests behind it included, so requests + // through the other banks overtake it unless something orders them. + void send_mem_req(const MemReq& req) { + #ifdef DBG_MEM_BANK0_WDELAY + if (bank_id_ == 0 && !config_.write_back) { + constexpr uint64_t DBG_WDELAY = 2048; + uint64_t now = SimPlatform::instance().cycles(); + uint64_t head = std::max(now, dbg_mem_req_free_); + uint64_t leave = head + (req.is_write() ? DBG_WDELAY : 0); + dbg_mem_req_free_ = leave + 1; + this->mem_req_out.send(req, MEM_REQ_DELAY + (leave - now)); + return; + } + #endif + this->mem_req_out.send(req, MEM_REQ_DELAY); + } + // Emit a write-through store downstream, with the optional core response. // All backpressure is checked before any send, so a false return stalls // the caller with no partial effects. @@ -1626,7 +1687,7 @@ class CacheBank : public SimObject { w.uuid = uuid; w.data = data; w.byteen = byteen; - this->mem_req_out.send(w, MEM_REQ_DELAY); + this->send_mem_req(w); DT(3, this->name() << " writethrough: " << w); #if VX_CFG_EXT_A_ENABLED // Committed write: break other harts' reservations. @@ -1656,6 +1717,7 @@ class CacheBank : public SimObject { switch (bank_req.type) { case bank_req_t::None: + case bank_req_t::AmoWb: pipe_req_->pop(); return; @@ -1705,7 +1767,7 @@ class CacheBank : public SimObject { wb.uuid = bank_req.uuid; wb.data = sec.data; wb.byteen = sec.dirty_mask; - this->mem_req_out.send(wb, MEM_REQ_DELAY); + this->send_mem_req(wb); DT(3, this->name() << " amo-probe-wb: " << wb); ++perf_stats_.evictions; } @@ -1731,7 +1793,7 @@ class CacheBank : public SimObject { amo_fwd.amo_cmp = bank_req.amo_cmp; amo_fwd.data = bank_req.data; // carries rhs (extracted at LLC) amo_fwd.flags = bank_req.flags; // carry amo_unsigned (signed vs unsigned MIN/MAX) - this->mem_req_out.send(amo_fwd, MEM_REQ_DELAY); + this->send_mem_req(amo_fwd); DT(3, this->name() << " amo-probe-fwd: " << amo_fwd); ++pending_fill_reqs_; // counts as an outstanding mem-roundtrip for perf --pending_amo_probes_; // reservation transferred to the table entry @@ -1771,7 +1833,7 @@ class CacheBank : public SimObject { fill.tag = bank_req.mshr_id; fill.hart_id = bank_req.hart_id; fill.uuid = bank_req.uuid; - this->mem_req_out.send(fill, MEM_REQ_DELAY); + this->send_mem_req(fill); DT(3, this->name() << " refetch-req: " << fill); ++pending_fill_reqs_; pipe_req_->pop(); @@ -1823,7 +1885,7 @@ class CacheBank : public SimObject { fill.tag = mshr_id; fill.hart_id = bank_req.hart_id; fill.uuid = bank_req.uuid; - this->mem_req_out.send(fill, MEM_REQ_DELAY); + this->send_mem_req(fill); ++pending_fill_reqs_; } pipe_req_->pop(); @@ -1976,7 +2038,7 @@ class CacheBank : public SimObject { w.uuid = bank_req.uuid; w.data = bank_req.data; w.byteen = bank_req.byteen; - this->mem_req_out.send(w, MEM_REQ_DELAY); + this->send_mem_req(w); DT(3, this->name() << " writethrough: " << w); #if VX_CFG_EXT_A_ENABLED // Writethrough commit: break other harts' reservations. @@ -2073,7 +2135,7 @@ class CacheBank : public SimObject { fill.tag = mshr_id; // routes the fill response back here fill.hart_id = bank_req.hart_id; fill.uuid = bank_req.uuid; - this->mem_req_out.send(fill, MEM_REQ_DELAY); + this->send_mem_req(fill); DT(3, this->name() << " fill-req: " << fill); ++pending_fill_reqs_; } @@ -2121,7 +2183,7 @@ class CacheBank : public SimObject { mem_req.op = MemOp::ST; mem_req.data = sec.data; mem_req.byteen = sec.dirty_mask; - this->mem_req_out.send(mem_req, MEM_REQ_DELAY); + this->send_mem_req(mem_req); DT(3, this->name() << " flush-wb: " << mem_req); ++perf_stats_.evictions; sec.dirty = false; @@ -2140,14 +2202,21 @@ class CacheBank : public SimObject { } void processTokenFlush() { + // the bank leaves its write-backs, then signals the end, a cycle each + if (token_flush_settle_ != 0) { + if (--token_flush_settle_ == 0) { + token_flushing_ = false; + DT(3, this->name() << (token_flush_skip_ ? " token-quiesce-done" : " token-flush-done")); + } + return; + } if (pending_fill_reqs_ != 0 || pipe_req_->size() != 0 || !mshr_.empty()) { return; } if (token_flush_skip_) { - token_flushing_ = false; - DT(3, this->name() << " token-quiesce-done"); + token_flush_settle_ = TOKEN_FLUSH_SETTLE; return; } uint32_t ways = config_.write_back ? params_.lines_per_set : 1; @@ -2167,7 +2236,7 @@ class CacheBank : public SimObject { mem_req.op = MemOp::ST; mem_req.data = sec.data; mem_req.byteen = sec.dirty_mask; - this->mem_req_out.send(mem_req, MEM_REQ_DELAY); + this->send_mem_req(mem_req); DT(3, this->name() << " flush-wb: " << mem_req); ++perf_stats_.evictions; } @@ -2181,8 +2250,7 @@ class CacheBank : public SimObject { } } if (++token_flush_idx_ == params_.sets_per_bank * ways) { - token_flushing_ = false; - DT(3, this->name() << " token-flush-done"); + token_flush_settle_ = TOKEN_FLUSH_SETTLE; } } @@ -2272,6 +2340,12 @@ class CacheBank : public SimObject { uint32_t chain_sector_; bool refetch_hold_; // a refetch fill is at its tag lookup: its chain waits bool refetch_hold_retry_; + bool amo_busy_; // a store-bearing LLC atomic holds core requests off + bool amo_wb_pending_; // its result still has to be written back + uint64_t amo_wb_tick_; // ... from this cycle on +#ifdef DBG_MEM_BANK0_WDELAY + uint64_t dbg_mem_req_free_ = 0; // the memory queue's next free departure cycle +#endif // Flush walk state. bool flushing_; @@ -2281,6 +2355,8 @@ class CacheBank : public SimObject { uint32_t flush_sector_idx_; bool token_flushing_; bool token_flush_skip_ = false; + static constexpr uint32_t TOKEN_FLUSH_SETTLE = 2; + uint32_t token_flush_settle_ = 0; uint32_t token_flush_idx_; #if VX_CFG_EXT_A_ENABLED diff --git a/tests/regression/fence_mp/common.h b/tests/regression/fence_mp/common.h index 8d2f0a9994..1b04613e07 100644 --- a/tests/regression/fence_mp/common.h +++ b/tests/regression/fence_mp/common.h @@ -5,6 +5,20 @@ #define FENCE_MP_MAGIC 0x5a5a0000u +// The data lives in the even 64-byte lines of its buffer and the flag in an +// odd one, so on a two-bank dcache they leave through different banks: a test +// can delay one bank's writes and check that only a release orders the flag +// behind them. +#define MP_LINE_WORDS 16 + +static inline uint32_t mp_data_index(uint32_t k) { + return (k / MP_LINE_WORDS) * (2 * MP_LINE_WORDS) + (k % MP_LINE_WORDS); +} + +static inline uint32_t mp_data_words(uint32_t count) { + return ((count + MP_LINE_WORDS - 1) / MP_LINE_WORDS) * (2 * MP_LINE_WORDS); +} + // How the producer publishes the flag and the consumer observes it. enum { MP_FENCE = 0, // full fences on both sides diff --git a/tests/regression/fence_mp/kernel.cpp b/tests/regression/fence_mp/kernel.cpp index 6d716a0aeb..b552343e02 100644 --- a/tests/regression/fence_mp/kernel.cpp +++ b/tests/regression/fence_mp/kernel.cpp @@ -48,13 +48,13 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto out = reinterpret_cast(arg->out_addr); if (gid == 0) { for (uint32_t k = 0; k < arg->count; ++k) { - data[k] = FENCE_MP_MAGIC + k; + data[mp_data_index(k)] = FENCE_MP_MAGIC + k; } publish(flag, arg->mode); } else if (gid == arg->num_tasks - 1) { uint32_t sum = 0; for (uint32_t k = 0; k < arg->count; ++k) { - sum += data[k]; + sum += data[mp_data_index(k)]; } out[2] = sum; uint32_t timeout = 1; @@ -67,9 +67,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { if (arg->mode == MP_FENCE) { vx_fence(); } + // newest first: the words written last are the ones still in flight uint32_t errors = 0; - for (uint32_t k = 0; k < arg->count; ++k) { - errors += (data[k] != FENCE_MP_MAGIC + k); + for (uint32_t k = arg->count; k-- > 0;) { + errors += (data[mp_data_index(k)] != FENCE_MP_MAGIC + k); } out[0] = errors; out[1] = timeout; diff --git a/tests/regression/fence_mp/main.cpp b/tests/regression/fence_mp/main.cpp index 6439256482..78f32f09e3 100644 --- a/tests/regression/fence_mp/main.cpp +++ b/tests/regression/fence_mp/main.cpp @@ -79,18 +79,20 @@ int main(int argc, char *argv[]) { kernel_arg.spin_limit = spin_limit; kernel_arg.mode = mode; - std::vector zeros(count, 0); + std::vector zeros(mp_data_words(count), 0); std::vector out(4, 0); - uint32_t data_size = count * sizeof(uint32_t); + uint32_t data_size = mp_data_words(count) * sizeof(uint32_t); uint32_t out_size = out.size() * sizeof(uint32_t); RT_CHECK(vx_buffer_create(device, data_size, VX_MEM_READ_WRITE, &data_buffer)); RT_CHECK(vx_buffer_address(data_buffer, &kernel_arg.data_addr)); - RT_CHECK(vx_buffer_create(device, sizeof(uint32_t), VX_MEM_READ_WRITE, &flag_buffer)); + // the flag sits in the buffer's second (odd) line + RT_CHECK(vx_buffer_create(device, 2 * MP_LINE_WORDS * sizeof(uint32_t), VX_MEM_READ_WRITE, &flag_buffer)); RT_CHECK(vx_buffer_address(flag_buffer, &kernel_arg.flag_addr)); + kernel_arg.flag_addr += MP_LINE_WORDS * sizeof(uint32_t); RT_CHECK(vx_buffer_create(device, out_size, VX_MEM_READ_WRITE, &out_buffer)); RT_CHECK(vx_buffer_address(out_buffer, &kernel_arg.out_addr)); RT_CHECK(vx_enqueue_write(queue, data_buffer, 0, zeros.data(), data_size, 0, nullptr, nullptr)); - RT_CHECK(vx_enqueue_write(queue, flag_buffer, 0, zeros.data(), sizeof(uint32_t), 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, flag_buffer, MP_LINE_WORDS * sizeof(uint32_t), zeros.data(), sizeof(uint32_t), 0, nullptr, nullptr)); RT_CHECK(vx_enqueue_write(queue, out_buffer, 0, out.data(), out_size, 0, nullptr, nullptr)); RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); From b9e7a2480cdd17bca48e6a7931a0fd2b08130140 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:52 -0700 Subject: [PATCH 106/336] ci: require a debug build to match release cycles (release_parity) A debug build widens the RTL UUID to 44 bits and turns on assertions and traces; none of that may move timing. A debug-only tag width did reach the dcache's read merge and changed cycle counts, and no check noticed. A blackbox case run with --debug can now set release_parity: true. The runner then reruns it built release at xlen 32 and requires identical instructions and cycles, reading the debug run's counts from its --log file. ci lint requires a --debug blackbox case and rejects combining it with a check. debug:sgemm_tcu-2 carries it, reshaped (8 threads, 8 warps, a 4 KB dcache, an L2) so the data cache is under pressure and its two channels' same-word reads merge. SimX: without a registered tree level, the cache's port crossbar is combinational. Each bank is now offered one request per cycle, and the ports are dequeued only when the bank takes it, so a later same-word read can still merge with a request waiting at its port. An offer still pending the next cycle is withdrawn and the ports re-arbitrate. Co-Authored-By: Claude Opus 5.5 --- ci/test_runner.py | 36 +++++++++++ ci/testcase.py | 13 ++++ ci/testcases/debug.yaml | 9 ++- docs/designs/continuous_integration.md | 12 ++++ sim/simx/mem/cache.cpp | 85 ++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 1 deletion(-) diff --git a/ci/test_runner.py b/ci/test_runner.py index 2cf559d5f1..5f761ba1b2 100644 --- a/ci/test_runner.py +++ b/ci/test_runner.py @@ -44,6 +44,39 @@ def _run_one(case, xlen, driver): return int(instrs), int(cycles) +def _perf_of(text, what): + perf = _PERF_RE.findall(text) + assert perf, "{}: no 'PERF: instrs=…, cycles=…' summary " \ + "(app must call vx_device_dump_perf)".format(what) + instrs, cycles = perf[-1] + return int(instrs), int(cycles) + + +def _release_parity(case, xlen): + """Run the debug case, then (xlen 32) the same run built release; both + must report identical instructions and cycles.""" + argv, env = case.run_command(xlen) + rc = tc.execute(argv, env) + assert rc == 0, "{} failed (exit {}): {}".format(case.id, rc, " ".join(argv)) + if xlen != 32: + return + # a debug run sends its output to the --log file (blackbox.sh) + log = next((a.split("=", 1)[1] for a in argv if a.startswith("--log=")), "run.log") + with open(log, errors="replace") as fh: + dbg_instrs, dbg_cycles = _perf_of(fh.read(), case.id + " [debug]") + rel_argv = [a for a in argv if not a.startswith(("--debug", "--log="))] + rc, out = tc.execute_capture(rel_argv, env) + assert rc == 0, "{} [release] failed (exit {}): {}".format( + case.id, rc, " ".join(rel_argv)) + rel_instrs, rel_cycles = _perf_of(out, case.id + " [release]") + print("RELEASE-PARITY: {}: instrs debug={} release={}, cycles debug={} " + "release={}".format(case.id, dbg_instrs, rel_instrs, dbg_cycles, rel_cycles)) + assert (dbg_instrs, dbg_cycles) == (rel_instrs, rel_cycles), \ + "{}: the debug build changes the run (instrs {} vs {}, cycles {} vs {}) " \ + "-- debug-only state must not reach timing".format( + case.id, dbg_instrs, rel_instrs, dbg_cycles, rel_cycles) + + def _model_parity(case, xlen): simx_instrs, simx_cycles = _run_one(case, xlen, "simx") rtl_instrs, rtl_cycles = _run_one(case, xlen, "rtlsim") @@ -99,6 +132,9 @@ def test_case(case, sim_build, request): if case.check == "perf_gate": _perf_gate(case, xlen, request.config.getoption("--update-baselines")) return + if case.release_parity: + _release_parity(case, xlen) + return argv, env = case.run_command(xlen) rc = tc.execute(argv, env) assert rc == 0, "{} failed (exit {}): {}".format(case.id, rc, " ".join(argv)) diff --git a/ci/testcase.py b/ci/testcase.py index 0e8da16c65..2581b6e61d 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -106,6 +106,12 @@ def __init__(self, category, entry, driver, defaults): # coverage can't creep back in per-file. if self.check in ("model_parity", "perf_gate"): self.xlens = [32] + # release_parity: a debug run (its flags carry --debug) must report + # exactly the cycles of the same run built release. A debug build widens + # the RTL UUID to 44 bits and enables assertions and traces; none of that + # may move timing. The runner adds the release leg itself, at xlen 32 + # only like the cycle gates; other xlens run the debug leg alone. + self.release_parity = bool(entry.get("release_parity", False)) self.tolerance = float(entry.get("tolerance", defaults.get("tolerance", DEFAULT_PARITY_TOLERANCE))) self.authored_drivers = ("driver" in entry) or ("drivers" in entry) @@ -575,6 +581,13 @@ def cmd_lint(args): errors.append("{}: script case missing 'run'".format(c.id)) if any(int(x) not in (32, 64) for x in c.xlens): errors.append("{}: xlen must be 32 and/or 64".format(c.id)) + if c.release_parity: + if c.via != "blackbox" or "--debug" not in c.flags: + errors.append("{}: release_parity needs a blackbox case run " + "with --debug".format(c.id)) + if c.check: + errors.append("{}: release_parity cannot combine with a " + "check".format(c.id)) if c.check: if c.check not in VALID_CHECK: errors.append("{}: invalid check {!r}".format(c.id, c.check)) diff --git a/ci/testcases/debug.yaml b/ci/testcases/debug.yaml index ee8d117b88..21b4ccd8fc 100644 --- a/ci/testcases/debug.yaml +++ b/ci/testcases/debug.yaml @@ -66,13 +66,20 @@ tests: app: sgemm_tcu configs: -DVX_CFG_EXT_TCU_ENABLE flags: --debug=3 --log=run_simx.log +# The debug build (44-bit UUID, assertions, traces) must not move RTL timing: +# the runner reruns this case built release and requires identical cycles. The +# shape keeps the data cache under pressure, with two channels whose same-word +# reads merge, so a debug-only tag width reaching the merge shows in cycles. - id: sgemm_tcu-2 via: blackbox drivers: - rtlsim app: sgemm_tcu - configs: -DVX_CFG_EXT_TCU_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_DCACHE_SIZE=4096 -DVX_CFG_EXT_TCU_ENABLE + shape: + l2cache: true flags: --debug=3 --log=run_rtlsim.log + release_parity: true - id: simx_mt-1 via: blackbox drivers: diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index c959934e45..20000d7464 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -230,6 +230,18 @@ kernel is all boot and dispatch skew, and its gap ratio is noise. **Never widen a tolerance to absorb a divergence.** Model the behavior, or mark the case `known_issue` while it is investigated. +#### `release_parity` + +A debug build must not change timing. A debug build widens the RTL UUID to 44 +bits and enables assertions and traces. A blackbox case run with `--debug` can +set `release_parity: true`. The runner then reruns it built release, at +xlen 32, and requires identical instructions and cycles; it reads the debug +run's counts from its `--log` file. This is a case field, not a check, so the +case stays in its category cell and costs one extra run there, with no new +cell. `debug:sgemm_tcu-2` carries it. That case is shaped to keep the data +cache under pressure, with two channels whose same-word reads merge; a +debug-only tag width reaching the read merge would show there. + ### 3.4 `perf_gate` rtlsim cycle counts are deterministic and host-independent, so the threshold diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index d957a056a6..abd71bf8af 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2423,6 +2423,21 @@ class CachePortArb : public SimObject { } } + // Without a registered tree level the hardware crossbar is combinational + // (no request buffer ahead of a bank): a request stays at its port until the + // bank takes it, where a later same-word read can still merge with it. The + // arbiter then offers each bank one request per cycle and dequeues the ports + // only when the bank pops the offer; an offer still pending next cycle is + // withdrawn and the ports are arbitrated afresh. + void attach_bank(uint32_t b, SimChannel* bank_in) { + if (!this->offers_enabled()) { + return; + } + offers_.resize(ReqOut.size()); + offers_.at(b).bank_in = bank_in; + bank_in->pop_callback([this, b]() { this->on_bank_pop(b); }); + } + // Bits of the merged-port mask carried below the request tag. uint32_t merge_bits() const { return (merge_group_ > 1) ? merge_group_ : 0; } @@ -2500,6 +2515,10 @@ class CachePortArb : public SimObject { std::fill(rr.begin(), rr.end(), 0); } collisions_ = 0; + for (auto& offer : offers_) { + offer.valid = false; + offer.withdrawing = false; + } } // Slices of the arbiter level s: level 0 grants the ports, each level above @@ -2524,6 +2543,14 @@ class CachePortArb : public SimObject { } uint32_t nb = ReqOut.size(); uint32_t nl = levels_.size(); + for (auto& offer : offers_) { + if (offer.valid) { + offer.withdrawing = true; + offer.bank_in->pop(); + offer.withdrawing = false; + offer.valid = false; + } + } // the tree's buffered levels, join first: each slice passes one buffer of // the level below per cycle for (uint32_t s = nl; s > 0; --s) { @@ -2622,6 +2649,10 @@ class CachePortArb : public SimObject { while (!(slice_want & (1u << j))) { j = (j + 1) % size; } + if (this->offers_enabled()) { + this->offer(b, sl * slice_ports + j, out, merged, wants.at(b) - 1, &rr, (j + 1) % size); + continue; + } this->grant(sl * slice_ports + j, out, merged); rr = (j + 1) % size; granted = true; @@ -2632,6 +2663,48 @@ class CachePortArb : public SimObject { } } + // Offer port i's request, with the ports merged into it, to bank b; the + // ports are dequeued when the bank takes it (on_bank_pop). + void offer(uint32_t b, uint32_t i, SimChannel& out, const std::vector& merged, + uint32_t collisions, uint32_t* rr, uint32_t rr_next) { + auto req = ReqIn.at(i).peek(); + uint64_t mask = (merge_group_ > 1) ? merged.at(i) : 0; + uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; + __assert((tag >> 32) == 0, "cache bank tag overflow"); + req.tag = uint32_t(tag); + auto& offer = offers_.at(b); + offer.valid = true; + offer.port = i; + offer.mask = mask; + offer.collisions = collisions; + offer.rr = rr; + offer.rr_next = rr_next; + out.send(req, 0); + } + + void on_bank_pop(uint32_t b) { + auto& offer = offers_.at(b); + if (offer.withdrawing) { + return; + } + __assert(offer.valid, "bank took a request the arbiter did not offer"); + uint32_t i = offer.port; + if (offer.mask != 0) { + uint32_t base = (i / merge_group_) * merge_group_; + for (uint64_t f = offer.mask; f != 0; f &= f - 1) { + uint32_t port = base + __builtin_ctzll(f); + DT(3, this->name() << " read-merge: port=" << port << ", leader=" << i << ", " << ReqIn.at(port).peek()); + ReqIn.at(port).pop(); + } + } + ReqIn.at(i).pop(); + *offer.rr = offer.rr_next; + collisions_ += offer.collisions; + offer.valid = false; + } + + bool offers_enabled() const { return levels_.empty() && delay_ == 0; } + // Send port i's request to its bank, with the ports merged into it. void grant(uint32_t i, SimChannel& out, const std::vector& merged) { auto req = ReqIn.at(i).peek(); @@ -2671,6 +2744,17 @@ class CachePortArb : public SimObject { std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; uint64_t collisions_ = 0; + struct offer_t { + SimChannel* bank_in = nullptr; + bool valid = false; + bool withdrawing = false; + uint32_t port = 0; + uint64_t mask = 0; + uint32_t collisions = 0; + uint32_t* rr = nullptr; + uint32_t rr_next = 0; + }; + std::vector offers_; // [bank], combinational crossbar only friend class SimObject; }; @@ -2973,6 +3057,7 @@ class Cache::Impl { // bind core ports port_arb_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); + port_arb_->attach_bank(i, &banks_.at(i)->core_req_in); banks_.at(i)->core_rsp_out.bind(rsp_mcast_ ? &rsp_mcast_->RspIn.at(i) : &bank_core_xbar_->RspIn.at(i)); // bind memory ports From 009a9d55a19038de44de42d8d2f9ac7437615302 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:53 -0700 Subject: [PATCH 107/336] tcu: fp32-only output, bit-exact TFR numerics in SimX, tcu_unit test The tensor core accumulates in fp32 (int32 for integer inputs) and has no other output format. Narrow fmt_d paths (fp16, bf16, fp8, bf8 and tf32 out, rounded at every step) existed only in SimX and in the test references, and modeled a datapath that does not exist. They are removed. vx_tensor.h rejects another output type at compile time, and VX_tcu_core and SimX assert on an instruction that encodes one. tensor:sgemm_tcu-4 and -6 now produce fp32. SimX on the TFR backend computed dot products as discrete fp32 multiply-adds, which round apart from the datapath's single windowed rounding on subnormal rows and near-cancellations. It now uses the TfrFedp model: products aligned to the largest exponent in a 25-bit window with a sticky bit, summed as integers, rounded once. The model moves from hw/unittest/tcu_fedp to sw/common/fedp.h and gains MX (E8M0) and NVFP4 (UE4M3) scales and fp4 lanes. Other backends keep the discrete fp32 order of the DPI datapath. RTL TFR fixes found by checking the datapath bit for bit: - MXFP4 E8M0 scales were added into the 6-bit term alignment field, so scales far from one (such as 2^-34) wrapped. They now join the lane exponent. - An MX fp8 lane scaled below the exponent range wrapped. The exponent sum is now one bit wider and signed, and such a lane only adds to the sticky bit. - NaN scales (E8M0 0xFF, UE4M3 0x7F) now make their block NaN. - TCU_EXP_BITS is 10 under MX. CI: unittest:hw-tcu-unit runs VX_tcu_unit end to end (dispatch, uops, operand routing, commit) at 0 ULP against TfrFedp on TFR and against the discrete reference on DPI. Directed vectors cover a subnormal fp16 row and near-cancellation. tcu_fedp gains MX/NVFP4 suites with directed scale corners. tcu_unit's format IDs now match VX_tcu_pkg, elements are packed per register, and a packet is dispatched exactly once. The WGMMA design doc gains fmt_d, the MX/NVFP4 scale semantics and TFR rounding. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor.yaml | 4 +- ci/testcases/unittest.yaml | 13 +- docs/designs/tensor_core_wgmma_engine.md | 38 +++- hw/rtl/tcu/VX_tcu_core.sv | 4 + hw/rtl/tcu/VX_tcu_pkg.sv | 3 + hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv | 33 ++- hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv | 45 ++-- hw/unittest/Makefile | 44 ++++ hw/unittest/tcu_fedp/Makefile | 8 +- hw/unittest/tcu_fedp/main.cpp | 76 ++++++- hw/unittest/tcu_unit/Makefile | 4 +- hw/unittest/tcu_unit/main.cpp | 213 +++++++++++++----- sim/simx/tcu/tcu_unit.cpp | 152 +++++++------ {hw/unittest/tcu_fedp => sw/common}/fedp.h | 193 ++++++++++++++-- sw/common/tensor_cfg.h | 6 + sw/kernel/include/vx_tensor.h | 6 + tests/regression/sgemm2_tcu/main.cpp | 28 +-- tests/regression/sgemm_tcu/main.cpp | 69 +----- tests/regression/sgemm_tcu_mx/main.cpp | 3 + tests/regression/sgemm_tcu_sp/main.cpp | 61 +---- tests/regression/sgemm_tcu_sp_mx/main.cpp | 3 + tests/regression/sgemm_tcu_wg/main.cpp | 61 +---- tests/regression/sgemm_tcu_wg_dxa/main.cpp | 17 +- .../sgemm_tcu_wg_dxa_mcast/main.cpp | 17 +- tests/regression/sgemm_tcu_wg_mx/main.cpp | 3 + 25 files changed, 687 insertions(+), 417 deletions(-) rename {hw/unittest/tcu_fedp => sw/common}/fedp.h (64%) diff --git a/ci/testcases/tensor.yaml b/ci/testcases/tensor.yaml index 52b0799215..dbcba1d2f5 100644 --- a/ci/testcases/tensor.yaml +++ b/ci/testcases/tensor.yaml @@ -31,7 +31,7 @@ tests: drivers: - simx app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=bf16 -DOTYPE=bf16 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=bf16 -DOTYPE=fp32 - id: sgemm_tcu-5 via: blackbox drivers: @@ -43,7 +43,7 @@ tests: drivers: - simx app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=bf8 -DOTYPE=bf8 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=bf8 -DOTYPE=fp32 - id: sgemm_tcu-7 via: blackbox drivers: diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 8765f57fac..cd3cf35d8c 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -23,14 +23,23 @@ tests: - id: hw-tcu via: script run: "make -C hw/unittest run-tcu" - touches: [hw/rtl/tcu, hw/unittest/tcu_fedp] + touches: [hw/rtl/tcu, hw/unittest/tcu_fedp, sw/common/fedp.h] + # VX_tcu_unit end to end (dispatch, uops, operand routing, commit), bit-exact + # against each datapath's reference: native TFR against the TfrFedp reference + # model, DPI against its discrete mul/add. Then the native FEDP's + # fp8 and block-scaled (MX/NVFP4) formats against the same model. The `hw` + # lane only builds these. + - id: hw-tcu-unit + via: script + run: "make -C hw/unittest run-tcu-unit" + touches: [hw/rtl/tcu, hw/dpi, hw/unittest/tcu_unit, hw/unittest/tcu_fedp, sw/common/fedp.h] # Same native-TFR formats with the multipliers mapped onto DSP48 (USE_DSP=1) — # the FPGA-synthesis path, off by default in sim. Bit-identical to LUT by # contract, so a mismatch is a DSP-packing bug. - id: hw-tcu-dsp via: script run: "make -C hw/unittest run-tcu-dsp" - touches: [hw/rtl/tcu, hw/unittest/tcu_fedp] + touches: [hw/rtl/tcu, hw/unittest/tcu_fedp, sw/common/fedp.h] # The command processor's DMA port into the last-level cache: a launch behind # DMA writes must wait for them to reach the cache. No runtime path launches # right behind a DMA write, so no system test observes this ordering. diff --git a/docs/designs/tensor_core_wgmma_engine.md b/docs/designs/tensor_core_wgmma_engine.md index ad890708d8..eefc1a8a15 100644 --- a/docs/designs/tensor_core_wgmma_engine.md +++ b/docs/designs/tensor_core_wgmma_engine.md @@ -67,6 +67,11 @@ Flags of an MMA instruction: | `rs2[2:1]` | `cd_nregs` — accumulator registers: 0 = 8, 1 = 16, 2 = 32 | | `rs2[3]` | `a_from_smem` — A comes from shared memory; B always does for WGMMA | +`fmt_d` is fp32 for float and MX inputs and int32 for integer inputs; the +tensor core has no other output format. The kernel API rejects any other output +type at compile time (`static_assert` in `vx_tensor.h`). An instruction that +encodes one anyway trips an assertion in the RTL (`VX_tcu_core`) and in SimX. + | `op_type` | value | operation | |---|---|---| | `INST_TCU_WMMA` | 0 | warp MMA | @@ -92,6 +97,16 @@ micro-op arguments do not carry a sparse bit. An MX format has bit 3 of the id set, which is how decode recognizes that the instruction depends on scale metadata. +Scales follow the OCP MX and NVFP4 definitions: + +- **mxfp8, mxbf8, mxfp4** take an E8M0 scale per 32 elements, worth + 2^(e-127). A product of two scales spans 2^-254 to 2^254. 0xFF is NaN. +- **nvfp4** takes a UE4M3 scale per 16 elements: unsigned, so bit 7 is + padding, and 0x7F is NaN (PTX `.ue4m3`). A zero exponent field is read as a + normal; subnormal scales are an open question. +- **Out-of-range products.** A NaN scale makes its block NaN. A product scaled + below the datapath's exponent range only reaches the sticky bit. + ### 2.2 Metadata dependencies Metadata is not an architectural register, so the scoreboard tracks it with @@ -398,9 +413,10 @@ MMA (§2.2). ## 8. FEDP backends -`VX_CFG_TCU_TYPE` selects one implementation of the dot product. All compute -the same function; each asserts the configured latency against its own -stage structure. +`VX_CFG_TCU_TYPE` selects one implementation of the dot product. Each asserts +the configured latency against its own stage structure. They agree up to +rounding, not bit for bit: `TFR` rounds once over a fixed window, the others +round each multiply and add. | backend | implementation | latency | |---|---|---| @@ -412,8 +428,13 @@ stage structure. `TFR` is the universal default. It is the only backend that does not pay a full floating-point rounding per tree level: it aligns every product to one -exponent and adds them as integers, rounding once. On an FPGA its -multipliers map onto DSP blocks through `VX_CFG_TCU_USE_DSP`. +exponent and adds them as integers, rounding once. The alignment keeps a +25-bit window below the largest product's exponent field, with a sticky bit +for what falls out of it. A row of subnormals, or products that nearly cancel, +can therefore round a few ULP away from the exact sum. +[`sw/common/fedp.h`](../../sw/common/fedp.h) (`TfrFedp`) models it bit for +bit. On an FPGA its multipliers map onto DSP blocks through +`VX_CFG_TCU_USE_DSP`. --- @@ -432,6 +453,13 @@ multipliers map onto DSP blocks through `VX_CFG_TCU_USE_DSP`. Operand loads go through simulation channels; the tensor path never reads memory directly. +Numerics follow the configured backend for fp32 results from float and +block-scaled (MX, NVFP4) sources. Under `TFR`, each dot product is the +`TfrFedp` model, the same one the RTL is checked against bit for bit in +`hw-tcu-unit`. Otherwise it is a discrete fp32 multiply-add: in the DPI +backend's order for unscaled formats, and per element for block-scaled ones. +C and D are fp32 (int32 for integer inputs) in both models; see §2. + Perf: MPM class `TCU` — `TCU_TBUF_STALLS` (0xB03, cycles a WGMMA micro-op waited on tile-buffer data), `TCU_TBUF_CACHE_HITS` (0xB04, B rows reused), `TCU_LMEM_READS` (0xB05). diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index b70a8e3a50..035186b074 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -153,6 +153,10 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire setup_enqueue = execute_fire && is_wgmma_setup; wire fedp_enqueue = execute_fire && !is_wgmma_setup; + // The accumulator is fp32 (int32 for integer inputs); no other fmt_d exists. + `RUNTIME_ASSERT (~fedp_enqueue || (fmt_d == (tcu_fmt_is_int(fmt_s) ? TCU_FMT_WIDTH'(TCU_I32_ID) : TCU_FMT_WIDTH'(TCU_FP32_ID))), + ("%t: *** %s unsupported TCU output format: fmt_s=%0d, fmt_d=%0d (fp32 only)", $time, INSTANCE_ID, fmt_s, fmt_d)) + // ----------------------------------------------------------------------- // Sparse metadata: VX_tcu_sp_meta (for WMMA_SP) + optional tile-buffer mux // ----------------------------------------------------------------------- diff --git a/hw/rtl/tcu/VX_tcu_pkg.sv b/hw/rtl/tcu/VX_tcu_pkg.sv index 7c0c76d7fa..10d4940158 100644 --- a/hw/rtl/tcu/VX_tcu_pkg.sv +++ b/hw/rtl/tcu/VX_tcu_pkg.sv @@ -216,10 +216,13 @@ package VX_tcu_pkg; localparam TCU_MX_MAX_SF = `MAX(`MAX(TCU_MX_SF_FP8, TCU_MX_SF_MXFP4), TCU_MX_SF_NVFP4); + // MX block scales span 2^-254 .. 2^256 on top of the elements' own range. `ifdef VX_CFG_TCU_TF32_ENABLE localparam TCU_EXP_BITS = 10; `elsif VX_CFG_TCU_FP16_ENABLE localparam TCU_EXP_BITS = 10; + `elsif VX_CFG_TCU_MX_ENABLE + localparam TCU_EXP_BITS = 10; `else localparam TCU_EXP_BITS = 9; `endif diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv index bdd2b2354e..8f65a751ac 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv @@ -94,6 +94,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; wire [3:0][23:0] term_mag_shifted; wire [3:0][EXP_TERM_W_MXFP4-1:0] term_exp_biased; wire [3:0] term_valid; + wire [3:0] term_on; wire [3:0] term_sign; wire [3:0][1:0] a_man, b_man; wire [3:0][3:0] f4_man_prod; @@ -102,6 +103,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; localparam OFF = (i % 2) * 16 + j * 4; wire lane_valid = vld_mask[i * 4 + j]; + assign term_on[j] = lane_valid; wire [3:0] raw_a = a_row[K_WORD][OFF +: 4]; wire [3:0] raw_b = b_col[K_WORD][OFF +: 4]; @@ -119,13 +121,11 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; assign b_exp[0] = raw_b[2] & ~raw_b[1]; assign b_exp[1] = raw_b[2] & raw_b[1]; - wire signed [9:0] sf_exp_a = $signed({1'b0, sf_a}) - 10'sd127; - wire signed [9:0] sf_exp_b = $signed({1'b0, sf_b}) - 10'sd127; - wire signed [5:0] exp_biased_raw = 6'(10'(EXP_ADJ_MXFP4) - + sf_exp_a - + sf_exp_b - + 10'(a_exp) - + 10'(b_exp)); + // the block scales are common to the lane's terms: they join its + // exponent, not this alignment field + wire [EXP_TERM_W_MXFP4-1:0] exp_biased_raw = EXP_ADJ_MXFP4 + + EXP_TERM_W_MXFP4'(a_exp) + + EXP_TERM_W_MXFP4'(b_exp); assign term_exp_biased[j] = term_valid[j] ? exp_biased_raw : '0; end @@ -250,10 +250,16 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; assign result_sig_mxfp4[i] = {sum_sign & ~is_zero_out, abs_sum[23:0]}; assign sig_zero_mxfp4[i] = is_zero_out; - assign result_exp_mxfp4[i] = EXP_W'(max_exp_biased) + EXP_W'(EXP_BASE_BIASED_MXFP4); + + // The E8M0 scales (2^-254 .. 2^256 together) join the lane exponent. + // A lane with a term stays within [15, 529]; an empty one wraps, but + // its exponent is cleared with its zero significand. + assign result_exp_mxfp4[i] = EXP_W'(max_exp_biased) + EXP_BASE_BIASED_MXFP4 + + EXP_W'(sf_a) + EXP_W'(sf_b) - EXP_W'(254); // The sign field is only consumed for infinity lanes downstream. - assign exceptions_mxfp4[i].is_nan = 1'b0; + // An E8M0 scale of 0xFF is NaN, and so is every element it scales. + assign exceptions_mxfp4[i].is_nan = ((sf_a == 8'hFF) || (sf_b == 8'hFF)) && (|term_on); assign exceptions_mxfp4[i].is_inf = 1'b0; assign exceptions_mxfp4[i].sign = 1'b0; end @@ -284,13 +290,14 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; for (genvar i = 0; i < TCK; ++i) begin : g_lane_nvfp4 localparam K_WORD = i / 2; - `UNUSED_VAR ({sf_a[7], sf_b[7]}) - // e4m3 scale factor mantissa mul + // UE4M3 scale factors (unsigned: bit 7 is padding); 0x7F is NaN. + `UNUSED_VAR ({sf_a[7], sf_b[7]}) wire [3:0] sf_man_a = {1'b1, sf_a[2:0]}; wire [3:0] sf_man_b = {1'b1, sf_b[2:0]}; wire [3:0] sf_exp_a = sf_a[6:3]; wire [3:0] sf_exp_b = sf_b[6:3]; + wire sf_nan = (sf_a[6:0] == 7'h7F) || (sf_b[6:0] == 7'h7F); wire [7:0] sf_man_prod; VX_tcu_tfr_wmul #( @@ -307,6 +314,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; wire [3:0][23:0] term_mag_shifted; wire [3:0][5:0] term_exp_biased; wire [3:0] term_valid; + wire [3:0] term_on; wire [3:0] term_sign; wire [3:0][1:0] a_man, b_man; wire [3:0][3:0] f4_man_prod; @@ -316,6 +324,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; localparam OFF = (i % 2) * 16 + j * 4; wire lane_valid = vld_mask[i * 4 + j]; + assign term_on[j] = lane_valid; wire [3:0] raw_a = a_row[K_WORD][OFF +: 4]; wire [3:0] raw_b = b_col[K_WORD][OFF +: 4]; @@ -511,7 +520,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; assign result_exp_nvfp4[i] = EXP_W'(max_exp_biased) + EXP_W'(EXP_BASE_BIASED); // The sign field is only consumed for infinity lanes downstream. - assign exceptions_nvfp4[i].is_nan = 1'b0; + assign exceptions_nvfp4[i].is_nan = sf_nan && (|term_on); assign exceptions_nvfp4[i].is_inf = 1'b0; assign exceptions_nvfp4[i].sign = 1'b0; end diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv index 60d3c41503..1c8c3cbbc6 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv @@ -194,24 +194,28 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; wire [7:0] bias_sel = is_bfloat ? BIAS_CONST_BF8 : BIAS_CONST_FP8; - wire [EXP_W-1:0] max_pre_sum_cpa, bias_sel_cpa; - wire [EXP_W-1:0] final_exp; + // One bit wider than the exponent, signed: the E8M0 scales (2^-254 .. + // 2^256 together) can take a lane below the exponent range, where it + // only adds to the sticky bit. + localparam SX = EXP_W + 1; + wire [SX-1:0] max_pre_sum_cpa, bias_sel_cpa; + wire [SX-1:0] final_exp_s; `ifdef VX_CFG_TCU_MX_ENABLE - wire [EXP_W-1:0] bias_cpa_sum, bias_cpa_carry; - wire [3*EXP_W-1:0] sf_comp = fmt_f[3] ? {EXP_W'(sf_a), EXP_W'(sf_b), -EXP_W'(254)} : (3*EXP_W)'(0); + wire [SX-1:0] bias_cpa_sum, bias_cpa_carry; + wire [3*SX-1:0] sf_comp = fmt_f[3] ? {SX'(sf_a), SX'(sf_b), -SX'(254)} : (3*SX)'(0); VX_csa_tree #( .N(4), - .W(EXP_W), - .S(EXP_W) + .W(SX), + .S(SX) ) exp_sf_csa ( - .operands ({EXP_W'(bias_sel), sf_comp}), + .operands ({SX'(bias_sel), sf_comp}), .sum (bias_cpa_sum), .carry (bias_cpa_carry) ); VX_ks_adder #( - .N(EXP_W), - .BYPASS(`FORCE_BUILTIN_ADDER(EXP_W)) + .N(SX), + .BYPASS(`FORCE_BUILTIN_ADDER(SX)) ) exp_bias_add ( .dataa(bias_cpa_sum), .datab(bias_cpa_carry), @@ -220,19 +224,20 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; `UNUSED_PIN(cout) ); `else - assign bias_sel_cpa = EXP_W'(bias_sel); + assign bias_sel_cpa = SX'(bias_sel); `endif - assign max_pre_sum_cpa = EXP_W'(max_pre_sum); + assign max_pre_sum_cpa = SX'(max_pre_sum); VX_ks_adder #( - .N(EXP_W), - .BYPASS(`FORCE_BUILTIN_ADDER(EXP_W)) + .N(SX), + .BYPASS(`FORCE_BUILTIN_ADDER(SX)) ) exp_final_add ( - .dataa(EXP_W'(max_pre_sum_cpa)), - .datab(EXP_W'(bias_sel_cpa)), + .dataa(max_pre_sum_cpa), + .datab(bias_sel_cpa), .cin(1'b0), - .sum(final_exp), + .sum(final_exp_s), `UNUSED_PIN(cout) ); + wire [EXP_W-1:0] final_exp = final_exp_s[SX-1] ? '0 : final_exp_s[EXP_W-1:0]; // The two fp8 mantissa products in a lane are independent unsigned 4x4 // multiplies — small enough to pack into a single DSP48 (USE_DSP), @@ -335,7 +340,13 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; wire add_nan = (pos_inf_0 && neg_inf_1) || (neg_inf_0 && pos_inf_1); - wire any_nan = (|(nan_sel & lane_valid)) || add_nan; + `ifdef VX_CFG_TCU_MX_ENABLE + // an E8M0 scale of 0xFF is NaN, and so is every element it scales + wire sf_nan = fmt_f[3] && ((sf_a == 8'hFF) || (sf_b == 8'hFF)) && (|lane_valid); + `else + wire sf_nan = 1'b0; + `endif + wire any_nan = (|(nan_sel & lane_valid)) || add_nan || sf_nan; wire any_inf = (|(inf_sel & lane_valid)) && ~any_nan; wire final_sign_inf = inf_sel[0] ? sign_sel[0] : sign_sel[1]; diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 4f2916de0f..a53bf2b353 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -125,6 +125,50 @@ run-tcu: $(call tcu_int_pair,DPI,INT8,17,18) $(call tcu_int_pair,DPI,INT4,19,20) +# VX_tcu_unit end to end (dispatch, uops, operand routing, commit), bit for bit +# (--ulp=0) against the reference of each datapath: the native TFR one against +# the TfrFedp reference model, the DPI one against its discrete fp32 mul/add. +# One build per tile shape and datapath: 4 threads (2x2) with tf32/fp16/bf16, +# 8 threads (4x2) fp16 on DPI. +TCU_UNIT_TFR := $(TCU_BASE) -DVX_CFG_TCU_TYPE_TFR -DVX_CFG_TCU_TF32_ENABLE +TCU_UNIT_DPI := $(TCU_BASE) -DVX_CFG_TCU_TYPE_DPI +run-tcu-unit: + $(MAKE) -C tcu_unit clean + $(MAKE) -C tcu_unit CONFIGS="$(TCU_UNIT_TFR)" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_TFR)" OPTS="--fmt=1 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_TFR)" OPTS="--fmt=2 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_TFR)" OPTS="--fmt=3 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit clean + $(MAKE) -C tcu_unit CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" OPTS="--fmt=1 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" OPTS="--fmt=2 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" OPTS="--fmt=3 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit clean + $(MAKE) -C tcu_unit CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_NUM_THREADS=8" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_NUM_THREADS=8" OPTS="--fmt=2 --ulp=0 --no-trace" + $(call tcu_mx_suite,2,1) + $(call tcu_mx_suite,4,2) + +# The native datapath's fp8 and block-scaled formats against the TfrFedp +# reference model, bit for bit: fp8/bf8, MXFP8/MXBF8 (E8M0 scales), MXFP4 (E8M0) +# and NVFP4 (E4M3). VX_tcu_unit takes its block scales from the metadata path, +# so these run on the FEDP. Two shapes: 2 registers on one scale slot, 4 on two. +# Directed legs pin zero products in an fp8 lane, MXFP4 scales far from one +# (2^-34), MXFP8/MXBF8 scales so small the lane underflows, and NaN scales. +TCU_MX := $(TCU_BASE) -DVX_CFG_TCU_TYPE_TFR -DUSE_FEDP -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE \ + -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_MXFP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE +# format:sf_a:sf_b +TCU_MX_DIRECTED := 10:110:110 8:0:3 9:4:2 8:0xff:127 11:0x7f:0x38 +tcu_mx_opts = --fmt=$(word 1,$(subst :, ,$(1))) --sfa=$(word 2,$(subst :, ,$(1))) --sfb=$(word 3,$(subst :, ,$(1))) +# $(1)=NUM_REGS $(2)=SF +define tcu_mx_suite + $(MAKE) -C tcu_fedp clean + $(MAKE) -C tcu_fedp NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" + $(foreach f,4 5 8 9 10 11,$(MAKE) -C tcu_fedp run NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" OPTS="--fmt=$(f) --W=25 --ulp=0 --no-trace" &&) true + $(foreach f,4 8 9,$(MAKE) -C tcu_fedp run NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" OPTS="--fmt=$(f) --W=25 --ulp=0 --features=zeros --tests=20000 --no-trace" &&) true + $(foreach o,$(TCU_MX_DIRECTED),$(MAKE) -C tcu_fedp run NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" OPTS="$(call tcu_mx_opts,$(o)) --W=25 --ulp=0 --tests=20000 --no-trace" &&) true +endef + # Same native-TFR formats as run-tcu, but with the multipliers mapped onto DSP48 # (USE_DSP=1). USE_DSP is bit-identical to the LUT path by contract, so the same # references apply — a mismatch here is a DSP-packing bug. diff --git a/hw/unittest/tcu_fedp/Makefile b/hw/unittest/tcu_fedp/Makefile index 7f095276fe..507bbee44c 100644 --- a/hw/unittest/tcu_fedp/Makefile +++ b/hw/unittest/tcu_fedp/Makefile @@ -26,8 +26,14 @@ XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME) # configured VX_CFG_TCU_LATENCY. LATENCY ?= $(patsubst -DVX_CFG_TCU_LATENCY=%,%,$(filter -DVX_CFG_TCU_LATENCY=%, $(XCONFIGS))) -CXXFLAGS += -DNUM_REGS=$(NUM_REGS) -DLATENCY=$(LATENCY) +# Block-scale slots of an MX datapath: the lanes split evenly across them. +SF ?= 1 + +CXXFLAGS += -DNUM_REGS=$(NUM_REGS) -DLATENCY=$(LATENCY) -DSF=$(SF) PARAMS := -GN=$(NUM_REGS) -GLATENCY=$(LATENCY) +ifneq (,$(filter -DVX_CFG_TCU_MX_ENABLE, $(XCONFIGS))) +PARAMS += -GSF=$(SF) +endif SRCS += $(DPI_DIR)/dpi_util.cpp SRCS += $(DPI_DIR)/dpi_float.cpp diff --git a/hw/unittest/tcu_fedp/main.cpp b/hw/unittest/tcu_fedp/main.cpp index 3e11bd1dde..5e60fbcaa1 100644 --- a/hw/unittest/tcu_fedp/main.cpp +++ b/hw/unittest/tcu_fedp/main.cpp @@ -158,6 +158,19 @@ void init_default_fp_format(uint32_t fmt, int *exp_bits, int *sig_bits) { *exp_bits = 5; *sig_bits = 2; break; + case 8: // mxfp8 (E4M3, E8M0 block scale) + *exp_bits = 4; + *sig_bits = 3; + break; + case 9: // mxbf8 (E5M2, E8M0 block scale) + *exp_bits = 5; + *sig_bits = 2; + break; + case 10: // mxfp4 (E2M1, E8M0 block scale) + case 11: // nvfp4 (E2M1, E4M3 block scale) + *exp_bits = 2; + *sig_bits = 1; + break; default: break; } @@ -187,6 +200,7 @@ struct TestConfig { int ulp = 1; // float precision error bits int sparsity = 0; // % of zero inputs int32_t test_id = -1; // Specific test ID to run (for debugging) + int sf_a = -1, sf_b = -1; // fixed block scales (debugging), random when negative }; std::ostream &operator<<(std::ostream &os, const TestConfig &config) { @@ -255,9 +269,17 @@ static void print_format(const std::string& prefix, std::vector& value static int float_fmt_width(int exp_bits, int sig_bits) { int element_bits = 1 + exp_bits + sig_bits; + if (element_bits <= 4) return 4; // fp4 packs eight to a register return 8 * ((element_bits + 7) / 8); } +// Block-scaled (MX/NV) formats carry the 2nd MSB of the format code. +static bool is_mx_fmt(uint32_t fmt) { return (fmt & 0x18) == 0x8; } + +#ifndef SF +#define SF 1 +#endif + static int int_fmt_width(int fmt) { switch (fmt) { case 16: return 32; // int32 @@ -486,6 +508,20 @@ class Testbench { return value; } + // A block scale: E8M0 (MX) or UE4M3 (NVFP4, NaN at 0x7F). Half the draws + // stay within 2^+-8 of one, the rest span the whole encoding. + uint8_t generate_scale(uint32_t fmt) { + std::uniform_int_distribution coin(0, 1); + if (fmt == 11) { + std::uniform_int_distribution any(0, 0x7f); + std::uniform_int_distribution near(0x30, 0x48); + return uint8_t(coin(rng_) ? near(rng_) : any(rng_)); + } + std::uniform_int_distribution any(0, 255); + std::uniform_int_distribution near(127 - 8, 127 + 8); + return uint8_t(coin(rng_) ? near(rng_) : any(rng_)); + } + uint32_t generate_fp_value(const std::string &feature, uint32_t exp_bits, uint32_t sig_bits, uint32_t test_id) { const uint32_t all_exp = (exp_bits == 32 ? 0xFFFFFFFFu : ((1u << exp_bits) - 1u)); const uint32_t max_frac = (sig_bits == 32 ? 0xFFFFFFFFu : ((1u << sig_bits) - 1u)); @@ -739,7 +775,7 @@ class Testbench { const uint32_t tests_per_feature = (NT + NF - 1) / NF; #ifdef USE_FEDP - FEDP fedp(config_.exp_bits, config_.sig_bits, (int)config_.frm, config_.W, config_.renorm); + TfrFedp fedp(config_.exp_bits, config_.sig_bits, (int)config_.frm, config_.W, config_.renorm); #endif uint32_t skipped = 0; @@ -780,6 +816,24 @@ class Testbench { uint32_t c_value_hex; std::memcpy(&c_value_hex, &c_value_float, sizeof(float)); + // Block scales, one per scale slot; lane l (two per register) takes + // slot l * SF / lanes, as the datapath maps them. + const bool mx = is_mx_fmt(config_.fmt_s); + const int lanes = 2 * NUM_REGS; + uint8_t sf_slot_a[SF] = {}, sf_slot_b[SF] = {}; + std::vector sf_lane_a(lanes), sf_lane_b(lanes); + if (mx) { + for (int k = 0; k < SF; ++k) { + sf_slot_a[k] = generate_scale(config_.fmt_s); + sf_slot_b[k] = generate_scale(config_.fmt_s); + if (config_.sf_a >= 0) sf_slot_a[k] = uint8_t(config_.sf_a); + if (config_.sf_b >= 0) sf_slot_b[k] = uint8_t(config_.sf_b); + } + for (int l = 0; l < lanes; ++l) { + sf_lane_a[l] = sf_slot_a[(l * SF) / lanes]; + sf_lane_b[l] = sf_slot_b[(l * SF) / lanes]; + } + } // skip if not in selected test id if (config_.test_id >= 0 && test_id != config_.test_id) continue; @@ -793,6 +847,11 @@ class Testbench { pack_elements(a_value_hex, element_bits, NUM_REGS, a_packed.data()); pack_elements(b_value_hex, element_bits, NUM_REGS, b_packed.data()); + #ifdef VX_CFG_TCU_MX_ENABLE + std::memcpy(&dut_->sf_a, sf_slot_a, SF); + std::memcpy(&dut_->sf_b, sf_slot_b, SF); + #endif + // Apply to DUT for (int i = 0; i < NUM_REGS; i++) { WRITE_WDATA(dut_->a_row, i, a_packed[i]); @@ -821,7 +880,9 @@ class Testbench { std::memcpy(&dut_result, &dut_result_bits, sizeof(float)); #ifdef USE_FEDP - float expected = fedp(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS); + float expected = mx ? fedp(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS, + sf_lane_a.data(), sf_lane_b.data(), config_.fmt_s == 11) + : fedp(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS); #else float expected = dot_product(a_value_hex.data(), b_value_hex.data(), c_value_hex, total_elements, config_.exp_bits, config_.sig_bits, config_.fused); @@ -848,6 +909,13 @@ class Testbench { print_float(" bf_values=", b_values_float, true); print_format(" bx_values=", b_value_hex, true); print_float(" c_value=", c_value_float, true); + if (mx) { + std::cout << " sf_a=" << std::hex; + for (int k = 0; k < SF; ++k) std::cout << " 0x" << int(sf_slot_a[k]); + std::cout << ", sf_b="; + for (int k = 0; k < SF; ++k) std::cout << " 0x" << int(sf_slot_b[k]); + std::cout << std::dec << std::endl; + } print_float(" expected=", expected, true); print_float(" actual=", dut_result, true); std::cout << " delta=" << delta << std::endl; @@ -927,6 +995,10 @@ TestConfig parse_args(int argc, char **argv) { config_.num_tests = std::stoi(arg.substr(8)); } else if (arg.substr(0, 7) == "--test=") { config_.test_id = std::stoi(arg.substr(7)); + } else if (arg.substr(0, 6) == "--sfa=") { + config_.sf_a = std::stoi(arg.substr(6), nullptr, 0); + } else if (arg.substr(0, 6) == "--sfb=") { + config_.sf_b = std::stoi(arg.substr(6), nullptr, 0); } else if (arg.substr(0, 7) == "--seed=") { config_.random_seed = std::stoi(arg.substr(7)); } else { diff --git a/hw/unittest/tcu_unit/Makefile b/hw/unittest/tcu_unit/Makefile index 25a468a2e5..936f24d4ea 100644 --- a/hw/unittest/tcu_unit/Makefile +++ b/hw/unittest/tcu_unit/Makefile @@ -56,12 +56,14 @@ RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/hardfloat/source/RISCV TOP := VX_tcu_fedp_bhf endif +# the native TFR datapath is checked against tcu_fedp's windowed FEDP model ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr +CXXFLAGS += -DUSE_FEDP endif # default test: fp16 -OPTS ?= --fmt=1 +OPTS ?= --fmt=2 TOP := VX_tcu_unit_top diff --git a/hw/unittest/tcu_unit/main.cpp b/hw/unittest/tcu_unit/main.cpp index 9b52e631e2..847e57b3fc 100644 --- a/hw/unittest/tcu_unit/main.cpp +++ b/hw/unittest/tcu_unit/main.cpp @@ -43,6 +43,9 @@ #include "VX_config.h" #include "softfloat_ext.h" #include +#ifdef USE_FEDP +#include "fedp.h" +#endif bool sim_trace_enabled() { return true; } @@ -74,9 +77,9 @@ constexpr int LANE_WORDS = VX_CFG_XLEN / 32; // TCU format IDs (must match VX_tcu_pkg.sv) constexpr uint32_t TCU_FP32_ID = 0; -constexpr uint32_t TCU_FP16_ID = 1; -constexpr uint32_t TCU_BF16_ID = 2; -constexpr uint32_t TCU_TF32_ID = 5; +constexpr uint32_t TCU_TF32_ID = 1; +constexpr uint32_t TCU_FP16_ID = 2; +constexpr uint32_t TCU_BF16_ID = 3; // WMMA op_type (INST_TCU_WMMA = 4'h0 from VX_gpu_pkg.sv) constexpr uint32_t INST_TCU_WMMA = 0; @@ -114,6 +117,8 @@ struct FmtInfo { int exp_bits; int sig_bits; int width_bits; // 16, 19 (tf32), or 32 + // a 32-bit register packs this many elements, element e in bits [e*width +: width] + int per_word() const { return 32 / width_bits; } }; static FmtInfo get_fmt_info(uint32_t fmt_id) { @@ -129,41 +134,54 @@ static FmtInfo get_fmt_info(uint32_t fmt_id) { } // Convert a packed source-format element to float (for software reference). -static float elem_to_float(uint32_t raw, const FmtInfo &fi) { +[[maybe_unused]] static float elem_to_float(uint32_t raw, const FmtInfo &fi) { return cvt_custom_to_f32(raw, fi.exp_bits, fi.sig_bits, 0 /*RNE*/, nullptr); } // ---- Software WMMA reference ----------------------------------------------- -// Computes D[t] = sum_k(A[i][k] * B[k][j]) + C[t] -// where t = i*TC_N + j, A in row-major, B in col-major (each column consecutive). -// -// rs1_data layout: lane t -> A[i][k] for t = i*TC_K + k (A in row-major) -// BUT each OUTPUT thread t=(i,j) shares the same A row. -// a_row[k] = rs1_data[i * TC_K + k] (all TC_N output threads in the same row share this) -// rs2_data layout: lane t -> B[k][j] for t = j*TC_K + k (B^T in row-major, i.e. B columns packed) -// b_col[k] = rs2_data[j * TC_K + k] -// rs3_data layout: lane t -> C[i][j] where t = i*TC_N + j -// +// D[t] = sum_k(A[i][k] * B[k][j]) + C[t], t = i*TC_N + j. Each register lane +// packs per_word() elements: +// rs1 lane i*TC_K + w holds A[i][w*per_word() ...] (row i of A) +// rs2 lane j*TC_K + w holds B[w*per_word() ...][j] (column j of B) +// rs3 lane t holds C[t] (fp32) +// The native TFR datapath is checked against the windowed FEDP model +// (USE_FEDP, as tcu_fedp); the DPI datapath against its discrete fp32 mul/add. static std::vector wmma_ref( - const std::vector &rs1, // VX_CFG_NUM_THREADS elements (A, row-major) - const std::vector &rs2, // VX_CFG_NUM_THREADS elements (B columns, col-major) - const std::vector &rs3, // VX_CFG_NUM_THREADS elements (C, row-major) - const FmtInfo &fi_s) // source format (for A and B) + const std::vector &rs1, + const std::vector &rs2, + const std::vector &rs3, + const FmtInfo &fi_s, + int window) { std::vector result(VX_CFG_NUM_THREADS); - +#ifdef USE_FEDP + TfrFedp fedp(fi_s.exp_bits, fi_s.sig_bits, 0 /*RNE*/, window); +#else + (void)window; +#endif for (int i = 0; i < TCU_TC_M; i++) { for (int j = 0; j < TCU_TC_N; j++) { int t = i * TCU_TC_N + j; - // Accumulate with fp32 rounding at each multiply-add step (matches DPI model). - float prod = 0.0f; + float c = bit_cast(rs3[t]); +#ifdef USE_FEDP + result[t] = fedp(&rs1[i * TCU_TC_K], &rs2[j * TCU_TC_K], c, TCU_TC_K); +#else + const int epw = fi_s.per_word(); + const uint32_t emask = (fi_s.width_bits == 32) ? ~0u : ((1u << fi_s.width_bits) - 1u); + // the DPI datapath's order: each register's products summed in + // fp32, then the register sums in order, then C + float sum = 0.0f; for (int k = 0; k < TCU_TC_K; k++) { - float a = elem_to_float(rs1[i * TCU_TC_K + k], fi_s); - float b = elem_to_float(rs2[j * TCU_TC_K + k], fi_s); - prod += a * b; + float word_sum = 0.0f; + for (int e = 0; e < epw; e++) { + uint32_t ea = (rs1[i * TCU_TC_K + k] >> (e * fi_s.width_bits)) & emask; + uint32_t eb = (rs2[j * TCU_TC_K + k] >> (e * fi_s.width_bits)) & emask; + word_sum += elem_to_float(ea, fi_s) * elem_to_float(eb, fi_s); + } + sum += word_sum; } - float c = bit_cast(rs3[t]); // C is always fp32 - result[t] = prod + c; + result[t] = sum + c; +#endif } } return result; @@ -208,7 +226,9 @@ struct TestConfig { RoundingMode frm = RoundingMode::RNE; uint32_t num_tests = 1000; int ulp = 1; + int window = 25; // TFR accumulator window (FEDP model) int32_t test_id = -1; + bool directed_only = false; std::vector features = {"infinities","nans","subnormals","normals","zeros"}; }; @@ -250,6 +270,30 @@ static uint32_t gen_fp_value(const std::string &feat, uint32_t eb, uint32_t sb, std::abort(); } +// ---- Directed vectors ------------------------------------------------------- +// Register contents (rs1 = A rows, rs2 = B columns, rs3 = C) on which the native +// datapath's windowed reduction and a discrete fp32 dot product round apart; +// each runs ahead of the random tests for its format and shape. +struct DirectedVec { + const char *name; + uint32_t fmt; + int num_threads; + uint32_t rs1[4], rs2[4], rs3[4]; +}; +static const DirectedVec kDirected[] = { + // fp16 rows of subnormals only: the window aligns on the products' + // exponent fields, so their few significant bits sit low in it + {"fp16 subnormal row", TCU_FP16_ID, 4, + {0x015e005e, 0x024080e2, 0x81fb0228, 0x820c8134}, + {0x41389f40, 0x96d5bc23, 0x285ac58a, 0x7044a57d}, + {0x4feb0a36, 0x4ca6e55a, 0x9a3df577, 0xfa232341}}, + // fp16 products that nearly cancel: the window truncates the residue + {"fp16 cancellation", TCU_FP16_ID, 4, + {0x2f13b5fb, 0x1f98ae04, 0xbdc6632b, 0xc171dfc8}, + {0xb03832e2, 0x2f380ce8, 0x609b8940, 0x7507d768}, + {0x387492a6, 0x2b2843b3, 0xad72f4f6, 0x858ca727}}, +}; + // ---- Testbench class ------------------------------------------------------- class Testbench { public: @@ -300,6 +344,25 @@ class Testbench { << " VX_CFG_NUM_THREADS=" << VX_CFG_NUM_THREADS << "\n"; + // directed vectors for this format and tile shape + uint32_t directed_done = 0; + for (const auto &v : kDirected) { + if (v.fmt != cfg_.fmt_s || v.num_threads != VX_CFG_NUM_THREADS) + continue; + std::vector rs1(v.rs1, v.rs1 + VX_CFG_NUM_THREADS); + std::vector rs2(v.rs2, v.rs2 + VX_CFG_NUM_THREADS); + std::vector rs3(v.rs3, v.rs3 + VX_CFG_NUM_THREADS); + if (!this->check(rs1, rs2, rs3, fi, std::string("Directed (") + v.name + ")")) + return false; + ++directed_done; + } + if (directed_done != 0) { + std::cout << directed_done << " directed test(s) PASSED!\n"; + } + if (cfg_.directed_only) { + return true; + } + uint32_t tests_done = 0; for (uint32_t test_id = 0; test_id < cfg_.num_tests; test_id++) { if (cfg_.test_id >= 0 && (int)test_id != cfg_.test_id) @@ -309,37 +372,30 @@ class Testbench { size_t feat_idx = test_id % cfg_.features.size(); const std::string &feat = cfg_.features[feat_idx]; - // Generate source elements for A and B (VX_CFG_NUM_THREADS values each) - std::vector rs1(VX_CFG_NUM_THREADS), rs2(VX_CFG_NUM_THREADS), rs3(VX_CFG_NUM_THREADS); - for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { - rs1[t] = gen_fp_value(feat, fi.exp_bits, fi.sig_bits, test_id * VX_CFG_NUM_THREADS + t, rng_); - rs2[t] = gen_fp_value("normals", fi.exp_bits, fi.sig_bits, test_id * VX_CFG_NUM_THREADS + t, rng_); - float c = bit_cast(gen_fp_value("normals", 8, 23, test_id * VX_CFG_NUM_THREADS + t + 1, rng_)); - rs3[t] = bit_cast(c); + // A carries the feature, B and C are normals; each register lane + // packs per_word() source elements. + const int epw = fi.per_word(); + std::vector rs1(VX_CFG_NUM_THREADS, 0), rs2(VX_CFG_NUM_THREADS, 0), rs3(VX_CFG_NUM_THREADS); + for (int t = 0; t < TCU_TC_M * TCU_TC_K; t++) { + for (int e = 0; e < epw; e++) { + uint32_t elem_id = (test_id * VX_CFG_NUM_THREADS + t) * epw + e; + rs1[t] |= gen_fp_value(feat, fi.exp_bits, fi.sig_bits, elem_id, rng_) << (e * fi.width_bits); + } } - - // Compute software reference - std::vector expected = wmma_ref(rs1, rs2, rs3, fi); - // Drive DUT - std::vector result = drive_and_collect(rs1, rs2, rs3, fi); - - bool test_pass = true; - for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { - float act = bit_cast(result[t]); - int delta = approx_equal(act, expected[t]); - if (std::abs(delta) > cfg_.ulp) { - if (test_pass) { - // First failure for this test - std::cout << "Test #" << test_id << " (" << feat << ") FAILED:\n"; - } - std::cout << " thread=" << t - << " expected="; print_float("", expected[t], false); - std::cout << " actual="; print_float("", act, false); - std::cout << " delta=" << delta << "\n"; - test_pass = false; + for (int t = 0; t < TCU_TC_N * TCU_TC_K; t++) { + for (int e = 0; e < epw; e++) { + uint32_t elem_id = (test_id * VX_CFG_NUM_THREADS + t) * epw + e; + rs2[t] |= gen_fp_value("normals", fi.exp_bits, fi.sig_bits, elem_id, rng_) << (e * fi.width_bits); } } - if (!test_pass) return false; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { + rs3[t] = gen_fp_value("normals", 8, 23, test_id * VX_CFG_NUM_THREADS + t + 1, rng_); + } + + std::ostringstream label; + label << "Test #" << test_id << " (" << feat << ")"; + if (!this->check(rs1, rs2, rs3, fi, label.str())) + return false; tests_done++; } @@ -354,6 +410,38 @@ class Testbench { } private: + // Drive one packet and compare every thread against the reference. + bool check(const std::vector &rs1, const std::vector &rs2, + const std::vector &rs3, const FmtInfo &fi, const std::string &label) { + std::vector expected = wmma_ref(rs1, rs2, rs3, fi, cfg_.window); + std::vector result = drive_and_collect(rs1, rs2, rs3, fi); + bool pass = true; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { + float act = bit_cast(result[t]); + int delta = approx_equal(act, expected[t]); + if (std::abs(delta) > cfg_.ulp) { + if (pass) { + std::cout << label << " FAILED:\n"; + } + std::cout << " thread=" << t + << " expected="; print_float("", expected[t], false); + std::cout << " actual="; print_float("", act, false); + std::cout << " delta=" << delta << "\n"; + pass = false; + } + } + if (!pass) { + std::cout << std::hex << " rs1="; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) std::cout << " 0x" << rs1[t]; + std::cout << "\n rs2="; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) std::cout << " 0x" << rs2[t]; + std::cout << "\n rs3="; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) std::cout << " 0x" << rs3[t]; + std::cout << std::dec << "\n"; + } + return pass; + } + TestConfig cfg_; uint64_t cycle_; std::mt19937 rng_; @@ -454,7 +542,8 @@ class Testbench { write_lane(&dut_->dispatch_rs3_data, t, rs3[t]); } - // Clock until dispatch fires (ready goes high) + // The packet fires on the edge that samples valid with ready: test + // ready ahead of each edge, so it is sent exactly once. dut_->commit_ready = 1; uint64_t start_cycle = cycle_; while (true) { @@ -462,10 +551,12 @@ class Testbench { std::cerr << "TIMEOUT waiting for dispatch_ready\n"; std::abort(); } + dut_->clk = 0; + dut_->eval(); + bool fire = dut_->dispatch_ready; tick(); - if (dut_->dispatch_ready) break; + if (fire) break; } - // Dispatch fired: de-assert valid idle_dispatch(); // Wait for commit result @@ -508,6 +599,10 @@ static TestConfig parse_args(int argc, char **argv) { cfg.random_seed = std::stoi(arg.substr(7)); } else if (arg.substr(0, 6) == "--ulp=") { cfg.ulp = std::stoi(arg.substr(6)); + } else if (arg == "--directed") { + cfg.directed_only = true; + } else if (arg.substr(0, 4) == "--W=") { + cfg.window = std::stoi(arg.substr(4)); } else if (arg.substr(0, 14) == "--trace-start=") { cfg.trace_start = std::stoull(arg.substr(14)); } else if (arg.substr(0, 12) == "--trace-end=") { @@ -519,12 +614,14 @@ static TestConfig parse_args(int argc, char **argv) { while (std::getline(ss, item, ',')) cfg.features.push_back(item); } else { std::cout << "Usage: " << argv[0] << " [options]\n" - << " --fmt= Source format (0=fp32,1=fp16,2=bf16,5=tf32)\n" + << " --fmt= Source format (1=tf32,2=fp16,3=bf16)\n" << " --frm= Rounding mode (RNE,RTZ,RDN,RUP,RMM)\n" << " --tests= Number of test cases (default 1000)\n" << " --test= Run a single test by id\n" << " --seed= Random seed\n" << " --ulp= ULP tolerance (default 1)\n" + << " --W= TFR accumulator window of the FEDP model (default 25)\n" + << " --directed Run only the directed vectors\n" << " --no-trace Disable VCD output\n" << " --features=LIST Comma-separated list of features to test\n" << " --trace-start=N Start VCD trace at timestamp N\n" diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index f5e33c6984..1d4a9a1880 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -16,6 +16,7 @@ #include "tcu_unit.h" #include "tensor_cfg.h" #include +#include #include "core.h" #include "scheduler.h" #include "local_mem.h" @@ -75,8 +76,8 @@ static inline uint8_t unpack_u8(uint32_t word, uint32_t idx) { return (word >> (idx * 8)) & 0xffu; } -// FMA: fused multiply-add returning an Ot-typed accumulator (bit-packed in uint32). -// Widens narrow inputs/accumulator to fp32, performs mul+add, rounds once to Ot. +// FMA: one multiply-add step into an Ot-typed accumulator (bit-packed +// in uint32): fp32 for float inputs, int32 for integer inputs. template struct FMA { using itype = typename It::dtype; @@ -97,15 +98,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint16_t a, uint16_t b, uint32_t c) { - auto fa = rv_htof_s(a, 0, nullptr); - auto fb = rv_htof_s(b, 0, nullptr); - auto fc = rv_htof_s(uint16_t(c), 0, nullptr); - return rv_ftoh_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- bf16 inputs -- template <> struct FMA { static uint32_t eval(uint16_t a, uint16_t b, uint32_t c) { @@ -114,15 +106,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint16_t a, uint16_t b, uint32_t c) { - auto fa = rv_btof_s(a, 0, nullptr); - auto fb = rv_btof_s(b, 0, nullptr); - auto fc = rv_btof_s(uint16_t(c), 0, nullptr); - return rv_ftob_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- fp8 inputs -- template <> struct FMA { static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { @@ -131,15 +114,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { - auto fa = rv_e4m3tof_s(a, 0, nullptr); - auto fb = rv_e4m3tof_s(b, 0, nullptr); - auto fc = rv_e4m3tof_s(uint8_t(c), 0, nullptr); - return rv_ftoe4m3_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- bf8 inputs -- template <> struct FMA { static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { @@ -148,15 +122,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { - auto fa = rv_e5m2tof_s(a, 0, nullptr); - auto fb = rv_e5m2tof_s(b, 0, nullptr); - auto fc = rv_e5m2tof_s(uint8_t(c), 0, nullptr); - return rv_ftoe5m2_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- tf32 inputs -- template <> struct FMA { static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { @@ -165,19 +130,21 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = rv_tf32tof_s(a, 0, nullptr); - auto fb = rv_tf32tof_s(b, 0, nullptr); - auto fc = rv_tf32tof_s(c, 0, nullptr); - return rv_ftotf32_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - -// Generic FEDP: universal rule keyed on output width. -// * Wide Ot (fp32): accumulate Σ(a_k*b_k) in fp32, add c_val last. -// * Narrow Ot (fp16/bf16/fp8/bf8/…): chain FMA so the accumulator is -// rounded to Ot each step. +// Float source formats the native TFR datapath reduces (exponent, mantissa bits). +template struct TfrFormat { static constexpr bool valid = false; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 5, sb = 10; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 8, sb = 7; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 4, sb = 3; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 5, sb = 2; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 8, sb = 10; }; + +// Generic FEDP, keyed on the output format. +// * fp32 Ot on the native TFR datapath: its fixed-point reduction — +// products aligned to the largest exponent in a 25-bit window, summed as +// integers, rounded once (the TfrFedp model the RTL is checked against). +// * fp32 Ot otherwise: accumulate Σ(a_k*b_k) in fp32 per register, +// the register sums in order, c_val last (the DPI datapath's order). +// * int32 Ot (integer inputs): exact integer accumulation. template struct FEDP { using itype = typename It::dtype; @@ -188,6 +155,22 @@ struct FEDP { static uint32_t eval_n(const reg_data_t *a_row, const reg_data_t *b_col, uint32_t c_val, uint32_t k_words) { constexpr uint32_t i_ratio = sizeof(uint32_t) / sizeof(itype); static_assert(i_ratio * sizeof(itype) == sizeof(uint32_t), "FEDP: tcK * i_ratio must be <= 32"); +#ifdef VX_CFG_TCU_TYPE_TFR + if constexpr (std::is_same_v && TfrFormat::valid) { + std::vector a(k_words), b(k_words); + for (uint32_t z = 0; z < k_words; ++z) { + a[z] = a_row[z].u32; + b[z] = b_col[z].u32; + } + TfrFedp fedp(TfrFormat::eb, TfrFormat::sb); + float c; + std::memcpy(&c, &c_val, sizeof(c)); + float d = fedp(a.data(), b.data(), c, k_words); + uint32_t d_bits; + std::memcpy(&d_bits, &d, sizeof(d_bits)); + return d_bits; + } else +#endif if constexpr (std::is_same_v) { uint32_t acc = 0; for (uint32_t z = 0; z < k_words; ++z) { @@ -201,6 +184,7 @@ struct FEDP { } return rv_fadd_s(c_val, acc, 0, nullptr); } else { + static_assert(std::is_same_v, "TCU output type must be fp32 (int32 for integer inputs)"); uint32_t acc = c_val; for (uint32_t z = 0; z < k_words; ++z) { auto a = reinterpret_cast(&a_row[z].u32); @@ -282,26 +266,6 @@ static PFN_FEDP_N select_FEDP_N(uint32_t IT, uint32_t OT) { break; } break; - case vt::fp16::id: - if (IT == vt::fp16::id) - return FEDP::eval_n; - break; - case vt::bf16::id: - if (IT == vt::bf16::id) - return FEDP::eval_n; - break; - case vt::fp8::id: - if (IT == vt::fp8::id) - return FEDP::eval_n; - break; - case vt::bf8::id: - if (IT == vt::bf8::id) - return FEDP::eval_n; - break; - case vt::tf32::id: - if (IT == vt::tf32::id) - return FEDP::eval_n; - break; case vt::int32::id: switch (IT) { case vt::int8::id: @@ -620,6 +584,12 @@ class TcuUnit::Impl { trace->dst_data.assign(num_threads, reg_data_t{}); auto& rd_data = trace->dst_data; + if (tcu_type != TcuType::TCU_LD && !vt::tcu_output_valid(tpuArgs.fmt_s, tpuArgs.fmt_d)) { + std::cerr << "*** TCU output format " << tpuArgs.fmt_d << " for input format " << tpuArgs.fmt_s + << " is not supported: the TCU returns fp32 (int32 for integer inputs)" << std::endl; + std::abort(); + } + switch (tcu_type) { case TcuType::WMMA: case TcuType::WMMA_SP: @@ -1371,6 +1341,46 @@ class TcuUnit::Impl { }; if (fmt_d == vt::fp32::id) { +#ifdef VX_CFG_TCU_TYPE_TFR + // The native datapath: the TfrFedp model, with each lane's (16 bits of + // a register) scales taken at its first element. + { + int eb = 0, sb = 0; + bool e4m3_scale = false; + if (fmt_s == vt::mxfp8::id) { + eb = 4; sb = 3; + } else if (fmt_s == vt::mxbf8::id) { + eb = 5; sb = 2; + } else if (fmt_s == vt::mxfp4::id) { + eb = 2; sb = 1; + } else if (fmt_s == vt::nvfp4::id) { + eb = 2; sb = 1; e4m3_scale = true; + } else { + std::abort(); + } + uint32_t sparse_ratio = is_sparse ? 2 : 1; + std::vector a(fedp_words), b(fedp_words); + std::vector sf_a(2 * fedp_words), sf_b(2 * fedp_words); + for (uint32_t z = 0; z < fedp_words; ++z) { + a[z] = a_row[z].u32; + b[z] = b_col[z].u32; + } + for (uint32_t lane = 0; lane < 2 * fedp_words; ++lane) { + uint32_t z = lane / 2; + uint32_t e = (lane % 2) * (ratio / 2); + uint32_t elem_k = ((step_k * fedp_words + z) * ratio + e) * sparse_ratio; + sf_a[lane] = uint8_t(scale_a(elem_k)); + sf_b[lane] = uint8_t(scale_b(elem_k)); + } + TfrFedp fedp(eb, sb); + float c; + std::memcpy(&c, &c_val, sizeof(c)); + float d = fedp(a.data(), b.data(), c, fedp_words, sf_a.data(), sf_b.data(), e4m3_scale); + uint32_t d_bits; + std::memcpy(&d_bits, &d, sizeof(d_bits)); + return d_bits; + } +#endif uint32_t acc = c_val; for (uint32_t z = 0; z < fedp_words; ++z) { for (uint32_t e = 0; e < ratio; ++e) { diff --git a/hw/unittest/tcu_fedp/fedp.h b/sw/common/fedp.h similarity index 64% rename from hw/unittest/tcu_fedp/fedp.h rename to sw/common/fedp.h index d93beefb23..486128c3a0 100644 --- a/hw/unittest/tcu_fedp/fedp.h +++ b/sw/common/fedp.h @@ -25,9 +25,9 @@ #include #ifdef FEDP_TRACE -#define LOG(...) std::fprintf(stderr, __VA_ARGS__); +#define FEDP_LOG(...) std::fprintf(stderr, __VA_ARGS__); #else -#define LOG(...) +#define FEDP_LOG(...) #endif #ifdef FEDP_RTL_WATCH @@ -36,26 +36,42 @@ #define RTL_WATCH(...) #endif -class FEDP { +class TfrFedp { public: - explicit FEDP(int exp_bits = 5, int sig_bits = 10, int frm = 0, int W = 25, bool renorm = false, bool no_window = false) + explicit TfrFedp(int exp_bits = 5, int sig_bits = 10, int frm = 0, int W = 25, bool renorm = false, bool no_window = false) : exp_bits_(exp_bits), sig_bits_(sig_bits), frm_(frm), W_(W), renorm_(renorm), no_window_(no_window) { - LOG("[ctor] fmt=e%dm%d, frm=%d, W=%d, renorm_=%s, no_window_=%s\n", + FEDP_LOG("[ctor] fmt=e%dm%d, frm=%d, W=%d, renorm_=%s, no_window_=%s\n", exp_bits_, sig_bits_, frm_, W_, (renorm_ ? "true" : "false"), (no_window_ ? "true" : "false")); assert(exp_bits_ > 0 && exp_bits_ <= 8); assert(sig_bits_ > 0 && sig_bits_ <= 10); assert(frm_ >= 0 && frm_ <= 4); } + // Block-scaled formats: one scale byte per lane for A and B, two lanes per + // 32-bit register (a lane is 16 bits of it: two fp8 or four fp4 elements). + // fp8/bf8 (MXFP8/MXBF8) and fp4 (MXFP4): E8M0 scales; + // fp4 with e4m3_scale (NVFP4): E4M3 scales. + float operator()(const uint32_t *a, const uint32_t *b, float c, uint32_t n, + const uint8_t *sf_a, const uint8_t *sf_b, bool e4m3_scale = false) { + sf_a_ = sf_a; + sf_b_ = sf_b; + e4m3_scale_ = e4m3_scale; + float d = (*this)(a, b, c, n); + sf_a_ = sf_b_ = nullptr; + e4m3_scale_ = false; + return d; + } + float operator()(const uint32_t *a, const uint32_t *b, float c, uint32_t n) { const auto c_enc = bitsFromF32(c); rtl_watch_s0(a, b, c_enc, n); - const auto terms = decode_inputs(a, b, n); const auto c_dec = decode_input(c_enc, 8, 23); const auto c_term = decodeC_to_common(c_dec); - const auto mul_res = multiply_to_common(terms, c_term); + const auto mul_res = (exp_bits_ == 2 && sig_bits_ == 1) + ? multiply_f4(a, b, n, c_term) + : multiply_to_common(decode_inputs(a, b, n), c_term); rtl_watch_s1(mul_res, c_term, c_enc); @@ -157,7 +173,7 @@ class FEDP { } // -------- [RTL-WATCH] helpers -------- - inline void rtl_watch_s0(const uint32_t* a, const uint32_t* b, uint32_t c_enc, uint32_t n) { + inline void rtl_watch_s0([[maybe_unused]] const uint32_t* a, [[maybe_unused]] const uint32_t* b, [[maybe_unused]] uint32_t c_enc, [[maybe_unused]] uint32_t n) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S0(%lu): a_row=", req_id_); RTL_WATCH("{"); @@ -174,7 +190,7 @@ class FEDP { #endif } - inline void rtl_watch_s1(const mul_res_t& res, const cterm_t& c_term, uint32_t c_enc) { + inline void rtl_watch_s1([[maybe_unused]] const mul_res_t& res, [[maybe_unused]] const cterm_t& c_term, [[maybe_unused]] uint32_t c_enc) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S1(%lu): max_exp=0x%x, shift_amt=", req_id_, res.L); RTL_WATCH("{"); @@ -202,7 +218,7 @@ class FEDP { #endif } - inline void rtl_watch_s2(const align_t& aln, const mul_res_t& res, const cterm_t& c_term, uint32_t c_enc) { + inline void rtl_watch_s2([[maybe_unused]] const align_t& aln, [[maybe_unused]] const mul_res_t& res, [[maybe_unused]] const cterm_t& c_term, [[maybe_unused]] uint32_t c_enc) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S2(%lu): max_exp=0x%x, aln_sig=", req_id_, res.L); RTL_WATCH("{"); @@ -228,7 +244,7 @@ class FEDP { #endif } - inline void rtl_watch_s3(const acc_t& acc, const mul_res_t& res, const cterm_t& c_term, uint32_t c_enc) { + inline void rtl_watch_s3([[maybe_unused]] const acc_t& acc, [[maybe_unused]] const mul_res_t& res, [[maybe_unused]] const cterm_t& c_term, [[maybe_unused]] uint32_t c_enc) { #ifdef FEDP_RTL_WATCH uint32_t sigs_sign = 0; for (size_t i = 0; i < res.terms.size() - 1; ++i) { @@ -239,7 +255,7 @@ class FEDP { #endif } - inline void rtl_watch_norm(const nrm_t& nrm, int flags, int max_exp) { + inline void rtl_watch_norm([[maybe_unused]] const nrm_t& nrm, [[maybe_unused]] int flags, [[maybe_unused]] int max_exp) { #ifdef FEDP_RTL_WATCH int lsb = nrm.kept & 1; RTL_WATCH("[RTL_WATCH] FEDP-NORM(%lu): abs_sum=0x%x, L=%d, G=%d, R=%d, S=%d, Rup=%d\n", @@ -248,12 +264,19 @@ class FEDP { #endif } - inline void rtl_watch_s4(uint32_t result) { + inline void rtl_watch_s4([[maybe_unused]] uint32_t result) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S4(%lu): result=0x%08x\n", req_id_, result); #endif } + // Exponent of a zero term: it stands for the datapath's exponent 0, which + // add_c_term places WA - 1 below this model's exponent origin. + static constexpr int kZeroE = -(1 << 20); + + // The datapath's accumulator width over `lanes` product terms and C. + int acc_width(uint32_t lanes) const { return W_ + 2 + (32 - lzcN(lanes, 32)); } + // -------- logic -------- static inline std::pair csa(uint32_t a, uint32_t b, uint32_t c) { uint32_t s = a ^ b ^ c; @@ -303,10 +326,11 @@ class FEDP { const int sb = sig_bits_, eb = exp_bits_; std::vector out; std::vector eps; // per-term *window LSB exponent* (not raw Ep/Ec) + std::vector sel_eps; + std::vector nonzero; out.reserve(v.size() + 1); eps.reserve(v.size() + 1); - bool c_is_zero = (c_term.cls == 0 && c_term.sp == 0); uint8_t c_nan = (c_term.sp == 3) ? FL_NAN : 0; uint8_t c_pinf = (c_term.sp == 1) ? FL_PINF : 0; @@ -354,8 +378,14 @@ class FEDP { Ep -= lzc_prod; } + // the fp8 lane pairs its two products by their exponent fields even + // when one is zero (a zero field counts as 1, as for a subnormal) + sel_eps.push_back((std::max(a.exp, 1) - bias(eb)) + (std::max(b.exp, 1) - bias(eb)) + + F32_BIAS + EXP_POS_BIAS + 1 - W_); + nonzero.push_back(!p_is_zero); + if (p_is_zero) { - Ep = 0; + Ep = kZeroE; Mp = 0; } @@ -372,7 +402,22 @@ class FEDP { for (size_t j = 0; j < out.size(); j += 2) { if (j + 1 < out.size()) { - auto r = reduce_f8(out[j], eps[j], out[j + 1], eps[j + 1]); + auto r = reduce_f8(out[j], sel_eps[j], nonzero[j], out[j + 1], sel_eps[j + 1], nonzero[j + 1]); + // MX: the lane's E8M0 scales add to its exponent field, which is + // 10 bits wide and sits WA - 1 above this model's exponents + if (sf_a_) { + // an E8M0 scale of 0xFF is NaN, and so is every element it scales + if (sf_a_[j / 2] == 0xff || sf_b_[j / 2] == 0xff) { + flags |= FL_NAN; + } + // a lane scaled below the exponent range sits at exponent 0, + // where it only adds to the sticky bit + if (r.second != kZeroE) { + const int WA = acc_width(uint32_t(out.size() / 2)); + int lane_exp = r.second + (WA - 1) + int(sf_a_[j / 2]) + int(sf_b_[j / 2]) - 254; + r.second = std::max(lane_exp, 0) - (WA - 1); + } + } red_out.push_back(r.first); red_eps.push_back(r.second); } else { @@ -385,13 +430,23 @@ class FEDP { eps.swap(red_eps); } + return add_c_term(std::move(out), std::move(eps), c_term, flags); + } + + // Append the C term and take every term's shift to the largest exponent. + mul_res_t add_c_term(std::vector out, std::vector eps, const cterm_t &c_term, uint8_t flags) { + bool c_is_zero = (c_term.cls == 0 && c_term.sp == 0); int Ec = c_term.Ec + F32_BIAS + EXP_POS_BIAS + 24 - W_; if (c_is_zero) { - Ec = 0; + Ec = kZeroE; } out.push_back({c_term.sign, c_term.Mc}); eps.push_back(Ec); + const int WA = acc_width(uint32_t(out.size() - 1)); + for (auto &e : eps) { + if (e == kZeroE) e = -(WA - 1); + } int L = *std::max_element(eps.begin(), eps.end()); @@ -404,12 +459,99 @@ class FEDP { return {out, L , shifts, flags}; } - std::pair reduce_f8(const pterm_t& t1, int e1, const pterm_t& t2, int e2) { + // Block-scaled fp4 (E2M1), one lane per 16 bits: its four products are + // aligned to their largest exponent in a 6-bit lane field, summed in 27-bit + // two's complement, and leave the datapath as one term. E2M1 has no + // infinity or NaN. MXFP4 adds the E8M0 scales to the lane exponent; NVFP4 + // folds the E4M3 scales' exponent fields into the 6-bit field (they fit) and + // multiplies each product by the scales' mantissa product. + mul_res_t multiply_f4(const uint32_t *a, const uint32_t *b, uint32_t n, const cterm_t &c_term) { + assert(sf_a_ && sf_b_ && "fp4 needs block scales"); + const int SIG_SHIFT = 11; + const int EXP_TERM_BIAS = 32; + const int exp_comp = e4m3_scale_ ? -(2 * 1 + 2 * (7 + 3)) : -(2 * 1 + 10); + const int exp_adj = (EXP_TERM_BIAS + exp_comp) & 63; + // the lane exponent the datapath emits is 10 bits wide; this + // model's lane exponents sit WA - 1 below it + const int WA = acc_width(2 * n); + const int bias_base = F32_BIAS + 2 * (23 - 22) - W_ + WA - 1 + EXP_POS_BIAS; + auto f4_man = [](uint32_t x) -> uint32_t { return ((x >> 1) & 3) == 0 ? 1u : (2u | (x & 1)); }; + auto f4_exp = [](uint32_t x) -> uint32_t { + uint32_t e2 = (x >> 2) & 1, e1 = (x >> 1) & 1; + return (e2 & ~e1 & 1) | ((e2 & e1) << 1); + }; + std::vector out; + std::vector eps; + uint8_t flags = 0; + if (c_term.sp == 3) flags |= FL_NAN; + if (c_term.sp == 1) flags |= FL_PINF; + if (c_term.sp == 2) flags |= FL_NINF; + for (uint32_t lane = 0; lane < 2 * n; ++lane) { + const uint32_t word = lane / 2, off = (lane % 2) * 16; + const uint32_t sfa = sf_a_[lane], sfb = sf_b_[lane]; + // NVFP4 UE4M3 scales (bit 7 is padding): 0x7F is NaN. MXFP4 E8M0 + // scales join the lane exponent; 0xFF is NaN. + uint32_t sf_man_prod = 1; + int sf_exp = 0; // into the 6-bit term exponent (NVFP4) + int sf_lane_exp = 0; // into the lane exponent (MXFP4) + if (e4m3_scale_) { + sf_man_prod = (8u | (sfa & 7)) * (8u | (sfb & 7)); + sf_exp = int((sfa >> 3) & 15) + int((sfb >> 3) & 15); + if ((sfa & 0x7f) == 0x7f || (sfb & 0x7f) == 0x7f) { + flags |= FL_NAN; + } + } else { + sf_lane_exp = int(sfa) + int(sfb) - 254; + if (sfa == 0xff || sfb == 0xff) { + flags |= FL_NAN; + } + } + uint32_t texp[4], tprod[4], tsign[4]; + bool tvalid[4]; + uint32_t max_exp = 0; + for (int j = 0; j < 4; ++j) { + uint32_t ra = (a[word] >> (off + 4 * j)) & 0xf; + uint32_t rb = (b[word] >> (off + 4 * j)) & 0xf; + tvalid[j] = (ra & 7) != 0 && (rb & 7) != 0; + tsign[j] = ((ra ^ rb) >> 3) & 1; + uint32_t prod = f4_man(ra) * f4_man(rb); + tprod[j] = e4m3_scale_ ? ((prod * sf_man_prod) & 0x7ff) : prod; + texp[j] = tvalid[j] ? (uint32_t(exp_adj + sf_exp + int(f4_exp(ra)) + int(f4_exp(rb))) & 63) : 0; + max_exp = std::max(max_exp, texp[j]); + } + uint32_t sum = 0; // 27-bit two's complement + for (int j = 0; j < 4; ++j) { + uint32_t shift = (max_exp - texp[j]) & 63; + uint32_t mag = tvalid[j] ? ((tprod[j] << SIG_SHIFT) & 0xffffff) : 0; + uint32_t aligned = (shift >= 24) ? 0 : (mag >> shift); + sum = (sum + (tsign[j] ? (0u - aligned) : aligned)) & 0x7ffffff; + } + uint32_t sum_sign = (sum >> 26) & 1; + uint32_t abs_sum = (sum_sign ? (0u - sum) : sum) & 0x3ffffff; + if (abs_sum == 0) { + out.push_back({0, 0}); + eps.push_back(kZeroE); + continue; + } + const int lane_exp = (int(max_exp) + bias_base + exp_comp + sf_lane_exp) & 1023; + out.push_back({int(sum_sign), int(abs_sum & 0xffffff)}); + eps.push_back(lane_exp - (WA - 1)); + } + return add_c_term(std::move(out), std::move(eps), c_term, flags); + } + + // One fp8 lane: its two products, aligned to the one the datapath picks as + // larger. A zero product still competes by its exponent fields when the + // other is nonzero and listed second, as the datapath compares them. + std::pair reduce_f8(const pterm_t& t1, int e1, bool v1, const pterm_t& t2, int e2, bool v2) { // Pick max exponent term int max_E, max_s, min_E, min_s; int64_t max_P, min_P; - if (e1 >= e2) { + if (!v1 && !v2) { + return {{0, 0}, kZeroE}; + } + if ((v1 && !v2) || (v2 && e1 >= e2)) { max_E = e1; max_P = t1.Mp; max_s = t1.sign; min_E = e2; min_P = t2.Mp; min_s = t2.sign; } else { @@ -435,7 +577,11 @@ class FEDP { int64_t val = val1 + val2; if (val == 0) { - return {{0, 0}, 0}; + // Only an exact cancellation of two equal-exponent products clears the + // lane exponent; a product aligned out of the lane keeps it, so the + // empty lane still sets the largest exponent. + bool cancel = (max_s != min_s) && (e1 == e2) && (t1.Mp == t2.Mp); + return {{0, 0}, cancel ? kZeroE : (max_E + 1)}; } int s = (val < 0) ? 1 : 0; @@ -446,7 +592,7 @@ class FEDP { P >>= 1; E += 1; - LOG("[reduce_f8 (%ld)] t1=(%d, %d, %d), t2=(%d, %d, %d) | diff=%d max_P=0x%lx min_P=0x%lx aligned=0x%lx val=0x%lx -> s=%d P=0x%lx E=%d\n", + FEDP_LOG("[reduce_f8 (%ld)] t1=(%d, %d, %d), t2=(%d, %d, %d) | diff=%d max_P=0x%lx min_P=0x%lx aligned=0x%lx val=0x%lx -> s=%d P=0x%lx E=%d\n", req_id_, t1.sign, t1.Mp, e1, t2.sign, t2.Mp, e2, diff, max_P, min_P, aligned, val, s, P, E); return {{s, (int)P}, E}; @@ -569,6 +715,8 @@ class FEDP { uint32_t kept_rounded = r.kept + (round_up ? 1 : 0); int be = r.e; if (kept_rounded & (1u << 24)) { kept_rounded >>= 1; be += 1; } + // the datapath's result exponent is a signed 10-bit quantity + be = ((be & 1023) ^ 512) - 512; if (be >= 255) return s_bit | 0x7f800000u; if (be <= 0) return s_bit; return s_bit | ((be & 0xff) << 23) | (kept_rounded & 0x7FFFFFu); @@ -577,5 +725,8 @@ class FEDP { // members -------- int exp_bits_, sig_bits_, frm_, W_, HR_; bool renorm_, no_window_; + const uint8_t *sf_a_ = nullptr; // per-lane block scales, null when unscaled + const uint8_t *sf_b_ = nullptr; + bool e4m3_scale_ = false; uint64_t req_id_ = 0; }; \ No newline at end of file diff --git a/sw/common/tensor_cfg.h b/sw/common/tensor_cfg.h index 6ca343bb12..e980bc5c77 100644 --- a/sw/common/tensor_cfg.h +++ b/sw/common/tensor_cfg.h @@ -156,6 +156,12 @@ inline const char* fmt_string(uint32_t fmt) { } } +// The TCU accumulates and returns fp32 (int32 for integer inputs); it has no +// other output format. +inline constexpr bool tcu_output_valid(uint32_t it, uint32_t ot) { + return (it >= int32::id) ? (ot == int32::id) : (ot == fp32::id); +} + inline constexpr bool mx_scale_format(uint32_t fmt) { switch (fmt) { case mxfp8::id: diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index 618a95b030..277b6ecc8b 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -173,6 +173,9 @@ struct wmma_context { public: + static_assert(tcu_output_valid(It::id, Ot::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + using input_t = typename It::dtype; using output_t = typename Ot::dtype; @@ -675,6 +678,9 @@ template struct wgmma_context { + static_assert(tcu_output_valid(It::id, Ot::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + private: static constexpr uint32_t NRA = 4; diff --git a/tests/regression/sgemm2_tcu/main.cpp b/tests/regression/sgemm2_tcu/main.cpp index 400cc84793..a38b6c40f4 100644 --- a/tests/regression/sgemm2_tcu/main.cpp +++ b/tests/regression/sgemm2_tcu/main.cpp @@ -255,17 +255,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -275,17 +264,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -330,6 +308,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { for (uint32_t m = 0; m < M; ++m) { for (uint32_t n = 0; n < N; ++n) { @@ -346,8 +327,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < K; ++k) { auto a = data_accessor_t::read(A, m * K + k); diff --git a/tests/regression/sgemm_tcu/main.cpp b/tests/regression/sgemm_tcu/main.cpp index 1815228882..0c9446a561 100644 --- a/tests/regression/sgemm_tcu/main.cpp +++ b/tests/regression/sgemm_tcu/main.cpp @@ -368,17 +368,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -388,17 +377,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -408,17 +386,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e4m3tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e4m3tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e4m3tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe4m3_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -428,17 +395,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e5m2tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e5m2tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e5m2tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe5m2_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint32_t a, uint32_t b, float c) { @@ -448,17 +404,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static int32_t eval(uint8_t a, uint8_t b, int32_t c) { @@ -495,6 +440,9 @@ inline typename T::dtype generate_B_value() { /////////////////////////////////////////////////////////////////////////////// +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; @@ -507,10 +455,10 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t for (uint32_t m = 0; m < M; ++m) { for (uint32_t n = 0; n < N; ++n) { if constexpr (std::is_same::value) { - // fp32 output: the tensor core accumulates the K products in a wide - // accumulator and rounds to fp32 once; a per-step-rounded reference - // drifts by several ULP over K. Each product is exact in fp32, so a - // double accumulation reproduces the single-rounding dot product. + // The tensor core accumulates the K products in a wide accumulator + // and rounds to fp32 once; a per-step-rounded reference drifts by + // several ULP over K. Each product is exact in fp32, so a double + // accumulation reproduces the single-rounding dot product. double acc = 0.0; for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); @@ -519,8 +467,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); diff --git a/tests/regression/sgemm_tcu_mx/main.cpp b/tests/regression/sgemm_tcu_mx/main.cpp index a0cff63253..f3d10d3b77 100644 --- a/tests/regression/sgemm_tcu_mx/main.cpp +++ b/tests/regression/sgemm_tcu_mx/main.cpp @@ -33,6 +33,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; static constexpr uint32_t kScalePack = 4; diff --git a/tests/regression/sgemm_tcu_sp/main.cpp b/tests/regression/sgemm_tcu_sp/main.cpp index c3237eea4b..dd357fe7a1 100644 --- a/tests/regression/sgemm_tcu_sp/main.cpp +++ b/tests/regression/sgemm_tcu_sp/main.cpp @@ -361,17 +361,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -381,17 +370,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -401,17 +379,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e4m3tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e4m3tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e4m3tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe4m3_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -421,17 +388,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e5m2tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e5m2tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e5m2tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe5m2_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint32_t a, uint32_t b, float c) { @@ -441,17 +397,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static int32_t eval(uint8_t a, uint8_t b, int32_t c) { @@ -493,6 +438,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + // Dense CPU reference matmul (pruned A has zeros at masked positions) static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { @@ -513,8 +461,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); diff --git a/tests/regression/sgemm_tcu_sp_mx/main.cpp b/tests/regression/sgemm_tcu_sp_mx/main.cpp index 8dabef023d..b7b4e9048c 100644 --- a/tests/regression/sgemm_tcu_sp_mx/main.cpp +++ b/tests/regression/sgemm_tcu_sp_mx/main.cpp @@ -34,6 +34,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; static constexpr uint32_t kScalePack = 4; diff --git a/tests/regression/sgemm_tcu_wg/main.cpp b/tests/regression/sgemm_tcu_wg/main.cpp index 18b07262a1..2085b13249 100644 --- a/tests/regression/sgemm_tcu_wg/main.cpp +++ b/tests/regression/sgemm_tcu_wg/main.cpp @@ -366,17 +366,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -386,17 +375,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -406,17 +384,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e4m3tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e4m3tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e4m3tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe4m3_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -426,17 +393,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e5m2tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e5m2tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e5m2tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe5m2_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint32_t a, uint32_t b, float c) { @@ -446,17 +402,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static int32_t eval(uint8_t a, uint8_t b, int32_t c) { @@ -501,6 +446,9 @@ static constexpr uint32_t per_warp_N = wg_cfg::xtileN; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { uint32_t subbytes = 8 / vt::ITYPE::bits; @@ -520,8 +468,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); diff --git a/tests/regression/sgemm_tcu_wg_dxa/main.cpp b/tests/regression/sgemm_tcu_wg_dxa/main.cpp index a651a7bda6..f9175cdf33 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/main.cpp @@ -174,17 +174,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - /////////////////////////////////////////////////////////////////////////////// using cfg = vt::wgmma_config_t; @@ -192,6 +181,9 @@ using cfg = vt::wgmma_config_t(acc); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < K; ++k) { sum = muladd_t::eval(A[m * K + k], B[k * N + n], sum); diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp index f32fbd3da4..aca8b0962f 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp @@ -174,17 +174,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - /////////////////////////////////////////////////////////////////////////////// using cfg = vt::wgmma_config_t; @@ -192,6 +181,9 @@ using cfg = vt::wgmma_config_t(acc); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < K; ++k) { sum = muladd_t::eval(A[m * K + k], B[k * N + n], sum); diff --git a/tests/regression/sgemm_tcu_wg_mx/main.cpp b/tests/regression/sgemm_tcu_wg_mx/main.cpp index f149b1dffc..ac8b594628 100644 --- a/tests/regression/sgemm_tcu_wg_mx/main.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/main.cpp @@ -35,6 +35,9 @@ static constexpr uint32_t kTileN = cfg::xtileN; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; static constexpr uint32_t kScalePack = 4; From 8206c7248dbdaf5e313c4cf4abc2fbd159398d89 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:55 -0700 Subject: [PATCH 108/336] ci: check fence_mp's release fence and .rl atomic behind a slow bank 0 fence_mp-5 (-m1, release/acquire fences) and fence_mp-6 (-m2, .rl/.aq atomics) run two cores with an L2 under DBG_MEM_BANK0_WDELAY, on simx and rtlsim. Bank 0's writes are held back, and the flag, in a bank-1 line, overtakes the data in bank-0 lines unless the release waits for them. Without the delay, no shape would fail a release that does not drain. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index d2d6c695d6..c9693b26a6 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -288,6 +288,31 @@ tests: cores: 2 l2cache: true l3cache: true +# A release must drain: bank 0's writes are held back by the debug delay, and +# the flag, in a bank-1 line, overtakes the data in bank-0 lines unless the +# release fence (-m1) or the .rl atomic (-m2) waits for them. +- id: fence_mp-5 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp + args: -n64 -m1 + configs: -DVX_CFG_EXT_A_ENABLE -DDBG_MEM_BANK0_WDELAY + shape: + cores: 2 + l2cache: true +- id: fence_mp-6 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp + args: -n64 -m2 + configs: -DVX_CFG_EXT_A_ENABLE -DDBG_MEM_BANK0_WDELAY + shape: + cores: 2 + l2cache: true # Kernel to kernel with no host transfer between (one command batch): the # launch flushes are L1-only, so the last-level cache alone carries each # round's writes to the next round, and the full flush runs before readback. From c9b78161c406f367ecf42ea1eb2212c390b08e17 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:56 -0700 Subject: [PATCH 109/336] cache: merge same-line icache fetches from a core's fetch slices With FETCH_WIDTH > 1 each fetch slice has its own icache port. Slices of one core often fetch different words of the same line in the same cycle (sgemm's three-line loop does almost every cycle), and each took its own bank access, serialized behind the others. READ_MERGE_LINE extends read merging from same-word to same-line reads. Plain reads of one line from a merge group in the same cycle share one bank access. The request carries every group port's {tag, word select} above the merged-port mask, the bank returns one word per slot, and the response crossbar hands each port its own word and tag. Only read-only caches may use it (static assert). The icache enables it through VX_CFG_ICACHE_READ_MERGE, default 1, which changes nothing at the default FETCH_WIDTH of 1. SimX merges by line in the port arbiter. The merged ports' tags ride the leader's request through the bank and back on its response, as in the RTL tag, and the response multicast hands each port its own. Under line merging the bank reads the whole line: decoding every slot's word select would sit on the data array's enable path. CI: config1:icache_merge-1 (sgemm, four fetch slices) and icache_merge-2 (raycast, four fetch slices on 16 icache banks), on simx and rtlsim. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 3 ++ ci/testcases/config1.yaml | 18 +++++++++ hw/rtl/VX_socket.sv | 3 ++ hw/rtl/cache/VX_cache.sv | 68 ++++++++++++++++++++++++++------ hw/rtl/cache/VX_cache_bank.sv | 47 ++++++++++++++++++++-- hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_data.sv | 6 ++- hw/rtl/cache/VX_cache_wrap.sv | 2 + sim/simx/mem/cache.cpp | 54 +++++++++++++++++++++---- sim/simx/mem/cache.h | 1 + sim/simx/socket.cpp | 4 +- sim/simx/types.h | 5 +++ 12 files changed, 186 insertions(+), 27 deletions(-) diff --git a/VX_config.toml b/VX_config.toml index ab8a07a5dc..3abeb0d774 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -218,6 +218,9 @@ VX_CFG_ICACHE_CRSQ_SIZE = 0 VX_CFG_ICACHE_MEM_PORTS = 1 # banks behind the fetch-slice request ports VX_CFG_ICACHE_NUM_BANKS = "expr: pow(2, clog2($VX_CFG_FETCH_WIDTH))" +# Merge same-line fetches from the core's slices in the same cycle into one +# bank access; the response goes to every merged slice, each with its word. +VX_CFG_ICACHE_READ_MERGE = 1 VX_CFG_ICACHE_LATENCY = "expr: 2 + max(0, clog2($VX_CFG_ICACHE_SIZE) - clog2(16384))" VX_CFG_DCACHE_SIZE = 16384 diff --git a/ci/testcases/config1.yaml b/ci/testcases/config1.yaml index f8ef17bbcb..a5d8bdcf45 100644 --- a/ci/testcases/config1.yaml +++ b/ci/testcases/config1.yaml @@ -401,3 +401,21 @@ tests: - simx app: regression/vecadd configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_LSU_BLOCKS=4 -DVX_CFG_NUM_LSU_LANES=4 +# Fetch slices reading one line in the same cycle share one icache access, and +# each slice must get its own word: sgemm's three-line loop merges almost every +# cycle. The second case puts the merged responses behind 16 banks. +- id: icache_merge-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + args: -n64 + configs: -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_THREADS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 +- id: icache_merge-2 + via: blackbox + drivers: + - simx + - rtlsim + app: raycast + configs: -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_THREADS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_ICACHE_NUM_BANKS=16 diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index a8fc37e113..0552c7e28e 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -171,6 +171,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NC_ENABLE (0), // read-only: no dirty data, and the dcache's flush walks the shared levels .FLUSH_FORWARD (0), + // a core's fetch slices read different words of the same lines + .READ_MERGE (`VX_CFG_ICACHE_READ_MERGE ? ICACHE_NUM_REQS : 0), + .READ_MERGE_LINE (1), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) icache ( diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 5184a6f56d..4b5b17c86e 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -61,6 +61,7 @@ module VX_cache import VX_gpu_pkg::*; #( parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) parameter READ_MERGE = 0, // merge same-word reads within groups of this many adjacent ports (0: off) + parameter READ_MERGE_LINE = 0, // merge any reads of the same line instead, each port keeping its own tag and word parameter AMO_ENABLE = 0 // synthesize atomic-op logic in the banks ) ( @@ -95,7 +96,13 @@ module VX_cache import VX_gpu_pkg::*; #( // leader's group) below the request tag through the bank. localparam MERGE_GROUP = (READ_MERGE != 0) ? `MIN(READ_MERGE, NUM_REQS) : 1; localparam DO_READ_MERGE = (MERGE_GROUP > 1); - localparam BANK_TAG_WIDTH = TAG_WIDTH + (DO_READ_MERGE ? MERGE_GROUP : 0); + // Line merging also carries every group port's {tag, word select}, one + // slot per port above the mask, and the bank answers with one word per slot. + localparam MERGE_LINE = DO_READ_MERGE && (READ_MERGE_LINE != 0); + localparam SLOT_WIDTH = TAG_WIDTH + WORD_SEL_WIDTH; + localparam SLOTS_WIDTH = MERGE_LINE ? (MERGE_GROUP * SLOT_WIDTH) : 0; + localparam RSP_WORDS = MERGE_LINE ? MERGE_GROUP : 1; + localparam BANK_TAG_WIDTH = TAG_WIDTH + SLOTS_WIDTH + (DO_READ_MERGE ? MERGE_GROUP : 0); localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + BANK_TAG_WIDTH + `UP(MEM_ATTR_WIDTH); localparam CORE_RSP_DATAW = WORD_WIDTH + TAG_WIDTH; localparam BANK_MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH; @@ -288,7 +295,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][`UP(MEM_ATTR_WIDTH)-1:0] per_bank_core_req_attr; wire [NUM_BANKS-1:0] per_bank_core_req_ready; wire [NUM_BANKS-1:0] per_bank_core_rsp_valid; - wire [NUM_BANKS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_rsp_data; + wire [NUM_BANKS-1:0][RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_rsp_data; wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_core_rsp_tag; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_core_rsp_idx; wire [NUM_BANKS-1:0] per_bank_core_rsp_ready; @@ -345,11 +352,11 @@ module VX_cache import VX_gpu_pkg::*; #( assign core_req_bid[i] = '0; end end - // Read merging: plain reads of the same word with the same tag from ports - // of one group in the same cycle are served by one bank access. The lowest - // such port leads; the others stay out of the request crossbar and are - // accepted in the cycle their leader is, so every port keeps its own - // request order into the bank. + // Read merging: plain reads of the same word with the same tag (or, under + // line merging, of the same line) from ports of one group in the same cycle + // are served by one bank access. The lowest such port leads; the others + // stay out of the request crossbar and are accepted in the cycle their + // leader is, so every port keeps its own request order into the bank. wire [NUM_REQS-1:0] core_req_follower; wire [NUM_REQS-1:0] core_req_xbar_ready; wire [NUM_REQS-1:0][BANK_TAG_WIDTH-1:0] core_req_bank_tag; @@ -369,9 +376,15 @@ module VX_cache import VX_gpu_pkg::*; #( for (genvar i = 0; i < NUM_REQS; ++i) begin : g_same_i for (genvar j = 0; j < NUM_REQS; ++j) begin : g_same_j if (j < i && (j / MERGE_GROUP) == (i / MERGE_GROUP)) begin : g_cmp - assign same_word[i][j] = plain_read[i] && plain_read[j] - && (core_req_addr[i] == core_req_addr[j]) - && (core_req_tag[i] == core_req_tag[j]); + if (MERGE_LINE) begin : g_line + assign same_word[i][j] = plain_read[i] && plain_read[j] + && (core_req_line_addr[i] == core_req_line_addr[j]) + && (core_req_bid[i] == core_req_bid[j]); + end else begin : g_word + assign same_word[i][j] = plain_read[i] && plain_read[j] + && (core_req_addr[i] == core_req_addr[j]) + && (core_req_tag[i] == core_req_tag[j]); + end end else begin : g_none assign same_word[i][j] = 1'b0; end @@ -387,8 +400,18 @@ module VX_cache import VX_gpu_pkg::*; #( for (genvar j = 0; j < NUM_REQS; ++j) begin : g_leader_fire assign leader_fire[j] = same_word[i][j] && ~core_req_follower[j] && core_req_xbar_ready[j]; end + wire [`UP(SLOTS_WIDTH)-1:0] slots; + if (MERGE_LINE) begin : g_slots + for (genvar k = 0; k < MERGE_GROUP; ++k) begin : g_slot + assign slots[k * SLOT_WIDTH +: SLOT_WIDTH] = {core_req_tag[GROUP_BASE + k], core_req_wsel[GROUP_BASE + k]}; + end + assign core_req_bank_tag[i] = {core_req_tag[i], slots, merged_mask}; + end else begin : g_no_slots + assign slots = '0; + `UNUSED_VAR (slots) + assign core_req_bank_tag[i] = {core_req_tag[i], merged_mask}; + end assign core_req_follower[i] = (| same_word[i]); - assign core_req_bank_tag[i] = {core_req_tag[i], merged_mask}; assign core_req_ready[i] = core_req_follower[i] ? (| leader_fire) : core_req_xbar_ready[i]; end end else begin : g_no_read_merge @@ -473,6 +496,9 @@ module VX_cache import VX_gpu_pkg::*; #( .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), .TAG_WIDTH (BANK_TAG_WIDTH), + .RSP_WORDS (RSP_WORDS), + .RSP_WSEL_OFFS (DO_READ_MERGE ? MERGE_GROUP : 0), + .RSP_WSEL_STRIDE (SLOT_WIDTH), .CORE_OUT_BUF (CORE_RSP_BUF_ENABLE ? 3 : 0), .MEM_OUT_BUF (MEM_REQ_BUF_ENABLE ? 3 : 0), .IS_LLC (IS_LLC), @@ -524,7 +550,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] core_rsp_ready; wire [NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] per_bank_core_rsp_pdata; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_core_rsp_pdata - assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i], per_bank_core_rsp_tag[i][BANK_TAG_WIDTH-1 -: TAG_WIDTH]}; + assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i][0], per_bank_core_rsp_tag[i][BANK_TAG_WIDTH-1 -: TAG_WIDTH]}; end // A flush token's answer takes bank 0's lane into the response crossbar // when the bank has no response of its own, so it leaves through the same @@ -557,6 +583,22 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_taken; wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_valid; wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_ready; + wire [NUM_REQS-1:0][NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] per_output_pdata; + // Under line merging each port takes its own slot's word and tag; the + // leader keeps the lane's tag, which a flush answer also uses. + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_output_pdata + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_lane + if (MERGE_LINE) begin : g_slot + localparam K = j % MERGE_GROUP; + wire [TAG_WIDTH-1:0] lane_tag = core_rsp_xbar_pdata[i][TAG_WIDTH-1:0]; + wire [TAG_WIDTH-1:0] slot_tag = per_bank_core_rsp_tag[i][MERGE_GROUP + K * SLOT_WIDTH + WORD_SEL_WIDTH +: TAG_WIDTH]; + wire is_leader = (core_rsp_xbar_idx[i] == REQ_SEL_WIDTH'(j)); + assign per_output_pdata[j][i] = {per_bank_core_rsp_data[i][K], (is_leader ? lane_tag : slot_tag)}; + end else begin : g_word + assign per_output_pdata[j][i] = core_rsp_xbar_pdata[i]; + end + end + end for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_lanes wire [MERGE_GROUP-1:0] merged_mask; if (i == 0) begin : g_flush_mask @@ -593,7 +635,7 @@ module VX_cache import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .valid_in (per_output_valid[j]), - .data_in (core_rsp_xbar_pdata), + .data_in (per_output_pdata[j]), .ready_in (per_output_ready[j]), .valid_out (core_rsp_valid[j]), .data_out (core_rsp_pdata[j]), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index d71663087e..893809a06c 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -41,6 +41,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // last-level cache: AMOs commit locally here parameter AMO_ENABLE = 0, // synthesize atomic-op logic parameter LATENCY = 2, // pipeline depth; >2 defers the data array to break the tag->data path + // Words per core response. Above 1, response word k is the line's word + // whose select rides in the request tag at RSP_WSEL_OFFS + k * RSP_WSEL_STRIDE. + parameter RSP_WORDS = 1, + parameter RSP_WSEL_OFFS = 0, + parameter RSP_WSEL_STRIDE = 0, parameter MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE), parameter MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH, parameter REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS), @@ -70,7 +75,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Core response output wire core_rsp_valid, - output wire [`CS_WORD_WIDTH-1:0] core_rsp_data, + output wire [RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] core_rsp_data, output wire [TAG_WIDTH-1:0] core_rsp_tag, output wire [REQ_SEL_WIDTH-1:0] core_rsp_idx, input wire core_rsp_ready, @@ -108,6 +113,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( `STATIC_ASSERT(LATENCY >= 2, ("invalid parameter: cache bank LATENCY must be >= 2")) `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) `STATIC_ASSERT(!WRITE_AROUND || (WRITEBACK && WRITE_ENABLE), ("invalid parameter: write-around requires a write-back cache")) + `STATIC_ASSERT(RSP_WORDS == 1 || (!WRITE_ENABLE && !AMO_ENABLE), ("invalid parameter: multi-word responses are read-only")) `UNUSED_PARAM (MRSQ_SIZE) // ======================================================================== @@ -909,6 +915,16 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mask_clr_line_stc = (mask_fill_stc && ~stC.lk.is_refill) || (stC.req.valid && stC.req.is_flush); wire mask_clr_sec_stc = mask_fill_stc && stC.lk.is_refill; + // The data slices a read returns: its word's, or under multi-word + // responses the whole line, since a merged read may take any word (and + // decoding the slots' selects here would sit on the array's enable path). + wire [`CS_WORDS_PER_LINE-1:0] read_words_std; + if (RSP_WORDS > 1) begin : g_read_words + assign read_words_std = '1; + end else begin : g_read_word + assign read_words_std = `CS_WORDS_PER_LINE'(1) << stD.req.word_idx; + end + VX_cache_data #( .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -944,6 +960,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .fill_data (fbuf_line), .write_word (stD.wdata), .word_idx (stD.req.word_idx), + .read_words (read_words_std), .sector_idx (sector_idx_std), .write_byteen (stD.req.byteen), .read_data (read_data_stc), @@ -1311,11 +1328,33 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign fwd_word = fbuf_data_r[`CS_WORD_WIDTH-1:0]; end + // A multi-word response takes each word its tag selects, from the line at + // commit or from the staged sector on a forward. + wire [RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] crsp_words, fwd_words; + if (RSP_WORDS > 1) begin : g_rsp_words + for (genvar k = 0; k < RSP_WORDS; ++k) begin : g_word + wire [WORD_SEL_WIDTH-1:0] wsel_stc = stC.req.tag[RSP_WSEL_OFFS + k * RSP_WSEL_STRIDE +: WORD_SEL_WIDTH]; + wire [WORD_SEL_WIDTH-1:0] wsel_fwd = replay_tag[RSP_WSEL_OFFS + k * RSP_WSEL_STRIDE +: WORD_SEL_WIDTH]; + assign crsp_words[k] = read_data_stc[wsel_stc]; + if (`CS_WORDS_PER_SECTOR > 1) begin : g_fwd + wire [`CLOG2(`CS_WORDS_PER_SECTOR)-1:0] fwd_k = wsel_fwd[`CLOG2(`CS_WORDS_PER_SECTOR)-1:0]; + assign fwd_words[k] = fbuf_data_r[fwd_k * `CS_WORD_WIDTH +: `CS_WORD_WIDTH]; + end else begin : g_fwd_1 + assign fwd_words[k] = fbuf_data_r[`CS_WORD_WIDTH-1:0]; + `UNUSED_VAR (wsel_fwd) + end + end + `UNUSED_VAR ({crsp_queue_data, fwd_word}) + end else begin : g_rsp_word + assign crsp_words = crsp_queue_data; + assign fwd_words = fwd_word; + end + `RUNTIME_ASSERT (~fwd_fire || (replay_addr == fbuf_addr_r), ("%t: %s fill-forward address mismatch: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(replay_addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fbuf_addr_r, BANK_ID))) `RUNTIME_ASSERT (~(flush_fire && fwd_pending), ("%t: %s flush during fill-forward drain", $time, INSTANCE_ID)) VX_elastic_buffer #( - .DATAW (TAG_WIDTH + `CS_WORD_WIDTH + REQ_SEL_WIDTH), + .DATAW (TAG_WIDTH + RSP_WORDS * `CS_WORD_WIDTH + REQ_SEL_WIDTH), .SIZE (CRSQ_QUEUE_SIZE), .OUT_REG (`TO_OUT_BUF_REG(CORE_OUT_BUF)) ) core_rsp_queue ( @@ -1323,8 +1362,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .reset (reset), .valid_in (crsp_queue_valid || fwd_head), .ready_in (crsp_queue_ready), - .data_in (crsp_queue_valid ? {stC.req.tag, crsp_queue_data, stC.req.req_idx} - : {replay_tag, fwd_word, replay_idx}), + .data_in (crsp_queue_valid ? {stC.req.tag, crsp_words, stC.req.req_idx} + : {replay_tag, fwd_words, replay_idx}), .data_out ({core_rsp_tag, core_rsp_data, core_rsp_idx}), .valid_out (core_rsp_valid), .ready_out (core_rsp_ready) diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index c7553e07e9..6065ef0f5e 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -63,6 +63,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). + parameter READ_MERGE_LINE = 0, // Merge any reads of the same line instead, each port keeping its own tag and word. parameter AMO_ENABLE = 0 // Enable atomic memory operations. ) ( @@ -189,6 +190,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .READ_MERGE (READ_MERGE), + .READ_MERGE_LINE (READ_MERGE_LINE), .AMO_ENABLE (AMO_ENABLE) ) cache_wrap ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index bc5bf12385..16d79555b3 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -71,6 +71,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( input wire [`CS_WORD_WIDTH-1:0] write_word, input wire [WORD_SIZE-1:0] write_byteen, input wire [`UP(`CS_WORD_SEL_BITS)-1:0] word_idx, + input wire [`CS_WORDS_PER_LINE-1:0] read_words, // slices a read returns (word_idx's, plus a merged read's) input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx, // sector being filled // outputs output wire [`CS_LINE_WIDTH-1:0] read_data, @@ -180,8 +181,8 @@ module VX_cache_data import VX_gpu_pkg::*; #( wire fill_sec_en = (`CS_SECTORS_PER_LINE == 1) || (sector_idx == `UP(`CS_SECTOR_SEL_BITS)'(SLICE_SECTOR)); - // load reads the selected slice; writeback/flush reads all slices. - wire slice_read = (read && word_en) || ((fill || flush) && WRITEBACK); + // a load reads its words' slices; writeback/flush reads all slices. + wire slice_read = (read && read_words[s]) || ((fill || flush) && WRITEBACK); wire slice_write; wire [WRENW-1:0] slice_wren; @@ -202,6 +203,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( `UNUSED_VAR (write_byteen) `UNUSED_VAR (hit_any) `UNUSED_VAR ({fill_write, merge_write}) + `UNUSED_VAR (word_en) assign slice_write = fill && fill_sec_en; assign slice_wren = 1'b1; assign slice_wdata = fill_data[s]; diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 597e63a042..f71dbea3b2 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -65,6 +65,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). + parameter READ_MERGE_LINE = 0, // Merge any reads of the same line instead, each port keeping its own tag and word. parameter AMO_ENABLE = 0 // Enable atomic operations in cache banks. ) ( @@ -170,6 +171,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .READ_MERGE (READ_MERGE), + .READ_MERGE_LINE (READ_MERGE_LINE), .AMO_ENABLE (AMO_ENABLE) ) cache ( .clk (clk), diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index abd71bf8af..6f09d976f1 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -423,6 +423,7 @@ struct bank_req_t { uint64_t addr; uint32_t hart_id; uint64_t req_tag; + std::shared_ptr> merged_tags; // ride to the response uint64_t uuid; uint32_t mshr_id; ReqType type; @@ -474,6 +475,7 @@ struct bank_req_t { addr = 0; hart_id = 0; req_tag = 0; + merged_tags.reset(); uuid = 0; mshr_id = 0; type = ReqType::None; @@ -1290,6 +1292,7 @@ class CacheBank : public SimObject { coalesced.hart_id = core_req.hart_id; coalesced.uuid = core_req.uuid; coalesced.req_tag = core_req.tag; + coalesced.merged_tags = core_req.merged_tags; coalesced.write = (core_req.op == MemOp::ST); coalesced.op = core_req.op; coalesced.data = core_req.data; @@ -1328,6 +1331,7 @@ class CacheBank : public SimObject { bank_req.hart_id = core_req.hart_id; bank_req.uuid = core_req.uuid; bank_req.req_tag = core_req.tag; + bank_req.merged_tags = core_req.merged_tags; // bank_req.write means "route through the cache write path" — true only // for plain stores. AMOs (op == AMO_SC / AMO RMW) are write-bearing // semantically (memop_is_write returns true) but use the dedicated AMO @@ -1537,6 +1541,7 @@ class CacheBank : public SimObject { mshr_.dequeue(&req); MemRsp rsp{req.req_tag, req.hart_id, req.uuid}; rsp.data = fwd_data_; + rsp.merged_tags = req.merged_tags; this->core_rsp_out.send(rsp); DT(3, this->name() << " fwd-rsp: " << rsp); return true; @@ -1934,6 +1939,7 @@ class CacheBank : public SimObject { MemRsp rsp{bank_req.req_tag, bank_req.hart_id, bank_req.uuid}; if (!bank_req.write) rsp.data = hit_sec.data; + rsp.merged_tags = bank_req.merged_tags; this->core_rsp_out.send(rsp); crsp_sent_ = true; DT(3, this->name() << " replay-rsp: " << rsp); @@ -2060,6 +2066,7 @@ class CacheBank : public SimObject { MemRsp rsp{bank_req.req_tag, bank_req.hart_id, bank_req.uuid}; if (!bank_req.write) rsp.data = hit_sec.data; + rsp.merged_tags = bank_req.merged_tags; this->core_rsp_out.send(rsp); crsp_sent_ = true; DT(3, this->name() << " core-rsp: " << rsp); @@ -2391,7 +2398,9 @@ class CacheBank : public SimObject { // With read merging, plain reads of the same word with the same tag from ports // of one group in the same cycle are served by one bank access: the lowest // such port leads, the others stay out of arbitration and are accepted with -// it. The merged ports' mask rides below the tag through the bank. +// it. The merged ports' mask rides below the tag through the bank. Line +// merging takes any plain read of the same line instead, whatever its word +// and tag. class CachePortArb : public SimObject { public: std::vector> ReqIn; // core ports @@ -2399,7 +2408,8 @@ class CachePortArb : public SimObject { std::vector> IoOut; // non-cacheable requests, per port CachePortArb(const SimContext& ctx, const char* name, const params_t& params, - uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group) + uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group, + bool merge_line) : SimObject(ctx, name) , ReqIn(num_inputs, this) , ReqOut(num_banks, this) @@ -2410,6 +2420,7 @@ class CachePortArb : public SimObject { , delay_((num_inputs > 2) ? 1 : 0) , levels_(arb_tree_levels(num_inputs)) , merge_group_(merge_group) + , merge_line_(merge_line) { __assert(merge_group_ <= 1 || (merge_group_ <= 64 && ispow2(merge_group_) && (num_inputs % merge_group_) == 0), "a read-merge group must be a power of two of at most 64 dividing the ports"); @@ -2595,8 +2606,11 @@ class CachePortArb : public SimObject { continue; } auto& req_j = ReqIn.at(j).peek(); - if ((req_j.addr >> params_.word_select_addr_start) == (req_i.addr >> params_.word_select_addr_start) - && req_j.tag == req_i.tag) { + bool same = merge_line_ + ? ((req_j.addr >> params_.bank_select_addr_start) == (req_i.addr >> params_.bank_select_addr_start)) + : ((req_j.addr >> params_.word_select_addr_start) == (req_i.addr >> params_.word_select_addr_start) + && req_j.tag == req_i.tag); + if (same) { merged.at(j) |= 1ull << (i - base); followers.at(i) = true; break; @@ -2672,6 +2686,7 @@ class CachePortArb : public SimObject { uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; __assert((tag >> 32) == 0, "cache bank tag overflow"); req.tag = uint32_t(tag); + req.merged_tags = this->merged_tags(i, mask); auto& offer = offers_.at(b); offer.valid = true; offer.port = i; @@ -2705,6 +2720,21 @@ class CachePortArb : public SimObject { bool offers_enabled() const { return levels_.empty() && delay_ == 0; } + // Under line merging, the merged ports' own tags, which ride the leader's + // bank access and its response. + std::shared_ptr> merged_tags(uint32_t i, uint64_t mask) const { + if (!merge_line_ || mask == 0) { + return nullptr; + } + uint32_t base = (i / merge_group_) * merge_group_; + auto tags = std::make_shared>(merge_group_, 0); + for (uint64_t f = mask; f != 0; f &= f - 1) { + uint32_t k = __builtin_ctzll(f); + tags->at(k) = ReqIn.at(base + k).peek().tag; + } + return tags; + } + // Send port i's request to its bank, with the ports merged into it. void grant(uint32_t i, SimChannel& out, const std::vector& merged) { auto req = ReqIn.at(i).peek(); @@ -2712,6 +2742,7 @@ class CachePortArb : public SimObject { if (merge_group_ > 1) { uint32_t base = (i / merge_group_) * merge_group_; mask = merged.at(i); + req.merged_tags = this->merged_tags(i, mask); for (uint64_t f = mask; f != 0; f &= f - 1) { uint32_t port = base + __builtin_ctzll(f); DT(3, this->name() << " read-merge: port=" << port << ", leader=" << i << ", " << req); @@ -2743,6 +2774,7 @@ class CachePortArb : public SimObject { std::vector>> tree_stage_; // [level][bank][slice] std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; + bool merge_line_; uint64_t collisions_ = 0; struct offer_t { SimChannel* bank_in = nullptr; @@ -2773,12 +2805,13 @@ class CacheRspMcast : public SimObject { std::vector> RspOut; // core ports CacheRspMcast(const SimContext& ctx, const char* name, uint32_t num_banks, - uint32_t num_outputs, uint32_t merge_group) + uint32_t num_outputs, uint32_t merge_group, bool merge_line) : SimObject(ctx, name) , RspIn(num_banks, this) , RspOut(num_outputs, this) , lg2_outputs_(log2ceil(num_outputs)) , merge_group_(merge_group) + , merge_line_(merge_line) , slices_(arb_tree_slices(num_banks)) , sent_(num_banks, 0) , last_grant_(num_outputs * slices_, num_banks - 1) @@ -2846,7 +2879,12 @@ class CacheRspMcast : public SimObject { continue; } auto rsp = RspIn.at(b).peek(); + uint32_t lane_tag = rsp.tag; rsp.tag >>= (lg2_outputs_ + merge_group_); + if (merge_line_ && o != (lane_tag & ((1u << lg2_outputs_) - 1))) { + __assert(rsp.merged_tags, "a line-merged response lost its merged tags"); + rsp.tag = rsp.merged_tags->at(o % merge_group_); + } if (out.try_send(rsp, (slices_ > 1) ? 1 : 0)) { DT(4, this->name() << " rsp" << b << "_" << o << ": " << rsp); sent_.at(b) |= 1ull << o; @@ -2867,6 +2905,7 @@ class CacheRspMcast : public SimObject { private: uint32_t lg2_outputs_; uint32_t merge_group_; + bool merge_line_; uint32_t slices_; std::vector sent_; // [bank] ports that took the lane's response std::vector last_grant_; // [port][slice] round-robin pointer @@ -3023,7 +3062,8 @@ class Cache::Impl { // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); uint32_t merge_group = std::min(config_.read_merge, config_.num_inputs); - port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group); + const bool merge_line = config_.merge_line && merge_group > 1; + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group, merge_line); for (uint32_t i = 0; i < config_.num_inputs; ++i) { simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); } @@ -3038,7 +3078,7 @@ class Cache::Impl { if (port_arb_->merge_bits() != 0) { // one buffered arbiter per core port over the bank lanes __assert(config_.core_out_buf, "read merging needs a registered core-response port"); - rsp_mcast_ = CacheRspMcast::Create(sname, num_banks, config_.num_inputs, port_arb_->merge_bits()); + rsp_mcast_ = CacheRspMcast::Create(sname, num_banks, config_.num_inputs, port_arb_->merge_bits(), merge_line); core_rsp_delay_ = 1; } else if (config_.core_out_buf) { bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel, 0, 0); diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index de87d79d01..a65ee53196 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -54,6 +54,7 @@ class Cache : public SimObject { bool mem_rsp_queue = false; // memory responses enter through a response queue bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register bool acq_forward = false; // an acquire flush also invalidates the next level + bool merge_line = false; // read merging takes any read of the same line, each merged port keeping its own tag and word }; struct PerfStats { diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 22b0f64820..b5b32226f3 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -75,11 +75,13 @@ class Socket::Impl { false, // shared_llc false, // lazy_fetch true, // core-response register - 0, // read_merge + (VX_CFG_ICACHE_READ_MERGE != 0) ? ICACHE_NUM_REQS : 0, // read_merge: the core's fetch slices 0, // mshr_lines false, // write_around VX_CFG_ICACHE_MRSQ_SIZE != 0, // memory-response queue true, // the socket's registered memory port + false, // acq_forward + true, // merge_line: slices fetch different words of a line }); snprintf(sname, 100, "%s-dcache", name.c_str()); diff --git a/sim/simx/types.h b/sim/simx/types.h index 0fb7ab61ea..b18936dbd3 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -1245,6 +1245,10 @@ struct MemReq { uint32_t hart_id; uint64_t uuid; MemFlags flags; + // Under cache line merging: the tags of the ports merged into this read, by + // offset in the leader's merge group. They ride the bank access and its + // response, as in the hardware's bank tag. + std::shared_ptr> merged_tags; MemReq(MemOp _op = MemOp::LD, uint64_t _addr = 0, @@ -1284,6 +1288,7 @@ struct MemRsp { uint32_t hart_id; uint64_t uuid; std::shared_ptr data; + std::shared_ptr> merged_tags; // see MemReq MemRsp(uint64_t _tag = 0, uint32_t _hart_id = 0, From 1babe84fbff329b803dd126a8fee8c0bb95bdeca Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:57 -0700 Subject: [PATCH 110/336] cache: keep a draining write-through chain pending, retime bank paths On a write-through cache the MSHR's pending test left out write entries, because their data had already gone to memory. But a chained store reaches the array only when it replays behind the fill. Once the chain's read head had replayed, a same-word load arriving before the store's replay found the line resident and hit, returning the value from before the thread's own store. Writes now count as pending on either policy, so the load links behind the store. SimX already chained such a hit. Several compare cones move off the bank's critical paths. Cycles do not change, and each carries a runtime assertion against the original combinational result: - VX_cache_amo registers its lane hits in every configuration and takes its coalesce keys a cycle early. - The dirty-byte mask is read one stage ahead of commit from a dual-port RAM, with write forwarding (MASK_SYNC, deferred-commit banks only), so commit takes it from a register and a large mask fits block RAM. - The MSHR takes its allocate match a cycle early against the request entering S0. The order-hazard probes drop the replay-dequeue exclusion, which the bank's link hold makes redundant, taking the S0 address compare out of the data array's write-enable cone. raw_order issues load, store, store, store, delay, load per warp and sweeps the delay across the chain's drain, then checks that every load returned its own store. CI: cache:raw_order-wt (write-through L1 under an L2, simx and rtlsim). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 15 +++ docs/designs/cache_subsystem.md | 5 +- hw/rtl/cache/VX_cache_amo.sv | 80 ++++++++----- hw/rtl/cache/VX_cache_bank.sv | 48 ++++++-- hw/rtl/cache/VX_cache_data.sv | 110 +++++++++++++----- hw/rtl/cache/VX_cache_mshr.sv | 64 +++++++++-- tests/regression/Makefile | 2 +- tests/regression/raw_order/Makefile | 16 +++ tests/regression/raw_order/common.h | 19 ++++ tests/regression/raw_order/kernel.cpp | 61 ++++++++++ tests/regression/raw_order/main.cpp | 158 ++++++++++++++++++++++++++ 11 files changed, 503 insertions(+), 75 deletions(-) create mode 100644 tests/regression/raw_order/Makefile create mode 100644 tests/regression/raw_order/common.h create mode 100644 tests/regression/raw_order/kernel.cpp create mode 100644 tests/regression/raw_order/main.cpp diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index d2e9377c61..12bd2c56b4 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -385,3 +385,18 @@ tests: shape: cores: 4 l2cache: true +# A thread's load must return its own earlier store to the same word. On the +# write-through L1 under an L2, the stores of a miss chain reach the array only +# as they replay behind the fill: a load landing after the chain's read head +# has replayed, and before the store has, must link behind the store rather +# than hit the line ahead of it. The delay sweep lands the load across that +# window. +- id: raw_order-wt + via: blackbox + drivers: + - simx + - rtlsim + app: raw_order + args: -n16384 -s64 -d2 + shape: + l2cache: true diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 80c5038263..8fc8e61c06 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -411,7 +411,10 @@ to new requests. This removes the read-after-fill round trip from miss latency. The window closes on the **first write or AMO head**. That head and every later chain entry replay through the pipeline as usual, preserving program order: an -older store must merge before a younger same-line read responds. +older store must merge before a younger same-line read responds. Stores still +waiting in the chain keep its line pending, on a write-through cache too, where +they have already been written through: a same-word access that arrives while +they drain links behind them instead of hitting the array ahead of them. Two details make it correct: diff --git a/hw/rtl/cache/VX_cache_amo.sv b/hw/rtl/cache/VX_cache_amo.sv index 63a857d330..a1d3338c8f 100644 --- a/hw/rtl/cache/VX_cache_amo.sv +++ b/hw/rtl/cache/VX_cache_amo.sv @@ -322,17 +322,13 @@ module VX_cache_amo import VX_gpu_pkg::*; #( wire [WBQ_SIZE-1:0][NUM_LANES-1:0] wbq_lane_hit; wire [NUM_LANES-1:0] post_lane_hit; - if ((PIPE_EX != 0) && (NUM_LANES > 1)) begin : g_hit_reg - // A deferred-commit bank has the next commit-stage key a cycle - // early. Where the word spans several lanes the forward merge is - // wide enough for the {line, word} compare to set the response - // path, so the lane hits are registered and the forward network - // and the response start from flops: each is taken between the - // queue's next contents and the request entering the commit stage - // next cycle, with the slot's occupancy folded in. The queue moves - // during a stall while the commit request holds, so both keys are - // compared and the stall selects. A single-lane word keeps the - // live compare, which is cheaper there. + // The lane hits are registered so the forward network and the + // response start from flops: each is taken between the queue's next + // contents and the request entering the commit stage next cycle (S0 + // itself when the commit is not deferred), with the slot's occupancy + // folded in. The queue moves during a stall while the commit request + // holds, so both keys are compared and the stall selects. + if (1) begin : g_hit_reg wire [WBQ_SIZE-1:0] wbq_key_n; for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_wbq_key_n assign wbq_key_n[i] = (wbq_addr[i] == res_addr_n) && (wbq_wsel[i] == word_idx_n); @@ -386,17 +382,6 @@ module VX_cache_amo import VX_gpu_pkg::*; #( end assign wbq_lane_hit = wbq_lane_hit_r; assign post_lane_hit = post_lane_hit_r; - end else begin : g_hit_live - for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_wbq - for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane - assign wbq_lane_hit[i][l] = (WBQ_CNTW'(i) < wbq_count) && wbq_key_st1[i] - && (wbq_lane[i] == LANE_IDXW'(l)); - end - end - for (genvar l = 0; l < NUM_LANES; ++l) begin : g_post_lane - assign post_lane_hit[l] = post_wb_valid && post_key_st1 && (post_wb_lane == LANE_IDXW'(l)); - end - `UNUSED_VAR (word_idx_n) end wire [WBQ_SIZE-1:0] wbq_word_hit; @@ -585,13 +570,54 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // independent of the drain: the highest match is the target either way // unless it is the draining head, so wb_fire, which arrives late with // the bank's stall, only picks between the two precomputed outcomes. + // The {line, word, lane} keys against the compute stage are taken a + // cycle early, when it loads from the commit stage, against each slot + // and the settling entry as they stand after this cycle's update, so + // the target select and the merge start from flops. A result pushes + // only the cycle after its load, the one cycle these keys are used. + wire [WBQ_SIZE-1:0] coal_eq_same, coal_eq_up; + for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_coal_eq + assign coal_eq_same[i] = (wbq_addr[i] == addr_st1) && (wbq_wsel[i] == word_idx_st1) + && (wbq_lane[i] == lane_st1); + if (i < WBQ_SIZE-1) begin : g_up + assign coal_eq_up[i] = (wbq_addr[i+1] == addr_st1) && (wbq_wsel[i+1] == word_idx_st1) + && (wbq_lane[i+1] == lane_st1); + end else begin : g_last + assign coal_eq_up[i] = coal_eq_same[i]; + end + end + wire coal_eq_cmp = (cmp_addr == addr_st1) && (cmp_wsel == word_idx_st1) && (cmp_lane == lane_st1); + wire coal_eq_post = (post_wb_addr == addr_st1) && (post_wb_wsel == word_idx_st1) + && (post_wb_lane == lane_st1); + reg [WBQ_SIZE-1:0] coal_key_r; + reg post_coal_key_r; + always @(posedge clk) begin + for (integer i = 0; i < WBQ_SIZE; ++i) begin + if (wb_push && (wb_slot == WBQ_IDXW'(i))) begin + coal_key_r[i] <= coal_eq_cmp; + end else if (wb_fire) begin + coal_key_r[i] <= coal_eq_up[i]; + end else begin + coal_key_r[i] <= coal_eq_same[i]; + end + end + post_coal_key_r <= wb_fire ? coal_eq_same[0] : coal_eq_post; + end + for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_coal_check + `RUNTIME_ASSERT (~wb_push || (coal_key_r[i] == ((wbq_addr[i] == cmp_addr) && (wbq_wsel[i] == cmp_wsel) + && (wbq_lane[i] == cmp_lane))), + ("%t: AMO coalesce key out of sync (slot=%0d)", $time, i)) + end + `RUNTIME_ASSERT (~wb_push || (post_coal_key_r == ((post_wb_addr == cmp_addr) && (post_wb_wsel == cmp_wsel) + && (post_wb_lane == cmp_lane))), + ("%t: AMO settling-entry coalesce key out of sync", $time)) + reg [WBQ_SIZE-1:0] wb_coal_hit; reg [WBQ_IDXW-1:0] wb_coal_idx; // pre-shift index of the coalesce target always @(*) begin wb_coal_idx = '0; for (integer i = 0; i < WBQ_SIZE; ++i) begin - wb_coal_hit[i] = (WBQ_CNTW'(i) < wbq_count) && (wbq_addr[i] == cmp_addr) - && (wbq_wsel[i] == cmp_wsel) && (wbq_lane[i] == cmp_lane); + wb_coal_hit[i] = (WBQ_CNTW'(i) < wbq_count) && coal_key_r[i]; if (wb_coal_hit[i]) begin wb_coal_idx = WBQ_IDXW'(i); end @@ -613,10 +639,8 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // than it: drop those bytes from the settling entry, so the forward // never has two writers for one byte. On a drain the settling entry // is reloaded from the head, which a same-key push never coalesces into. - wire post_push_post = wb_push && (post_wb_addr == cmp_addr) && (post_wb_wsel == cmp_wsel) - && (post_wb_lane == cmp_lane); - wire post_push_head = wb_push && (wbq_addr[0] == cmp_addr) && (wbq_wsel[0] == cmp_wsel) - && (wbq_lane[0] == cmp_lane); + wire post_push_post = wb_push && post_coal_key_r; + wire post_push_head = wb_push && coal_key_r[0]; always @(posedge clk) begin if (reset) begin diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 893809a06c..2b90f0e660 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -500,6 +500,17 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // pending wb (chained AMO replays from MSHR after a fill) and must not be // cleared by amo_wb_pending; the synthetic writeback carries amo.valid=0 so // it never re-commits at S1. + // The core-request slot's address: a pending LLC AMO writeback or the core + // request. The MSHR takes its allocate match against it a cycle early. + wire [`CS_LINE_ADDR_WIDTH-1:0] creq_addr_sel = amo_wb_pending ? amo_wb_addr : core_req_addr; + wire [WORD_SEL_WIDTH-1:0] creq_wsel_sel = amo_wb_pending ? amo_wb_word_idx : core_req_wsel; + wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] creq_sector_sel; + if (`CS_SECTOR_SEL_BITS != 0) begin : g_creq_sector + assign creq_sector_sel = creq_wsel_sel[`CS_WORD_SEL_BITS-1 -: `CS_SECTOR_SEL_BITS]; + end else begin : g_creq_sector_0 + assign creq_sector_sel = '0; + end + always @(*) begin sel_req = '0; sel_req.req.valid = init_fire || replay_fire || fill_fire || flush_fire || core_req_fire; @@ -513,14 +524,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( sel_req.req.attr = amo_wb_pending ? amo_wb_attr : (core_req_valid ? core_req_attr : '0); sel_req.req.way_idx = flush_way; sel_req.req.addr = (init_valid | flush_valid) ? `CS_LINE_ADDR_WIDTH'(flush_sel) - : (replay_mux ? replay_addr : (fill_mux ? mem_rsp_addr - : (amo_wb_pending ? amo_wb_addr : core_req_addr))); + : (replay_mux ? replay_addr : (fill_mux ? mem_rsp_addr : creq_addr_sel)); sel_req.req.byteen = replay_mux ? replay_byteen : (amo_wb_pending ? amo_wb_byteen : core_req_byteen); // a fill carries the installed sector in its word_idx high bits so the // tag/data stages mark/write the right sector (0 when 1 sector/line). sel_req.req.word_idx = replay_mux ? replay_wsel : (fill_mux ? fill_word_idx - : (amo_wb_pending ? amo_wb_word_idx : core_req_wsel)); + : creq_wsel_sel); sel_req.req.req_idx = replay_mux ? replay_idx : (amo_wb_pending ? amo_wb_idx : core_req_idx); sel_req.req.tag = (init_valid | flush_valid) ? (flush_valid ? flush_tag : '0) : (replay_mux ? replay_tag : (fill_mux ? (lazy_sel ? '0 : mem_rsp_tag_s) @@ -699,8 +709,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // disjoint words — the common case in streaming, where the whole warp's // beats land on one sector — keep the fast release path, as does // read-after-read on the same word. + // (the probes carry their own pending test; see VX_cache_mshr) assign creq_hit_order_hazard_st0 = st0.req.is_creq && ~st0.req.is_replay - && mshr_pending_raw && (mshr_pending_wr_raw || (st0.req.rw && mshr_pending_ovl_raw)) && ~is_amo_fwd_st0 && ~((AMO_ENABLE != 0) && st0.req.amo.amo_valid); @@ -774,15 +784,30 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_out (st1) ); + // stP is the stage that feeds stC: the dirty-byte mask is read there, so + // stC gets it from a register. + commit_t stP; VX_pipe_register #( .DATAW ($bits(commit_t)), .RESETW (1), - .DEPTH (PIPE_EX) - ) reg_cmt ( + .DEPTH ((PIPE_EX != 0) ? (PIPE_EX - 1) : 0) + ) reg_cmt_p ( .clk (clk), .reset (reset), .enable (~pipe_stall), .data_in (st1), + .data_out (stP) + ); + + VX_pipe_register #( + .DATAW ($bits(commit_t)), + .RESETW (1), + .DEPTH ((PIPE_EX != 0) ? 1 : 0) + ) reg_cmt ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (stP), .data_out (stC) ); @@ -935,7 +960,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), - .LAZY_FETCH (LAZY_FETCH) + .LAZY_FETCH (LAZY_FETCH), + .MASK_SYNC (PIPE_EX != 0) ) cache_data ( .clk (clk), .reset (reset), @@ -950,6 +976,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .mask_word (stC.req.word_idx), .mask_byteen (stC.req.byteen), .mask_sector (sector_idx_stc), + .mask_read (~pipe_stall), + .mask_rway (stP.req.way_idx), + .mask_rline (stP.req.addr[`CS_LINE_SEL_BITS-1:0]), .merge_write (do_merge_stc && ~pipe_stall), .flush (do_flush_std && ~pipe_stall), .read (do_read_std && ~pipe_stall), @@ -1045,7 +1074,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .NUM_BANKS (NUM_BANKS), .MSHR_SIZE (MSHR_SIZE), .MSHR_LINES (MSHR_LINES), - .WRITEBACK (WRITEBACK), .AMO_ENABLE (AMO_ENABLE != 0), .AMO_PASSTHRU ((AMO_ENABLE != 0) && (IS_LLC == 0)), .WORD_SEL_WIDTH (WORD_SEL_WIDTH), @@ -1077,12 +1105,16 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .dequeue_data ({replay_wsel, replay_byteen, replay_data, replay_tag, replay_idx, replay_amo}), .dequeue_id (replay_id), .dequeue_ready (replay_ready || fwd_fire), + .dequeue_fwd (fwd_fire), .dequeue_park (park_sel), .dequeue_park_retry (chain_retry), .dequeue_block (replay_block), .allocate_valid (mshr_allocate_st0 && ~pipe_stall), .allocate_addr (st0.req.addr), .allocate_sector (sector_idx_st0), + .allocate_addr_n (creq_addr_sel), + .allocate_sector_n (creq_sector_sel), + .allocate_adv (~pipe_stall), .allocate_word_idx (st0.req.word_idx), .allocate_rw (st0.req.rw), // Only non-LLC AMOs must not coalesce; at the LLC same-line AMOs coalesce diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index 16d79555b3..247a847dd1 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -40,7 +40,8 @@ module VX_cache_data import VX_gpu_pkg::*; #( parameter WRITE_ENABLE = 1, // Enable cache writeable parameter WRITEBACK = 0, // Enable cache writeback parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback - parameter LAZY_FETCH = 0 // Fills may be data-less or merge under the dirty bytes + parameter LAZY_FETCH = 0, // Fills may be data-less or merge under the dirty bytes + parameter MASK_SYNC = 0 // Dirty-byte mask read one stage ahead of the commit stage ) ( input wire clk, input wire reset, @@ -58,6 +59,11 @@ module VX_cache_data import VX_gpu_pkg::*; #( input wire [`UP(`CS_WORD_SEL_BITS)-1:0] mask_word, input wire [WORD_SIZE-1:0] mask_byteen, input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mask_sector, + // MASK_SYNC: the {way, line} that enters the commit stage next, and its + // advance enable + input wire mask_read, + input wire [`CS_WAY_SEL_WIDTH-1:0] mask_rway, + input wire [`CS_LINE_SEL_BITS-1:0] mask_rline, // merge of a refetched sector into its resident copy at the commit stage // (the stage behind it is empty): only the clean bytes input wire merge_write, @@ -112,8 +118,8 @@ module VX_cache_data import VX_gpu_pkg::*; #( end // Per-byte dirty mask (writeback only), one mask word per {way, line}. It - // is read and updated only at the commit stage, so each access sees every - // older update with no bypass. A line's mask is cleared when it is + // is updated only at the commit stage, in pipeline order, so each access + // sees every older update. A line's mask is cleared when it is // installed, which leaves nothing for the reset walk to clear. if (DIRTY_BYTES != 0) begin : g_dirty_bytes wire [`CS_WORDS_PER_LINE-1:0][WORD_SIZE-1:0] set_word; @@ -131,39 +137,89 @@ module VX_cache_data import VX_gpu_pkg::*; #( wire [LINE_SIZE-1:0] set_mask = set_word; wire [LINE_SIZE-1:0] clr_mask = clr_sector; - wire [DATA_ADDR_WIDTH-1:0] mask_addr; + wire [DATA_ADDR_WIDTH-1:0] mask_addr, mask_raddr; if (NUM_WAYS > 1) begin : g_mask_way_addr - assign mask_addr = DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}); + assign mask_addr = DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}); + assign mask_raddr = DATA_ADDR_WIDTH'({mask_rway[WAY_SEL_BITS-1:0], mask_rline}); end else begin : g_mask_line_addr - assign mask_addr = mask_line; + `UNUSED_VAR (mask_rway) + assign mask_addr = mask_line; + assign mask_raddr = mask_rline; end - // A whole-word read-modify-write shares one write enable across the - // mask, so it maps onto wide distributed RAM rather than one cell per - // byte (or a shattered block RAM). wire [LINE_SIZE-1:0] mask_rdata; - VX_sp_ram #( - .DATAW (LINE_SIZE), - .SIZE (DATA_RAM_DEPTH), - .OUT_REG (0), - .LUTRAM (1), - .RDW_MODE ("W"), - .INIT_ENABLE (1), - .INIT_VALUE (0) - ) byteen_store ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (mask_write), - .wren (1'b1), - .addr (mask_addr), - .wdata ((mask_rdata & ~clr_mask) | set_mask), - .rdata (mask_rdata) - ); + wire [LINE_SIZE-1:0] mask_wdata = (mask_rdata & ~clr_mask) | set_mask; + + if (MASK_SYNC != 0) begin : g_mask_sync + // Read at the stage ahead, so the commit stage takes the mask from + // a register rather than through a deep distributed-RAM read mux + // (and a large mask fits block RAM). A write by the request now + // committing to the address read behind it is forwarded. The + // pipeline holds as a whole, so nothing writes while the read is + // held. + wire [LINE_SIZE-1:0] mask_ram_rdata; + VX_dp_ram #( + .DATAW (LINE_SIZE), + .SIZE (DATA_RAM_DEPTH), + .OUT_REG (1), + .RDW_MODE ("R"), + .INIT_ENABLE (1), + .INIT_VALUE (0) + ) byteen_store ( + .clk (clk), + .reset (reset), + .read (mask_read), + .write (mask_write), + .wren (1'b1), + .waddr (mask_addr), + .wdata (mask_wdata), + .raddr (mask_raddr), + .rdata (mask_ram_rdata) + ); + + reg mask_fwd_r; + reg [LINE_SIZE-1:0] mask_fwd_data_r; + reg [DATA_ADDR_WIDTH-1:0] mask_raddr_r; + always @(posedge clk) begin + if (mask_read) begin + mask_fwd_r <= mask_write && (mask_addr == mask_raddr); + mask_fwd_data_r <= mask_wdata; + mask_raddr_r <= mask_raddr; + end + end + assign mask_rdata = mask_fwd_r ? mask_fwd_data_r : mask_ram_rdata; + + `RUNTIME_ASSERT (~mask_write || mask_read, ("%t: dirty-byte mask written while its read is held", $time)) + `RUNTIME_ASSERT (~mask_write || (mask_addr == mask_raddr_r), ("%t: dirty-byte mask read for another line: 0x%0h, 0x%0h", $time, mask_addr, mask_raddr_r)) + end else begin : g_mask_async + `UNUSED_VAR ({mask_read, mask_raddr}) + // A whole-word read-modify-write shares one write enable across the + // mask, so it maps onto wide distributed RAM rather than one cell per + // byte (or a shattered block RAM). + VX_sp_ram #( + .DATAW (LINE_SIZE), + .SIZE (DATA_RAM_DEPTH), + .OUT_REG (0), + .LUTRAM (1), + .RDW_MODE ("W"), + .INIT_ENABLE (1), + .INIT_VALUE (0) + ) byteen_store ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (mask_write), + .wren (1'b1), + .addr (mask_addr), + .wdata (mask_wdata), + .rdata (mask_rdata) + ); + end assign evict_byteen = mask_rdata; end else begin : g_no_dirty_bytes `UNUSED_VAR ({mask_write, mask_set, mask_clr_line, mask_clr_sector}) `UNUSED_VAR ({mask_line, mask_word, mask_byteen, mask_sector}) + `UNUSED_VAR ({mask_read, mask_rway, mask_rline}) assign evict_byteen = '1; // update whole line end diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index c314df553c..05616d0bdd 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -61,7 +61,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter MSHR_SIZE = 4, // Miss Reserv Queue Knob parameter MSHR_LINES = 0, // distinct line entries (0 = address held per request) parameter DATA_WIDTH = 1, // MSHR parameters - parameter WRITEBACK = 0, // Enable cache writeback parameter AMO_ENABLE = 0, // Per-entry AMO tracking (probe outputs) parameter AMO_PASSTHRU = 0, // Non-LLC passthrough: AMO entries never coalesce parameter WORD_SEL_WIDTH = 1, // word-in-line select width (order-hazard granularity) @@ -115,6 +114,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire [DATA_WIDTH-1:0] dequeue_data, output wire [MSHR_ADDR_WIDTH-1:0] dequeue_id, input wire dequeue_ready, + // the dequeue this cycle is the bank's fill-forward port, not a replay + input wire dequeue_fwd, // park the chain at its head instead of consuming it: the entry stays // allocated (it becomes the root of a refetch) and the walk stops. input wire dequeue_park, @@ -126,6 +127,10 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( input wire allocate_valid, input wire [`CS_LINE_ADDR_WIDTH-1:0] allocate_addr, input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] allocate_sector, + // the core request entering S0 next cycle, and the S0 advance enable + input wire [`CS_LINE_ADDR_WIDTH-1:0] allocate_addr_n, + input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] allocate_sector_n, + input wire allocate_adv, input wire [WORD_SEL_WIDTH-1:0] allocate_word_idx, input wire allocate_rw, input wire allocate_is_amo, // AMO: never coalesce @@ -321,6 +326,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( if (MSHR_LINES != 0) begin : g_line_table localparam LINE_ID_W = `LOG2UP(MSHR_LINES); + `UNUSED_VAR ({allocate_addr_n, allocate_sector_n, allocate_adv}) localparam LCNT_W = `CLOG2(MSHR_SIZE + 1); reg [`CS_LINE_ADDR_WIDTH-1:0] line_addr [0:MSHR_LINES-1]; @@ -440,10 +446,27 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( end end + // The allocate match is taken a cycle early, against the core request + // entering S0 and the table as it stands after this cycle's allocate, + // so S0 starts it from flops: it gates the data array's write enable + // through the order-hazard probes. A request of another kind entering + // S0 does not allocate, so its match is never used. Nothing allocates + // while S0 holds. + wire alloc_self_match = (allocate_addr == allocate_addr_n) && (allocate_sector == allocate_sector_n); + reg [MSHR_SIZE-1:0] entry_match_r; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match - assign entry_match[i] = (addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector); + wire match_n = (allocate_fire && (allocate_id == MSHR_ADDR_WIDTH'(i))) ? alloc_self_match + : ((addr_table[i] == allocate_addr_n) && (sector_table[i] == allocate_sector_n)); + always @(posedge clk) begin + if (allocate_adv) begin + entry_match_r[i] <= match_n; + end + end + assign entry_match[i] = entry_match_r[i]; assign entry_probe[i] = (addr_table[i] == probe_addr); assign entry_addr[i] = addr_table[i]; + `RUNTIME_ASSERT (~allocate_valid || (entry_match_r[i] == ((addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector))), + ("%t: *** %s allocate match out of sync: id=%0d", $time, INSTANCE_ID, i)) end assign fill_addr = addr_table[fill_id]; @@ -619,12 +642,12 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign allocate_id = allocate_id_r; assign allocate_previd = prev_idx; - if (WRITEBACK) begin : g_pending_wb - assign allocate_pending = |addr_matches; - end else begin : g_pending_wt - // exclude write requests if writethrough - assign allocate_pending = |(addr_matches & ~write_table); - end + // Any undrained entry of the line is pending, writes included. On a + // write-through cache a write entry only lives behind a read root, so once + // the root has replayed the writes still waiting are a chain mid-drain: a + // same-word access must link behind them, not overtake them through the + // array. + assign allocate_pending = |addr_matches; // Word-granular order-hazard probes. A chained entry only constrains a // requester that touches the SAME word: the entry's replay reads or @@ -634,14 +657,35 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_word_matches assign word_matches[i] = (word_table[i] == allocate_word_idx); end + // These probes gate the data array's write enable at S0, so they leave the + // replay dequeue out of their exclusion: a replay that could change them (a + // same-word entry, with a write on either side, against an allocating core + // request) is held that cycle by the bank's link hold, so the exclusion + // never alters them, and dropping it takes the S0 address compare behind + // that hold out of the write-enable cone. The forward port is not held, so + // its dequeue stays excluded, as does a release. + wire [MSHR_SIZE-1:0] hazard_matches; + for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_hazard_matches + assign hazard_matches[i] = valid_table[i] && entry_match[i] && ~amo_mask[i] + && ~(dequeue_fwd && ~((LAZY_FETCH != 0) && dequeue_park) && (dequeue_id == MSHR_ADDR_WIDTH'(i))) + && ~(finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i))); + end + wire hazard_pending = |hazard_matches; // The probed word's chain holds a WRITE that has not reached the data array // yet. A same-word hit completing ahead of it would read pre-store data (or // a write-hit would be overwritten by the older store's replay), inverting // arrival order — the bank demotes such a hit to a chained miss. - assign allocate_pending_wr = |(addr_matches & write_table & word_matches); + assign allocate_pending_wr = hazard_pending && |(hazard_matches & write_table & word_matches); // Any undrained same-word entry: an incoming WRITE hit must not land ahead // of an older chained read of the same word. - assign allocate_pending_ovl = |(addr_matches & word_matches); + assign allocate_pending_ovl = hazard_pending && |(hazard_matches & word_matches); + // the order hazard the bank derives from the two probes is the one the full + // exclusion gives + `RUNTIME_ASSERT (~allocate_valid + || ((allocate_pending_wr || (allocate_rw && allocate_pending_ovl)) + == (allocate_pending && (|(addr_matches & write_table & word_matches) + || (allocate_rw && |(addr_matches & word_matches))))), + ("%t: *** %s order-hazard probe diverged from the full exclusion", $time, INSTANCE_ID)) assign dequeue_valid = dequeue_val && ~((LAZY_FETCH != 0) && dequeue_block); // Sampled with dequeue_id_n, alongside the payload in mshr_store, rather diff --git a/tests/regression/Makefile b/tests/regression/Makefile index a11c131c2f..c6f6777677 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -15,7 +15,7 @@ TESTS := \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ async_barrier async_gbarrier gbar_phase bar_slot_phase packld wgather wsync quad_lod \ occupancy multikernel reopen module_reload single_cta \ - flush_coherence lazy_fetch sector_merge \ + flush_coherence lazy_fetch sector_merge raw_order \ vm_test vm_fault vm_stress # --- common exclude list --------------------------------------------- diff --git a/tests/regression/raw_order/Makefile b/tests/regression/raw_order/Makefile new file mode 100644 index 0000000000..48e56c03b6 --- /dev/null +++ b/tests/regression/raw_order/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := raw_order + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n2048 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/raw_order/common.h b/tests/regression/raw_order/common.h new file mode 100644 index 0000000000..dde800c43d --- /dev/null +++ b/tests/regression/raw_order/common.h @@ -0,0 +1,19 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define RAW_MARK 0x80000000u + +typedef struct { + uint32_t num_words; // words in the data buffer + uint32_t num_threads; // total threads launched + uint32_t warp_size; // threads per warp + uint32_t delay_sweep; // iteration k delays the checked load (k % delay_sweep) * delay_step steps + uint32_t delay_step; + uint64_t data_addr; // in: initial words; out: regions 0, 1 and 3 overwritten with RAW_MARK + index + uint64_t new_addr; // out: the value each region-0 load returned + uint64_t sink_addr; // out: per-thread sum of the region-2 loads (keeps them alive) +} kernel_arg_t; + +#endif diff --git a/tests/regression/raw_order/kernel.cpp b/tests/regression/raw_order/kernel.cpp new file mode 100644 index 0000000000..f704172ba3 --- /dev/null +++ b/tests/regression/raw_order/kernel.cpp @@ -0,0 +1,61 @@ +#include +#include "common.h" + +// Each warp owns a group of four regions (one word per lane each, one cache +// line per group) and, per iteration, issues in program order: +// +// load r2 - misses: heads the group's miss chain (answered from the fill) +// store r3 - chained behind it +// store r1 - chained behind it +// store r0 - chained behind it +// delay - warp-uniform countdown, swept per iteration +// load r0 - the checked load: must return the store to r0 above +// +// On a write-through cache the stores leave for memory at once and update +// the array when they replay behind the fill. The sweep lands the checked +// load at varying points of the chain's drain, including after the head has +// replayed and before the store to r0 has. The sequence is inline assembly +// so no compiler scheduling moves the requests relative to each other. +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto data = reinterpret_cast(arg->data_addr); + auto nval = reinterpret_cast(arg->new_addr); + auto sink = reinterpret_cast(arg->sink_addr); + const uint32_t W = arg->warp_size; + const uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const uint32_t lane = tid % W; + const uint32_t warp = tid / W; + const uint32_t num_warps = arg->num_threads / W; + const uint32_t group_words = 4 * W; + uint32_t acc = 0; + uint32_t k = 0; + for (uint32_t g = warp; (g + 1) * group_words <= arg->num_words; g += num_warps, ++k) { + uint32_t* r0 = data + g * group_words + lane; + uint32_t* r1 = r0 + W; + uint32_t* r2 = r0 + 2 * W; + uint32_t* r3 = r0 + 3 * W; + uint32_t i0 = g * group_words + lane; + uint32_t mark0 = RAW_MARK + i0; + uint32_t mark1 = RAW_MARK + i0 + W; + uint32_t mark3 = RAW_MARK + i0 + 3 * W; + uint32_t delay = (k % arg->delay_sweep) * arg->delay_step; + uint32_t a, v; + __asm__ volatile( + "lw %[a], 0(%[r2])\n\t" + "sw %[m3], 0(%[r3])\n\t" + "sw %[m1], 0(%[r1])\n\t" + "sw %[m0], 0(%[r0])\n\t" + "beqz %[n], 2f\n\t" + "1:\n\t" + "addi %[n], %[n], -1\n\t" + "bnez %[n], 1b\n\t" + "2:\n\t" + "lw %[v], 0(%[r0])\n\t" + : [a] "=&r"(a), [v] "=&r"(v), [n] "+r"(delay) + : [r0] "r"(r0), [r1] "r"(r1), [r2] "r"(r2), [r3] "r"(r3), + [m0] "r"(mark0), [m1] "r"(mark1), [m3] "r"(mark3) + : "memory"); + acc += a; + nval[i0] = v; + } + sink[tid] = acc; +} diff --git a/tests/regression/raw_order/main.cpp b/tests/regression/raw_order/main.cpp new file mode 100644 index 0000000000..24a9c44404 --- /dev/null +++ b/tests/regression/raw_order/main.cpp @@ -0,0 +1,158 @@ +// raw_order — a thread's load must observe its own earlier store to the same +// word, even while the line is still draining its miss chain. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t num_words = 1024; +uint32_t delay_sweep = 4; +uint32_t delay_step = 1; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:s:d:k:h")) != -1) { + switch (c) { + case 'n': num_words = std::atoi(optarg); break; + case 's': delay_sweep = std::atoi(optarg); break; + case 'd': delay_step = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-n words] [-s delay sweep] [-d delay step] [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} + +uint32_t init_word(uint32_t i) { return i * 7 + 1; } +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + if (delay_sweep == 0) delay_sweep = 1; + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // Fill every warp of every core. + uint64_t nt = 0, nw = 0, nc = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + const uint32_t num_threads = static_cast(nt * nw * nc); + std::cout << "raw_order: words=" << num_words << " threads=" << num_threads + << " delay=" << delay_sweep << "x" << delay_step << std::endl; + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + const uint64_t data_size = uint64_t(num_words) * sizeof(uint32_t); + const uint64_t sink_size = uint64_t(num_threads) * sizeof(uint32_t); + vx_buffer_h data_buf = nullptr, new_buf = nullptr, sink_buf = nullptr; + CHECK(vx_buffer_create(dev, data_size, VX_MEM_READ_WRITE, &data_buf)); + CHECK(vx_buffer_create(dev, data_size, VX_MEM_WRITE, &new_buf)); + CHECK(vx_buffer_create(dev, sink_size, VX_MEM_WRITE, &sink_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.num_words = num_words; + kernel_arg.num_threads = num_threads; + kernel_arg.warp_size = static_cast(nt); + kernel_arg.delay_sweep = delay_sweep; + kernel_arg.delay_step = delay_step; + CHECK(vx_buffer_address(data_buf, &kernel_arg.data_addr)); + CHECK(vx_buffer_address(new_buf, &kernel_arg.new_addr)); + CHECK(vx_buffer_address(sink_buf, &kernel_arg.sink_addr)); + + std::vector h_data(num_words), h_new(num_words); + for (uint32_t i = 0; i < num_words; ++i) h_data[i] = init_word(i); + CHECK(vx_enqueue_write(q, data_buf, 0, h_data.data(), data_size, 0, nullptr, nullptr)); + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &num_threads, grid, block)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0]= block[0]; + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, h_new.data(), new_buf, 0, data_size, 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_data.data(), data_buf, 0, data_size, 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + // Each warp owns groups of four regions of warp_size words: regions 0, 1 + // and 3 are overwritten, region 0 is then loaded back, region 2 is only + // read. + const uint32_t W = static_cast(nt); + const uint32_t group_words = 4 * W; + const uint32_t groups = num_words / group_words; + if (groups < num_threads / W) { + std::cout << "words (" << num_words << ") must cover one group per warp (" + << (num_threads / W) * group_words << ")" << std::endl; + return 1; + } + int errors = 0; + auto expect = [&](const char* what, uint32_t i, uint32_t actual, uint32_t ref) { + if (actual == ref) return; + if (errors < 16) + std::printf("*** %s [%u] expected=0x%x actual=0x%x%s\n", what, i, ref, actual, + actual == init_word(i) ? " (before its own earlier store)" : ""); + ++errors; + }; + for (uint32_t g = 0; g < groups; ++g) { + for (uint32_t l = 0; l < W; ++l) { + uint32_t i0 = g * group_words + l; + expect("load r0", i0, h_new[i0], RAW_MARK + i0); + expect("data r0", i0, h_data[i0], RAW_MARK + i0); + expect("data r1", i0 + W, h_data[i0 + W], RAW_MARK + i0 + W); + expect("data r2", i0 + 2 * W, h_data[i0 + 2 * W], init_word(i0 + 2 * W)); + expect("data r3", i0 + 3 * W, h_data[i0 + 3 * W], RAW_MARK + i0 + 3 * W); + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(sink_buf); + vx_buffer_release(new_buf); + vx_buffer_release(data_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} From 32432bb9d9af5406a42c504d1b484383faf6a3d5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:59 -0700 Subject: [PATCH 111/336] simx: match LSU ingress, SFU result buffers and lmem crossbar to RTL Diffing per-core issue order against the RTL on sgemm n64 with two cores and an L2 found three structures SimX did not model: - The LSU's lane-dispatch skid buffer, the scheduler's request queue and its pending table all register their full flags, so a slot freed in one cycle takes a new request in the next. SimX freed the entry, sent the next request and ingested a new trace in one tick. The LSU now admits against room sampled at the start of the tick, and its inputs are registered-ready. - The WCTL and CSR units each hold two results in their own buffer ahead of the SFU's shared response switch, which holds two more. SimX let one six-entry output channel absorb them, so up to ten SFU ops sat between dispatch and commit, where the RTL holds eight. A WCTL or CSR op is now admitted only while fewer than two of that unit's results wait beyond the switch's entries. Results reach commit after different latencies, so each is tracked at its arrival cycle, in the order commit pops them. - The local-memory request crossbar registers its output toward the banks. With it, a shared-memory load's minimum latency matches the RTL's 14 cycles. CI: core:parity-sgemm-mc64 (n64, two cores, L2, two operand collectors per issue slice, 3% tolerance) runs at +1.72% against the RTL. parity-sgemmx moves from -0.62% to -3.40%, still within its 5%: the registered LSU input is RTL-faithful, so another SimX error on that local-memory kernel had been offsetting it. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 12 ++++++++ sim/simx/lsu_unit.cpp | 20 +++++++++---- sim/simx/lsu_unit.h | 2 +- sim/simx/mem/local_mem.cpp | 14 ++++----- sim/simx/sfu_unit.cpp | 60 +++++++++++++++++++++++++++++++++----- sim/simx/sfu_unit.h | 19 ++++++++++++ 6 files changed, 106 insertions(+), 21 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 676531abed..e6210b28a3 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -188,6 +188,18 @@ tests: args: -n128 configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 shape: {cores: 2, l2cache: true} +# The same shape at n64: L2 hits on a miss-heavy working set, two operand +# collectors per issue slice, and the load-response path's exact entry counts. +- id: parity-sgemm-mc64 + check: model_parity + # Tighter than the default: the shape exercises those entries and the + # collectors' output register exactly. + tolerance: 0.03 + via: blackbox + app: sgemm + args: -n64 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 + shape: {cores: 2, l2cache: true} # SGEMM on two frontend slices feeding four issue slices: per-slice fetch on a # two-port icache, CTAs as wide as the core, a 16-port dcache. - id: parity-sgemm-fw2 diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 21240dd7b4..93252a9eb8 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -108,7 +108,12 @@ LsuUnit::LsuUnit(const SimContext& ctx, const char* name, Core* core) , TcuRspOut(TCU_AGU_PORTS, this) #endif , pending_loads_(0) -{} +{ + // the lane-dispatch skid buffer's ready is registered + for (auto& input : Inputs) { + input.set_registered_ready(true); + } +} LsuUnit::~LsuUnit() {} @@ -354,10 +359,10 @@ void LsuUnit::process_response_step(uint32_t b) { // At most one transaction/cycle: the trace is not dispatchable until the next // cycle, so req_queue adds a real cycle of latency. A while-loop here would // fabricate write bandwidth and collapse that latency to zero. -void LsuUnit::ingest_inputs(uint32_t b) { +void LsuUnit::ingest_inputs(uint32_t b, bool queue_room, uint32_t reads_held) { auto& state = states_.at(b); auto& input = Inputs.at(b); - if (input.empty() || state.req_queue.full()) + if (input.empty() || !queue_room) return; auto trace = input.peek(); // Hold a fence at the input head until req_queue is empty so the @@ -368,7 +373,7 @@ void LsuUnit::ingest_inputs(uint32_t b) { return; const bool is_read = !lsu_type_tag || *lsu_type_tag == LsuType::LOAD; if (is_read) { - if (state.pending_reqs.size() + state.staged_reads >= VX_CFG_LSU_PENDING_SIZE) { + if (reads_held >= VX_CFG_LSU_PENDING_SIZE) { return; } ++state.staged_reads; @@ -715,8 +720,13 @@ void LsuUnit::on_tick() { // as it stood at the start of this cycle, THEN ingest one new trace for // next cycle. Dispatching before ingest makes req_queue a 1-cycle stage // rather than a same-tick passthrough. + // The scheduler's request queue and its pending table register their + // full flags: a slot freed this cycle takes a new request next cycle. + auto& state = states_.at(b); + bool queue_room = !state.req_queue.full(); + uint32_t reads_held = state.pending_reqs.size() + state.staged_reads; this->process_response_step(b); this->process_request_step(b); - this->ingest_inputs(b); + this->ingest_inputs(b, queue_room, reads_held); } } diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index cb81581e53..045ddc9880 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -111,7 +111,7 @@ class LsuUnit : public FuncUnit { void send_fence_token(uint32_t b); // Drain Inputs[b] into req_queue. - void ingest_inputs(uint32_t b); + void ingest_inputs(uint32_t b, bool queue_room, uint32_t reads_held); struct mem_addr_size_t { uint64_t addr; diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 211f46c5d7..5d25807a80 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -88,12 +88,12 @@ class LocalMem::Impl { snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); uint32_t lg2_line_size = log2ceil(config_.line_size); uint32_t num_banks = 1 << config.B; - // The crossbar's registered request and response outputs are taken by - // the switch and bank stages around it, so its delays here are only the - // extra tree level: an arbiter over more than 1.5x the fan-out limit is - // split into a buffered tree, one cycle deeper. The request side - // arbitrates the lanes at each bank, the response side the banks at - // each lane. + // The request crossbar registers its output toward the banks, one cycle + // ahead of the SRAM read. The response crossbar's register is the + // adapter's response stage. An arbiter over more than 1.5x the fan-out + // limit is split into a buffered tree, one cycle deeper. The request + // side arbitrates the lanes at each bank, the response side the banks + // at each lane. auto tree_delay = [](uint32_t requesters) -> uint32_t { constexpr uint32_t kMaxFanout = 8; return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; @@ -102,7 +102,7 @@ class LocalMem::Impl { [lg2_line_size, num_banks](const MemCrossBar::ReqType& req) { // Custom logic to calculate the output index using bank interleaving return (uint32_t)((req.addr >> lg2_line_size) & (num_banks-1)); - }, tree_delay(config.num_reqs), tree_delay(num_banks)); + }, 1 + tree_delay(config.num_reqs), tree_delay(num_banks)); for (uint32_t i = 0; i < config.num_reqs; ++i) { simobject->Inputs.at(i).bind(&mem_xbar_->ReqIn.at(i)); mem_xbar_->RspOut.at(i).bind(&simobject->Outputs.at(i)); diff --git a/sim/simx/sfu_unit.cpp b/sim/simx/sfu_unit.cpp index 4c57d1ab8b..14d76713c8 100644 --- a/sim/simx/sfu_unit.cpp +++ b/sim/simx/sfu_unit.cpp @@ -96,6 +96,40 @@ bool SfuUnit::rtu_trace2_reserve_slot(uint32_t wid) { #endif +// Results sent toward commit with the unit that produced them, in the order +// they reach the output channel (arrival cycle, then send order), which is the +// order commit pops them in. The oldest PE_SWITCH_SIZE stand in the shared +// response switch; the rest still wait in their unit's own result buffer. +void SfuUnit::sync_out(uint32_t b) { + auto& q = out_pe_.at(b); + uint32_t n = Outputs.at(b).size(); + while (q.size() > n) { + q.pop_front(); + } +} + +bool SfuUnit::pe_buf_full(uint32_t b, PeClass pe) { + this->sync_out(b); + auto& q = out_pe_.at(b); + uint32_t held = 0; + for (size_t i = PE_SWITCH_SIZE; i < q.size(); ++i) { + held += (q[i].pe == pe); + } + return held >= PE_BUF_SIZE; +} + +void SfuUnit::send_out(uint32_t b, instr_trace_t* trace, uint32_t delay, PeClass pe) { + this->sync_out(b); + Outputs.at(b).send(trace, delay); + auto& q = out_pe_.at(b); + uint64_t arrive = SimPlatform::instance().cycles() + delay; + auto it = q.end(); + while (it != q.begin() && std::prev(it)->arrive > arrive) { + --it; + } + q.insert(it, OutPe{arrive, pe}); +} + void SfuUnit::on_tick() { #ifdef VX_CFG_EXT_RTU_ENABLE // Drain RTU rsps. Two flavors, both completing the warp's parked WAIT @@ -125,7 +159,7 @@ void SfuUnit::on_tick() { auto wb = is_candidate ? rtu_unit_->on_candidate_rsp(rsp) : rtu_unit_->on_terminal_rsp(rsp); if (wb.trace) { - Outputs.at(wb.block_id).send(wb.trace, this->latency_of(wb.trace)); + this->send_out(wb.block_id, wb.trace, this->latency_of(wb.trace), PE_OTHER); DT(3, "rtu-rsp deliver: core=" << core_->id() << ", wid=" << wb.trace->wid << ", cand=" << is_candidate); } else { @@ -153,7 +187,7 @@ void SfuUnit::on_tick() { { // Unit latency is already modeled by the TEX pipeline; charge only // the gather/writeback hop. - output.send(trace, 2); + this->send_out(rsp.block_id, trace, 2, PE_OTHER); DT(3, "tex-rsp deliver: core=" << core_->id() << ", wid=" << trace->wid); } tex_rsp_in.pop(); @@ -337,7 +371,7 @@ void SfuUnit::on_tick() { continue; // OM back-pressure — nothing was sent; retry } } - output.send(trace, 1); + this->send_out(b, trace, 1, PE_OTHER); input.pop(); continue; } @@ -369,7 +403,7 @@ void SfuUnit::on_tick() { if (output.full()) continue; if (!rtu_unit_->process_cb_ret(trace, b)) continue; // bus backpressure — nothing was sent - output.send(trace, this->latency_of(trace)); + this->send_out(b, trace, this->latency_of(trace), PE_OTHER); input.pop(); continue; } @@ -384,7 +418,7 @@ void SfuUnit::on_tick() { if (output.full()) continue; if (!rtu_unit_->process_trace_uop(trace, b, args.uop)) continue; - output.send(trace, this->latency_of(trace)); + this->send_out(b, trace, this->latency_of(trace), PE_OTHER); input.pop(); continue; } @@ -400,7 +434,7 @@ void SfuUnit::on_tick() { } instr_trace_t* wb = rtu_unit_->process_wait(trace, b); if (wb) { - output.send(wb, this->latency_of(wb)); + this->send_out(b, wb, this->latency_of(wb), PE_OTHER); } input.pop(); continue; @@ -414,7 +448,7 @@ void SfuUnit::on_tick() { auto args = std::get(trace->instr_ptr->get_args()); if (output.full()) continue; rtu_window_.process_getw_uop(trace, args.uop, *rtu_p == GfxwType::GETWF); - output.send(trace, this->latency_of(trace)); + this->send_out(b, trace, this->latency_of(trace), PE_OTHER); input.pop(); continue; } @@ -465,6 +499,13 @@ void SfuUnit::on_tick() { if (output.full()) continue; // stall — no side effects this tick + // The warp-control and CSR units each answer through their own + // two-entry result buffer ahead of the shared response switch. + const bool is_wctl = std::get_if(&trace->op_type) != nullptr; + const PeClass pe = is_csr ? PE_CSR : (is_wctl ? PE_WCTL : PE_OTHER); + if (pe != PE_OTHER && this->pe_buf_full(b, pe)) + continue; + // WSYNC has a structural gate: cannot complete until prior insts retire. // BAR (vx_barrier and vx_barrier_arrive) drains LSU before continuing — // implements CUDA __syncthreads / OpenCL barrier(CLK_LOCAL_MEM_FENCE) semantic. @@ -502,7 +543,7 @@ void SfuUnit::on_tick() { } uint32_t delay = this->latency_of(trace); - output.send(trace, delay); + this->send_out(b, trace, delay, pe); // Warp-control refines the default (fetch_stall) release decision: a // sync-barrier, a not-yet-last barrier arrival, a deferred wspawn, or a // warp that disabled itself (tmask=0) keeps the warp parked — it is @@ -536,6 +577,9 @@ void SfuUnit::on_reset() { std::fill(std::begin(res_rr_), std::end(res_rr_), 0); csr_read_done_.fill(nullptr); bar_arrive_.fill(BarArrive{}); + for (auto& q : out_pe_) { + q.clear(); + } } SfuUnit::SharedRes SfuUnit::shared_res(uint32_t block) { diff --git a/sim/simx/sfu_unit.h b/sim/simx/sfu_unit.h index 5e208b614e..7ea94a6b8d 100644 --- a/sim/simx/sfu_unit.h +++ b/sim/simx/sfu_unit.h @@ -13,6 +13,8 @@ #pragma once +#include + #include #include #include "func_unit.h" @@ -133,6 +135,23 @@ class SfuUnit : public FuncUnit { std::unique_ptr wctl_unit_; std::unique_ptr csr_unit_; + // The unit a result in the output channel came from: the warp-control and + // CSR units each hold PE_BUF_SIZE results ahead of the shared response + // switch, which holds PE_SWITCH_SIZE. Results reach the channel after + // different delays, so each is kept at its arrival cycle, in the channel's + // own order. + enum PeClass : uint8_t { PE_OTHER = 0, PE_WCTL, PE_CSR }; + static constexpr uint32_t PE_BUF_SIZE = 2; + static constexpr uint32_t PE_SWITCH_SIZE = 2; + struct OutPe { + uint64_t arrive; + PeClass pe; + }; + std::array, VX_CFG_NUM_SFU_BLOCKS> out_pe_; + void sync_out(uint32_t b); + bool pe_buf_full(uint32_t b, PeClass pe); + void send_out(uint32_t b, instr_trace_t* trace, uint32_t delay, PeClass pe); + // Per block, the CSR op that has spent its one-cycle CTA-context read. std::array csr_read_done_{}; // Per block, the barrier arrive waiting for the shared barrier unit's From d0cb49c70df9cd4ff339c3230c24547c4e314749 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:00 -0700 Subject: [PATCH 112/336] cache: fix a lazy-fetch dcache deadlock between AMO write-back and fills On a lazy-fetch dcache with a deferred commit stage (latency 3), a queued AMO result drains through the core-request slot. Like a core request, it waited for the lazy queue's pipeline reservation. Meanwhile a full AMO write-back queue held off the fills that drain the lazy queue, and the bank deadlocked. With the pipe empty, only the write-back itself can push, so it now needs only a free slot (lazyq_wb_block). The lazy queue is now twice the pipe depth, so a few queued entries leave admission and the refetch loopback open. SimX sizes its loopback the same way. CI: cache:lazy_fetch-5 (latency-3 lazy-fetch dcache with atomics, simx and rtlsim). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 10 ++++++++++ hw/rtl/cache/VX_cache_bank.sv | 21 +++++++++++++++------ sim/simx/mem/cache.cpp | 5 +++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 12bd2c56b4..c4326f73c0 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -362,6 +362,16 @@ tests: shape: cores: 4 l2cache: true +# The same on a dcache with a deferred commit stage (latency 3): a queued AMO +# result drains through the core-request slot while the lazy queue holds its +# pipeline reservation, and fully written sectors still loop back data-less. +- id: lazy_fetch-5 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 -DVX_CFG_DCACHE_LATENCY=3 # Both sectors of one sectored L2 line miss together from different warps and # cores: each must get its own fill, never chain behind the other sector's. # Run with the address held per request and with the MSHR line table. diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 2b90f0e660..c9150cc112 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -197,7 +197,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mem_rsp_sector; // sector this fill installs wire [MSHR_ADDR_WIDTH-1:0] mshr_alloc_id, mshr_previd; wire mshr_fill_refetch, mshr_fill_retry, mshr_dequeue_reads; - wire lazyq_empty, lazyq_alm_full; + wire lazyq_empty, lazyq_alm_full, lazyq_full; wire mshr_pending_raw; wire mshr_pending_wr_raw; wire mshr_pending_ovl_raw; @@ -433,6 +433,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; // a core request may become a write-miss root, which pushes to lazyq. wire creq_alm_full = mreq_queue_alm_full || lazyq_alm_full || replay_block; + // A queued AMO result drains through the core-request slot, while a full + // AMO writeback queue holds off the fills that drain the lazy queue. So + // the writeback is not held to the lazy queue's pipeline reservation when + // the pipe is empty: then only the writeback itself can push, and a free + // slot is room enough. + wire lazyq_wb_block = lazyq_alm_full && (pipe_inflight || lazyq_full); + wire amo_wb_alm_full = mreq_queue_alm_full || lazyq_wb_block || replay_block; assign core_req_ready = creq_grant && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; @@ -445,9 +452,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire flush_fire = flush_valid && flush_ready; // amo_wb_path already excludes the cycle a fresh AMO commits at S1, so the // writeback never races the chain update. - wire amo_wb_fire = amo_wb_path && creq_grant && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall; - wire core_req_fire = (amo_creq_path || amo_wb_path) && creq_grant - && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall; + wire amo_wb_fire = amo_wb_path && creq_grant && ~amo_wb_alm_full && ~mshr_alm_full && ~pipe_stall; + wire core_req_fire = ((amo_creq_path && ~creq_alm_full) || (amo_wb_path && ~amo_wb_alm_full)) + && creq_grant && ~mshr_alm_full && ~pipe_stall; wire [MSHR_ADDR_WIDTH-1:0] mem_rsp_id = mem_rsp_tag[MSHR_ADDR_WIDTH-1:0]; wire [MSHR_ADDR_WIDTH-1:0] fill_id = lazy_sel ? lazyq_id : mem_rsp_id; @@ -1628,8 +1635,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Sized like the memory request queue's reservation: a core request is // admitted only while every request already in the pipe still fits. if (LAZY_FETCH != 0) begin : g_lazyq - wire lazyq_full; - localparam LAZYQ_SIZE = 1 << `CLOG2(PIPELINE_STAGES + 1); + // Room for twice the pipe, so a few queued entries leave the + // reservation (and the refetch loopback, which needs it) open. + localparam LAZYQ_SIZE = 1 << `CLOG2(2 * PIPELINE_STAGES); VX_fifo_queue #( .DATAW (MSHR_ADDR_WIDTH), .DEPTH (LAZYQ_SIZE), @@ -1702,6 +1710,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign lazyq_id = '0; assign lazyq_empty = 1'b1; assign lazyq_alm_full = 1'b0; + assign lazyq_full = 1'b0; assign chain_partial = 1'b0; assign chain_retry = 1'b0; assign refetch_inflight = 1'b0; diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 6f09d976f1..b2a7d24945 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -1495,9 +1495,10 @@ class CacheBank : public SimObject { chain_sector_ = sector_id; } - // Lazy-fetch loopback depth: room for every request in the pipe plus one. + // Lazy-fetch loopback depth: room for twice the pipe, so a few queued + // entries leave admission and the refetch loopback open. static uint32_t lazyq_size(const Cache::Config &config) { - return 1u << log2ceil(config.latency + 1); + return 1u << log2ceil(2 * config.latency); } // The loopback still fits every request in the pipe: a new request may be From 5391be583ddf75ff05f2dff9564ed90da942b856 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:02 -0700 Subject: [PATCH 113/336] syn: read vortex_afu.v as SystemVerilog in the Vivado flows The platform tops pass unpacked per-port arrays to VX_afu_wrap, which Vivado accepts only from SystemVerilog sources. The elaboration check, the per-block DUT project, the XRT kernel packager and the AVED IP packager now set vortex_afu.v's file type accordingly. The DUT `top` build also resolves AFU_MEM_PORTS with afu_mem_ports.sh, as the XRT flow does. The unused `unittest` DUT, which had no top module, leaves the Xilinx and Altera synthesis catalogs. Co-Authored-By: Claude Opus 5.5 --- hw/scripts/xilinx_elab_check.tcl | 6 ++++++ hw/syn/altera/dut/catalog.mk | 6 +----- hw/syn/xilinx/README | 2 +- hw/syn/xilinx/aved/package_ip.tcl | 7 +++++++ hw/syn/xilinx/dut/build.mk | 11 +++++++++++ hw/syn/xilinx/dut/catalog.mk | 6 ++---- hw/syn/xilinx/dut/project.tcl | 6 ++++++ hw/syn/xilinx/xrt/package_kernel.tcl | 6 ++++++ 8 files changed, 40 insertions(+), 10 deletions(-) diff --git a/hw/scripts/xilinx_elab_check.tcl b/hw/scripts/xilinx_elab_check.tcl index 8933471fbf..39ce791404 100644 --- a/hw/scripts/xilinx_elab_check.tcl +++ b/hw/scripts/xilinx_elab_check.tcl @@ -46,6 +46,12 @@ if {$mic_mode eq "off" || $mic_mode eq "0"} { create_project -in_memory add_files -norecurse ${mic_sources} + # The platform tops (vortex_afu.v) pass unpacked per-port arrays to + # VX_afu_wrap, which needs SystemVerilog. + set mic_afu_tops [get_files -quiet */vortex_afu.v] + if {[llength $mic_afu_tops] > 0} { + set_property file_type SystemVerilog $mic_afu_tops + } # Black-box every generated vendor IP via its BLOCK_STUB so a -rtl elaboration # resolves the IP instances (FPU FMA/DIV/SQRT/MUL/ADD, etc.) without pulling diff --git a/hw/syn/altera/dut/catalog.mk b/hw/syn/altera/dut/catalog.mk index 2bca1ee61b..74563d021f 100644 --- a/hw/syn/altera/dut/catalog.mk +++ b/hw/syn/altera/dut/catalog.mk @@ -9,7 +9,7 @@ # Add a DUT: append its name to DUTS and define its variables. No new directory, # no dispatcher edit. -DUTS := cache core fpu issue lmem mem_unit scope top unittest vortex +DUTS := cache core fpu issue lmem mem_unit scope top vortex # --------------------------------------------------------------------------- # shared include fragments @@ -75,10 +75,6 @@ top_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RT -I$(AFU_DIR) -I$(AFU_DIR)/ccip -I$(IP_CACHE_DIR) $(FPU_INC) $(TCU_INC) $(DXA_INC) top_PKG = $(TCU_PKG) $(DXA_PKG) -unittest_PRJ := Unittest -unittest_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache \ - -I$(IP_CACHE_DIR) $(FPU_INC) - vortex_PRJ := Vortex vortex_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache \ -I$(IP_CACHE_DIR) $(FPU_INC) $(TCU_INC) $(DXA_INC) diff --git a/hw/syn/xilinx/README b/hw/syn/xilinx/README index 0ae222a173..d698de9fde 100644 --- a/hw/syn/xilinx/README +++ b/hw/syn/xilinx/README @@ -9,7 +9,7 @@ vivado -mode batch -source xilinx_ip_gen.tcl -tclargs ip/xilinx_u50_gen3x16_xdma # build per-block (dut): Vivado project-mode synthesis of an individual block. # Self-backgrounds; writes //build.log. Default DEVICE: xcu55c. -# Targets: core vortex top issue cache fpu tcu dxa tex raster om lmem mem_unit scope unittest +# Targets: core vortex top issue cache fpu tcu dxa tex raster om lmem mem_unit scope cd hw/syn/xilinx/dut PREFIX=smoke1 make core PREFIX=smoke1 make vortex diff --git a/hw/syn/xilinx/aved/package_ip.tcl b/hw/syn/xilinx/aved/package_ip.tcl index 0e6bdb33cc..4df28092a9 100644 --- a/hw/syn/xilinx/aved/package_ip.tcl +++ b/hw/syn/xilinx/aved/package_ip.tcl @@ -63,6 +63,13 @@ foreach line [split $data "\n"] { } } +# The platform tops (vortex_afu.v) pass unpacked per-port arrays to +# VX_afu_wrap, which needs SystemVerilog. +set afu_tops [get_files -quiet */vortex_afu.v] +if {[llength $afu_tops] > 0} { + set_property file_type SystemVerilog $afu_tops +} + # Pull in the floating-point cores generated above so they are packaged # alongside the RTL. The set matches the XRT flow's package_kernel.tcl. set ip_files [list] diff --git a/hw/syn/xilinx/dut/build.mk b/hw/syn/xilinx/dut/build.mk index 878af652d0..dbff6f5ae3 100644 --- a/hw/syn/xilinx/dut/build.mk +++ b/hw/syn/xilinx/dut/build.mk @@ -32,3 +32,14 @@ RTL_PKGS += $($(DUT)_PKG) ifeq ($($(DUT)_EXT),1) include $(VORTEX_HOME)/hw/syn/extensions.mk endif + +# An AFU build reads the platform tops too, which declare one AXI master per +# GPU memory port literally: resolve the count from the elaborated +# configuration, as the xrt flow does. +ifeq ($($(DUT)_AFU),1) +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c .afu_mem_ports $(VERILATOR_PATH)/bin/verilator $(filter-out -J% -DSYNTHESIS -DVIVADO,$(CFLAGS)) $(RTL_PKGS)) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) +endif diff --git a/hw/syn/xilinx/dut/catalog.mk b/hw/syn/xilinx/dut/catalog.mk index 1bb5edaa98..abdef0dae8 100644 --- a/hw/syn/xilinx/dut/catalog.mk +++ b/hw/syn/xilinx/dut/catalog.mk @@ -11,7 +11,7 @@ # directory, no dispatcher edit. DUTS := cache core cp dxa fpu gfx issue lmem mem_unit om raster rtu \ - scope tcu tensor tex top unittest vm vortex + scope tcu tensor tex top vm vortex # --------------------------------------------------------------------------- # shared include fragments @@ -131,12 +131,10 @@ tex_PKG = $(RTL_DIR)/tex/VX_tex_pkg.sv top_PRJ := VX_afu_wrap top_IP := 1 top_EXT := 1 +top_AFU := 1 top_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache \ -I$(RTL_DIR)/cp -I$(AFU_DIR) -I$(AFU_COMMON_DIR) $(FPU_INC) -unittest_PRJ := Unittest -unittest_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache $(FPU_INC) - vm_PRJ := VX_vm_top vm_CFG := -DVX_CFG_VM_ENABLE -DVX_CFG_EXT_A_ENABLE -DVX_CFG_NUM_CORES=2 \ -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 \ diff --git a/hw/syn/xilinx/dut/project.tcl b/hw/syn/xilinx/dut/project.tcl index 56339544ef..9f512e3f94 100644 --- a/hw/syn/xilinx/dut/project.tcl +++ b/hw/syn/xilinx/dut/project.tcl @@ -173,6 +173,12 @@ proc run_setup {} { # Add the design sources add_files -norecurse -verbose $vsources_list + # The platform tops (vortex_afu.v) pass unpacked per-port arrays to + # VX_afu_wrap, which needs SystemVerilog. + set afu_tops [get_files -quiet */vortex_afu.v] + if {[llength $afu_tops] > 0} { + set_property file_type SystemVerilog $afu_tops + } # process defines set_property verilog_define ${vdefines_list} [current_fileset] diff --git a/hw/syn/xilinx/xrt/package_kernel.tcl b/hw/syn/xilinx/xrt/package_kernel.tcl index d9b9777c72..c9b0ab20cb 100644 --- a/hw/syn/xilinx/xrt/package_kernel.tcl +++ b/hw/syn/xilinx/xrt/package_kernel.tcl @@ -62,6 +62,12 @@ foreach def $vdefines_list { create_project -force kernel_pack $path_to_tmp_project add_files -norecurse ${vsources_list} +# The platform tops (vortex_afu.v) pass unpacked per-port arrays to +# VX_afu_wrap, which needs SystemVerilog. +set afu_tops [get_files -quiet */vortex_afu.v] +if {[llength $afu_tops] > 0} { + set_property file_type SystemVerilog $afu_tops +} set obj [get_filesets sources_1] set ip_files [list \ From 7820027c0f35bdf2326bd60885d668b806b1159c Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:03 -0700 Subject: [PATCH 114/336] simx: keep the last-level cache across a launch reset A launch after CP DMA writes into the last-level cache drained the whole model before its reset, including the transfer's line fills. The hardware does not reset between a transfer and the launch that follows it. The last-level cache, the memory below it and the CP port are now built under a LaunchKeepScope and survive reset_launch(), so the kernel meets fills still in flight. The crossbar's 64-input bound, which its per-output round-robin state never needed, is dropped. Co-Authored-By: Claude Opus 5.5 --- sim/common/simobject.h | 42 +++++++++++++- sim/common/simobject_functional.h | 42 +++++++++++++- sim/simx/cluster.cpp | 59 +++++++++++--------- sim/simx/processor.cpp | 93 ++++++++++++++++++------------- sim/simx/processor_impl.h | 5 +- sim/simx/types.h | 2 +- 6 files changed, 172 insertions(+), 71 deletions(-) diff --git a/sim/common/simobject.h b/sim/common/simobject.h index 225dab157a..5cc55e36ec 100644 --- a/sim/common/simobject.h +++ b/sim/common/simobject.h @@ -88,6 +88,7 @@ class SimObjectBase { std::string name_; bool tick_active_ = true; uint32_t domain_ = 0; + bool launch_kept_ = false; // built under a LaunchKeepScope virtual void do_reset() = 0; virtual void do_tick() = 0; friend class SimPlatform; @@ -265,6 +266,7 @@ class SimPlatform { uint32_t alloc_domain() { return num_domains_++; } uint32_t num_domains() const { return num_domains_; } uint32_t build_domain() const { return build_domain_; } + bool build_launch_kept() const { return build_launch_kept_; } // Cap the parallel thread count (must be called before the first tick). // A limit of 1 forces serial execution — required while the configuration @@ -280,6 +282,25 @@ class SimPlatform { req_workers_ = std::max(1u, num_workers); } + // Objects built under a LaunchKeepScope (the last-level cache and the + // memory below it, when the command processor writes into that cache) + // carry their state and in-flight work across reset_launch(). + class LaunchKeepScope { + public: + explicit LaunchKeepScope(bool keep) { + auto& p = SimPlatform::instance(); + saved_ = p.build_launch_kept_; + p.build_launch_kept_ = saved_ || keep; + } + ~LaunchKeepScope() { + SimPlatform::instance().build_launch_kept_ = saved_; + } + LaunchKeepScope(const LaunchKeepScope&) = delete; + LaunchKeepScope& operator=(const LaunchKeepScope&) = delete; + private: + bool saved_; + }; + class DomainScope { public: explicit DomainScope(uint32_t domain) { @@ -331,6 +352,7 @@ class SimPlatform { void schedule(Func&& func, const Pkt& pkt, uint64_t delay, uint32_t domain); void reset(); + void reset_launch(); void tick(); private: @@ -395,6 +417,7 @@ class SimPlatform { uint32_t num_domains_ = 1; uint32_t build_domain_ = 0; + bool build_launch_kept_ = false; static inline thread_local uint32_t tl_exec_domain_ = HOST_DOMAIN; static inline thread_local SimObjectBase* tl_exec_object_ = nullptr; static inline thread_local uint64_t tl_cross_seq_ = 0; @@ -1183,6 +1206,22 @@ inline void SimPlatform::validate_topology() { } } +// A kernel launch resets every object but those built under a +// LaunchKeepScope: the hardware does not reset between a transfer and the +// launch that follows it, so the transfer's line fills may still be in +// flight in the kept memory side when the kernel starts. Their events stay +// scheduled and time keeps counting; the rest of the model is quiescent. +inline void SimPlatform::reset_launch() { + for (auto* object : active_reset_) { + if (!object->launch_kept_) { + object->do_reset(); + } + } + for (auto& object : objects_) { + object->tick_active_ = true; + } +} + inline void SimPlatform::reset() { this->validate_topology(); @@ -1361,7 +1400,8 @@ void SimChannelEvent::fire() { inline SimObjectBase::SimObjectBase(const SimContext&, const std::string& name) : name_(name) - , domain_(SimPlatform::instance().build_domain()) {} + , domain_(SimPlatform::instance().build_domain()) + , launch_kept_(SimPlatform::instance().build_launch_kept()) {} inline SimChannelBase::SimChannelBase(SimObjectBase* module) : module_(module) diff --git a/sim/common/simobject_functional.h b/sim/common/simobject_functional.h index 924f09ba56..11668176e9 100644 --- a/sim/common/simobject_functional.h +++ b/sim/common/simobject_functional.h @@ -103,6 +103,7 @@ class SimObjectBase { std::string name_; bool tick_active_ = true; uint32_t domain_ = 0; + bool launch_kept_ = false; // built under a LaunchKeepScope virtual void do_reset() = 0; virtual void do_tick() = 0; friend class SimPlatform; @@ -276,6 +277,7 @@ class SimPlatform { uint32_t alloc_domain() { return num_domains_++; } uint32_t num_domains() const { return num_domains_; } uint32_t build_domain() const { return build_domain_; } + bool build_launch_kept() const { return build_launch_kept_; } // Cap the parallel thread count (must be called before the first tick). // A limit of 1 forces serial execution — required while the configuration @@ -291,6 +293,25 @@ class SimPlatform { req_workers_ = std::max(1u, num_workers); } + // Objects built under a LaunchKeepScope (the last-level cache and the + // memory below it, when the command processor writes into that cache) + // carry their state and in-flight work across reset_launch(). + class LaunchKeepScope { + public: + explicit LaunchKeepScope(bool keep) { + auto& p = SimPlatform::instance(); + saved_ = p.build_launch_kept_; + p.build_launch_kept_ = saved_ || keep; + } + ~LaunchKeepScope() { + SimPlatform::instance().build_launch_kept_ = saved_; + } + LaunchKeepScope(const LaunchKeepScope&) = delete; + LaunchKeepScope& operator=(const LaunchKeepScope&) = delete; + private: + bool saved_; + }; + class DomainScope { public: explicit DomainScope(uint32_t domain) { @@ -342,6 +363,7 @@ class SimPlatform { void schedule(Func&& func, const Pkt& pkt, uint64_t delay, uint32_t domain); void reset(); + void reset_launch(); void tick(); private: @@ -408,6 +430,7 @@ class SimPlatform { uint32_t num_domains_ = 1; uint32_t build_domain_ = 0; + bool build_launch_kept_ = false; static inline thread_local uint32_t tl_exec_domain_ = HOST_DOMAIN; static inline thread_local SimObjectBase* tl_exec_object_ = nullptr; static inline thread_local uint64_t tl_cross_seq_ = 0; @@ -1171,6 +1194,22 @@ inline void SimPlatform::validate_topology() { } } +// A kernel launch resets every object but those built under a +// LaunchKeepScope: the hardware does not reset between a transfer and the +// launch that follows it, so the transfer's line fills may still be in +// flight in the kept memory side when the kernel starts. Their events stay +// scheduled and time keeps counting; the rest of the model is quiescent. +inline void SimPlatform::reset_launch() { + for (auto* object : active_reset_) { + if (!object->launch_kept_) { + object->do_reset(); + } + } + for (auto& object : objects_) { + object->tick_active_ = true; + } +} + inline void SimPlatform::reset() { this->validate_topology(); @@ -1348,7 +1387,8 @@ void SimChannelEvent::fire() { inline SimObjectBase::SimObjectBase(const SimContext&, const std::string& name) : name_(name) - , domain_(SimPlatform::instance().build_domain()) {} + , domain_(SimPlatform::instance().build_domain()) + , launch_kept_(SimPlatform::instance().build_launch_kept()) {} inline SimChannelBase::SimChannelBase(SimObjectBase* module) : module_(module) diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 0f409b8f0e..8ecebf429d 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -73,33 +73,38 @@ class Cluster::Impl { snprintf(sname, 100, "%s-l2cache", name.c_str()); // L2 is the LLC iff L2 is enabled and L3 is not. - l2cache_ = Cache::Create(sname, Cache::Config{ - !VX_CFG_L2_ENABLED, - log2ceil(VX_CFG_L2_SIZE),// C - log2ceil(VX_CFG_L2_LINE_SIZE),// L - log2ceil(VX_CFG_L2_SECTOR_SIZE),// S - log2ceil(VX_CFG_L1_LINE_SIZE), // W - log2ceil(VX_CFG_L2_NUM_WAYS), // A - log2ceil(VX_CFG_L2_NUM_BANKS), // B - VX_CFG_XLEN, // address bits - L2_CACHE_REQS, // request size - VX_CFG_L2_MEM_PORTS, // memory ports - VX_CFG_L2_WRITEBACK, // write-back - false, // write response - VX_CFG_L2_MSHR_SIZE, // mshr size - VX_CFG_L2_LATENCY, // pipeline latency - VX_CFG_L2_REPL_POLICY, // replacement policy - (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc - true, // non-cacheable bypass - (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash - L2_SHARED_LLC, // shared_llc - VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch - true, // core-response register (behind the bypass layer) - 0, // read_merge - VX_CFG_L2_MSHR_LINES, // mshr_lines - CP_ON_L2 != 0, // write_around - VX_CFG_L2_MRSQ_SIZE != 0, // memory-response queue - }); + { + // the CP writes into this cache when it is the last level: its state + // and a transfer's fills carry across the launch reset + SimPlatform::LaunchKeepScope keep(CP_ON_L2 != 0); + l2cache_ = Cache::Create(sname, Cache::Config{ + !VX_CFG_L2_ENABLED, + log2ceil(VX_CFG_L2_SIZE),// C + log2ceil(VX_CFG_L2_LINE_SIZE),// L + log2ceil(VX_CFG_L2_SECTOR_SIZE),// S + log2ceil(VX_CFG_L1_LINE_SIZE), // W + log2ceil(VX_CFG_L2_NUM_WAYS), // A + log2ceil(VX_CFG_L2_NUM_BANKS), // B + VX_CFG_XLEN, // address bits + L2_CACHE_REQS, // request size + VX_CFG_L2_MEM_PORTS, // memory ports + VX_CFG_L2_WRITEBACK, // write-back + false, // write response + VX_CFG_L2_MSHR_SIZE, // mshr size + VX_CFG_L2_LATENCY, // pipeline latency + VX_CFG_L2_REPL_POLICY, // replacement policy + (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + true, // non-cacheable bypass + (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash + L2_SHARED_LLC, // shared_llc + VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) + 0, // read_merge + VX_CFG_L2_MSHR_LINES, // mshr_lines + CP_ON_L2 != 0, // write_around + VX_CFG_L2_MRSQ_SIZE != 0, // memory-response queue + }); + } // connect l2cache memory interface diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index b015d165ca..7f74d987af 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -50,10 +50,14 @@ ProcessorImpl::ProcessorImpl() constexpr uint32_t total_cores = VX_CFG_NUM_CLUSTERS * NUM_SOCKETS * VX_CFG_SOCKET_SIZE; kmu_ = Kmu::Create("kmu", total_cores); - // create memory simulator + // create memory simulator; with the CP writing into the last-level cache, + // the memory side carries a transfer's fills across the launch reset auto dram_config = DramSim::platform_config(); dram_config.block_size = VX_CFG_MEM_BLOCK_SIZE; - memsim_ = Memory::Create("dram", Memory::Config{VX_CFG_L3_MEM_PORTS, dram_config}); + { + SimPlatform::LaunchKeepScope keep(CP_ON_L2 != 0 || CP_ON_L3 != 0); + memsim_ = Memory::Create("dram", Memory::Config{VX_CFG_L3_MEM_PORTS, dram_config}); + } char sname[100]; @@ -95,34 +99,37 @@ ProcessorImpl::ProcessorImpl() // create L3 cache; when L3 is enabled it is the LLC, otherwise it is a // transparent bypass arbiter and the L2 (or L1) is the LLC. - l3cache_ = Cache::Create("l3cache", Cache::Config{ - !VX_CFG_L3_ENABLED, - log2ceil(VX_CFG_L3_SIZE), // C - log2ceil(VX_CFG_L3_LINE_SIZE), // L - log2ceil(VX_CFG_L3_SECTOR_SIZE), // S - log2ceil(VX_CFG_L2_SECTOR_SIZE), // W - log2ceil(VX_CFG_L3_NUM_WAYS), // A - log2ceil(VX_CFG_L3_NUM_BANKS), // B - VX_CFG_XLEN, // address bits - L3_CACHE_REQS, // request size - VX_CFG_L3_MEM_PORTS, // memory ports - VX_CFG_L3_WRITEBACK, // write-back - false, // write response - VX_CFG_L3_MSHR_SIZE, // mshr size - VX_CFG_L3_LATENCY, // pipeline latency - VX_CFG_L3_REPL_POLICY, // replacement policy - VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC - true, // non-cacheable bypass - false, // bank_hash - L3_SHARED_LLC, // shared_llc - VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch - true, // core-response register (behind the bypass layer) - 0, // read_merge - VX_CFG_L3_MSHR_LINES, // mshr_lines - CP_ON_L3 != 0, // write_around - VX_CFG_L3_MRSQ_SIZE != 0, // memory-response queue - } - ); + { + SimPlatform::LaunchKeepScope keep(CP_ON_L2 != 0 || CP_ON_L3 != 0); + l3cache_ = Cache::Create("l3cache", Cache::Config{ + !VX_CFG_L3_ENABLED, + log2ceil(VX_CFG_L3_SIZE), // C + log2ceil(VX_CFG_L3_LINE_SIZE), // L + log2ceil(VX_CFG_L3_SECTOR_SIZE), // S + log2ceil(VX_CFG_L2_SECTOR_SIZE), // W + log2ceil(VX_CFG_L3_NUM_WAYS), // A + log2ceil(VX_CFG_L3_NUM_BANKS), // B + VX_CFG_XLEN, // address bits + L3_CACHE_REQS, // request size + VX_CFG_L3_MEM_PORTS, // memory ports + VX_CFG_L3_WRITEBACK, // write-back + false, // write response + VX_CFG_L3_MSHR_SIZE, // mshr size + VX_CFG_L3_LATENCY, // pipeline latency + VX_CFG_L3_REPL_POLICY, // replacement policy + VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC + true, // non-cacheable bypass + false, // bank_hash + L3_SHARED_LLC, // shared_llc + VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) + 0, // read_merge + VX_CFG_L3_MSHR_LINES, // mshr_lines + CP_ON_L3 != 0, // write_around + VX_CFG_L3_MRSQ_SIZE != 0, // memory-response queue + } + ); + } #if VX_CFG_EXT_A_ENABLED // Build-time invariant: every cache above the LLC must be write-through. @@ -175,6 +182,7 @@ ProcessorImpl::ProcessorImpl() // The CP DMA's client slot on the shared last-level cache. if constexpr (CP_ON_L2 != 0 || CP_ON_L3 != 0) { + SimPlatform::LaunchKeepScope keep(true); cp_llc_port_ = CpLlcPort::Create("cp-llc-port"); if constexpr (CP_ON_L2 != 0) { cp_llc_port_->MemReqOut.bind(&clusters_.at(0)->cp_mem_req_in()); @@ -301,8 +309,7 @@ void ProcessorImpl::flush_caches(bool l1_only) { } int ProcessorImpl::run() { - this->cp_mem_fence(); - this->reset(); + this->launch_reset(); kmu_->start(); this->forward_delegated_launch(); @@ -373,19 +380,27 @@ void ProcessorImpl::cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* this->llc_keep_lines(); } -void ProcessorImpl::cp_mem_fence() { +void ProcessorImpl::launch_reset() { if (!cp_llc_port_) { + this->reset(); return; } + // The launch fence follows the CP's DMA writes, and the kernel starts once + // the last-level cache has taken them. A write that missed still has its + // line fill in flight, as on the hardware, so the memory side keeps its + // state across the reset and the kernel meets those fills. if (cp_llc_port_->writes_pending()) { cp_llc_port_->fence(); } - // The launch resets the model, so a transfer's fills land first: memory - // holds a read in flight without a channel packet. - while (!cp_llc_port_->idle() || !SimPlatform::instance().idle() || perf_mem_pending_reads_ != 0) { + while (!cp_llc_port_->idle()) { SimPlatform::instance().tick(); } - this->llc_keep_lines(); + SimPlatform::instance().reset_launch(); + platform_reset_ = true; + perf_mem_reads_ = 0; + perf_mem_writes_ = 0; + perf_mem_latency_ = 0; + is_cycle_initialized_ = false; } void ProcessorImpl::llc_keep_lines() { @@ -427,10 +442,10 @@ void ProcessorImpl::reset() { bool ProcessorImpl::cycle() { // Lazy first-call init mirrors run()'s top-of-loop sequence so the // external driver doesn't need to choreograph reset + kmu start - // separately. reset() clears is_cycle_initialized_ so a back-to-back + // separately. A reset clears is_cycle_initialized_ so a back-to-back // kernel launch re-dispatches. if (!is_cycle_initialized_) { - this->reset(); + this->launch_reset(); kmu_->start(); this->forward_delegated_launch(); is_cycle_initialized_ = true; diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index 052d06f8f4..f2ae7212db 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -92,8 +92,9 @@ class ProcessorImpl { // read returned; the last-level cache keeps what it took into the next launch. void cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* dst, uint64_t size, bool noalloc); - // A launch that follows DMA writes waits for the port's fence. - void cp_mem_fence(); + // Reset the model for a launch. After DMA writes it waits for the port's + // fence and keeps the memory side, whose line fills may still be in flight. + void launch_reset(); void llc_keep_lines(); diff --git a/sim/simx/types.h b/sim/simx/types.h index b18936dbd3..75acbf2080 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -1637,7 +1637,7 @@ class TxCrossBar : public SimObject> { , lg2_outputs_(log2ceil(num_outputs)) , output_sel_(output_sel) , collisions_(0) { - assert(num_inputs <= 64); + // per-output round-robin pointers only: no bound on the input count assert(num_outputs <= 64); assert(ispow2(num_outputs)); assert(output_sel != nullptr); From b6717056f725de2bd76bdf9613f1d164f5d6a656 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:03 -0700 Subject: [PATCH 115/336] ci: re-record the perf_gate baselines for the MLP changes Regenerated with pytest ci -m perf_gate --update-baselines on the CI toolchain (LLVM v3.0.1) after the MLP series. Every case's config hash moves with the new defaults; retired instruction counts are unchanged. Cycles move with the miss-path, fence, lazy-fetch and tensor-core changes in the preceding commits: the 49 cases are 3.4% faster than master at the geometric mean, led by sgemv-nt16 (-44%), fp16-mc-nt16 (-8.6%) and stencil3d-nt16 (-3.1%). The same tree passed model_parity and the regression suites. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 64 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 42 ++++++++++---------- ci/baselines/perf/graphics.json | 36 ++++++++--------- ci/baselines/perf/raytracing.json | 12 +++--- ci/baselines/perf/tensor.json | 20 +++++----- ci/baselines/perf/tensor_mx.json | 4 +- ci/baselines/perf/tensor_sp.json | 4 +- ci/baselines/perf/tensor_wg.json | 12 +++--- 8 files changed, 97 insertions(+), 97 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index c9c7dc8a7f..5c239ea9bc 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 11185799, + "cycles": 11182301, "instrs": 2241925 }, "app": "raycast", "args": "", - "config_hash": "ef7e97261abf3c7e", + "config_hash": "969509e046ab06ca", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 21106013, + "cycles": 21106989, "instrs": 5664693 }, "app": "raycast", "args": "", - "config_hash": "4a346b970c5d5f4e", + "config_hash": "9fde6e6f7f37e2d4", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 632919, + "cycles": 635244, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "12eb01a0cf320bec", + "config_hash": "d063e6340ecd4137", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2251795, + "cycles": 2204469, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "a5cc1a944ac4fc92", + "config_hash": "15237d49dceccaee", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1665973, + "cycles": 1661746, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "dcf77cabb080bf39", + "config_hash": "aa47b9962f16edb2", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 5943507, + "cycles": 5942823, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "9b7d85b091d9be08", + "config_hash": "5ad71cacfd524b08", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1225395, + "cycles": 1224642, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "1181ce2dd247a573", + "config_hash": "22fd9b73cb7d2ba8", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4218769, + "cycles": 4217077, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "b8d3bbe9aaa587a9", + "config_hash": "a14ffd5b6f36fa72", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 408992, + "cycles": 381483, "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "a2e93b1416c1222b", + "config_hash": "df3b981f27e8e320", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 883260, + "cycles": 866110, "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "212b79b7b9e0df8d", + "config_hash": "3297af456dd6427a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1270578, + "cycles": 1270624, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "4a980cd3436e0491", + "config_hash": "558d5f9fda5e3deb", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4860082, + "cycles": 4860368, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "3d84ea77567e034e", + "config_hash": "79a0a9910744e8d5", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2283980, + "cycles": 2235935, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "5b78acb9a50222ff", + "config_hash": "461212f296e5def6", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4185784, + "cycles": 4180121, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "dc41c57ce7819a59", + "config_hash": "0556d0ca4d4eee68", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 95240, + "cycles": 96002, "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "652eaf7c4099617d", + "config_hash": "36eb32e6e90447f5", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 368717, + "cycles": 368700, "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "c602c9c37a145297", + "config_hash": "36ba610d0e7114cb", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index ec5e66786f..35d5419be0 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -1,23 +1,23 @@ { "dxa:perf_gate-copy-mcast:rtlsim": { "32": { - "cycles": 3381, + "cycles": 3389, "instrs": 961 }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "28dc0787284f20c8", + "config_hash": "c269fb880dec8b2c", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 1953, + "cycles": 1961, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "912948e13624b9a4", + "config_hash": "ae2903a3d445d89b", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,95 +28,95 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "277ced0b15aed8d9", + "config_hash": "3bf4cf26dbfc656b", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14016579, + "cycles": 14016558, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "f247e0e4b7829023", + "config_hash": "662e5c2b0c7b8fc1", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14051140, + "cycles": 14051366, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "3660e26ec6c4ba10", + "config_hash": "d6b2042d190ac5ce", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3947991, + "cycles": 3935575, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "55af78d744fa9e46", + "config_hash": "f292acf98eaaf8e3", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14352123, + "cycles": 14351025, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "8c7ee15d08de3277", + "config_hash": "604991b30fdc7c3a", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 78840, + "cycles": 78350, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "d27df3466a8eed79", + "config_hash": "2cc358de1155b70b", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 358897, + "cycles": 358377, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "776a839834d35db8", + "config_hash": "93e2a7ee4a7b218d", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 29487, + "cycles": 29643, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "56519748fbd3c39b", + "config_hash": "9106c858ba91e5ae", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 48573, + "cycles": 48722, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "f1664e1fcc03f8a6", + "config_hash": "2091dd383d7ad57e", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 4d13a12aaf..32a26f3bb2 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,100 +1,100 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 529816, + "cycles": 529951, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "9b91ab9ac3aa5103", + "config_hash": "b05e6b5e6b9ffc06", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 141632, + "cycles": 141529, "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "1f2de39a01c16b67", + "config_hash": "6e98bbba0a7822d3", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 529359, + "cycles": 529326, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "73f771f2643f6527", + "config_hash": "208358458841bcde", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 189979, + "cycles": 189991, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "132840fd850713c9", + "config_hash": "9d0a1a59914e4ff0", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 740701, + "cycles": 740707, "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "d8e520e04c9979f4", + "config_hash": "bbaf6fac9da36d0f", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14487, + "cycles": 14481, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "e2f119110f028280", + "config_hash": "6f2b1e9c2df8bb37", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55553, + "cycles": 55551, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "5109e566c9637f1b", + "config_hash": "e3036cf5282b4118", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 35254, + "cycles": 35278, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "3876028c632f011b", + "config_hash": "82f630774395560e", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 136492, + "cycles": 136502, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "fe367b2efd48140d", + "config_hash": "3a44bdd6d3900bb7", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 71ef9f1d16..ae04bd9c87 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,34 +1,34 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 85040, + "cycles": 85069, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "c0bd39f0f980f01d", + "config_hash": "68b742a1457ab4a3", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 52906, + "cycles": 52944, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "b899d2a47bcc51ae", + "config_hash": "53b9672f25769532", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt4:rtlsim": { "32": { - "cycles": 159655, + "cycles": 159677, "instrs": 39330 }, "app": "rt_raycast", "args": "", - "config_hash": "47b882de7074c1db", + "config_hash": "afe227c6459fe67e", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 8ce4e9b7b0..6678ef084d 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 197355, + "cycles": 196204, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "b80548e5fa6b84ae", + "config_hash": "e7b2b07afe27a336", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 36479, + "cycles": 38764, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "2935e28e509b5699", + "config_hash": "60e7762f69f5cccf", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 222420, + "cycles": 221653, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "d6ccf2e6532a7a88", + "config_hash": "a70b10177d28a4d9", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 254124, + "cycles": 252792, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "9c339e61cd76a65c", + "config_hash": "55ac763cadf62490", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 593588, + "cycles": 594162, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "c27f67158e4b632a", + "config_hash": "d9980e4be1d6630d", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 3b8f55c440..5d8ca88fab 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 76410, + "cycles": 76658, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "63975cb90eb9477f", + "config_hash": "433197d2eaf189c5", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index bf7540cbf3..257076acad 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 181992, + "cycles": 183077, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "3bb64bb8ac3c1a95", + "config_hash": "200d2f36d7530327", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 2047a4ef2a..00f45bea82 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 158197, + "cycles": 158241, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "6e007939f3dbec76", + "config_hash": "9a317f76414111c9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 834916, + "cycles": 838095, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "76bcef5705d497dc", + "config_hash": "a747dcd3d6a89d6b", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 406430, + "cycles": 399994, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "d983c017c55aa5a7", + "config_hash": "1e905ff223e20367", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From c08c24521991dfe7d4d9430b4954a2253e6b899d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:04 -0700 Subject: [PATCH 116/336] docs: add the memory-level parallelism design document docs/designs/mlp_optimizations.md describes the memory-level parallelism features of the cache, memory, DMA and LSU paths: what each one fixes, how it works, the cycles it saves on the measured shapes, and the VX_CFG knobs that control it. Each feature has a diagram under docs/assets/img/mlp_*.svg. The index links it under the memory-system designs. Co-Authored-By: Claude Opus 5.5 --- docs/assets/img/mlp_afu_inflight.svg | 34 ++ docs/assets/img/mlp_afu_ports.svg | 57 ++++ docs/assets/img/mlp_arg_slots.svg | 36 +++ docs/assets/img/mlp_cp_dma_llc.svg | 46 +++ docs/assets/img/mlp_fence_split.svg | 40 +++ docs/assets/img/mlp_fences.svg | 41 +++ docs/assets/img/mlp_fill_forward.svg | 45 +++ docs/assets/img/mlp_fill_staging.svg | 40 +++ docs/assets/img/mlp_flush_tokens.svg | 45 +++ docs/assets/img/mlp_hashed_banks.svg | 53 +++ docs/assets/img/mlp_icache_merge.svg | 54 ++++ docs/assets/img/mlp_launch_flush.svg | 30 ++ docs/assets/img/mlp_lazy_fetch.svg | 48 +++ docs/assets/img/mlp_llc_atomics.svg | 33 ++ docs/assets/img/mlp_lmem_round_robin.svg | 39 +++ docs/assets/img/mlp_lsu_paths.svg | 42 +++ docs/assets/img/mlp_mshr_line_table.svg | 51 +++ docs/assets/img/mlp_read_merge.svg | 54 ++++ docs/assets/img/mlp_sectoring.svg | 46 +++ docs/assets/img/mlp_wt_order.svg | 41 +++ docs/designs/mlp_optimizations.md | 389 +++++++++++++++++++++++ docs/index.md | 1 + 22 files changed, 1265 insertions(+) create mode 100644 docs/assets/img/mlp_afu_inflight.svg create mode 100644 docs/assets/img/mlp_afu_ports.svg create mode 100644 docs/assets/img/mlp_arg_slots.svg create mode 100644 docs/assets/img/mlp_cp_dma_llc.svg create mode 100644 docs/assets/img/mlp_fence_split.svg create mode 100644 docs/assets/img/mlp_fences.svg create mode 100644 docs/assets/img/mlp_fill_forward.svg create mode 100644 docs/assets/img/mlp_fill_staging.svg create mode 100644 docs/assets/img/mlp_flush_tokens.svg create mode 100644 docs/assets/img/mlp_hashed_banks.svg create mode 100644 docs/assets/img/mlp_icache_merge.svg create mode 100644 docs/assets/img/mlp_launch_flush.svg create mode 100644 docs/assets/img/mlp_lazy_fetch.svg create mode 100644 docs/assets/img/mlp_llc_atomics.svg create mode 100644 docs/assets/img/mlp_lmem_round_robin.svg create mode 100644 docs/assets/img/mlp_lsu_paths.svg create mode 100644 docs/assets/img/mlp_mshr_line_table.svg create mode 100644 docs/assets/img/mlp_read_merge.svg create mode 100644 docs/assets/img/mlp_sectoring.svg create mode 100644 docs/assets/img/mlp_wt_order.svg create mode 100644 docs/designs/mlp_optimizations.md diff --git a/docs/assets/img/mlp_afu_inflight.svg b/docs/assets/img/mlp_afu_inflight.svg new file mode 100644 index 0000000000..ffc14da73f --- /dev/null +++ b/docs/assets/img/mlp_afu_inflight.svg @@ -0,0 +1,34 @@ + + Before: each request waits a full 100-cycle round trip before the next is sent. After: four requests overlap and finish in about the time of one round trip plus three cycles. + + + + + Before: one in flight + R1 · 100 cyc + R2 + R3 + R4 + ≈ 400 cyc + After: many in flight + + R1–R4 overlap: ≈ 104 cyc + time → + + diff --git a/docs/assets/img/mlp_afu_ports.svg b/docs/assets/img/mlp_afu_ports.svg new file mode 100644 index 0000000000..1a51bb3d18 --- /dev/null +++ b/docs/assets/img/mlp_afu_ports.svg @@ -0,0 +1,57 @@ + + The AFU now has one AXI port per GPU memory port, each mapped to its own HBM group, and a write's data now leaves in the same cycle as its address. + + + +Before: fixed per board +After: follows the GPU + +GPU port 0 +GPU port 1 +GPU port 2 +GPU port 3 +1 AXI port +HBM + + + +GPU port 0 +GPU port 1 +GPU port 2 +GPU port 3 +AXI 0 +AXI 1 +AXI 2 +AXI 3 +HBM group 0 +HBM group 1 +HBM group 2 +HBM group 3 + + +Write fall-through +before +AW +W +2 cycles per write +after +AW + W +1 cycle per write + + diff --git a/docs/assets/img/mlp_arg_slots.svg b/docs/assets/img/mlp_arg_slots.svg new file mode 100644 index 0000000000..a770b2c019 --- /dev/null +++ b/docs/assets/img/mlp_arg_slots.svg @@ -0,0 +1,36 @@ + + Before, uploading a launch's arguments forced a full LLC flush; now arguments go into a slot no kernel has read since the last full flush, so the launch follows right away. + + + + +Before +upload args +full LLC flush +launch + + +After +upload args to a clean slot +launch + +no kernel has read this slot +since the last full flush + + diff --git a/docs/assets/img/mlp_cp_dma_llc.svg b/docs/assets/img/mlp_cp_dma_llc.svg new file mode 100644 index 0000000000..58c436f523 --- /dev/null +++ b/docs/assets/img/mlp_cp_dma_llc.svg @@ -0,0 +1,46 @@ + + Before: CP DMA writes DRAM directly, leaving stale lines in L2 that force a full LLC walk. After: CP DMA enters L2 on its own client slot, so only the L1s are flushed. + + + +Before: DMA beside the LLC + After: DMA as an LLC client + + host + CP DMA + core L1s + L2: stale lines + DRAM + + + writes DRAM + + needs full walk + + host + CP DMA + core L1s + L2: coherent + DRAM + + L1-only flush + client slot + + + diff --git a/docs/assets/img/mlp_fence_split.svg b/docs/assets/img/mlp_fence_split.svg new file mode 100644 index 0000000000..2027ea9bf6 --- /dev/null +++ b/docs/assets/img/mlp_fence_split.svg @@ -0,0 +1,40 @@ + + An acquire only invalidates afterward, a release only drains beforehand, and only a full fence does both. + + + + +Acquire +Release +Full +fence r,rw · amo.aq +fence rw,w · amo.rl +fence rw,rw · amo.aqrl +before +nothing +drain writes, no walk +drain writes, no walk +after +invalidate L1 +nothing +invalidate L1 +No L1 action at all where the dcache is itself the coherence point +(one socket, one dcache, no other memory client) + + diff --git a/docs/assets/img/mlp_fences.svg b/docs/assets/img/mlp_fences.svg new file mode 100644 index 0000000000..5c4b1d5056 --- /dev/null +++ b/docs/assets/img/mlp_fences.svg @@ -0,0 +1,41 @@ + + Before: a fence token walks every line in L1 and in the L2 and writes back to DRAM. After: L1 writes back to L2, and the shared L2 acknowledges the token without a walk; DRAM is not touched. + + + +Before: full flush per fence + After: L1-only token + + core: fence + L1: walk all lines + L2 (LLC): walk all lines + DRAM: write-backs + token + token + dirty + token + dirty + + core: fence + L1: write back dirty + L2 (LLC): ack, no walk + DRAM: not touched + L1-only token + dirty lines + ack + + diff --git a/docs/assets/img/mlp_fill_forward.svg b/docs/assets/img/mlp_fill_forward.svg new file mode 100644 index 0000000000..1923b1c975 --- /dev/null +++ b/docs/assets/img/mlp_fill_forward.svg @@ -0,0 +1,45 @@ + + Before, every waiting request replayed through the bank pipeline after a fill. With fill forwarding, the waiting reads take their data from the staged fill, and only the first write replays. + + + +Before: every waiter replays +After: reads forward from the fill + +fill: line L +ld L+0 (waiting) +ld L+4 (waiting) +st L+8 (waiting) +bank pipeline3 replay passes + + + +new requests wait + +fill: line L +ld L+0: done +ld L+4: done +st L+8 (waiting) + + +forward +bank pipeline1 replay, keeps order + + + diff --git a/docs/assets/img/mlp_fill_staging.svg b/docs/assets/img/mlp_fill_staging.svg new file mode 100644 index 0000000000..ac92df36e8 --- /dev/null +++ b/docs/assets/img/mlp_fill_staging.svg @@ -0,0 +1,40 @@ + + Before, a bank took a new fill every other cycle; now fills enter back to back, one per cycle. + + + + +Before +fill 1 +busy +fill 2 +busy +fill 3 +busy +fill 4 +4 fills in 7 cycles +After +fill 1 +fill 2 +fill 3 +fill 4 +4 fills in 4 cycles +cycles → + + diff --git a/docs/assets/img/mlp_flush_tokens.svg b/docs/assets/img/mlp_flush_tokens.svg new file mode 100644 index 0000000000..7a77a0177c --- /dev/null +++ b/docs/assets/img/mlp_flush_tokens.svg @@ -0,0 +1,45 @@ + + A broadcast flush overlaps every core's flush, and a flush token on every memory port makes sure no write arrives after the next level has finished its walk. + + + +Before: cores flush in turn +After: one broadcast, all overlap + +core 0 +core 1 +core 2 +core 3 + +core 0 +core 1 +core 2 +core 3 +Before: token on port 0 only +After: a token on every port +L1 (2 memory ports) +next level: line left dirty +token +write,late +L1 (2 memory ports) +next level: answers after both +token +token + + diff --git a/docs/assets/img/mlp_hashed_banks.svg b/docs/assets/img/mlp_hashed_banks.svg new file mode 100644 index 0000000000..259a3d19a4 --- /dev/null +++ b/docs/assets/img/mlp_hashed_banks.svg @@ -0,0 +1,53 @@ + + Four requests with a power-of-two stride all queue on bank 0 with plain bank select; with the hash they spread over banks 0 to 3. + + + +Before: bank = line[1:0] + After: bank = line[1:0] XOR hash(line) + + line 0x000 + line 0x100 + line 0x200 + line 0x300 + bank 0 · 4 queued + bank 1 idle + bank 2 idle + bank 3 idle + + + + + + line 0x000 + line 0x100 + line 0x200 + line 0x300 + bank 0 + bank 1 + bank 2 + bank 3 + + + + + serialized: 4 cycles + parallel: 1 cycle + + diff --git a/docs/assets/img/mlp_icache_merge.svg b/docs/assets/img/mlp_icache_merge.svg new file mode 100644 index 0000000000..da923d6298 --- /dev/null +++ b/docs/assets/img/mlp_icache_merge.svg @@ -0,0 +1,54 @@ + + Before: four fetch slices request the same icache line and are served one per cycle. After: one access returns word 0 to 3 to slices 0 to 3 in the same cycle. + + + +Before: slices take turns + After: one access, a word per slice + + slice 0: line L + slice 1: line L + slice 2: line L + slice 3: line L + icache + + + + + S0 + S1 + S2 + S3 + 4 cycles + + slice 0: line L + slice 1: line L + slice 2: line L + slice 3: line L + icache + + + + + + w0w1w2w3 + S0–3 + 1 cycle + + diff --git a/docs/assets/img/mlp_launch_flush.svg b/docs/assets/img/mlp_launch_flush.svg new file mode 100644 index 0000000000..5d10af1f49 --- /dev/null +++ b/docs/assets/img/mlp_launch_flush.svg @@ -0,0 +1,30 @@ + + Flush cycles per launch, drawn to scale: a full flush walks the L2; an L1-only flush skips it. + + + + +1 core, full flush422 +1 core, L1-only152 +4 cores + L2, full flush2,750 +4 cores + L2, L1-only180 + +02,750 cycles per launch + + diff --git a/docs/assets/img/mlp_lazy_fetch.svg b/docs/assets/img/mlp_lazy_fetch.svg new file mode 100644 index 0000000000..4a58bc1d2b --- /dev/null +++ b/docs/assets/img/mlp_lazy_fetch.svg @@ -0,0 +1,48 @@ + + Stores fill the sector's byte mask without a DRAM read. When a load arrives on a fully written sector, the old design reads DRAM anyway; design G loops back inside the bank with no DRAM read. + + + + + 1 · store miss + 2 · more stores + 3 · load arrives + + sector installed, partial = 1 + no DRAM read + + byte mask full + partial still = 1 + ld → partial line + + + + + + + Without the mask check + With the mask check + refetch from DRAM + merge under 0 bytes + + mask full → loop back + clear partial, no DRAM + + + diff --git a/docs/assets/img/mlp_llc_atomics.svg b/docs/assets/img/mlp_llc_atomics.svg new file mode 100644 index 0000000000..250253c402 --- /dev/null +++ b/docs/assets/img/mlp_llc_atomics.svg @@ -0,0 +1,33 @@ + + An atomic executes once at the last-level cache, and every L1 it passes through drops its copy of the line. + + + + +core 0: amoadd X +core 1: ld X +L1: pass, drop X +L1: miss on X +L2 (LLC): read-modify-write X + +AMO + +sees new X + + diff --git a/docs/assets/img/mlp_lmem_round_robin.svg b/docs/assets/img/mlp_lmem_round_robin.svg new file mode 100644 index 0000000000..6009b161f7 --- /dev/null +++ b/docs/assets/img/mlp_lmem_round_robin.svg @@ -0,0 +1,39 @@ + + Before: DXA holds the LMEM port for about 540 cycles while TCU reads wait. After: grants alternate between DXA and TCU every cycle. + + + + + Before: fixed priority + DXA writes + holds the port ~540 cycles + TCU reads + stalled ~510 cycles + go + After: round-robin + DXA writes + TCU reads + + + + + grants alternate + time → + + diff --git a/docs/assets/img/mlp_lsu_paths.svg b/docs/assets/img/mlp_lsu_paths.svg new file mode 100644 index 0000000000..641b348ff8 --- /dev/null +++ b/docs/assets/img/mlp_lsu_paths.svg @@ -0,0 +1,42 @@ + + Each LSU block now owns its path to its dcache ports, so the core no longer funnels every block through one shared memory path. + + + +Before: one shared path +After: a path per LSU block + +LSU block 0 +LSU block 1 +LSU block 2 +LSU block 3 +core memorypath +dcache + + + +LSU block 0 +LSU block 1 +LSU block 2 +LSU block 3 +dcacheports 0–3 + +own path, own status + + diff --git a/docs/assets/img/mlp_mshr_line_table.svg b/docs/assets/img/mlp_mshr_line_table.svg new file mode 100644 index 0000000000..0097fe146b --- /dev/null +++ b/docs/assets/img/mlp_mshr_line_table.svg @@ -0,0 +1,51 @@ + + Before: a flat MSHR where each of 16 entries stores a full line address. After: 64 small request entries point by index into a 16-entry line table of full addresses. + + + +Before: flat MSHR (16) + After: requests (64) → line table (16) + + full line addr | request + full line addr | request + full line addr | request + ⋮ + full line addr | request + 16 wide comparators + full at 16 requests + + idx 0 | request + idx 0 | request + idx 1 | request + idx 0 | request + ⋮ + idx 2 | request + 64 narrow entries + 0: line A · refs 3 + 1: line B · refs 1 + 2: line C · refs 1 + ⋮ 16 lines + + + + + + 16 wide comparators + + diff --git a/docs/assets/img/mlp_read_merge.svg b/docs/assets/img/mlp_read_merge.svg new file mode 100644 index 0000000000..fad93e4e00 --- /dev/null +++ b/docs/assets/img/mlp_read_merge.svg @@ -0,0 +1,54 @@ + + Before: four ports load word A and the bank serves them in four cycles. After: one access at cycle 1, with the reply multicast to all four ports. + + + +Before: one access per port + After: one access, multicast + + port 0: ld A + port 1: ld A + port 2: ld A + port 3: ld A + bank + + + + + P0 + P1 + P2 + P3 + 4 cycles + + port 0: ld A + port 1: ld A + port 2: ld A + port 3: ld A + bank + + + + + + P0–3 + 1 cycle + dashed: one reply to every port + + diff --git a/docs/assets/img/mlp_sectoring.svg b/docs/assets/img/mlp_sectoring.svg new file mode 100644 index 0000000000..228691552e --- /dev/null +++ b/docs/assets/img/mlp_sectoring.svg @@ -0,0 +1,46 @@ + + L2 and L3 lines carry one tag over two 64-byte sectors; dcache banks scale with the thread count. + + + +L2/L3: one tag, two sectors +Dcache: banks follow the lanes + +before +tag +64 B line +tag +64 B line +2 tags per 128 B +after +tag +sector 0 · 64 B +sector 1 · 64 B +1 tag per 128 B: half the tags +fills and evictions move one 64 B sector + +4 threads → 4 banks + +8 threads → 8 banks + +16 threads → 16 banks + +banks = line footprint / word size, line size unchanged + + diff --git a/docs/assets/img/mlp_wt_order.svg b/docs/assets/img/mlp_wt_order.svg new file mode 100644 index 0000000000..1abc739ba6 --- /dev/null +++ b/docs/assets/img/mlp_wt_order.svg @@ -0,0 +1,41 @@ + + In a write-through cache, a load to a word with an older store still waiting must wait behind that store, not hit the old data. + + + +Before: the load hits stale data +After: the load waits for the store + +read miss, line L +store X (waiting) + +load X (new) +hit: returns old X + + +read miss, line L +store X (waiting) +load X: linked behind + + +returns new X + +a waiting storenow counts asa hazard + + diff --git a/docs/designs/mlp_optimizations.md b/docs/designs/mlp_optimizations.md new file mode 100644 index 0000000000..e93fc29890 --- /dev/null +++ b/docs/designs/mlp_optimizations.md @@ -0,0 +1,389 @@ +# Memory-Level Parallelism Optimizations + +A GPU hides memory latency by keeping many memory requests in flight at once. This is memory-level parallelism (MLP). It runs from the lanes of one warp, through the caches and the command processor's DMA, out to the DRAM channels. This document describes the features that raise Vortex's MLP: +- how the cache hierarchy is organized; +- how misses are handled; +- how same-address requests are merged; +- how coherence and ordering are kept cheap; +- how the DMA engines share the memory system; +- how the FPGA shell reaches HBM. + +Each section says what problem the feature removes, how it works, and what it gains. + +## Reading the numbers + +- Gains are changes in total kernel cycles, so −10% means 10% faster. Each gain is measured against the same configuration with the feature off. +- **rtlsim** is the Verilator model of the RTL. **SimX** is the cycle-level C++ model. **xrtsim** is the RTL plus the FPGA shell. +- The GPU shapes (`a100`, `h100`, `rtx5080`, …) are hardware profiles. A profile sets cores, warps, threads, cache sizes, and the DRAM type and timing (HBM2, HBM3, GDDR6) from data-sheet values. See [simulation.md](../simulation.md#hardware-profiles). + +## Summary + +| Feature | Level | What it removes | Best measured gain | Default | +|---|---|---|---|---| +| [Sectored L2/L3](#sectored-l2-and-l3-caches) | L2/L3 | Half of the LLC tags | Tag area halved | On | +| [Word-size dcache banking](#dcache-banks-follow-the-thread-count) | L1D | Fixed bank count as threads grow | Bank count scales with threads | On | +| [Hashed bank select](#hashed-bank-select) | L1D, L2 | Power-of-two strides on one bank | sgemv −32.8% | On (L2; dcache ≥ 4 banks) | +| [Fill forwarding](#fill-forwarding) | All caches | Replaying waiters after a fill | sgemv −5.0% | On | +| [Back-to-back fills](#back-to-back-fills) | All caches | A bubble after each fill | rtx5080 stream −6.2% | On | +| [MSHR line table](#mshr-line-table) | All caches | One wide address compare per waiting request | sgemm −15.3% at 400-cycle latency | Off | +| [Write-allocate without fetch](#write-allocate-without-fetch-lazy-fetch) | Write-back caches | Reading a line only to overwrite it | DRAM reads −33% | Off | +| [Full-sector refetch skip](#full-sector-refetch-skip) | Write-back caches | Refetching a fully written sector | rtx5080 sgemm −43.5% with lazy L2 | Off (with lazy fetch) | +| [Dcache read merge](#dcache-read-merge) | L1D | Same-word loads serialized on one bank | sgemm −17.2% | On | +| [Icache same-line merge](#icache-same-line-merge) | L1I | Fetch slices serialized on one line | rtx5080 sgemm −25.8% | On | +| [Broadcast flush, per-port tokens](#broadcast-flush-and-per-port-flush-tokens) | Whole hierarchy | Serial per-core flushes; lost writes | Flush 2.76M → 16.5k cycles per launch | On | +| [L1-only launch flush](#l1-only-flush-between-launches) | Whole hierarchy | Walking the LLC at every launch | Flush 2,750 → 180 cycles per launch | On | +| [Argument upload slots](#launch-argument-slots) | Runtime + LLC | A full flush after each argument upload | multikernel −7.5% | On | +| [Targeted fences](#targeted-fences) | L1D, LLC | A full flush per fence | printf 13.6× faster | On | +| [Acquire/release fences](#acquire-and-release-fences) | L1D | Doing both halves of a fence | Ordering-correct atomics at minimal cost | On | +| [Atomics at the LLC](#atomics-at-the-last-level-cache) | L1D, LLC | Atomics racing across private caches | Correct atomics across the hierarchy | With the A extension | +| [Write-through ordering](#write-through-ordering) | L1D | A load returning data older than its own store | Correctness | On | +| [CP DMA through the LLC](#command-processor-dma-through-the-llc) | CP, LLC | An LLC walk for every host transfer | multikernel −42.7% | Off (on in GPU profiles) | +| [LMEM DMA round-robin](#local-memory-dma-round-robin) | Local memory | DXA starving tensor-core reads | WGMMA with DXA −10.2% | On | +| [Per-block LSU memory paths](#per-block-lsu-memory-paths) | Core | One shared core memory path | Scales with LSU blocks | On | +| [Many AXI transactions in flight](#many-axi-transactions-in-flight) | FPGA shell | One outstanding request per port | 4-core sgemv 1.30× | On | +| [Ports follow the GPU; write fall-through](#memory-ports-follow-the-gpu-and-write-fall-through) | FPGA shell | A fixed port count; a cycle per write | Write throughput 2× | On | + +## Configuration knobs + +Some of these features can be set with a `VX_CFG_*` knob (see [Build Configuration System](build_configuration_system.md)). The rest are always on, or are selected automatically from the cache layout. + +| Feature | Knob | Default | +|---|---|---| +| Hashed bank select | `VX_CFG_DCACHE_BANK_HASH`, `VX_CFG_L2_BANK_HASH` | dcache: on with 4 or more banks; L2: on. The L3 has no bank hash. | +| MSHR line table | `VX_CFG_{DCACHE,L2,L3}_MSHR_LINES` | 0 (off): flat MSHR | +| Write-allocate without fetch | `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH` | 0 (off). Enabling it also enables the dirty-byte masks (`*_DIRTYBYTES`). | +| Full-sector refetch skip | Part of lazy fetch | Follows `*_LAZY_FETCH` | +| Dcache read merge | `VX_CFG_DCACHE_READ_MERGE` | 1 (on) | +| Icache same-line merge | `VX_CFG_ICACHE_READ_MERGE` | 1 (on). No effect with one fetch slice. | +| CP DMA through the LLC | `VX_CFG_CP_LLC_CLIENT` | 0 (off); 1 in the GPU profiles. Takes effect only when one LLC serves every core. | +| Atomics at the LLC | `VX_CFG_EXT_A_ENABLE` | Off | +| Sectored L2/L3 | `VX_CFG_{L2,L3}_SECTOR_SIZE` (a size) | One bus granule. Setting it to the line size removes sectoring. | +| Dcache word-size banking | `VX_CFG_DCACHE_WORD_SIZE` (a size) | Derived from the lane count and XLEN | + +**Always on:** fill forwarding, back-to-back fills, broadcast flush and per-port flush tokens, launch argument slots, targeted fences, acquire/release fences, write-through ordering, local-memory DMA round-robin, per-block LSU memory paths, many AXI transactions in flight, and write fall-through. + +**Selected automatically:** +- The L1-only launch flush and the fence shortcuts apply whenever a single LLC is the point of coherence. +- The FPGA shell's memory port count follows the GPU's memory ports. + +--- + +## Cache organization + +### Sectored L2 and L3 caches + +![L2/L3 sectoring and dcache banking](../assets/img/mlp_sectoring.svg) + +**Problem.** An LLC line as small as the memory bus granule needs one tag per 64 bytes. At GPU LLC sizes (megabytes), that is a large tag array. + +**How it works.** L2 and L3 lines are twice the bus granule. One tag covers the line, and the line is split into two sectors of one bus granule each, with a valid bit per sector. Fills and evictions move one sector at a time, so memory traffic stays at bus granularity, while the tag array is half the size. Sector size is set with `VX_CFG_{L2,L3}_SECTOR_SIZE`. + +**Gain.** The LLC tag area is halved at the same capacity and the same memory traffic. + +### Dcache banks follow the thread count + +**Problem.** A dcache with a fixed bank count serializes a warp's lanes once the thread count outgrows it. + +**How it works.** The coalescer's word size sets the dcache banking: banks = line footprint / word size, and there is one bank per request lane. Wider warps get more banks without a smaller line. See the right half of the figure above. + +**Gain.** Dcache parallelism tracks `NUM_THREADS`, so a 16- or 32-thread warp is not throttled by a 4-bank cache. + +### Hashed bank select + +![Hashed bank select](../assets/img/mlp_hashed_banks.svg) + +**Problem.** A cache picks its bank from the lowest line-address bits. GPU kernels walk arrays with power-of-two strides: matrix rows, tiles, per-thread blocks. Every request in such a stride has the same low bits, so all of them queue on one bank while the others sit idle. + +**How it works.** The bank index is the low line-address bits XORed with a hash of the higher bits. The hash is an IPOLY polynomial: IPOLY(5) for 16 banks, IPOLY(37) for 32, and an XOR fold for other counts. Strided requests spread over all banks. Each line still has exactly one bank, so the rest of the cache is unchanged. It is controlled by `VX_CFG_{DCACHE,L2}_BANK_HASH`: on for the L2, and on for a dcache with 4 or more banks. Below 4 banks, the extra queuing outweighs the spreading. + +**Gain (rtlsim):** +- sgemv 512×512 at 16 threads: −32.8% +- fp16 tensor sgemm on 2 cores + L2: −13.8% +- WGMMA with DXA: −13.5% +- multi-core sgemm: −8.2% +- SimX, rtx5080, N=256: −34% + +--- + +## Miss handling + +### Fill forwarding + +![Fill forwarding](../assets/img/mlp_fill_forward.svg) + +**Problem.** When a miss returns, every request waiting on that line (the MSHR chain) used to replay through the bank pipeline one at a time. Each replay is a full pipeline pass, and new requests are blocked while the chain drains. + +**How it works.** The reads at the head of the chain take their data straight from the staged fill sector, which is fill-buffer forwarding, standard in GPU L1s. The first write or atomic in the chain, and everything after it, still replays, so same-line accesses keep program order. The input arbiter stays open to new requests while the chain completes. Always on. + +**Gain (rtlsim):** sgemm −2.7%, sgemv −5.0%, multi-core sgemm −2.2%. + +### Back-to-back fills + +![Back-to-back fills](../assets/img/mlp_fill_staging.svg) + +**Problem.** A bank could not take a fill in the cycle right after another fill. Under heavy miss traffic, fills arrived in bursts and the bank accepted only one every other cycle. + +**How it works.** Fills follow each other into the bank pipeline back to back. Each staged fill keeps its own sector until it installs, so consecutive fills don't conflict. + +**Gain:** +- rtx5080 profile, 4 MiB stream: −6.2% +- the stock configuration (shallow L2): within ±0.5% +- the cache also closed timing faster: Fmax up by 3% + +### MSHR line table + +![MSHR line table](../assets/img/mlp_mshr_line_table.svg) + +**Problem.** Each MSHR entry holds a full line address, and every new miss compares against all of them. To keep more misses in flight, the MSHR needs more entries, and each one adds a wide comparator. At high memory latency, a 16-entry MSHR was full a fifth of the time, often with only one or two distinct lines in it. + +**How it works.** Distinct {line, sector} pairs go into a small reference-counted line table. Each request entry keeps only an index into that table. The cache holds many more waiting requests (for example 64) while comparing addresses against only the line table (for example 16 entries). Controlled by `VX_CFG_{DCACHE,L2,L3}_MSHR_LINES`. + +**Gain (rtlsim at 400-cycle memory latency, MSHR 16 → 64 requests over 16 lines):** sgemm −15.3%, raycast −8.1%. Without the dcache read merge, the same sgemm gains 24.8%. At the stock latency the MSHR does not limit, the gain is under 0.5%, and the feature is off by default. + +### Write-allocate without fetch (lazy fetch) + +**Problem.** A write-back cache that misses on a store normally reads the whole line from memory, then overwrites part or all of it. Streaming writes therefore cost a read for every line they write. + +**How it works.** A store miss installs the sector at once, with a per-byte dirty mask and a *partial* bit, without reading memory. A later load or atomic on a partial sector refetches it and merges memory data only under the bytes that were never written. An eviction writes back only the dirty bytes. A refill of one sector keeps the dirty bytes of the line's other sectors. Controlled by `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH`. + +**Gain:** 1-core vecadd DRAM reads −33% and memory requests −25%. Cycles are unchanged because that kernel is latency-bound. Off by default. + +### Full-sector refetch skip + +![Full-sector refetch skip](../assets/img/mlp_lazy_fetch.svg) + +**Problem.** With lazy fetch, once stores have written every byte of a sector, the partial bit still triggered a refetch on the next load. The cache read memory only to merge it under zero unwritten bytes. On L2-heavy kernels, these useless reads made lazy fetch slower than leaving it off. + +**How it works.** When the refetch commits, the bank checks the byte mask. If the mask is full, the refetch loops back through the bank as a fill with no data, clears the partial bit, and reads nothing from memory. If the loopback queue has no room, the refetch falls back to memory as before. + +**Gain:** +- DRAM reads: 145 → 17 at the dcache and 539 → 27 at the L2, on a fully written stream. +- SimX, rtx5080 profile, 4 cores, sgemm -n192 with L2 lazy fetch: 285,006 → 161,121 cycles (−43.5%). Lazy fetch is now slightly faster than having it off. + +--- + +## Request merging + +### Dcache read merge + +![Dcache read merge](../assets/img/mlp_read_merge.svg) + +**Problem.** Uniform loads make every lane of a warp read the same word in the same cycle. Kernel arguments and the shared operand of a matrix multiply are typical. Each LSU port sent its own request to the same bank, and the bank served them one per cycle. + +**How it works.** Plain reads of the same word from the ports of one LSU block in the same cycle are served by one bank access, and the reply goes to every port that asked. Each port's order into the bank is unchanged, so loads stay ordered against stores. The merge compares request tags in the same form on every port, so it doesn't depend on how wide the tag is. Controlled by `VX_CFG_DCACHE_READ_MERGE`. + +**Gain (rtlsim sgemm -n128):** +- 16 threads with an 8-byte dcache word: −17.2% +- 2 banks: −9.8% +- 32 threads: −7.9% +- 16 threads: −6.5% +- 4 cores + L2: −6.5% + +At 16 threads, dcache reads fall 25% and bank stalls 55%. + +### Icache same-line merge + +![Icache same-line merge](../assets/img/mlp_icache_merge.svg) + +**Problem.** With several fetch slices, the front end fetches up to one instruction per slice per cycle. A dense kernel's inner loop often spans only a few cache lines (sgemm: 3). So the slices keep requesting the same line and take turns at its bank, losing 1 to 2 of 4 fetch slots every cycle even with a 100% hit rate. A warp has only one fetch in flight, so each cycle of waiting lowers its fetch rate directly. + +**How it works.** Reads of the same icache line from different fetch slices in the same cycle become one bank access. The bank returns each slice the word it asked for. This differs from the dcache merge: the slices want different words of one line, not the same word. Controlled by `VX_CFG_ICACHE_READ_MERGE`. It has no effect with one fetch slice. + +**Gain (rtlsim / SimX):** + +| Shape | sgemm | vecadd | conv3 | raycast | +|---|---|---|---|---| +| Issue and fetch width 4 | −12.1% / −12.4% | | | −4.7% / −4.7% | +| rtx5080 core | −25.8% / −20.2% | −15.7% / −20.1% | −3.4% / −4.6% | | + +With the merge, 16 icache banks are no longer slower than 4. + +--- + +## Coherence and ordering + +### Broadcast flush and per-port flush tokens + +![Broadcast flush and per-port tokens](../assets/img/mlp_flush_tokens.svg) + +**Problem:** +- The command processor flushed the cores one after another, so a launch boundary cost the sum of every core's flush. +- A cache forwarded its flush token on memory port 0 only. A write still queued on another port could reach the next level after that level had finished its walk, and stay dirty there. + +**How it works:** +- One broadcast `CACHE_FLUSH` write starts every core's flush at once. The command processor then collects the completions, so the flushes overlap. +- A cache walks its banks and sends a flush token down **every** memory port. It answers only once all ports have answered. +- Every independently buffered write path carries its own token. +- Read-only caches (icache, texture, ray-tracing caches) no longer forward flushes at all. + +**Gain:** flush cost per launch dropped from 2.76M to 16.5k cycles on the measured shape. It also fixed the lost-write case above. + +### L1-only flush between launches + +![L1-only launch flush](../assets/img/mlp_launch_flush.svg) + +**Problem.** Every kernel launch flushed the whole hierarchy, including a walk of the LLC. Back-to-back launches paid for it each time. + +**How it works.** When a single LLC serves every memory client, it is the point of coherence: launches only need the L1s cleaned. The launch flush becomes an L1-only token, which the shared LLC answers without walking. LLCs that are replicated per core or per cluster, and LLCs with peer clients beside them (texture, ray tracing, raster, output merger, DXA), still walk. A dcache counts as the LLC only when it is the single dcache of a single socket with no other memory client. The runtime posts flushes lazily and flushes fully before any device-memory transfer that needs it. The device reports busy until every core write, including uncached I/O stores, is visible in memory. + +**Gain:** flush cycles per launch 422 → 152 at 1 core and 2,750 → 180 at 4 cores + L2. Chained launches at 4 cores + L2: −11%. + +### Launch argument slots + +![Launch argument slots](../assets/img/mlp_arg_slots.svg) + +**Problem.** Each launch uploads its arguments and launch descriptors to device memory. That upload made the LLC dirty, so the next launch needed a full flush. Workloads that launch many small kernels behaved as if the L1-only flush didn't exist. + +**How it works.** Arguments and descriptors go into a pool of slots. A slot is reused only if no kernel has read it since the last full flush, so no cache can hold a stale copy and the upload needs no flush. This is the same dirty tracking AMD's driver does for its argument buffers. + +**Gain:** multikernel at 4 cores + L2: −7.5% on rtlsim, −8.5% on SimX. A workload that reads results back after every launch still needs the full flush and is unchanged. + +### Targeted fences + +![Targeted fences](../assets/img/mlp_fences.svg) + +**Problem.** A memory fence flushed the whole hierarchy: every level walked all of its lines and wrote dirty data to DRAM. `printf` fences after every device write, so printf-heavy kernels spent most of their time walking caches. + +**How it works.** +- A memory fence sends an L1-only token. The L1s write their dirty lines back to the shared LLC, and the LLC answers without walking. DRAM is not touched. +- A fence that orders only device (I/O) accesses becomes its own LSU operation. Every level drains its in-flight writes to its memory ports, with no walk anywhere. + +**Gain (rtlsim):** +- printf at 2 cores + L2: 4,006,822 → 295,165 cycles (13.6×) +- printf at the default configuration: −42.9% +- fence test at 2 cores + L2: −55% +- async global barrier: −13.3% + +### Acquire and release fences + +![Acquire and release fences](../assets/img/mlp_fence_split.svg) + +**Problem.** A fence did both halves of a full barrier, even when the program asked for only one. Atomics marked acquire or release (`.aq`/`.rl`) were not ordered at all. + +**How it works.** The fence's predecessor and successor sets choose the action: +- **Acquire** (`fence r,rw`, `amo.aq`): invalidates clean L1 lines after the operation. Nothing written back. +- **Release** (`fence rw,w`, `amo.rl`): drains pending writes before the operation, without a walk. Nothing invalidated. +- **Full** (`fence rw,rw`, `amo.aqrl`): does both. + +Atomics with `.aq`/`.rl` get these steps around them. All of it is skipped when the dcache is itself the point of coherence. + +**Gain:** correct acquire/release ordering for atomics, at the minimum cost the memory model allows. Fences, printf, global barriers and sgemm are unchanged; their kernels use full fences. A test that makes every atomic sequentially consistent pays for the ordering it now gets. + +### Atomics at the last-level cache + +![Atomics at the LLC](../assets/img/mlp_llc_atomics.svg) + +**How it works.** RISC-V `A`-extension atomics run once, as a read-modify-write at the last-level cache. Each private L1 they pass through drops its copy of the line, so the next load of that word from any core misses and sees the new value. `LR`/`SC` use a reservation table. This keeps atomics correct across L1, L2 and L3 without a coherence protocol between the L1s. Controlled by `VX_CFG_EXT_A_ENABLE`. + +### Write-through ordering + +![Write-through ordering](../assets/img/mlp_wt_order.svg) + +**Problem.** In a write-through L1 (every configuration with more than one core, or with an L2 or L3), a store can be waiting in the MSHR behind an earlier read miss to the same line. A load of the same word that arrived then could hit the cache and return the value from before the store. + +**How it works.** A waiting store counts as a hazard. A later access to the same word links behind it in the MSHR chain and sees the stored value. The same rule keeps a newer write from overtaking an older one. + +**Gain:** correctness. The ordering test fails without the rule and passes with it. + +--- + +## DMA + +### Command-processor DMA through the LLC + +![CP DMA through the LLC](../assets/img/mlp_cp_dma_llc.svg) + +**Problem.** The command processor copied host data straight into DRAM, beside the L2. The L2 could hold stale copies of those lines, so every host transfer needed a full flush and invalidate of the LLC. Launches with uploads between them paid this every time. + +**How it works:** +- When one LLC serves every core, the command processor's DMA becomes a client of it on its own slot. A page-table walker attaches the same way. +- Host data flows through the LLC and keeps it coherent, so a transfer needs only an L1-only flush. +- Transfers of at least half the LLC write around it, so they don't evict the working set. +- A launch issued after DMA writes waits for an L1-only flush token sent on the DMA's slot. +- The runtime detects the feature through the `DMA_LLC_CLIENT` capability. +- Controlled by `VX_CFG_CP_LLC_CLIENT`: off by default, on in the GPU profiles. + +**Gain (rtlsim):** + +| Shape | dogfood | multikernel | sgemm -n32 | +|---|---|---|---| +| 2 clusters + L2 + L3 | −30.0% | −42.7% | −14.1% | +| 4 cores + L2 | −9.1% | −23.4% | | +| 8 MB L3 | −64% | | | + +**Virtual memory.** SimX's command processor translates DMA addresses before they enter the LLC. The RTL command processor has no MMU, so host DMA with virtual memory is a SimX-only configuration for now. + +### Local-memory DMA round-robin + +![LMEM DMA round-robin](../assets/img/mlp_lmem_round_robin.svg) + +**Problem.** Two clients share the local-memory DMA port: the DXA engine, which writes tiles into shared memory, and the tensor core, which reads WGMMA operands from it. The DXA had fixed priority. One tiled-scatter transfer held the port for about 540 cycles, and a tensor-core batch caught behind it stalled for about 510. That stall delayed the warpgroup barrier, and the barrier delayed the next DXA request. + +**How it works.** The port grants round-robin, so a tensor-core read waits at most one DXA write. It costs one bit of state, and a single-client configuration is unchanged. + +**Gain (rtlsim):** WGMMA with DXA −10.2%, sparse WGMMA with DXA −11.6%. + +--- + +## Load/store unit + +### Per-block LSU memory paths + +![Per-block LSU memory paths](../assets/img/mlp_lsu_paths.svg) + +**Problem.** With several LSU blocks per core (one per issue slice), all of them went through one shared core memory path. Both bandwidth and floorplanning were limited by that single funnel. + +**How it works.** Each LSU block owns its path to its own dcache ports, along with its own status signals. The tensor-core blocks also get their own metadata ports into their slice's LSU. This is the processing-block organization of NVIDIA SMs: each partition has its own load/store path. + +**Gain:** cycle-identical at today's shapes. It is the structure that lets memory bandwidth scale with the number of LSU blocks, and it shortens the wires in the core. + +--- + +## FPGA memory interface + +### Many AXI transactions in flight + +![Many AXI transactions in flight](../assets/img/mlp_afu_inflight.svg) + +**Problem.** The FPGA shell limited each AXI memory port to very few outstanding transactions. By Little's law, a port's throughput is the number of requests in flight divided by latency. One request in flight at 100 cycles of latency delivers one line per 100 cycles, however much bandwidth the HBM has. + +**How it works.** The limit is gone. Requests are arbitrated with their AXI IDs, and responses are routed back by ID, so many transactions overlap their latency. A grant is held while a request is being offered, which keeps the AXI rule that a valid request stays stable. + +**Gain (xrtsim):** + +| Case | Speedup | +|---|---| +| One port, 100-cycle latency | 1.25–1.27× | +| 4-core sgemv | 1.30× | +| Several ports | 1.06–1.08× | +| Default timing | 1.00–1.13× | + +### Memory ports follow the GPU, and write fall-through + +![Memory ports and write fall-through](../assets/img/mlp_afu_ports.svg) + +**Problem.** +- The number of AXI memory ports was fixed per board, not set by the GPU configuration. A larger GPU could not use more HBM channels, and some boards left channels unmapped. +- A write's data waited a cycle behind its address. + +**How it works.** +- The shell gets one AXI port per GPU memory port, rounded up to a power of two. +- Each port owns a contiguous part of memory, mapped to its own HBM channel group, and the board connectivity is generated from the port count. +- A write's data now leaves in the same cycle as its address. Data stays in address order through a small queue. +- Reset waits for any write already in progress, so it never splits a burst. + +**Gain:** +- Single-beat writes: twice the throughput (1,000 writes in 1,000 cycles instead of 2,000). +- Small GPUs (1–4 cores) are not port-bound and see within ±1.2%. The port scaling is what lets larger GPUs use the HBM. + +--- + +## How the gains were measured + +- **DRAM model.** All simulators share one DRAM model: + - independent channels, refresh, and open-page bank interleave; + - JEDEC-style HBM2, HBM3 and GDDR6 timing from data-sheet values; + - per-channel response queues. + + The A100 profile's idle latency (216/257 core cycles open-row/conflict) matches GPGPU-Sim's SM80 model (about 215/255). +- **Model agreement.** SimX models every feature here with the same structure as the RTL, and CI checks SimX against rtlsim on each feature's shapes. So gains measured on SimX at full GPU scale reflect the hardware. Each feature has a test that fails when it is broken or turned off. diff --git a/docs/index.md b/docs/index.md index 82631b4f45..103fbc2c2b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,6 +47,7 @@ Detailed architectural specifications for Vortex subsystems live under - [Custom Accelerator ISA Extensions](designs/custom_accelerator_isa_extensions.md) - **Memory system** - [Cache Subsystem](designs/cache_subsystem.md) + - [Memory-Level Parallelism Optimizations](designs/mlp_optimizations.md) - [Atomic Memory Operations](designs/atomic_memory_operations.md) - [Multi-Cache AMO Coherence](designs/multicache_amo_coherence.md) - [Memory Fabric Attributes](designs/memory_fabric_attributes.md) From 2339de0b996b01475bf2de38a9ca79473136e330 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 06:27:41 -0700 Subject: [PATCH 117/336] syn: leave the XRT platform top out of the yosys AFU DUT The yosys top DUT synthesizes VX_afu_wrap, but its include path also pulled in hw/rtl/afu/xrt, so sv2v read the XRT platform top too. That top now spells out one AXI master per GPU memory port through AFU_MEM_PORTS, a count only the Xilinx flows resolve from the elaborated configuration, so asic_gate's top build stopped at source conversion on the undefined macro. The platform top is not part of the synthesized design: drop its directory from the include path. asic_gate -b top: 618.3 MHz against the 620.9 MHz baseline, cell area +0.5%, PASS. The other eleven asic_gate DUTs pass on the same tree. Co-Authored-By: Claude Opus 5.5 --- hw/syn/yosys/dut/catalog.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/syn/yosys/dut/catalog.mk b/hw/syn/yosys/dut/catalog.mk index d67a94c75b..6e28351a77 100644 --- a/hw/syn/yosys/dut/catalog.mk +++ b/hw/syn/yosys/dut/catalog.mk @@ -83,7 +83,7 @@ tensor_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TCU_ENA # Multi-core system top behind the AFU control/memory wrapper, as in the Xilinx # catalog: 2 cores with atomics and virtual memory. top_TOP := VX_afu_wrap -top_INC := -I$(VORTEX_HOME)/hw/rtl/cp -I$(AFU_RTL_DIR)/xrt -I$(AFU_RTL_DIR)/common +top_INC := -I$(VORTEX_HOME)/hw/rtl/cp -I$(AFU_RTL_DIR)/common top_CFG := -DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_CORES=2 \ -DVX_CFG_EXT_A_ENABLE -DVX_CFG_VM_ENABLE -DVX_CFG_DCACHE_LATENCY=3 From 1578b79cb5ae8ce71bf7634f6984b01fe39285b5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 11:04:10 -0700 Subject: [PATCH 118/336] ci: split the model_parity and perf_gate cells into hour-long shards Each cell's two shards ran about 2h25 on the CI runners, close to the tests job's timeout and the longest jobs of a run. Their cases are long rtlsim runs, so these two cells get a 60-minute shard budget instead of the default 150 minutes: both now plan four shards, balanced on the recorded runtimes (model_parity 54-58 min, perf_gate 50 min each). The shard count still comes from the planner and reaches pytest through --shard, so every shard derives the same split. Co-Authored-By: Claude Opus 5.5 --- ci/testcase.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ci/testcase.py b/ci/testcase.py index f5e6a10717..3998238844 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -61,6 +61,8 @@ # into the tests job's timeout. RUNTIMES_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtimes.json") SHARD_BUDGET_S = 150 * 60 +# Cells of long rtlsim runs split finer: each shard of these stays near an hour. +CELL_SHARD_BUDGET_S = {"model_parity": 60 * 60, "perf_gate": 60 * 60} # Weight of a case with no recorded time (new, or not yet run on that xlen). DEFAULT_CASE_S = 120 @@ -421,19 +423,19 @@ def cell_name(case): return case.check or case.category -def shard_plan(case_ids, xlen, shards=None): +def shard_plan(case_ids, xlen, shards=None, budget=SHARD_BUDGET_S): """Split a cell's cases into shards balanced on their recorded runtimes. Longest case first onto the least-loaded shard, with ties broken on case id and shard index, so the planner and every shard derive the same split from the same ids. Unless `shards` is given, the count is the fewest that keep - the cell's recorded total within SHARD_BUDGET_S per shard. + the cell's recorded total within `budget` per shard. Returns (shards, {case id: 0-based shard}). """ runtimes = load_runtimes(xlen) weight = {i: runtimes.get(i, DEFAULT_CASE_S) for i in case_ids} if shards is None: - shards = max(1, math.ceil(sum(weight.values()) / SHARD_BUDGET_S)) + shards = max(1, math.ceil(sum(weight.values()) / budget)) load = [0.0] * shards plan = {} for i in sorted(weight, key=lambda i: (-weight[i], i)): @@ -472,7 +474,7 @@ def cmd_matrix(args): # the CHECK, not on a category that happens to share its name -- a check is a # marker across suites, and no category name may control whether it runs. # - # A cell whose recorded runtime exceeds SHARD_BUDGET_S is emitted as `shards` + # A cell whose recorded runtime exceeds its budget is emitted as `shards` # cells, each running shard `shard` of the split (shard_plan). The split is # over every case the cell's -m expression selects, not only those this # event's filters planned the cell for, because that is what the cell runs. @@ -499,7 +501,8 @@ def cmd_matrix(args): out = [] for key, cell in cells.items(): cell["needs"] = sorted(cell["needs"]) - shards, _ = shard_plan(members.get(key, []), key[2]) + shards, _ = shard_plan(members.get(key, []), key[2], + budget=CELL_SHARD_BUDGET_S.get(key[0], SHARD_BUDGET_S)) if shards == 1: out.append(cell) continue From 3500f0aa9d022ae5c414527a10e5f802b360a3ad Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 12:55:56 -0700 Subject: [PATCH 119/336] simx: keep a straddling instruction's low half until decode takes it The RVC decompressor joined a 32-bit instruction that straddles a word from the buffered low half and the refetched high half, and cleared the buffer as it did. When the decode latch was full the response stayed and was offered again, but the cleared buffer sent the retry down the fresh fetch path, which read the wrong half-word: at XLEN 64, raycast with RVC decoded a c.srli in place of an fmul.s, jumped to a wild PC and spun until the CI cell's 3-hour timeout. The RTL decompressor holds its state under backpressure. The buffer is now freed when the instruction enters the decode latch, so a retry sees the same state. Decode backpressure while a response waits arises on this branch once every warp's ibuffer is full and ready warps still fetch. rvc and simx: 5/5 at XLEN 64 and 32; raycast -w 64 -h 64 with RVC at XLEN 64 passes with master's instruction count (5,676,110). Non-RVC SimX cycles are unchanged (sgemm 796,238, raycast 21,106,995). Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 3 +++ sim/simx/decompressor.cpp | 5 ++++- sim/simx/decompressor.h | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 8e89758a9b..9aed139299 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -533,6 +533,9 @@ class Core::Impl { // queues a refetch internally for cross-word 32-bit completion. bool emit = decompressor->on_icache_rsp(trace, *mem_rsp.data); bool progressed = !emit || decode_latch.try_push(trace); + if (emit && progressed) { + decompressor->on_emit(trace); + } #else bool emit = true; bool progressed = false; diff --git a/sim/simx/decompressor.cpp b/sim/simx/decompressor.cpp index 0de1b35499..2c98736fc2 100644 --- a/sim/simx/decompressor.cpp +++ b/sim/simx/decompressor.cpp @@ -444,6 +444,10 @@ void Decompressor::commit_request(bool from_refetch) { } } +void Decompressor::on_emit(const instr_trace_t* trace) { + state_.at(trace->wid) = RvcSlot{}; +} + Word Decompressor::fetch_addr(const instr_trace_t* trace) const { const auto& rvc = state_.at(trace->wid); if (rvc.needs_second) { @@ -466,7 +470,6 @@ bool Decompressor::on_icache_rsp(instr_trace_t* trace, const mem_block_t& line) // Combine: low half from buffer, high half = low16 of new word. uint16_t high = word & 0xFFFFu; trace->code = (uint32_t(high) << 16) | rvc.low_half; - rvc = RvcSlot{}; return true; } diff --git a/sim/simx/decompressor.h b/sim/simx/decompressor.h index ea5ad0508d..2aed4892d1 100644 --- a/sim/simx/decompressor.h +++ b/sim/simx/decompressor.h @@ -72,8 +72,14 @@ class Decompressor : public SimObject { // 32-bit. The decoder later detects RVC from code[1:0] and // decompresses internally; // - queues `trace` for refetch internally and returns false. + // An emitted response may be refused by a full decode latch and offered + // again: the warp's buffered low half is kept until on_emit(). bool on_icache_rsp(instr_trace_t* trace, const mem_block_t& line); + // Notify that the instruction emitted for `trace` entered the decode + // latch, which frees the warp's half-word buffer. + void on_emit(const instr_trace_t* trace); + protected: void on_reset(); friend class SimObject; From 29c250d9a71fee2a71970afdb0860c511ca9a3f2 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 13:10:18 -0700 Subject: [PATCH 120/336] core: drop the MUFU instance the master merge left in VX_sfu_unit Merging master auto-merged its VX_mufu_unit instance into VX_sfu_unit, but on this branch the SFU's processing elements live in VX_sfu_block, which already instantiates MUFU. VX_sfu_unit has none of the PE interfaces the instance names, so any build with VX_CFG_EXT_MUFU_ENABLE failed to elaborate: parity-mufu on rtlsim and the mufu app in the rtlsim regression could not build. VX_sfu_unit is back to its pre-merge form, and the mufu cases now run when VX_sfu_block changes. mufu on simx, rtlsim and xrtsim: 9/9. parity-mufu 0.01%, -nt8 0.14%, -nt16 0.15%. mufu and zfhmin regression apps pass on rtlsim. Builds without MUFU are unchanged: the removed block was entirely under VX_CFG_EXT_MUFU_ENABLE. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/mufu.yaml | 2 +- hw/rtl/core/VX_sfu_unit.sv | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/ci/testcases/mufu.yaml b/ci/testcases/mufu.yaml index c7f70873b9..42f075d0a1 100644 --- a/ci/testcases/mufu.yaml +++ b/ci/testcases/mufu.yaml @@ -8,7 +8,7 @@ defaults: - 32 - 64 tier: smoke - touches: [hw/rtl/mufu, hw/rtl/core/VX_sfu_unit.sv, sim/simx/mufu_unit.cpp, sim/simx/sfu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] + touches: [hw/rtl/mufu, hw/rtl/core/VX_sfu_unit.sv, hw/rtl/core/VX_sfu_block.sv, sim/simx/mufu_unit.cpp, sim/simx/sfu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] tests: - id: mufu-1 via: blackbox diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index 820be2b1a1..59ba4c8dc3 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -132,18 +132,6 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( ); `endif -`ifdef VX_CFG_EXT_MUFU_ENABLE - VX_mufu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-mufu", INSTANCE_ID))), - .NUM_LANES (NUM_LANES) - ) mufu_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_MUFU]), - .result_if (pe_result_if[PE_IDX_MUFU]) - ); -`endif - VX_lane_gather #( .BLOCK_SIZE (BLOCK_SIZE), .NUM_LANES (NUM_LANES), From 48f4e425510a2ad94567b4ec8bddf3421fe51e3e Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 13:46:06 -0700 Subject: [PATCH 121/336] ci: re-record the perf_gate baselines after the master merge Regenerated with pytest ci -m perf_gate --update-baselines on the merged tree (CI toolchain, LLVM v3.0.1), in four shards whose records were combined case by case. Every case's config hash changes with the configuration keys master added; retired instructions and cycles are identical to the pre-merge baselines in all 49 cases. Against master's baselines the 47 comparable cases are 3.5% faster at the geometric mean. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 32 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 22 ++++++++++----------- ci/baselines/perf/graphics.json | 18 ++++++++--------- ci/baselines/perf/raytracing.json | 6 +++--- ci/baselines/perf/tensor.json | 10 +++++----- ci/baselines/perf/tensor_mx.json | 2 +- ci/baselines/perf/tensor_sp.json | 2 +- ci/baselines/perf/tensor_wg.json | 6 +++--- 8 files changed, 49 insertions(+), 49 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 5c239ea9bc..8257cb1e42 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -6,7 +6,7 @@ }, "app": "raycast", "args": "", - "config_hash": "969509e046ab06ca", + "config_hash": "b9d9ee09370158c4", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "raycast", "args": "", - "config_hash": "9fde6e6f7f37e2d4", + "config_hash": "d0bce0794d7bfdbc", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "d063e6340ecd4137", + "config_hash": "14d91b803afc0b16", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "15237d49dceccaee", + "config_hash": "ea174630d2a4f571", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "aa47b9962f16edb2", + "config_hash": "912a20e26fc4d53b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "5ad71cacfd524b08", + "config_hash": "23fd0d2a32b65356", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "22fd9b73cb7d2ba8", + "config_hash": "0dae7e6b06c6facd", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "a14ffd5b6f36fa72", + "config_hash": "e2fe551b6ed5319a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "df3b981f27e8e320", + "config_hash": "6075eb2ef46a71f7", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -105,7 +105,7 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "3297af456dd6427a", + "config_hash": "800b8c9f7edc548a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "558d5f9fda5e3deb", + "config_hash": "3580b572151bfcde", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -127,7 +127,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "79a0a9910744e8d5", + "config_hash": "15f1137753d108a7", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -138,7 +138,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "461212f296e5def6", + "config_hash": "a6ddac00bf4be79b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -149,7 +149,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "0556d0ca4d4eee68", + "config_hash": "63352c381701eb2a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -160,7 +160,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "36eb32e6e90447f5", + "config_hash": "d14a86b1d94d9967", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -171,7 +171,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "36ba610d0e7114cb", + "config_hash": "dcca7a6e7c88ad28", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index 35d5419be0..89fa130f36 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -6,7 +6,7 @@ }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "c269fb880dec8b2c", + "config_hash": "b6b95c0822edf48b", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "ae2903a3d445d89b", + "config_hash": "a6138216544c603d", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "3bf4cf26dbfc656b", + "config_hash": "461ebf20c7b39b93", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "662e5c2b0c7b8fc1", + "config_hash": "6d3b0c78bc6bce82", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "d6b2042d190ac5ce", + "config_hash": "44732317d093efc0", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "f292acf98eaaf8e3", + "config_hash": "ad5d3a1ba925762f", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "604991b30fdc7c3a", + "config_hash": "84141dd86fe75797", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "2cc358de1155b70b", + "config_hash": "cb07e8565e928aa5", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "93e2a7ee4a7b218d", + "config_hash": "0f396d75e95e93fe", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -105,7 +105,7 @@ }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "9106c858ba91e5ae", + "config_hash": "93664ea8afb18300", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "2091dd383d7ad57e", + "config_hash": "73097ce78c7e35b3", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 32a26f3bb2..e0e39c0977 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -6,7 +6,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "b05e6b5e6b9ffc06", + "config_hash": "02ff3eb8cc7c891c", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "6e98bbba0a7822d3", + "config_hash": "33bc329a5c31e08d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "208358458841bcde", + "config_hash": "bd063408e042748e", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "9d0a1a59914e4ff0", + "config_hash": "a283b22b5fc238d4", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "bbaf6fac9da36d0f", + "config_hash": "5b99293235b7c2fa", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "6f2b1e9c2df8bb37", + "config_hash": "e6d78d84f54ee283", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "e3036cf5282b4118", + "config_hash": "15f7ab48d7b2edce", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "82f630774395560e", + "config_hash": "c808376069d5b819", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "3a44bdd6d3900bb7", + "config_hash": "9d3f7a8db252c33d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index ae04bd9c87..a9a61b8666 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -6,7 +6,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "68b742a1457ab4a3", + "config_hash": "75e25538007e3fd7", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "53b9672f25769532", + "config_hash": "4d66c19f8df23f2e", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "afe227c6459fe67e", + "config_hash": "fb0b2247eb1c36e6", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 6678ef084d..89bfffe2f7 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "e7b2b07afe27a336", + "config_hash": "51141703c93beb72", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "60e7762f69f5cccf", + "config_hash": "8afdd549e2e91df6", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "a70b10177d28a4d9", + "config_hash": "33b3e7a8d1fd702d", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "55ac763cadf62490", + "config_hash": "0371a3688be32c73", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "d9980e4be1d6630d", + "config_hash": "c410f503b55870cf", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 5d8ca88fab..c10ac69fc3 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "433197d2eaf189c5", + "config_hash": "31c2447f43720b96", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 257076acad..9e03868a83 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "200d2f36d7530327", + "config_hash": "069b267054841b9f", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 00f45bea82..690531b2c5 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "9a317f76414111c9", + "config_hash": "5a6929889f5a9768", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "a747dcd3d6a89d6b", + "config_hash": "531700427d640d57", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "1e905ff223e20367", + "config_hash": "1bece58b69db57f0", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From 40ec5f4c989aec0f8451a688a34afa6199043ee8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 14:38:40 -0700 Subject: [PATCH 122/336] core: read the RTU perf counters through the MPM DCR window The master merge brought in the RTU perf counters (#435), but the RTU class case in VX_csr_mpm was lost while resolving the branch's MPM rework. With PERF_ENABLE and the RTU on, sysmem_perf.rtu/rtcache went unread and lint failed (UNUSEDSIGNAL), which broke model_parity-rt_raycast. SimX and the runtime already read this class. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_csr_mpm.sv | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hw/rtl/core/VX_csr_mpm.sv b/hw/rtl/core/VX_csr_mpm.sv index 9105e11851..4acdd07c1c 100644 --- a/hw/rtl/core/VX_csr_mpm.sv +++ b/hw/rtl/core/VX_csr_mpm.sv @@ -235,6 +235,21 @@ module VX_csr_mpm import VX_gpu_pkg::*; #( default:; endcase end + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + `VX_DCR_MPM_CLASS_RTU: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_RTU_RAYS, dcr_data_w, sysmem_perf.rtu.rays); + `CSR_READ_64(`VX_CSR_MPM_RTU_HITS, dcr_data_w, sysmem_perf.rtu.hits); + `CSR_READ_64(`VX_CSR_MPM_RTU_BOX_TESTS, dcr_data_w, sysmem_perf.rtu.box_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_TRI_TESTS, dcr_data_w, sysmem_perf.rtu.tri_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_CALLBACKS, dcr_data_w, sysmem_perf.rtu.callbacks); + `CSR_READ_64(`VX_CSR_MPM_RTU_BUSY, dcr_data_w, sysmem_perf.rtu.busy); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_READS, dcr_data_w, sysmem_perf.rtcache.reads); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_MISS_R, dcr_data_w, sysmem_perf.rtcache.read_misses); + default:; + endcase + end `endif default:; endcase From ffd8a8666189d8950db8f8ba36808a82b8c11303 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 17:41:17 -0700 Subject: [PATCH 123/336] AGENTS: drop the proposals section Proposals are working documents that are never committed, so docs/proposals/ has no tracked files and the rule sent every agent to write untracked drafts into its own worktree, where they scattered. Where contributors keep drafts is their own workflow, not a repo rule. The documentation map keeps docs/designs/ for accepted designs. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1324ebf732..59757bf89d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,8 +34,7 @@ This is the canonical entry point for **both human contributors and AI coding ag - [CONTRIBUTING.md](CONTRIBUTING.md) — public fork/PR contribution flow - [docs/bug_fixes.md](docs/bug_fixes.md) — bug-fix discipline (root-cause vs patch) - [docs/continuous_integration.md](docs/continuous_integration.md) — CI pipeline -- [docs/proposals/](docs/proposals/) — design and migration proposals (drafts and in-progress) -- [docs/designs/](docs/designs/) — accepted designs (post-proposal, post-implementation) +- [docs/designs/](docs/designs/) — accepted designs (implemented) --- @@ -154,19 +153,11 @@ See [docs/coding_guidelines_cpp.md](docs/coding_guidelines_cpp.md) and [docs/cod --- -## 7. Proposals - -Design and migration proposals belong in [`docs/proposals/`](docs/proposals/) — never in build dirs or the repo root. A proposal captures the *why* of a non-trivial change before the code lands. - -Accepted designs that outlive their proposal phase live in [`docs/designs/`](docs/designs/). - ---- - -## 8. Living Document Policy +## 7. Living Document Policy This file is versioned alongside the code and evolves with it. Both humans and agents are expected to update it: - **Add a rule** when you discover a footgun the next contributor will hit. Include the *why*. - **Update or remove a rule** that's gone stale. A stale rule is worse than no rule. - **Don't duplicate** content from the topic docs in `docs/`. AGENTS.md links; the topic doc explains. -- **Branch-specific addenda are forbidden.** Branch context goes in a `docs/proposals/_proposal.md`, not in branch-local `AGENTS.md` variants. If the rule is general, promote it here; if it's a project-tracking detail, keep it in the proposal. +- **Branch-specific addenda are forbidden.** No branch-local `AGENTS.md` variants: if a rule is general, promote it here; project-tracking details stay out of the repo docs. From 3f25c838ca87a28318cc9c3f07181580743a7ecc Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 18:43:22 -0700 Subject: [PATCH 124/336] afu: pass the AXI memory ports to VX_afu_wrap as flat buses VX_afu_wrap took its AXI memory ports as SystemVerilog unpacked arrays, which plain Verilog cannot connect, so the platform tops (vortex_afu.v) were read as SystemVerilog. The Vitis kernel packager rejects an SystemVerilog top: it then refuses every SV interface with struct ports in the design, and no XRT bitstream could be packaged. Each AXI field now crosses into VX_afu_wrap as one flat vector with port i in slice i, and the wrapper splits it back into the per-port arrays its logic uses. The platform tops and the xrtsim shim are plain Verilog again, and the file-type overrides in the Vivado flows are gone. The port count still follows C_M_AXI_MEM_NUM_BANKS, so VX_afu_wrap builds standalone as the synthesis `top` DUT as before. Verified: xrtsim CI xrt cases 24/24; U55C configs on xrtsim (4c socket 4 + AMO + VM; 2c + TCU fp16 sparse; 1c IW4 + TCU sparse + WGMMA + DXA) pass vecadd, sgemm, amo, vm_test, sgemm_tcu(_sp), sgemm_tcu_wg_sp(_dxa) and dxa_copy; Vivado elaboration check and vortex_afu.xo packaging pass for the 4-core config (they failed before). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/afu/aved/vortex_afu.v | 6 +- hw/rtl/afu/common/VX_afu_wrap.sv | 114 ++++++++++++++++------ hw/rtl/afu/common/vortex_afu.vh | 136 ++++++++++++++++----------- hw/rtl/afu/xrt/vortex_afu.v | 6 +- hw/scripts/xilinx_elab_check.tcl | 6 -- hw/syn/xilinx/aved/package_ip.tcl | 7 -- hw/syn/xilinx/dut/project.tcl | 6 -- hw/syn/xilinx/xrt/package_kernel.tcl | 6 -- sim/xrtsim/vortex_afu_shim.sv | 6 +- 9 files changed, 175 insertions(+), 118 deletions(-) diff --git a/hw/rtl/afu/aved/vortex_afu.v b/hw/rtl/afu/aved/vortex_afu.v index 6490e304ea..54f5a5f288 100644 --- a/hw/rtl/afu/aved/vortex_afu.v +++ b/hw/rtl/afu/aved/vortex_afu.v @@ -61,8 +61,8 @@ module vortex_afu #( output wire interrupt ); - `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) - `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + `AXI_MEM_BUSES (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_BUS, MP_SEMI); VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CONTROL_ADDR_WIDTH), @@ -74,7 +74,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (~ap_rst_n), - `AXI_MEM_ARRAY_ARGS, + `AXI_MEM_BUS_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_control_awvalid), .s_axi_ctrl_awready (s_axi_control_awready), diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index 2d3940170a..208d8b4542 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -48,36 +48,36 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( input wire clk, input wire reset, - // AXI4 memory masters - output wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS], - input wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS], - output wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS], - - output wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS], - input wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS], - output wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS], - - input wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS], - output wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS], - input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS], - input wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS], - - output wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS], - input wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS], - output wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS], - - input wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS], - output wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS], - input wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS], - input wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS], - input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS], - input wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS], + // AXI4 memory masters, port i in slice i of each bus + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_awvalid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_awready_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*8-1:0] m_axi_mem_awlen_bus, + + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_wvalid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_wready_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*(C_M_AXI_MEM_DATA_WIDTH/8)-1:0] m_axi_mem_wstrb_bus, + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_wlast_bus, + + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_bvalid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_bready_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*2-1:0] m_axi_mem_bresp_bus, + + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_arvalid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_arready_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*8-1:0] m_axi_mem_arlen_bus, + + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_rvalid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_rready_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_rlast_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*2-1:0] m_axi_mem_rresp_bus, // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -113,6 +113,60 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `STATIC_ASSERT(C_M_AXI_MEM_NUM_BANKS == `AFU_MEM_PORTS_OF(VX_MEM_PORTS), ("AXI memory port count %0d does not match the GPU's %0d", C_M_AXI_MEM_NUM_BANKS, `AFU_MEM_PORTS_OF(VX_MEM_PORTS))) + wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS]; + + for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_axi_mem_bus + assign m_axi_mem_awvalid_bus[i] = m_axi_mem_awvalid_a[i]; + assign m_axi_mem_awready_a[i] = m_axi_mem_awready_bus[i]; + assign m_axi_mem_awaddr_bus[i * C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH] = m_axi_mem_awaddr_a[i]; + assign m_axi_mem_awid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_awid_a[i]; + assign m_axi_mem_awlen_bus[i * 8 +: 8] = m_axi_mem_awlen_a[i]; + assign m_axi_mem_wvalid_bus[i] = m_axi_mem_wvalid_a[i]; + assign m_axi_mem_wready_a[i] = m_axi_mem_wready_bus[i]; + assign m_axi_mem_wdata_bus[i * C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH] = m_axi_mem_wdata_a[i]; + assign m_axi_mem_wstrb_bus[i * (C_M_AXI_MEM_DATA_WIDTH/8) +: (C_M_AXI_MEM_DATA_WIDTH/8)] = m_axi_mem_wstrb_a[i]; + assign m_axi_mem_wlast_bus[i] = m_axi_mem_wlast_a[i]; + assign m_axi_mem_bvalid_a[i] = m_axi_mem_bvalid_bus[i]; + assign m_axi_mem_bready_bus[i] = m_axi_mem_bready_a[i]; + assign m_axi_mem_bid_a[i] = m_axi_mem_bid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; + assign m_axi_mem_bresp_a[i] = m_axi_mem_bresp_bus[i * 2 +: 2]; + assign m_axi_mem_arvalid_bus[i] = m_axi_mem_arvalid_a[i]; + assign m_axi_mem_arready_a[i] = m_axi_mem_arready_bus[i]; + assign m_axi_mem_araddr_bus[i * C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH] = m_axi_mem_araddr_a[i]; + assign m_axi_mem_arid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_arid_a[i]; + assign m_axi_mem_arlen_bus[i * 8 +: 8] = m_axi_mem_arlen_a[i]; + assign m_axi_mem_rvalid_a[i] = m_axi_mem_rvalid_bus[i]; + assign m_axi_mem_rready_bus[i] = m_axi_mem_rready_a[i]; + assign m_axi_mem_rdata_a[i] = m_axi_mem_rdata_bus[i * C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH]; + assign m_axi_mem_rlast_a[i] = m_axi_mem_rlast_bus[i]; + assign m_axi_mem_rid_a[i] = m_axi_mem_rid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; + assign m_axi_mem_rresp_a[i] = m_axi_mem_rresp_bus[i * 2 +: 2]; + end + reg [`VX_CFG_RESET_DELAY-1:0] vx_reset_shift_r; wire vx_reset; wire vx_start; diff --git a/hw/rtl/afu/common/vortex_afu.vh b/hw/rtl/afu/common/vortex_afu.vh index 4ff28ab33f..c9a6c5075e 100644 --- a/hw/rtl/afu/common/vortex_afu.vh +++ b/hw/rtl/afu/common/vortex_afu.vh @@ -163,6 +163,33 @@ assign m_axi_mem_bresp_a[i] = m_axi_mem_``i``_bresp; \ assign m_axi_mem_bid_a[i] = m_axi_mem_``i``_bid +`define AXI_MEM_TO_BUS(i) \ + assign m_axi_mem_``i``_awvalid = m_axi_mem_awvalid_bus[i]; \ + assign m_axi_mem_awready_bus[i] = m_axi_mem_``i``_awready; \ + assign m_axi_mem_``i``_awaddr = m_axi_mem_awaddr_bus[(i)*C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH]; \ + assign m_axi_mem_``i``_awid = m_axi_mem_awid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; \ + assign m_axi_mem_``i``_awlen = m_axi_mem_awlen_bus[(i)*8 +: 8]; \ + assign m_axi_mem_``i``_wvalid = m_axi_mem_wvalid_bus[i]; \ + assign m_axi_mem_wready_bus[i] = m_axi_mem_``i``_wready; \ + assign m_axi_mem_``i``_wdata = m_axi_mem_wdata_bus[(i)*C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH]; \ + assign m_axi_mem_``i``_wstrb = m_axi_mem_wstrb_bus[(i)*(C_M_AXI_MEM_DATA_WIDTH/8) +: (C_M_AXI_MEM_DATA_WIDTH/8)]; \ + assign m_axi_mem_``i``_wlast = m_axi_mem_wlast_bus[i]; \ + assign m_axi_mem_``i``_arvalid = m_axi_mem_arvalid_bus[i]; \ + assign m_axi_mem_arready_bus[i] = m_axi_mem_``i``_arready; \ + assign m_axi_mem_``i``_araddr = m_axi_mem_araddr_bus[(i)*C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH]; \ + assign m_axi_mem_``i``_arid = m_axi_mem_arid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; \ + assign m_axi_mem_``i``_arlen = m_axi_mem_arlen_bus[(i)*8 +: 8]; \ + assign m_axi_mem_rvalid_bus[i] = m_axi_mem_``i``_rvalid; \ + assign m_axi_mem_``i``_rready = m_axi_mem_rready_bus[i]; \ + assign m_axi_mem_rdata_bus[(i)*C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH] = m_axi_mem_``i``_rdata; \ + assign m_axi_mem_rlast_bus[i] = m_axi_mem_``i``_rlast; \ + assign m_axi_mem_rid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_``i``_rid; \ + assign m_axi_mem_rresp_bus[(i)*2 +: 2] = m_axi_mem_``i``_rresp; \ + assign m_axi_mem_bvalid_bus[i] = m_axi_mem_``i``_bvalid; \ + assign m_axi_mem_``i``_bready = m_axi_mem_bready_bus[i]; \ + assign m_axi_mem_bresp_bus[(i)*2 +: 2] = m_axi_mem_``i``_bresp; \ + assign m_axi_mem_bid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_``i``_bid + // AXI memory masters the AFU exposes: one per GPU memory port, rounded up to // a power of two so that address bits select the port, and at most one per // platform memory channel. A platform top must spell its ports out, so the @@ -170,61 +197,62 @@ // (hw/scripts/afu_mem_ports.sh) and VX_afu_wrap checks that the two agree. `define AFU_MEM_PORTS_OF(n) `MIN(1 << `CLOG2(n), `VX_CFG_PLATFORM_MEMORY_NUM_BANKS) -// Per-port arrays connecting a platform top's flat m_axi_mem_ ports to -// VX_afu_wrap (see AXI_MEM_TO_ARRAY). -`define AXI_MEM_ARRAYS(n) \ - wire m_axi_mem_awvalid_a [n]; \ - wire m_axi_mem_awready_a [n]; \ - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [n]; \ - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [n]; \ - wire [7:0] m_axi_mem_awlen_a [n]; \ - wire m_axi_mem_wvalid_a [n]; \ - wire m_axi_mem_wready_a [n]; \ - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [n]; \ - wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [n]; \ - wire m_axi_mem_wlast_a [n]; \ - wire m_axi_mem_arvalid_a [n]; \ - wire m_axi_mem_arready_a [n]; \ - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [n]; \ - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [n]; \ - wire [7:0] m_axi_mem_arlen_a [n]; \ - wire m_axi_mem_rvalid_a [n]; \ - wire m_axi_mem_rready_a [n]; \ - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [n]; \ - wire m_axi_mem_rlast_a [n]; \ - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [n]; \ - wire [1:0] m_axi_mem_rresp_a [n]; \ - wire m_axi_mem_bvalid_a [n]; \ - wire m_axi_mem_bready_a [n]; \ - wire [1:0] m_axi_mem_bresp_a [n]; \ - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [n]; +// Flat per-field buses connecting a platform top's m_axi_mem_ ports to +// VX_afu_wrap, port i in slice i. They are plain Verilog, so the platform +// tops stay Verilog, which the Vitis and AVED kernel packagers require. +`define AXI_MEM_BUSES(n) \ + wire [(n)-1:0] m_axi_mem_awvalid_bus; \ + wire [(n)-1:0] m_axi_mem_awready_bus; \ + wire [(n)*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_bus; \ + wire [(n)*8-1:0] m_axi_mem_awlen_bus; \ + wire [(n)-1:0] m_axi_mem_wvalid_bus; \ + wire [(n)-1:0] m_axi_mem_wready_bus; \ + wire [(n)*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_bus; \ + wire [(n)*(C_M_AXI_MEM_DATA_WIDTH/8)-1:0] m_axi_mem_wstrb_bus; \ + wire [(n)-1:0] m_axi_mem_wlast_bus; \ + wire [(n)-1:0] m_axi_mem_arvalid_bus; \ + wire [(n)-1:0] m_axi_mem_arready_bus; \ + wire [(n)*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_bus; \ + wire [(n)*8-1:0] m_axi_mem_arlen_bus; \ + wire [(n)-1:0] m_axi_mem_rvalid_bus; \ + wire [(n)-1:0] m_axi_mem_rready_bus; \ + wire [(n)*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_bus; \ + wire [(n)-1:0] m_axi_mem_rlast_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_bus; \ + wire [(n)*2-1:0] m_axi_mem_rresp_bus; \ + wire [(n)-1:0] m_axi_mem_bvalid_bus; \ + wire [(n)-1:0] m_axi_mem_bready_bus; \ + wire [(n)*2-1:0] m_axi_mem_bresp_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_bus; -`define AXI_MEM_ARRAY_ARGS \ - .m_axi_mem_awvalid_a(m_axi_mem_awvalid_a), \ - .m_axi_mem_awready_a(m_axi_mem_awready_a), \ - .m_axi_mem_awaddr_a(m_axi_mem_awaddr_a), \ - .m_axi_mem_awid_a(m_axi_mem_awid_a), \ - .m_axi_mem_awlen_a(m_axi_mem_awlen_a), \ - .m_axi_mem_wvalid_a(m_axi_mem_wvalid_a), \ - .m_axi_mem_wready_a(m_axi_mem_wready_a), \ - .m_axi_mem_wdata_a(m_axi_mem_wdata_a), \ - .m_axi_mem_wstrb_a(m_axi_mem_wstrb_a), \ - .m_axi_mem_wlast_a(m_axi_mem_wlast_a), \ - .m_axi_mem_arvalid_a(m_axi_mem_arvalid_a), \ - .m_axi_mem_arready_a(m_axi_mem_arready_a), \ - .m_axi_mem_araddr_a(m_axi_mem_araddr_a), \ - .m_axi_mem_arid_a(m_axi_mem_arid_a), \ - .m_axi_mem_arlen_a(m_axi_mem_arlen_a), \ - .m_axi_mem_rvalid_a(m_axi_mem_rvalid_a), \ - .m_axi_mem_rready_a(m_axi_mem_rready_a), \ - .m_axi_mem_rdata_a(m_axi_mem_rdata_a), \ - .m_axi_mem_rlast_a(m_axi_mem_rlast_a), \ - .m_axi_mem_rid_a(m_axi_mem_rid_a), \ - .m_axi_mem_rresp_a(m_axi_mem_rresp_a), \ - .m_axi_mem_bvalid_a(m_axi_mem_bvalid_a), \ - .m_axi_mem_bready_a(m_axi_mem_bready_a), \ - .m_axi_mem_bresp_a(m_axi_mem_bresp_a), \ - .m_axi_mem_bid_a(m_axi_mem_bid_a) +`define AXI_MEM_BUS_ARGS \ + .m_axi_mem_awvalid_bus(m_axi_mem_awvalid_bus), \ + .m_axi_mem_awready_bus(m_axi_mem_awready_bus), \ + .m_axi_mem_awaddr_bus(m_axi_mem_awaddr_bus), \ + .m_axi_mem_awid_bus(m_axi_mem_awid_bus), \ + .m_axi_mem_awlen_bus(m_axi_mem_awlen_bus), \ + .m_axi_mem_wvalid_bus(m_axi_mem_wvalid_bus), \ + .m_axi_mem_wready_bus(m_axi_mem_wready_bus), \ + .m_axi_mem_wdata_bus(m_axi_mem_wdata_bus), \ + .m_axi_mem_wstrb_bus(m_axi_mem_wstrb_bus), \ + .m_axi_mem_wlast_bus(m_axi_mem_wlast_bus), \ + .m_axi_mem_arvalid_bus(m_axi_mem_arvalid_bus), \ + .m_axi_mem_arready_bus(m_axi_mem_arready_bus), \ + .m_axi_mem_araddr_bus(m_axi_mem_araddr_bus), \ + .m_axi_mem_arid_bus(m_axi_mem_arid_bus), \ + .m_axi_mem_arlen_bus(m_axi_mem_arlen_bus), \ + .m_axi_mem_rvalid_bus(m_axi_mem_rvalid_bus), \ + .m_axi_mem_rready_bus(m_axi_mem_rready_bus), \ + .m_axi_mem_rdata_bus(m_axi_mem_rdata_bus), \ + .m_axi_mem_rlast_bus(m_axi_mem_rlast_bus), \ + .m_axi_mem_rid_bus(m_axi_mem_rid_bus), \ + .m_axi_mem_rresp_bus(m_axi_mem_rresp_bus), \ + .m_axi_mem_bvalid_bus(m_axi_mem_bvalid_bus), \ + .m_axi_mem_bready_bus(m_axi_mem_bready_bus), \ + .m_axi_mem_bresp_bus(m_axi_mem_bresp_bus), \ + .m_axi_mem_bid_bus(m_axi_mem_bid_bus) `include "VX_define.vh" diff --git a/hw/rtl/afu/xrt/vortex_afu.v b/hw/rtl/afu/xrt/vortex_afu.v index 5e37bd818c..05863d8791 100644 --- a/hw/rtl/afu/xrt/vortex_afu.v +++ b/hw/rtl/afu/xrt/vortex_afu.v @@ -67,8 +67,8 @@ module vortex_afu #( reset_r <= ~ap_rst_n; end - `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) - `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + `AXI_MEM_BUSES (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_BUS, MP_SEMI); VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), @@ -80,7 +80,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (reset_r), - `AXI_MEM_ARRAY_ARGS, + `AXI_MEM_BUS_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_ctrl_awvalid), .s_axi_ctrl_awready (s_axi_ctrl_awready), diff --git a/hw/scripts/xilinx_elab_check.tcl b/hw/scripts/xilinx_elab_check.tcl index 39ce791404..8933471fbf 100644 --- a/hw/scripts/xilinx_elab_check.tcl +++ b/hw/scripts/xilinx_elab_check.tcl @@ -46,12 +46,6 @@ if {$mic_mode eq "off" || $mic_mode eq "0"} { create_project -in_memory add_files -norecurse ${mic_sources} - # The platform tops (vortex_afu.v) pass unpacked per-port arrays to - # VX_afu_wrap, which needs SystemVerilog. - set mic_afu_tops [get_files -quiet */vortex_afu.v] - if {[llength $mic_afu_tops] > 0} { - set_property file_type SystemVerilog $mic_afu_tops - } # Black-box every generated vendor IP via its BLOCK_STUB so a -rtl elaboration # resolves the IP instances (FPU FMA/DIV/SQRT/MUL/ADD, etc.) without pulling diff --git a/hw/syn/xilinx/aved/package_ip.tcl b/hw/syn/xilinx/aved/package_ip.tcl index 4df28092a9..0e6bdb33cc 100644 --- a/hw/syn/xilinx/aved/package_ip.tcl +++ b/hw/syn/xilinx/aved/package_ip.tcl @@ -63,13 +63,6 @@ foreach line [split $data "\n"] { } } -# The platform tops (vortex_afu.v) pass unpacked per-port arrays to -# VX_afu_wrap, which needs SystemVerilog. -set afu_tops [get_files -quiet */vortex_afu.v] -if {[llength $afu_tops] > 0} { - set_property file_type SystemVerilog $afu_tops -} - # Pull in the floating-point cores generated above so they are packaged # alongside the RTL. The set matches the XRT flow's package_kernel.tcl. set ip_files [list] diff --git a/hw/syn/xilinx/dut/project.tcl b/hw/syn/xilinx/dut/project.tcl index 9f512e3f94..56339544ef 100644 --- a/hw/syn/xilinx/dut/project.tcl +++ b/hw/syn/xilinx/dut/project.tcl @@ -173,12 +173,6 @@ proc run_setup {} { # Add the design sources add_files -norecurse -verbose $vsources_list - # The platform tops (vortex_afu.v) pass unpacked per-port arrays to - # VX_afu_wrap, which needs SystemVerilog. - set afu_tops [get_files -quiet */vortex_afu.v] - if {[llength $afu_tops] > 0} { - set_property file_type SystemVerilog $afu_tops - } # process defines set_property verilog_define ${vdefines_list} [current_fileset] diff --git a/hw/syn/xilinx/xrt/package_kernel.tcl b/hw/syn/xilinx/xrt/package_kernel.tcl index c9b0ab20cb..d9b9777c72 100644 --- a/hw/syn/xilinx/xrt/package_kernel.tcl +++ b/hw/syn/xilinx/xrt/package_kernel.tcl @@ -62,12 +62,6 @@ foreach def $vdefines_list { create_project -force kernel_pack $path_to_tmp_project add_files -norecurse ${vsources_list} -# The platform tops (vortex_afu.v) pass unpacked per-port arrays to -# VX_afu_wrap, which needs SystemVerilog. -set afu_tops [get_files -quiet */vortex_afu.v] -if {[llength $afu_tops] > 0} { - set_property file_type SystemVerilog $afu_tops -} set obj [get_filesets sources_1] set ip_files [list \ diff --git a/sim/xrtsim/vortex_afu_shim.sv b/sim/xrtsim/vortex_afu_shim.sv index 73a8b8f071..d6a453507d 100644 --- a/sim/xrtsim/vortex_afu_shim.sv +++ b/sim/xrtsim/vortex_afu_shim.sv @@ -54,8 +54,8 @@ module vortex_afu_shim #( output wire interrupt `IGNORE_WARNINGS_END ); - `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) - `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + `AXI_MEM_BUSES (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_BUS, MP_SEMI); VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), @@ -68,7 +68,7 @@ module vortex_afu_shim #( .clk (ap_clk), .reset (~ap_rst_n), - `AXI_MEM_ARRAY_ARGS, + `AXI_MEM_BUS_ARGS, `AXI_HOST_ARGS, From b95dd2b4d7ff73bc0300bc0456520868b19fcf49 Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 27 Oct 2026 11:27:37 -0700 Subject: [PATCH 125/336] lmem: merge same-word reads of an LSU block into one bank access Local memory served every lane separately, so a warp-uniform read -- the A operand of a tiled GEMM, a broadcast constant, a look-up with a uniform index -- was serialized lane by lane at its bank: 32 accesses at NT=32. That made LMEM-staged kernels slower than plain global-memory ones on wide shapes, the opposite of every GPU they are written for. Follow the AMD/NVIDIA rule: a bank serves one distinct word per cycle to every lane of the request that reads it. Plain reads of the same word with the same tag from the ports of one LSU block share one bank access. The lowest such port leads and the others stay out of the request crossbar, accepted in their leader's cycle; the merged-port mask rides under the bank tag, and the response is multicast to all of them through per-lane arbiters with a per-bank sent mask. It is the dcache read merge's structure (VX_cache g_read_merge / g_core_rsp_mcast), applied to VX_local_mem. Stores, atomics, LR/SC and DMA are unchanged. A merged read counts once in the reads counter, as the bank access that serves it, as in the dcache. VX_CFG_LMEM_READ_MERGE (default 1) enables it, matching the icache and dcache merges. SimX mirrors the rule in LmemMergeXbar, which also presents at most one request per lane per cycle, as the RTL does. SimX, rtx5080 profile (4c x 32w x 32t), cycles merge off -> on: sgemm2 -t16 n128 352,650 -> 182,847 (1.93x) CUDA mapping -t32 449,483 -> 74,038 (6.07x) register-blocked -r2 93,310 -> 30,806 (3.03x) geomean, 8 LMEM kernels 2.72x naive sgemm (no LMEM) unchanged Detection logic: ~4% of VX_local_mem area at that shape (yosys/ASAP7), a shorter path than one leader per block. New regression lmem_merge: shared-word reads in one and several banks, partial thread masks, lanes of different LSU blocks, and each lane's order against its own stores, plus a full LMEM dump check. CI: lmem-merge-1/2 and lmem-merge-off (simx, rtlsim), parity-lmem-merge and parity-sgemm2-lmem-merge. Verified: tests/regression run-simx; all 65 model_parity cases (SimX vs rtlsim within 0.2% on the new cases, exact instruction match); the 7 new CI cases on simx and rtlsim. Not verified: perf_gate baselines are stale by construction (the resolved config hash includes the new knob) and need a reviewed --update-baselines re-record; fpga_gate/asic_gate for core and lmem not run (the lmem DUT now synthesizes READ_MERGE, so its synthesis baseline moves). Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 4 + ci/testcases/core.yaml | 13 ++ ci/testcases/regression.yaml | 29 +++ hw/rtl/core/VX_mem_unit.sv | 1 + hw/rtl/mem/VX_local_mem.sv | 181 +++++++++++++++---- hw/unittest/local_mem/VX_local_mem_top.sv | 1 + sim/simx/core.cpp | 3 +- sim/simx/mem/local_mem.cpp | 211 +++++++++++++++++++++- sim/simx/mem/local_mem.h | 1 + tests/regression/Makefile | 2 +- tests/regression/lmem_merge/Makefile | 16 ++ tests/regression/lmem_merge/common.h | 22 +++ tests/regression/lmem_merge/kernel.cpp | 90 +++++++++ tests/regression/lmem_merge/main.cpp | 152 ++++++++++++++++ 14 files changed, 684 insertions(+), 42 deletions(-) create mode 100644 tests/regression/lmem_merge/Makefile create mode 100644 tests/regression/lmem_merge/common.h create mode 100644 tests/regression/lmem_merge/kernel.cpp create mode 100644 tests/regression/lmem_merge/main.cpp diff --git a/VX_config.toml b/VX_config.toml index 35bacd10f2..bac730f67c 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -324,6 +324,10 @@ lmem_bytes_per_bank = 2048 lmem_min_bytes = 16384 VX_CFG_LMEM_NUM_BANKS = "expr: $VX_CFG_NUM_LSU_LANES" VX_CFG_LMEM_LOG_SIZE = "expr: clog2(max($lmem_min_bytes, $VX_CFG_LMEM_NUM_BANKS * $lmem_bytes_per_bank))" +# Same-word reads of one LSU block's lanes share one bank access, and the +# response returns to all of them: each bank serves one distinct word per cycle +# to every lane reading it. +VX_CFG_LMEM_READ_MERGE = 1 [tcu] # string enum: 'DPI'|'DSP'|'BHF'|'TFR'|'FPNEW'; TFR is the universal default. On diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index e6210b28a3..26cd790c1e 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -244,6 +244,19 @@ tests: via: blackbox app: uniform_ld configs: -DVX_CFG_NUM_THREADS=16 +# Same-word local-memory reads of one LSU block, served by one bank access and +# returned to every lane. +- id: parity-lmem-merge + check: model_parity + via: blackbox + app: lmem_merge + configs: -DVX_CFG_NUM_THREADS=16 +- id: parity-sgemm2-lmem-merge + check: model_parity + via: blackbox + app: sgemm2 + args: -n32 -t4 + configs: -DVX_CFG_NUM_THREADS=16 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index c9693b26a6..c925823d24 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -18,6 +18,35 @@ tests: - rtlsim dir: tests/regression target: run-{driver} +# Local-memory read merging: shared-word reads, partial masks, lanes of +# different LSU blocks, and per-lane order against the lane's own stores. +- id: lmem-merge-1 + via: blackbox + drivers: + - simx + - rtlsim + app: lmem_merge + shape: + threads: 16 +- id: lmem-merge-2 + via: blackbox + drivers: + - simx + app: lmem_merge + configs: -DVX_CFG_ISSUE_WIDTH=2 + shape: + threads: 32 + warps: 8 +# The same test with merging off: every lane takes its own bank access. +- id: lmem-merge-off + via: blackbox + drivers: + - simx + - rtlsim + app: lmem_merge + configs: -DVX_CFG_LMEM_READ_MERGE=0 + shape: + threads: 16 - id: occupancy-1 via: blackbox drivers: diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 6ed8f18c83..ef42d48fe1 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -175,6 +175,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .DMA_ENABLE (LMEM_DMA_EN), .DMA_TAG_WIDTH (LMEM_DMA_TAG_WIDTH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), + .READ_MERGE (`VX_CFG_LMEM_READ_MERGE ? `VX_CFG_NUM_LSU_LANES : 0), .OUT_BUF (3) ) local_mem ( .clk (clk), diff --git a/hw/rtl/mem/VX_local_mem.sv b/hw/rtl/mem/VX_local_mem.sv index 5614595503..26a798537a 100644 --- a/hw/rtl/mem/VX_local_mem.sv +++ b/hw/rtl/mem/VX_local_mem.sv @@ -39,6 +39,9 @@ module VX_local_mem import VX_gpu_pkg::*; #( // Synthesize atomic-op + LR/SC logic in the banks parameter AMO_ENABLE = 0, + // Merge same-word reads within groups of this many adjacent ports (0: off) + parameter READ_MERGE = 0, + // Response buffer parameter OUT_BUF = 0 ) ( @@ -78,10 +81,17 @@ module VX_local_mem import VX_gpu_pkg::*; #( localparam AMO_CMP_WIDTH = 0; `endif localparam AMO_SB_WIDTH = 1 + $bits(amo_op_e) + 1 + HART_ID_WIDTH + AMO_CMP_WIDTH; - localparam REQ_DATAW = 1 + BANK_ADDR_WIDTH + WORD_SIZE + WORD_WIDTH + TAG_WIDTH + AMO_SB_WIDTH; + // A merged read carries its group's merged-port mask under its tag, through + // the bank, to the response multicast. + localparam MERGE_GROUP = (READ_MERGE != 0) ? `MIN(READ_MERGE, NUM_REQS) : 1; + localparam DO_READ_MERGE = (MERGE_GROUP > 1); + localparam MERGE_BITS = DO_READ_MERGE ? MERGE_GROUP : 0; + localparam BANK_TAG_WIDTH = TAG_WIDTH + MERGE_BITS; + localparam REQ_DATAW = 1 + BANK_ADDR_WIDTH + WORD_SIZE + WORD_WIDTH + BANK_TAG_WIDTH + AMO_SB_WIDTH; localparam RSP_DATAW = WORD_WIDTH + TAG_WIDTH; `STATIC_ASSERT(ADDR_WIDTH == (BANK_ADDR_WIDTH + `CLOG2(NUM_BANKS)), ("invalid parameter")) + `STATIC_ASSERT(!DO_READ_MERGE || (`IS_POW2(MERGE_GROUP) && `IS_DIVISBLE(NUM_REQS, MERGE_GROUP)), ("invalid parameter: read merge group")) // bank selection @@ -131,7 +141,7 @@ module VX_local_mem import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] per_bank_req_addr; wire [NUM_BANKS-1:0][WORD_SIZE-1:0] per_bank_req_byteen; wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] per_bank_req_data; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_req_tag; + wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_req_tag; wire [NUM_BANKS-1:0] per_bank_req_amo_valid; wire [NUM_BANKS-1:0][$bits(amo_op_e)-1:0] per_bank_req_amo_op; wire [NUM_BANKS-1:0] per_bank_req_amo_unsigned; @@ -152,8 +162,56 @@ module VX_local_mem import VX_gpu_pkg::*; #( wire [PERF_CTR_BITS-1:0] perf_collisions; `endif + // Read merging: plain reads of the same word with the same tag from ports + // of one group in the same cycle are served by one bank access. The lowest + // such port leads; the others stay out of the request crossbar and are + // accepted in the cycle their leader is, so every port keeps its own + // request order into the bank. + wire [NUM_REQS-1:0] req_follower; + wire [NUM_REQS-1:0][BANK_TAG_WIDTH-1:0] req_bank_tag; + if (DO_READ_MERGE) begin : g_read_merge + wire [NUM_REQS-1:0] plain_read; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_plain_read + assign plain_read[i] = lsu_bus_if[i].req_valid && ~lsu_bus_if[i].req_data.rw && ~req_amo_valid[i]; + end + // same_word[i][j]: port i reads what a lower port j of its group reads + wire [NUM_REQS-1:0][NUM_REQS-1:0] same_word; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_same_i + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_same_j + if (j < i && (j / MERGE_GROUP) == (i / MERGE_GROUP)) begin : g_cmp + assign same_word[i][j] = plain_read[i] && plain_read[j] + && (lsu_bus_if[i].req_data.addr == lsu_bus_if[j].req_data.addr) + && (lsu_bus_if[i].req_data.tag == lsu_bus_if[j].req_data.tag); + end else begin : g_none + assign same_word[i][j] = 1'b0; + end + end + end + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_merge + localparam GROUP_BASE = (i / MERGE_GROUP) * MERGE_GROUP; + wire [MERGE_GROUP-1:0] merged_mask; + wire [NUM_REQS-1:0] leader_fire; + for (genvar k = 0; k < MERGE_GROUP; ++k) begin : g_merged_mask + assign merged_mask[k] = same_word[GROUP_BASE + k][i]; + end + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_leader_fire + assign leader_fire[j] = same_word[i][j] && ~req_follower[j] && req_ready_in[j]; + end + assign req_bank_tag[i] = {lsu_bus_if[i].req_data.tag, merged_mask}; + assign req_follower[i] = (| same_word[i]); + assign lsu_bus_if[i].req_ready = req_follower[i] ? (| leader_fire) : req_ready_in[i]; + end + end else begin : g_no_read_merge + assign req_follower = '0; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_bank_tag + assign req_bank_tag[i] = lsu_bus_if[i].req_data.tag; + assign lsu_bus_if[i].req_ready = req_ready_in[i]; + end + end + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_data_in - assign req_valid_in[i] = lsu_bus_if[i].req_valid; + // a merged read enters the crossbar once, as its leader + assign req_valid_in[i] = lsu_bus_if[i].req_valid && ~req_follower[i]; assign req_data_in[i] = { req_amo_valid[i], req_amo_op[i], @@ -166,9 +224,8 @@ module VX_local_mem import VX_gpu_pkg::*; #( req_bank_addr[i], lsu_bus_if[i].req_data.data, lsu_bus_if[i].req_data.byteen, - lsu_bus_if[i].req_data.tag + req_bank_tag[i] }; - assign lsu_bus_if[i].req_ready = req_ready_in[i]; end VX_stream_xbar #( @@ -220,7 +277,7 @@ module VX_local_mem import VX_gpu_pkg::*; #( // Back-pressure-safe copy of the LSU response data (see g_data_store). wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] bank_lsu_rsp_data; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_rsp_idx; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_rsp_tag; + wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_rsp_tag; wire [NUM_BANKS-1:0] per_bank_rsp_ready; // DMA port handshake @@ -608,7 +665,7 @@ module VX_local_mem import VX_gpu_pkg::*; #( // Delay tag/idx to align with SRAM 1-cycle output latency VX_pipe_buffer #( - .DATAW (REQ_SEL_WIDTH + TAG_WIDTH) + .DATAW (REQ_SEL_WIDTH + BANK_TAG_WIDTH) ) bram_buf ( .clk (clk), .reset (reset), @@ -659,35 +716,86 @@ module VX_local_mem import VX_gpu_pkg::*; #( // bank responses gather - wire [NUM_BANKS-1:0][RSP_DATAW-1:0] per_bank_rsp_data_aos; - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_rsp_data_aos - assign per_bank_rsp_data_aos[i] = {bank_lsu_rsp_data[i], per_bank_rsp_tag[i]}; - end - wire [NUM_REQS-1:0] rsp_valid_out; wire [NUM_REQS-1:0][RSP_DATAW-1:0] rsp_data_out; wire [NUM_REQS-1:0] rsp_ready_out; - VX_stream_xbar #( - .NUM_INPUTS (NUM_BANKS), - .NUM_OUTPUTS (NUM_REQS), - .DATAW (RSP_DATAW), - .ARBITER ("P"), // this priority arbiter has negligeable impact on performance - .OUT_BUF (OUT_BUF) - ) rsp_xbar ( - .clk (clk), - .reset (reset), - `UNUSED_PIN (collisions), - .sel_in (per_bank_rsp_idx), - .valid_in (per_bank_rsp_valid), - .data_in (per_bank_rsp_data_aos), - .ready_in (per_bank_rsp_ready), - .valid_out (rsp_valid_out), - .data_out (rsp_data_out), - .ready_out (rsp_ready_out), - `UNUSED_PIN (sel_out) - ); + if (DO_READ_MERGE) begin : g_rsp_mcast + // A merged read's response goes to its leader's port and to every + // merged port: each output takes it when its arbiter grants it, and the + // bank's response is released once all its targets have taken it. + localparam GROUP_BITS = `CLOG2(MERGE_GROUP); + wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_pending; + wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_taken; + wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_valid; + wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_ready; + wire [NUM_BANKS-1:0][RSP_DATAW-1:0] per_bank_rsp_pdata; + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_banks + wire [MERGE_GROUP-1:0] merged_mask = per_bank_rsp_tag[i][MERGE_GROUP-1:0]; + wire [REQ_SEL_WIDTH-1:0] group_base = (per_bank_rsp_idx[i] >> GROUP_BITS) << GROUP_BITS; + wire [NUM_REQS-1:0] targets = (NUM_REQS'(1) << per_bank_rsp_idx[i]) + | (NUM_REQS'(merged_mask) << group_base); + reg [NUM_REQS-1:0] sent_r; + assign per_bank_rsp_pdata[i] = {bank_lsu_rsp_data[i], per_bank_rsp_tag[i][MERGE_BITS +: TAG_WIDTH]}; + assign lane_pending[i] = targets & ~sent_r & {NUM_REQS{per_bank_rsp_valid[i]}}; + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_taken + assign per_output_valid[j][i] = lane_pending[i][j]; + assign lane_taken[i][j] = lane_pending[i][j] && per_output_ready[j][i]; + end + assign per_bank_rsp_ready[i] = per_bank_rsp_valid[i] && ((lane_pending[i] & ~lane_taken[i]) == '0); + always @(posedge clk) begin + if (reset) begin + sent_r <= '0; + end else if (per_bank_rsp_valid[i]) begin + sent_r <= per_bank_rsp_ready[i] ? '0 : (sent_r | lane_taken[i]); + end + end + end + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_outputs + VX_stream_arb #( + .NUM_INPUTS (NUM_BANKS), + .NUM_OUTPUTS (1), + .DATAW (RSP_DATAW), + .ARBITER ("P"), + .OUT_BUF (OUT_BUF) + ) rsp_arb ( + .clk (clk), + .reset (reset), + .valid_in (per_output_valid[j]), + .data_in (per_bank_rsp_pdata), + .ready_in (per_output_ready[j]), + .valid_out (rsp_valid_out[j]), + .data_out (rsp_data_out[j]), + `UNUSED_PIN (sel_out), + .ready_out (rsp_ready_out[j]) + ); + end + end else begin : g_rsp_xbar + wire [NUM_BANKS-1:0][RSP_DATAW-1:0] per_bank_rsp_data_aos; + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_rsp_data_aos + assign per_bank_rsp_data_aos[i] = {bank_lsu_rsp_data[i], per_bank_rsp_tag[i]}; + end + + VX_stream_xbar #( + .NUM_INPUTS (NUM_BANKS), + .NUM_OUTPUTS (NUM_REQS), + .DATAW (RSP_DATAW), + .ARBITER ("P"), // this priority arbiter has negligeable impact on performance + .OUT_BUF (OUT_BUF) + ) rsp_xbar ( + .clk (clk), + .reset (reset), + `UNUSED_PIN (collisions), + .sel_in (per_bank_rsp_idx), + .valid_in (per_bank_rsp_valid), + .data_in (per_bank_rsp_data_aos), + .ready_in (per_bank_rsp_ready), + .valid_out (rsp_valid_out), + .data_out (rsp_data_out), + .ready_out (rsp_ready_out), + `UNUSED_PIN (sel_out) + ); + end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_lsu_bus_if assign lsu_bus_if[i].rsp_valid = rsp_valid_out[i]; @@ -709,6 +817,7 @@ module VX_local_mem import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] perf_reads_per_req, perf_writes_per_req; wire [NUM_REQS-1:0] perf_crsp_stall_per_req = rsp_valid_out & ~rsp_ready_out; + // a merged read counts once, as the bank access that serves it `BUFFER(perf_reads_per_req, req_valid_in & req_ready_in & ~req_rw); `BUFFER(perf_writes_per_req, req_valid_in & req_ready_in & req_rw); @@ -748,11 +857,11 @@ module VX_local_mem import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][UUID_WIDTH-1:0] per_bank_rsp_uuid; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_req_uuid - assign per_bank_req_tag_value[i] = per_bank_req_tag[i][TAG_WIDTH-UUID_WIDTH-1:0]; - assign per_bank_rsp_tag_value[i] = per_bank_rsp_tag[i][TAG_WIDTH-UUID_WIDTH-1:0]; + assign per_bank_req_tag_value[i] = per_bank_req_tag[i][MERGE_BITS +: (TAG_WIDTH-UUID_WIDTH)]; + assign per_bank_rsp_tag_value[i] = per_bank_rsp_tag[i][MERGE_BITS +: (TAG_WIDTH-UUID_WIDTH)]; if (UUID_WIDTH != 0) begin : g_uuid - assign per_bank_req_uuid[i] = per_bank_req_tag[i][TAG_WIDTH-1 -: UUID_WIDTH]; - assign per_bank_rsp_uuid[i] = per_bank_rsp_tag[i][TAG_WIDTH-1 -: UUID_WIDTH]; + assign per_bank_req_uuid[i] = per_bank_req_tag[i][BANK_TAG_WIDTH-1 -: UUID_WIDTH]; + assign per_bank_rsp_uuid[i] = per_bank_rsp_tag[i][BANK_TAG_WIDTH-1 -: UUID_WIDTH]; end else begin : g_no_uuid assign per_bank_req_uuid[i] = 0; assign per_bank_rsp_uuid[i] = 0; diff --git a/hw/unittest/local_mem/VX_local_mem_top.sv b/hw/unittest/local_mem/VX_local_mem_top.sv index 3bbf4b9c1f..cf323cb596 100644 --- a/hw/unittest/local_mem/VX_local_mem_top.sv +++ b/hw/unittest/local_mem/VX_local_mem_top.sv @@ -104,6 +104,7 @@ module VX_local_mem_top import VX_gpu_pkg::*; #( .ADDR_WIDTH (ADDR_WIDTH), .TAG_WIDTH (TAG_WIDTH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), + .READ_MERGE (`VX_CFG_LMEM_READ_MERGE ? NUM_REQS : 0), .OUT_BUF (3) ) local_mem ( .clk (clk), diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 9aed139299..ec8af29b71 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -164,7 +164,8 @@ class Core::Impl { LSU_NUM_REQS, log2ceil(VX_CFG_LMEM_NUM_BANKS), false, - VX_CFG_EXT_DXA_ENABLED + VX_CFG_TCU_WGMMA_ENABLED + VX_CFG_EXT_DXA_ENABLED + VX_CFG_TCU_WGMMA_ENABLED, + (VX_CFG_LMEM_READ_MERGE != 0 && LSU_CHANNELS > 1) ? LSU_CHANNELS : 0 // read_merge: one LSU block's lanes }); // create lmem switch diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 6454fe0ee7..64434e76bd 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include "types.h" @@ -26,6 +27,190 @@ using namespace vortex; +// The lane-to-bank crossbar under read merging. Each bank grants its lowest +// requesting lane per cycle, as the plain crossbar does. A plain read of the +// same word with the same tag as a lower lane of its LSU block follows the +// lowest such lane: it stays out of arbitration, is accepted when that leader +// is, and the bank's response returns to every one of them. +class LmemMergeXbar : public SimObject { +public: + using Ptr = std::shared_ptr; + + std::vector> ReqIn; // lanes + std::vector> RspOut; // lanes + std::vector> ReqOut; // banks + std::vector> RspIn; // banks + + LmemMergeXbar(const SimContext& ctx, const char* name, uint32_t num_inputs, uint32_t num_banks, + uint32_t group, uint32_t lg2_word, uint32_t req_delay, uint32_t rsp_delay) + : SimObject(ctx, name) + , ReqIn(num_inputs, this) + , RspOut(num_inputs, this) + , ReqOut(num_banks, this) + , RspIn(num_banks, this) + , group_(group) + , lg2_word_(lg2_word) + , lg2_inputs_(log2ceil(num_inputs)) + , req_delay_(req_delay) + , rsp_delay_(rsp_delay) + , followers_(num_banks) + , sent_(num_banks, 0) + { + __assert(group_ >= 1 && group_ <= 64 && (num_inputs % group_) == 0, + "a merge group must be at most 64 lanes dividing the ports"); + } + + uint64_t collisions() const { return collisions_; } + +protected: + void on_reset() { + for (auto& f : followers_) { + f.clear(); + } + std::fill(sent_.begin(), sent_.end(), 0); + } + + void on_tick() { + uint32_t I = ReqIn.size(); + uint32_t B = ReqOut.size(); + uint32_t R = 1 << lg2_inputs_; + + // responses: each lane takes one bank per cycle, lowest first, and a + // bank's response leaves once every lane it serves has taken it + std::vector targets(B, 0); + std::vector bases(B, 0); + bool idle = true; + for (uint32_t b = 0; b < B; ++b) { + if (RspIn.at(b).empty()) { + continue; + } + idle = false; + auto& rsp = RspIn.at(b).peek(); + uint32_t lane = rsp.tag & (R - 1); + bases.at(b) = (lane / group_) * group_; + uint64_t mask = 0; + auto it = followers_.at(b).find(rsp.tag); + if (it != followers_.at(b).end()) { + mask = it->second; + } + targets.at(b) = (mask | (1ull << (lane - bases.at(b)))) & ~sent_.at(b); + } + for (uint32_t o = 0; o < I; ++o) { + for (uint32_t b = 0; b < B; ++b) { + if (o < bases.at(b) || o >= bases.at(b) + group_ + || !(targets.at(b) & (1ull << (o - bases.at(b))))) { + continue; + } + MemRsp out(RspIn.at(b).peek()); + out.tag >>= lg2_inputs_; + if (RspOut.at(o).try_send(out, rsp_delay_)) { + DT(4, this->name() << " rsp" << b << "_" << o << ": " << out); + sent_.at(b) |= 1ull << (o - bases.at(b)); + } + break; + } + } + for (uint32_t b = 0; b < B; ++b) { + if (targets.at(b) != 0 && (targets.at(b) & ~sent_.at(b)) == 0) { + followers_.at(b).erase(RspIn.at(b).peek().tag); + RspIn.at(b).pop(); + sent_.at(b) = 0; + } + } + + // requests: find each follower's leader, then let each bank grant its + // lowest requesting lane that follows no other. A lane presents one + // request per cycle, so a lane granted or released this cycle sits out + // the remaining banks even when its next request is already queued. + std::vector lead(I, -1); + std::vector taken(I, false); + for (uint32_t base = 0; base < I; base += group_) { + for (uint32_t i = base; i < base + group_; ++i) { + if (!this->is_plain_read(i)) { + continue; + } + auto& req = ReqIn.at(i).peek(); + for (uint32_t j = base; j < i; ++j) { + if (!this->is_plain_read(j)) { + continue; + } + auto& lreq = ReqIn.at(j).peek(); + if ((lreq.addr >> lg2_word_) == (req.addr >> lg2_word_) && lreq.tag == req.tag) { + lead.at(i) = j; + break; + } + } + } + } + for (uint32_t o = 0; o < B; ++o) { + int32_t g = -1; + bool has_collision = false; + for (uint32_t i = 0; i < I; ++i) { + if (taken.at(i) || ReqIn.at(i).empty() || lead.at(i) != -1 || this->bank_of(ReqIn.at(i).peek()) != o) { + continue; + } + if (g != -1) { + has_collision = true; + break; + } + g = i; + } + if (g == -1) { + continue; + } + idle = false; + MemReq out(ReqIn.at(g).peek()); + out.tag = (out.tag << lg2_inputs_) | g; + if (ReqOut.at(o).try_send(out, req_delay_)) { + DT(4, this->name() << " req" << g << "_" << o << ": " << out); + uint32_t base = (g / group_) * group_; + uint64_t mask = 0; + for (uint32_t i = base; i < base + group_; ++i) { + if (lead.at(i) == g && !taken.at(i)) { + DT(3, this->name() << " read-merge: lane=" << i << ", leader=" << g << ", " << ReqIn.at(i).peek()); + mask |= 1ull << (i - base); + ReqIn.at(i).pop(); + taken.at(i) = true; + } + } + if (mask != 0) { + followers_.at(o)[out.tag] = mask; + } + ReqIn.at(g).pop(); + taken.at(g) = true; + } + collisions_ += has_collision; + } + + for (uint32_t i = 0; i < I && idle; ++i) { + idle = ReqIn.at(i).empty(); + } + if (idle) { + this->tick_sleep(); + } + } + + uint32_t bank_of(const MemReq& req) const { + return (uint32_t)((req.addr >> lg2_word_) & (ReqOut.size() - 1)); + } + + bool is_plain_read(uint32_t i) const { + return !ReqIn.at(i).empty() && ReqIn.at(i).peek().op == MemOp::LD; + } + +private: + uint32_t group_; + uint32_t lg2_word_; + uint32_t lg2_inputs_; + uint32_t req_delay_; + uint32_t rsp_delay_; + std::vector> followers_; // [bank] leader's tag -> lanes riding it, from the group base + std::vector sent_; // [bank] lanes that took the response, from the group base + uint64_t collisions_ = 0; + + friend class SimObject; +}; + class LocalMem::Impl { protected: LocalMem* simobject_; @@ -54,6 +239,7 @@ class LocalMem::Impl { std::vector rdw_cur_addr_; std::vector rdw_cur_valid_; MemCrossBar::Ptr mem_xbar_; + LmemMergeXbar::Ptr merge_xbar_; mutable PerfStats perf_stats_; uint32_t dma_last_grant_ = 0; @@ -98,6 +284,15 @@ class LocalMem::Impl { constexpr uint32_t kMaxFanout = 8; return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; }; + if (config.read_merge != 0) { + merge_xbar_ = LmemMergeXbar::Create(sname, config.num_reqs, num_banks, config.read_merge, + lg2_line_size, 1 + tree_delay(config.num_reqs), tree_delay(num_banks)); + for (uint32_t i = 0; i < config.num_reqs; ++i) { + simobject->Inputs.at(i).bind(&merge_xbar_->ReqIn.at(i)); + merge_xbar_->RspOut.at(i).bind(&simobject->Outputs.at(i)); + } + return; + } mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::Priority, config.num_reqs, num_banks, [lg2_line_size, num_banks](const MemCrossBar::ReqType& req) { // Custom logic to calculate the output index using bank interleaving @@ -109,6 +304,14 @@ class LocalMem::Impl { } } + SimChannel& bank_req(uint32_t b) { + return merge_xbar_ ? merge_xbar_->ReqOut.at(b) : mem_xbar_->ReqOut.at(b); + } + + SimChannel& bank_rsp(uint32_t b) { + return merge_xbar_ ? merge_xbar_->RspIn.at(b) : mem_xbar_->RspIn.at(b); + } + virtual ~Impl() {} void reset() { @@ -217,7 +420,7 @@ class LocalMem::Impl { if ((dma_banks >> i) & 1) { continue; } - auto& xbar_req_out = mem_xbar_->ReqOut.at(i); + auto& xbar_req_out = this->bank_req(i); if (xbar_req_out.empty()) continue; @@ -275,7 +478,7 @@ class LocalMem::Impl { const uint64_t ret_word = is_sc ? (sc_fail ? 1ull : 0ull) : rmw.ret_word; amo_store_word(rsp_data->data(), byte_off, width, ret_word); bank_rsp.data = rsp_data; - if (!mem_xbar_->RspIn.at(i).try_send(bank_rsp)) + if (!this->bank_rsp(i).try_send(bank_rsp)) continue; // stall; no state mutated yet // An SC gives up its own reservation whether it succeeds or @@ -335,7 +538,7 @@ class LocalMem::Impl { ram_.read(rsp_data->data(), line_addr, VX_CFG_MEM_BLOCK_SIZE); bank_rsp.data = rsp_data; } - if (!mem_xbar_->RspIn.at(i).try_send(bank_rsp)) + if (!this->bank_rsp(i).try_send(bank_rsp)) continue; // stall } @@ -351,7 +554,7 @@ class LocalMem::Impl { } const PerfStats& perf_stats() const { - perf_stats_.bank_stalls = mem_xbar_->collisions(); + perf_stats_.bank_stalls = merge_xbar_ ? merge_xbar_->collisions() : mem_xbar_->collisions(); return perf_stats_; } diff --git a/sim/simx/mem/local_mem.h b/sim/simx/mem/local_mem.h index 6772c8b04f..7e725d30f9 100644 --- a/sim/simx/mem/local_mem.h +++ b/sim/simx/mem/local_mem.h @@ -32,6 +32,7 @@ class LocalMem : public SimObject { uint32_t B; // log2 number of banks bool write_reponse; uint32_t dma_clients; // row-wide DMA masters, served round-robin + uint32_t read_merge; // same-word reads merge within groups of this many adjacent ports (0: off) }; struct PerfStats { diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 95f880e341..eb32835a84 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax uniform_ld mufu zfhmin \ + softmax uniform_ld lmem_merge mufu zfhmin \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/lmem_merge/Makefile b/tests/regression/lmem_merge/Makefile new file mode 100644 index 0000000000..5177c093ca --- /dev/null +++ b/tests/regression/lmem_merge/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := lmem_merge + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n16 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/lmem_merge/common.h b/tests/regression/lmem_merge/common.h new file mode 100644 index 0000000000..99d53a6627 --- /dev/null +++ b/tests/regression/lmem_merge/common.h @@ -0,0 +1,22 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define LMEM_MARK 0x80000000u + +// Local-memory layout per block, in words: read-only words first, then one +// RAW and one WAR word per warp and iteration. +#define RO_WORDS 128 + +typedef struct { + uint32_t num_iters; // iterations per warp + uint32_t warp_size; // threads per warp + uint32_t lmem_words; // local-memory words per block + uint32_t raw_base; // first RAW word + uint32_t war_base; // first WAR word + uint64_t err_addr; // out: per-thread error count + uint64_t dump_addr; // out: each block's final local memory +} kernel_arg_t; + +#endif diff --git a/tests/regression/lmem_merge/kernel.cpp b/tests/regression/lmem_merge/kernel.cpp new file mode 100644 index 0000000000..474a4aed44 --- /dev/null +++ b/tests/regression/lmem_merge/kernel.cpp @@ -0,0 +1,90 @@ +#include +#include "common.h" + +// Local-memory reads that share words, around single-lane stores. Per +// iteration k, lane s = k % W takes a turn: +// +// shared words - lanes in groups of 1, 2, 4, ... W read one word; then 2 +// and 4 distinct words per warp, in one bank (32 words +// apart) and in different banks: every lane must get its +// own word; a third of the lanes read one word (partial +// thread mask); every warp reads the same word (lanes of +// different LSU blocks) +// RAW - lane s stores its word, then the whole warp reads it: +// lane s must see its own store +// WAR - the whole warp reads a word, then lane s stores it: lane +// s's read must not see its own later store +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = reinterpret_cast(__local_mem()); + auto err = reinterpret_cast(arg->err_addr); + auto dump = reinterpret_cast(arg->dump_addr); + const uint32_t W = arg->warp_size; + const uint32_t lid = threadIdx.x; + const uint32_t bsize = blockDim.x; + const uint32_t tid = blockIdx.x * bsize + lid; + const uint32_t lane = lid % W; + const uint32_t warp = lid / W; + const uint32_t words = arg->lmem_words; + + for (uint32_t i = lid; i < words; i += bsize) { + lmem[i] = i * 3 + 2; + } + __syncthreads(); + + uint32_t errors = 0; + for (uint32_t k = 0; k < arg->num_iters; ++k) { + uint32_t s = k % W; + uint32_t slot = warp * arg->num_iters + k; + + uint32_t base = (slot * 7) % (RO_WORDS / 2); + uint32_t shared_idx[5] = { + base + (lane >> (k % 6)) % (RO_WORDS / 2), + base + (lane & 1) * 32, + base + (lane & 3) * 16, + base + (lane & 1), + base + (lane & 3), + }; + for (uint32_t p = 0; p < 5; ++p) { + uint32_t idx = shared_idx[p] % RO_WORDS; + if (lmem[idx] != idx * 3 + 2) { + ++errors; + } + } + if (lane % 3 == k % 3) { + uint32_t idx = (base + 5) % RO_WORDS; + if (lmem[idx] != idx * 3 + 2) { + ++errors; + } + } + { + uint32_t idx = (k * 5) % RO_WORDS; + if (lmem[idx] != idx * 3 + 2) { + ++errors; + } + } + + uint32_t ri = arg->raw_base + slot; + if (lane == s) { + lmem[ri] = LMEM_MARK + ri; + } + uint32_t rv = lmem[ri]; + if (lane == s && rv != LMEM_MARK + ri) { + ++errors; + } + + uint32_t wi = arg->war_base + slot; + uint32_t wv = lmem[wi]; + if (lane == s) { + lmem[wi] = LMEM_MARK + wi; + } + if (lane == s && wv != wi * 3 + 2) { + ++errors; + } + } + err[tid] = errors; + + __syncthreads(); + for (uint32_t i = lid; i < words; i += bsize) { + dump[blockIdx.x * words + i] = lmem[i]; + } +} diff --git a/tests/regression/lmem_merge/main.cpp b/tests/regression/lmem_merge/main.cpp new file mode 100644 index 0000000000..391bb57bda --- /dev/null +++ b/tests/regression/lmem_merge/main.cpp @@ -0,0 +1,152 @@ +// lmem_merge — local-memory reads that share words must each get their own +// word, and keep each thread's order with its own stores. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t num_iters = 16; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h")) != -1) { + switch (c) { + case 'n': num_iters = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-n iterations] [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + if (num_iters == 0) { + num_iters = 1; + } + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // One block per core holding all its warps, two waves of blocks. + uint64_t nt = 0, nw = 0, nc = 0, lmem_size = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + CHECK(vx_device_query(dev, VX_CAPS_LOCAL_MEM_SIZE, &lmem_size)); + const uint32_t W = static_cast(nt); + const uint32_t block = static_cast(nt * nw); + const uint32_t grid = static_cast(nc * 2); + const uint32_t num_threads = block * grid; + // shrink the iteration count until a block's RAW and WAR words fit + while (num_iters > 1 && (RO_WORDS + 2 * nw * num_iters) * sizeof(uint32_t) > lmem_size) { + num_iters /= 2; + } + const uint32_t words = RO_WORDS + 2 * static_cast(nw) * num_iters; + std::cout << "lmem_merge: iters=" << num_iters << " block=" << block << " grid=" << grid + << " words=" << words << std::endl; + if (words * sizeof(uint32_t) > lmem_size) { + std::cout << "local memory too small\nFAILED!" << std::endl; + return 1; + } + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + vx_buffer_h err_buf = nullptr, dump_buf = nullptr; + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); + CHECK(vx_buffer_create(dev, grid * words * sizeof(uint32_t), VX_MEM_WRITE, &dump_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.num_iters = num_iters; + kernel_arg.warp_size = W; + kernel_arg.lmem_words = words; + kernel_arg.raw_base = RO_WORDS; + kernel_arg.war_base = RO_WORDS + static_cast(nw) * num_iters; + CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); + CHECK(vx_buffer_address(dump_buf, &kernel_arg.dump_addr)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid; + li.block_dim[0]= block; + li.lmem_size = words * sizeof(uint32_t); + + std::vector h_err(num_threads), h_dump(grid * words); + vx_event_h launch_ev = nullptr, read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_dump.data(), dump_buf, 0, grid * words * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] == 0) { + continue; + } + if (errors < 16) { + std::printf("*** thread %u: %u wrong reads\n", t, h_err[t]); + } + errors += h_err[t]; + } + // every RAW and WAR word was stored by its lane; the read-only words kept their value + for (uint32_t b = 0; b < grid; ++b) { + for (uint32_t i = 0; i < words; ++i) { + uint32_t expected = (i < RO_WORDS) ? (i * 3 + 2) : (LMEM_MARK + i); + uint32_t actual = h_dump[b * words + i]; + if (actual != expected) { + if (errors < 16) { + std::printf("*** block %u lmem[%u] expected=0x%x actual=0x%x\n", b, i, expected, actual); + } + ++errors; + } + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(dump_buf); + vx_buffer_release(err_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} From 6109891efc3700ffa9bff3f87f1e7ce583343f92 Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 27 Oct 2026 11:27:49 -0700 Subject: [PATCH 126/336] sgemm2: rewrite as a standard CUDA tiled SGEMM with register blocking sgemm2 ran 3-8x slower than the naive sgemm it is meant to beat. With runtime tile and chunk sizes its inner loop could not unroll, its index arithmetic outweighed its 4-16 multiply-adds per barrier pair, and its transposed thread mapping (threadIdx.x walking rows) put a warp's B reads in one bank, so the kernel retired 3-4x sgemm's instructions. Write it the way a CUDA shared-memory GEMM is written: threadIdx.x walks columns, the block stages a TILE x CHUNK slice of A and a CHUNK x TILE slice of B with coalesced cooperative loads, and each thread accumulates a REG x REG block of C in registers from fully unrolled loops. TILE, CHUNK and REG are template parameters, picked from a table of supported shapes by the -t/-c and new -r options; the host rejects shapes outside the table and lists them. Element copies move through integer registers to keep the floating-point registers for the accumulators. The exit status is 1 on failure instead of the error count, which reads as success whenever the count is a multiple of 256. SimX, cycles sgemm2 / sgemm (below 1: sgemm2 faster), LMEM read merge on: NW=NT=4 n128 -t16 -r4 0.63 (was 4.60) rtx5080 n128 -t64 -r2 0.74 (was 7.13) rtx5080 n256 -t64 -r4 0.64 (was 7.57) At NW=NT=4 n128 it retires 1.12M instructions, against 2.49M for sgemm and 9.48M before. The default -t4 is kept, so the CI case's shape is unchanged. Verified: SimX at both shapes; the CI sgemm2 case (tests/regression -n16) on SimX at XLEN 32 and 64, and on rtlsim at XLEN 32 (SimX/rtlsim within 0.1%). Not verified: perf_gate (see the previous commit). Co-Authored-By: Claude Opus 5.5 --- tests/regression/sgemm2/common.h | 16 +++- tests/regression/sgemm2/kernel.cpp | 136 +++++++++++++++++++++-------- tests/regression/sgemm2/main.cpp | 52 ++++++++--- 3 files changed, 158 insertions(+), 46 deletions(-) diff --git a/tests/regression/sgemm2/common.h b/tests/regression/sgemm2/common.h index 59a521c347..258961a1a3 100644 --- a/tests/regression/sgemm2/common.h +++ b/tests/regression/sgemm2/common.h @@ -5,11 +5,25 @@ #define TYPE float #endif +// Supported (tile, chunk_k, reg) shapes. The kernel is instantiated for each +// one, so the tile loops have compile-time bounds; the host rejects others. +// tile : the block computes a tile x tile block of C +// chunk: the k-slice of A and B staged in local memory per step +// reg : each thread computes reg x reg outputs (block = (tile/reg)^2 threads) +#define SGEMM2_CONFIGS(X) \ + X(4, 4, 1) X(4, 16, 1) \ + X(8, 8, 1) X(8, 32, 1) \ + X(16, 16, 1) X(16, 32, 1) \ + X(32, 32, 1) \ + X(8, 8, 2) X(16, 16, 2) X(32, 16, 2) X(64, 16, 2) \ + X(16, 16, 4) X(32, 16, 4) X(64, 16, 4) X(64, 8, 4) \ + X(128, 8, 4) + typedef struct { uint32_t size; uint32_t tile_size; uint32_t chunk_k; - uint32_t pad_; + uint32_t reg_size; uint64_t A_addr; uint64_t B_addr; uint64_t C_addr; diff --git a/tests/regression/sgemm2/kernel.cpp b/tests/regression/sgemm2/kernel.cpp index afe6c336e8..b88055bc5e 100644 --- a/tests/regression/sgemm2/kernel.cpp +++ b/tests/regression/sgemm2/kernel.cpp @@ -1,49 +1,115 @@ #include #include "common.h" -__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - auto A_ptr = reinterpret_cast(arg->A_addr); - auto B_ptr = reinterpret_cast(arg->B_addr); - auto C_ptr = reinterpret_cast(arg->C_addr); - - size_t size = arg->size; - size_t tile_size = arg->tile_size; - size_t chunk_k = arg->chunk_k; - - // Global and local thread coordinates. - size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; - size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; - size_t l_row = threadIdx.x; - size_t l_col = threadIdx.y; - - // Shared memory layout: - // local_A: [tile_size × chunk_k], row-major (row = l_row, col = k_local) - // local_B: [chunk_k × tile_size], row-major (row = k_local, col = l_col) - auto tile_elems_a = tile_size * chunk_k; - auto local_ptr = reinterpret_cast(__local_mem()); - auto local_A = local_ptr; - auto local_B = local_A + tile_elems_a; - - TYPE sum(0); - - for (size_t k = 0; k < size; k += chunk_k) { - // Cooperative load: each thread loads chunk_k/tile_size elements per tile. - // Thread (l_row, l_col) loads columns l_col, l_col+tile_size, ... of its row. - for (size_t kk = l_col; kk < chunk_k; kk += tile_size) { - local_A[l_row * chunk_k + kk] = A_ptr[g_row * size + (k + kk)]; +static_assert(sizeof(TYPE) == sizeof(uint32_t), "copy_word moves 32-bit elements"); + +typedef uint32_t __attribute__((may_alias)) word_t; + +// Copies one element through an integer register, which leaves the floating- +// point register file to the accumulators. +static inline __attribute__((always_inline)) +void copy_word(TYPE* dst, const TYPE* src) { + *reinterpret_cast(dst) = *reinterpret_cast(src); +} + +// Shared-memory tiled SGEMM with register blocking. +// +// A block computes a TILE x TILE tile of C with (TILE/REG)^2 threads; each +// thread owns REG x REG outputs at rows ty + i*TX and columns tx + j*TX +// (TX = TILE/REG). threadIdx.x walks columns, so a warp reads consecutive B +// words and shares the A words of its row(s). Each k-step stages a +// TILE x CHUNK slice of A and a CHUNK x TILE slice of B in local memory. +template +static inline __attribute__((always_inline)) +void sgemm_tile(const TYPE* A, const TYPE* B, TYPE* C, uint32_t size) { + constexpr uint32_t TX = TILE / REG; + constexpr uint32_t NT = TX * TX; + constexpr uint32_t LOADS = (TILE * CHUNK) / NT; + static_assert((TILE % REG) == 0, "tile must be a multiple of reg"); + static_assert(((TILE * CHUNK) % NT) == 0, "slice must split evenly over threads"); + static_assert((NT % CHUNK) == 0 && (NT % TILE) == 0, "a copy pass must cover whole rows"); + + uint32_t tx = threadIdx.x; + uint32_t ty = threadIdx.y; + uint32_t tid = ty * TX + tx; + uint32_t row0 = blockIdx.y * TILE; + uint32_t col0 = blockIdx.x * TILE; + + auto local_A = reinterpret_cast(__local_mem()); // [TILE][CHUNK] + auto local_B = local_A + TILE * CHUNK; // [CHUNK][TILE] + + // Cooperative-load sources: consecutive threads copy consecutive words, and + // each pass covers NT/CHUNK rows of A and NT/TILE rows of B. + const TYPE* src_A = A + (row0 + tid / CHUNK) * size + tid % CHUNK; + const TYPE* src_B = B + (tid / TILE) * size + col0 + tid % TILE; + uint32_t step_A = (NT / CHUNK) * size; + uint32_t step_B = (NT / TILE) * size; + + TYPE acc[REG][REG]; + #pragma unroll + for (uint32_t i = 0; i < REG; ++i) { + #pragma unroll + for (uint32_t j = 0; j < REG; ++j) { + acc[i][j] = 0; } - for (size_t kk = l_row; kk < chunk_k; kk += tile_size) { - local_B[kk * tile_size + l_col] = B_ptr[(k + kk) * size + g_col]; + } + + for (uint32_t k0 = 0; k0 < size; k0 += CHUNK) { + #pragma unroll + for (uint32_t l = 0; l < LOADS; ++l) { + copy_word(&local_A[tid + l * NT], &src_A[l * step_A]); } + #pragma unroll + for (uint32_t l = 0; l < LOADS; ++l) { + copy_word(&local_B[tid + l * NT], &src_B[l * step_B]); + } + src_A += CHUNK; + src_B += CHUNK * size; __syncthreads(); - for (size_t j = 0; j < chunk_k; ++j) { - sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; + #pragma unroll + for (uint32_t k = 0; k < CHUNK; ++k) { + TYPE a[REG], b[REG]; + #pragma unroll + for (uint32_t i = 0; i < REG; ++i) { + a[i] = local_A[(ty + i * TX) * CHUNK + k]; + } + #pragma unroll + for (uint32_t j = 0; j < REG; ++j) { + b[j] = local_B[k * TILE + tx + j * TX]; + } + #pragma unroll + for (uint32_t i = 0; i < REG; ++i) { + #pragma unroll + for (uint32_t j = 0; j < REG; ++j) { + acc[i][j] += a[i] * b[j]; + } + } } __syncthreads(); } - C_ptr[g_row * size + g_col] = sum; + #pragma unroll + for (uint32_t i = 0; i < REG; ++i) { + #pragma unroll + for (uint32_t j = 0; j < REG; ++j) { + C[(row0 + ty + i * TX) * size + col0 + tx + j * TX] = acc[i][j]; + } + } +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto A = reinterpret_cast(arg->A_addr); + auto B = reinterpret_cast(arg->B_addr); + auto C = reinterpret_cast(arg->C_addr); + uint32_t size = arg->size; + uint32_t tile = arg->tile_size; + uint32_t chunk = arg->chunk_k; + uint32_t reg = arg->reg_size; +#define SGEMM2_DISPATCH(T, K, R) \ + if (tile == T && chunk == K && reg == R) { sgemm_tile(A, B, C, size); return; } + SGEMM2_CONFIGS(SGEMM2_DISPATCH) +#undef SGEMM2_DISPATCH } diff --git a/tests/regression/sgemm2/main.cpp b/tests/regression/sgemm2/main.cpp index 04e7b5376e..e454658795 100644 --- a/tests/regression/sgemm2/main.cpp +++ b/tests/regression/sgemm2/main.cpp @@ -88,6 +88,7 @@ const char* kernel_file = "kernel.vxbin"; uint32_t size = 64; uint32_t tile_size = 4; uint32_t chunk_k = 0; +uint32_t reg_size = 1; vx_device_h device = nullptr; vx_buffer_h A_buffer = nullptr; @@ -100,13 +101,20 @@ kernel_arg_t kernel_arg = {}; static void show_usage() { std::cout << "Vortex Test." << std::endl; - std::cout << "Usage: [-k kernel] [-n matrix_size] [-t tile_size] [-c chunk_k] [-h help]" << std::endl; - std::cout << " chunk_k=0 means chunk_k=tile_size (original behavior)" << std::endl; + std::cout << "Usage: [-k kernel] [-n matrix_size] [-t tile_size] [-c chunk_k] [-r reg_size] [-h help]" << std::endl; + std::cout << " tile_size: each block computes a tile_size x tile_size tile of C" << std::endl; + std::cout << " chunk_k: k-slice staged in local memory per step (0: the first supported chunk for tile_size, reg_size)" << std::endl; + std::cout << " reg_size: outputs per thread per dimension; block = (tile_size/reg_size)^2 threads" << std::endl; + std::cout << " supported (tile_size, chunk_k, reg_size):"; +#define SGEMM2_PRINT(T, K, R) std::cout << " (" << T << "," << K << "," << R << ")"; + SGEMM2_CONFIGS(SGEMM2_PRINT) +#undef SGEMM2_PRINT + std::cout << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "n:t:c:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:t:c:r:k:h")) != -1) { switch (c) { case 'n': size = atoi(optarg); @@ -117,6 +125,9 @@ static void parse_args(int argc, char **argv) { case 'c': chunk_k = atoi(optarg); break; + case 'r': + reg_size = atoi(optarg); + break; case 'k': kernel_file = optarg; break; @@ -144,6 +155,20 @@ void cleanup() { } } +static bool is_supported(uint32_t tile, uint32_t chunk, uint32_t reg) { +#define SGEMM2_MATCH(T, K, R) if (tile == T && chunk == K && reg == R) { return true; } + SGEMM2_CONFIGS(SGEMM2_MATCH) +#undef SGEMM2_MATCH + return false; +} + +static uint32_t default_chunk(uint32_t tile, uint32_t reg) { +#define SGEMM2_FIRST(T, K, R) if (tile == T && reg == R) { return K; } + SGEMM2_CONFIGS(SGEMM2_FIRST) +#undef SGEMM2_FIRST + return tile; +} + int main(int argc, char *argv[]) { // parse command arguments parse_args(argc, argv); @@ -153,9 +178,14 @@ int main(int argc, char *argv[]) { return -1; } - // Default chunk_k to tile_size. - if (chunk_k == 0) - chunk_k = tile_size; + if (chunk_k == 0) { + chunk_k = default_chunk(tile_size, reg_size); + } + if (!is_supported(tile_size, chunk_k, reg_size)) { + printf("Error: unsupported tile_size=%d, chunk_k=%d, reg_size=%d\n", tile_size, chunk_k, reg_size); + show_usage(); + return -1; + } if ((size % chunk_k) != 0) { printf("Error: matrix size %d must be a multiple of chunk_k %d\n", size, chunk_k); return -1; @@ -172,14 +202,15 @@ int main(int argc, char *argv[]) { uint32_t size_sq = size * size; uint32_t buf_size = size_sq * sizeof(TYPE); - uint32_t group_size = tile_size * tile_size; + uint32_t block_size = tile_size / reg_size; + uint32_t group_size = block_size * block_size; uint32_t tile_elems_a = tile_size * chunk_k; uint32_t tile_elems_b = chunk_k * tile_size; uint32_t local_mem = (tile_elems_a + tile_elems_b) * sizeof(TYPE); std::cout << "data type: " << Comparator::type_str() << std::endl; std::cout << "matrix size: " << size << "x" << size << std::endl; - std::cout << "tile size: " << tile_size << ", chunk_k: " << chunk_k << std::endl; + std::cout << "tile size: " << tile_size << ", chunk_k: " << chunk_k << ", reg size: " << reg_size << std::endl; std::cout << "local memory: " << local_mem << " bytes" << std::endl; // check work group occupancy @@ -226,9 +257,10 @@ int main(int argc, char *argv[]) { kernel_arg.size = size; kernel_arg.tile_size = tile_size; kernel_arg.chunk_k = chunk_k; + kernel_arg.reg_size = reg_size; uint32_t grid_dim[2] = {size / tile_size, size / tile_size}; - uint32_t block_dim[2] = {tile_size, tile_size}; + uint32_t block_dim[2] = {block_size, block_size}; // launch kernel — args passed as a host blob (UVA), no args device buffer std::cout << "launch kernel" << std::endl; @@ -280,7 +312,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return 1; } std::cout << "PASSED!" << std::endl; From dd11947f5c33392547774166dc6c1fa3320381f5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 27 Oct 2026 23:32:08 -0700 Subject: [PATCH 127/336] sgemm2: use the CUDA samples' tiled matrixMul as is The register-blocked kernel was not the code people know, so it could not show whether Vortex local memory speeds up a standard shared-memory kernel. Each thread now computes one element of C and the block steps BLOCK_SIZE x BLOCK_SIZE tiles of A and B through local memory, exactly as in the CUDA samples; -t selects BLOCK_SIZE (4, 8, 16 or 32). Co-Authored-By: Claude Opus 5.5 --- tests/regression/sgemm2/common.h | 18 +---- tests/regression/sgemm2/kernel.cpp | 116 ++++++----------------------- tests/regression/sgemm2/main.cpp | 70 ++++------------- 3 files changed, 37 insertions(+), 167 deletions(-) diff --git a/tests/regression/sgemm2/common.h b/tests/regression/sgemm2/common.h index 258961a1a3..1f25306211 100644 --- a/tests/regression/sgemm2/common.h +++ b/tests/regression/sgemm2/common.h @@ -5,25 +5,9 @@ #define TYPE float #endif -// Supported (tile, chunk_k, reg) shapes. The kernel is instantiated for each -// one, so the tile loops have compile-time bounds; the host rejects others. -// tile : the block computes a tile x tile block of C -// chunk: the k-slice of A and B staged in local memory per step -// reg : each thread computes reg x reg outputs (block = (tile/reg)^2 threads) -#define SGEMM2_CONFIGS(X) \ - X(4, 4, 1) X(4, 16, 1) \ - X(8, 8, 1) X(8, 32, 1) \ - X(16, 16, 1) X(16, 32, 1) \ - X(32, 32, 1) \ - X(8, 8, 2) X(16, 16, 2) X(32, 16, 2) X(64, 16, 2) \ - X(16, 16, 4) X(32, 16, 4) X(64, 16, 4) X(64, 8, 4) \ - X(128, 8, 4) - typedef struct { uint32_t size; - uint32_t tile_size; - uint32_t chunk_k; - uint32_t reg_size; + uint32_t block_size; uint64_t A_addr; uint64_t B_addr; uint64_t C_addr; diff --git a/tests/regression/sgemm2/kernel.cpp b/tests/regression/sgemm2/kernel.cpp index b88055bc5e..3961f24ece 100644 --- a/tests/regression/sgemm2/kernel.cpp +++ b/tests/regression/sgemm2/kernel.cpp @@ -1,115 +1,43 @@ #include #include "common.h" -static_assert(sizeof(TYPE) == sizeof(uint32_t), "copy_word moves 32-bit elements"); - -typedef uint32_t __attribute__((may_alias)) word_t; - -// Copies one element through an integer register, which leaves the floating- -// point register file to the accumulators. -static inline __attribute__((always_inline)) -void copy_word(TYPE* dst, const TYPE* src) { - *reinterpret_cast(dst) = *reinterpret_cast(src); -} - -// Shared-memory tiled SGEMM with register blocking. -// -// A block computes a TILE x TILE tile of C with (TILE/REG)^2 threads; each -// thread owns REG x REG outputs at rows ty + i*TX and columns tx + j*TX -// (TX = TILE/REG). threadIdx.x walks columns, so a warp reads consecutive B -// words and shares the A words of its row(s). Each k-step stages a -// TILE x CHUNK slice of A and a CHUNK x TILE slice of B in local memory. -template -static inline __attribute__((always_inline)) -void sgemm_tile(const TYPE* A, const TYPE* B, TYPE* C, uint32_t size) { - constexpr uint32_t TX = TILE / REG; - constexpr uint32_t NT = TX * TX; - constexpr uint32_t LOADS = (TILE * CHUNK) / NT; - static_assert((TILE % REG) == 0, "tile must be a multiple of reg"); - static_assert(((TILE * CHUNK) % NT) == 0, "slice must split evenly over threads"); - static_assert((NT % CHUNK) == 0 && (NT % TILE) == 0, "a copy pass must cover whole rows"); +// Shared-memory tiled matrix multiply, as in the CUDA samples' matrixMul: each +// thread computes one element of C, and the block steps BLOCK_SIZE x +// BLOCK_SIZE tiles of A and B through local memory. +template +static void matrix_mul(const TYPE* A, const TYPE* B, TYPE* C, uint32_t size) { + auto As = reinterpret_cast(__local_mem()); // [BLOCK_SIZE][BLOCK_SIZE] + auto Bs = As + BLOCK_SIZE * BLOCK_SIZE; // [BLOCK_SIZE][BLOCK_SIZE] uint32_t tx = threadIdx.x; uint32_t ty = threadIdx.y; - uint32_t tid = ty * TX + tx; - uint32_t row0 = blockIdx.y * TILE; - uint32_t col0 = blockIdx.x * TILE; - - auto local_A = reinterpret_cast(__local_mem()); // [TILE][CHUNK] - auto local_B = local_A + TILE * CHUNK; // [CHUNK][TILE] - - // Cooperative-load sources: consecutive threads copy consecutive words, and - // each pass covers NT/CHUNK rows of A and NT/TILE rows of B. - const TYPE* src_A = A + (row0 + tid / CHUNK) * size + tid % CHUNK; - const TYPE* src_B = B + (tid / TILE) * size + col0 + tid % TILE; - uint32_t step_A = (NT / CHUNK) * size; - uint32_t step_B = (NT / TILE) * size; - - TYPE acc[REG][REG]; - #pragma unroll - for (uint32_t i = 0; i < REG; ++i) { - #pragma unroll - for (uint32_t j = 0; j < REG; ++j) { - acc[i][j] = 0; - } - } - - for (uint32_t k0 = 0; k0 < size; k0 += CHUNK) { - #pragma unroll - for (uint32_t l = 0; l < LOADS; ++l) { - copy_word(&local_A[tid + l * NT], &src_A[l * step_A]); - } - #pragma unroll - for (uint32_t l = 0; l < LOADS; ++l) { - copy_word(&local_B[tid + l * NT], &src_B[l * step_B]); - } - src_A += CHUNK; - src_B += CHUNK * size; + uint32_t row = blockIdx.y * BLOCK_SIZE + ty; + uint32_t col = blockIdx.x * BLOCK_SIZE + tx; + TYPE sum = 0; + for (uint32_t t = 0; t < size; t += BLOCK_SIZE) { + As[ty * BLOCK_SIZE + tx] = A[row * size + t + tx]; + Bs[ty * BLOCK_SIZE + tx] = B[(t + ty) * size + col]; __syncthreads(); #pragma unroll - for (uint32_t k = 0; k < CHUNK; ++k) { - TYPE a[REG], b[REG]; - #pragma unroll - for (uint32_t i = 0; i < REG; ++i) { - a[i] = local_A[(ty + i * TX) * CHUNK + k]; - } - #pragma unroll - for (uint32_t j = 0; j < REG; ++j) { - b[j] = local_B[k * TILE + tx + j * TX]; - } - #pragma unroll - for (uint32_t i = 0; i < REG; ++i) { - #pragma unroll - for (uint32_t j = 0; j < REG; ++j) { - acc[i][j] += a[i] * b[j]; - } - } + for (uint32_t k = 0; k < BLOCK_SIZE; ++k) { + sum += As[ty * BLOCK_SIZE + k] * Bs[k * BLOCK_SIZE + tx]; } - __syncthreads(); } - #pragma unroll - for (uint32_t i = 0; i < REG; ++i) { - #pragma unroll - for (uint32_t j = 0; j < REG; ++j) { - C[(row0 + ty + i * TX) * size + col0 + tx + j * TX] = acc[i][j]; - } - } + C[row * size + col] = sum; } __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - uint32_t size = arg->size; - uint32_t tile = arg->tile_size; - uint32_t chunk = arg->chunk_k; - uint32_t reg = arg->reg_size; -#define SGEMM2_DISPATCH(T, K, R) \ - if (tile == T && chunk == K && reg == R) { sgemm_tile(A, B, C, size); return; } - SGEMM2_CONFIGS(SGEMM2_DISPATCH) -#undef SGEMM2_DISPATCH + switch (arg->block_size) { + case 4: matrix_mul<4>(A, B, C, arg->size); break; + case 8: matrix_mul<8>(A, B, C, arg->size); break; + case 16: matrix_mul<16>(A, B, C, arg->size); break; + case 32: matrix_mul<32>(A, B, C, arg->size); break; + } } diff --git a/tests/regression/sgemm2/main.cpp b/tests/regression/sgemm2/main.cpp index e454658795..667a85f4fe 100644 --- a/tests/regression/sgemm2/main.cpp +++ b/tests/regression/sgemm2/main.cpp @@ -86,9 +86,7 @@ static void matmul_cpu(TYPE* out, const TYPE* A, const TYPE* B, uint32_t width, const char* kernel_file = "kernel.vxbin"; uint32_t size = 64; -uint32_t tile_size = 4; -uint32_t chunk_k = 0; -uint32_t reg_size = 1; +uint32_t block_size = 4; vx_device_h device = nullptr; vx_buffer_h A_buffer = nullptr; @@ -101,32 +99,19 @@ kernel_arg_t kernel_arg = {}; static void show_usage() { std::cout << "Vortex Test." << std::endl; - std::cout << "Usage: [-k kernel] [-n matrix_size] [-t tile_size] [-c chunk_k] [-r reg_size] [-h help]" << std::endl; - std::cout << " tile_size: each block computes a tile_size x tile_size tile of C" << std::endl; - std::cout << " chunk_k: k-slice staged in local memory per step (0: the first supported chunk for tile_size, reg_size)" << std::endl; - std::cout << " reg_size: outputs per thread per dimension; block = (tile_size/reg_size)^2 threads" << std::endl; - std::cout << " supported (tile_size, chunk_k, reg_size):"; -#define SGEMM2_PRINT(T, K, R) std::cout << " (" << T << "," << K << "," << R << ")"; - SGEMM2_CONFIGS(SGEMM2_PRINT) -#undef SGEMM2_PRINT - std::cout << std::endl; + std::cout << "Usage: [-k kernel] [-n matrix_size] [-t block_size] [-h help]" << std::endl; + std::cout << " block_size: 4, 8, 16 or 32; a block of block_size x block_size threads computes a tile of C" << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "n:t:c:r:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:t:k:h")) != -1) { switch (c) { case 'n': size = atoi(optarg); break; case 't': - tile_size = atoi(optarg); - break; - case 'c': - chunk_k = atoi(optarg); - break; - case 'r': - reg_size = atoi(optarg); + block_size = atoi(optarg); break; case 'k': kernel_file = optarg; @@ -155,39 +140,17 @@ void cleanup() { } } -static bool is_supported(uint32_t tile, uint32_t chunk, uint32_t reg) { -#define SGEMM2_MATCH(T, K, R) if (tile == T && chunk == K && reg == R) { return true; } - SGEMM2_CONFIGS(SGEMM2_MATCH) -#undef SGEMM2_MATCH - return false; -} - -static uint32_t default_chunk(uint32_t tile, uint32_t reg) { -#define SGEMM2_FIRST(T, K, R) if (tile == T && reg == R) { return K; } - SGEMM2_CONFIGS(SGEMM2_FIRST) -#undef SGEMM2_FIRST - return tile; -} - int main(int argc, char *argv[]) { // parse command arguments parse_args(argc, argv); - if ((size / tile_size) * tile_size != size) { - printf("Error: matrix size %d must be a multiple of tile size %d\n", size, tile_size); - return -1; - } - - if (chunk_k == 0) { - chunk_k = default_chunk(tile_size, reg_size); - } - if (!is_supported(tile_size, chunk_k, reg_size)) { - printf("Error: unsupported tile_size=%d, chunk_k=%d, reg_size=%d\n", tile_size, chunk_k, reg_size); + if (block_size != 4 && block_size != 8 && block_size != 16 && block_size != 32) { + printf("Error: unsupported block size %d\n", block_size); show_usage(); return -1; } - if ((size % chunk_k) != 0) { - printf("Error: matrix size %d must be a multiple of chunk_k %d\n", size, chunk_k); + if ((size % block_size) != 0) { + printf("Error: matrix size %d must be a multiple of block size %d\n", size, block_size); return -1; } @@ -202,15 +165,12 @@ int main(int argc, char *argv[]) { uint32_t size_sq = size * size; uint32_t buf_size = size_sq * sizeof(TYPE); - uint32_t block_size = tile_size / reg_size; uint32_t group_size = block_size * block_size; - uint32_t tile_elems_a = tile_size * chunk_k; - uint32_t tile_elems_b = chunk_k * tile_size; - uint32_t local_mem = (tile_elems_a + tile_elems_b) * sizeof(TYPE); + uint32_t local_mem = 2 * group_size * sizeof(TYPE); std::cout << "data type: " << Comparator::type_str() << std::endl; std::cout << "matrix size: " << size << "x" << size << std::endl; - std::cout << "tile size: " << tile_size << ", chunk_k: " << chunk_k << ", reg size: " << reg_size << std::endl; + std::cout << "block size: " << block_size << std::endl; std::cout << "local memory: " << local_mem << " bytes" << std::endl; // check work group occupancy @@ -254,12 +214,10 @@ int main(int argc, char *argv[]) { RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); - kernel_arg.size = size; - kernel_arg.tile_size = tile_size; - kernel_arg.chunk_k = chunk_k; - kernel_arg.reg_size = reg_size; + kernel_arg.size = size; + kernel_arg.block_size = block_size; - uint32_t grid_dim[2] = {size / tile_size, size / tile_size}; + uint32_t grid_dim[2] = {size / block_size, size / block_size}; uint32_t block_dim[2] = {block_size, block_size}; // launch kernel — args passed as a host blob (UVA), no args device buffer From 8fb34d817ae743b93f884fb865f0c408ee616e34 Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 27 Oct 2026 23:32:21 -0700 Subject: [PATCH 128/336] amo: cover every AMO op on local memory Only amoadd, LR/SC and amocas were exercised on the LMEM banks. The new amo_ops_lmem case has every hart of a group apply swap, and, or, xor, min, max, minu and maxu to shared local-memory words, so the banks serialize each operation; the group leader rebuilds the expected words and checks the swap chain's sum. Co-Authored-By: Claude Opus 5.5 --- tests/regression/amo/common.h | 3 ++ tests/regression/amo/kernel.cpp | 88 ++++++++++++++++++++++++++++++-- tests/regression/amo/testcases.h | 35 ++++++++++++- 3 files changed, 120 insertions(+), 6 deletions(-) diff --git a/tests/regression/amo/common.h b/tests/regression/amo/common.h index e9828a2d2f..f9017a823c 100644 --- a/tests/regression/amo/common.h +++ b/tests/regression/amo/common.h @@ -13,6 +13,9 @@ #define CAS_LMEM_SLOTS 16 #define CAS_LMEM_GROUP_MARK 0x10000u +// Initial word of the local-memory swap chain in the AMO op sweep. +#define AMO_OPS_SWAP_INIT 0x5EED5EEDu + // Args shared between host (main.cpp) and the GPU kernel (kernel.cpp). // Layout matches dogfood's, with `iters` added for the contention loops. typedef struct { diff --git a/tests/regression/amo/kernel.cpp b/tests/regression/amo/kernel.cpp index 4e9733015f..56ba507fed 100644 --- a/tests/regression/amo/kernel.cpp +++ b/tests/regression/amo/kernel.cpp @@ -336,8 +336,87 @@ void kernel_lrsc_lmem(kernel_arg_t* __UNIFORM__ arg) { } } +// 15) LMEM AMO op sweep. +// Every hart of a group applies swap, and, or, xor, min, max, minu and maxu +// to one local-memory word per operation, all harts on the same words, so +// the LMEM banks serialize every operation. The leader then rebuilds each +// final word from the group's contributions, checks that the swapped-out +// values and the final word sum to the initial word plus every value swapped +// in, and publishes its mismatch count with the group marker. +#define AMO_LMEM(op, addr, val) ({ \ + uint32_t _old; \ + asm volatile (op " %0, %2, (%1)" \ + : "=r"(_old) : "r"(addr), "r"(val) : "memory"); \ + _old; }) + +static inline uint32_t amo_ops_value(uint32_t t) { + return t * 0x9E3779B1u + 0x01234567u; +} + +void kernel_amo_ops_lmem(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = (uint32_t*)__local_mem(); + auto per_hart = (uint32_t*)arg->per_hart_addr; + const uint32_t hid = hart_id(); + const uint32_t t = threadIdx.x; + const uint32_t n = blockDim.x; + // 0 swap, 1 and, 2 or, 3 xor, 4 min, 5 max, 6 minu, 7 maxu, then each + // hart's swapped-out value + auto olds = &lmem[8]; + + if (t == 0) { + lmem[0] = AMO_OPS_SWAP_INIT; + lmem[1] = ~0u; + lmem[2] = 0; + lmem[3] = 0; + lmem[4] = 0x7fffffffu; + lmem[5] = 0x80000000u; + lmem[6] = ~0u; + lmem[7] = 0; + } + __syncthreads(); + + uint32_t v = amo_ops_value(t); + uint32_t bit = 1u << (t % 32); + olds[t] = AMO_LMEM("amoswap.w", &lmem[0], v); + AMO_LMEM("amoand.w", &lmem[1], ~bit); + AMO_LMEM("amoor.w", &lmem[2], bit); + AMO_LMEM("amoxor.w", &lmem[3], v); + AMO_LMEM("amomin.w", &lmem[4], v); + AMO_LMEM("amomax.w", &lmem[5], v); + AMO_LMEM("amominu.w", &lmem[6], v); + AMO_LMEM("amomaxu.w", &lmem[7], v); + __syncthreads(); + + uint32_t mismatches = 0; + if (t == 0) { + uint32_t and_e = ~0u, or_e = 0, xor_e = 0, minu_e = ~0u, maxu_e = 0, sum_in = 0, sum_out = 0; + int32_t min_e = 0x7fffffff, max_e = (int32_t)0x80000000u; + for (uint32_t i = 0; i < n; ++i) { + uint32_t vi = amo_ops_value(i); + and_e &= ~(1u << (i % 32)); + or_e |= 1u << (i % 32); + xor_e ^= vi; + min_e = ((int32_t)vi < min_e) ? (int32_t)vi : min_e; + max_e = ((int32_t)vi > max_e) ? (int32_t)vi : max_e; + minu_e = (vi < minu_e) ? vi : minu_e; + maxu_e = (vi > maxu_e) ? vi : maxu_e; + sum_in += vi; + sum_out += olds[i]; + } + mismatches += (sum_out + lmem[0]) != (AMO_OPS_SWAP_INIT + sum_in); + mismatches += lmem[1] != and_e; + mismatches += lmem[2] != or_e; + mismatches += lmem[3] != xor_e; + mismatches += (int32_t)lmem[4] != min_e; + mismatches += (int32_t)lmem[5] != max_e; + mismatches += lmem[6] != minu_e; + mismatches += lmem[7] != maxu_e; + } + per_hart[hid] = mismatches + ((t == 0) ? CAS_LMEM_GROUP_MARK : 0); +} + #if VX_CFG_EXT_ZACAS_ENABLED -// 15) Native compare-and-swap contention. +// 16) Native compare-and-swap contention. // Every hart races to claim each slot of a ladder with its own id, so for // slot i exactly one hart sees the empty value and swaps, and the others see // that winner's id. Each hart records how many slots it won. @@ -367,7 +446,7 @@ void kernel_cas_ladder(kernel_arg_t* __UNIFORM__ arg) { per_hart[hid] = wins; } -// 16) LMEM compare-and-swap ladder. +// 17) LMEM compare-and-swap ladder. // The cas_ladder race moved onto local memory, so the swap and the old word // it returns come from the LMEM banks. Each group races for its own ladder // and exactly one of its harts wins each slot, so the win counts sum to @@ -417,9 +496,10 @@ static const PFN_Kernel sc_tests[] = { kernel_self_consistency, // 12 issuer self-consistency (load->AMO->load) kernel_amoadd_lmem, // 13 AMOADD on local memory (LMEM banks) kernel_lrsc_lmem, // 14 LR/SC on local memory (LMEM banks) + kernel_amo_ops_lmem, // 15 every other AMO op on local memory (LMEM banks) #if VX_CFG_EXT_ZACAS_ENABLED - kernel_cas_ladder, // 15 native compare-and-swap (Zacas amocas) - kernel_cas_lmem, // 16 compare-and-swap on local memory (LMEM banks) + kernel_cas_ladder, // 16 native compare-and-swap (Zacas amocas) + kernel_cas_lmem, // 17 compare-and-swap on local memory (LMEM banks) #endif }; diff --git a/tests/regression/amo/testcases.h b/tests/regression/amo/testcases.h index ff69860e5e..bc3bdb4e7e 100644 --- a/tests/regression/amo/testcases.h +++ b/tests/regression/amo/testcases.h @@ -404,7 +404,37 @@ class Test_LRSC_LMEM : public ITestCase { } }; -// 15) Native compare-and-swap ladder: every hart races for each slot, so each +// 15) LMEM AMO op sweep: swap, and, or, xor, min, max, minu and maxu on local +// memory. Each group's leader checks its group's final words in the kernel +// and publishes the mismatch count with the group marker. +class Test_AMO_OPS_LMEM : public ITestCase { +public: + Test_AMO_OPS_LMEM(TestSuite* s) : ITestCase(s, "amo_ops_lmem") {} + void setup(uint32_t n, uint32_t* /*shared*/, uint32_t* per_hart) override { + for (uint32_t h = 0; h < n; ++h) per_hart[h] = 0; + } + int verify(uint32_t n, uint32_t /*iters*/, + const uint32_t* /*shared*/, const uint32_t* per_hart) override { + uint32_t groups = 0, mismatches = 0; + for (uint32_t h = 0; h < n; ++h) { + uint32_t v = per_hart[h]; + if (v >= CAS_LMEM_GROUP_MARK) { ++groups; v -= CAS_LMEM_GROUP_MARK; } + mismatches += v; + } + if (groups == 0) { + std::cout << " no group published -- the kernel did not run" << std::endl; + return 1; + } + if (mismatches != 0) { + std::cout << " " << mismatches << " wrong final word(s) across " + << groups << " group(s)" << std::endl; + return 1; + } + return 0; + } +}; + +// 16) Native compare-and-swap ladder: every hart races for each slot, so each // slot has exactly one winner. Summing the per-hart win counts checks the // per-lane return value, which a compare-and-swap that returned the wrong // old word would get wrong even while leaving the ladder itself plausible. @@ -438,7 +468,7 @@ class Test_CAS_LADDER : public ITestCase { } }; -// 16) LMEM compare-and-swap ladder: the cas_ladder race on local memory, so +// 17) LMEM compare-and-swap ladder: the cas_ladder race on local memory, so // the LMEM banks resolve the swap and the returned old word. Each group // has its own ladder with exactly one winner per slot; the leaders' publish // marker carries the group count, so the win counts must sum to @@ -495,6 +525,7 @@ inline TestSuite::TestSuite(vx_device_h device) : device_(device) { add_test(new Test_SELF_CONSISTENCY(this)); add_test(new Test_AMOADD_LMEM(this)); add_test(new Test_LRSC_LMEM(this)); + add_test(new Test_AMO_OPS_LMEM(this)); #if VX_CFG_EXT_ZACAS_ENABLED // Native compare-and-swap only exists when Zacas is built in. Without it a // compare-exchange lowers to an LR/SC retry loop, which is the construct From db094ba39f8ce92aa7dad93b0367638676b13e7e Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 27 Oct 2026 23:32:27 -0700 Subject: [PATCH 129/336] config: fetch one slice per issue slice by default FETCH_WIDTH stayed at 1, so an IW>1 core fetched every warp through one scheduler and icache port while each issue slice waited on it. It now defaults to the largest power of two dividing ISSUE_WIDTH, so each issue slice schedules and fetches its own warps. IW=1 configs, every gate DUT among them, are unchanged. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/VX_config.toml b/VX_config.toml index bac730f67c..0a70158668 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -57,8 +57,9 @@ VX_CFG_IBUF_SIZE = 4 VX_CFG_DISPATCH_QUEUE_SIZE = 4 VX_CFG_ISSUE_WIDTH = "expr: up($VX_CFG_NUM_WARPS / 16)" # Fetch slices, each scheduling and fetching its own warps on its own icache -# port: a power of two that divides ISSUE_WIDTH. -VX_CFG_FETCH_WIDTH = 1 +# port: a power of two that divides ISSUE_WIDTH. One per issue slice by +# default (the largest power of two dividing ISSUE_WIDTH). +VX_CFG_FETCH_WIDTH = "expr: $VX_CFG_ISSUE_WIDTH & -$VX_CFG_ISSUE_WIDTH" VX_CFG_SIMD_WIDTH = "expr: $VX_CFG_NUM_THREADS" VX_CFG_NUM_OPCS = "expr: up($VX_CFG_NUM_WARPS / (4 * $VX_CFG_ISSUE_WIDTH))" VX_CFG_NUM_GPR_BANKS = 4 From e84ef1dcdb65cceac6fe4aa1e94d3e072e2c82c8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 02:52:00 -0700 Subject: [PATCH 130/336] cp: translate the DMA's device operands through the page table The RTL command processor had no MMU: it reported VM_ENABLED=0, so on xrt and the FPGA the runtime never minted virtual addresses and a VM build ran without virtual memory. The emulation CP used by simx and rtlsim already walked the table, so VM worked only in simulation. VX_cp_dma now walks the Sv32/Sv39 table (superpages included) for each device operand of a CMD_MEM_* before the transfer, reading the PTEs on the device port so it sees what the last-level cache sees. As in the emulation CP, an allocation is one contiguous physical range, so the base translation covers the transfer; F_MEM_PHYSICAL skips the walk and an unmapped address passes through. The regfile gains CP_SATP_LO/HI and advertises VM_ENABLED in a VM build, so the runtime turns VM on with no change of its own. Verified: cp_dma unit test (4 new page-table scenarios, Sv32 and Sv39, and it fails with the walk disabled), cp_axil_regfile and cp_core unit tests with VM on and off; xrtsim vm_test, vecadd, sgemm and dogfood with randomized VAs; the new xrt CI cases cp-dma-walk and cp-dma-walk-megapage at XLEN 32 and 64. The cp DUT meets 300 MHz with VM on (+0.188 ns, worst path outside the DMA). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/vm.yaml | 18 +++ docs/designs/command_processor.md | 67 ++++---- docs/designs/mlp_optimizations.md | 2 +- docs/designs/virtual_memory_subsystem.md | 15 +- hw/rtl/cp/VX_cp_axil_regfile.sv | 27 ++-- hw/rtl/cp/VX_cp_core.sv | 4 + hw/rtl/cp/VX_cp_dma.sv | 125 ++++++++++++-- hw/rtl/cp/VX_cp_pkg.sv | 2 + .../cp_axil_regfile/VX_cp_axil_regfile_top.sv | 3 + hw/unittest/cp_axil_regfile/main.cpp | 12 ++ hw/unittest/cp_dma/VX_cp_dma_top.sv | 2 + hw/unittest/cp_dma/main.cpp | 152 ++++++++++++++---- sw/runtime/common/device.cpp | 6 +- 13 files changed, 332 insertions(+), 103 deletions(-) diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index 6f98894b00..3540d0e161 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -258,3 +258,21 @@ tests: via: blackbox app: vecadd args: -n16384 +# The RTL command processor's DMA walks the page table for every upload and +# download; only xrt runs it (simx and rtlsim use the C++ command processor). +- id: cp-dma-walk + via: blackbox + drivers: + - xrt + app: vm_test + args: -t1 -n16 -p1 + env: + VORTEX_RANDOMIZE_VA: 1 +- id: cp-dma-walk-megapage + via: blackbox + drivers: + - xrt + app: sgemm + args: -n16 + env: + VORTEX_VM_MEGAPAGE_HEAP: 1 diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index 9e0dd9c5a2..e366c9794c 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -108,7 +108,7 @@ architecturally defined; FENCE reads only `arg0[1:0]`, which is inside its |---|---|---| | 0 | `F_PROFILE` | Appends an 8 B `profile_slot` trailer; every size below grows by 8. | | 1 | `F_FENCE_PRE` | Declared in the package; not acted on by any consumer. | -| 2 | `F_MEM_PHYSICAL` | `CMD_MEM_*` device operand is a physical address — the MMU-aware DMA skips translation (§8). Emulation CP only ([`cmd_processor.h:160`](../../sim/common/cmd_processor.h#L160)). | +| 2 | `F_MEM_PHYSICAL` | `CMD_MEM_*` device operand is a physical address — the MMU-aware DMA skips translation (§8). | ### 2.2 Opcode map @@ -388,10 +388,8 @@ Reads are registered (AR latches, R returns the decoded value the next cycle). Undecoded addresses return `0xDEADBEEF` with DECERR on both channels. -> **FPGA/sim divergence:** the RTL regfile has **no** `CP_SATP_LO/HI` -> registers. The Emulation CP *does* (`0x028/0x02C`, -> [`cmd_processor.cpp:72-73`](../../sim/common/cmd_processor.cpp#L72)), -> and the runtime writes them only when VM is advertised. See §8. +`CP_SATP_LO/HI` (`0x028/0x02C`) hold the DMA's page-table root; the runtime +writes them only when `VM_ENABLED` is advertised. See §8. --- @@ -462,11 +460,9 @@ upper byte of the CP's own caps word is free, and the Emulation CP uses it | 25 | `SUPPORTS_DRAW` | The CP decodes `CMD_DRAW` / `OP_DRAW` (§8.1) | | 26 | `SUPPORTS_QMD` | The CP decodes `CMD_LAUNCH_QMD` (§2.3) | -**The RTL CP hardwires all three to zero.** Its `CP_DEV_CAPS` read returns -`{8'd0, AXI_TID_W, RING_SIZE_LOG2_MAX, NUM_QUEUES}` -([`VX_cp_axil_regfile.sv:177-180`](../../hw/rtl/cp/VX_cp_axil_regfile.sv#L177)), -so on FPGA every launch takes the 20-DCR fallback, SATP is never -programmed, and draws stream as a ring batch. The runtime consumes the +**The RTL CP sets `VM_ENABLED` from `VX_CFG_VM_ENABLE` and hardwires +`SUPPORTS_DRAW` / `SUPPORTS_QMD` to zero**, so on FPGA every launch takes the +20-DCR fallback and draws stream as a ring batch. The runtime consumes the bits at open ([`device.cpp:274-282`](../../sw/runtime/common/device.cpp#L274)) and branches accordingly, so the graphics driver calls `vx_enqueue_draw` unconditionally on every backend. @@ -557,19 +553,17 @@ a divergence — with one queue the RTL's arbiters are degenerate too. The Emulation CP's MMIO map matches the RTL regfile **plus**: -- `CP_SATP_LO/HI` at `0x028/0x02C` - ([`cmd_processor.cpp:72-73`](../../sim/common/cmd_processor.cpp#L72)), -- the `CP_DEV_CAPS` feature bits 24-26 (§6), -- a software page-table walk `cp_translate` (Sv32/Sv39, megapage-aware, - [`:157-201`](../../sim/common/cmd_processor.cpp#L157)) honoring the - `MEM_FLAG_PHYSICAL` flag ([`:519-520`](../../sim/common/cmd_processor.cpp#L519)), +- the `CP_DEV_CAPS` feature bits 25-26 (§6), - byte-exact `CMD_MEM_*` copies, staged through a 64 KiB bounce buffer ([`:531-541`](../../sim/common/cmd_processor.cpp#L531)), - the two extra opcodes, `CMD_LAUNCH_QMD` (§2.3) and `CMD_DRAW` (§8.1). -This means **CP DMA is MMU-aware in simulation but not on FPGA** — a -deliberate phased rollout (runtime + emulation first, RTL walker later; -see §10). Two smaller divergences: `CP_STATUS` returns only the busy bit +Both CPs are MMU-aware: with a root in `CP_SATP`, each `CMD_MEM_*` walks +the Sv32/Sv39 table (megapage-aware) for its device operands, once per +command, since an allocation is one contiguous physical range, and +`F_MEM_PHYSICAL` skips the walk (`cp_translate` in the emulation CP; the +`S_XLATE_*` states of `VX_cp_dma`, which read the PTEs on the device port). +Two smaller divergences: `CP_STATUS` returns only the busy bit (no error bit), and unknown MMIO offsets are silently ignored on write rather than raising DECERR. @@ -642,8 +636,7 @@ is written **last**, after the per-queue enable. It then reads `CP_DEV_CAPS` once and latches the VM / DRAW / QMD feature bits ([`:274-282`](../../sw/runtime/common/device.cpp#L274)); `CP_SATP_LO/HI` is written only inside the `if (vm_enabled_)` branch -([`:305-306`](../../sw/runtime/common/device.cpp#L305)) — so on every RTL -CP today (§6) SATP is never programmed. +([`:305-306`](../../sw/runtime/common/device.cpp#L305)). **Submit path.** Each `vx_enqueue_*` pushes a `Command` lambda onto a per-queue worker FIFO ([`queue.cpp:28`](../../sw/runtime/common/queue.cpp#L28) @@ -746,50 +739,46 @@ The following gaps remain **open**. ([`cmd_processor.cpp:531-541`](../../sim/common/cmd_processor.cpp#L531)). Non-cache-line-aligned transfers can over-write on FPGA. Needs tail `wstrb` on the last beat. -2. **VM in RTL.** Add `CP_SATP_LO/HI` regfile decode + a hardware - page-table walker + TLB in `VX_cp_dma`, and route `F_MEM_PHYSICAL`, so - FPGA matches the simulator's MMU-aware DMA. Today VM works on - simx/rtlsim/gem5 and silently no-ops on FPGA. -3. **Real `CMD_FENCE` semantics.** The engine retires FENCE as a NOP +2. **Real `CMD_FENCE` semantics.** The engine retires FENCE as a NOP ([`VX_cp_engine.sv:109-112`](../../hw/rtl/cp/VX_cp_engine.sv#L109)); it should honor `FENCE_DMA_BIT` / `FENCE_GPU_BIT` ordering. -4. **`dcr_req_ready` backpressure.** `VX_cp_dcr_proxy` assumes the DCR bus +3. **`dcr_req_ready` backpressure.** `VX_cp_dcr_proxy` assumes the DCR bus accepts every request in one cycle ([`:115-122`](../../hw/rtl/cp/VX_cp_dcr_proxy.sv#L115)); `VX_cp_core` discards `dcr_req_ready` outright ([`:340`](../../hw/rtl/cp/VX_cp_core.sv#L340)) even though `VX_cp_gpu_slice` now carries a real elastic buffer that can stall. -5. **Resource re-entry under multi-queue** (§4.4, item 2). A grant landing +4. **Resource re-entry under multi-queue** (§4.4, item 2). A grant landing on a busy resource unit is silently dropped while the granted CPE still retires on the previous command's `done`. Latent at `NUM_QUEUES=1`; blocking for anything more. **Performance / feature gaps:** -6. **QMD `CMD_LAUNCH_QMD` in the RTL CP.** The atomic-launch opcode, the +5. **QMD `CMD_LAUNCH_QMD` in the RTL CP.** The atomic-launch opcode, the in-memory descriptor format, the runtime encoder, and the `SUPPORTS_QMD` gate all exist and are validated on the Emulation CP (§2.3); the RTL CP does not decode 0x0B and advertises 0. Until it does, every FPGA launch pays the 20-`CMD_DCR_WRITE` dance, and per-queue launch concurrency stays blocked on item 5. -7. **`CMD_DRAW` in the RTL CP** (§8.1) — same shape: shipped in emulation, +6. **`CMD_DRAW` in the RTL CP** (§8.1) — same shape: shipped in emulation, cap bit cleared in RTL. -8. **Multi-queue everywhere.** RTL is parameterized on `NUM_QUEUES` but +7. **Multi-queue everywhere.** RTL is parameterized on `NUM_QUEUES` but defaults to 1; the Emulation CP models only `q0_`; the runtime programs only queue 0 and funnels every host queue through one ring lock (§9). Real concurrency depends on items 5 and 6. -9. **Batch the launch path.** `enqueue_launch` submits its 21 commands +8. **Batch the launch path.** `enqueue_launch` submits its 21 commands unbatched — 21 doorbells, 21 busy-polls — while `cp_batch_begin/end` already exists and is used by `enqueue_commands`. Cheap win independent of item 6. -10. **EVENT_WAIT fairness + backoff.** `EVENT_WAIT` holds its arbiter +9. **EVENT_WAIT fairness + backoff.** `EVENT_WAIT` holds its arbiter grant for the whole wait ([`VX_cp_event_unit.sv:124-129`](../../hw/rtl/cp/VX_cp_event_unit.sv#L124)); it should release between polls and back off. -11. **Priority arbitration.** `VX_cp_engine` already drives `bid_priority` +10. **Priority arbitration.** `VX_cp_engine` already drives `bid_priority` from `q_state.prio`; `VX_cp_arbiter` ties it off ([`:41`](../../hw/rtl/cp/VX_cp_arbiter.sv#L41)). Wire it through. -12. **Profiling writeback.** [`VX_cp_profiling.sv`](../../hw/rtl/cp/VX_cp_profiling.sv) +11. **Profiling writeback.** [`VX_cp_profiling.sv`](../../hw/rtl/cp/VX_cp_profiling.sv) is **not instantiated by anything** — `VX_cp_axil_regfile` carries its own free-running counter for `CP_CYCLE_LO/HI` at `0x010/0x014` (the profiling module's header comment claiming `0x040/0x044` is stale). @@ -798,19 +787,19 @@ The following gaps remain **open**. To give `vx_event_get_profiling` hardware backing: instantiate the module, emit the 32 B `{queued,submit,start,end}` record, and add a `CP_CYCLE_FREQ_HZ` register. -13. **Interrupt path.** Add IP_ISR/IER/GIER in the AFU and +12. **Interrupt path.** Add IP_ISR/IER/GIER in the AFU and `xrt::ip::interrupt` so the runtime sleeps instead of busy-polling `Q_SEQNUM` (the `irq` pulse already exists, [`VX_cp_core.sv:505`](../../hw/rtl/cp/VX_cp_core.sv#L505)). -14. **Host-coherent completion mailbox / `head_addr` writeback.** The CP +13. **Host-coherent completion mailbox / `head_addr` writeback.** The CP tracks `head` internally and never DMAs it to `head_addr`; the runtime likewise ignores the cmpl slot it programs (§9). A cacheable host mailbox would let both replace the per-poll MMIO read. -15. **A CP DCR range.** `command_processor` has no DCR block of its own for +14. **A CP DCR range.** `command_processor` has no DCR block of its own for CP↔GPU coordination / multi-context KMU. The range once earmarked for it (`0x080–0x0BF`) is now occupied by `[dcr_om]` in `VX_types.toml`, so this needs a fresh allocation rather than the old reservation. -16. **Opcode-constant duplication.** The command opcodes are re-declared in +15. **Opcode-constant duplication.** The command opcodes are re-declared in `VX_cp_pkg.sv`, `cmd_processor.h`, `device.cpp` **and** `queue.cpp`, with no shared header — four copies of one ABI, and a standing drift hazard. diff --git a/docs/designs/mlp_optimizations.md b/docs/designs/mlp_optimizations.md index e93fc29890..8bd6967c35 100644 --- a/docs/designs/mlp_optimizations.md +++ b/docs/designs/mlp_optimizations.md @@ -311,7 +311,7 @@ Atomics with `.aq`/`.rl` get these steps around them. All of it is skipped when | 4 cores + L2 | −9.1% | −23.4% | | | 8 MB L3 | −64% | | | -**Virtual memory.** SimX's command processor translates DMA addresses before they enter the LLC. The RTL command processor has no MMU, so host DMA with virtual memory is a SimX-only configuration for now. +**Virtual memory.** Both command processors translate DMA addresses before they enter the LLC, so a client of the LLC sees physical addresses with virtual memory on. ### Local-memory DMA round-robin diff --git a/docs/designs/virtual_memory_subsystem.md b/docs/designs/virtual_memory_subsystem.md index 8b6a34a6e4..99701ab776 100644 --- a/docs/designs/virtual_memory_subsystem.md +++ b/docs/designs/virtual_memory_subsystem.md @@ -563,14 +563,15 @@ DCRs. Then: ### 8.5 The command processor -The emulation CP ([`cmd_processor.cpp`](../../sim/common/cmd_processor.cpp)) -is MMU-aware: `cp_translate()` performs the same Sv32/Sv39 walk as +Both command processors are MMU-aware: the emulation CP +([`cmd_processor.cpp`](../../sim/common/cmd_processor.cpp), `cp_translate()`) +and the RTL DMA ([`VX_cp_dma.sv`](../../hw/rtl/cp/VX_cp_dma.sv), the +`S_XLATE_*` states) perform the same Sv32/Sv39 walk as `VMManager::page_table_walk` against device memory for every `CMD_MEM_*` VA -operand, skipped by `F_MEM_PHYSICAL`; its root is `CP_SATP_LO/HI` (regfile -0x028/0x02C), programmed at open. The RTL CP has **no MMU**: `CP_SATP` reads -zero, `DEV_CAPS.VM_ENABLED` is 0, and the runtime never mints VAs on FPGA. -VM is therefore a simulation-only feature today -([`command_processor.md`](command_processor.md) §8, §10). +operand, skipped by `F_MEM_PHYSICAL`. Their root is `CP_SATP_LO/HI` (regfile +0x028/0x02C), programmed at open, and both advertise `DEV_CAPS.VM_ENABLED` +in a VM build, so VM runs the same on simulators and FPGA +([`command_processor.md`](command_processor.md) §8). --- diff --git a/hw/rtl/cp/VX_cp_axil_regfile.sv b/hw/rtl/cp/VX_cp_axil_regfile.sv index 96a0ca2fbc..e874dd36c8 100644 --- a/hw/rtl/cp/VX_cp_axil_regfile.sv +++ b/hw/rtl/cp/VX_cp_axil_regfile.sv @@ -15,7 +15,8 @@ // 0x000 CP_CTRL RW bit0=enable_global, bit1=reset_all // 0x004 CP_STATUS RO bit0=busy, bit1=error // 0x008 CP_DEV_CAPS RO [7:0]NUM_QUEUES | [15:8]RING_SIZE_LOG2_MAX -// [23:16]AXI_TID_WIDTH | [28]DMA_LLC_CLIENT +// [23:16]AXI_TID_WIDTH | [24]VM_ENABLED +// [28]DMA_LLC_CLIENT // 0x00C GPU_DEV_CAPS_EXT RO device-config caps extension: [7:0]SOCKET_SIZE // 0x010 CP_CYCLE_LO RO free-running cycle counter low 32 bits // 0x014 CP_CYCLE_HI RO high 32 bits @@ -78,6 +79,9 @@ module VX_cp_axil_regfile // Programmed state out to every CPE. output cpe_state_t q_state [NUM_QUEUES], + // CP_SATP: page-table root of the DMA's device operands. + output wire [63:0] satp, + // One-cycle reset pulse per queue when the host writes Q_CONTROL.reset. output logic q_reset_pulse [NUM_QUEUES], @@ -117,6 +121,8 @@ module VX_cp_axil_regfile // ---- Global registers ---- logic [31:0] r_cp_ctrl; + logic [63:0] r_satp; + assign satp = r_satp; logic [63:0] r_cycle_count; always_ff @(posedge clk) begin @@ -215,7 +221,8 @@ module VX_cp_axil_regfile 6'h04: return {30'd0, cp_error, cp_busy}; 6'h08: return {3'd0, 1'(DMA_LLC_CLIENT), - 4'd0, + 3'd0, + 1'(`VX_CFG_VM_ENABLED), 8'(AXI_TID_W), 8'(RING_SIZE_LOG2_MAX), 8'(NUM_QUEUES)}; @@ -226,15 +233,8 @@ module VX_cp_axil_regfile 6'h1C: return gpu_dev_caps[63:32]; 6'h20: return gpu_isa_caps[31:0]; 6'h24: return gpu_isa_caps[63:32]; - // Block padding -- see the 16-byte block note on is_decoded(). Read - // as zero; they exist so their enclosing block is fully populated. - // 0x28/0x2C are CP_SATP_LO/HI in the software ABI (sw/runtime/common/ - // device.cpp), i.e. the CP DMA MMU page-table root. This CP has no - // MMU -- there is no satp anywhere in hw/rtl/cp -- so it correctly - // reports VM_ENABLED=0 in CP_DEV_CAPS and the runtime never writes - // them. Reading zero is the honest answer; implement them for real if - // an MMU is added. - 6'h28, 6'h2C: return 32'd0; + 6'h28: return r_satp[31:0]; + 6'h2C: return r_satp[63:32]; // AFU host-port drain counters (debug, read-only). 6'h30: return dbg_host_w_counts; 6'h34: return dbg_host_r_counts; @@ -359,6 +359,7 @@ module VX_cp_axil_regfile automatic logic [5:0] off; if (reset) begin r_cp_ctrl <= '0; + r_satp <= '0; for (int i = 0; i < NUM_QUEUES; ++i) begin r_ring_base[i] <= '0; r_head_addr[i] <= '0; @@ -390,6 +391,10 @@ module VX_cp_axil_regfile if (wr_data_buf[1]) begin for (int i = 0; i < NUM_QUEUES; ++i) q_reset_pulse[i] <= 1'b1; end + end else if (is_global(wr_addr_buf, 8'h28)) begin + r_satp[31:0] <= wr_data_buf; + end else if (is_global(wr_addr_buf, 8'h2C)) begin + r_satp[63:32] <= wr_data_buf; end else if (decode_queue(wr_addr_buf, qid, off)) begin case (off) 6'h00: r_ring_base[qid][31:0] <= wr_data_buf; diff --git a/hw/rtl/cp/VX_cp_core.sv b/hw/rtl/cp/VX_cp_core.sv index 417a3584f9..28995c62fe 100644 --- a/hw/rtl/cp/VX_cp_core.sv +++ b/hw/rtl/cp/VX_cp_core.sv @@ -149,6 +149,8 @@ module VX_cp_core logic fetch_idle [NUM_QUEUES]; logic engine_idle [NUM_QUEUES]; + wire [63:0] cp_satp; + VX_cp_axil_regfile #( .NUM_QUEUES (NUM_QUEUES), .ADDR_W (AXIL_AW), @@ -164,6 +166,7 @@ module VX_cp_core .q_error (q_error_to_reg), .last_dcr_rsp (dcr_last_rsp_data), .q_state (q_state), + .satp (cp_satp), .q_reset_pulse (q_reset_pulse), .q_clear_ack (q_clear), .dbg_host_w_counts (dbg_host_w_counts), @@ -378,6 +381,7 @@ module VX_cp_core .grant (any_dma_grant), .cmd (granted_dma_cmd), .done (dma_done), + .satp (cp_satp), .axi_host (dma_host_axi), .axi_dev (dma_dev_axi) ); diff --git a/hw/rtl/cp/VX_cp_dma.sv b/hw/rtl/cp/VX_cp_dma.sv index 0357b9427a..de454a37f4 100644 --- a/hw/rtl/cp/VX_cp_dma.sv +++ b/hw/rtl/cp/VX_cp_dma.sv @@ -27,7 +27,11 @@ // not pipelined. // // FSM: -// S_IDLE : grant -> latch op/dst/src/size -> S_SETUP +// S_IDLE : grant -> latch op/dst/src/size -> S_XLATE_AR +// (no device operand to translate) -> S_SETUP +// S_XLATE_AR/S_XLATE_R : page-table walk of each device operand, one PTE +// read per level on the device port; the leaf rewrites the +// operand with its physical address -> S_SETUP // S_SETUP : size 0 -> S_DONE; else size the chunk -> S_REQ_AR // S_REQ_AR : drive AR on the read port; arready -> S_READ // S_READ : capture rdata beats into buf_r; last beat -> S_REQ_AW @@ -56,6 +60,12 @@ // host polls, and the completion line that follows it) fire. // Device-direction writes don't need this: nothing reads them through a // foreign port on the strength of Q_SEQNUM alone. +// +// VIRTUAL MEMORY. With a page-table root in CP_SATP, a device operand is a +// virtual address, as on a GPU copy engine. Every allocation maps to one +// contiguous physical range, so translating the operand's base covers the +// whole transfer. Commands flagged F_MEM_PHYSICAL (the page tables +// themselves) skip the walk. // ============================================================================ module VX_cp_dma @@ -71,6 +81,9 @@ module VX_cp_dma input cmd_t cmd, output logic done, + // CP_SATP: page-table root for the device operands (0 or bare: physical). + input wire [63:0] satp, + // Host-memory AXI master (command-ring side / upload source / download dst). VX_mem_axi_if.master axi_host, // Device-memory AXI master. @@ -82,10 +95,20 @@ module VX_cp_dma localparam int BCNT_W = 7; // chunk length 1..64 typedef enum logic [3:0] { - S_IDLE, S_SETUP, S_REQ_AR, S_READ, S_REQ_AW, S_WRITE, S_WAIT_B, - S_FLUSH_AR, S_FLUSH_R, S_DONE + S_IDLE, S_XLATE_AR, S_XLATE_R, S_SETUP, S_REQ_AR, S_READ, S_REQ_AW, + S_WRITE, S_WAIT_B, S_FLUSH_AR, S_FLUSH_R, S_DONE } state_e; + // Page-table geometry: Sv32 at XLEN=32, Sv39 at XLEN=64. + localparam int PAGE_LOG2 = `VX_VM_PAGE_LOG2_SIZE; + localparam int PT_LEVELS = `VX_VM_PT_LEVEL; + localparam int LEVEL_BITS = `CLOG2(`VX_VM_PT_SIZE / `VX_VM_PTE_SIZE); + localparam int PTE_BITS = `VX_VM_PTE_SIZE * 8; + localparam int PTE_SHIFT = `CLOG2(`VX_VM_PTE_SIZE); + localparam int PTE_SEL_W = 6 - PTE_SHIFT; + localparam int PPN_W = (PTE_BITS == 32) ? 22 : 44; + localparam int LVL_W = `UP(`CLOG2(PT_LEVELS)); + state_e state; logic [7:0] op_r; // latched opcode (host/dev routing) logic [63:0] dst_r, src_r; @@ -124,6 +147,50 @@ module VX_cp_dma wire rd_from_host = (cp_opcode_e'(op_r) == CMD_MEM_WRITE); // upload: read host wire wr_to_host = (cp_opcode_e'(op_r) == CMD_MEM_READ); // download: write host + // ---- Page-table walk of the device operands ---- +`ifdef VX_VM_ADDR_MODE_SV32 + wire satp_on = satp[31]; + wire [PPN_W-1:0] satp_ppn = satp[PPN_W-1:0]; + `UNUSED_VAR (satp[63:32]) + `UNUSED_VAR (satp[30:PPN_W]) +`else + wire satp_on = (satp[63:60] != '0); + wire [PPN_W-1:0] satp_ppn = satp[PPN_W-1:0]; + `UNUSED_VAR (satp[59:PPN_W]) +`endif + + wire cmd_is_mem = (cp_opcode_e'(cmd.hdr.opcode) == CMD_MEM_WRITE) + || (cp_opcode_e'(cmd.hdr.opcode) == CMD_MEM_READ) + || (cp_opcode_e'(cmd.hdr.opcode) == CMD_MEM_COPY); + wire cmd_xlate = (`VX_CFG_VM_ENABLED != 0) && satp_on && cmd_is_mem + && ~cmd.hdr.flags[F_MEM_PHYSICAL]; + wire cmd_xl_dst = cmd_xlate && (cp_opcode_e'(cmd.hdr.opcode) != CMD_MEM_READ); + wire cmd_xl_src = cmd_xlate && (cp_opcode_e'(cmd.hdr.opcode) != CMD_MEM_WRITE); + + logic xl_dst_r, xl_src_r; // operands still to translate (dst first) + logic [LVL_W-1:0] xl_level_r; + logic [PPN_W-1:0] xl_ppn_r; // table base at the current level + + wire [63:0] xl_va = xl_dst_r ? dst_r : src_r; + wire [LEVEL_BITS-1:0] xl_vpn = xl_va[PAGE_LOG2 + xl_level_r * LEVEL_BITS +: LEVEL_BITS]; + wire [63:0] pte_addr = (64'(xl_ppn_r) << PAGE_LOG2) | (64'(xl_vpn) << PTE_SHIFT); + wire [PTE_SEL_W-1:0] pte_sel = pte_addr[5:PTE_SHIFT]; + wire [PTE_BITS-1:0] pte = axi_dev.rdata[pte_sel * PTE_BITS +: PTE_BITS]; + wire [PPN_W-1:0] pte_ppn = pte[10 +: PPN_W]; + wire pte_v = pte[0]; + wire pte_leaf = pte[1] | pte[2] | pte[3]; // R | W | X + // A leaf above level 0 is a superpage: the VA supplies its low bits. + wire [63:0] off_mask = (64'(1) << (PAGE_LOG2 + xl_level_r * LEVEL_BITS)) - 64'(1); + wire [63:0] leaf_pa = ((64'(pte_ppn) << PAGE_LOG2) & ~off_mask) | (xl_va & off_mask); + // An unmapped or leafless walk leaves the address as it is, as the + // functional model does. + wire xl_last = ~pte_v || pte_leaf || (xl_level_r == '0); + wire [63:0] xl_result = (pte_v && pte_leaf) ? leaf_pa : xl_va; + if (PTE_BITS > 10 + PPN_W) begin : g_pte_unused + `UNUSED_VAR (pte[PTE_BITS-1:10+PPN_W]) + end + `UNUSED_VAR (pte[9:4]) + // Last beat of the current chunk. wire last_beat = (BCNT_W'({1'b0, beat_idx}) == (chunk_beats - BCNT_W'(1))); @@ -137,16 +204,49 @@ module VX_cp_dma rem_beats <= '0; chunk_beats <= '0; beat_idx <= '0; + xl_dst_r <= 1'b0; + xl_src_r <= 1'b0; + xl_level_r <= '0; + xl_ppn_r <= '0; end else begin case (state) S_IDLE: begin if (grant) begin - op_r <= cmd.hdr.opcode; - dst_r <= cmd.arg0; - src_r <= cmd.arg1; + op_r <= cmd.hdr.opcode; + dst_r <= cmd.arg0; + src_r <= cmd.arg1; // Round the byte count up to a whole cache line. - rem_beats <= (cmd.arg2 + 64'd63) >> 6; - state <= S_SETUP; + rem_beats <= (cmd.arg2 + 64'd63) >> 6; + xl_dst_r <= cmd_xl_dst; + xl_src_r <= cmd_xl_src; + xl_level_r <= LVL_W'(PT_LEVELS - 1); + xl_ppn_r <= satp_ppn; + state <= (cmd_xl_dst || cmd_xl_src) ? S_XLATE_AR : S_SETUP; + end + end + S_XLATE_AR: begin + if (axi_dev.arvalid && axi_dev.arready) begin + state <= S_XLATE_R; + end + end + S_XLATE_R: begin + if (axi_dev.rvalid && axi_dev.rready) begin + if (xl_last) begin + if (xl_dst_r) begin + dst_r <= xl_result; + xl_dst_r <= 1'b0; + end else begin + src_r <= xl_result; + xl_src_r <= 1'b0; + end + xl_level_r <= LVL_W'(PT_LEVELS - 1); + xl_ppn_r <= satp_ppn; + state <= (xl_dst_r && xl_src_r) ? S_XLATE_AR : S_SETUP; + end else begin + xl_level_r <= xl_level_r - LVL_W'(1); + xl_ppn_r <= pte_ppn; + state <= S_XLATE_AR; + end end end S_SETUP: begin @@ -249,7 +349,7 @@ module VX_cp_dma // Replicating the decode lets each copy enable a nearby group. Logically // identical to inlining these expressions; affects implementation only. (* MAX_FANOUT = 32 *) wire host_rready = (rd_rready & rd_from_host) || (state == S_FLUSH_R); - (* MAX_FANOUT = 32 *) wire dev_rready = rd_rready & ~rd_from_host; + (* MAX_FANOUT = 32 *) wire dev_rready = (rd_rready & ~rd_from_host) || (state == S_XLATE_R); // ---- Drive both AXI masters; only the routed port asserts valid ---- always_comb begin @@ -275,10 +375,10 @@ module VX_cp_dma axi_host.bready = wr_bready & wr_to_host; // ----- axi_dev ----- - axi_dev.arvalid = rd_arvalid & ~rd_from_host; - axi_dev.araddr = src_r; + axi_dev.arvalid = (rd_arvalid & ~rd_from_host) || (state == S_XLATE_AR); + axi_dev.araddr = (state == S_XLATE_AR) ? {pte_addr[63:6], 6'd0} : src_r; axi_dev.arid = TID_PREFIX; - axi_dev.arlen = burst_len; + axi_dev.arlen = (state == S_XLATE_AR) ? 8'd0 : burst_len; axi_dev.arsize = 3'd6; axi_dev.arburst = 2'b01; axi_dev.rready = dev_rready; @@ -300,6 +400,7 @@ module VX_cp_dma // Sanity / unused. `UNUSED_VAR (cmd.hdr.flags) + `UNUSED_VAR (pte_addr[5:0]) `UNUSED_VAR (cmd.hdr.reserved) `UNUSED_VAR (cmd.profile_slot) `UNUSED_VAR (axi_host.bid) diff --git a/hw/rtl/cp/VX_cp_pkg.sv b/hw/rtl/cp/VX_cp_pkg.sv index 8a8f1d9452..64ee445bf6 100644 --- a/hw/rtl/cp/VX_cp_pkg.sv +++ b/hw/rtl/cp/VX_cp_pkg.sv @@ -82,6 +82,8 @@ package VX_cp_pkg; localparam int F_FENCE_PRE = 1; // CMD_CACHE_FLUSH: flush only the levels above the last-level cache localparam int F_FLUSH_L1_ONLY = 2; + // CMD_MEM_*: the device operands are physical addresses (no page walk). + localparam int F_MEM_PHYSICAL = 2; typedef struct packed { logic [15:0] reserved; diff --git a/hw/unittest/cp_axil_regfile/VX_cp_axil_regfile_top.sv b/hw/unittest/cp_axil_regfile/VX_cp_axil_regfile_top.sv index 68a6fe802b..3a0638cf2d 100644 --- a/hw/unittest/cp_axil_regfile/VX_cp_axil_regfile_top.sv +++ b/hw/unittest/cp_axil_regfile/VX_cp_axil_regfile_top.sv @@ -23,6 +23,8 @@ module VX_cp_axil_regfile_top input wire clk, input wire reset, + output wire [63:0] satp, + // AXI-Lite W/AW/B input wire awvalid, output wire awready, @@ -114,6 +116,7 @@ module VX_cp_axil_regfile_top .q_error (q_error_arr), .last_dcr_rsp (32'd0), .q_state (q_state_arr), + .satp (satp), .q_reset_pulse (q_reset_arr), .dbg_host_w_counts (32'd0), .dbg_host_r_counts (32'd0), diff --git a/hw/unittest/cp_axil_regfile/main.cpp b/hw/unittest/cp_axil_regfile/main.cpp index 920a9ee3af..6a32120570 100644 --- a/hw/unittest/cp_axil_regfile/main.cpp +++ b/hw/unittest/cp_axil_regfile/main.cpp @@ -158,6 +158,8 @@ static constexpr uint16_t CP_CYCLE_LO = 0x010; static constexpr uint16_t CP_CYCLE_HI = 0x014; static constexpr uint16_t GPU_DEV_CAPS_LO = 0x018; static constexpr uint16_t GPU_DEV_CAPS_HI = 0x01C; +static constexpr uint16_t CP_SATP_LO = 0x028; +static constexpr uint16_t CP_SATP_HI = 0x02C; static constexpr uint16_t GPU_ISA_CAPS_LO = 0x020; static constexpr uint16_t GPU_ISA_CAPS_HI = 0x024; @@ -199,6 +201,16 @@ int main(int argc, char** argv) { EXPECT((v & 0xff) == 1, "T1: NUM_QUEUES low byte"); EXPECT(((v >> 8) & 0xff) == 16, "T1: RING_SIZE_LOG2_MAX byte"); EXPECT(((v >> 16) & 0xff) == 6, "T1: AXI_TID_WIDTH byte"); + EXPECT(((v >> 24) & 1) == VX_CFG_VM_ENABLED, "T1: VM_ENABLED bit"); + } + + // ----- Test 1b: CP_SATP holds the DMA page-table root ----- + { + EXPECT(axil_write(sim, tick, CP_SATP_LO, 0x80012345) == 0, "T1b: SATP_LO write"); + EXPECT(axil_write(sim, tick, CP_SATP_HI, 0x00000007) == 0, "T1b: SATP_HI write"); + EXPECT((axil_read(sim, tick, CP_SATP_LO) & 0xffffffffull) == 0x80012345, "T1b: SATP_LO readback"); + EXPECT((axil_read(sim, tick, CP_SATP_HI) & 0xffffffffull) == 0x00000007, "T1b: SATP_HI readback"); + EXPECT(sim->satp == 0x0000000780012345ull, "T1b: satp output"); } // ----- Test 2: CP_CYCLE counter advances ----- diff --git a/hw/unittest/cp_dma/VX_cp_dma_top.sv b/hw/unittest/cp_dma/VX_cp_dma_top.sv index da9faf2c7c..9c438cbaae 100644 --- a/hw/unittest/cp_dma/VX_cp_dma_top.sv +++ b/hw/unittest/cp_dma/VX_cp_dma_top.sv @@ -27,6 +27,7 @@ module VX_cp_dma_top input wire grant, input wire [$bits(cmd_t)-1:0] cmd_packed, output wire done, + input wire [63:0] satp, // ---- Host-memory AXI master (flat) ---- output wire h_awvalid, @@ -181,6 +182,7 @@ module VX_cp_dma_top .grant (grant), .cmd (cmd_typed), .done (done), + .satp (satp), .axi_host (axi_host), .axi_dev (axi_dev) ); diff --git a/hw/unittest/cp_dma/main.cpp b/hw/unittest/cp_dma/main.cpp index 0e725d8ab1..56683f5e27 100644 --- a/hw/unittest/cp_dma/main.cpp +++ b/hw/unittest/cp_dma/main.cpp @@ -19,6 +19,11 @@ // bytes match src bytes byte-for-byte. // 2. Second back-to-back COPY (different addrs / pattern) re-arms // cleanly — DMA returns to IDLE and accepts the next grant. +// 3. With a page-table root in satp, a COPY between two virtual pages +// walks the table for both operands and moves the physical lines. +// 4. A superpage leaf takes the low address bits from the VA. +// 5. F_MEM_PHYSICAL skips the walk. +// 6. An unmapped VA passes through untranslated, as in the C++ model. // ============================================================================ #include "vl_simulator.h" @@ -28,6 +33,10 @@ #include #include #include +#include +#include +#include +#include #ifndef TRACE_START_TIME #define TRACE_START_TIME 0ull @@ -66,11 +75,22 @@ static void pack_cmd(uint32_t out_words[9], out_words[8] = (uint32_t)opcode | ((uint32_t)flags << 8); } -// ---- AXI4 slave model ---- +// ---- AXI4 slave model (sparse, cache-line granular) ---- struct AxiSlave { - static constexpr uint64_t MEM_BASE = 0x1000; - static constexpr int MEM_SIZE = 4096; - uint8_t mem[MEM_SIZE] = {0}; + std::unordered_map> lines; + + uint8_t& byte(uint64_t a) { + auto it = lines.find(a >> 6); + if (it == lines.end()) it = lines.emplace(a >> 6, std::array{}).first; + return it->second[a & 63]; + } + uint8_t peek(uint64_t a) const { + auto it = lines.find(a >> 6); + return (it == lines.end()) ? 0 : it->second[a & 63]; + } + void write_word(uint64_t a, uint64_t v, int bytes) { + for (int b = 0; b < bytes; ++b) byte(a + b) = uint8_t(v >> (8 * b)); + } bool r_inflight = false; uint64_t r_addr = 0; @@ -83,28 +103,18 @@ struct AxiSlave { uint8_t b_id = 0; void mem_write_cl(uint64_t addr, const uint8_t* src) { - for (int i = 0; i < 64; ++i) { - int64_t a = (int64_t)addr - (int64_t)MEM_BASE + i; - if (a >= 0 && a < MEM_SIZE) mem[a] = src[i]; - } + for (int i = 0; i < 64; ++i) byte(addr + i) = src[i]; } void mem_read_cl(uint64_t addr, uint32_t* dst) const { for (int w = 0; w < 16; ++w) { uint32_t v = 0; - for (int b = 0; b < 4; ++b) { - int64_t a = (int64_t)addr - (int64_t)MEM_BASE + w*4 + b; - if (a >= 0 && a < MEM_SIZE) v |= (uint32_t)mem[a] << (8 * b); - } + for (int b = 0; b < 4; ++b) v |= (uint32_t)peek(addr + w*4 + b) << (8 * b); dst[w] = v; } } int mem_cmp_cl(uint64_t addr_a, uint64_t addr_b) const { for (int i = 0; i < 64; ++i) { - int64_t aa = (int64_t)addr_a - (int64_t)MEM_BASE + i; - int64_t ab = (int64_t)addr_b - (int64_t)MEM_BASE + i; - uint8_t va = (aa >= 0 && aa < MEM_SIZE) ? mem[aa] : 0; - uint8_t vb = (ab >= 0 && ab < MEM_SIZE) ? mem[ab] : 0; - if (va != vb) return i; + if (peek(addr_a + i) != peek(addr_b + i)) return i; } return -1; } @@ -146,10 +156,7 @@ struct AxiSlave { // Write 64 bytes from wdata[0..15] into memory at aw_addr. for (int w = 0; w < 16; ++w) { uint32_t v = top->d_wdata[w]; - for (int b = 0; b < 4; ++b) { - int64_t a = (int64_t)aw_addr - (int64_t)MEM_BASE + w*4 + b; - if (a >= 0 && a < MEM_SIZE) mem[a] = (uint8_t)(v >> (8 * b)); - } + for (int b = 0; b < 4; ++b) byte(aw_addr + w*4 + b) = (uint8_t)(v >> (8 * b)); } aw_taken = false; b_pending = true; @@ -172,10 +179,54 @@ static void cycle(vl_simulator& sim, AxiSlave& s, uint64_t& tick) { top->eval(); } +// Page-table builder in the slave's memory: Sv32 at XLEN=32, Sv39 at 64. +struct PageTable { + static constexpr int LEVEL_BITS = __builtin_ctz(VX_VM_PT_SIZE / VX_VM_PTE_SIZE); + AxiSlave& mem; + uint64_t root; + uint64_t next_table; + + PageTable(AxiSlave& m, uint64_t root_pa) : mem(m), root(root_pa), next_table(root_pa + VX_VM_PT_SIZE) {} + + uint64_t satp() const { +#if VX_VM_PT_LEVEL == 2 + return (1ull << 31) | (root >> VX_VM_PAGE_LOG2_SIZE); +#else + return (8ull << 60) | (root >> VX_VM_PAGE_LOG2_SIZE); +#endif + } + uint64_t pte_addr(uint64_t table, uint64_t va, int level) const { + uint64_t idx = (va >> (VX_VM_PAGE_LOG2_SIZE + level * LEVEL_BITS)) & ((1ull << LEVEL_BITS) - 1); + return table + idx * VX_VM_PTE_SIZE; + } + uint64_t read_pte(uint64_t a) const { + uint64_t v = 0; + for (int b = 0; b < VX_VM_PTE_SIZE; ++b) v |= uint64_t(mem.peek(a + b)) << (8 * b); + return v; + } + // Map one page (leaf_level 0) or one superpage (leaf_level > 0). + void map(uint64_t va, uint64_t pa, int leaf_level) { + uint64_t table = root; + for (int level = VX_VM_PT_LEVEL - 1; level > leaf_level; --level) { + uint64_t a = pte_addr(table, va, level); + uint64_t pte = read_pte(a); + if ((pte & 1) == 0) { + pte = ((next_table >> VX_VM_PAGE_LOG2_SIZE) << 10) | 0x1; // V, pointer + mem.write_word(a, pte, VX_VM_PTE_SIZE); + next_table += VX_VM_PT_SIZE; + } + table = (pte >> 10) << VX_VM_PAGE_LOG2_SIZE; + } + uint64_t leaf = ((pa >> VX_VM_PAGE_LOG2_SIZE) << 10) | 0xC7; // D A W R V + mem.write_word(pte_addr(table, va, leaf_level), leaf, VX_VM_PTE_SIZE); + } +}; + template static void run_copy(vl_simulator& sim, AxiSlave& slave, uint64_t& tick, - uint64_t src, uint64_t dst, const uint8_t* pattern) { - slave.mem_write_cl(src, pattern); + uint64_t src, uint64_t dst, const uint8_t* pattern, + uint64_t pattern_pa, uint8_t flags = 0) { + slave.mem_write_cl(pattern_pa, pattern); // Drain any leftover state (a previous run_copy returns with the FSM // in S_DONE; one idle cycle takes it back to S_IDLE before we drive @@ -184,7 +235,7 @@ static void run_copy(vl_simulator& sim, AxiSlave& slave, uint64_t& tick, for (int i = 0; i < 2; ++i) cycle(sim, slave, tick); uint32_t c[9]; - pack_cmd(c, /*opcode=*/0x03 /*MEM_COPY*/, 0, /*arg0=dst*/dst, + pack_cmd(c, /*opcode=*/0x03 /*MEM_COPY*/, flags, /*arg0=dst*/dst, /*arg1=src*/src, /*arg2=size*/64); for (int i = 0; i < 9; ++i) sim->cmd_packed[i] = c[i]; @@ -202,11 +253,11 @@ static void run_copy(vl_simulator& sim, AxiSlave& slave, uint64_t& tick, EXPECT(latched, "DMA never asserted arvalid (grant capture failed)"); bool got_done = false; - for (int g = 0; g < 50 && !got_done; ++g) { + for (int g = 0; g < 200 && !got_done; ++g) { cycle(sim, slave, tick); if (sim->done) got_done = true; } - EXPECT(got_done, "DMA did not signal done within 50 cycles"); + EXPECT(got_done, "DMA did not signal done within 200 cycles"); } int main(int argc, char** argv) { @@ -225,7 +276,7 @@ int main(int argc, char** argv) { { uint8_t pat[64]; for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0xA0 + i); - run_copy(sim, slave, tick, /*src=*/0x1000, /*dst=*/0x1100, pat); + run_copy(sim, slave, tick, /*src=*/0x1000, /*dst=*/0x1100, pat, 0x1000); int diff = slave.mem_cmp_cl(0x1000, 0x1100); EXPECT(diff < 0, "T1: dst doesn't match src after copy"); @@ -235,12 +286,55 @@ int main(int argc, char** argv) { { uint8_t pat[64]; for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0x5A ^ (i << 1)); - run_copy(sim, slave, tick, /*src=*/0x1200, /*dst=*/0x1300, pat); + run_copy(sim, slave, tick, /*src=*/0x1200, /*dst=*/0x1300, pat, 0x1200); int diff = slave.mem_cmp_cl(0x1200, 0x1300); EXPECT(diff < 0, "T2: second copy mismatch"); } - std::printf("PASSED — 2 scenarios\n"); +#if VX_CFG_VM_ENABLED + PageTable pt(slave, 0x100000); + pt.map(0x80001000, 0x20000, 0); + pt.map(0x80005000, 0x23000, 0); + pt.map(0xC0000000, 0x400000, 1); + sim->satp = pt.satp(); + + // ----- Test 3: both operands walk the table ----- + { + uint8_t pat[64]; + for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0x3C + 7 * i); + run_copy(sim, slave, tick, /*src=*/0x80001040, /*dst=*/0x80005080, pat, 0x20040); + EXPECT(slave.mem_cmp_cl(0x20040, 0x23080) < 0, "T3: VA copy did not land at the mapped PA"); + EXPECT(slave.lines.count(0x80005080 >> 6) == 0, "T3: VA copy wrote the untranslated address"); + } + + // ----- Test 4: superpage leaf keeps the VA's low bits ----- + { + uint8_t pat[64]; + for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0x91 ^ i); + run_copy(sim, slave, tick, /*src=*/0xC0012340, /*dst=*/0x80005100, pat, 0x412340); + EXPECT(slave.mem_cmp_cl(0x412340, 0x23100) < 0, "T4: superpage copy mismatch"); + } + + // ----- Test 5: F_MEM_PHYSICAL skips the walk ----- + { + uint8_t pat[64]; + for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0x17 + i); + run_copy(sim, slave, tick, /*src=*/0x30000, /*dst=*/0x30400, pat, 0x30000, /*flags=*/0x04); + EXPECT(slave.mem_cmp_cl(0x30000, 0x30400) < 0, "T5: physical copy mismatch"); + } + + // ----- Test 6: an unmapped VA passes through ----- + { + uint8_t pat[64]; + for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0xE1 - i); + run_copy(sim, slave, tick, /*src=*/0x00031000, /*dst=*/0x80005140, pat, 0x31000); + EXPECT(slave.mem_cmp_cl(0x31000, 0x23140) < 0, "T6: unmapped source did not pass through"); + } + + std::printf("PASSED — 6 scenarios\n"); +#else + std::printf("PASSED — 2 scenarios (VX_CFG_VM_ENABLE off: no page walk)\n"); +#endif return 0; } diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index 957564a3a7..81e34a506d 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -347,16 +347,14 @@ vx_result_t Device::cp_init() { uint32_t dev_caps = 0; if (p->cp_reg_read(CP_DEV_CAPS, &dev_caps) != VX_SUCCESS) return VX_ERR_DEVICE_LOST; - // An all-ones read is never a valid capability word: bits [31:24] are + // An all-ones read is never a valid capability word: bits [31:29] are // hardwired zero by VX_cp_axil_regfile, so 0xFFFFFFFF means the read // did not reach the register (AXI DECERR / no response substitutes // all-ones on the way back). Decoding it would set every capability // bit -- and a spurious VM_ENABLED sends VMManager off to identity-map // 65,536 PTEs one CP round-trip at a time, which presents as a hang at // 100% CPU rather than as the bus error it actually is. - // Treat it as "no optional capabilities", which is also the correct - // answer for this CP: it has no SATP register, no OP_DRAW, and no - // CMD_LAUNCH_QMD (grep hw/rtl/cp -- all three are absent). + // Treat it as "no optional capabilities". if (dev_caps == 0xFFFFFFFFu) { printf("[VXDRV] Warning: CP_DEV_CAPS read returned all-ones; the " "AXI-Lite read did not reach the register. Assuming no " From ddbcd6888eed7c46b1a0c6160ade6ff5cb975397 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 05:08:05 -0700 Subject: [PATCH 131/336] cache: synthesize the cache DUT with the dcache read merge VX_cache_top never passed READ_MERGE, so the cache gate synthesized a cache without the same-word merge that every dcache instance carries by default. Drive it from VX_CFG_DCACHE_READ_MERGE across all request ports, as VX_mem_unit does for one LSU block, so the gate measures the shipped dcache. Co-Authored-By: Claude Opus 5.5 --- hw/unittest/cache/VX_cache_top.sv | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index 68e16685af..e5fcbea01c 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -56,6 +56,10 @@ module VX_cache_top import VX_gpu_pkg::*; #( // Write misses allocate without a fill (write-validate) parameter LAZY_FETCH = `VX_CFG_DCACHE_LAZY_FETCH, + // Same-word read merge across the ports, as the dcache merges the + // channels of one LSU block + parameter READ_MERGE = `VX_CFG_DCACHE_READ_MERGE ? NUM_REQS : 0, + // Bank pipeline depth (capacity-scaled) parameter LATENCY = `VX_CFG_DCACHE_LATENCY, @@ -189,6 +193,7 @@ module VX_cache_top import VX_gpu_pkg::*; #( .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), .LAZY_FETCH (LAZY_FETCH), + .READ_MERGE (READ_MERGE), .AMO_ENABLE (AMO_ENABLE), .IS_LLC (IS_LLC), .BANK_HASH (BANK_HASH), From b5a21b16c88d3fa85e29797261e13270b2beadf1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 11:02:38 -0700 Subject: [PATCH 132/336] cp: register the DMA request path into the last-level cache slot With CP_LLC_CLIENT, the DMA reaches the shared L2 through two combinational hops that failed timing: - VX_afu_wrap fed the device bridge's request straight into the cache slot, so a combinational valid and 512 muxed data nets crossed the module boundary (MODULE-IFCHECK failure). - VX_cp_llc_port's request buffer was a single pipe register whose ready passes through, so the L2 core-request crossbar's round-robin grant drove the enable of its 628-bit register (-1.556 ns at 300 MHz). Both are now two-entry elastic buffers with a registered output, as on the socket ports into the same crossbar. Latency stays one cycle and throughput is unchanged. Top DUT (2 cores, VM, L2) with the CP on L2: 201.7 -> 210.0 MHz. The worst CP path (-1.401 ns) now ties the worst non-CP path (-1.397 ns, socket icache tag->data). xrtsim, CP on L2, with and without VM: dogfood and sgemm pass. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/afu/common/VX_afu_wrap.sv | 24 +++++++++++++++++------- hw/rtl/mem/VX_cp_llc_port.sv | 7 +++++-- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index 208d8b4542..90c8615e0e 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -977,13 +977,23 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( if (CP_DMA_LLC != 0) begin : g_cp_llc `STATIC_ASSERT(CP_MEM_DATA_SIZE == FAB_DATA_SIZE, ("CP DMA beat must match the last-level cache word")) `STATIC_ASSERT(CP_MEM_TAG_WIDTH >= `VX_CP_AXI_TID_WIDTH, ("CP DMA id does not fit the last-level cache slot tag")) - assign cp_llc_req_valid = cp_mem_req_valid; - assign cp_llc_req_rw = cp_mem_req_rw; - assign cp_llc_req_addr = cp_mem_req_addr[CP_MEM_ADDR_WIDTH-1:0]; - assign cp_llc_req_data = cp_mem_req_data; - assign cp_llc_req_byteen = cp_mem_req_byteen; - assign cp_llc_req_tag = CP_MEM_TAG_WIDTH'(cp_mem_req_tag); - assign cp_mem_req_ready = cp_llc_req_ready; + // The bridge's request valid is combinational; register the + // request before it enters the last-level cache's slot. + VX_elastic_buffer #( + .DATAW (1 + CP_MEM_ADDR_WIDTH + CP_MEM_DATA_SIZE * 9 + CP_MEM_TAG_WIDTH), + .SIZE (`TO_OUT_BUF_SIZE(3)), + .OUT_REG (`TO_OUT_BUF_REG(3)) + ) cp_llc_req_buf ( + .clk (clk), + .reset (subsys_reset), + .valid_in (cp_mem_req_valid), + .ready_in (cp_mem_req_ready), + .data_in ({cp_mem_req_rw, cp_mem_req_addr[CP_MEM_ADDR_WIDTH-1:0], cp_mem_req_data, + cp_mem_req_byteen, CP_MEM_TAG_WIDTH'(cp_mem_req_tag)}), + .data_out ({cp_llc_req_rw, cp_llc_req_addr, cp_llc_req_data, cp_llc_req_byteen, cp_llc_req_tag}), + .valid_out (cp_llc_req_valid), + .ready_out (cp_llc_req_ready) + ); assign cp_mem_rsp_valid = cp_llc_rsp_valid; assign cp_mem_rsp_data = cp_llc_rsp_data; assign cp_mem_rsp_tag = cp_llc_rsp_tag[`VX_CP_AXI_TID_WIDTH-1:0]; diff --git a/hw/rtl/mem/VX_cp_llc_port.sv b/hw/rtl/mem/VX_cp_llc_port.sv index 5542758f62..ac764277f8 100644 --- a/hw/rtl/mem/VX_cp_llc_port.sv +++ b/hw/rtl/mem/VX_cp_llc_port.sv @@ -93,9 +93,12 @@ module VX_cp_llc_port import VX_gpu_pkg::*; #( wire req_fire = req_valid && req_ready; wire fence_fire = fence_req && buf_ready_in; + // Registered ready: the slot's ready is the last-level cache's crossbar + // grant, which must not reach this buffer's enable combinationally. VX_elastic_buffer #( - .DATAW (REQ_DATAW), - .SIZE (1) + .DATAW (REQ_DATAW), + .SIZE (`TO_OUT_BUF_SIZE(3)), + .OUT_REG (`TO_OUT_BUF_REG(3)) ) req_buf ( .clk (clk), .reset (reset), From 5defb95818e3540d0b26a2ede04f78d54a000bb0 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 11:59:35 -0700 Subject: [PATCH 133/336] cache: erase the MSHR line table The line table (VX_CFG_{DCACHE,L2,L3}_MSHR_LINES) held each distinct {line, sector} once and gave request entries an index into it, so the MSHR could grow without widening its address compare. Measured on the rtx5080 shape (16 cores, 192-entry L2 MSHR) and the 4-bank cache DUT, it does not pay off at any size: - 48 lines (MSHR/4), the smallest that keeps cycles flat (vecadd +3%, sgemm 0%): area savings shrink below the 32-line point, which saves 5% LUT over a plain 192-entry MSHR. - 16 lines saves 19% LUT, but vecadd runs 2.8x slower and the two-level lookup lengthens the MSHR->data-write path from 9 to 15 logic levels (201 -> 163 MHz). Remove it from RTL and SimX together, with its CI cases, config knobs and documentation. The default per-entry address table is unchanged. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 - VX_config.toml | 6 - ci/testcases/cache.yaml | 11 -- docs/assets/img/mlp_mshr_line_table.svg | 51 ------ docs/designs/mlp_optimizations.md | 12 -- hw/rtl/VX_cluster.sv | 1 - hw/rtl/VX_socket.sv | 1 - hw/rtl/Vortex.sv | 1 - hw/rtl/cache/VX_cache.sv | 2 - hw/rtl/cache/VX_cache_bank.sv | 8 +- hw/rtl/cache/VX_cache_cluster.sv | 2 - hw/rtl/cache/VX_cache_mshr.sv | 208 ++++-------------------- hw/rtl/cache/VX_cache_wrap.sv | 2 - hw/unittest/cache/VX_cache_top.sv | 2 - sim/simx/cluster.cpp | 1 - sim/simx/mem/cache.cpp | 95 +---------- sim/simx/mem/cache.h | 1 - sim/simx/processor.cpp | 1 - sim/simx/socket.cpp | 2 - 19 files changed, 44 insertions(+), 366 deletions(-) delete mode 100644 docs/assets/img/mlp_mshr_line_table.svg diff --git a/CHANGELOG.md b/CHANGELOG.md index ec2d93d728..b9e9efc2dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,9 +29,6 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, the sector with its dirty-byte mask and a partial bit instead of fetching the line; a later load or AMO merges memory data under the unwritten bytes, and evictions write back only dirty bytes. *Why:* streaming writes no longer read every line first: 1-core vecadd DRAM reads -33%. -- **MSHR line table (`VX_CFG_{DCACHE,L2,L3}_MSHR_LINES`, off by default).** Misses are tracked in a reference-counted table of - distinct {line, sector} entries, and request entries keep only an index into it, so the MSHR holds more requests without widening - its address compares. *Why:* at 400-cycle memory latency, 64 requests over 16 lines: sgemm -n128 -15.3%, raycast -8.1%. - **Command-processor DMA through the shared LLC (`VX_CFG_CP_LLC_CLIENT`, off by default).** When one LLC serves every core, the CP's transfers enter it on their own client slot, so host transfers need only an L1 flush, not a walk of the whole LLC; transfers of at least half the LLC write around it. A launch behind DMA writes waits for an L1-only flush token sent on the slot. The runtime diff --git a/VX_config.toml b/VX_config.toml index 0a70158668..d63ef51de1 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -247,10 +247,6 @@ VX_CFG_DCACHE_DIRTYBYTES = "expr: $VX_CFG_DCACHE_LAZY_FETCH" VX_CFG_DCACHE_LAZY_FETCH = 0 VX_CFG_DCACHE_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_DCACHE_MSHR_SIZE = 16 -# MSHR line entries: distinct {line, sector} misses a bank tracks, each naming -# the chain of requests merged on it; MSHR_SIZE then sizes the request pool. -# 0 keeps the address in every request entry (lines = requests). -VX_CFG_DCACHE_MSHR_LINES = 0 # dcache is unsectored: line = sector = mem transaction granule = L1_LINE (which # defaults to MEM_BLOCK but can be configured smaller). Banking comes from the # reduced WORD above (>= sector is clamped), not the line. Deriving from L1_LINE @@ -286,7 +282,6 @@ VX_CFG_L2_DIRTYBYTES = "expr: $VX_CFG_L2_LAZY_FETCH" VX_CFG_L2_LAZY_FETCH = 0 VX_CFG_L2_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L2_MSHR_SIZE = 16 -VX_CFG_L2_MSHR_LINES = 0 VX_CFG_L2_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L2_SIZE) - clog2(1048576))" VX_CFG_L2_MRSQ_SIZE = 4 VX_CFG_L2_MREQ_SIZE = 0 @@ -306,7 +301,6 @@ VX_CFG_L3_DIRTYBYTES = "expr: $VX_CFG_L3_LAZY_FETCH" VX_CFG_L3_LAZY_FETCH = 0 VX_CFG_L3_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L3_MSHR_SIZE = 16 -VX_CFG_L3_MSHR_LINES = 0 VX_CFG_L3_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L3_SIZE) - clog2(2097152))" VX_CFG_L3_MRSQ_SIZE = 4 VX_CFG_L3_MREQ_SIZE = 0 diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index c4326f73c0..c5b159884d 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -374,7 +374,6 @@ tests: configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 -DVX_CFG_DCACHE_LATENCY=3 # Both sectors of one sectored L2 line miss together from different warps and # cores: each must get its own fill, never chain behind the other sector's. -# Run with the address held per request and with the MSHR line table. - id: sector_merge-1 via: blackbox drivers: @@ -385,16 +384,6 @@ tests: shape: cores: 4 l2cache: true -- id: sector_merge-2 - via: blackbox - drivers: - - simx - - rtlsim - app: sector_merge - configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_DCACHE_MSHR_SIZE=32 -DVX_CFG_DCACHE_MSHR_LINES=8 -DVX_CFG_L2_MSHR_SIZE=32 -DVX_CFG_L2_MSHR_LINES=8 - shape: - cores: 4 - l2cache: true # A thread's load must return its own earlier store to the same word. On the # write-through L1 under an L2, the stores of a miss chain reach the array only # as they replay behind the fill: a load landing after the chain's read head diff --git a/docs/assets/img/mlp_mshr_line_table.svg b/docs/assets/img/mlp_mshr_line_table.svg deleted file mode 100644 index 0097fe146b..0000000000 --- a/docs/assets/img/mlp_mshr_line_table.svg +++ /dev/null @@ -1,51 +0,0 @@ - - Before: a flat MSHR where each of 16 entries stores a full line address. After: 64 small request entries point by index into a 16-entry line table of full addresses. - - - -Before: flat MSHR (16) - After: requests (64) → line table (16) - - full line addr | request - full line addr | request - full line addr | request - ⋮ - full line addr | request - 16 wide comparators - full at 16 requests - - idx 0 | request - idx 0 | request - idx 1 | request - idx 0 | request - ⋮ - idx 2 | request - 64 narrow entries - 0: line A · refs 3 - 1: line B · refs 1 - 2: line C · refs 1 - ⋮ 16 lines - - - - - - 16 wide comparators - - diff --git a/docs/designs/mlp_optimizations.md b/docs/designs/mlp_optimizations.md index 8bd6967c35..bbd4253197 100644 --- a/docs/designs/mlp_optimizations.md +++ b/docs/designs/mlp_optimizations.md @@ -25,7 +25,6 @@ Each section says what problem the feature removes, how it works, and what it ga | [Hashed bank select](#hashed-bank-select) | L1D, L2 | Power-of-two strides on one bank | sgemv −32.8% | On (L2; dcache ≥ 4 banks) | | [Fill forwarding](#fill-forwarding) | All caches | Replaying waiters after a fill | sgemv −5.0% | On | | [Back-to-back fills](#back-to-back-fills) | All caches | A bubble after each fill | rtx5080 stream −6.2% | On | -| [MSHR line table](#mshr-line-table) | All caches | One wide address compare per waiting request | sgemm −15.3% at 400-cycle latency | Off | | [Write-allocate without fetch](#write-allocate-without-fetch-lazy-fetch) | Write-back caches | Reading a line only to overwrite it | DRAM reads −33% | Off | | [Full-sector refetch skip](#full-sector-refetch-skip) | Write-back caches | Refetching a fully written sector | rtx5080 sgemm −43.5% with lazy L2 | Off (with lazy fetch) | | [Dcache read merge](#dcache-read-merge) | L1D | Same-word loads serialized on one bank | sgemm −17.2% | On | @@ -50,7 +49,6 @@ Some of these features can be set with a `VX_CFG_*` knob (see [Build Configurati | Feature | Knob | Default | |---|---|---| | Hashed bank select | `VX_CFG_DCACHE_BANK_HASH`, `VX_CFG_L2_BANK_HASH` | dcache: on with 4 or more banks; L2: on. The L3 has no bank hash. | -| MSHR line table | `VX_CFG_{DCACHE,L2,L3}_MSHR_LINES` | 0 (off): flat MSHR | | Write-allocate without fetch | `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH` | 0 (off). Enabling it also enables the dirty-byte masks (`*_DIRTYBYTES`). | | Full-sector refetch skip | Part of lazy fetch | Follows `*_LAZY_FETCH` | | Dcache read merge | `VX_CFG_DCACHE_READ_MERGE` | 1 (on) | @@ -130,16 +128,6 @@ Some of these features can be set with a `VX_CFG_*` knob (see [Build Configurati - the stock configuration (shallow L2): within ±0.5% - the cache also closed timing faster: Fmax up by 3% -### MSHR line table - -![MSHR line table](../assets/img/mlp_mshr_line_table.svg) - -**Problem.** Each MSHR entry holds a full line address, and every new miss compares against all of them. To keep more misses in flight, the MSHR needs more entries, and each one adds a wide comparator. At high memory latency, a 16-entry MSHR was full a fifth of the time, often with only one or two distinct lines in it. - -**How it works.** Distinct {line, sector} pairs go into a small reference-counted line table. Each request entry keeps only an index into that table. The cache holds many more waiting requests (for example 64) while comparing addresses against only the line table (for example 16 entries). Controlled by `VX_CFG_{DCACHE,L2,L3}_MSHR_LINES`. - -**Gain (rtlsim at 400-cycle memory latency, MSHR 16 → 64 requests over 16 lines):** sgemm −15.3%, raycast −8.1%. Without the dcache read merge, the same sgemm gains 24.8%. At the stock latency the MSHR does not limit, the gain is under 0.5%, and the feature is off by default. - ### Write-allocate without fetch (lazy fetch) **Problem.** A write-back cache that misses on a store normally reads the whole line from memory, then overwrites part or all of it. Streaming writes therefore cost a read for every line they write. diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 5dfcb2f4b9..6121d36926 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -224,7 +224,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .MEM_PORTS (L2_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L2_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L2_MSHR_SIZE), - .MSHR_LINES (`VX_CFG_L2_MSHR_LINES), .MRSQ_SIZE (`VX_CFG_L2_MRSQ_SIZE), .MREQ_SIZE (`VX_CFG_L2_MREQ_SIZE), .LATENCY (`VX_CFG_L2_LATENCY), diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index c40b679395..3b360dfee8 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -230,7 +230,6 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .MEM_PORTS (L1_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_DCACHE_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_DCACHE_MSHR_SIZE), - .MSHR_LINES (`VX_CFG_DCACHE_MSHR_LINES), .MRSQ_SIZE (`VX_CFG_DCACHE_MRSQ_SIZE), .MREQ_SIZE (`VX_CFG_DCACHE_MREQ_SIZE), .LATENCY (`VX_CFG_DCACHE_LATENCY), diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 320693faa3..0c9ba0c713 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -160,7 +160,6 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .MEM_PORTS (L3_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L3_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L3_MSHR_SIZE), - .MSHR_LINES (`VX_CFG_L3_MSHR_LINES), .MRSQ_SIZE (`VX_CFG_L3_MRSQ_SIZE), .MREQ_SIZE (`VX_CFG_L3_MREQ_SIZE), .LATENCY (`VX_CFG_L3_LATENCY), diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 4b5b17c86e..2b3c81cfa7 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -29,7 +29,6 @@ module VX_cache import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Core Response Queue Size parameter MSHR_SIZE = 16, // Miss Reserv Queue Knob - parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 4, // Memory Response Queue Size parameter MREQ_SIZE = 0, // Memory Request Queue Size (0 = derived minimum) @@ -491,7 +490,6 @@ module VX_cache import VX_gpu_pkg::*; #( .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), - .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index c9150cc112..5a3b5041cf 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -26,7 +26,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter SECTOR_SIZE = LINE_SIZE,// sector (fill/eviction granule); = LINE_SIZE => 1 sector parameter CRSQ_SIZE = 0, // extra core-response queue slots over minimum parameter MSHR_SIZE = 1, // miss reservation queue size - parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 1, // memory response queue size (sized at wrapper) parameter MREQ_SIZE = 0, // memory request queue size (0 = derived minimum) parameter WRITE_ENABLE = 1, @@ -181,7 +180,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Shared signals // ------------------------------------------------------------------------ wire crsp_queue_stall, mshr_alm_full; - wire mshr_req_full, mshr_line_ready; wire mshr_allocate_st0; (* max_fanout = 64 *) wire pipe_stall; // Authoritative "no entry held", straight off the MSHR's valid mask. The @@ -1065,13 +1063,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .decr (mshr_dequeue), `UNUSED_PIN (empty), `UNUSED_PIN (alm_empty), - .full (mshr_req_full), + .full (mshr_alm_full), `UNUSED_PIN (alm_full), `UNUSED_PIN (size) ); - assign mshr_alm_full = mshr_req_full || ~mshr_line_ready; - VX_cache_mshr #( .INSTANCE_ID (`SFORMATF(("%s-mshr", INSTANCE_ID))), .BANK_ID (BANK_ID), @@ -1080,7 +1076,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .SECTOR_SIZE (SECTOR_SIZE), .NUM_BANKS (NUM_BANKS), .MSHR_SIZE (MSHR_SIZE), - .MSHR_LINES (MSHR_LINES), .AMO_ENABLE (AMO_ENABLE != 0), .AMO_PASSTHRU ((AMO_ENABLE != 0) && (IS_LLC == 0)), .WORD_SEL_WIDTH (WORD_SEL_WIDTH), @@ -1135,7 +1130,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .allocate_pending_ovl (mshr_pending_ovl_raw), .allocate_previd (mshr_previd), `UNUSED_PIN (allocate_ready), - .allocate_line_ready (mshr_line_ready), .finalize_valid (mshr_finalize_st1 && ~pipe_stall), .finalize_is_release (mshr_release_st1), // Only link an entry into the pending chain if it is KEPT (a miss). A diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 6065ef0f5e..8352630768 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -33,7 +33,6 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Core Response Queue Size parameter MSHR_SIZE = 16, // Miss Reserv Queue Knob - parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 4, // Memory Response Queue Size parameter MREQ_SIZE = 0, // Memory Request Queue Size (0 = derived minimum) @@ -174,7 +173,6 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), - .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index 05616d0bdd..64ffe48898 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -38,15 +38,6 @@ // The finalize request is used to persit or release the currently allocated MSHR entry // if we had a cache miss or a hit, respectively. // -// With MSHR_LINES != 0 the line address lives in a separate table of -// MSHR_LINES entries, one per distinct {line, sector} held, and each request -// entry stores only an index into it. The address CAM then spans the lines -// rather than the requests, so the request pool can grow (many requests merge -// on few lines) without growing the wide compare. A line entry is reference -// counted by the request entries naming it and frees when the count drains. -// The bank admits a core request only while two line entries are free: the -// request already past admission may claim one before this one reaches S0. -// // Warning: This MSHR implementation is strongly coupled with the bank pipeline // and as such changes to either module requires careful evaluation. // @@ -59,7 +50,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (coalescing/fill granule); = LINE_SIZE => 1 sector parameter NUM_BANKS = 1, // Number of banks parameter MSHR_SIZE = 4, // Miss Reserv Queue Knob - parameter MSHR_LINES = 0, // distinct line entries (0 = address held per request) parameter DATA_WIDTH = 1, // MSHR parameters parameter AMO_ENABLE = 0, // Per-entry AMO tracking (probe outputs) parameter AMO_PASSTHRU = 0, // Non-LLC passthrough: AMO entries never coalesce @@ -142,8 +132,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire allocate_pending_ovl, output wire [MSHR_ADDR_WIDTH-1:0] allocate_previd, output wire allocate_ready, - // a line entry is free for every request the bank can still admit - output wire allocate_line_ready, // finalize input wire finalize_valid, @@ -161,8 +149,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( `UNUSED_PARAM (BANK_ID) `UNUSED_PARAM (BANK_HASH) - `STATIC_ASSERT((MSHR_LINES == 0) || ((MSHR_LINES >= 2) && (MSHR_LINES <= MSHR_SIZE)), ("invalid parameter: MSHR_LINES")) - // Per-entry word-in-line select, kept outside the payload RAM so the // allocate probe can compare it: the hit-demotion order hazard is // word-granular (a chained entry constrains only requests that touch @@ -196,12 +182,10 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( wire dequeue_park_fire = (LAZY_FETCH != 0) && dequeue_fire && dequeue_park; wire dequeue_free = dequeue_fire && ~dequeue_park_fire; - // Entry holds the allocate request's {line, sector} / the probe's line. + // Entry holds the allocate request's {line, sector}. // Coalescing matches on {line, sector} so same-line different-sector misses // get independent fills (each replay then hits its own filled sector). wire [MSHR_SIZE-1:0] entry_match; - wire [MSHR_SIZE-1:0] entry_probe; - wire [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_n; wire [MSHR_SIZE-1:0] addr_matches; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_addr_matches @@ -321,168 +305,48 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( next_table <= next_table_n; end - // Per-entry address, for checks and traces only. - wire [`CS_LINE_ADDR_WIDTH-1:0] entry_addr [MSHR_SIZE]; - - if (MSHR_LINES != 0) begin : g_line_table - localparam LINE_ID_W = `LOG2UP(MSHR_LINES); - `UNUSED_VAR ({allocate_addr_n, allocate_sector_n, allocate_adv}) - localparam LCNT_W = `CLOG2(MSHR_SIZE + 1); - - reg [`CS_LINE_ADDR_WIDTH-1:0] line_addr [0:MSHR_LINES-1]; - reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] line_sector [0:MSHR_LINES-1]; - reg [LCNT_W-1:0] line_cnt [0:MSHR_LINES-1]; - reg [LINE_ID_W-1:0] entry_line [0:MSHR_SIZE-1]; - - reg [LINE_ID_W-1:0] line_free_id_r; - reg line_free_r, line_ready_r; - - wire [MSHR_LINES-1:0] line_valid, line_hits, line_probes, line_free_n; - for (genvar l = 0; l < MSHR_LINES; ++l) begin : g_line_match - assign line_valid[l] = (line_cnt[l] != '0); - assign line_hits[l] = line_valid[l] && (line_addr[l] == allocate_addr) && (line_sector[l] == allocate_sector); - assign line_probes[l] = line_valid[l] && (line_addr[l] == probe_addr); - end - - // a {line, sector} occupies at most one line entry - wire line_hit; - wire [LINE_ID_W-1:0] line_hit_id; - VX_priority_encoder #( - .N (MSHR_LINES) - ) line_hit_sel ( - .data_in (line_hits), - .index_out (line_hit_id), - .valid_out (line_hit), - `UNUSED_PIN (onehot_out) - ); - - wire [LINE_ID_W-1:0] alloc_line = line_hit ? line_hit_id : line_free_id_r; + reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; + reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_table [0:MSHR_SIZE-1]; - for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match - assign entry_match[i] = line_hit && (entry_line[i] == line_hit_id); - assign entry_probe[i] = line_probes[entry_line[i]]; - assign entry_addr[i] = line_addr[entry_line[i]]; - end - - wire [LINE_ID_W-1:0] dequeue_line = entry_line[dequeue_id]; - wire [LINE_ID_W-1:0] finalize_line = entry_line[finalize_id]; - wire finalize_free = finalize_valid && finalize_is_release; - - for (genvar l = 0; l < MSHR_LINES; ++l) begin : g_line_cnt - wire inc = allocate_fire && (alloc_line == LINE_ID_W'(l)); - wire dec_d = dequeue_free && (dequeue_line == LINE_ID_W'(l)); - wire dec_f = finalize_free && (finalize_line == LINE_ID_W'(l)); - wire [LCNT_W-1:0] cnt_n = line_cnt[l] + LCNT_W'(inc) - LCNT_W'(dec_d) - LCNT_W'(dec_f); - assign line_free_n[l] = (cnt_n == '0); - always @(posedge clk) begin - if (reset) begin - line_cnt[l] <= '0; - end else begin - line_cnt[l] <= cnt_n; - end - end - end - - wire [MSHR_LINES-1:0] line_free_oh; - wire [LINE_ID_W-1:0] line_free_id_n; - wire line_free_any_n; - VX_priority_encoder #( - .N (MSHR_LINES) - ) line_free_sel ( - .data_in (line_free_n), - .onehot_out (line_free_oh), - .index_out (line_free_id_n), - .valid_out (line_free_any_n) - ); - - always @(posedge clk) begin - if (reset) begin - line_free_r <= 1'b0; - line_ready_r <= 1'b0; - end else begin - line_free_r <= line_free_any_n; - line_ready_r <= line_free_any_n && (| (line_free_n & ~line_free_oh)); - end - line_free_id_r <= line_free_id_n; - if (allocate_fire) begin - entry_line[allocate_id] <= alloc_line; - if (~line_hit) begin - line_addr[line_free_id_r] <= allocate_addr; - line_sector[line_free_id_r] <= allocate_sector; - end - end - end - - assign fill_addr = line_addr[entry_line[fill_id]]; - assign fill_sector = line_sector[entry_line[fill_id]]; - assign dequeue_addr_n = line_addr[entry_line[dequeue_id_n]]; - assign allocate_line_ready = line_ready_r; - - `RUNTIME_ASSERT(~(allocate_fire && ~line_hit && ~line_free_r), ("*** %s line table overflow: addr=0x%0h (#%0d)", INSTANCE_ID, - `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID), alc_req_uuid)) - - `ifdef SIMULATION - // Each line count equals the live entries naming the line. - for (genvar l = 0; l < MSHR_LINES; ++l) begin : g_line_check - integer refs; - always @(*) begin - refs = 0; - for (integer i = 0; i < MSHR_SIZE; ++i) begin - refs += 32'(valid_table[i] && (entry_line[i] == LINE_ID_W'(l))); - end - end - `RUNTIME_ASSERT(refs == 32'(line_cnt[l]), ("*** %s line count mismatch: line=%0d, count=%0d, refs=%0d", INSTANCE_ID, - l, line_cnt[l], refs)) + always @(posedge clk) begin + if (allocate_fire) begin + addr_table[allocate_id] <= allocate_addr; + sector_table[allocate_id] <= allocate_sector; end - `endif - end else begin : g_entry_table - reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; - reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_table [0:MSHR_SIZE-1]; + end + // The allocate match is taken a cycle early, against the core request + // entering S0 and the table as it stands after this cycle's allocate, + // so S0 starts it from flops: it gates the data array's write enable + // through the order-hazard probes. A request of another kind entering + // S0 does not allocate, so its match is never used. Nothing allocates + // while S0 holds. + wire alloc_self_match = (allocate_addr == allocate_addr_n) && (allocate_sector == allocate_sector_n); + reg [MSHR_SIZE-1:0] entry_match_r; + for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match + wire match_n = (allocate_fire && (allocate_id == MSHR_ADDR_WIDTH'(i))) ? alloc_self_match + : ((addr_table[i] == allocate_addr_n) && (sector_table[i] == allocate_sector_n)); always @(posedge clk) begin - if (allocate_fire) begin - addr_table[allocate_id] <= allocate_addr; - sector_table[allocate_id] <= allocate_sector; + if (allocate_adv) begin + entry_match_r[i] <= match_n; end end - - // The allocate match is taken a cycle early, against the core request - // entering S0 and the table as it stands after this cycle's allocate, - // so S0 starts it from flops: it gates the data array's write enable - // through the order-hazard probes. A request of another kind entering - // S0 does not allocate, so its match is never used. Nothing allocates - // while S0 holds. - wire alloc_self_match = (allocate_addr == allocate_addr_n) && (allocate_sector == allocate_sector_n); - reg [MSHR_SIZE-1:0] entry_match_r; - for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match - wire match_n = (allocate_fire && (allocate_id == MSHR_ADDR_WIDTH'(i))) ? alloc_self_match - : ((addr_table[i] == allocate_addr_n) && (sector_table[i] == allocate_sector_n)); - always @(posedge clk) begin - if (allocate_adv) begin - entry_match_r[i] <= match_n; - end - end - assign entry_match[i] = entry_match_r[i]; - assign entry_probe[i] = (addr_table[i] == probe_addr); - assign entry_addr[i] = addr_table[i]; - `RUNTIME_ASSERT (~allocate_valid || (entry_match_r[i] == ((addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector))), - ("%t: *** %s allocate match out of sync: id=%0d", $time, INSTANCE_ID, i)) - end - - assign fill_addr = addr_table[fill_id]; - assign fill_sector = sector_table[fill_id]; - assign dequeue_addr_n = addr_table[dequeue_id_n]; - assign allocate_line_ready = 1'b1; + assign entry_match[i] = entry_match_r[i]; + `RUNTIME_ASSERT (~allocate_valid || (entry_match_r[i] == ((addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector))), + ("%t: *** %s allocate match out of sync: id=%0d", $time, INSTANCE_ID, i)) end + assign fill_addr = addr_table[fill_id]; + assign fill_sector = sector_table[fill_id]; + `RUNTIME_ASSERT(~(allocate_fire && valid_table[allocate_id_r]), ("*** %s inuse allocation: addr=0x%0h, id=%0d (#%0d)", INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID), allocate_id_r, alc_req_uuid)) `RUNTIME_ASSERT(~(finalize_valid && ~valid_table[finalize_id]), ("*** %s invalid release: addr=0x%0h, id=%0d (#%0d)", INSTANCE_ID, - `CS_BANK_TO_FULL_ADDR(entry_addr[finalize_id], BANK_ID), finalize_id, fin_req_uuid)) + `CS_BANK_TO_FULL_ADDR(addr_table[finalize_id], BANK_ID), finalize_id, fin_req_uuid)) `RUNTIME_ASSERT(~(fill_valid && ~valid_table[fill_id]), ("*** %s invalid fill: addr=0x%0h, id=%0d", INSTANCE_ID, - `CS_BANK_TO_FULL_ADDR(entry_addr[fill_id], BANK_ID), fill_id)) + `CS_BANK_TO_FULL_ADDR(addr_table[fill_id], BANK_ID), fill_id)) // A walk confined to its own live chain never lands on an entry that is // already invalid. next_table and next_index are cleared only at allocate, @@ -500,11 +364,11 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( reg [`CS_LINE_ADDR_WIDTH-1:0] chain_addr_r; always @(posedge clk) begin if (fill_valid) begin - chain_addr_r <= entry_addr[fill_id]; + chain_addr_r <= addr_table[fill_id]; end end - `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (entry_addr[dequeue_id] != chain_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, - dequeue_id, `CS_BANK_TO_FULL_ADDR(entry_addr[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(chain_addr_r, BANK_ID))) + `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (addr_table[dequeue_id] != chain_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, + dequeue_id, `CS_BANK_TO_FULL_ADDR(addr_table[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(chain_addr_r, BANK_ID))) // The slot the dequeue pointer names must not be handed to another line // while the pointer still rests on it: a combinational `dequeue_addr` on @@ -599,7 +463,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( wire [MSHR_SIZE-1:0] probe_ld, probe_amo; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_probe_matches - wire addr_match = valid_table[i] && persisted_table[i] && entry_probe[i]; + wire addr_match = valid_table[i] && persisted_table[i] && (addr_table[i] == probe_addr); assign probe_ld[i] = addr_match && ~amo_table[i]; assign probe_amo[i] = addr_match && amo_table[i]; end @@ -635,7 +499,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign probe_pending_amo = 1'b0; assign {probe_any_ld, probe_any_amo, persist_ld, persist_amo} = '0; `UNUSED_PARAM (AMO_PASSTHRU) - `UNUSED_VAR ({allocate_is_amo, probe_addr, entry_probe}) + `UNUSED_VAR ({allocate_is_amo, probe_addr}) end assign allocate_ready = allocate_rdy; @@ -708,7 +572,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( reg [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_r; reg dequeue_rw_r; always @(posedge clk) begin - dequeue_addr_r <= dequeue_addr_n; + dequeue_addr_r <= addr_table[dequeue_id_n]; dequeue_rw_r <= write_table[dequeue_id_n]; end @@ -746,7 +610,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( `TRACE(3, ("%t: %s table: ", $time, INSTANCE_ID)) for (integer i = 0; i < MSHR_SIZE; ++i) begin if (valid_table[i]) begin - `TRACE(3, (" %0d=0x%0h", i, `CS_BANK_TO_FULL_ADDR(entry_addr[i], BANK_ID))) + `TRACE(3, (" %0d=0x%0h", i, `CS_BANK_TO_FULL_ADDR(addr_table[i], BANK_ID))) if (write_table[i]) begin `TRACE(3, ("(w)")) end else begin diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index f71dbea3b2..6d705b20d6 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -31,7 +31,6 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Core Response Queue Size parameter MSHR_SIZE = 16, // Miss Reserv Queue Knob - parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 4, // Memory Response Queue Size parameter MREQ_SIZE = 0, // Memory Request Queue Size (0 = derived minimum) @@ -158,7 +157,6 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), - .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index e5fcbea01c..3ba5f9c37b 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -39,7 +39,6 @@ module VX_cache_top import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Miss Reserv Queue Knob parameter MSHR_SIZE = 16, - parameter MSHR_LINES = 0, // Memory Response Queue Size parameter MRSQ_SIZE = 8, parameter MREQ_SIZE = 0, @@ -184,7 +183,6 @@ module VX_cache_top import VX_gpu_pkg::*; #( .MEM_PORTS (MEM_PORTS), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), - .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 8ecebf429d..4919c6b6aa 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -100,7 +100,6 @@ class Cluster::Impl { VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch true, // core-response register (behind the bypass layer) 0, // read_merge - VX_CFG_L2_MSHR_LINES, // mshr_lines CP_ON_L2 != 0, // write_around VX_CFG_L2_MRSQ_SIZE != 0, // memory-response queue }); diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 10f8b81291..7e09ea14ed 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -530,21 +530,10 @@ struct mshr_entry_t { } }; -// With a line cap, the {line, sector} misses live in a table of that many -// entries, each reference-counted by the request entries chained on it. class MSHR { public: - MSHR(uint32_t size, uint32_t line_cap) - : entries_(size), ready_reqs_(0), size_(0), line_cap_(line_cap) {} - - // Distinct {line, sector} misses held. - uint32_t lines() const { - return lines_.size(); - } - - bool holds_line(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) const { - return this->find_line(set_id, addr_tag, sector_id) != lines_.end(); - } + MSHR(uint32_t size) + : entries_(size), ready_reqs_(0), size_(0) {} uint32_t capacity() const { return entries_.size(); @@ -639,7 +628,6 @@ class MSHR { entry.refetch = refetch; entry.refetch_retry = false; ++size_; - this->line_ref(set_id, addr_tag, sector_id); return i; } } @@ -742,7 +730,6 @@ class MSHR { picked->bank_req.type = bank_req_t::None; --ready_reqs_; --size_; - this->line_unref(picked->set_id, picked->addr_tag, picked->sector_id); } void reset() { @@ -751,51 +738,12 @@ class MSHR { } ready_reqs_ = 0; size_ = 0; - lines_.clear(); } private: - struct line_ref_t { - uint32_t set_id; - uint64_t addr_tag; - uint32_t sector_id; - uint32_t count; - }; - - std::vector::const_iterator find_line(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) const { - return std::find_if(lines_.begin(), lines_.end(), [&](const line_ref_t& l) { - return l.set_id == set_id && l.addr_tag == addr_tag && l.sector_id == sector_id; - }); - } - - void line_ref(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) { - for (auto &l : lines_) { - if (l.set_id == set_id && l.addr_tag == addr_tag && l.sector_id == sector_id) { - ++l.count; - return; - } - } - __assert(line_cap_ == 0 || lines_.size() < line_cap_, "MSHR line table overflow"); - lines_.push_back({set_id, addr_tag, sector_id, 1}); - } - - void line_unref(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) { - for (auto it = lines_.begin(); it != lines_.end(); ++it) { - if (it->set_id == set_id && it->addr_tag == addr_tag && it->sector_id == sector_id) { - if (--it->count == 0) { - lines_.erase(it); - } - return; - } - } - __assert(false, "MSHR line count underflow"); - } - std::vector entries_; uint32_t ready_reqs_; uint32_t size_; - uint32_t line_cap_; - std::vector lines_; }; class CacheBank : public SimObject { @@ -826,7 +774,7 @@ class CacheBank : public SimObject { const Cache::Config &config, const params_t ¶ms, uint32_t bank_id) - : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size, config.mshr_lines), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), lazy_q_(TFifo::Create("", 1, lazyq_size(config))), rand_ctr_(0), adm_seq_ctr_(0) + : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), lazy_q_(TFifo::Create("", 1, lazyq_size(config))), rand_ctr_(0), adm_seq_ctr_(0) #if VX_CFG_EXT_A_ENABLED , amo_unit_(__MAX(2u, (uint32_t)VX_CFG_AMO_RS_SIZE)) #endif @@ -1244,13 +1192,6 @@ class CacheBank : public SimObject { ++perf_stats_.mshr_stalls; return; } - // With a line table, a line entry must stay free for this request once - // every request ahead of it has claimed its own. - if (needs_mshr && config_.mshr_lines != 0 - && this->mshrLinesClaimed() >= config_.mshr_lines) { - ++perf_stats_.mshr_stalls; - return; - } // Any admitted request may become a write-miss root: the lazy queue // keeps room for every request already in the pipe. if (config_.lazy_fetch && !this->lazyRoom()) { @@ -1351,8 +1292,7 @@ class CacheBank : public SimObject { if (!is_amo_passthru) { ++pending_mshr_size_; pipe_core_lines_.push_back({params_.addr_set_id(core_req.addr), - params_.addr_tag(core_req.addr), - params_.addr_sector_id(core_req.addr)}); + params_.addr_tag(core_req.addr)}); } #if VX_CFG_EXT_A_ENABLED if (is_amo_passthru) ++pending_amo_probes_; @@ -2271,14 +2211,12 @@ class CacheBank : public SimObject { uint32_t pending_mshr_size_; uint32_t mshr_occupancy_; uint32_t pending_amo_probes_; // AmoProbe requests in the pipe, each holding a table reservation - // {set, tag, sector} of Core requests inside the pipe: their MSHR entries - // exist only from pipe exit, so ordering checks against in-flight lines (the - // AMO admission defer) and the line-table admission must consult this - // mirror as well. + // {set, tag} of Core requests inside the pipe: their MSHR entries exist + // only from pipe exit, so ordering checks against in-flight lines (the AMO + // admission defer) must consult this mirror as well. struct pipe_line_t { uint32_t set_id; uint64_t addr_tag; - uint32_t sector_id; }; std::vector pipe_core_lines_; @@ -2302,25 +2240,6 @@ class CacheBank : public SimObject { assert(false && "in-pipe line mirror underflow"); } - // MSHR line entries held or claimed: the lines in the MSHR, plus a line for - // each distinct one a request inside the pipe may still allocate. The RTL - // allocates at the pipe's first stage, so these already hold their entries. - uint32_t mshrLinesClaimed() const { - uint32_t n = mshr_.lines(); - for (uint32_t i = 0, k = pipe_core_lines_.size(); i < k; ++i) { - auto &l = pipe_core_lines_.at(i); - if (mshr_.holds_line(l.set_id, l.addr_tag, l.sector_id)) { - continue; - } - bool seen = false; - for (uint32_t j = 0; j < i && !seen; ++j) { - auto &o = pipe_core_lines_.at(j); - seen = (o.set_id == l.set_id && o.addr_tag == l.addr_tag && o.sector_id == l.sector_id); - } - n += !seen; - } - return n; - } TFifo::Ptr pipe_req_; TFifo::Ptr lazy_q_; // roots awaiting a data-less fill: write misses and written refetches diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index a65ee53196..fd8fafae0c 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -49,7 +49,6 @@ class Cache : public SimObject { bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) bool core_out_buf = false; // registered core-response port uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) - uint16_t mshr_lines = 0; // MSHR distinct {line, sector} entries (0: one per request) bool write_around = false; // a no-allocate write miss goes to memory (write-back only) bool mem_rsp_queue = false; // memory responses enter through a response queue bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 43dad2545c..2e1591f361 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -124,7 +124,6 @@ ProcessorImpl::ProcessorImpl() VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch true, // core-response register (behind the bypass layer) 0, // read_merge - VX_CFG_L3_MSHR_LINES, // mshr_lines CP_ON_L3 != 0, // write_around VX_CFG_L3_MRSQ_SIZE != 0, // memory-response queue } diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index b5b32226f3..9c631bea4c 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -76,7 +76,6 @@ class Socket::Impl { false, // lazy_fetch true, // core-response register (VX_CFG_ICACHE_READ_MERGE != 0) ? ICACHE_NUM_REQS : 0, // read_merge: the core's fetch slices - 0, // mshr_lines false, // write_around VX_CFG_ICACHE_MRSQ_SIZE != 0, // memory-response queue true, // the socket's registered memory port @@ -109,7 +108,6 @@ class Socket::Impl { VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch true, // core-response register (VX_CFG_DCACHE_READ_MERGE != 0) ? DCACHE_CHANNELS : 0, // read_merge - VX_CFG_DCACHE_MSHR_LINES, // mshr_lines false, // write_around VX_CFG_DCACHE_MRSQ_SIZE != 0, // memory-response queue true, // the socket's registered memory port From a99c1be470a5ff3b576411d6b9472f388bc5d1fa Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 12:00:33 -0700 Subject: [PATCH 134/336] cache: turn lazy fetch on wherever the cache is a write-back LLC Write-allocate without fetch (a store miss installs the sector with a dirty-byte mask instead of reading memory) was off by default. It is the write-validate policy GPU L2s use, and the DRAM reads it removes count even where cycles are flat (1-core vecadd: DRAM reads -33%). LAZY_FETCH now defaults to on in the cache that is a write-back last level (the dcache with no L2, the L2 with no L3, or the L3), the only place it is legal; DIRTYBYTES follows it. The CI cases that test fetch-on-write pin LAZY_FETCH=0 next to their DIRTYBYTES=0. rtlsim perf cases (49): instruction counts unchanged; five run 2-9% faster (wgmma-dxa, wgmma-sp-dxa, wgmma-dxa-mcast-nt16, fp16-isw4, fp16-mc-nt16), the rest within 2%. Baselines need the reviewed re-record. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- VX_config.toml | 15 ++++++++------- ci/testcases/amo.yaml | 2 +- ci/testcases/cache.yaml | 2 +- docs/designs/mlp_optimizations.md | 9 +++++---- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9e9efc2dd..ff31bb1c16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, the same cycle are served by one bank access, and the response is multicast to every port; each port's order into the bank is unchanged. *Why:* uniform loads (kernel arguments, SGEMM's shared operand) no longer serialize on one bank: rtlsim sgemm -n128 at 16 threads -6.5%, -17.2% with an 8-byte dcache word. -- **Write-allocate without fetch (`VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH`, off by default).** A store miss in a write-back cache installs +- **Write-allocate without fetch (`VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH`, on where the cache is a write-back LLC).** A store miss in a write-back cache installs the sector with its dirty-byte mask and a partial bit instead of fetching the line; a later load or AMO merges memory data under the unwritten bytes, and evictions write back only dirty bytes. *Why:* streaming writes no longer read every line first: 1-core vecadd DRAM reads -33%. diff --git a/VX_config.toml b/VX_config.toml index d63ef51de1..0f75d3e32d 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -239,12 +239,13 @@ VX_CFG_DCACHE_NUM_WAYS = 4 # stays moderate: NT 1/2/4/8/16/32/64 -> word 4/8/8/16/16/32/32, banks 1/1/2/2/4/4/8. VX_CFG_DCACHE_WORD_SIZE = "expr: min($VX_CFG_L1_LINE_SIZE, min($VX_CFG_MEM_BLOCK_SIZE, ($VX_CFG_XLEN / 8) * pow(2, (clog2($VX_CFG_NUM_LSU_LANES) + 1) / 2)))" VX_CFG_DCACHE_WRITEBACK = "expr: int($dcache_is_llc and $single_core)" -# LAZY_FETCH (write-back LLC only): a store miss allocates its line without a -# fill and a later read fetches and merges under the dirty bytes. It needs the -# per-byte dirty masks, so it turns DIRTYBYTES on; DIRTYBYTES alone keeps -# fetch-on-write. The same pair exists for L2 and L3. +# LAZY_FETCH: a store miss allocates its line without a fill and a later read +# fetches and merges under the dirty bytes (write-validate, as GPU L2s do). It +# is on wherever the cache is a write-back LLC, the only place it is legal. It +# needs the per-byte dirty masks, so it turns DIRTYBYTES on; DIRTYBYTES alone +# keeps fetch-on-write. The same pair exists for L2 and L3. VX_CFG_DCACHE_DIRTYBYTES = "expr: $VX_CFG_DCACHE_LAZY_FETCH" -VX_CFG_DCACHE_LAZY_FETCH = 0 +VX_CFG_DCACHE_LAZY_FETCH = "expr: int($VX_CFG_DCACHE_WRITEBACK and $dcache_is_llc)" VX_CFG_DCACHE_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_DCACHE_MSHR_SIZE = 16 # dcache is unsectored: line = sector = mem transaction granule = L1_LINE (which @@ -279,7 +280,7 @@ VX_CFG_L2_SIZE = 1048576 VX_CFG_L2_NUM_WAYS = 8 VX_CFG_L2_WRITEBACK = "expr: int($l2_is_llc and $single_cluster)" VX_CFG_L2_DIRTYBYTES = "expr: $VX_CFG_L2_LAZY_FETCH" -VX_CFG_L2_LAZY_FETCH = 0 +VX_CFG_L2_LAZY_FETCH = "expr: int($VX_CFG_L2_WRITEBACK and $l2_is_llc)" VX_CFG_L2_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L2_MSHR_SIZE = 16 VX_CFG_L2_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L2_SIZE) - clog2(1048576))" @@ -298,7 +299,7 @@ VX_CFG_L3_SIZE = 2097152 VX_CFG_L3_NUM_WAYS = 8 VX_CFG_L3_WRITEBACK = "expr: int($l3_is_llc)" VX_CFG_L3_DIRTYBYTES = "expr: $VX_CFG_L3_LAZY_FETCH" -VX_CFG_L3_LAZY_FETCH = 0 +VX_CFG_L3_LAZY_FETCH = "expr: int($VX_CFG_L3_WRITEBACK and $l3_is_llc)" VX_CFG_L3_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L3_MSHR_SIZE = 16 VX_CFG_L3_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L3_SIZE) - clog2(2097152))" diff --git a/ci/testcases/amo.yaml b/ci/testcases/amo.yaml index 8ce2da3fbd..1e63cc43c0 100644 --- a/ci/testcases/amo.yaml +++ b/ci/testcases/amo.yaml @@ -30,7 +30,7 @@ tests: - id: wb-dirtybytes0 app: amo drivers: [rtlsim] - configs+: "-DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_DIRTYBYTES=0 -DVX_CFG_DCACHE_NUM_WAYS=4" + configs+: "-DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_LAZY_FETCH=0 -DVX_CFG_DCACHE_DIRTYBYTES=0 -DVX_CFG_DCACHE_NUM_WAYS=4" - id: wb-dirtybytes1 app: amo drivers: [rtlsim] diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index c5b159884d..cd69bd7f37 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -161,7 +161,7 @@ tests: drivers: - rtlsim app: mstress - configs: -DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_DIRTYBYTES=0 -DVX_CFG_DCACHE_NUM_WAYS=4 + configs: -DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_LAZY_FETCH=0 -DVX_CFG_DCACHE_DIRTYBYTES=0 -DVX_CFG_DCACHE_NUM_WAYS=4 - id: mstress-2 via: blackbox drivers: diff --git a/docs/designs/mlp_optimizations.md b/docs/designs/mlp_optimizations.md index bbd4253197..515541c908 100644 --- a/docs/designs/mlp_optimizations.md +++ b/docs/designs/mlp_optimizations.md @@ -25,8 +25,8 @@ Each section says what problem the feature removes, how it works, and what it ga | [Hashed bank select](#hashed-bank-select) | L1D, L2 | Power-of-two strides on one bank | sgemv −32.8% | On (L2; dcache ≥ 4 banks) | | [Fill forwarding](#fill-forwarding) | All caches | Replaying waiters after a fill | sgemv −5.0% | On | | [Back-to-back fills](#back-to-back-fills) | All caches | A bubble after each fill | rtx5080 stream −6.2% | On | -| [Write-allocate without fetch](#write-allocate-without-fetch-lazy-fetch) | Write-back caches | Reading a line only to overwrite it | DRAM reads −33% | Off | -| [Full-sector refetch skip](#full-sector-refetch-skip) | Write-back caches | Refetching a fully written sector | rtx5080 sgemm −43.5% with lazy L2 | Off (with lazy fetch) | +| [Write-allocate without fetch](#write-allocate-without-fetch-lazy-fetch) | Write-back caches | Reading a line only to overwrite it | DRAM reads −33% | On where the cache is a write-back LLC | +| [Full-sector refetch skip](#full-sector-refetch-skip) | Write-back caches | Refetching a fully written sector | rtx5080 sgemm −43.5% with lazy L2 | On with lazy fetch | | [Dcache read merge](#dcache-read-merge) | L1D | Same-word loads serialized on one bank | sgemm −17.2% | On | | [Icache same-line merge](#icache-same-line-merge) | L1I | Fetch slices serialized on one line | rtx5080 sgemm −25.8% | On | | [Broadcast flush, per-port tokens](#broadcast-flush-and-per-port-flush-tokens) | Whole hierarchy | Serial per-core flushes; lost writes | Flush 2.76M → 16.5k cycles per launch | On | @@ -49,7 +49,7 @@ Some of these features can be set with a `VX_CFG_*` knob (see [Build Configurati | Feature | Knob | Default | |---|---|---| | Hashed bank select | `VX_CFG_DCACHE_BANK_HASH`, `VX_CFG_L2_BANK_HASH` | dcache: on with 4 or more banks; L2: on. The L3 has no bank hash. | -| Write-allocate without fetch | `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH` | 0 (off). Enabling it also enables the dirty-byte masks (`*_DIRTYBYTES`). | +| Write-allocate without fetch | `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH` | On where the cache is a write-back LLC (the dcache with no L2, the L2 with no L3, or the L3), off elsewhere. It also enables the dirty-byte masks (`*_DIRTYBYTES`). | | Full-sector refetch skip | Part of lazy fetch | Follows `*_LAZY_FETCH` | | Dcache read merge | `VX_CFG_DCACHE_READ_MERGE` | 1 (on) | | Icache same-line merge | `VX_CFG_ICACHE_READ_MERGE` | 1 (on). No effect with one fetch slice. | @@ -62,6 +62,7 @@ Some of these features can be set with a `VX_CFG_*` knob (see [Build Configurati **Selected automatically:** - The L1-only launch flush and the fence shortcuts apply whenever a single LLC is the point of coherence. +- Write-allocate without fetch, the full-sector refetch skip and the dirty-byte masks turn on in whichever cache is a write-back LLC. - The FPGA shell's memory port count follows the GPU's memory ports. --- @@ -134,7 +135,7 @@ Some of these features can be set with a `VX_CFG_*` knob (see [Build Configurati **How it works.** A store miss installs the sector at once, with a per-byte dirty mask and a *partial* bit, without reading memory. A later load or atomic on a partial sector refetches it and merges memory data only under the bytes that were never written. An eviction writes back only the dirty bytes. A refill of one sector keeps the dirty bytes of the line's other sectors. Controlled by `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH`. -**Gain:** 1-core vecadd DRAM reads −33% and memory requests −25%. Cycles are unchanged because that kernel is latency-bound. Off by default. +**Gain:** 1-core vecadd DRAM reads −33% and memory requests −25%. Cycles are unchanged because that kernel is latency-bound. On where the cache is a write-back LLC, the same write-validate policy NVIDIA's L2 uses: the DRAM traffic it removes counts even where cycles are flat. ### Full-sector refetch skip From 60d79470ab0776bde71fe78684ff9b5e9f6c744b Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 12:01:25 -0700 Subject: [PATCH 135/336] cp: route the command-processor DMA through the LLC by default With the DMA request path registered (b5a21b16c), the CP slot no longer sets a critical path of its own: on the 2-core VM top with an L2 its worst path ties the worst non-CP path. Make CP_LLC_CLIENT the default, so host transfers land in the LLC and stay coherent without a full cache flush, as the GPU profiles already configured. It takes effect only where one LLC serves every core. The CI cases that set CP_LLC_CLIENT=1 drop the now-default flag, and model_parity-dogfood-cp-direct keeps the direct-to-memory DMA path covered on the same L2 shape. L2/L3 perf cases may shift cycles (host data is L2-resident at launch); their baselines are re-recorded with the other phase-2 defaults. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- VX_config.toml | 2 +- ci/testcases/regression.yaml | 15 ++++++++++++--- docs/designs/mlp_optimizations.md | 4 ++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff31bb1c16..f2e579169a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, the sector with its dirty-byte mask and a partial bit instead of fetching the line; a later load or AMO merges memory data under the unwritten bytes, and evictions write back only dirty bytes. *Why:* streaming writes no longer read every line first: 1-core vecadd DRAM reads -33%. -- **Command-processor DMA through the shared LLC (`VX_CFG_CP_LLC_CLIENT`, off by default).** When one LLC serves every core, the +- **Command-processor DMA through the shared LLC (`VX_CFG_CP_LLC_CLIENT`, on by default).** When one LLC serves every core, the CP's transfers enter it on their own client slot, so host transfers need only an L1 flush, not a walk of the whole LLC; transfers of at least half the LLC write around it. A launch behind DMA writes waits for an L1-only flush token sent on the slot. The runtime sees the new `DMA_LLC_CLIENT` CP capability. *Why:* rtlsim at 2 clusters + L2 + L3: dogfood -30%, multikernel -42.7%; dogfood diff --git a/VX_config.toml b/VX_config.toml index 0f75d3e32d..e297ef35ad 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -212,7 +212,7 @@ single_cluster = "expr: $VX_CFG_NUM_CLUSTERS == 1" # is then the point of coherence for host transfers: they need no cache flush. # Only a last-level cache that alone serves every core can take it (an L2 with # one cluster, or an L3); VX_CFG_CP_ON_LLC is set where it took effect. -VX_CFG_CP_LLC_CLIENT = 0 +VX_CFG_CP_LLC_CLIENT = 1 VX_CFG_CP_ON_LLC = "expr: int($VX_CFG_CP_LLC_CLIENT != 0 and (($l2_is_llc and $single_cluster) or $l3_is_llc))" [l1cache] diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index c925823d24..b83322e89b 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -416,7 +416,16 @@ tests: via: blackbox app: dogfood args: -n16 -xbar -xgbar - configs: -DVX_CFG_CP_LLC_CLIENT=1 + shape: + cores: 4 + l2cache: true +# The same with the DMA off the L2, reaching memory on its own path. +- id: model_parity-dogfood-cp-direct + check: model_parity + via: blackbox + app: dogfood + args: -n16 -xbar -xgbar + configs: -DVX_CFG_CP_LLC_CLIENT=0 shape: cores: 4 l2cache: true @@ -428,7 +437,7 @@ tests: via: blackbox app: sgemm args: -n32 - configs: -DVX_CFG_CP_LLC_CLIENT=1 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 shape: cores: 2 l2cache: true @@ -438,7 +447,7 @@ tests: - simx app: sgemm args: -n32 - configs: -DVX_CFG_CP_LLC_CLIENT=1 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 shape: cores: 8 l2cache: true diff --git a/docs/designs/mlp_optimizations.md b/docs/designs/mlp_optimizations.md index 515541c908..c71469b390 100644 --- a/docs/designs/mlp_optimizations.md +++ b/docs/designs/mlp_optimizations.md @@ -53,7 +53,7 @@ Some of these features can be set with a `VX_CFG_*` knob (see [Build Configurati | Full-sector refetch skip | Part of lazy fetch | Follows `*_LAZY_FETCH` | | Dcache read merge | `VX_CFG_DCACHE_READ_MERGE` | 1 (on) | | Icache same-line merge | `VX_CFG_ICACHE_READ_MERGE` | 1 (on). No effect with one fetch slice. | -| CP DMA through the LLC | `VX_CFG_CP_LLC_CLIENT` | 0 (off); 1 in the GPU profiles. Takes effect only when one LLC serves every core. | +| CP DMA through the LLC | `VX_CFG_CP_LLC_CLIENT` | 1 (on). Takes effect only when one LLC serves every core. | | Atomics at the LLC | `VX_CFG_EXT_A_ENABLE` | Off | | Sectored L2/L3 | `VX_CFG_{L2,L3}_SECTOR_SIZE` (a size) | One bus granule. Setting it to the line size removes sectoring. | | Dcache word-size banking | `VX_CFG_DCACHE_WORD_SIZE` (a size) | Derived from the lane count and XLEN | @@ -290,7 +290,7 @@ Atomics with `.aq`/`.rl` get these steps around them. All of it is skipped when - Transfers of at least half the LLC write around it, so they don't evict the working set. - A launch issued after DMA writes waits for an L1-only flush token sent on the DMA's slot. - The runtime detects the feature through the `DMA_LLC_CLIENT` capability. -- Controlled by `VX_CFG_CP_LLC_CLIENT`: off by default, on in the GPU profiles. +- Controlled by `VX_CFG_CP_LLC_CLIENT`: on by default. **Gain (rtlsim):** From fea650f0947c2d11ee6d6057de632159b4261832 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 12:13:50 -0700 Subject: [PATCH 136/336] core: cut logic depth on the three paths that held the core under 300 MHz The phase-2 slice restructure exposed deep paths that master's placement hid; the core gate fell to 280.3 MHz. Each fix keeps the same cycle behaviour: - LSU address: lane-dispatch register -> AGU (3:2 compressor, 32-bit add, stack interleave) -> address-type and byte-enable decode -> the memory scheduler's queue write. The AGU now runs ahead of the stage register: LSU dispatch uses OUT_BUF 0 and VX_lsu_block opens with its own two-entry registered address stage, the buffer OUT_BUF 3 used to place, holding the generated addresses instead of rs1. Same latency and backpressure; the queue write starts from flops. - Issue -> ALU: a 287-fanout op bit drove the immediate mux ahead of the adder. The decoder now pre-selects the immediate form in two former padding bits of alu_args_t/br_args_t (imm_hi: LUI/AUIPC; imm_x2: JAL and conditional branches). The adder enters the result mux at its last level, and branch-equal is a direct compare. - Decompressor: the scheduler's warp id went through the PC compare into the buffer's read-address mux. The buffer now keeps a second LUTRAM copy, one read port per requester; stage 1 picks the port from its registered kind. Core gate: 292.8 MHz (PASS), LUT 172,852 (+0.7%); FF, BRAM, DSP unchanged. ASIC gate: 587.0 MHz, PASS. Lint on five configs, ISA suites, apps on simx and rtlsim, 53 model_parity cases (the one gap is the LMEM merge being off in that tree, reproduced without this change) and all 49 perf cases are cycle-identical to the reference. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 8 +- hw/rtl/core/VX_alu_int.sv | 45 +++++---- hw/rtl/core/VX_decode.sv | 20 ++++ hw/rtl/core/VX_decompressor.sv | 42 +++++++-- hw/rtl/core/VX_execute_slice.sv | 3 +- hw/rtl/core/VX_lsu_agu.sv | 6 +- hw/rtl/core/VX_lsu_block.sv | 160 ++++++++++++++++++++------------ hw/rtl/core/VX_lsu_unit.sv | 3 +- 8 files changed, 193 insertions(+), 94 deletions(-) diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 29ccffe8c3..4dee25d424 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1106,8 +1106,11 @@ package VX_gpu_pkg; // tcu_args_t at 27 bits = 1+1+1+2+5+5+4+4+4. Sparse variants are distinct opcodes. localparam INST_ARGS_BITS = 3 + ALU_TYPE_BITS + 20 + 2; + // imm_hi: imm20 is bits [31:12] of the immediate (LUI, AUIPC). + // imm_x2: imm20 counts halfwords (JAL, conditional branches). typedef struct packed { - logic [1:0] __padding; + logic imm_hi; + logic imm_x2; logic use_PC; logic use_imm; logic is_w; @@ -1118,7 +1121,8 @@ package VX_gpu_pkg; // Branch instructions (JAL/JALR/B/SYS) execute on the ALU pipeline. typedef struct packed { - logic [1:0] __padding; + logic imm_hi; + logic imm_x2; logic use_PC; logic use_imm; logic is_rvc; diff --git a/hw/rtl/core/VX_alu_int.sv b/hw/rtl/core/VX_alu_int.sv index 0c081cc07f..52b03a96b8 100644 --- a/hw/rtl/core/VX_alu_int.sv +++ b/hw/rtl/core/VX_alu_int.sv @@ -48,7 +48,7 @@ module VX_alu_int import VX_gpu_pkg::*; #( reg [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] vote_result; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] shfl_result; - reg [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] alu_result; + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] alu_result; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] alu_result_r; `ifdef VX_CFG_XLEN_64 @@ -65,7 +65,6 @@ module VX_alu_int import VX_gpu_pkg::*; #( wire [INST_ALU_BITS-1:0] alu_op = INST_ALU_BITS'(execute_if.data.op_type); wire [INST_BR_BITS-1:0] br_op = INST_BR_BITS'(execute_if.data.op_type); wire is_br_op = (execute_if.data.op_args.alu.xtype == ALU_TYPE_BRANCH); - wire is_alu_op = (execute_if.data.op_args.alu.xtype == ALU_TYPE_ARITH); wire is_sub_op = inst_alu_is_sub(alu_op); wire is_signed = inst_alu_signed(alu_op); wire [1:0] op_class = is_br_op ? inst_br_class(alu_op) : inst_alu_class(alu_op); @@ -75,16 +74,15 @@ module VX_alu_int import VX_gpu_pkg::*; #( wire [1:0] wg_src_offset = execute_if.data.op_args.alu.imm20[1:0]; // group-relative source index - wire is_br_jal_op = is_br_op && (br_op <= INST_BR_JAL); - wire is_lui_op = is_alu_op && (alu_op == INST_ALU_LUI || alu_op == INST_ALU_AUIPC); - wire [31:0] lui_imm32 = {execute_if.data.op_args.alu.imm20, 12'd0}; wire [20:0] br_imm21 = {execute_if.data.op_args.alu.imm20, 1'b0}; wire [`VX_CFG_XLEN-1:0] alu_imm = `SEXT(`VX_CFG_XLEN, execute_if.data.op_args.alu.imm20); wire [`VX_CFG_XLEN-1:0] lui_imm = `SEXT(`VX_CFG_XLEN, lui_imm32); wire [`VX_CFG_XLEN-1:0] br_imm = `SEXT(`VX_CFG_XLEN, br_imm21); - wire [`VX_CFG_XLEN-1:0] add_imm = is_lui_op ? lui_imm : (is_br_jal_op ? br_imm : alu_imm); + // The decoder pre-selects the immediate form, so only flops steer this mux. + wire [`VX_CFG_XLEN-1:0] add_imm = execute_if.data.op_args.alu.imm_hi ? lui_imm + : (execute_if.data.op_args.alu.imm_x2 ? br_imm : alu_imm); wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] add_in1_PC = execute_if.data.op_args.alu.use_PC ? {NUM_LANES{to_fullPC(execute_if.data.header.PC)}} : alu_in1; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] add_in2_imm = execute_if.data.op_args.alu.use_imm ? {NUM_LANES{add_imm}} : alu_in2; @@ -227,30 +225,39 @@ module VX_alu_int import VX_gpu_pkg::*; #( assign shfl_result[0] = alu_in1[0]; end + // The adder settles last, so its select is decoded from the op alone and + // it enters the result mux at the final level. + wire is_add_result = (execute_if.data.op_args.alu.xtype != ALU_TYPE_OTHER) + && ({is_alu_w, op_class} == 3'b000); + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_alu_result - wire [`VX_CFG_XLEN-1:0] slt_br_result = `VX_CFG_XLEN'({is_br_op && ~(| sub_result[i][`VX_CFG_XLEN-1:0]), sub_result[i][`VX_CFG_XLEN]}); + // the difference's low XLEN bits are zero exactly when the operands match + wire is_equal = (alu_in1[i] == sub_in2_imm[i]); + wire [`VX_CFG_XLEN-1:0] slt_br_result = `VX_CFG_XLEN'({is_br_op && is_equal, sub_result[i][`VX_CFG_XLEN]}); wire [`VX_CFG_XLEN-1:0] sub_slt_br_result = (is_sub_op && ~is_br_op) ? sub_result[i][`VX_CFG_XLEN-1:0] : slt_br_result; + reg [`VX_CFG_XLEN-1:0] other_result; always @(*) begin if (execute_if.data.op_args.alu.xtype == ALU_TYPE_OTHER) begin case (alu_op[3:2]) - 2'b00: alu_result[i] = vote_result[i]; - 2'b01: alu_result[i] = shfl_result[i]; - 2'b10: alu_result[i] = wgather_result[i]; - default: alu_result[i] = vote_result[i]; + 2'b00: other_result = vote_result[i]; + 2'b01: other_result = shfl_result[i]; + 2'b10: other_result = wgather_result[i]; + default: other_result = vote_result[i]; endcase end else begin case ({is_alu_w, op_class}) - 3'b000: alu_result[i] = add_result[i]; // ADD, LUI, AUIPC - 3'b001: alu_result[i] = sub_slt_br_result; // SUB, SLTU, SLTI, BR* - 3'b010: alu_result[i] = shr_zic_result[i]; // SRL, SRA, SRLI, SRAI, CZERO* - 3'b011: alu_result[i] = msc_result[i]; // AND, OR, XOR, SLL, SLLI - 3'b100: alu_result[i] = add_result_w[i]; // ADDIW, ADDW - 3'b101: alu_result[i] = sub_result_w[i]; // SUBW - 3'b110: alu_result[i] = shr_result_w[i]; // SRLW, SRAW, SRLIW, SRAIW - 3'b111: alu_result[i] = msc_result_w[i]; // SLLW + 3'b001: other_result = sub_slt_br_result; // SUB, SLTU, SLTI, BR* + 3'b010: other_result = shr_zic_result[i]; // SRL, SRA, SRLI, SRAI, CZERO* + 3'b011: other_result = msc_result[i]; // AND, OR, XOR, SLL, SLLI + 3'b100: other_result = add_result_w[i]; // ADDIW, ADDW + 3'b101: other_result = sub_result_w[i]; // SUBW + 3'b110: other_result = shr_result_w[i]; // SRLW, SRAW, SRLIW, SRAIW + 3'b111: other_result = msc_result_w[i]; // SLLW + default: other_result = 'x; // ADD, LUI, AUIPC: is_add_result endcase end end + assign alu_result[i] = is_add_result ? add_result[i] : other_result; end // branch diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 328a29503a..636e6f1075 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -231,6 +231,8 @@ module VX_decode import op_args.alu.is_w = 0; op_args.alu.use_PC = 0; op_args.alu.use_imm = 1; + op_args.alu.imm_hi = 0; + op_args.alu.imm_x2 = 0; op_args.alu.imm20 = `SEXT(20, i_imm); `USED_IREG (rd); `USED_IREG (rs1); @@ -240,6 +242,8 @@ module VX_decode import op_args.alu.is_w = 0; op_args.alu.use_PC = 0; op_args.alu.use_imm = 0; + op_args.alu.imm_hi = 0; + op_args.alu.imm_x2 = 0; `USED_IREG (rd); `USED_IREG (rs1); `USED_IREG (rs2); @@ -273,6 +277,8 @@ module VX_decode import op_args.alu.is_w = 1; op_args.alu.use_PC = 0; op_args.alu.use_imm = 1; + op_args.alu.imm_hi = 0; + op_args.alu.imm_x2 = 0; op_args.alu.imm20 = `SEXT(20, iw_imm); `USED_IREG (rd); `USED_IREG (rs1); @@ -282,6 +288,8 @@ module VX_decode import op_args.alu.is_w = 1; op_args.alu.use_PC = 0; op_args.alu.use_imm = 0; + op_args.alu.imm_hi = 0; + op_args.alu.imm_x2 = 0; `USED_IREG (rd); `USED_IREG (rs1); `USED_IREG (rs2); @@ -308,6 +316,8 @@ module VX_decode import op_args.alu.is_w = 0; op_args.alu.use_PC = 0; op_args.alu.use_imm = 1; + op_args.alu.imm_hi = 1; + op_args.alu.imm_x2 = 0; op_args.alu.imm20 = ui_imm; `USED_IREG (rd); end @@ -318,6 +328,8 @@ module VX_decode import op_args.alu.is_w = 0; op_args.alu.use_PC = 1; op_args.alu.use_imm = 1; + op_args.alu.imm_hi = 1; + op_args.alu.imm_x2 = 0; op_args.alu.imm20 = ui_imm; `USED_IREG (rd); end @@ -327,6 +339,8 @@ module VX_decode import op_args.br.xtype = ALU_TYPE_BRANCH; op_args.br.use_PC = 1; op_args.br.use_imm= 1; + op_args.br.imm_hi = 0; + op_args.br.imm_x2 = 1; op_args.br.imm20 = jal_imm; op_args.br.is_rvc = decode_is_rvc; is_wstall = 1; @@ -338,6 +352,8 @@ module VX_decode import op_args.br.xtype = ALU_TYPE_BRANCH; op_args.br.use_PC = 0; op_args.br.use_imm= 1; + op_args.br.imm_hi = 0; + op_args.br.imm_x2 = 0; op_args.br.imm20 = `SEXT(20, u_12); op_args.br.is_rvc = decode_is_rvc; is_wstall = 1; @@ -350,6 +366,8 @@ module VX_decode import op_args.br.xtype = ALU_TYPE_BRANCH; op_args.br.use_PC = 1; op_args.br.use_imm= 1; + op_args.br.imm_hi = 0; + op_args.br.imm_x2 = 1; op_args.br.imm20 = `SEXT(20, b_imm); op_args.br.is_rvc = decode_is_rvc; is_wstall = 1; @@ -427,6 +445,8 @@ module VX_decode import op_type = INST_OP_BITS'(s_type); op_args.br.xtype = ALU_TYPE_BRANCH; op_args.br.use_imm= 1; + op_args.br.imm_hi = 0; + op_args.br.imm_x2 = 0; op_args.br.use_PC = 1; op_args.br.imm20 = 20'd4; op_args.br.is_rvc = decode_is_rvc; diff --git a/hw/rtl/core/VX_decompressor.sv b/hw/rtl/core/VX_decompressor.sv index 229400d48e..a1bf786487 100644 --- a/hw/rtl/core/VX_decompressor.sv +++ b/hw/rtl/core/VX_decompressor.sv @@ -372,10 +372,14 @@ module VX_decompressor import VX_gpu_pkg::*; #( logic buf_we; logic [WID_W-1:0] buf_waddr; buf_data_t buf_wdata; - logic [WID_W-1:0] buf_raddr; - buf_data_t buf_rdata; // registered (OUT_REG=1) + buf_data_t buf_rdata; logic buf_read; // gates rdata_r update during stall + // One read port per requester, so neither read address waits on the + // stage-0 case decode; stage 1 picks the port its kind was read from. + buf_data_t sched_rdata; // registered (OUT_REG=1) + buf_data_t rsp_rdata; // registered (OUT_REG=1) + VX_dp_ram #( .DATAW ($bits(buf_data_t)), .SIZE (NUM_WARPS), @@ -390,8 +394,26 @@ module VX_decompressor import VX_gpu_pkg::*; #( .wren (1'b1), .waddr (buf_waddr), .wdata (buf_wdata), - .raddr (buf_raddr), - .rdata (buf_rdata) + .raddr (sched_wid), + .rdata (sched_rdata) + ); + + VX_dp_ram #( + .DATAW ($bits(buf_data_t)), + .SIZE (NUM_WARPS), + .OUT_REG (1), + .LUTRAM (1), + .RDW_MODE("R") + ) rsp_buffer_ram ( + .clk (clk), + .reset (reset), + .read (buf_read), + .write (buf_we), + .wren (1'b1), + .waddr (buf_waddr), + .wdata (buf_wdata), + .raddr (rsp_wid), + .rdata (rsp_rdata) ); // ------------------------------------------------------------------ @@ -500,11 +522,9 @@ module VX_decompressor import VX_gpu_pkg::*; #( && (state[rsp_wid] == BUF_EMPTY) && !pc_low && !rsp_high_c; - // BRAM raddr (cycle-N read latches data for cycle-N+1 stage 1 use). Both - // scheduler-gated emits (have_C, have_B_emit) read at sched_wid. - assign buf_raddr = (have_C || have_B_emit) ? sched_wid : rsp_wid; - // Only update raddr_r when stage 0 actually advances; keeps the - // registered rdata_r aligned with the registered s1_ctx. + // BRAM read (cycle-N read latches data for cycle-N+1 stage 1 use). Only + // update rdata_r when stage 0 actually advances; keeps the registered + // rdata_r aligned with the registered s1_ctx. assign buf_read = st0_advance; // ------------------------------------------------------------------ @@ -708,6 +728,10 @@ module VX_decompressor import VX_gpu_pkg::*; #( // Stage 1 (combinational): build fsm_data from registered ctx + BRAM // ------------------------------------------------------------------ + // Both scheduler-gated emits (S1_C, S1_B) read at sched_wid; the + // response-gated straddle (S1_BR) reads at rsp_wid. + assign buf_rdata = (s1_kind == S1_BR) ? rsp_rdata : sched_rdata; + // Both RVC and straddle sources are registered 32-bit payloads pre-formed in // stage 0; this stage only selects, no decompress16. wire [31:0] instr_rvc = (s1_kind == S1_C) ? buf_rdata.payload // buffered RVC diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index 5542697bc2..ef032a55a1 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -156,10 +156,11 @@ import VX_tcu_pkg::*; .data_t (lsu_result_t) ) lsu_result_if[1](); + // The LSU block registers its dispatch in its address stage. VX_lane_dispatch #( .BLOCK_SIZE (1), .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .OUT_BUF (3), + .OUT_BUF (0), .NUM_PORTS (1) ) lsu_dispatch ( .clk (clk), diff --git a/hw/rtl/core/VX_lsu_agu.sv b/hw/rtl/core/VX_lsu_agu.sv index 960e9c1b6e..96cbf2906c 100644 --- a/hw/rtl/core/VX_lsu_agu.sv +++ b/hw/rtl/core/VX_lsu_agu.sv @@ -23,9 +23,9 @@ // and a single carry-propagate adder serves both. Addresses in the per-thread // stack window are then interleaved by word across threads. // -// This module sits between the LSU lane dispatch registers and the memory -// scheduler's request queue, so every full-width operation here is reduced to -// the narrowest width that is provably equivalent; see g_stack_interleave. +// This module sits between the issue dispatch registers and the LSU address +// stage register, so every full-width operation here is reduced to the +// narrowest width that is provably equivalent; see g_stack_interleave. module VX_lsu_agu import VX_gpu_pkg::*; ( input wire [`VX_CFG_XLEN-1:0] base, // rs1 diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 6cf3b9446d..a50c108027 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -80,36 +80,78 @@ module VX_lsu_block import VX_gpu_pkg::*; #( .data_t (lsu_result_t) ) result_no_rsp_if(); - // The third operand slot is read only by compare-and-swap, which needs - // the atomic extension to exist at all. + // Address stage: the per-lane AGUs read the operands as dispatched, and the + // stage register holds the generated addresses, so the request into the + // memory scheduler starts from flops. The third operand slot is read only + // by compare-and-swap, which needs the atomic extension to exist at all. + typedef struct packed { + lsu_header_t header; + logic [INST_OP_BITS-1:0] op_type; + op_args_t op_args; + logic [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] addr; + logic [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] rs2_data; + `ifdef VX_CFG_EXT_A_ENABLE + `ifdef VX_CFG_EXT_ZACAS_ENABLE + logic [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] rs3_data; + `endif + `endif + } lsu_req_t; + + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] agu_addr; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr + VX_lsu_agu lsu_agu ( + .base (execute_if.data.rs1_data[i]), + .stride (execute_if.data.rs2_data[i]), + .offset (execute_if.data.op_args.lsu.offset), + .pack (execute_if.data.op_args.lsu.pack), + .addr (agu_addr[i]) + ); + end + + lsu_req_t agu_data; + assign agu_data.header = execute_if.data.header; + assign agu_data.op_type = execute_if.data.op_type; + assign agu_data.op_args = execute_if.data.op_args; + assign agu_data.addr = agu_addr; + assign agu_data.rs2_data = execute_if.data.rs2_data; `ifdef VX_CFG_EXT_A_ENABLE -`ifndef VX_CFG_EXT_ZACAS_ENABLE +`ifdef VX_CFG_EXT_ZACAS_ENABLE + assign agu_data.rs3_data = execute_if.data.rs3_data; +`else `UNUSED_VAR (execute_if.data.rs3_data) `endif `else `UNUSED_VAR (execute_if.data.rs3_data) `endif - // full address calculation — per-lane AGU (all address forms live in - // VX_lsu_agu; this slice contains no address arithmetic). + wire req_valid; + lsu_req_t req_data; + wire req_ready; + + VX_elastic_buffer #( + .DATAW ($bits(lsu_req_t)), + .SIZE (2), + .OUT_REG (1) + ) agu_buf ( + .clk (clk), + .reset (reset), + .valid_in (execute_if.valid), + .ready_in (execute_if.ready), + .data_in (agu_data), + .data_out (req_data), + .valid_out (req_valid), + .ready_out (req_ready) + ); + `UNUSED_VAR ({req_data.op_args.lsu.offset, req_data.op_args.lsu.pack}) + + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr = req_data.addr; wire req_is_fence, rsp_is_fence; - wire req_fence_io = inst_lsu_is_fence_io(execute_if.data.op_type); + wire req_fence_io = inst_lsu_is_fence_io(req_data.op_type); // a memory fence that orders only later reads (acquire) or only earlier // writes (release) - wire req_fence_acq = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_rl; - wire req_fence_rel = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_aq; - - wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr; - for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr - VX_lsu_agu lsu_agu ( - .base (execute_if.data.rs1_data[i]), - .stride (execute_if.data.rs2_data[i]), - .offset (execute_if.data.op_args.lsu.offset), - .pack (execute_if.data.op_args.lsu.pack), - .addr (full_addr[i]) - ); - end + wire req_fence_acq = ~req_fence_io && ~req_data.op_args.lsu.amo_rl; + wire req_fence_rel = ~req_fence_io && ~req_data.op_args.lsu.amo_aq; // address type + AMO classification — per-lane mem_bus_attr_t // carried through the scheduler's USER channel and out via the @@ -157,20 +199,20 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_A_ENABLE amo_req_t lane_amo; - assign lane_amo.amo_valid = execute_if.data.op_args.lsu.amo_valid; - assign lane_amo.amo_op = execute_if.data.op_args.lsu.amo_op; + assign lane_amo.amo_valid = req_data.op_args.lsu.amo_valid; + assign lane_amo.amo_op = req_data.op_args.lsu.amo_op; // amo_unsigned distinguishes signed/unsigned AMOMIN/MAX variants // (decoder collapses MINU/MAXU into MIN/MAX + this bit). - assign lane_amo.amo_unsigned = execute_if.data.op_args.lsu.amo_unsigned; + assign lane_amo.amo_unsigned = req_data.op_args.lsu.amo_unsigned; `ifdef VX_CFG_EXT_ZACAS_ENABLE // Compare-and-swap comparand. The decoder routes rd into the third // operand slot for it; every other atomic leaves the slot unread. - assign lane_amo.amo_cmp = execute_if.data.rs3_data[i]; + assign lane_amo.amo_cmp = req_data.rs3_data[i]; `endif // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid. assign lane_amo.hart_id = HART_ID_WIDTH'( (HART_ID_WIDTH'(core_id) << (NW_BITS + NT_BITS)) - | (HART_ID_WIDTH'(execute_if.data.header.wid) << NT_BITS) + | (HART_ID_WIDTH'(req_data.header.wid) << NT_BITS) | HART_ID_WIDTH'(i) ); assign mem_req_attr_struct[i].amo = lane_amo; @@ -213,13 +255,13 @@ module VX_lsu_block import VX_gpu_pkg::*; #( reg fence_lock; - assign req_is_fence = inst_lsu_is_fence(execute_if.data.op_type); + assign req_is_fence = inst_lsu_is_fence(req_data.op_type); always @(posedge clk) begin if (reset) begin fence_lock <= 0; end else begin - if (mem_req_fire && req_is_fence && execute_if.data.header.eop) begin + if (mem_req_fire && req_is_fence && req_data.header.eop) begin fence_lock <= 1; end if (mem_rsp_fire && rsp_is_fence && mem_rsp_eop_pkt) begin @@ -228,31 +270,31 @@ module VX_lsu_block import VX_gpu_pkg::*; #( end end - wire req_skip = req_is_fence && ~execute_if.data.header.eop; - wire no_rsp_buf_enable = (mem_req_rw && ~execute_if.data.header.wb) || req_skip; + wire req_skip = req_is_fence && ~req_data.header.eop; + wire no_rsp_buf_enable = (mem_req_rw && ~req_data.header.wb) || req_skip; - assign mem_req_valid = execute_if.valid + assign mem_req_valid = req_valid && ~req_skip && ~(no_rsp_buf_enable && ~no_rsp_buf_ready) && ~fence_lock; - assign no_rsp_buf_valid = execute_if.valid + assign no_rsp_buf_valid = req_valid && no_rsp_buf_enable && (req_skip || mem_req_ready) && ~fence_lock; - assign execute_if.ready = (mem_req_ready || req_skip) + assign req_ready = (mem_req_ready || req_skip) && ~(no_rsp_buf_enable && ~no_rsp_buf_ready) && ~fence_lock; - assign mem_req_mask = execute_if.data.header.tmask; + assign mem_req_mask = req_data.header.tmask; `ifdef VX_CFG_EXT_A_ENABLE // AMO MemReq.rw is unconditionally 0: a missing line under SC must // miss-and-return-failure, not write-and-succeed. - assign mem_req_rw = execute_if.data.op_args.lsu.is_store - && ~execute_if.data.op_args.lsu.amo_valid; + assign mem_req_rw = req_data.op_args.lsu.is_store + && ~req_data.op_args.lsu.amo_valid; `else - assign mem_req_rw = execute_if.data.op_args.lsu.is_store; + assign mem_req_rw = req_data.op_args.lsu.is_store; `endif // address formatting @@ -269,7 +311,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( reg [LSU_WORD_SIZE-1:0] mem_req_byteen_w; always @(*) begin mem_req_byteen_w = '0; - case (inst_lsu_wsize(execute_if.data.op_type)) + case (inst_lsu_wsize(req_data.op_type)) 0: begin // 8-bit mem_req_byteen_w[req_align[i]] = 1'b1; end @@ -296,25 +338,25 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // memory misalignment not supported! for (genvar i = 0; i < NUM_LANES; ++i) begin : g_missalign - wire lsu_req_fire = execute_if.valid && execute_if.ready; - `RUNTIME_ASSERT((~lsu_req_fire || ~execute_if.data.header.tmask[i] || req_is_fence || (full_addr[i] % (1 << inst_lsu_wsize(execute_if.data.op_type))) == 0), + wire lsu_req_fire = req_valid && req_ready; + `RUNTIME_ASSERT((~lsu_req_fire || ~req_data.header.tmask[i] || req_is_fence || (full_addr[i] % (1 << inst_lsu_wsize(req_data.op_type))) == 0), ("%t: misaligned memory access, wid=%0d, PC=0x%0h, addr=0x%0h, wsize=%0d! (#%0d)", - $time, execute_if.data.header.wid, to_fullPC(execute_if.data.header.PC), full_addr[i], inst_lsu_wsize(execute_if.data.op_type), execute_if.data.header.uuid)) + $time, req_data.header.wid, to_fullPC(req_data.header.PC), full_addr[i], inst_lsu_wsize(req_data.op_type), req_data.header.uuid)) end // store data formatting for (genvar i = 0; i < NUM_LANES; ++i) begin : g_mem_req_data always @(*) begin - mem_req_data[i] = execute_if.data.rs2_data[i]; + mem_req_data[i] = req_data.rs2_data[i]; case (req_align[i]) - 1: mem_req_data[i][`VX_CFG_XLEN-1:8] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-9:0]; - 2: mem_req_data[i][`VX_CFG_XLEN-1:16] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-17:0]; - 3: mem_req_data[i][`VX_CFG_XLEN-1:24] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-25:0]; + 1: mem_req_data[i][`VX_CFG_XLEN-1:8] = req_data.rs2_data[i][`VX_CFG_XLEN-9:0]; + 2: mem_req_data[i][`VX_CFG_XLEN-1:16] = req_data.rs2_data[i][`VX_CFG_XLEN-17:0]; + 3: mem_req_data[i][`VX_CFG_XLEN-1:24] = req_data.rs2_data[i][`VX_CFG_XLEN-25:0]; `ifdef VX_CFG_XLEN_64 - 4: mem_req_data[i][`VX_CFG_XLEN-1:32] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-33:0]; - 5: mem_req_data[i][`VX_CFG_XLEN-1:40] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-41:0]; - 6: mem_req_data[i][`VX_CFG_XLEN-1:48] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-49:0]; - 7: mem_req_data[i][`VX_CFG_XLEN-1:56] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-57:0]; + 4: mem_req_data[i][`VX_CFG_XLEN-1:32] = req_data.rs2_data[i][`VX_CFG_XLEN-33:0]; + 5: mem_req_data[i][`VX_CFG_XLEN-1:40] = req_data.rs2_data[i][`VX_CFG_XLEN-41:0]; + 6: mem_req_data[i][`VX_CFG_XLEN-1:48] = req_data.rs2_data[i][`VX_CFG_XLEN-49:0]; + 7: mem_req_data[i][`VX_CFG_XLEN-1:56] = req_data.rs2_data[i][`VX_CFG_XLEN-57:0]; `endif default:; endcase @@ -329,8 +371,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( reg [`VX_CFG_LSU_PENDING_SIZE-1:0] pkt_sop, pkt_eop; wire mem_req_rd_fire = mem_req_fire && ~mem_req_rw; - wire mem_req_rd_sop_fire = mem_req_rd_fire && execute_if.data.header.sop; - wire mem_req_rd_eop_fire = mem_req_rd_fire && execute_if.data.header.eop; + wire mem_req_rd_sop_fire = mem_req_rd_fire && req_data.header.sop; + wire mem_req_rd_eop_fire = mem_req_rd_fire && req_data.header.eop; wire mem_rsp_eop_fire = mem_rsp_fire && mem_rsp_eop; wire mem_rsp_eop_pkt_fire= mem_rsp_fire && mem_rsp_eop_pkt; wire full; @@ -398,8 +440,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // pack memory request tag assign mem_req_tag = { - execute_if.data.header, - execute_if.data.op_type, + req_data.header, + req_data.op_type, req_align, pkt_waddr, req_is_fence @@ -545,7 +587,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( .reset (reset), .valid_in (no_rsp_buf_valid), .ready_in (no_rsp_buf_ready), - .data_in (execute_if.data.header), + .data_in (req_data.header), .data_out (result_no_rsp_if.data.header), .valid_out (result_no_rsp_if.valid), .ready_out (result_no_rsp_if.ready) @@ -571,24 +613,24 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `ifdef DBG_TRACE_MEM always @(posedge clk) begin - if (execute_if.valid && fence_lock) begin + if (req_valid && fence_lock) begin `TRACE(2, ("%t: *** %s fence wait\n", $time, INSTANCE_ID)) end if (mem_req_fire) begin if (mem_req_rw) begin - `TRACE(2, ("%t: %s Wr Req: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, addr=", $time, INSTANCE_ID, execute_if.data.header.wid, execute_if.data.header.cta_id, to_fullPC(execute_if.data.header.PC), mem_req_mask)) + `TRACE(2, ("%t: %s Wr Req: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, addr=", $time, INSTANCE_ID, req_data.header.wid, req_data.header.cta_id, to_fullPC(req_data.header.PC), mem_req_mask)) `TRACE_ARRAY1D(2, "0x%h", full_addr, NUM_LANES) `TRACE(2, (", attr=")) `TRACE_ARRAY1D(2, "%b", mem_req_attr, NUM_LANES) `TRACE(2, (", byteen=0x%0h, data=", mem_req_byteen)) `TRACE_ARRAY1D(2, "0x%0h", mem_req_data, NUM_LANES) - `TRACE(2, (", sop=%b, pid=%0d, eop=%b, tag=0x%0h (#%0d)\n", execute_if.data.header.pid, execute_if.data.header.sop, execute_if.data.header.eop, mem_req_tag, execute_if.data.header.uuid)) + `TRACE(2, (", sop=%b, pid=%0d, eop=%b, tag=0x%0h (#%0d)\n", req_data.header.pid, req_data.header.sop, req_data.header.eop, mem_req_tag, req_data.header.uuid)) end else begin - `TRACE(2, ("%t: %s Rd Req: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, addr=", $time, INSTANCE_ID, execute_if.data.header.wid, execute_if.data.header.cta_id, to_fullPC(execute_if.data.header.PC), mem_req_mask)) + `TRACE(2, ("%t: %s Rd Req: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, addr=", $time, INSTANCE_ID, req_data.header.wid, req_data.header.cta_id, to_fullPC(req_data.header.PC), mem_req_mask)) `TRACE_ARRAY1D(2, "0x%h", full_addr, NUM_LANES) `TRACE(2, (", attr=")) `TRACE_ARRAY1D(2, "%b", mem_req_attr, NUM_LANES) - `TRACE(2, (", byteen=0x%0h, rd=%0d, pid=%0d, sop=%b, eop=%b, tag=0x%0h (#%0d)\n", mem_req_byteen, execute_if.data.header.rd, execute_if.data.header.pid, execute_if.data.header.sop, execute_if.data.header.eop, mem_req_tag, execute_if.data.header.uuid)) + `TRACE(2, (", byteen=0x%0h, rd=%0d, pid=%0d, sop=%b, eop=%b, tag=0x%0h (#%0d)\n", mem_req_byteen, req_data.header.rd, req_data.header.pid, req_data.header.sop, req_data.header.eop, mem_req_tag, req_data.header.uuid)) end end if (mem_rsp_fire) begin @@ -621,7 +663,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( full_addr, mem_req_byteen, mem_req_data, - execute_if.data.header.uuid, + req_data.header.uuid, rsp_data, rsp_hdr.uuid }, @@ -636,7 +678,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `ifdef DBG_SCOPE_LSU ila_lsu ila_lsu_inst ( .clk (clk), - .probe0 ({execute_if.valid, execute_if.data, execute_if.ready}), + .probe0 ({req_valid, req_data, req_ready}), .probe1 ({client_if.req_valid, client_if.req_data, client_if.req_ready}), .probe2 ({client_if.rsp_valid, client_if.rsp_data, client_if.rsp_ready}) ); diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index ccd14130a3..9f8f525898 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -58,10 +58,11 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( .data_t (lsu_execute_t) ) per_block_execute_if[BLOCK_SIZE](); + // Each LSU block registers its dispatch in its address stage. VX_lane_dispatch #( .BLOCK_SIZE (BLOCK_SIZE), .NUM_LANES (NUM_LANES), - .OUT_BUF (3) + .OUT_BUF (0) ) lane_dispatch ( .clk (clk), .reset (reset), From 5079a94c672be7b329655ec8697ae35c5c3b7ba5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 14:21:19 -0700 Subject: [PATCH 137/336] ci: hold Fmax to 2% in the FPGA and ASIC synthesis gates A 5% Fmax window let a 300 MHz design slip to 285 MHz and still pass, both against its baseline and against the target clock. Fmax now has its own 2% default in synth_gate.py (METRIC_TOLERANCE), shared by fpga_gate.py and asic_gate.py; area metrics keep 5%. The command line and a catalog entry's "thresholds" still override it, most specific first. Co-Authored-By: Claude Opus 5.5 --- ci/synth_gate.py | 19 +++++++++++-------- ci/testcases/fpga_gate.yaml | 2 +- docs/designs/continuous_integration.md | 2 +- docs/synthesis_analysis.md | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ci/synth_gate.py b/ci/synth_gate.py index 820089a69d..dc3283be0c 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -40,16 +40,17 @@ import yaml -# Gated metrics must stay within +/-5% of baseline. The regression side is the -# gate; the improvement side is the ratchet -- a win beyond it must be recorded -# so the gain is locked in and a later silent slide back to the old number still -# fails. +# Gated metrics must stay within +/-5% of baseline, and Fmax within +/-2% (of +# both the baseline and the target clock). The regression side is the gate; the +# improvement side is the ratchet -- a win beyond it must be recorded so the gain +# is locked in and a later silent slide back to the old number still fails. # -# This is the GLOBAL default. It is overridden per-metric on the command line -# (--metric-threshold lut=0.10) and per-build in the catalog ("thresholds": +# These are the GLOBAL defaults. They are overridden per-metric on the command +# line (--metric-threshold lut=0.10) and per-build in the catalog ("thresholds": # {"fmax_mhz": 0.03}) -- most specific wins. Same shape as model_parity's # `tolerance` (per-case entry > category defaults > DEFAULT_PARITY_TOLERANCE). TOLERANCE = 0.05 +METRIC_TOLERANCE = {"fmax_mhz": 0.02} # Higher-is-better metrics. Everything else is lower-is-better. HIGHER_IS_BETTER = ("fmax_mhz",) @@ -1137,7 +1138,9 @@ def main(argv=None, default_tool="xilinx"): ap.add_argument("--metric-threshold", action="append", default=[], metavar="M=F", help="global tolerance for one metric, e.g. lut=0.10 " - "(repeatable); still overridden per-build") + "(repeatable; default: %s); still overridden " + "per-build" % ", ".join("%s=%g" % kv for kv in + METRIC_TOLERANCE.items())) ap.add_argument("--gate", metavar="M,M", help="metrics to assert on (default: the tool's, %s for %s); " "all metrics are always recorded and reported" @@ -1178,7 +1181,7 @@ def main(argv=None, default_tool="xilinx"): # happens whenever this is piped -- CI, nohup, a log file). sys.stdout.reconfigure(line_buffering=True) - overrides = {} + overrides = dict(METRIC_TOLERANCE) for spec in args.metric_threshold: metric, _, value = spec.partition("=") if metric not in tool.metrics or not value: diff --git a/ci/testcases/fpga_gate.yaml b/ci/testcases/fpga_gate.yaml index cae9b0f0a8..79c7d77d5b 100644 --- a/ci/testcases/fpga_gate.yaml +++ b/ci/testcases/fpga_gate.yaml @@ -195,7 +195,7 @@ builds: clock_mhz: 250 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # Route-dominated at this size: identical-area netlists swung 207-242 MHz - # under the default strategy (placement variance), which no 5% gate + # under the default strategy (placement variance), which no Fmax gate # tolerance survives. The worst arc is 80%+ route (L1 tag way-decode to the # data-store write enables), so the closure pick is the post-route physical # optimizer: 251.8 vs 244.2 for NetDelay_high on the same netlist. diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index 20000d7464..dc51152740 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -276,7 +276,7 @@ and assert the results against golden baselines. | also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | | comparable when | same device, optimization level, xlen | same PDK, VT, corner, xlen | | runner | self-hosted, licensed Vivado | hosted, one job per design | -| default tolerance | 5 % | 5 % | +| default tolerance | Fmax 2 %, LUT 5 % | Fmax 2 %, cell area 5 % | Neither is a pytest cell. A cell is a build tree plus a driver; a synthesis build is neither, and both gates are hours long. They are standalone scripts diff --git a/docs/synthesis_analysis.md b/docs/synthesis_analysis.md index b2bb361da5..26c73652df 100644 --- a/docs/synthesis_analysis.md +++ b/docs/synthesis_analysis.md @@ -451,7 +451,7 @@ make -C build/hw/syn/yosys/dut om CLOCK_FREQ=500 # override the target clock #### Synthesis-regression gate (`asic_gate`) -`ci/asic_gate.py` runs that catalog against checked-in goldens in `ci/baselines/synthesis/yosys/` and fails on a Fmax or cell-area move beyond ±5%. It is the ASIC sibling of `ci/fpga_gate.py` and shares its implementation (`ci/synth_gate.py`). +`ci/asic_gate.py` runs that catalog against checked-in goldens in `ci/baselines/synthesis/yosys/` and fails on a Fmax move beyond ±2% or a cell-area move beyond ±5%. It is the ASIC sibling of `ci/fpga_gate.py` and shares its implementation (`ci/synth_gate.py`). ```bash ci/asic_gate.py --list # builds and their recorded baselines From 4700ff041579e0a85ffcd2f2e922759966a5122f Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 15:24:11 -0700 Subject: [PATCH 138/336] ci: gate Fmax on the target clock and area on a fixed budget, one-sided Fmax was compared with the last recorded baseline both ways, so a build that came out faster than its baseline by more than the tolerance failed as IMPROVED, which placement noise alone triggers at 2%. Every build already declares the clock it is built for, so Fmax is now held only to that target: it fails when it falls more than 2% below it, never when it is higher. Area (LUT on the FPGA, cell area on the ASIC) was relative to the last recording, so each reviewed re-record could bake in up to 5% more and area could creep upward without any single step failing. Each build now declares a fixed budget in the catalog (lut_budget, area_budget_um2), seeded from today's baselines; the gate fails when area grows more than 5% over it and never when it shrinks. Re-recording cannot move a budget; raising one is a reviewed catalog edit. The budgets are not in the config hash, so no baseline goes stale. Baselines are still recorded and their deltas still printed; a metric added with --gate is still compared with its baseline both ways. Co-Authored-By: Claude Opus 5.5 --- ci/synth_gate.py | 53 ++++++++++++++++++-------- ci/testcases/asic_gate.yaml | 17 +++++++++ ci/testcases/fpga_gate.yaml | 20 +++++++++- docs/designs/continuous_integration.md | 23 ++++++----- docs/synthesis_analysis.md | 2 +- 5 files changed, 88 insertions(+), 27 deletions(-) diff --git a/ci/synth_gate.py b/ci/synth_gate.py index dc3283be0c..36913c2f91 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -40,10 +40,13 @@ import yaml -# Gated metrics must stay within +/-5% of baseline, and Fmax within +/-2% (of -# both the baseline and the target clock). The regression side is the gate; the -# improvement side is the ratchet -- a win beyond it must be recorded so the gain -# is locked in and a later silent slide back to the old number still fails. +# Fmax is held to the build's target clock and area to its fixed budget, both +# from the catalog and both one-sided: Fmax may not fall more than 2% below the +# target, area may not grow more than 5% over the budget, and doing better never +# fails. Neither is measured against the last recorded baseline, so re-recording +# cannot ratchet the limit; raising a budget is a reviewed catalog edit. Any +# other metric gated with --gate is compared with its baseline both ways: the +# improvement side is the ratchet, so a win must be recorded to lock it in. # # These are the GLOBAL defaults. They are overridden per-metric on the command # line (--metric-threshold lut=0.10) and per-build in the catalog ("thresholds": @@ -89,6 +92,8 @@ class Tool: metrics = () # recorded, in report order; build_time_s is bookkeeping gated = () # asserted by default (--gate overrides) + area = "" # the gated area metric, held to the catalog's budget + budget_key = "" # the catalog field holding that budget integral = () # recorded as ints; every other metric is a rounded float columns = () # (header, metric, format, width) for the report table @@ -191,7 +196,10 @@ class Xilinx(Tool): metrics = ("fmax_mhz", "wns_ns", "lut", "lutram", "ff", "bram", "uram", "dsp", "build_time_s") - gated = ("fmax_mhz", "lut") + # Fmax is held to the build's clock target (gate()'s BELOW TARGET check). + gated = ("lut",) + area = "lut" + budget_key = "lut_budget" integral = ("lut", "lutram", "ff", "bram", "uram", "dsp", "build_time_s") columns = (("fmax", "fmax_mhz", "%.0f", 9), ("lut", "lut", "%d", 9), ("ff", "ff", "%d", 9), ("lutram", "lutram", "%d", 9), @@ -310,6 +318,8 @@ class Yosys(Tool): # whatever else the design contains: #424 left the RTU's worst path in an # unchanged FMA stage yet moved it 957 -> 903 MHz. gated = ("cell_area_um2",) + area = "cell_area_um2" + budget_key = "area_budget_um2" integral = ("cell_count", "build_time_s") columns = (("fmax", "fmax_mhz", "%.0f", 9), ("area_um2", "cell_area_um2", "%.0f", 11), @@ -517,7 +527,7 @@ def load_catalog(root, tool): if bid in builds: sys.exit("ERROR: duplicate build id '%s' in %s" % (bid, spec_path(root, tool.gate))) - missing = {"dut", "clock_mhz", "configs"} - set(entry) + missing = {"dut", "clock_mhz", "configs", tool.budget_key} - set(entry) if missing: sys.exit("ERROR: %s: missing %s" % (bid, ", ".join(sorted(missing)))) group = entry.get("group", "default") @@ -987,16 +997,31 @@ def verdict(name, reasons): if result["error"]: return "BUILD-FAIL", [result["error"]] + reasons = [] + vs_base = [m for m in gated if m != tool.area] + + if tool.area in gated: + budget = float(build[tool.budget_key]) + cur = result["metrics"].get(tool.area) + if cur is None: + reasons.append("%s: not measured by this run" % tool.area) + else: + tol = threshold(build, tool.area, args) + excess = (cur - budget) / budget + if excess > tol: + reasons.append("%s: %g vs %g budget (%+.1f%%, tolerance %.0f%%) " + "-- OVER BUDGET" + % (tool.area, cur, budget, 100 * excess, 100 * tol)) + base = build.get("baseline") or {} - if not any(base.get(m) is not None for m in gated): + if vs_base and not any(base.get(m) is not None for m in vs_base): return verdict("NO-BASELINE", ["no recorded baseline (run --update-baseline)"]) - if build.get("config_hash") != config_hash(build, env, tool): + if vs_base and build.get("config_hash") != config_hash(build, env, tool): return verdict("STALE", ["config changed since the baseline was recorded " "(re-record with --update-baseline)"]) - reasons = [] - for m in gated: + for m in vs_base: cur, ref = result["metrics"].get(m), base.get(m) if ref is None: reasons.append("%s: missing from baseline" % m) @@ -1015,10 +1040,8 @@ def verdict(name, reasons): if better else "REGRESSION")) # Target-frequency gate: Fmax must be within tolerance of the clock the design - # was BUILT for, independent of the baseline. A baseline recorded below target - # (a design that never met its clock) must not let a build pass just by matching - # that number -- the gate answers "does it meet the frequency it targets?", not - # only "did it get worse than last time?". + # was BUILT for, never of the last recording. One-sided: a faster build passes, + # since placement noise alone moves Fmax by more than the tolerance. target = build.get("clock_mhz") fmax = result["metrics"].get("fmax_mhz") if target and fmax is not None: @@ -1116,7 +1139,7 @@ def cell(key, fmt): def main(argv=None, default_tool="xilinx"): tool = TOOLS[default_tool] ap = argparse.ArgumentParser( - description="Synthesis-regression gate (Fmax/area vs golden baseline)") + description="Synthesis-regression gate (Fmax vs target clock, area vs budget)") ap.add_argument("-b", "--build", action="append", default=[], metavar="ID", help="build id or group to run (repeatable; default: all)") ap.add_argument("-j", "--jobs", type=int, default=2, metavar="N", diff --git a/ci/testcases/asic_gate.yaml b/ci/testcases/asic_gate.yaml index 00def8e7cc..6166cf9a39 100644 --- a/ci/testcases/asic_gate.yaml +++ b/ci/testcases/asic_gate.yaml @@ -78,6 +78,10 @@ tests: # clock_mhz target clock the design is constrained to, via CLOCK_FREQ. Fmax # is derived from it and the worst slack, and the gate also asserts # the design MEETS it — so this is a commitment, not a hint. +# area_budget_um2 cell-area ceiling: the gate fails when cell area grows +# more than 5% over it. Fixed here rather than taken from the +# baseline, so re-recording cannot creep it upward; raising it +# is a reviewed edit. # configs EXTRA defines, appended to the catalog's. Usually empty: the DUT # configuration belongs in one place, and that place is catalog.mk. # group which golden file the results land in (mirrors the Xilinx groups) @@ -103,6 +107,7 @@ builds: group: core dut: cache clock_mhz: 800 + area_budget_um2: 48726 configs: "" # Wide-SIMD core: issue/operand-collect/LSU/FPU at NT=NW=16, plus the RVC @@ -115,6 +120,7 @@ builds: group: core dut: core clock_mhz: 400 + area_budget_um2: 84624 configs: "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8" # Multi-core system top: 2 cores with atomics and virtual memory behind the AFU @@ -124,6 +130,7 @@ builds: group: core dut: top clock_mhz: 400 + area_budget_um2: 187230 # 11.2 GB Yosys peak at NT=NW=4 (8/8: 18.6 GB). configs: "-DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=4" @@ -134,6 +141,7 @@ builds: group: dxa dut: dxa clock_mhz: 800 + area_budget_um2: 12449 configs: "" # --- mufu -------------------------------------------------------------------- @@ -144,6 +152,7 @@ builds: group: mufu dut: mufu clock_mhz: 800 + area_budget_um2: 2073 configs: "" # --- tensor ------------------------------------------------------------------ @@ -154,6 +163,7 @@ builds: group: tensor dut: tcu clock_mhz: 500 + area_budget_um2: 74407 configs: "" # Full tensor subsystem: TCU (WGMMA + MX) + dxa behind a shared L2, 2 cores per @@ -162,6 +172,7 @@ builds: group: tensor dut: tensor clock_mhz: 400 + area_budget_um2: 104438 configs: "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8" # --- graphics ---------------------------------------------------------------- @@ -171,6 +182,7 @@ builds: group: graphics dut: raster clock_mhz: 800 + area_budget_um2: 10527 configs: "" # Output merger: depth/stencil test, blend, ocache port. @@ -178,6 +190,7 @@ builds: group: graphics dut: om clock_mhz: 800 + area_budget_um2: 5926 configs: "" # Texture unit: address generation, LOD, bilinear filter, tcache port. @@ -186,6 +199,7 @@ builds: group: graphics dut: tex clock_mhz: 800 + area_budget_um2: 61159 configs: "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8" # Ray-tracing unit: CW-BVH4 walker, box/triangle PEs, short stack. @@ -193,6 +207,7 @@ builds: group: graphics dut: rtu clock_mhz: 400 + area_budget_um2: 98965 configs: "" # Full graphics subsystem: raster + om + tex + rtu + dxa behind a shared L2, @@ -201,6 +216,7 @@ builds: group: graphics dut: gfx clock_mhz: 400 + area_budget_um2: 242203 # 14.6 GB Yosys peak at NT=NW=4 (8/8: 19.5 GB). configs: "-DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=4" @@ -212,4 +228,5 @@ builds: group: vm dut: vm clock_mhz: 400 + area_budget_um2: 146445 configs: "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8" diff --git a/ci/testcases/fpga_gate.yaml b/ci/testcases/fpga_gate.yaml index 79c7d77d5b..4f0d04a661 100644 --- a/ci/testcases/fpga_gate.yaml +++ b/ci/testcases/fpga_gate.yaml @@ -48,7 +48,12 @@ tests: # builds — read by ci/fpga_gate.py, ignored by the pytest catalog loader. # # dut hw/syn/xilinx/dut/ target to synthesize -# clock_mhz target clock, baked into the constraint via CLK_FREQ_MHZ +# clock_mhz target clock, baked into the constraint via CLK_FREQ_MHZ. The +# gate fails when Fmax falls more than 2% below it. +# lut_budget LUT ceiling: the gate fails when LUT grows more than 5% over +# it. Fixed here rather than taken from the baseline, so +# re-recording cannot creep it upward; raising it is a reviewed +# edit. # group which golden file the results land in # impl_strategy Vivado implementation strategy for this build (IMPL_STRATEGY # env into project.tcl); absent = the OPT_LEVEL default. Part of @@ -71,6 +76,7 @@ builds: group: core dut: cache clock_mhz: 300 + lut_budget: 28369 configs: -DVX_CFG_EXT_A_ENABLE # Wide-SIMD core: issue/operand-collect/LSU/FPU at NT=NW=16, plus the RVC @@ -79,6 +85,7 @@ builds: group: core dut: core clock_mhz: 300 + lut_budget: 171712 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE # Multi-core system top: 2 cores at NT=NW=4, atomics + virtual memory (MMU). @@ -88,6 +95,7 @@ builds: group: core dut: top clock_mhz: 250 + lut_budget: 136762 configs: >- -DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_CORES=2 @@ -101,6 +109,7 @@ builds: group: dxa dut: dxa clock_mhz: 300 + lut_budget: 10102 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # --- mufu -------------------------------------------------------------------- @@ -111,6 +120,7 @@ builds: group: mufu dut: mufu clock_mhz: 300 + lut_budget: 2030 configs: "-DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16" # --- tensor ------------------------------------------------------------------ @@ -121,6 +131,7 @@ builds: group: tensor dut: tcu clock_mhz: 300 + lut_budget: 117377 configs: >- -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_TCU_TYPE_TFR -DVX_CFG_TCU_FP16_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DVX_CFG_TCU_INT8_ENABLE @@ -138,6 +149,7 @@ builds: group: graphics dut: rtu clock_mhz: 300 + lut_budget: 67881 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # Rasterizer: bin/block/quad walker, edge evaluators, rcache port. @@ -145,6 +157,7 @@ builds: group: graphics dut: raster clock_mhz: 300 + lut_budget: 8346 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # Output merger: depth/stencil test, blend, ocache port. @@ -152,6 +165,7 @@ builds: group: graphics dut: om clock_mhz: 300 + lut_budget: 2618 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # Texture unit: address generation, LOD, bilinear filter, tcache port. @@ -159,6 +173,7 @@ builds: group: graphics dut: tex clock_mhz: 300 + lut_budget: 61245 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # --- vm ---------------------------------------------------------------------- @@ -170,6 +185,7 @@ builds: group: vm dut: vm clock_mhz: 250 + lut_budget: 119695 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_CORES=2 # --- graphics (combined subsystem) ------------------------------------------- @@ -182,6 +198,7 @@ builds: group: graphics dut: gfx clock_mhz: 250 + lut_budget: 204871 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # --- tensor (combined subsystem) --------------------------------------------- @@ -193,6 +210,7 @@ builds: group: tensor dut: tensor clock_mhz: 250 + lut_budget: 239260 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # Route-dominated at this size: identical-area netlists swung 207-242 MHz # under the default strategy (placement variance), which no Fmax gate diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index dc51152740..7283e97d0b 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -266,13 +266,14 @@ so the parity and perf views of one workload are two cases. `perf_gate` catches a change that costs cycles. The synthesis gates catch one that costs **timing closure or area**: they synthesize a catalog of designs -and assert the results against golden baselines. +and assert the results against the target clock and area budget each build +declares, recording every run's metrics in golden baselines. | | `fpga_gate` | `asic_gate` | |---|---|---| | flow | Vivado synthesis, place and route | sv2v → Yosys → ABC → OpenSTA, on ASAP7 | | design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | -| gated against the baseline | `fmax_mhz`, `lut` | `cell_area_um2` (Fmax only against the target clock) | +| gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | | also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | | comparable when | same device, optimization level, xlen | same PDK, VT, corner, xlen | | runner | self-hosted, licensed Vivado | hosted, one job per design | @@ -297,7 +298,7 @@ tool. | file | written by | holds | |---|---|---| -| `ci/testcases/{fpga,asic}_gate.yaml` | a person | per build: a design, a target clock, `CONFIGS`, a group, optional `thresholds` and `known_issue` | +| `ci/testcases/{fpga,asic}_gate.yaml` | a person | per build: a design, a target clock, an area budget, `CONFIGS`, a group, optional `thresholds` and `known_issue` | | `ci/baselines/synthesis/{xilinx,yosys}/.json` | the gate | measured metrics, the config hash, the tool versions and environment | Groups are `core`, `tensor`, `graphics`, `dxa` and `vm`. `asic_gate` uses the @@ -306,7 +307,9 @@ gates share a design, a divergence between the tools on the same module is a finding, not noise. The config hash ties the two files together: edit the spec, and the gate -refuses to compare against numbers recorded for the old one. Its +refuses to compare against numbers recorded for the old one. It matters only +for a metric gated against its baseline (`--gate`); the default assertions +read the target clock and the budget, which are not part of the hash. Its tool-specific half is the device and optimization level for Vivado and the PDK, VT and corner for Yosys. @@ -318,9 +321,9 @@ developer runs cannot diverge. | assertion | reason | |---|---| -| each gated metric within tolerance of its baseline | regression detection | -| an improvement beyond tolerance also fails | the gain must be recorded | -| Fmax within tolerance of the **target clock**, independent of the baseline | a baseline recorded below target must not let a build pass by matching it | +| Fmax no more than its tolerance **below the target clock** | the design meets the clock it is built for; a faster result never fails, since placement noise alone moves Fmax by more than 2 % | +| area no more than its tolerance **over the build's budget** | the budget is fixed in the catalog, so re-recording a baseline cannot creep area upward; raising it is a reviewed edit | +| any other metric gated with `--gate`: within tolerance of its baseline, both ways | an improvement beyond tolerance must be recorded | Thresholds resolve most-specific first: a build's own `thresholds`, then a global per-metric threshold, then the global default. @@ -339,10 +342,10 @@ so the flow reports the signed worst slack instead. | verdict | meaning | fails the run | |---|---|---| | `PASS` | within tolerance | no | -| `REGRESSION` | a gated metric got worse | yes | -| `IMPROVED` | a gated metric got better; record it | yes | +| `FAIL` | Fmax below target, area over budget, or a baseline-gated metric worse | yes | +| `IMPROVED` | a baseline-gated metric got better; record it | yes | | `STALE` | the spec or the tool changed since the baseline | yes | -| `FAIL` | the build produced no metrics | yes | +| `BUILD-FAIL` | the build produced no metrics | yes | | `KNOWN-ISSUE` | a tracked expected failure | no | | `XPASS` | a known issue stopped reproducing; clear the flag | no | | `RECORDED` | a baseline was written | no | diff --git a/docs/synthesis_analysis.md b/docs/synthesis_analysis.md index 26c73652df..607da4efe4 100644 --- a/docs/synthesis_analysis.md +++ b/docs/synthesis_analysis.md @@ -451,7 +451,7 @@ make -C build/hw/syn/yosys/dut om CLOCK_FREQ=500 # override the target clock #### Synthesis-regression gate (`asic_gate`) -`ci/asic_gate.py` runs that catalog against checked-in goldens in `ci/baselines/synthesis/yosys/` and fails on a Fmax move beyond ±2% or a cell-area move beyond ±5%. It is the ASIC sibling of `ci/fpga_gate.py` and shares its implementation (`ci/synth_gate.py`). +`ci/asic_gate.py` runs that catalog against checked-in goldens in `ci/baselines/synthesis/yosys/` and fails when Fmax falls more than 2% below the build's target clock or cell area grows more than 5% over its `area_budget_um2`; both limits live in the catalog. It is the ASIC sibling of `ci/fpga_gate.py` and shares its implementation (`ci/synth_gate.py`). ```bash ci/asic_gate.py --list # builds and their recorded baselines From 7536302ceacbcb69cc8759d6e0b2c5ea97fe9042 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 21:22:21 -0700 Subject: [PATCH 139/336] core: restore the LSU lane-dispatch output register fea650f09 moved the LSU's address stage into VX_lsu_block, which left the lane dispatch without its own output register and had the LSU slice register a bus it does not drive. The guidelines give that register to the driver: the dispatch registers its outgoing request, and a consumer never re-registers an incoming interface. This restores the lane dispatch's output buffer and the slice's direct use of it. The change is cycle-neutral (fea650f09 was cycle-identical on every perf case) and has no SimX counterpart. The AGU path it was cutting is shortened instead by aligning the stack window, in the next commit. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_execute_slice.sv | 3 +- hw/rtl/core/VX_lsu_agu.sv | 6 +- hw/rtl/core/VX_lsu_block.sv | 160 ++++++++++++-------------------- hw/rtl/core/VX_lsu_unit.sv | 3 +- 4 files changed, 64 insertions(+), 108 deletions(-) diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index ef032a55a1..5542697bc2 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -156,11 +156,10 @@ import VX_tcu_pkg::*; .data_t (lsu_result_t) ) lsu_result_if[1](); - // The LSU block registers its dispatch in its address stage. VX_lane_dispatch #( .BLOCK_SIZE (1), .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .OUT_BUF (0), + .OUT_BUF (3), .NUM_PORTS (1) ) lsu_dispatch ( .clk (clk), diff --git a/hw/rtl/core/VX_lsu_agu.sv b/hw/rtl/core/VX_lsu_agu.sv index 96cbf2906c..960e9c1b6e 100644 --- a/hw/rtl/core/VX_lsu_agu.sv +++ b/hw/rtl/core/VX_lsu_agu.sv @@ -23,9 +23,9 @@ // and a single carry-propagate adder serves both. Addresses in the per-thread // stack window are then interleaved by word across threads. // -// This module sits between the issue dispatch registers and the LSU address -// stage register, so every full-width operation here is reduced to the -// narrowest width that is provably equivalent; see g_stack_interleave. +// This module sits between the LSU lane dispatch registers and the memory +// scheduler's request queue, so every full-width operation here is reduced to +// the narrowest width that is provably equivalent; see g_stack_interleave. module VX_lsu_agu import VX_gpu_pkg::*; ( input wire [`VX_CFG_XLEN-1:0] base, // rs1 diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index a50c108027..6cf3b9446d 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -80,78 +80,36 @@ module VX_lsu_block import VX_gpu_pkg::*; #( .data_t (lsu_result_t) ) result_no_rsp_if(); - // Address stage: the per-lane AGUs read the operands as dispatched, and the - // stage register holds the generated addresses, so the request into the - // memory scheduler starts from flops. The third operand slot is read only - // by compare-and-swap, which needs the atomic extension to exist at all. - typedef struct packed { - lsu_header_t header; - logic [INST_OP_BITS-1:0] op_type; - op_args_t op_args; - logic [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] addr; - logic [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] rs2_data; - `ifdef VX_CFG_EXT_A_ENABLE - `ifdef VX_CFG_EXT_ZACAS_ENABLE - logic [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] rs3_data; - `endif - `endif - } lsu_req_t; - - wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] agu_addr; - for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr - VX_lsu_agu lsu_agu ( - .base (execute_if.data.rs1_data[i]), - .stride (execute_if.data.rs2_data[i]), - .offset (execute_if.data.op_args.lsu.offset), - .pack (execute_if.data.op_args.lsu.pack), - .addr (agu_addr[i]) - ); - end - - lsu_req_t agu_data; - assign agu_data.header = execute_if.data.header; - assign agu_data.op_type = execute_if.data.op_type; - assign agu_data.op_args = execute_if.data.op_args; - assign agu_data.addr = agu_addr; - assign agu_data.rs2_data = execute_if.data.rs2_data; + // The third operand slot is read only by compare-and-swap, which needs + // the atomic extension to exist at all. `ifdef VX_CFG_EXT_A_ENABLE -`ifdef VX_CFG_EXT_ZACAS_ENABLE - assign agu_data.rs3_data = execute_if.data.rs3_data; -`else +`ifndef VX_CFG_EXT_ZACAS_ENABLE `UNUSED_VAR (execute_if.data.rs3_data) `endif `else `UNUSED_VAR (execute_if.data.rs3_data) `endif - wire req_valid; - lsu_req_t req_data; - wire req_ready; - - VX_elastic_buffer #( - .DATAW ($bits(lsu_req_t)), - .SIZE (2), - .OUT_REG (1) - ) agu_buf ( - .clk (clk), - .reset (reset), - .valid_in (execute_if.valid), - .ready_in (execute_if.ready), - .data_in (agu_data), - .data_out (req_data), - .valid_out (req_valid), - .ready_out (req_ready) - ); - `UNUSED_VAR ({req_data.op_args.lsu.offset, req_data.op_args.lsu.pack}) - - wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr = req_data.addr; + // full address calculation — per-lane AGU (all address forms live in + // VX_lsu_agu; this slice contains no address arithmetic). wire req_is_fence, rsp_is_fence; - wire req_fence_io = inst_lsu_is_fence_io(req_data.op_type); + wire req_fence_io = inst_lsu_is_fence_io(execute_if.data.op_type); // a memory fence that orders only later reads (acquire) or only earlier // writes (release) - wire req_fence_acq = ~req_fence_io && ~req_data.op_args.lsu.amo_rl; - wire req_fence_rel = ~req_fence_io && ~req_data.op_args.lsu.amo_aq; + wire req_fence_acq = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_rl; + wire req_fence_rel = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_aq; + + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr + VX_lsu_agu lsu_agu ( + .base (execute_if.data.rs1_data[i]), + .stride (execute_if.data.rs2_data[i]), + .offset (execute_if.data.op_args.lsu.offset), + .pack (execute_if.data.op_args.lsu.pack), + .addr (full_addr[i]) + ); + end // address type + AMO classification — per-lane mem_bus_attr_t // carried through the scheduler's USER channel and out via the @@ -199,20 +157,20 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_A_ENABLE amo_req_t lane_amo; - assign lane_amo.amo_valid = req_data.op_args.lsu.amo_valid; - assign lane_amo.amo_op = req_data.op_args.lsu.amo_op; + assign lane_amo.amo_valid = execute_if.data.op_args.lsu.amo_valid; + assign lane_amo.amo_op = execute_if.data.op_args.lsu.amo_op; // amo_unsigned distinguishes signed/unsigned AMOMIN/MAX variants // (decoder collapses MINU/MAXU into MIN/MAX + this bit). - assign lane_amo.amo_unsigned = req_data.op_args.lsu.amo_unsigned; + assign lane_amo.amo_unsigned = execute_if.data.op_args.lsu.amo_unsigned; `ifdef VX_CFG_EXT_ZACAS_ENABLE // Compare-and-swap comparand. The decoder routes rd into the third // operand slot for it; every other atomic leaves the slot unread. - assign lane_amo.amo_cmp = req_data.rs3_data[i]; + assign lane_amo.amo_cmp = execute_if.data.rs3_data[i]; `endif // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid. assign lane_amo.hart_id = HART_ID_WIDTH'( (HART_ID_WIDTH'(core_id) << (NW_BITS + NT_BITS)) - | (HART_ID_WIDTH'(req_data.header.wid) << NT_BITS) + | (HART_ID_WIDTH'(execute_if.data.header.wid) << NT_BITS) | HART_ID_WIDTH'(i) ); assign mem_req_attr_struct[i].amo = lane_amo; @@ -255,13 +213,13 @@ module VX_lsu_block import VX_gpu_pkg::*; #( reg fence_lock; - assign req_is_fence = inst_lsu_is_fence(req_data.op_type); + assign req_is_fence = inst_lsu_is_fence(execute_if.data.op_type); always @(posedge clk) begin if (reset) begin fence_lock <= 0; end else begin - if (mem_req_fire && req_is_fence && req_data.header.eop) begin + if (mem_req_fire && req_is_fence && execute_if.data.header.eop) begin fence_lock <= 1; end if (mem_rsp_fire && rsp_is_fence && mem_rsp_eop_pkt) begin @@ -270,31 +228,31 @@ module VX_lsu_block import VX_gpu_pkg::*; #( end end - wire req_skip = req_is_fence && ~req_data.header.eop; - wire no_rsp_buf_enable = (mem_req_rw && ~req_data.header.wb) || req_skip; + wire req_skip = req_is_fence && ~execute_if.data.header.eop; + wire no_rsp_buf_enable = (mem_req_rw && ~execute_if.data.header.wb) || req_skip; - assign mem_req_valid = req_valid + assign mem_req_valid = execute_if.valid && ~req_skip && ~(no_rsp_buf_enable && ~no_rsp_buf_ready) && ~fence_lock; - assign no_rsp_buf_valid = req_valid + assign no_rsp_buf_valid = execute_if.valid && no_rsp_buf_enable && (req_skip || mem_req_ready) && ~fence_lock; - assign req_ready = (mem_req_ready || req_skip) + assign execute_if.ready = (mem_req_ready || req_skip) && ~(no_rsp_buf_enable && ~no_rsp_buf_ready) && ~fence_lock; - assign mem_req_mask = req_data.header.tmask; + assign mem_req_mask = execute_if.data.header.tmask; `ifdef VX_CFG_EXT_A_ENABLE // AMO MemReq.rw is unconditionally 0: a missing line under SC must // miss-and-return-failure, not write-and-succeed. - assign mem_req_rw = req_data.op_args.lsu.is_store - && ~req_data.op_args.lsu.amo_valid; + assign mem_req_rw = execute_if.data.op_args.lsu.is_store + && ~execute_if.data.op_args.lsu.amo_valid; `else - assign mem_req_rw = req_data.op_args.lsu.is_store; + assign mem_req_rw = execute_if.data.op_args.lsu.is_store; `endif // address formatting @@ -311,7 +269,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( reg [LSU_WORD_SIZE-1:0] mem_req_byteen_w; always @(*) begin mem_req_byteen_w = '0; - case (inst_lsu_wsize(req_data.op_type)) + case (inst_lsu_wsize(execute_if.data.op_type)) 0: begin // 8-bit mem_req_byteen_w[req_align[i]] = 1'b1; end @@ -338,25 +296,25 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // memory misalignment not supported! for (genvar i = 0; i < NUM_LANES; ++i) begin : g_missalign - wire lsu_req_fire = req_valid && req_ready; - `RUNTIME_ASSERT((~lsu_req_fire || ~req_data.header.tmask[i] || req_is_fence || (full_addr[i] % (1 << inst_lsu_wsize(req_data.op_type))) == 0), + wire lsu_req_fire = execute_if.valid && execute_if.ready; + `RUNTIME_ASSERT((~lsu_req_fire || ~execute_if.data.header.tmask[i] || req_is_fence || (full_addr[i] % (1 << inst_lsu_wsize(execute_if.data.op_type))) == 0), ("%t: misaligned memory access, wid=%0d, PC=0x%0h, addr=0x%0h, wsize=%0d! (#%0d)", - $time, req_data.header.wid, to_fullPC(req_data.header.PC), full_addr[i], inst_lsu_wsize(req_data.op_type), req_data.header.uuid)) + $time, execute_if.data.header.wid, to_fullPC(execute_if.data.header.PC), full_addr[i], inst_lsu_wsize(execute_if.data.op_type), execute_if.data.header.uuid)) end // store data formatting for (genvar i = 0; i < NUM_LANES; ++i) begin : g_mem_req_data always @(*) begin - mem_req_data[i] = req_data.rs2_data[i]; + mem_req_data[i] = execute_if.data.rs2_data[i]; case (req_align[i]) - 1: mem_req_data[i][`VX_CFG_XLEN-1:8] = req_data.rs2_data[i][`VX_CFG_XLEN-9:0]; - 2: mem_req_data[i][`VX_CFG_XLEN-1:16] = req_data.rs2_data[i][`VX_CFG_XLEN-17:0]; - 3: mem_req_data[i][`VX_CFG_XLEN-1:24] = req_data.rs2_data[i][`VX_CFG_XLEN-25:0]; + 1: mem_req_data[i][`VX_CFG_XLEN-1:8] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-9:0]; + 2: mem_req_data[i][`VX_CFG_XLEN-1:16] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-17:0]; + 3: mem_req_data[i][`VX_CFG_XLEN-1:24] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-25:0]; `ifdef VX_CFG_XLEN_64 - 4: mem_req_data[i][`VX_CFG_XLEN-1:32] = req_data.rs2_data[i][`VX_CFG_XLEN-33:0]; - 5: mem_req_data[i][`VX_CFG_XLEN-1:40] = req_data.rs2_data[i][`VX_CFG_XLEN-41:0]; - 6: mem_req_data[i][`VX_CFG_XLEN-1:48] = req_data.rs2_data[i][`VX_CFG_XLEN-49:0]; - 7: mem_req_data[i][`VX_CFG_XLEN-1:56] = req_data.rs2_data[i][`VX_CFG_XLEN-57:0]; + 4: mem_req_data[i][`VX_CFG_XLEN-1:32] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-33:0]; + 5: mem_req_data[i][`VX_CFG_XLEN-1:40] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-41:0]; + 6: mem_req_data[i][`VX_CFG_XLEN-1:48] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-49:0]; + 7: mem_req_data[i][`VX_CFG_XLEN-1:56] = execute_if.data.rs2_data[i][`VX_CFG_XLEN-57:0]; `endif default:; endcase @@ -371,8 +329,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( reg [`VX_CFG_LSU_PENDING_SIZE-1:0] pkt_sop, pkt_eop; wire mem_req_rd_fire = mem_req_fire && ~mem_req_rw; - wire mem_req_rd_sop_fire = mem_req_rd_fire && req_data.header.sop; - wire mem_req_rd_eop_fire = mem_req_rd_fire && req_data.header.eop; + wire mem_req_rd_sop_fire = mem_req_rd_fire && execute_if.data.header.sop; + wire mem_req_rd_eop_fire = mem_req_rd_fire && execute_if.data.header.eop; wire mem_rsp_eop_fire = mem_rsp_fire && mem_rsp_eop; wire mem_rsp_eop_pkt_fire= mem_rsp_fire && mem_rsp_eop_pkt; wire full; @@ -440,8 +398,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // pack memory request tag assign mem_req_tag = { - req_data.header, - req_data.op_type, + execute_if.data.header, + execute_if.data.op_type, req_align, pkt_waddr, req_is_fence @@ -587,7 +545,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( .reset (reset), .valid_in (no_rsp_buf_valid), .ready_in (no_rsp_buf_ready), - .data_in (req_data.header), + .data_in (execute_if.data.header), .data_out (result_no_rsp_if.data.header), .valid_out (result_no_rsp_if.valid), .ready_out (result_no_rsp_if.ready) @@ -613,24 +571,24 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `ifdef DBG_TRACE_MEM always @(posedge clk) begin - if (req_valid && fence_lock) begin + if (execute_if.valid && fence_lock) begin `TRACE(2, ("%t: *** %s fence wait\n", $time, INSTANCE_ID)) end if (mem_req_fire) begin if (mem_req_rw) begin - `TRACE(2, ("%t: %s Wr Req: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, addr=", $time, INSTANCE_ID, req_data.header.wid, req_data.header.cta_id, to_fullPC(req_data.header.PC), mem_req_mask)) + `TRACE(2, ("%t: %s Wr Req: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, addr=", $time, INSTANCE_ID, execute_if.data.header.wid, execute_if.data.header.cta_id, to_fullPC(execute_if.data.header.PC), mem_req_mask)) `TRACE_ARRAY1D(2, "0x%h", full_addr, NUM_LANES) `TRACE(2, (", attr=")) `TRACE_ARRAY1D(2, "%b", mem_req_attr, NUM_LANES) `TRACE(2, (", byteen=0x%0h, data=", mem_req_byteen)) `TRACE_ARRAY1D(2, "0x%0h", mem_req_data, NUM_LANES) - `TRACE(2, (", sop=%b, pid=%0d, eop=%b, tag=0x%0h (#%0d)\n", req_data.header.pid, req_data.header.sop, req_data.header.eop, mem_req_tag, req_data.header.uuid)) + `TRACE(2, (", sop=%b, pid=%0d, eop=%b, tag=0x%0h (#%0d)\n", execute_if.data.header.pid, execute_if.data.header.sop, execute_if.data.header.eop, mem_req_tag, execute_if.data.header.uuid)) end else begin - `TRACE(2, ("%t: %s Rd Req: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, addr=", $time, INSTANCE_ID, req_data.header.wid, req_data.header.cta_id, to_fullPC(req_data.header.PC), mem_req_mask)) + `TRACE(2, ("%t: %s Rd Req: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, addr=", $time, INSTANCE_ID, execute_if.data.header.wid, execute_if.data.header.cta_id, to_fullPC(execute_if.data.header.PC), mem_req_mask)) `TRACE_ARRAY1D(2, "0x%h", full_addr, NUM_LANES) `TRACE(2, (", attr=")) `TRACE_ARRAY1D(2, "%b", mem_req_attr, NUM_LANES) - `TRACE(2, (", byteen=0x%0h, rd=%0d, pid=%0d, sop=%b, eop=%b, tag=0x%0h (#%0d)\n", mem_req_byteen, req_data.header.rd, req_data.header.pid, req_data.header.sop, req_data.header.eop, mem_req_tag, req_data.header.uuid)) + `TRACE(2, (", byteen=0x%0h, rd=%0d, pid=%0d, sop=%b, eop=%b, tag=0x%0h (#%0d)\n", mem_req_byteen, execute_if.data.header.rd, execute_if.data.header.pid, execute_if.data.header.sop, execute_if.data.header.eop, mem_req_tag, execute_if.data.header.uuid)) end end if (mem_rsp_fire) begin @@ -663,7 +621,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( full_addr, mem_req_byteen, mem_req_data, - req_data.header.uuid, + execute_if.data.header.uuid, rsp_data, rsp_hdr.uuid }, @@ -678,7 +636,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `ifdef DBG_SCOPE_LSU ila_lsu ila_lsu_inst ( .clk (clk), - .probe0 ({req_valid, req_data, req_ready}), + .probe0 ({execute_if.valid, execute_if.data, execute_if.ready}), .probe1 ({client_if.req_valid, client_if.req_data, client_if.req_ready}), .probe2 ({client_if.rsp_valid, client_if.rsp_data, client_if.rsp_ready}) ); diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index 9f8f525898..ccd14130a3 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -58,11 +58,10 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( .data_t (lsu_execute_t) ) per_block_execute_if[BLOCK_SIZE](); - // Each LSU block registers its dispatch in its address stage. VX_lane_dispatch #( .BLOCK_SIZE (BLOCK_SIZE), .NUM_LANES (NUM_LANES), - .OUT_BUF (0) + .OUT_BUF (3) ) lane_dispatch ( .clk (clk), .reset (reset), From 367891fec281e612097e98bdd672e5c0d0894fca Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 21:26:27 -0700 Subject: [PATCH 140/336] lsu: align the stack window top so the stack interleave is a bit permutation The stack window top moves from 0xFFFF0000 to 0xFFFC0000 (0x1FFFC0000 on XLEN 64), a multiple of 256 KB. The window then lies on whole groups of NUM_THREADS stacks for up to 32 threads, so the interleave's group index is the address's own bits above the group and the remap needs no subtract and re-add of the window base: the AGU keeps its one adder, the window compare and a narrow XOR. The LMEM window follows the stack base, as before. This path, lane dispatch -> AGU -> memory scheduler queue, is the LSU's critical path. Out of context the AGU drops from 3.021 to 2.704 ns and from 2391 to 1720 LUTs. The core FPGA gate (with the LMEM read path) reaches 290.7 MHz at 167868 LUTs; the remaining paths under the 300 MHz target are spread across the LSU, SFU, operand collector and FPU at about -0.1 ns. SimX's stack map is rewritten the same way and asserts the alignment. The map was checked equal to the previous one over NT 2-32, multi-core and XLEN 64 (a mutant is caught); SimX regression and rtlsim vecadd, sgemm, sgemm2, dogfood, fence, printf, diverge and amo pass. perf_gate was not run: stack bank mapping can move cycles, and the baselines are already stale on the config hash pending the LMEM re-record. Co-Authored-By: Claude Opus 5.5 --- VX_types.toml | 5 +- docs/designs/lsu_pipeline_design.md | 17 +++--- hw/rtl/core/VX_lsu_agu.sv | 95 +++++------------------------ sim/simx/stack_interleave.h | 19 +++--- 4 files changed, 39 insertions(+), 97 deletions(-) diff --git a/VX_types.toml b/VX_types.toml index 14d0398ba7..04072db399 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -19,7 +19,10 @@ VX_ISA_IMPL_ID = 0 [memmap] VX_MEM_USER_BASE_ADDR = "expr: 0x0000000000010000 if ($XLEN == 64) else 0x00010000" VX_MEM_STACK_LOG2_SIZE = 13 -VX_MEM_STACK_BASE_ADDR = "expr: 0x00000001FFFF0000 if ($XLEN == 64) else 0xFFFF0000" +# The stack window's top is 256 KB aligned, so the window lies on whole groups +# of NUM_THREADS stacks (up to 32 threads) and the stack interleave is a +# pure bit permutation. +VX_MEM_STACK_BASE_ADDR = "expr: 0x00000001FFFC0000 if ($XLEN == 64) else 0xFFFC0000" VX_MEM_LMEM_BASE_ADDR = "expr: $VX_MEM_STACK_BASE_ADDR" VX_MEM_IO_BASE_ADDR = "expr: 0x0000000000000040 if ($XLEN == 64) else 0x00000040" VX_MEM_PAGE_TABLE_BASE_ADDR = "expr: 0x00000000F0000000 if ($XLEN == 64) else 0xF0000000" diff --git a/docs/designs/lsu_pipeline_design.md b/docs/designs/lsu_pipeline_design.md index 1e3531a12d..97268076f3 100644 --- a/docs/designs/lsu_pipeline_design.md +++ b/docs/designs/lsu_pipeline_design.md @@ -87,12 +87,12 @@ Load-bearing properties: - **The XOR with the group index is what spreads warps across sets.** A group spans a power of two larger than a cache way; without the skew, the same frame slot of every warp would share a set. -- **It costs two narrow operations, not three wide ones.** Written naively - the map subtracts the window base, permutes, and adds the base back. The - permutation moves bits only inside the group field and the group index - keeps its position, so the subtract and add cancel on every bit above the - field; the window base has trailing zeros, so the field itself needs - arithmetic only on the bits above them. +- **It is a bit permutation plus one narrow XOR, with no arithmetic.** The + window top `VX_MEM_STACK_BASE_ADDR` is a multiple of the group size + (`NUM_THREADS` stacks), so groups start on absolute group boundaries and + the map needs no subtract or add of the window base: it rewires the + thread and word fields and XORs the word field with the low bits of the + group index. The 256 KB-aligned base covers up to 32 threads. - **Window membership is a narrow compare.** Both window bounds are asserted to be multiples of the stack size, so the test uses only the bits above it. @@ -393,8 +393,9 @@ pool does. | `VX_CFG_LSU_QUEUE_IN_SIZE` | §4.3 | staging depth | | `VX_CFG_LSU_PENDING_SIZE` | §4.3 | outstanding-pool depth | -`NUM_THREADS` must be a power of two for the stack interleave, and a stack -access must fit a word (`FLEN ≤ XLEN`). +`NUM_THREADS` must be a power of two for the stack interleave, the stack +window top must be a multiple of `NUM_THREADS` stacks, and a stack access +must fit a word (`FLEN ≤ XLEN`). --- diff --git a/hw/rtl/core/VX_lsu_agu.sv b/hw/rtl/core/VX_lsu_agu.sv index 960e9c1b6e..204db31184 100644 --- a/hw/rtl/core/VX_lsu_agu.sv +++ b/hw/rtl/core/VX_lsu_agu.sv @@ -24,8 +24,7 @@ // stack window are then interleaved by word across threads. // // This module sits between the LSU lane dispatch registers and the memory -// scheduler's request queue, so every full-width operation here is reduced to -// the narrowest width that is provably equivalent; see g_stack_interleave. +// scheduler's request queue, so it holds no arithmetic beyond the one adder. module VX_lsu_agu import VX_gpu_pkg::*; ( input wire [`VX_CFG_XLEN-1:0] base, // rs1 @@ -66,103 +65,41 @@ module VX_lsu_agu import VX_gpu_pkg::*; ( // depends on the address alone, so any thread may dereference another // thread's stack pointer. // - // Written naively the map costs three dependent full-width operations after - // lin_addr: subtract the window base, permute, add the base back. It does - // not need them. The permutation moves bits only WITHIN the group field -- - // the group index occupies the same position on both sides -- so subtracting - // the base and adding it back cancel on the high bits, and the window base - // has ALIGN_BITS trailing zeros, so the low field needs arithmetic only on - // the CORR_BITS above them. What survives of two XLEN-wide operations is one - // CORR_BITS-wide subtract, one CORR_BITS-wide add, and the +-1 the two - // disagree by. + // The window is aligned to a whole group, so the group index is the + // address's own bits above the group and the map is a bit permutation: + // no arithmetic follows the adder but the window compare. if (`VX_CFG_NUM_THREADS > 1) begin : g_stack_interleave localparam WORD_BITS = `CLOG2(`VX_CFG_XLEN / 8); localparam THREAD_BITS = `CLOG2(`VX_CFG_NUM_THREADS); localparam STACK_BITS = `VX_MEM_STACK_LOG2_SIZE; localparam SLOT_BITS = STACK_BITS - WORD_BITS; localparam GROUP_BITS = STACK_BITS + THREAD_BITS; - localparam HI_BITS = `VX_CFG_XLEN - GROUP_BITS; - // Trailing zeros of the window base. - localparam [`VX_CFG_XLEN-1:0] BOT_LSB = STACK_WINDOW_BOTTOM & -STACK_WINDOW_BOTTOM; - localparam ALIGN_BITS = (BOT_LSB == '0) ? `VX_CFG_XLEN : `CLOG2(BOT_LSB); - localparam CORR_BITS = (GROUP_BITS > ALIGN_BITS) ? (GROUP_BITS - ALIGN_BITS) : 0; localparam WIN_BITS = `VX_CFG_XLEN - STACK_BITS; `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_THREADS), ("invalid parameter: NUM_THREADS=%0d", `VX_CFG_NUM_THREADS)) `STATIC_ASSERT(`VX_CFG_FLEN <= `VX_CFG_XLEN, ("invalid parameter: stack accesses must fit a word")) `STATIC_ASSERT(STACK_WINDOW_SPAN <= STACK_WINDOW_TOP, ("invalid parameter: stack window underflows")) - // The window test compares only the bits above the stack size, so a - // memory-map edit that breaks either bound's alignment must fail here. - `STATIC_ASSERT((STACK_WINDOW_BOTTOM & ((`VX_CFG_XLEN'(1) << STACK_BITS) - 1)) == '0, - ("stack window base is not a multiple of the stack size")) - `STATIC_ASSERT((STACK_WINDOW_TOP & ((`VX_CFG_XLEN'(1) << STACK_BITS) - 1)) == '0, - ("stack window top is not a multiple of the stack size")) + `STATIC_ASSERT(GROUP_BITS + SLOT_BITS <= `VX_CFG_XLEN, ("invalid parameter: NUM_THREADS=%0d", `VX_CFG_NUM_THREADS)) + // The span is a whole number of groups, so a group-aligned top makes + // the bottom group-aligned too. + `STATIC_ASSERT((STACK_WINDOW_TOP & ((`VX_CFG_XLEN'(1) << GROUP_BITS) - 1)) == '0, + ("stack window top is not aligned to a group of NUM_THREADS stacks")) - localparam [GROUP_BITS-1:0] BOT_LO = STACK_WINDOW_BOTTOM[GROUP_BITS-1:0]; - localparam [HI_BITS-1:0] BOT_HI = STACK_WINDOW_BOTTOM[`VX_CFG_XLEN-1:GROUP_BITS]; - - // Both bounds are multiples of the stack size, so membership is decided - // by the bits above it -- a WIN_BITS compare, not a full-width one. wire [WIN_BITS-1:0] lin_win = lin_addr[`VX_CFG_XLEN-1:STACK_BITS]; wire in_stack = (lin_win >= WIN_BITS'(STACK_WINDOW_BOTTOM >> STACK_BITS)) && (lin_win < WIN_BITS'(STACK_WINDOW_TOP >> STACK_BITS)); - wire [GROUP_BITS-1:0] lin_lo = lin_addr[GROUP_BITS-1:0]; - wire [HI_BITS-1:0] lin_hi = lin_addr[`VX_CFG_XLEN-1:GROUP_BITS]; - - // stack_off = lin_addr - STACK_WINDOW_BOTTOM, group field only. - wire [GROUP_BITS-1:0] off_lo; - wire off_borrow; - if (CORR_BITS == 0) begin : g_off_aligned - `UNUSED_PARAM (BOT_LO) - assign off_lo = lin_lo; - assign off_borrow = 1'b0; - end else begin : g_off_corr - wire [CORR_BITS:0] diff = {1'b0, lin_lo[GROUP_BITS-1:ALIGN_BITS]} - - {1'b0, BOT_LO[GROUP_BITS-1:ALIGN_BITS]}; - assign off_lo = {diff[CORR_BITS-1:0], lin_lo[ALIGN_BITS-1:0]}; - assign off_borrow = diff[CORR_BITS]; - end - - // Only SLOT_BITS of the group index reach the XOR, and truncation - // commutes with the subtract, so it is computed at that width. - wire [SLOT_BITS-1:0] stack_group; - if (SLOT_BITS <= HI_BITS) begin : g_group_narrow - assign stack_group = lin_hi[SLOT_BITS-1:0] - BOT_HI[SLOT_BITS-1:0] - - SLOT_BITS'(off_borrow); - end else begin : g_group_wide - assign stack_group = SLOT_BITS'(lin_hi - BOT_HI - HI_BITS'(off_borrow)); - end + wire [SLOT_BITS-1:0] stack_slot = lin_addr[STACK_BITS-1:WORD_BITS] + ^ lin_addr[GROUP_BITS +: SLOT_BITS]; - wire [SLOT_BITS-1:0] stack_slot = off_lo[STACK_BITS-1:WORD_BITS] ^ stack_group; - - wire [GROUP_BITS-1:0] swz_lo = { + wire [`VX_CFG_XLEN-1:0] stack_addr = { + lin_addr[`VX_CFG_XLEN-1:GROUP_BITS], stack_slot, - off_lo[GROUP_BITS-1:STACK_BITS], - off_lo[WORD_BITS-1:0] + lin_addr[GROUP_BITS-1:STACK_BITS], + lin_addr[WORD_BITS-1:0] }; - // addr = STACK_WINDOW_BOTTOM + swz_off, group field only. - wire [GROUP_BITS-1:0] addr_lo; - wire addr_carry; - if (CORR_BITS == 0) begin : g_out_aligned - assign addr_lo = swz_lo; - assign addr_carry = 1'b0; - end else begin : g_out_corr - wire [CORR_BITS:0] sum = {1'b0, swz_lo[GROUP_BITS-1:ALIGN_BITS]} - + {1'b0, BOT_LO[GROUP_BITS-1:ALIGN_BITS]}; - assign addr_lo = {sum[CORR_BITS-1:0], swz_lo[ALIGN_BITS-1:0]}; - assign addr_carry = sum[CORR_BITS]; - end - - // Only the two group-field corrections survive on the high bits. They - // settle last and differ by at most one, so they select a precomputed neighbour. - wire [HI_BITS-1:0] lin_hi_dec = lin_hi - HI_BITS'(1); - wire [HI_BITS-1:0] lin_hi_inc = lin_hi + HI_BITS'(1); - wire [HI_BITS-1:0] addr_hi = (off_borrow == addr_carry) ? lin_hi - : (off_borrow ? lin_hi_dec : lin_hi_inc); - - assign addr = in_stack ? {addr_hi, addr_lo} : lin_addr; + assign addr = in_stack ? stack_addr : lin_addr; end else begin : g_stack_linear assign addr = lin_addr; end diff --git a/sim/simx/stack_interleave.h b/sim/simx/stack_interleave.h index 39b4c6a4e2..4d37a9730b 100644 --- a/sim/simx/stack_interleave.h +++ b/sim/simx/stack_interleave.h @@ -38,12 +38,14 @@ class StackInterleave { static constexpr uint64_t NUM_THREADS = VX_CFG_NUM_THREADS; static constexpr uint64_t NUM_STACKS = uint64_t(VX_CFG_NUM_CLUSTERS) * NUM_SOCKETS * VX_CFG_SOCKET_SIZE * VX_CFG_NUM_WARPS * NUM_THREADS; - static constexpr uint64_t TOP = VX_MEM_STACK_BASE_ADDR; - static constexpr uint64_t SPAN = NUM_STACKS * STACK_SIZE; - static constexpr uint64_t BOTTOM = TOP - SPAN; + static constexpr uint64_t GROUP_SIZE = NUM_THREADS * STACK_SIZE; + static constexpr uint64_t TOP = VX_MEM_STACK_BASE_ADDR; + static constexpr uint64_t SPAN = NUM_STACKS * STACK_SIZE; + static constexpr uint64_t BOTTOM = TOP - SPAN; static_assert(ispow2(NUM_THREADS), "stack interleave needs a power-of-two thread count"); static_assert(SPAN <= TOP, "stack window underflows the address space"); + static_assert(!ENABLED || TOP % GROUP_SIZE == 0, "stack window top is not aligned to a group of stacks"); static bool contains(uint64_t addr) { return ENABLED && addr >= BOTTOM && addr < TOP; @@ -53,13 +55,12 @@ class StackInterleave { if (!contains(addr)) { return addr; } - uint64_t off = addr - BOTTOM; - uint64_t stack = off / STACK_SIZE; - uint64_t thread = stack % NUM_THREADS; - uint64_t group = stack / NUM_THREADS; + // The window is group-aligned, so the group index is the address's own. + uint64_t group = addr / GROUP_SIZE; + uint64_t thread = (addr / STACK_SIZE) % NUM_THREADS; uint64_t words = STACK_SIZE / WORD_SIZE; - uint64_t word = ((off % STACK_SIZE) / WORD_SIZE) ^ (group % words); - return BOTTOM + ((group * words + word) * NUM_THREADS + thread) * WORD_SIZE + (off % WORD_SIZE); + uint64_t word = ((addr % STACK_SIZE) / WORD_SIZE) ^ (group % words); + return group * GROUP_SIZE + (word * NUM_THREADS + thread) * WORD_SIZE + (addr % WORD_SIZE); } }; From 5aecea3598cf9e7c09b06ce418b0a1f98df80c82 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 21 Sep 2026 17:08:44 -0700 Subject: [PATCH 141/336] firesim: take architecture overrides and build extensions The FireSim flow could only build the default configuration, and on master could not build at all: - The AFU shell moved to afu/common, but the include path still named afu/xrt, so vortex_afu.vh was not found. Include afu/common and derive the shell modules to drop from that directory instead of a stale list. - Verilator was resolved from PATH only; include hw/syn/common.mk like the other synthesis flows. - CONFIGS was the full resolved config filtered down to the type selectors, so no architecture override could reach the RTL. CONFIGS now carries user overrides and XCONFIGS the resolved set, as in the XRT flow. - Extensions never had their packages or include paths; include hw/syn/extensions.mk and list their packages in dependency order (raster imports om). - The flattened tree ignored command-line define changes; a flags stamp now invalidates it. - The FPU and RTU select Xilinx floating_point IP on Vivado; add a xilinx-ips target (the XRT flow's script) and a kernel-env target naming what the FireSim kernel packaging reads. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit a57dc2cbe98787c75a1e7d6b0f75d50502fe67fd) --- hw/syn/firesim/Makefile | 80 +++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 18 deletions(-) diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index b945a90189..2336062da5 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -7,6 +7,7 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk +include $(VORTEX_HOME)/hw/syn/common.mk # configure copies this Makefile into the build tree, so ROOT_DIR resolves there # while sources are read from VORTEX_HOME. @@ -38,7 +39,7 @@ VIVADO_SRCS_LIST := $(BUILD_DIR)/vivado_srcs.f SRCS_DIR := $(BUILD_DIR)/vortex_srcs SRCS_LIST := $(BUILD_DIR)/vortex_srcs.f -.PHONY: stage patch sources check-firesim clean-staged +.PHONY: stage patch sources vivado-sources xilinx-ips check-firesim clean-staged stage: check-firesim @mkdir -p $(TARGET_DEST) $(CC_DEST) @@ -61,19 +62,28 @@ patch: check-firesim done XLEN ?= 32 -CONFIGS ?= $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='-DVX_CFG_XLEN=$(XLEN)') + +# Architecture overrides, as the other synthesis flows take them, e.g. +# CONFIGS="-DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TEX_ENABLE". XCONFIGS is the fully +# resolved set, used only to select extension sources and type selectors. +CONFIGS ?= +XCONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') # VX_config.vh and VX_types.vh are generated into the build tree, so those come # from ROOT_DIR while the RTL itself comes from VORTEX_HOME. RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs \ -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm \ - -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/xrt -I$(RTL_DIR)/afu/firesim + -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(RTL_DIR)/afu/firesim + +RTL_PKGS := +include $(VORTEX_HOME)/hw/syn/extensions.mk -# Only XLEN and the TCU/FPU selectors are passed. Every other scalar is computed -# by the generated VX_config.vh behind an ifndef, so defining them here replaces -# those computations with the generator's report values -- which sets the memory -# port counts to zero and yields a design with no memory interface at all. -TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(CONFIGS)) +# Only the overrides and the TCU/FPU selectors are passed. Every other scalar is +# computed by the generated VX_config.vh behind an ifndef, so passing the whole +# resolved XCONFIGS here replaces those computations with the generator's report +# values -- which sets the memory port counts to zero and yields a design with no +# memory interface at all. +TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) # The same defines every other synthesis flow passes (hw/syn/xilinx, hw/syn/altera). # VX_dp_ram and VX_sp_ram gate all block-RAM inference behind SYNTHESIS, and @@ -130,7 +140,14 @@ PLATFORM_DEFINES += -DPLATFORM_MEMORY_ID_WIDTH=4 PERF_DEFINES ?= PLATFORM_DEFINES += $(PERF_DEFINES) -GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(TYPE_DEFINES) $(PLATFORM_DEFINES) $(SYNTH_DEFINES) $(RTL_INCLUDE) +GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) $(SYNTH_DEFINES) $(RTL_INCLUDE) + +# Defines arrive on the command line, which no file timestamp reflects, so a +# changed CONFIGS would otherwise reuse the previous flattened tree. The stamp is +# rewritten only when the flags differ, which keeps an unchanged build current. +CONFIG_STAMP := $(BUILD_DIR)/gen_cflags.stamp +$(shell mkdir -p $(BUILD_DIR) && echo '$(GEN_CFLAGS)' > $(CONFIG_STAMP).tmp && \ + (cmp -s $(CONFIG_STAMP).tmp $(CONFIG_STAMP) && rm -f $(CONFIG_STAMP).tmp || mv -f $(CONFIG_STAMP).tmp $(CONFIG_STAMP))) # Only the packages and the wrapper are named; every other Vortex module is # resolved by library search over RTL_INCLUDE. Handing the whole tree over as a @@ -139,8 +156,13 @@ GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(TYPE_DEFINES) $(PLAT # Packages have to be parsed before the modules that import them, so they are # listed rather than left to library search, in dependency order. Vortex.sv # imports gpu, trace and tlb; fpu is pulled in below the core. +# Extension packages follow in dependency order, which is not extensions.mk's +# append order: raster imports om. +EXT_PKG_ORDER := VX_tcu_pkg.sv VX_dxa_pkg.sv VX_rtu_pkg.sv VX_om_pkg.sv \ + VX_raster_pkg.sv VX_tex_pkg.sv +EXT_PKG_NAMES := $(filter $(notdir $(RTL_PKGS)),$(EXT_PKG_ORDER)) EXPLICIT_NAMES := VX_gpu_pkg.sv VX_fpu_pkg.sv VX_tlb_pkg.sv VX_trace_pkg.sv \ - VX_firesim_wrap.sv + $(EXT_PKG_NAMES) VX_firesim_wrap.sv EXPLICIT_SRCS := $(addprefix $(VIVADO_SRCS_DIR)/,$(EXPLICIT_NAMES)) sources: $(SRCS_LIST) @@ -161,12 +183,12 @@ incdirs: # the constant-parameter failure that ruled flattening out for Verilator is an # elaboration behaviour, not a synthesis one. -# The XRT AFU is a second kernel top with its own control shell. It arrives here -# only because its directory is on the include path for the shared port macros, -# and leaving it in gives Vivado two candidate tops, which makes it elaborate -# modules standalone and reject their interface ports. The header stays; the -# modules go. -XRT_AFU_MODULES := vortex_afu.v VX_afu_ctrl.sv VX_afu_wrap.sv +# The shared AFU shell (VX_afu_wrap and its control, demux, drain and reset +# modules) is a second kernel top. It arrives here only because its directory is +# on the include path for the shared port macros in vortex_afu.vh, and leaving it +# in gives Vivado two candidate tops, which makes it elaborate modules standalone +# and reject their interface ports. The header stays; the modules go. +AFU_SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv)) # Every input the flattener reads. Without these the rule below has no # prerequisites at all, so make runs it exactly once and then reports "nothing to @@ -175,14 +197,14 @@ XRT_AFU_MODULES := vortex_afu.v VX_afu_ctrl.sv VX_afu_wrap.sv # listed too because GEN_CFLAGS resolves widths out of them. RTL_SRCS := $(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh') \ $(wildcard $(ROOT_DIR)/hw/VX_config.vh) $(wildcard $(ROOT_DIR)/hw/VX_types.vh) \ - $(firstword $(MAKEFILE_LIST)) + $(firstword $(MAKEFILE_LIST)) $(CONFIG_STAMP) vivado-sources: $(VIVADO_SRCS_LIST) $(VIVADO_SRCS_LIST): $(RTL_SRCS) @mkdir -p $(BUILD_DIR) cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) \ -TVX_firesim_wrap -Cvivado_srcs -Ovivado_srcs.f - @for f in $(XRT_AFU_MODULES); do \ + @for f in $(AFU_SHELL_MODULES); do \ rm -f $(VIVADO_SRCS_DIR)/$$f; \ grep -v "/$$f$$" $(VIVADO_SRCS_LIST) > $(VIVADO_SRCS_LIST).tmp && \ mv $(VIVADO_SRCS_LIST).tmp $(VIVADO_SRCS_LIST); \ @@ -197,6 +219,28 @@ $(VIVADO_SRCS_LIST): $(RTL_SRCS) fi @echo "wrote $(VIVADO_SRCS_LIST) ($$(grep -cv '^+' $(VIVADO_SRCS_LIST)) files)" +# Vendor floating-point cores. On Vivado the FPU and RTU select Xilinx's +# floating_point IP (xil_fma, xil_fdiv, ...) instead of the soft core, and the +# flattened sources only instantiate them, so the kernel packaging must be handed +# the generated cores or synthesis fails on the missing modules. Same script and +# core set as the XRT flow. The part must match the FireSim kernel's part. +FPGA_PART ?= xcu55c-fsvh2892-2L-e +XILINX_IP_DIR := $(BUILD_DIR)/ip +XILINX_IP_STAMP := $(XILINX_IP_DIR)/.part-$(FPGA_PART) + +xilinx-ips: $(XILINX_IP_STAMP) +$(XILINX_IP_STAMP): $(SCRIPT_DIR)/xilinx_ip_gen.tcl + rm -rf $(XILINX_IP_DIR) + cd $(BUILD_DIR) && vivado -mode batch -nojournal -nolog -source $(SCRIPT_DIR)/xilinx_ip_gen.tcl \ + -tclargs $(XILINX_IP_DIR) $(FPGA_PART) + @test -n "$$(find $(XILINX_IP_DIR) -name '*.xci')" || { echo "ERROR: no .xci generated under $(XILINX_IP_DIR)"; exit 1; } + touch $@ + +# The environment the FireSim Vitis kernel packaging reads (package_kernel.tcl). +kernel-env: + @echo 'export VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST)' + @echo 'export VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR)' + check-firesim: @test -d $(FIRESIM_PATH)/sim || { \ echo "FireSim checkout not found at $(FIRESIM_PATH)"; \ From 66dbebfa0b4b0ff2b1120420664d889afd7a2cf5 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 21 Sep 2026 22:37:08 -0700 Subject: [PATCH 142/336] firesim: resolve async-BRAM placeholders before opt_design The FireSim link never ran the post-synthesis patch that fills the VX_placeholder black boxes VX_async_ram_patch emits, so any configuration big enough to put those RAMs in BRAM failed opt_design with DRC INBB-3 (seen at 1 core, 16 warps x 16 threads, raster+tex+om+rtu on the U55C). kernel-env now exports a pre-opt hook that sources xilinx_async_bram_patch.tcl, as the xrt flow does. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 286071d1ed70d1e6d98bab84f06bdc6aeddc7c7b) --- hw/syn/firesim/Makefile | 6 +++++- hw/syn/firesim/pre_opt_hook.tcl | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 hw/syn/firesim/pre_opt_hook.tcl diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 2336062da5..da7a618da9 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -236,10 +236,14 @@ $(XILINX_IP_STAMP): $(SCRIPT_DIR)/xilinx_ip_gen.tcl @test -n "$$(find $(XILINX_IP_DIR) -name '*.xci')" || { echo "ERROR: no .xci generated under $(XILINX_IP_DIR)"; exit 1; } touch $@ -# The environment the FireSim Vitis kernel packaging reads (package_kernel.tcl). +# The environment the FireSim Vitis kernel build reads: the sources and vendor +# cores package_kernel.tcl attaches, and the pre-opt hook the link runs to +# resolve the async-BRAM placeholders (it locates hw/scripts through TOOL_DIR). kernel-env: @echo 'export VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST)' @echo 'export VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR)' + @echo 'export VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl' + @echo 'export TOOL_DIR=$(SCRIPT_DIR)' check-firesim: @test -d $(FIRESIM_PATH)/sim || { \ diff --git a/hw/syn/firesim/pre_opt_hook.tcl b/hw/syn/firesim/pre_opt_hook.tcl new file mode 100644 index 0000000000..f5b5046a15 --- /dev/null +++ b/hw/syn/firesim/pre_opt_hook.tcl @@ -0,0 +1,11 @@ +# Runs on impl_1 immediately before opt_design, attached by the FireSim platform +# build through VORTEX_PRE_OPT_TCL. +# +# VX_async_ram_patch emits VX_placeholder black boxes wherever an +# asynchronous-read RAM has to be realized on a synchronous BRAM, and their +# contents are filled in here from the synthesized netlist. Without this, +# opt_design aborts with "DRC INBB-3 Black Box Instances" -- but only once a +# configuration is large enough to push those RAMs into BRAM. + +set tool_dir $::env(TOOL_DIR) +source ${tool_dir}/xilinx_async_bram_patch.tcl From 13a0df8f8d661f0d3cd955f8f7b06f3eb29846ac Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 21 Sep 2026 17:17:27 -0700 Subject: [PATCH 143/336] hw/syn/xilinx: card descriptors replace the per-flow platforms.mk Card facts -- memory geometry, device base, connectivity tags, the host tags a shell cannot use, kernel clock -- now live in one TOML descriptor per card under hw/syn/xilinx/platforms/. hw/scripts/gen_platform.py resolves the card the way each flow already names it (XRT: platform XSA, then FPGA family; AVED: device part) and emits the make fragment, and for AVED the linker config that config.cfg.tmpl and its sed step produced. Both flows include it through hw/syn/xilinx/platform.mk, which stops the build on a descriptor error rather than letting $(shell) discard it, and refuses a listed host tag at parse time for any card, not just the V80. Effective flags are unchanged for every card branch (CONFIGS, VPP_FLAGS, CFLAGS, tags and the V80 config.cfg dumped before and after), except: - -DPLATFORM_MEMORY_DATA_WIDTH=512 is no longer emitted. No RTL reads it. - The VCK5000 offset was written 40'hC000000000 without the escape the other cards use, leaving an unmatched quote in the shell command that consumes CFLAGS. It is now escaped; the base is stored as an integer and the sized literal derived from it. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 57b2274a38e24bfc0d31b6472d34e99d4c38e1d4) Integration (phase 2): the descriptors take phase 2's AFU memory model, one AXI master per GPU memory port. An xrt card names its memory channel tag ([memory] tag) and the generator emits PLATFORM_MEMORY_TAG and PLATFORM_MEMORY_CHANNELS, from which the xrt Makefile binds each master to its contiguous channel group; PLATFORM_MERGED_MEMORY_INTERFACE and the per-card connectivity strings are gone. The aved card keeps its single MEM_TAG master. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/afu/common/vortex_afu.vh | 2 +- hw/scripts/gen_platform.py | 307 +++++++++++++++++++++++++ hw/syn/xilinx/aved/Makefile | 44 ++-- hw/syn/xilinx/aved/config.cfg.tmpl | 15 -- hw/syn/xilinx/aved/platforms.mk | 74 ------ hw/syn/xilinx/platform.mk | 19 ++ hw/syn/xilinx/platforms/alveo.toml | 15 ++ hw/syn/xilinx/platforms/u200.toml | 18 ++ hw/syn/xilinx/platforms/u250.toml | 20 ++ hw/syn/xilinx/platforms/u280.toml | 19 ++ hw/syn/xilinx/platforms/u50.toml | 19 ++ hw/syn/xilinx/platforms/u55c.toml | 26 +++ hw/syn/xilinx/platforms/v80.toml | 44 ++++ hw/syn/xilinx/platforms/vck5000.toml | 15 ++ hw/syn/xilinx/platforms/versal.toml | 13 ++ hw/syn/xilinx/platforms/zynquplus.toml | 13 ++ hw/syn/xilinx/xrt/Makefile | 12 +- hw/syn/xilinx/xrt/platforms.mk | 53 ----- 18 files changed, 553 insertions(+), 175 deletions(-) create mode 100644 hw/scripts/gen_platform.py delete mode 100644 hw/syn/xilinx/aved/config.cfg.tmpl delete mode 100644 hw/syn/xilinx/aved/platforms.mk create mode 100644 hw/syn/xilinx/platform.mk create mode 100644 hw/syn/xilinx/platforms/alveo.toml create mode 100644 hw/syn/xilinx/platforms/u200.toml create mode 100644 hw/syn/xilinx/platforms/u250.toml create mode 100644 hw/syn/xilinx/platforms/u280.toml create mode 100644 hw/syn/xilinx/platforms/u50.toml create mode 100644 hw/syn/xilinx/platforms/u55c.toml create mode 100644 hw/syn/xilinx/platforms/v80.toml create mode 100644 hw/syn/xilinx/platforms/vck5000.toml create mode 100644 hw/syn/xilinx/platforms/versal.toml create mode 100644 hw/syn/xilinx/platforms/zynquplus.toml delete mode 100644 hw/syn/xilinx/xrt/platforms.mk diff --git a/hw/rtl/afu/common/vortex_afu.vh b/hw/rtl/afu/common/vortex_afu.vh index c9a6c5075e..dd17de49aa 100644 --- a/hw/rtl/afu/common/vortex_afu.vh +++ b/hw/rtl/afu/common/vortex_afu.vh @@ -81,7 +81,7 @@ .m_axi_mem_``i``_bid(m_axi_mem_``i``_bid) // Host-memory AXI4 master port (CP command ring + host side of DMA). -// XRT pins this kernel port to HOST[0] (see hw/syn/xilinx/xrt/platforms.mk). +// XRT pins this kernel port to the card descriptor's host tag (hw/syn/xilinx/platforms). `define GEN_AXI_HOST \ output wire m_axi_host_awvalid, \ input wire m_axi_host_awready, \ diff --git a/hw/scripts/gen_platform.py b/hw/scripts/gen_platform.py new file mode 100644 index 0000000000..23f3376c17 --- /dev/null +++ b/hw/scripts/gen_platform.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Resolve a Xilinx card descriptor and emit what a build flow consumes. + +Card facts live in hw/syn/xilinx/platforms/.toml, one file per card. A +flow names the card the way it already does -- the XRT flow by platform XSA +and FPGA family, the AVED flow by device part -- and this script resolves the +descriptor and emits one artifact: + + mk make fragment for hw/syn/xilinx/{xrt,aved}/Makefile + cfg slashkit linker config (AVED flow) + firesim-mk make fragment for hw/syn/firesim and sim/firesim + card the resolved card name +""" + +from __future__ import annotations + +import argparse +import fnmatch +import os +import sys +import tomllib +from dataclasses import dataclass, field +from typing import List, Optional + +PLATFORM_DIR = os.path.normpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "syn", "xilinx", "platforms")) + +FLOWS = ("xrt", "aved") + +# The AFU kernel instance name each linker assigns. +KERNEL_INSTANCE = {"xrt": "vortex_afu_1", "aved": "vortex_afu_0"} + + +# Width of the PLATFORM_MEMORY_OFFSET literal the AFU adds to device addresses. +OFFSET_BITS = 40 + + +class DescriptorError(Exception): + pass + + +def verilog_offset(base: int) -> str: + # Sized hex, escaped for make and the shell: Vivado truncates an oversized + # decimal constant to 0 with only a warning. + return f"{OFFSET_BITS}\\'h{base:X}" + + +@dataclass +class Card: + name: str + path: str + flow: str + part: str + family: str + match_xsa: List[str] + match_family: List[str] + match_device: List[str] + banks: int + addr_width: int + base: int + mem_tag: str + host_tag: str + host_refuse: List[str] + host_refuse_why: str + kernel_mhz: int + axil_block_bytes: int + firesim: dict = field(default_factory=dict) + + def specificity(self) -> int: + # An exact XSA or device pattern beats a family match, which beats a + # catch-all, so a card-specific descriptor always wins. + if any(p != "*" for p in self.match_xsa + self.match_device): + return 2 + if self.match_family: + return 1 + return 0 + + +def _require(d: dict, key: str, path: str, ctx: str): + if key not in d: + raise DescriptorError(f"{path}: missing [{ctx}] {key}") + return d[key] + + +def load_card(path: str) -> Card: + with open(path, "rb") as f: + t = tomllib.load(f) + card = _require(t, "card", path, "card") + mem = _require(t, "memory", path, "memory") + host = t.get("host", {}) + clock = t.get("clock", {}) + control = t.get("control", {}) + flow = _require(card, "flow", path, "card") + if flow not in FLOWS: + raise DescriptorError(f"{path}: flow must be one of {FLOWS} (got '{flow}')") + c = Card( + name=_require(card, "name", path, "card"), + path=path, + flow=flow, + part=card.get("part", ""), + family=card.get("family", ""), + match_xsa=list(card.get("match_xsa", [])), + match_family=list(card.get("match_family", [])), + match_device=list(card.get("match_device", [])), + banks=int(_require(mem, "banks", path, "memory")), + addr_width=int(_require(mem, "addr_width", path, "memory")), + base=int(mem.get("base", 0)), + mem_tag=str(mem.get("tag", "")), + host_tag=str(host.get("tag", "")), + host_refuse=list(host.get("refuse", [])), + host_refuse_why=str(host.get("refuse_why", "")), + kernel_mhz=int(clock.get("kernel_mhz", 300)), + axil_block_bytes=int(control.get("axil_block_bytes", 0)), + firesim=dict(t.get("firesim", {})), + ) + if not 0 <= c.base < (1 << OFFSET_BITS): + raise DescriptorError(f"{path}: [memory] base 0x{c.base:x} does not fit {OFFSET_BITS} bits") + if c.host_tag and c.host_tag in c.host_refuse: + raise DescriptorError(f"{path}: host tag '{c.host_tag}' is in its own refuse list") + return c + + +def load_all(platform_dir: str) -> List[Card]: + cards = [] + for fn in sorted(os.listdir(platform_dir)): + if fn.endswith(".toml"): + cards.append(load_card(os.path.join(platform_dir, fn))) + names = [c.name for c in cards] + dups = {n for n in names if names.count(n) > 1} + if dups: + raise DescriptorError(f"duplicate card names: {sorted(dups)}") + return cards + + +def resolve(cards: List[Card], flow: str, xsa: str, family: str, device: str, name: str) -> Card: + pool = [c for c in cards if c.flow == flow] + if name: + hits = [c for c in pool if c.name == name] + if not hits: + raise DescriptorError(f"no {flow} card named '{name}'; known: {sorted(c.name for c in pool)}") + return hits[0] + + def matches(c: Card) -> bool: + if xsa and any(fnmatch.fnmatchcase(xsa, p) for p in c.match_xsa): + return True + if device and any(fnmatch.fnmatchcase(device, p) for p in c.match_device): + return True + if family and family in c.match_family: + return True + return False + + hits = [c for c in pool if matches(c)] + if not hits: + what = xsa or device or family or "" + raise DescriptorError(f"no {flow} card descriptor matches '{what}'; known: " + f"{sorted(c.name for c in pool)} (add one under {PLATFORM_DIR})") + best = max(c.specificity() for c in hits) + top = [c for c in hits if c.specificity() == best] + if len(top) > 1: + raise DescriptorError(f"ambiguous card for '{xsa or device or family}': " + f"{sorted(c.name for c in top)}") + return top[0] + + +def emit_mk(c: Card) -> str: + ov = "override " if c.flow == "aved" else "" + inst = KERNEL_INSTANCE[c.flow] + lines = [f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", + f"PLATFORM_CARD := {c.name}"] + if c.part: + lines.append(f"PLATFORM_PART := {c.part}") + lines.append(f"PLATFORM_AXIL_BLOCK_BYTES := {c.axil_block_bytes}") + lines.append(f"PLATFORM_HOST_REFUSE := {' '.join(c.host_refuse)}") + lines.append(f"PLATFORM_HOST_REFUSE_WHY := {c.host_refuse_why}") + + lines.append(f"{ov}CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS={c.banks} " + f"-DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH={c.addr_width}") + if c.base: + lines.append(f"{ov}CONFIGS += -DPLATFORM_MEMORY_OFFSET={verilog_offset(c.base)}") + + if c.flow == "xrt": + if c.host_tag: + lines.append(f"HOST_TAG = {c.host_tag}") + lines.append(f"VPP_FLAGS += --connectivity.sp {inst}.m_axi_host:$(HOST_TAG)") + # The flow binds each AFU memory master to a contiguous group of the + # card's memory channels. + if c.mem_tag: + lines.append(f"PLATFORM_MEMORY_TAG := {c.mem_tag}") + lines.append(f"PLATFORM_MEMORY_CHANNELS := {c.banks}") + else: + if not c.mem_tag: + raise DescriptorError(f"{c.path}: the aved flow binds its memory port to a [memory] tag") + lines.append(f"MEM_TAG = {c.mem_tag}") + lines.append(f"HOST_TAG = {c.host_tag}") + if c.part: + lines.append(f"DEVICE ?= {c.part}") + lines.append(f"KERNEL_FREQ ?= {c.kernel_mhz}") + return "\n".join(lines) + "\n" + + +def emit_cfg(c: Card, mem_tag: str, host_tag: str, kernel_mhz: int) -> str: + if c.flow != "aved": + raise DescriptorError(f"card '{c.name}' is an {c.flow} card; cfg is for the aved flow") + check_host_tag(c, host_tag) + inst = KERNEL_INSTANCE["aved"] + return "\n".join([ + f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", + "[connectivity]", + "# Bind to the compute-only static shell. The service shell adds the DCMAC", + "# ethernet stack the AFU has no use for.", + "shell=compute", + f"nk=vortex_afu:1:{inst}", + f"sp={inst}.m_axi_mem_0:{mem_tag}", + f"sp={inst}.m_axi_host:{host_tag}", + "", + "# Kernel clock, in Hz rather than via the linker's --clock-hz flag, whose", + "# name and help text disagree on the unit.", + "[clock]", + f"krnl={inst}", + f"freqhz={kernel_mhz * 1000000}", + "", + ]) + + +def emit_firesim_mk(c: Card) -> str: + fs = c.firesim + if not fs: + raise DescriptorError(f"{c.path}: no [firesim] section; card '{c.name}' has no FireSim platform") + lines = [f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", + f"FIRESIM_CARD := {c.name}", + f"PLATFORM := {_require(fs, 'platform', c.path, 'firesim')}", + f"FPGA_PART := {c.part}"] + if "device" in fs: + lines.append(f"DEVICE := {fs['device']}") + if "host_mem_sp" in fs: + lines.append(f"HOST_MEM_SP := {fs['host_mem_sp']}") + if "platform_config" in fs: + lines.append(f"PLATFORM_CONFIG := {fs['platform_config']}") + return "\n".join(lines) + "\n" + + +def check_host_tag(c: Card, host_tag: str) -> None: + if host_tag in c.host_refuse: + raise DescriptorError(f"host tag '{host_tag}' is refused on card '{c.name}': {c.host_refuse_why}") + + +def main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--flow", required=True, choices=FLOWS) + ap.add_argument("--xsa", default="", help="XRT platform name (xrt flow)") + ap.add_argument("--family", default="", help="FPGA family reported by platforminfo (xrt flow)") + ap.add_argument("--device", default="", help="device part (aved flow)") + ap.add_argument("--card", default="", help="select a card by name instead of matching") + ap.add_argument("--emit", required=True, choices=("mk", "cfg", "firesim-mk", "card")) + ap.add_argument("--mem-tag", default="") + ap.add_argument("--host-tag", default="") + ap.add_argument("--kernel-mhz", type=int, default=0) + ap.add_argument("--platform-dir", default=PLATFORM_DIR) + ap.add_argument("-o", "--output", default="-") + a = ap.parse_args(argv) + + try: + cards = load_all(a.platform_dir) + c = resolve(cards, a.flow, a.xsa, a.family.strip(), a.device, a.card) + if a.emit == "mk": + out = emit_mk(c) + elif a.emit == "cfg": + out = emit_cfg(c, a.mem_tag or c.mem_tag, a.host_tag or c.host_tag, + a.kernel_mhz or c.kernel_mhz) + elif a.emit == "firesim-mk": + out = emit_firesim_mk(c) + else: + out = c.name + "\n" + except DescriptorError as e: + print(f"gen_platform: error: {e}", file=sys.stderr) + return 1 + + if a.output == "-": + sys.stdout.write(out) + else: + # Rewrite only on change so dependents are not needlessly invalidated. + old = None + if os.path.exists(a.output): + with open(a.output) as f: + old = f.read() + if old != out: + os.makedirs(os.path.dirname(os.path.abspath(a.output)), exist_ok=True) + with open(a.output, "w") as f: + f.write(out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hw/syn/xilinx/aved/Makefile b/hw/syn/xilinx/aved/Makefile index 0787371146..03ba252f52 100644 --- a/hw/syn/xilinx/aved/Makefile +++ b/hw/syn/xilinx/aved/Makefile @@ -85,7 +85,7 @@ CONFIGS_64c := -DVX_CFG_NUM_CLUSTERS=4 -DVX_CFG_NUM_CORES=16 override CONFIGS += $(CONFIGS_$(NUM_CORES)c) endif -# `override`, as in platforms.mk: a command-line `make CONFIGS=...` otherwise +# `override`, as in the generated card settings: a command-line `make CONFIGS=...` otherwise # discards these appends, and a build silently missing -DSYNTHESIS elaborates # the simulation-only paths. override CONFIGS += -DSYNTHESIS -DVIVADO @@ -115,34 +115,22 @@ RTL_INCLUDE += -I$(RTL_DIR)/cp # the resolved XCONFIGS. include $(VORTEX_HOME)/hw/syn/extensions.mk -# load platform settings -include $(SRC_DIR)/platforms.mk - -# Memory connectivity tags resolved into the linker config. -MEM_TAG ?= HBM0 -HOST_TAG ?= HOST - -# HOST routes m_axi_host to the QDMA slave bridge, whose reads never complete -# on the compute shell: the CP hangs on its first ring fetch and reports -# nothing -- the queue reads back armed and correct while head stays at 0. That -# is a full synthesis run to discover, and it has already cost one, so refuse -# the tag at parse time rather than emitting the config. See platforms.mk. -ifeq ($(HOST_TAG),HOST) -$(error HOST_TAG=HOST routes m_axi_host to the QDMA slave bridge, whose reads never complete on the compute shell; the CP would hang on its first ring fetch. Use HOST_TAG=HBM8 (the platforms.mk default)) -endif +# Card settings (memory geometry, connectivity tags, kernel clock), resolved +# from the device part. A tag the card refuses -- on the V80, HOST, whose reads +# never complete on the compute shell -- stops the build at parse time rather +# than after a full synthesis run. +PLATFORM_GEN_ARGS := --flow=aved --device=$(DEVICE) +include $(VORTEX_HOME)/hw/syn/xilinx/platform.mk # MEM covers the 32-bit device space from the HBM base, i.e. HBM0..HBM7; a CP # staging aperture there aliases device memory and large uploads overwrite the -# heap. See platforms.mk. +# heap. See hw/syn/xilinx/platforms/v80.toml. ifneq ($(filter HBM0 HBM1 HBM2 HBM3 HBM4 HBM5 HBM6 HBM7,$(HOST_TAG)),) ifneq ($(filter MEM HBM0,$(MEM_TAG)),) $(error HOST_TAG=$(HOST_TAG) lies inside the device-memory window of MEM_TAG=$(MEM_TAG) (HBM0..HBM7); CP staging would alias the Vortex heap. Use HBM8 or above) endif endif -# platforms.mk states the kernel clock in MHz; the linker config wants Hz. -KERNEL_FREQ_HZ := $(shell echo $$(( $(KERNEL_FREQ) * 1000000 ))) - # Debugging ifdef DEBUG CFLAGS += -DVX_DBG_DEBUG_LEVEL=$(DEBUG) $(DBG_TRACE_FLAGS) @@ -178,13 +166,13 @@ AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(BUILD_DIR)/.afu_mem ifeq ($(AFU_MEM_PORTS),) $(error cannot resolve the AFU memory port count for this configuration) endif -# config.cfg.tmpl binds the single device-memory master to MEM_TAG. +# The linker config binds the single device-memory master to MEM_TAG. ifneq ($(AFU_MEM_PORTS),1) $(error the V80 link binds one device-memory master, but this configuration has AFU_MEM_PORTS=$(AFU_MEM_PORTS)) endif CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) -.PHONY: all clean gen-sources gen-ast scope-json package-ip gen-bin check-axil-blocks +.PHONY: all clean gen-sources gen-ast scope-json package-ip gen-bin check-axil-blocks FORCE # Fail before synthesis if any 16-byte block in the CP AXI-Lite regfile is only # partially decoded. Measured on V80 silicon: such a block DECERRs on EVERY @@ -220,11 +208,13 @@ scope-json: $(BIN_DIR)/scope.json $(BIN_DIR)/scope.json: $(BUILD_DIR)/vortex.xml mkdir -p $(BIN_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/scope.py vortex.xml -o bin/scope.json -$(BUILD_DIR)/config.cfg: $(SRC_DIR)/config.cfg.tmpl - mkdir -p $(BUILD_DIR) - sed -e 's|@MEM_TAG@|$(MEM_TAG)|g' -e 's|@HOST_TAG@|$(HOST_TAG)|g' \ - -e 's|@KERNEL_FREQ_HZ@|$(KERNEL_FREQ_HZ)|g' $< > $@ - @grep -q '@[A-Z_]*@' $@ && { echo "ERROR: unsubstituted placeholder in $@"; grep -n '@[A-Z_]*@' $@; rm -f $@; exit 1; } || true +# Regenerated on every run and rewritten only when its content changes, so a +# command-line tag or clock override reaches the linker without invalidating an +# unchanged build. +$(BUILD_DIR)/config.cfg: FORCE + $(PLATFORM_GEN) --emit=cfg --mem-tag='$(MEM_TAG)' --host-tag='$(HOST_TAG)' --kernel-mhz=$(KERNEL_FREQ) -o $@ + +FORCE: package-ip: $(IP_DIR)/component.xml $(IP_DIR)/component.xml: $(BUILD_DIR)/sources.txt diff --git a/hw/syn/xilinx/aved/config.cfg.tmpl b/hw/syn/xilinx/aved/config.cfg.tmpl deleted file mode 100644 index f401f4702b..0000000000 --- a/hw/syn/xilinx/aved/config.cfg.tmpl +++ /dev/null @@ -1,15 +0,0 @@ -[connectivity] -# Bind to the compute-only static shell. This is slashkit's current default, -# but it selects which abstract shell the AFU links against, so state it -# rather than inherit it. The service shell adds the DCMAC ethernet stack the -# AFU has no use for. -shell=compute -nk=vortex_afu:1:vortex_afu_0 -sp=vortex_afu_0.m_axi_mem_0:@MEM_TAG@ -sp=vortex_afu_0.m_axi_host:@HOST_TAG@ - -# Kernel clock. Stated in Hz here rather than via the linker's --clock-hz -# flag, whose name and help text disagree on the unit. -[clock] -krnl=vortex_afu_0 -freqhz=@KERNEL_FREQ_HZ@ diff --git a/hw/syn/xilinx/aved/platforms.mk b/hw/syn/xilinx/aved/platforms.mk deleted file mode 100644 index 64029f51f0..0000000000 --- a/hw/syn/xilinx/aved/platforms.mk +++ /dev/null @@ -1,74 +0,0 @@ -# Platform specific configurations for the AMD Alveo V80. -# -# `override` so a command-line `make CONFIGS=...` cannot drop the platform -# settings; without it the AFU is packaged for the wrong memory topology. - -override CONFIGS += -DPLATFORM_MEMORY_DATA_WIDTH=512 - -# The V80 exposes its HBM stacks through the linker's HBM connectivity tags. -# A single wide master fanned across the channels avoids the per-bank base -# address problem: the linker assigns each memory range independently, and -# those bases are not known at synthesis time. One memory channel caps the -# AFU at one memory master whatever the GPU's memory port count. -override CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -override CONFIGS += -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 - -# Device-memory aperture base. The V80 maps the AFU's m_axi_mem_0 port at -# 0x40_0000_0000 in both build flavours -- simulation puts the BRAM model there -# (vbin.prj/run_pre.tcl) and hardware puts HBM_AXI_00 there (slash's generated -# address map). Vortex's device allocator is based at VX_MEM_USER_BASE_ADDR -# (0x10000) and a 32-bit core cannot emit the aperture base itself, so the AFU -# wrapper rebases every device access -- from the cores and from the CP -- by -# this synthesis-time offset. Without it nothing decodes: writes still collect -# a BRESP from the interconnect while the data goes nowhere. -# -# Sized Verilog literal, NOT decimal. A decimal constant this large is silently -# truncated: "WARNING: [VRFC 10-8884] decimal constant 274877906944 should be -# smaller than 2147483648; using 0 instead". -# -# Tied to MEM_TAG=HBM0; a different tag lands on a different aperture base. -override CONFIGS += -DPLATFORM_MEMORY_OFFSET=40\'h4000000000 - -# Device-memory connectivity. NOT HBM0: that tag is a single HBM_AXI channel, -# which the compute shell maps as 1 GB at 0x40_0000_0000. Vortex's 32-bit memory -# map puts the stack at the top of a 4 GB space -- vx_start.S sets sp to -# VX_MEM_STACK_BASE_ADDR (0xFFFF0000) and grows down, and that lands just below -# the local-memory window (VX_lsu_block.sv decodes LMEM *upward* from the same -# base), so it is a global-memory access. Rebased it becomes 0x40_FFFE_Fxxx, -# which a 1 GB aperture cannot answer: the read gets no response, the LSU stalls -# forever, and the AFU never goes idle -- taking the shell's AXI path with it. -# -# The MEM tag routes through the HBM VNOC, which the shell maps at the same -# 0x40_0000_0000 base with 32 GB of range, so it covers the whole 32-bit space -# and PLATFORM_MEMORY_OFFSET above stays correct unchanged. -# -# Set before the Makefile's `MEM_TAG ?= HBM0`, which is why plain `=` suffices. -MEM_TAG = MEM - -# Host-memory connectivity for the CP's command-ring master. NOT HOST: that tag -# routes m_axi_host to the QDMA slave bridge, and on this compute shell a read -# through that bridge never returns a response at all. The CP's very first ring -# fetch then hangs with no error to show for it -- the queue reads back armed -# and correct (Q_CONTROL=0x1, Q_TAIL=0x40, Q_ERROR=0) while head stays at 0 -# forever, and the next process's reset is refused because that read is still -# genuinely outstanding. -# -# HBM1 routes the port to a second HBM aperture instead, and the runtime stages -# the ring there and publishes it on the doorbell. That path works. -# -# This default was carried on the command line for one build and lost on the -# next, which cost a bitstream and several hours of hardware debugging. It -# belongs here next to MEM_TAG, for the same reason and by the same mechanism: -# stated before the Makefile's `HOST_TAG ?= HOST`, so plain `=` suffices. -# The aperture must lie OUTSIDE the device-memory window. Each HBM_AXI port -# addresses its own 512 MB slice of the stack, HBM at 0x40_0000_0000 + -# k*512 MB, and the MEM tag above exposes Vortex's whole 32-bit space from that -# same base -- so HBM0..HBM7 ARE device memory. With HBM1, VRT placed the CP's -# staging buffers at +512 MB, inside the heap: any scene whose buffers grew past -# 512 MB was overwritten by its own upload (wrong images; corrupted BVHs that -# traversal never finished). HBM8 starts at +4 GB, past the last device byte. -HOST_TAG = HBM8 - -# Kernel clock target (MHz). The linker also accepts a frequency request; -# the runtime can retune within the platform's supported range. -KERNEL_FREQ ?= 300 diff --git a/hw/syn/xilinx/platform.mk b/hw/syn/xilinx/platform.mk new file mode 100644 index 0000000000..a2434a3056 --- /dev/null +++ b/hw/syn/xilinx/platform.mk @@ -0,0 +1,19 @@ +# Card facts for the Xilinx flows, resolved from hw/syn/xilinx/platforms/.toml. +# +# The including flow sets PLATFORM_GEN_ARGS (how it names the card) and +# BUILD_DIR before including this. $(shell) discards exit codes, so the +# generator's status is captured explicitly: a descriptor error must stop the +# build here, not surface later as a design built for the wrong card. + +PLATFORM_MK := $(BUILD_DIR)/platform.mk +PLATFORM_GEN := python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py $(PLATFORM_GEN_ARGS) + +PLATFORM_GEN_RC := $(shell mkdir -p $(BUILD_DIR) && $(PLATFORM_GEN) --emit=mk -o $(PLATFORM_MK) 2> $(PLATFORM_MK).err; echo $$?) +ifneq ($(PLATFORM_GEN_RC),0) +$(error $(shell cat $(PLATFORM_MK).err)) +endif +include $(PLATFORM_MK) + +ifneq ($(filter $(HOST_TAG),$(PLATFORM_HOST_REFUSE)),) +$(error HOST_TAG=$(HOST_TAG) is refused on card $(PLATFORM_CARD): $(PLATFORM_HOST_REFUSE_WHY)) +endif diff --git a/hw/syn/xilinx/platforms/alveo.toml b/hw/syn/xilinx/platforms/alveo.toml new file mode 100644 index 0000000000..0b7732f939 --- /dev/null +++ b/hw/syn/xilinx/platforms/alveo.toml @@ -0,0 +1,15 @@ +# Any other Alveo XDMA platform. +[card] +name = "alveo" +flow = "xrt" +match_xsa = ["*"] + +[memory] +banks = 1 +addr_width = 32 + +[host] +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u200.toml b/hw/syn/xilinx/platforms/u200.toml new file mode 100644 index 0000000000..0936cf87e0 --- /dev/null +++ b/hw/syn/xilinx/platforms/u200.toml @@ -0,0 +1,18 @@ +# AMD Alveo U200: 64 GB DDR4 in four banks of 16 GB. +[card] +name = "u200" +flow = "xrt" +family = "virtexuplus" +part = "xcu200-fsgd2104-2-e" +match_xsa = ["xilinx_u200_*"] + +[memory] +banks = 4 +addr_width = 36 +tag = "DDR" + +[host] +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u250.toml b/hw/syn/xilinx/platforms/u250.toml new file mode 100644 index 0000000000..10fb0715f9 --- /dev/null +++ b/hw/syn/xilinx/platforms/u250.toml @@ -0,0 +1,20 @@ +# AMD Alveo U250: 64 GB DDR4 in four banks. Only bank 0 is bound: multi-bank +# needs per-bank device-address offsets that are not known at synthesis time. +[card] +name = "u250" +flow = "xrt" +family = "virtexuplus" +part = "xcu250-figd2104-2L-e" +match_xsa = ["xilinx_u250_*"] + +[memory] +banks = 1 +addr_width = 34 +tag = "DDR" +base = 0x4000000000 + +[host] +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u280.toml b/hw/syn/xilinx/platforms/u280.toml new file mode 100644 index 0000000000..6d1807d60c --- /dev/null +++ b/hw/syn/xilinx/platforms/u280.toml @@ -0,0 +1,19 @@ +# AMD Alveo U280: 8 GB HBM2, 32 pseudo-channels of 256 MB (plus DDR4, unused). +[card] +name = "u280" +flow = "xrt" +family = "virtexuplusHBM" +part = "xcu280-fsvh2892-2L-e" +match_xsa = ["xilinx_u280_*"] + +[memory] +banks = 32 +addr_width = 33 +# The AFU memory masters split the channels into contiguous groups. +tag = "HBM" + +[host] +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u50.toml b/hw/syn/xilinx/platforms/u50.toml new file mode 100644 index 0000000000..1b2a2b2581 --- /dev/null +++ b/hw/syn/xilinx/platforms/u50.toml @@ -0,0 +1,19 @@ +# AMD Alveo U50: 8 GB HBM2, 32 pseudo-channels of 256 MB. +[card] +name = "u50" +flow = "xrt" +family = "virtexuplusHBM" +part = "xcu50-fsvh2104-2-e" +match_xsa = ["xilinx_u50_*"] + +[memory] +banks = 32 +addr_width = 33 +# The AFU memory masters split the channels into contiguous groups. +tag = "HBM" + +[host] +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u55c.toml b/hw/syn/xilinx/platforms/u55c.toml new file mode 100644 index 0000000000..0578213bd6 --- /dev/null +++ b/hw/syn/xilinx/platforms/u55c.toml @@ -0,0 +1,26 @@ +# AMD Alveo U55C: 16 GB HBM2, 32 pseudo-channels of 512 MB. +[card] +name = "u55c" +flow = "xrt" +family = "virtexuplusHBM" +part = "xcu55c-fsvh2892-2L-e" +match_xsa = ["xilinx_u55c_*"] + +[memory] +banks = 32 +addr_width = 34 +# The AFU memory masters split the channels into contiguous groups. +tag = "HBM" + +[host] +# The CP's m_axi_host reaches host DRAM through the XDMA slave bridge. +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 + +[firesim] +platform = "vitis" +device = "xilinx_u55c_gen3x16_xdma_3_202210_1" +# 32-bit target spans 4 GiB: eight 512 MB pseudo-channels. +host_mem_sp = "HBM[0:7]" diff --git a/hw/syn/xilinx/platforms/v80.toml b/hw/syn/xilinx/platforms/v80.toml new file mode 100644 index 0000000000..ae66e2c1fc --- /dev/null +++ b/hw/syn/xilinx/platforms/v80.toml @@ -0,0 +1,44 @@ +# AMD Alveo V80 through the SLASH compute shell. The authoritative rationale for +# each value is in docs/designs/aved_driver_architecture.md (§4, §9). +[card] +name = "v80" +flow = "aved" +family = "versal" +part = "xcv80-lsva4737-2MHP-e-S" +match_device = ["xcv80-*"] + +[memory] +# One memory channel caps the AFU at one memory master whatever the GPU's +# memory port count: the linker assigns each memory range independently, and +# per-bank bases are not known at synthesis time. +banks = 1 +addr_width = 34 +# The shell maps m_axi_mem_0 at 0x40_0000_0000 in hardware and simulation, and a +# 32-bit core cannot emit that base, so the AFU rebases every device access. +base = 0x4000000000 +# MEM, not HBM0: HBM0 is a 1 GB aperture and the stack sits just under 4 GB. MEM +# routes through the HBM VNOC at the same base with 32 GB of range. +tag = "MEM" + +[host] +# HBM8 with runtime staging. HOST is the QDMA slave bridge, whose reads never +# complete on the compute shell: the CP hangs on its first ring fetch silently. +# The aperture must lie outside the device-memory window: HBM sits at +# 0x40_0000_0000 + k*512 MB and MEM exposes the whole 32-bit space from that +# base, so HBM0..HBM7 are device memory. With HBM1 the CP's staging buffers +# landed inside the heap and uploads past 512 MB overwrote it; HBM8 starts at +# +4 GB, past the last device byte. +tag = "HBM8" +refuse = ["HOST"] +refuse_why = "m_axi_host through the QDMA slave bridge never completes reads on the compute shell; the CP would hang on its first ring fetch" + +[control] +# A 16-byte AXI-Lite block answers only if all four of its words decode. +axil_block_bytes = 16 + +[clock] +kernel_mhz = 300 + +[firesim] +platform = "aved" +host_mem_tag = "MEM" diff --git a/hw/syn/xilinx/platforms/vck5000.toml b/hw/syn/xilinx/platforms/vck5000.toml new file mode 100644 index 0000000000..3983523988 --- /dev/null +++ b/hw/syn/xilinx/platforms/vck5000.toml @@ -0,0 +1,15 @@ +# AMD VCK5000 (Versal AI Core). Device memory sits at 0xC0_0000_0000. +[card] +name = "vck5000" +flow = "xrt" +family = "versal" +part = "xcvc1902-vsvd1760-2MP-e-S" +match_xsa = ["xilinx_vck5000_*"] + +[memory] +banks = 1 +addr_width = 32 +base = 0xC000000000 + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/versal.toml b/hw/syn/xilinx/platforms/versal.toml new file mode 100644 index 0000000000..f3a14d7077 --- /dev/null +++ b/hw/syn/xilinx/platforms/versal.toml @@ -0,0 +1,13 @@ +# Any other Versal XRT platform. +[card] +name = "versal" +flow = "xrt" +family = "versal" +match_family = ["versal"] + +[memory] +banks = 1 +addr_width = 32 + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/zynquplus.toml b/hw/syn/xilinx/platforms/zynquplus.toml new file mode 100644 index 0000000000..2e31d6739f --- /dev/null +++ b/hw/syn/xilinx/platforms/zynquplus.toml @@ -0,0 +1,13 @@ +# Zynq UltraScale+ XRT platforms (embedded; no PCIe host). +[card] +name = "zynquplus" +flow = "xrt" +family = "zynquplus" +match_family = ["zynquplus"] + +[memory] +banks = 1 +addr_width = 32 + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/xrt/Makefile b/hw/syn/xilinx/xrt/Makefile index a1ec0c00b2..1ef50465af 100644 --- a/hw/syn/xilinx/xrt/Makefile +++ b/hw/syn/xilinx/xrt/Makefile @@ -117,10 +117,9 @@ RTL_INCLUDE += -I$(RTL_DIR)/cp # single source of truth, appended based on the resolved XCONFIGS. include $(VORTEX_HOME)/hw/syn/extensions.mk -# Kernel clock target frequency (MHz). Defaults to the U55C platform native -# 300 MHz. Override on the command line (e.g. KERNEL_FREQ=250) for extra +# Kernel clock target frequency (MHz). The card descriptor supplies the +# default; override on the command line (e.g. KERNEL_FREQ=250) for extra # timing-closure margin. -KERNEL_FREQ ?= 300 # Timing-miss policy. Default (1): a design that misses KERNEL_FREQ is # auto-scaled down to the achieved frequency and the build succeeds with @@ -146,8 +145,11 @@ ifdef IMPL_STRATEGY VPP_FLAGS += --xp "vivado_prop:run.impl_1.STRATEGY=$(IMPL_STRATEGY)" endif -# load platform settings -include $(SRC_DIR)/platforms.mk +# Card settings, resolved from the platform's XSA and FPGA family. +ifneq ($(PLATFORM),) +PLATFORM_GEN_ARGS := --flow=xrt --xsa=$(XSA) --family='$(DEV_ARCH)' +include $(VORTEX_HOME)/hw/syn/xilinx/platform.mk +endif VPP_FLAGS += --report_level 2 VPP_FLAGS += --config $(SRC_DIR)/vitis.ini diff --git a/hw/syn/xilinx/xrt/platforms.mk b/hw/syn/xilinx/xrt/platforms.mk deleted file mode 100644 index f50397a122..0000000000 --- a/hw/syn/xilinx/xrt/platforms.mk +++ /dev/null @@ -1,53 +0,0 @@ -# Platform specific configurations -# Add your platform specific configurations here - -CONFIGS += -DPLATFORM_MEMORY_DATA_WIDTH=512 - -ifeq ($(DEV_ARCH), zynquplus) -# zynquplus -CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=32 -else ifeq ($(DEV_ARCH), versal) -# versal -CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=32 -ifneq ($(findstring xilinx_vck5000,$(XSA)),) - CONFIGS += -DPLATFORM_MEMORY_OFFSET=40'hC000000000 -endif -else -# alveo -# The Command Processor's host-memory master (m_axi_host) reaches host DRAM -# through the platform slave-bridge / Host Memory Access aperture. All Alveo -# XDMA shells expose this as the HOST[0] connectivity tag. -VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_host:HOST[0] -# PLATFORM_MEMORY_TAG/CHANNELS name the device-memory channels the AFU's -# memory masters bind to; the Makefile gives each master its own group. -ifneq ($(findstring xilinx_u55c,$(XSA)),) - # 16 GB of HBM2 with 32 channels (512 MB per channel) - PLATFORM_MEMORY_TAG := HBM - PLATFORM_MEMORY_CHANNELS := 32 - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 -else ifneq ($(findstring xilinx_u50,$(XSA)),) - # 8 GB of HBM2 with 32 channels (256 MB per channel) - PLATFORM_MEMORY_TAG := HBM - PLATFORM_MEMORY_CHANNELS := 32 - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 -else ifneq ($(findstring xilinx_u280,$(XSA)),) - # 8 GB of HBM2 with 32 channels (256 MB per channel) - PLATFORM_MEMORY_TAG := HBM - PLATFORM_MEMORY_CHANNELS := 32 - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 -else ifneq ($(findstring xilinx_u250,$(XSA)),) - # 16 GB of DDR4 (single channel, bank 0). Multi-bank requires per-bank XRT - # VA offsets that aren't known at synthesis time. - PLATFORM_MEMORY_TAG := DDR - PLATFORM_MEMORY_CHANNELS := 1 - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 - CONFIGS += -DPLATFORM_MEMORY_OFFSET=40\'h4000000000 -else ifneq ($(findstring xilinx_u200,$(XSA)),) - # 64 GB of DDR4 with 4 channels (16 GB per channel) - PLATFORM_MEMORY_TAG := DDR - PLATFORM_MEMORY_CHANNELS := 4 - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=36 -else - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=32 -endif -endif From fd4477c65e3b12743e0df278dd54ab36e53c83ad Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 21 Sep 2026 17:20:29 -0700 Subject: [PATCH 144/336] hw/syn/xilinx: share the flow core; fix card defects the descriptors exposed The xrt and aved Makefiles duplicated ~100 lines -- core-count presets, debug flag lists, XCONFIGS, RTL packages and includes, CP sources, extension wiring, compile flags and the gen-sources/gen-ast/scope-json rules. They now share hw/syn/xilinx/common.mk and keep only what differs: the linker and its targets. Behaviour changes, each visible in a before/after flag dump: - A command-line CONFIGS=... no longer discards the xrt flow's -DSYNTHESIS -DVIVADO and card settings; both flows append with `override`, as aved already did for this reason. - U280: one merged memory port fanned across HBM[0:31], as on the U55C and U50. It exposed 32 ports and bound only port 0. - An XSA no descriptor matches is an error listing the known cards, instead of a silent 1-bank, 32-bit default. That default is how the U200 came to have no memory binding. - The AXI-Lite block check runs in any flow whose card declares block decoding, not only in aved; the debug trace list is the union of both. The U200 descriptor stays as it was, marked incomplete: its fix (one merged port on DDR[0] with that bank's base) needs the platform's memory map, which is not installed here to confirm the base. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 5e99db8b90c43af364a4910764b5c90f42b3595d) Integration (phase 2): the shared flow core keeps phase 2's one AXI master per GPU memory port. Each flow resolves AFU_MEM_PORTS once its CFLAGS are complete (aved still requires one master, xrt binds each master to its channel group), and xilinx/common.mk regenerates the sources when the port count changes. The U280 keeps phase 2's per-port HBM groups rather than one merged port, and the U200 note now names only the unconfirmed DDR base. Co-Authored-By: Claude Opus 5.5 --- hw/scripts/gen_platform.py | 3 +- hw/syn/xilinx/aved/Makefile | 125 ++-------------------------- hw/syn/xilinx/common.mk | 126 +++++++++++++++++++++++++++++ hw/syn/xilinx/platforms/alveo.toml | 15 ---- hw/syn/xilinx/platforms/u200.toml | 4 + hw/syn/xilinx/xrt/Makefile | 118 ++------------------------- 6 files changed, 144 insertions(+), 247 deletions(-) create mode 100644 hw/syn/xilinx/common.mk delete mode 100644 hw/syn/xilinx/platforms/alveo.toml diff --git a/hw/scripts/gen_platform.py b/hw/scripts/gen_platform.py index 23f3376c17..a9898b7c2f 100644 --- a/hw/scripts/gen_platform.py +++ b/hw/scripts/gen_platform.py @@ -176,7 +176,8 @@ def matches(c: Card) -> bool: def emit_mk(c: Card) -> str: - ov = "override " if c.flow == "aved" else "" + # override: a command-line CONFIGS=... must not drop the card settings. + ov = "override " inst = KERNEL_INSTANCE[c.flow] lines = [f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", f"PLATFORM_CARD := {c.name}"] diff --git a/hw/syn/xilinx/aved/Makefile b/hw/syn/xilinx/aved/Makefile index 03ba252f52..1dae65a0fc 100644 --- a/hw/syn/xilinx/aved/Makefile +++ b/hw/syn/xilinx/aved/Makefile @@ -19,29 +19,14 @@ TARGET ?= hw PREFIX ?= build$(XLEN) -MAX_JOBS ?= 8 - # V80 device. Override for a different speed grade or package. DEVICE ?= xcv80-lsva4737-2MHP-e-S SRC_DIR := $(VORTEX_HOME)/hw/syn/xilinx/aved +AFU_DIR := $(VORTEX_HOME)/hw/rtl/afu/aved -RTL_DIR := $(VORTEX_HOME)/hw/rtl -DPI_DIR := $(VORTEX_HOME)/hw/dpi -AFU_DIR := $(RTL_DIR)/afu/aved -AFU_COMMON_DIR := $(RTL_DIR)/afu/common -SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts - -VIVADO := $(XILINX_VIVADO)/bin/vivado SLASHKIT ?= slashkit -CP = cp -rf -RMDIR = rm -rf -ECHO = @echo - -NCPUS := $(shell lscpu | grep "^Core(s) per socket:" | awk '{print $$4}') -JOBS ?= $(shell echo $$(( $(NCPUS) > $(MAX_JOBS) ? $(MAX_JOBS) : $(NCPUS) ))) - BUILD_DIR := $(PREFIX)_aved_$(TARGET) BIN_DIR := $(BUILD_DIR)/bin @@ -60,60 +45,12 @@ ifneq ($(TARGET),$(findstring $(TARGET), hw sim)) $(error TARGET must be hw or sim (got '$(TARGET)')) endif -# Control RTL debug tracing states -DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE -DBG_TRACE_FLAGS += -DDBG_TRACE_MEM -DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE -DBG_TRACE_FLAGS += -DDBG_TRACE_AFU -DBG_TRACE_FLAGS += -DDBG_TRACE_TCU - -# Control logic analyzer monitors -DBG_SCOPE_FLAGS += -DDBG_SCOPE_AFU -DBG_SCOPE_FLAGS += -DDBG_SCOPE_ISSUE -DBG_SCOPE_FLAGS += -DDBG_SCOPE_FETCH -DBG_SCOPE_FLAGS += -DDBG_SCOPE_LSU - -ifdef NUM_CORES -# cluster configuration -CONFIGS_1c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=1 -CONFIGS_2c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=2 -CONFIGS_4c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=4 -CONFIGS_8c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=8 -CONFIGS_16c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=16 -CONFIGS_32c := -DVX_CFG_NUM_CLUSTERS=2 -DVX_CFG_NUM_CORES=16 -CONFIGS_64c := -DVX_CFG_NUM_CLUSTERS=4 -DVX_CFG_NUM_CORES=16 -override CONFIGS += $(CONFIGS_$(NUM_CORES)c) -endif - -# `override`, as in the generated card settings: a command-line `make CONFIGS=...` otherwise -# discards these appends, and a build silently missing -DSYNTHESIS elaborates -# the simulation-only paths. -override CONFIGS += -DSYNTHESIS -DVIVADO - ifeq ($(TARGET), sim) override CONFIGS += -DSIMULATION endif -XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') - -# include sources -RTL_PKGS = $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv -FPU_INCLUDE = -I$(RTL_DIR)/fpu -ifneq (,$(filter -DVX_CFG_FPU_TYPE_FPNEW, $(XCONFIGS))) - RTL_PKGS += $(THIRD_PARTY_DIR)/cvfpu/src/fpnew_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src/cf_math_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl/defs_div_sqrt_mvp.sv - FPU_INCLUDE += -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/include -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src -J$(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl -J$(THIRD_PARTY_DIR)/cvfpu/src -endif -RTL_INCLUDE = -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(AFU_DIR) -I$(AFU_COMMON_DIR) -RTL_INCLUDE += $(FPU_INCLUDE) - -# Command Processor sources. RTL_PKGS holds packages only; the CP -# interface files are auto-discovered via the -I path below. -RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv -RTL_INCLUDE += -I$(RTL_DIR)/cp - -# Per-extension RTL sources — single source of truth, appended based on -# the resolved XCONFIGS. -include $(VORTEX_HOME)/hw/syn/extensions.mk +.DEFAULT_GOAL := all +include $(VORTEX_HOME)/hw/syn/xilinx/common.mk # Card settings (memory geometry, connectivity tags, kernel clock), resolved # from the device part. A tag the card refuses -- on the V80, HOST, whose reads @@ -138,27 +75,7 @@ else CFLAGS += -DNDEBUG endif -# Enable scope analyzer -ifdef SCOPE - CFLAGS += -DSCOPE $(DBG_SCOPE_FLAGS) - SCOPE_JSON += $(BIN_DIR)/scope.json -endif - -# Enable perf counters -ifdef PERF - CFLAGS += -DPERF_ENABLE -endif - -# compilation flags -CFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) -# raw CONFIGS carries only an enum *value* (e.g. VX_CFG_TCU_TYPE=TFR); append the -# resolved type selectors so an enum override selects its RTL implementation -# (without this, VX_tcu_fedp elaborates 'no_implementation_selected'). -CFLAGS += $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) -CFLAGS += $(RTL_INCLUDE) - -# ast dump flags -XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI +CFLAGS += $(COMPILE_CFLAGS) # One AXI memory master per GPU memory port. The kernel top must declare its # ports literally, so resolve the count from the elaborated configuration. @@ -172,42 +89,10 @@ $(error the V80 link binds one device-memory master, but this configuration has endif CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) -.PHONY: all clean gen-sources gen-ast scope-json package-ip gen-bin check-axil-blocks FORCE - -# Fail before synthesis if any 16-byte block in the CP AXI-Lite regfile is only -# partially decoded. Measured on V80 silicon: such a block DECERRs on EVERY -# word, including the words that ARE implemented -- the shell -> s_axi_control -# path resolves at 128-bit granularity and rejects a block it cannot fully -# resolve, so the per-word decode never gets consulted. No simulator reproduces -# it; the symptom is a register reading 0xFFFFFFFF on hardware for no reason -# visible in the RTL. Finding that cost ten days. A synthesis run costs ~37 -# minutes, so this check pays for itself the first time it fires. -check-axil-blocks: - @python3 $(SCRIPT_DIR)/check_axil_blocks.py $(RTL_DIR)/cp/VX_cp_axil_regfile.sv +.PHONY: all clean package-ip gen-bin FORCE all: check-axil-blocks $(VBIN_CONTAINER) -gen-sources: $(BUILD_DIR)/sources.txt -$(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt - -# The preprocessed sources bake in the kernel's port count: regenerate them -# whenever the configuration changes it. -$(BUILD_DIR)/afu_mem_ports.stamp: force-afu-mem-ports - @mkdir -p $(BUILD_DIR) - @if [ "$$(cat $@ 2>/dev/null)" != "$(AFU_MEM_PORTS)" ]; then echo "$(AFU_MEM_PORTS)" > $@; fi - -.PHONY: force-afu-mem-ports -force-afu-mem-ports: - -gen-ast: $(BUILD_DIR)/vortex.xml -$(BUILD_DIR)/vortex.xml: - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml - -scope-json: $(BIN_DIR)/scope.json -$(BIN_DIR)/scope.json: $(BUILD_DIR)/vortex.xml - mkdir -p $(BIN_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/scope.py vortex.xml -o bin/scope.json - # Regenerated on every run and rewritten only when its content changes, so a # command-line tag or clock override reaches the linker without invalidating an # unchanged build. diff --git a/hw/syn/xilinx/common.mk b/hw/syn/xilinx/common.mk new file mode 100644 index 0000000000..66cf4806e2 --- /dev/null +++ b/hw/syn/xilinx/common.mk @@ -0,0 +1,126 @@ +# Shared by the Xilinx flows (xrt, aved): configuration, RTL sources and the +# source-generation rules. The including flow sets BUILD_DIR, BIN_DIR, AFU_DIR +# and any TARGET-specific CONFIGS before including this, and includes +# platform.mk after it. + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi +AFU_COMMON_DIR := $(RTL_DIR)/afu/common +SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts + +VIVADO := $(XILINX_VIVADO)/bin/vivado + +CP = cp -rf +RMDIR = rm -rf +ECHO = @echo + +MAX_JOBS ?= 8 +NCPUS := $(shell lscpu | grep "^Core(s) per socket:" | awk '{print $$4}') +JOBS ?= $(shell echo $$(( $(NCPUS) > $(MAX_JOBS) ? $(MAX_JOBS) : $(NCPUS) ))) + +# Control RTL debug tracing states +DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE +DBG_TRACE_FLAGS += -DDBG_TRACE_MEM +DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE +DBG_TRACE_FLAGS += -DDBG_TRACE_AFU +DBG_TRACE_FLAGS += -DDBG_TRACE_TEX +DBG_TRACE_FLAGS += -DDBG_TRACE_RASTER +DBG_TRACE_FLAGS += -DDBG_TRACE_OM +DBG_TRACE_FLAGS += -DDBG_TRACE_GBAR +DBG_TRACE_FLAGS += -DDBG_TRACE_TCU + +# Control logic analyzer monitors +DBG_SCOPE_FLAGS += -DDBG_SCOPE_AFU +DBG_SCOPE_FLAGS += -DDBG_SCOPE_ISSUE +DBG_SCOPE_FLAGS += -DDBG_SCOPE_FETCH +DBG_SCOPE_FLAGS += -DDBG_SCOPE_LSU + +ifdef NUM_CORES +# cluster configuration +CONFIGS_1c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=1 +CONFIGS_2c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=2 +CONFIGS_4c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=4 +CONFIGS_8c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=8 +CONFIGS_16c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=16 +CONFIGS_32c := -DVX_CFG_NUM_CLUSTERS=2 -DVX_CFG_NUM_CORES=16 +CONFIGS_64c := -DVX_CFG_NUM_CLUSTERS=4 -DVX_CFG_NUM_CORES=16 +override CONFIGS += $(CONFIGS_$(NUM_CORES)c) +endif + +# `override` throughout: a command-line `make CONFIGS=...` otherwise discards +# these appends, and a build silently missing -DSYNTHESIS elaborates the +# simulation-only paths. +override CONFIGS += -DSYNTHESIS -DVIVADO + +XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') + +# include sources +RTL_PKGS = $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv +FPU_INCLUDE = -I$(RTL_DIR)/fpu +ifneq (,$(filter -DVX_CFG_FPU_TYPE_FPNEW, $(XCONFIGS))) + RTL_PKGS += $(THIRD_PARTY_DIR)/cvfpu/src/fpnew_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src/cf_math_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl/defs_div_sqrt_mvp.sv + FPU_INCLUDE += -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/include -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src -J$(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl -J$(THIRD_PARTY_DIR)/cvfpu/src +endif +RTL_INCLUDE = -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(AFU_DIR) -I$(AFU_COMMON_DIR) +RTL_INCLUDE += $(FPU_INCLUDE) + +# Command Processor sources. RTL_PKGS holds packages only; the CP +# interface files are auto-discovered via the -I path below. +RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv +RTL_INCLUDE += -I$(RTL_DIR)/cp + +# Per-extension RTL sources (TCU/DXA/V/RTU/TEX/RASTER/OM) — +# single source of truth, appended based on the resolved XCONFIGS. +include $(VORTEX_HOME)/hw/syn/extensions.mk + +# Enable perf counters +ifdef PERF + CFLAGS += -DPERF_ENABLE +endif + +# Enable scope analyzer +ifdef SCOPE + CFLAGS += -DSCOPE $(DBG_SCOPE_FLAGS) + SCOPE_JSON += $(BIN_DIR)/scope.json +endif + +# compilation flags (the flow adds its DEBUG/NDEBUG flags) +COMPILE_CFLAGS = -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) +# raw CONFIGS carries only an enum *value* (e.g. VX_CFG_TCU_TYPE=TFR); append the +# resolved type selectors so an enum override selects its RTL implementation +# (without this, VX_tcu_fedp elaborates 'no_implementation_selected'). +COMPILE_CFLAGS += $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) +COMPILE_CFLAGS += $(RTL_INCLUDE) + +# ast dump flags +XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI + +# Fail before synthesis if any 16-byte block in the CP AXI-Lite regfile is only +# partially decoded, on cards whose shell resolves AXI-Lite in blocks (the +# descriptor's axil_block_bytes). Measured on V80 silicon: such a block DECERRs +# on EVERY word, including the words that ARE implemented. No simulator +# reproduces it; the symptom is a register reading 0xFFFFFFFF on hardware for no +# reason visible in the RTL. +check-axil-blocks: + $(if $(filter-out 0,$(PLATFORM_AXIL_BLOCK_BYTES)),@python3 $(SCRIPT_DIR)/check_axil_blocks.py $(RTL_DIR)/cp/VX_cp_axil_regfile.sv) + +gen-sources: $(BUILD_DIR)/sources.txt +$(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp + mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt + +# The preprocessed sources bake in the kernel's memory port count (the flow's +# AFU_MEM_PORTS): regenerate them whenever the configuration changes it. +$(BUILD_DIR)/afu_mem_ports.stamp: force-afu-mem-ports + @mkdir -p $(BUILD_DIR) + @if [ "$$(cat $@ 2>/dev/null)" != "$(AFU_MEM_PORTS)" ]; then echo "$(AFU_MEM_PORTS)" > $@; fi + +gen-ast: $(BUILD_DIR)/vortex.xml +$(BUILD_DIR)/vortex.xml: + mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml + +scope-json: $(BIN_DIR)/scope.json +$(BIN_DIR)/scope.json: $(BUILD_DIR)/vortex.xml + mkdir -p $(BIN_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/scope.py vortex.xml -o bin/scope.json + +.PHONY: check-axil-blocks gen-sources gen-ast scope-json force-afu-mem-ports +force-afu-mem-ports: diff --git a/hw/syn/xilinx/platforms/alveo.toml b/hw/syn/xilinx/platforms/alveo.toml deleted file mode 100644 index 0b7732f939..0000000000 --- a/hw/syn/xilinx/platforms/alveo.toml +++ /dev/null @@ -1,15 +0,0 @@ -# Any other Alveo XDMA platform. -[card] -name = "alveo" -flow = "xrt" -match_xsa = ["*"] - -[memory] -banks = 1 -addr_width = 32 - -[host] -tag = "HOST[0]" - -[clock] -kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u200.toml b/hw/syn/xilinx/platforms/u200.toml index 0936cf87e0..be36aa2471 100644 --- a/hw/syn/xilinx/platforms/u200.toml +++ b/hw/syn/xilinx/platforms/u200.toml @@ -1,4 +1,8 @@ # AMD Alveo U200: 64 GB DDR4 in four banks of 16 GB. +# +# KNOWN INCOMPLETE: the DDR banks' device base (PLATFORM_MEMORY_OFFSET) is not +# set; confirming it needs the U200 platform's memory map (platforminfo / +# xclbinutil). [card] name = "u200" flow = "xrt" diff --git a/hw/syn/xilinx/xrt/Makefile b/hw/syn/xilinx/xrt/Makefile index 1ef50465af..9123d3a25e 100644 --- a/hw/syn/xilinx/xrt/Makefile +++ b/hw/syn/xilinx/xrt/Makefile @@ -23,7 +23,6 @@ PREFIX ?= build$(XLEN) SAIF_FILE ?= SAIF_INST ?= -MAX_JOBS ?= 8 # Synthesis optimization level (standardized across hw/syn): # 0 -- fastest compile, no optimization (Vitis --optimize 0) @@ -32,28 +31,14 @@ MAX_JOBS ?= 8 OPT_LEVEL ?= 3 SRC_DIR := $(VORTEX_HOME)/hw/syn/xilinx/xrt +AFU_DIR := $(VORTEX_HOME)/hw/rtl/afu/xrt -RTL_DIR := $(VORTEX_HOME)/hw/rtl -DPI_DIR := $(VORTEX_HOME)/hw/dpi -AFU_DIR := $(RTL_DIR)/afu/xrt -AFU_COMMON_DIR := $(RTL_DIR)/afu/common -SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts - -VIVADO := $(XILINX_VIVADO)/bin/vivado VPP := $(XILINX_VITIS)/bin/v++ -CP = cp -rf -RMDIR = rm -rf -ECHO = @echo - -NCPUS := $(shell lscpu | grep "^Core(s) per socket:" | awk '{print $$4}') -JOBS ?= $(shell echo $$(( $(NCPUS) > $(MAX_JOBS) ? $(MAX_JOBS) : $(NCPUS) ))) - PLATFORM_TO_XSA = $(strip $(patsubst %.xpfm, % , $(shell basename $(PLATFORM)))) XSA := $(call PLATFORM_TO_XSA, $(PLATFORM)) DEV_ARCH := $(shell platforminfo -p $(PLATFORM) | grep 'FPGA Family' | sed 's/.*://' | sed '/ai_engine/d' | sed 's/^[[:space:]]*//') -CPU_TYPE := $(shell platforminfo -p $(PLATFORM) | grep 'CPU Type' | sed 's/.*://' | sed '/ai_engine/d' | sed 's/^[[:space:]]*//') BUILD_DIR := $(PREFIX)_$(XSA)_$(TARGET) BIN_DIR := $(BUILD_DIR)/bin @@ -61,61 +46,12 @@ BIN_DIR := $(BUILD_DIR)/bin XO_CONTAINER := $(BIN_DIR)/vortex_afu.xo XCLBIN_CONTAINER := $(BIN_DIR)/vortex_afu.xclbin -# Control RTL debug tracing states -DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE -DBG_TRACE_FLAGS += -DDBG_TRACE_MEM -DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE -DBG_TRACE_FLAGS += -DDBG_TRACE_AFU -DBG_TRACE_FLAGS += -DDBG_TRACE_TEX -DBG_TRACE_FLAGS += -DDBG_TRACE_RASTER -DBG_TRACE_FLAGS += -DDBG_TRACE_OM -DBG_TRACE_FLAGS += -DDBG_TRACE_GBAR -DBG_TRACE_FLAGS += -DDBG_TRACE_TCU - -# Control logic analyzer monitors -DBG_SCOPE_FLAGS += -DDBG_SCOPE_AFU -DBG_SCOPE_FLAGS += -DDBG_SCOPE_ISSUE -DBG_SCOPE_FLAGS += -DDBG_SCOPE_FETCH -DBG_SCOPE_FLAGS += -DDBG_SCOPE_LSU - -ifdef NUM_CORES -# cluster configuration -CONFIGS_1c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=1 -CONFIGS_2c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=2 -CONFIGS_4c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=4 -CONFIGS_8c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=8 -CONFIGS_16c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=16 -CONFIGS_32c := -DVX_CFG_NUM_CLUSTERS=2 -DVX_CFG_NUM_CORES=16 -CONFIGS_64c := -DVX_CFG_NUM_CLUSTERS=4 -DVX_CFG_NUM_CORES=16 -CONFIGS += $(CONFIGS_$(NUM_CORES)c) -endif - -CONFIGS += -DSYNTHESIS -DVIVADO - ifeq ($(TARGET), hw_emu) - CONFIGS += -DSIMULATION + override CONFIGS += -DSIMULATION endif -XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') - -# include sources -RTL_PKGS = $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv -FPU_INCLUDE = -I$(RTL_DIR)/fpu -ifneq (,$(filter -DVX_CFG_FPU_TYPE_FPNEW, $(XCONFIGS))) - RTL_PKGS += $(THIRD_PARTY_DIR)/cvfpu/src/fpnew_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src/cf_math_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl/defs_div_sqrt_mvp.sv - FPU_INCLUDE += -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/include -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src -J$(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl -J$(THIRD_PARTY_DIR)/cvfpu/src -endif -RTL_INCLUDE = -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(AFU_DIR) -I$(AFU_COMMON_DIR) -RTL_INCLUDE += $(FPU_INCLUDE) - -# Command Processor sources. RTL_PKGS holds packages only; the CP -# interface files are auto-discovered via the -I path below. -RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv -RTL_INCLUDE += -I$(RTL_DIR)/cp - -# Per-extension RTL sources (TCU/DXA/V/RTU/TEX/RASTER/OM) — -# single source of truth, appended based on the resolved XCONFIGS. -include $(VORTEX_HOME)/hw/syn/extensions.mk +.DEFAULT_GOAL := all +include $(VORTEX_HOME)/hw/syn/xilinx/common.mk # Kernel clock target frequency (MHz). The card descriptor supplies the # default; override on the command line (e.g. KERNEL_FREQ=250) for extra @@ -154,11 +90,6 @@ endif VPP_FLAGS += --report_level 2 VPP_FLAGS += --config $(SRC_DIR)/vitis.ini -# Enable perf counters -ifdef PERF - CFLAGS += -DPERF_ENABLE -endif - # Generates profile summary report ifdef PROFILE VPP_FLAGS += --profile_kernel data:all:all:all @@ -180,21 +111,7 @@ else CFLAGS += -DNDEBUG endif -# Enable scope analyzer -ifdef SCOPE - CFLAGS += -DSCOPE $(DBG_SCOPE_FLAGS) - SCOPE_JSON += $(BIN_DIR)/scope.json -endif - -# compilation flags -CFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) -# raw CONFIGS carries only an enum *value* (e.g. VX_CFG_TCU_TYPE=TFR); append the -# resolved type selectors so an enum override selects its RTL implementation. -CFLAGS += $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) -CFLAGS += $(RTL_INCLUDE) - -# ast dump flags -XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI +CFLAGS += $(COMPILE_CFLAGS) # One AXI memory master per GPU memory port. The kernel top must declare its # ports literally, so resolve the count from the elaborated configuration. @@ -221,30 +138,9 @@ ifneq ($(TARGET),$(findstring $(TARGET), hw hw_emu)) $(warning WARNING:Application supports only hw hw_emu TARGET. Please use the target for running the application) endif -.PHONY: all clean gen-sources gen-ast emconfig check-devices power - -all: check-devices emconfig $(XCLBIN_CONTAINER) report - -gen-sources: $(BUILD_DIR)/sources.txt -$(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt - -# The preprocessed sources bake in the kernel's port count: regenerate them -# whenever the configuration changes it. -$(BUILD_DIR)/afu_mem_ports.stamp: force-afu-mem-ports - @mkdir -p $(BUILD_DIR) - @if [ "$$(cat $@ 2>/dev/null)" != "$(AFU_MEM_PORTS)" ]; then echo "$(AFU_MEM_PORTS)" > $@; fi - -.PHONY: force-afu-mem-ports -force-afu-mem-ports: - -gen-ast: $(BUILD_DIR)/vortex.xml -$(BUILD_DIR)/vortex.xml: - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml +.PHONY: all clean emconfig check-devices power -scope-json: $(BIN_DIR)/scope.json -$(BIN_DIR)/scope.json: $(BUILD_DIR)/vortex.xml - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/scope.py vortex.xml -o bin/scope.json +all: check-devices check-axil-blocks emconfig $(XCLBIN_CONTAINER) report gen-xo: $(XO_CONTAINER) $(XO_CONTAINER): $(BUILD_DIR)/sources.txt From b3bd1b7b1a511a0fc32376e64a614068a8ba773c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 21 Sep 2026 17:36:04 -0700 Subject: [PATCH 145/336] firesim: pin the fork and take the card from its descriptor FireSim's source of truth is now the vortexgpgpu/firesim fork, pinned by FIRESIM_REV in VERSION (substituted into config.mk like GEM5_REV). The patch series under hw/syn/firesim/patches is deleted: it had already drifted from the fork it claimed to reproduce byte-for-byte. `make stage` refuses a checkout that does not contain the pinned commit. The FireSim flow takes its card from the card descriptor (FIRESIM_CARD, default u55c): platform, part, device, host-memory binding and base. The base reaches FireSim as a generated per-card platform config class (U55CPlatformConfig = WithHostMemBase(...) ++ DefaultVitisConfig), because FireSim instantiates PLATFORM_CONFIG fragments by name and a name cannot carry a value. A card with no [firesim] section is refused. `make -C hw/syn/firesim bitstream` now runs the whole flow -- flatten, source list, vendor IP for the card's part, stage, Golden Gate, place-and-route -- instead of four manual steps in two trees. sim/firesim builds the transport for the same platform and config, linking XRT only for vitis cards. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit e7650339bfa58c37596c34a3ab0a07a0c76fb3b5) --- VERSION | 1 + config.mk.in | 2 + configure | 4 +- docs/building_toolchain.md | 10 +- docs/designs/firesim_integration.md | 68 +++++--- hw/scripts/gen_platform.py | 57 +++++-- hw/syn/firesim/Makefile | 66 +++++--- .../0001-rebase-target-cycle-counter.patch | 73 -------- .../patches/0002-vitis-u55c-build-fixes.patch | 68 -------- ...3-vitis-memcheck-and-buffer-lifetime.patch | 156 ----------------- .../0004-vitis-host-mem-base-address.patch | 66 -------- .../patches/0005-inflight-target-cycle.patch | 54 ------ .../patches/0006-package-design-sources.patch | 112 ------------- .../0007-vitis-target-dram-aperture.patch | 157 ------------------ .../0008-driver-teardown-before-exit.patch | 18 -- hw/syn/firesim/patches/README | 104 ------------ hw/syn/xilinx/platforms/v80.toml | 2 +- sim/firesim/Makefile | 24 ++- 18 files changed, 169 insertions(+), 873 deletions(-) delete mode 100644 hw/syn/firesim/patches/0001-rebase-target-cycle-counter.patch delete mode 100644 hw/syn/firesim/patches/0002-vitis-u55c-build-fixes.patch delete mode 100644 hw/syn/firesim/patches/0003-vitis-memcheck-and-buffer-lifetime.patch delete mode 100644 hw/syn/firesim/patches/0004-vitis-host-mem-base-address.patch delete mode 100644 hw/syn/firesim/patches/0005-inflight-target-cycle.patch delete mode 100644 hw/syn/firesim/patches/0006-package-design-sources.patch delete mode 100644 hw/syn/firesim/patches/0007-vitis-target-dram-aperture.patch delete mode 100644 hw/syn/firesim/patches/0008-driver-teardown-before-exit.patch delete mode 100644 hw/syn/firesim/patches/README diff --git a/VERSION b/VERSION index 60550bfcb5..177bead6ef 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 +FIRESIM_REV=4d4b0f9521313f3043114460c338b368b8506391 diff --git a/config.mk.in b/config.mk.in index 1dc8f4efbd..dad8cccf06 100644 --- a/config.mk.in +++ b/config.mk.in @@ -42,6 +42,8 @@ RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv$(XLEN)-gnu-toolchain # and a path derived from VORTEX_HOME would make the dependency's location a # function of which tree is being built. FIRESIM_PATH ?= $(TOOLDIR)/firesim +# vortexgpgpu/firesim commit this tree is built and tested against (VERSION). +FIRESIM_REV := @FIRESIM_REV@ RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf RISCV_SYSROOT ?= $(RISCV_TOOLCHAIN_PATH)/$(RISCV_PREFIX) diff --git a/configure b/configure index c5e1e21147..a585284055 100755 --- a/configure +++ b/configure @@ -77,7 +77,7 @@ copy_files() { continue fi mkdir -p "$dest_dir" - sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g; s|@PROFILE@|$PROFILE|g" "$file" > "$dest_file" + sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g; s|@PROFILE@|$PROFILE|g; s|@FIRESIM_REV@|$FIRESIM_REV|g" "$file" > "$dest_file" # apply permissions to bash scripts read -r firstline < "$dest_file" if [[ "$firstline" =~ ^#!.*bash ]]; then @@ -208,7 +208,7 @@ SOURCE_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) THIRD_PARTY_DIR=$SOURCE_DIR/third_party -# load version pins (VORTEX_VERSION, TOOLCHAIN_REV) +# load version pins (VORTEX_VERSION, TOOLCHAIN_REV, GEM5_REV, FIRESIM_REV) source "$SOURCE_DIR/VERSION" # stamp file tracks config parameter changes; touched only when params change diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index 9508651472..f699645215 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -899,10 +899,13 @@ it, and only an FPGA flow can use it. Install it by hand: ```bash git clone --branch vortex_3.x https://github.com/vortexgpgpu/firesim.git ~/dev/firesim -cd ~/dev/firesim && ./build-setup.sh --skip-validate +cd ~/dev/firesim && ./build-setup.sh # needs `conda` on PATH (e.g. Miniforge) ln -s ~/dev/firesim $TOOLDIR/firesim ``` +The checkout must contain the fork commit pinned as `FIRESIM_REV` in `VERSION`; +`make -C hw/syn/firesim stage` checks it. + `vortex_3.x` is based on upstream tag 1.21.0. Clone it wherever you keep sources; `$TOOLDIR/firesim` only has to resolve to the checkout. @@ -947,8 +950,9 @@ make -C hw/syn/firesim check-firesim host with `libgmp-dev` installed, building the transport works without activating conda at all. - **Vortex's changes to FireSim live on the branch, not as patches.** - `git diff 1.21.0..vortex_3.x` is the authoritative delta — 8 files, mostly - Vitis platform fixes and cycle-counter accounting, all upstreamable. + `git diff 1.21.0..$FIRESIM_REV` is the authoritative delta: Vitis platform + fixes, the host-memory base as a config field, driver build and teardown, + and cycle-counter accounting, all upstreamable. ## 13. SLASH / Alveo V80 (the `aved` backend) diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index a467c86948..d5abee0f2f 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -16,8 +16,8 @@ FireSim itself is an external dependency, developed in a clone of `vortex_3.x`, based on upstream 1.21.0) at `~/dev/firesim`, and consumed by the Vortex build through `$(FIRESIM_PATH)`, which `config.mk` defaults to `$(TOOLDIR)/firesim` — the same source/tool split POCL, Mesa and LLVM use. -Vortex's own changes to FireSim are small — 8 files, ~324 lines — and are bug -fixes and U55C platform plumbing rather than divergence; see §6. +Vortex's own changes to FireSim are small bug fixes and platform plumbing +rather than divergence, pinned by `FIRESIM_REV` in `VERSION`; see §6. For running kernels on hardware generally, see [`fpga_setup.md`](../fpga_setup.md); for the other simulation vehicles, see @@ -226,17 +226,35 @@ driver says so explicitly rather than leaving it to be inferred. ## 4. Build flow +One command builds a card's image, for the card named by `FIRESIM_CARD` +(default `u55c`), from a configured build tree: + +```bash +source $FIRESIM_PATH/env.sh # FireSim's conda environment +source /opt/xilinx/2024.2/Vitis/settings64.sh # plus XRT's setup.sh +make -C hw/syn/firesim bitstream FIRESIM_CARD=u55c \ + CONFIGS="-DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TEX_ENABLE" FREQUENCY=60.0 +``` + +`bitstream` runs, in order: + ``` -make -C hw/syn/firesim stage copy Chisel/C++ targets into $(FIRESIM_PATH) -make -C hw/syn/firesim sources Verilator source list (packages + wrapper) -make -C hw/syn/firesim vivado-sources - flatten Vortex with macros inlined (-P) -cd $(FIRESIM_PATH)/sim && make replace-rtl PLATFORM=vitis DESIGN=VortexTarget - Golden Gate elaboration -cd platforms/vitis/… && make bitstream DEVICE=… FREQUENCY=60.0 STRATEGY=TIMING - synthesis + place-and-route +vivado-sources flatten Vortex with macros inlined (-P) +sources Verilator source list (packages + wrapper) +xilinx-ips generate the Xilinx floating-point cores for the card's part +stage copy the Chisel/C++ targets and the card's platform config + class into $(FIRESIM_PATH) +replace-rtl Golden Gate elaboration (in $(FIRESIM_PATH)/sim) +bitstream synthesis + place-and-route (in the platform's cl_ dir) ``` +The card's FireSim platform, part, device, host-memory binding and base address +come from its descriptor in `hw/syn/xilinx/platforms/`; the platform config +class (`U55CPlatformConfig`) is generated from it. `CONFIGS` takes architecture +overrides as the other synthesis flows do; extensions pull in their sources +through `hw/syn/extensions.mk`, and a changed `CONFIGS` invalidates the +flattened tree. + ### 4.1 Two source lists, and why Verilator resolves Vortex by **library search**, so `sources` names only the @@ -322,20 +340,24 @@ For scale: `sgemm -n32` takes 1228 ms on rtlsim and 8 ms on the card, ~153×. ## 6. Relationship to upstream FireSim -The FireSim checkout is a **git clone, not a vendored copy**. Vortex's changes -live on branch `vortex_3.x` of `vortexgpgpu/firesim`, based on upstream tag -1.21.0, and amount to 8 files: +The FireSim checkout is a **git clone of the fork**, `vortexgpgpu/firesim` +branch `vortex_3.x`, based on upstream tag 1.21.0. Every change Vortex needs in +FireSim is a commit there, and `VERSION` pins the one this tree is built and +tested against (`FIRESIM_REV`). `make stage` refuses a checkout that does not +contain the pinned commit; being ahead of it is normal while developing the +fork. -| area | files | why | -|---|---|---| -| Vitis platform | `simif_vitis.cc`, `VitisShim.scala` | DRAM aperture across HBM[0:7] with a 4 GiB window; host-memory base address | -| driver lifetime | `entry.cc` | tear the platform down at the end of `entry()`; a global `unique_ptr` is destroyed during static destruction, after XRT's own statics, and segfaults in the device registry | -| cycle accounting | `simulation.{cc,h}` | `end_tcycle` is read absolute and never rebased, so it accumulates across runs and FMR divides a per-run quantity by a cumulative one | -| build | `cl_firesim/Makefile`, `gen_xo.tcl`, `package_kernel.tcl` | U55C build fixes; package the flattened Vortex sources into the kernel | - -All are candidates for upstreaming: only the `HOST_MEM_SP` default is -U55C-specific, and it is parameterized rather than hardcoded. The branch should -shrink over time as changes are accepted upstream. +The fork's changes are bug fixes and platform plumbing rather than divergence: + +| area | why | +|---|---| +| Vitis platform | host-memory base as a config field (`HostMemBaseAddressKey`), checked by the driver against the memory the binding resolved to; 4 GiB DRAM aperture; part read from the platform; vendor IP cores packaged with the kernel | +| driver lifetime | tear the platform down at the end of `entry()`; a global `unique_ptr` is destroyed during static destruction, after XRT's own statics, and segfaults in the device registry | +| driver build | the Vitis driver is compiled by the host toolchain: XRT needs the host glibc, and conda's start files do not link against glibc 2.34+ | +| cycle accounting | `end_tcycle` is read absolute and never rebased, so it accumulates across runs and FMR divides a per-run quantity by a cumulative one | +| build | Vitis 2024.2 fixes; package the flattened Vortex sources into the kernel | + +All are candidates for upstreaming. --- diff --git a/hw/scripts/gen_platform.py b/hw/scripts/gen_platform.py index a9898b7c2f..c0e06810cf 100644 --- a/hw/scripts/gen_platform.py +++ b/hw/scripts/gen_platform.py @@ -22,6 +22,7 @@ mk make fragment for hw/syn/xilinx/{xrt,aved}/Makefile cfg slashkit linker config (AVED flow) firesim-mk make fragment for hw/syn/firesim and sim/firesim + scala the card's FireSim platform config class card the resolved card name """ @@ -146,11 +147,11 @@ def load_all(platform_dir: str) -> List[Card]: def resolve(cards: List[Card], flow: str, xsa: str, family: str, device: str, name: str) -> Card: - pool = [c for c in cards if c.flow == flow] + pool = [c for c in cards if not flow or c.flow == flow] if name: hits = [c for c in pool if c.name == name] if not hits: - raise DescriptorError(f"no {flow} card named '{name}'; known: {sorted(c.name for c in pool)}") + raise DescriptorError(f"no {flow or 'Xilinx'} card named '{name}'; known: {sorted(c.name for c in pool)}") return hits[0] def matches(c: Card) -> bool: @@ -236,23 +237,52 @@ def emit_cfg(c: Card, mem_tag: str, host_tag: str, kernel_mhz: int) -> str: ]) -def emit_firesim_mk(c: Card) -> str: +# FireSim platform -> the midasexamples config every card on it builds on. +FIRESIM_BASE_CONFIG = {"vitis": "DefaultVitisConfig", "aved": "DefaultAvedConfig"} + + +def firesim_config_class(c: Card) -> str: + return f"{c.name.upper()}PlatformConfig" + + +def _firesim(c: Card) -> dict: fs = c.firesim if not fs: raise DescriptorError(f"{c.path}: no [firesim] section; card '{c.name}' has no FireSim platform") + platform = _require(fs, "platform", c.path, "firesim") + if platform not in FIRESIM_BASE_CONFIG: + raise DescriptorError(f"{c.path}: [firesim] platform must be one of {sorted(FIRESIM_BASE_CONFIG)}") + _require(fs, "host_mem_sp", c.path, "firesim") + return fs + + +def emit_firesim_mk(c: Card) -> str: + fs = _firesim(c) lines = [f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", f"FIRESIM_CARD := {c.name}", - f"PLATFORM := {_require(fs, 'platform', c.path, 'firesim')}", - f"FPGA_PART := {c.part}"] + f"PLATFORM := {fs['platform']}", + f"PLATFORM_CONFIG := {firesim_config_class(c)}", + f"FPGA_PART := {c.part}", + f"HOST_MEM_SP := {fs['host_mem_sp']}"] if "device" in fs: lines.append(f"DEVICE := {fs['device']}") - if "host_mem_sp" in fs: - lines.append(f"HOST_MEM_SP := {fs['host_mem_sp']}") - if "platform_config" in fs: - lines.append(f"PLATFORM_CONFIG := {fs['platform_config']}") return "\n".join(lines) + "\n" +def emit_scala(c: Card) -> str: + fs = _firesim(c) + return "\n".join([ + f"// Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", + "package firesim.midasexamples", + "", + "import org.chipsalliance.cde.config.Config", + "", + f"class {firesim_config_class(c)} extends Config(", + f" new midas.WithHostMemBase(BigInt(\"{c.base:x}\", 16)) ++ new {FIRESIM_BASE_CONFIG[fs['platform']]})", + "", + ]) + + def check_host_tag(c: Card, host_tag: str) -> None: if host_tag in c.host_refuse: raise DescriptorError(f"host tag '{host_tag}' is refused on card '{c.name}': {c.host_refuse_why}") @@ -260,12 +290,13 @@ def check_host_tag(c: Card, host_tag: str) -> None: def main(argv: Optional[List[str]] = None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--flow", required=True, choices=FLOWS) + ap.add_argument("--flow", default="", choices=("",) + FLOWS, + help="restrict matching to one flow; required unless --card is given") ap.add_argument("--xsa", default="", help="XRT platform name (xrt flow)") ap.add_argument("--family", default="", help="FPGA family reported by platforminfo (xrt flow)") ap.add_argument("--device", default="", help="device part (aved flow)") ap.add_argument("--card", default="", help="select a card by name instead of matching") - ap.add_argument("--emit", required=True, choices=("mk", "cfg", "firesim-mk", "card")) + ap.add_argument("--emit", required=True, choices=("mk", "cfg", "firesim-mk", "scala", "card")) ap.add_argument("--mem-tag", default="") ap.add_argument("--host-tag", default="") ap.add_argument("--kernel-mhz", type=int, default=0) @@ -273,6 +304,8 @@ def main(argv: Optional[List[str]] = None) -> int: ap.add_argument("-o", "--output", default="-") a = ap.parse_args(argv) + if not a.flow and not a.card: + ap.error("--flow is required unless --card names the card") try: cards = load_all(a.platform_dir) c = resolve(cards, a.flow, a.xsa, a.family.strip(), a.device, a.card) @@ -283,6 +316,8 @@ def main(argv: Optional[List[str]] = None) -> int: a.kernel_mhz or c.kernel_mhz) elif a.emit == "firesim-mk": out = emit_firesim_mk(c) + elif a.emit == "scala": + out = emit_scala(c) else: out = c.name + "\n" except DescriptorError as e: diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index da7a618da9..5a263d1d69 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -1,9 +1,13 @@ # FireSim platform backend. # -# The Chisel targets and the upstream patches live here, in the Vortex tree, so a -# fresh checkout can reproduce a build. The FireSim checkout itself is a working -# directory, not a source of truth: `stage` copies the targets into it and applies -# the patches, and anything left only in the checkout is expected to be lost. +# FireSim itself is the vortexgpgpu/firesim fork, consumed through FIRESIM_PATH +# and pinned by FIRESIM_REV in VERSION: every change Vortex needs in FireSim is a +# commit there, and the checkout must contain the pinned one. The Chisel targets, +# harnesses and the card's platform config live here and are staged into it. +# +# The card comes from its descriptor (hw/syn/xilinx/platforms/.toml): +# FIRESIM_CARD selects it, and the descriptor supplies the FireSim platform, the +# FPGA part and device, the host-memory binding and its base address. ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk @@ -13,7 +17,6 @@ include $(VORTEX_HOME)/hw/syn/common.mk # while sources are read from VORTEX_HOME. SRC_DIR := $(VORTEX_HOME)/hw/syn/firesim -PATCH_DIR := $(SRC_DIR)/patches # Targets are compiled as part of FireSim's midasexamples project, which is where # its harness and bridge helpers are resolvable from. @@ -30,6 +33,20 @@ SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts BUILD_DIR ?= $(ROOT_DIR)/hw/syn/firesim +FIRESIM_CARD ?= u55c +PLATFORM_GEN := python3 $(SCRIPT_DIR)/gen_platform.py --card=$(FIRESIM_CARD) +FIRESIM_MK := $(BUILD_DIR)/firesim_platform.mk +FIRESIM_GEN_RC := $(shell mkdir -p $(BUILD_DIR) && $(PLATFORM_GEN) --emit=firesim-mk -o $(FIRESIM_MK) 2> $(FIRESIM_MK).err; echo $$?) +ifneq ($(FIRESIM_GEN_RC),0) +$(error $(shell cat $(FIRESIM_MK).err)) +endif +include $(FIRESIM_MK) + +# The FireSim flow's make variables for this card's build. +FIRESIM_TARGET_ARGS := PLATFORM=$(PLATFORM) TARGET_PROJECT=midasexamples \ + DESIGN=VortexTarget TARGET_CONFIG=VortexConfig PLATFORM_CONFIG=$(PLATFORM_CONFIG) +QUINTUPLET := $(PLATFORM)-midasexamples-VortexTarget-VortexConfig-$(PLATFORM_CONFIG) + # Flattened Vortex sources for the BlackBox. Golden Gate takes a file list, not # include paths, so the tree is copied into one directory with the macros already # inlined (-P) and emitted in elaboration order: packages, interfaces, modules. @@ -39,7 +56,7 @@ VIVADO_SRCS_LIST := $(BUILD_DIR)/vivado_srcs.f SRCS_DIR := $(BUILD_DIR)/vortex_srcs SRCS_LIST := $(BUILD_DIR)/vortex_srcs.f -.PHONY: stage patch sources vivado-sources xilinx-ips check-firesim clean-staged +.PHONY: stage sources vivado-sources xilinx-ips kernel-env bitstream check-firesim clean-staged stage: check-firesim @mkdir -p $(TARGET_DEST) $(CC_DEST) @@ -51,15 +68,8 @@ stage: check-firesim cp -f "$$f" $(CC_DEST)/ || exit 1; \ echo "staged $$(basename $$f)"; \ done - -# Applies in series against a pristine checkout. Ordering matters: 0003 and 0005 -# both build on 0001. Re-running against an already-patched checkout will report -# reversed hunks; that is the intended signal, not a failure to hide. -patch: check-firesim - @cd $(FIRESIM_PATH) && for p in $(sort $(wildcard $(PATCH_DIR)/[0-9]*.patch)); do \ - echo "applying $$(basename $$p)"; \ - patch -p2 --no-backup-if-mismatch < "$$p" || exit 1; \ - done + @$(PLATFORM_GEN) --emit=scala -o $(TARGET_DEST)/$(PLATFORM_CONFIG).scala + @echo "staged $(PLATFORM_CONFIG).scala ($(FIRESIM_CARD))" XLEN ?= 32 @@ -223,8 +233,7 @@ $(VIVADO_SRCS_LIST): $(RTL_SRCS) # floating_point IP (xil_fma, xil_fdiv, ...) instead of the soft core, and the # flattened sources only instantiate them, so the kernel packaging must be handed # the generated cores or synthesis fails on the missing modules. Same script and -# core set as the XRT flow. The part must match the FireSim kernel's part. -FPGA_PART ?= xcu55c-fsvh2892-2L-e +# core set as the XRT flow, for the card's part (FPGA_PART, from the descriptor). XILINX_IP_DIR := $(BUILD_DIR)/ip XILINX_IP_STAMP := $(XILINX_IP_DIR)/.part-$(FPGA_PART) @@ -245,13 +254,32 @@ kernel-env: @echo 'export VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl' @echo 'export TOOL_DIR=$(SCRIPT_DIR)' +# One command from sources to the card's image. Expects the FireSim environment +# (its env.sh) and the Xilinx tools for the card's platform in the environment. +bitstream: vivado-sources sources xilinx-ips stage + $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) + VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ + $(MAKE) -C $(FIRESIM_PATH)/platforms/$(PLATFORM)/cl_$(QUINTUPLET) bitstream \ + DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) + +FREQUENCY ?= 60.0 +STRATEGY ?= TIMING + +# The checkout must contain the pinned fork commit. Being ahead of it is normal +# while developing the fork; being behind it, or on unrelated history, builds +# against FireSim changes this tree does not expect. check-firesim: @test -d $(FIRESIM_PATH)/sim || { \ echo "FireSim checkout not found at $(FIRESIM_PATH)"; \ - echo "set FIRESIM_PATH to a FireSim 1.21.0 checkout"; \ + echo "clone vortexgpgpu/firesim there (see docs/building_toolchain.md)"; \ + exit 1; } + @git -C $(FIRESIM_PATH) merge-base --is-ancestor $(FIRESIM_REV) HEAD 2>/dev/null || { \ + echo "FireSim checkout at $(FIRESIM_PATH) does not contain FIRESIM_REV=$(FIRESIM_REV)"; \ + echo " HEAD is $$(git -C $(FIRESIM_PATH) rev-parse --short HEAD)"; \ + echo " fetch the fork and check out $(FIRESIM_REV) or a descendant"; \ exit 1; } clean-staged: - @for f in $(SCALA_SRCS); do \ + @for f in $(SCALA_SRCS) $(PLATFORM_CONFIG).scala; do \ rm -f $(TARGET_DEST)/$$(basename "$$f"); \ done diff --git a/hw/syn/firesim/patches/0001-rebase-target-cycle-counter.patch b/hw/syn/firesim/patches/0001-rebase-target-cycle-counter.patch deleted file mode 100644 index 6527f07302..0000000000 --- a/hw/syn/firesim/patches/0001-rebase-target-cycle-counter.patch +++ /dev/null @@ -1,73 +0,0 @@ ---- build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h.orig 2026-08-02 05:13:08.369209268 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h 2026-08-02 05:16:23.260800446 -0700 -@@ -145,7 +145,18 @@ - midas_time_t end_time; - std::optional start_hcycle; - std::optional end_hcycle; -+ uint64_t start_tcycle = 0; - uint64_t end_tcycle = 0; -+ -+ /// Target cycles consumed by *this* run. -+ /// The counter lives in fabric and persists while the bitstream stays -+ /// programmed, so it must be rebased against the start sample -- host -+ /// cycles are already handled this way. -+ /// Saturates rather than wrapping: a device reprogrammed mid-run can leave -+ /// end < start, and an unsigned wrap would report a nonsense duration. -+ uint64_t elapsed_tcycles() const { -+ return (end_tcycle >= start_tcycle) ? (end_tcycle - start_tcycle) : 0; -+ } - }; - - #endif // __SIMULATION_H ---- build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc.orig 2026-08-02 05:13:08.373209261 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc 2026-08-02 05:16:23.260800446 -0700 -@@ -41,6 +41,7 @@ - - void simulation_t::record_start_times() { - start_hcycle = clock.hcycle(); -+ start_tcycle = clock.tcycle(); - start_time = timestamp(); - } - -@@ -58,10 +59,13 @@ - - const uint64_t hcycles = *end_hcycle - *start_hcycle; - const double sim_time = diff_secs(end_time, start_time); -- const double sim_speed = ((double)end_tcycle) / (sim_time * 1000.0); -+ const uint64_t tcycles = elapsed_tcycles(); -+ const double sim_speed = ((double)tcycles) / (sim_time * 1000.0); - const double measured_host_frequency = - ((double)hcycles) / (sim_time * 1000.0); -- const double fmr = ((double)hcycles / end_tcycle); -+ // A run that steps no target cycles is legal (e.g. fingerprint-only); -+ // reporting inf/nan for it would be worse than reporting zero. -+ const double fmr = tcycles ? ((double)hcycles / tcycles) : 0.0; - - fprintf(stderr, "\nEmulation Performance Summary\n"); - fprintf(stderr, "------------------------------\n"); -@@ -74,7 +78,7 @@ - fprintf(stderr, "%.3f KHz\n", measured_host_frequency); - } - -- fprintf(stderr, "Target Cycles Emulated: %" PRIu64 "\n", end_tcycle); -+ fprintf(stderr, "Target Cycles Emulated: %" PRIu64 "\n", tcycles); - fprintf(stderr, "Effective Target Frequency: "); - if (sim_speed > 1000.0) { - fprintf(stderr, "%.3f MHz\n", sim_speed / 1000.0); -@@ -154,13 +158,13 @@ - fprintf(stderr, - "*** FAILED *** (code = %d) after %" PRIu64 " cycles\n", - exit_code, -- end_tcycle); -+ elapsed_tcycles()); - } else if (timeout) { - fprintf(stderr, - "*** FAILED *** simulation timed out after %" PRIu64 " cycles\n", -- end_tcycle); -+ elapsed_tcycles()); - } else { -- fprintf(stderr, "*** PASSED *** after %" PRIu64 " cycles\n", end_tcycle); -+ fprintf(stderr, "*** PASSED *** after %" PRIu64 " cycles\n", elapsed_tcycles()); - } - - print_simulation_performance_summary(); diff --git a/hw/syn/firesim/patches/0002-vitis-u55c-build-fixes.patch b/hw/syn/firesim/patches/0002-vitis-u55c-build-fixes.patch deleted file mode 100644 index 524e715dd3..0000000000 --- a/hw/syn/firesim/patches/0002-vitis-u55c-build-fixes.patch +++ /dev/null @@ -1,68 +0,0 @@ -Vitis flow fixes required to build a FireSim xclbin for the Alveo U55C -under Vitis 2024.2. Upstream FireSim targets Vitis 2022.1 and a U250. - - 1. host_mem_0 connectivity was hardcoded to DDR[0]; the U55C is HBM-only - (platforminfo reports "Bus SP Tag: HBM" and no DDR entry). Parameterized - as HOST_MEM_SP so non-HBM platforms can override it. - 2. package_kernel.tcl hardcoded the U250 part, so kernel IP was generated - for the wrong device regardless of DEVICE=. - 3. package_xo silently produced no .xo: create_project chdirs into the temp - project and the closing "close_project -delete" removes it, leaving the - process CWD on a deleted path so package_xo dies inside - boost::filesystem::current_path(). Save and restore the CWD. - 4. v++ --temp_dir pointed at the same directory the .xo is written to, and - Vitis 2024.2 clears its temp dir at startup -- deleting the .xo before - the link step reads it. - ---- build_firesim/firesim/platforms/vitis/cl_firesim/Makefile.orig 2026-08-02 05:18:37.916499449 -0700 -+++ build_firesim/firesim/platforms/vitis/cl_firesim/Makefile 2026-08-02 05:18:37.948499376 -0700 -@@ -47,6 +47,11 @@ - # current support on firesim1 - DEVICE ?= xilinx_u250_gen3x16_xdma_3_1_202020_1 - -+# Memory resource backing the simulator's host_mem_0 port. Upstream hardcodes -+# DDR[0]; HBM-only cards (Alveo U55C) expose no DDR bank at all, so this must -+# be overridable per platform. -+HOST_MEM_SP ?= HBM[0] -+ - # Provide's the desired simulator frequency. This is generated from the clock - # provided to the kernel using an internal MMCM. - FREQUENCY ?= 90 -@@ -168,7 +173,7 @@ - mkdir -p $(@D) - # Move into temp directory to avoid polluting top-level workdir - cd $(@D)/..; $(VPP) $(vpp_config_overlay_args) $(VPP_TARGET_SPECIFIC_ARGS) $(VPP_COMMON_ARGS) \ -- --connectivity.sp $(PROJECT_NAME)_1.host_mem_0:DDR[0] \ -+ --connectivity.sp $(PROJECT_NAME)_1.host_mem_0:$(HOST_MEM_SP) \ - --link $(VPP_LDFLAGS) -o $@ $< - - ---- build_firesim/firesim/platforms/vitis/cl_firesim/scripts/package_kernel.tcl.orig 2026-08-02 00:47:00.546532830 -0700 -+++ build_firesim/firesim/platforms/vitis/cl_firesim/scripts/package_kernel.tcl 2026-08-02 00:47:00.718532277 -0700 -@@ -19,7 +19,9 @@ - set path_to_packaged "./packaged_kernel_${suffix}" - set path_to_tmp_project "./tmp_kernel_pack_${suffix}" - --set projPart "xcu250-figd2104-2L-e" -+# Upstream hardcodes the U250 part; the kernel-packaging project must use -+# the part of the target platform or its IP is generated for the wrong device. -+set projPart "xcu55c-fsvh2892-2L-e" - - set base_filename "FireSim-generated" - set synth_xdc_path "${path_to_hdl}/${base_filename}.synthesis.xdc" ---- build_firesim/firesim/platforms/vitis/cl_firesim/scripts/gen_xo.tcl.orig 2026-08-02 00:51:18.973709853 -0700 -+++ build_firesim/firesim/platforms/vitis/cl_firesim/scripts/gen_xo.tcl 2026-08-02 00:51:19.061709575 -0700 -@@ -33,7 +33,13 @@ - set script_path [file dirname [file normalize [info script]]] - - #source -notrace ./scripts/package_kernel.tcl -+# Vivado's create_project chdirs into the project dir, and package_kernel.tcl's -+# closing "close_project -delete" then removes it -- leaving the process CWD -+# pointing at a deleted directory. package_xo subsequently dies inside -+# boost::filesystem::current_path(). Save the CWD and restore it below. -+set origin_cwd [pwd] - source ${script_path}/package_kernel.tcl -+cd $origin_cwd - - if {[file exists "${xoname}"]} { - file delete -force "${xoname}" diff --git a/hw/syn/firesim/patches/0003-vitis-memcheck-and-buffer-lifetime.patch b/hw/syn/firesim/patches/0003-vitis-memcheck-and-buffer-lifetime.patch deleted file mode 100644 index 73a23e69bf..0000000000 --- a/hw/syn/firesim/patches/0003-vitis-memcheck-and-buffer-lifetime.patch +++ /dev/null @@ -1,156 +0,0 @@ -Diagnostic and correctness fixes for FireSim on the Alveo U55C. -Applies on top of 0001-rebase-target-cycle-counter.patch. - - 1. simif_vitis.cc: hold the host_mem_0 xrt::bo as a member. Upstream - allocates it as a constructor local, so the device allocation is freed - when the constructor returns and XRT may reassign that memory for the - entire life of the simulation. - - 2. simif_vitis.cc: drop the hardcoded U250 DDR base assertion. The kernel's - host_mem_0_offset register is IP-XACT metadata only -- it is absent from - the generated RTL, and writing it lands in FireSim's own widget MMIO - space. Address mapping is done by the platform memory subsystem via - --connectivity.sp. - - 3. simulation.{h,cc}: add +memcheck, which round-trips a known pattern - through the LoadMem widget before target execution, testing the target - DRAM path independently of the design under test. This is what identified - the U55C HBM memory path as non-functional while the control plane works - (see docs/proposals/firesim_vitis_host_mem_offset.md). - ---- build_firesim/firesim/sim/midas/src/main/cc/simif_vitis.cc.orig 2026-08-02 03:11:40.536120157 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/simif_vitis.cc 2026-08-02 06:01:44.813949894 -0700 -@@ -42,16 +42,16 @@ - xrt::uuid uuid; - xrt::ip kernel_handle; - xrt::run run_handle; -+ /// The device-side memory backing host_mem_0. Must outlive the -+ /// simulation: as a constructor local its allocation is freed on -+ /// return, leaving the window pointing at memory XRT may reuse. -+ xrt::bo fpga_mem_0; - }; - --constexpr size_t u250_dram_channel_size_bytes = 16ULL * 1024 * 1024 * 1024; --/** -- * We currently support only a single FPGA DRAM channel. When the buffer is -- * allocated in XRT, we'd generally have to write the offset back to the kernel -- * using MMIO. For now we've hardcoded the offset for a U250 with no other -- * concurrently running kernel. -- */ --constexpr uint64_t u250_dram_expected_offset = 0x4000000000L; -+/// Size of the single device-memory window backing host_mem_0. -+/// One U55C HBM pseudo-channel is 512 MB; a U250 DDR bank is larger, but a -+/// single channel is all this simif supports either way. -+constexpr size_t host_mem_0_size_bytes = 512ULL * 1024 * 1024; - - simif_vitis_t::simif_vitis_t(const TargetConfig &config, - const std::vector &args) -@@ -92,22 +92,21 @@ - kernel_handle = xrt::ip(device_handle, uuid, "firesim"); - - // Intialize FPGA-DRAM regions. -- // The final argument here is the bank index for the dram channel. -- // I used xclbinutil to find this -- // https://xilinx.github.io/XRT/master/html/xclbintools.html -- auto fpga_mem_0 = xrt::bo(device_handle, -- u250_dram_channel_size_bytes, -- xrt::bo::flags::device_only, -- 0); -- -- if (fpga_mem_0.address() != u250_dram_expected_offset) { -- std::cerr -- << "Allocated device_only buffer address not match the expected offset." -- << std::endl; -- std::cerr << "Expected: " << u250_dram_expected_offset -- << "Received: " << fpga_mem_0.address() << std::endl; -- exit(1); -- } -+ // The final argument is the bank index for the DRAM channel, found with -+ // xclbinutil: https://xilinx.github.io/XRT/master/html/xclbintools.html -+ fpga_mem_0 = xrt::bo(device_handle, -+ host_mem_0_size_bytes, -+ xrt::bo::flags::device_only, -+ 0); -+ -+ // NOTE: the kernel's host_mem_0_offset register (package_kernel.tcl, s_axi_lite -+ // 0x010) is IP-XACT metadata only -- it does not appear in the generated RTL, -+ // and writing it would land in FireSim's own widget MMIO space. Address -+ // mapping into HBM is done by the platform memory subsystem via -+ // --connectivity.sp, so nothing needs programming here. -+ std::cerr << "[vitis] host_mem_0 base = 0x" << std::hex -+ << fpga_mem_0.address() << " size = 0x" << host_mem_0_size_bytes -+ << std::dec << std::endl; - } - - void simif_vitis_t::write(size_t addr, uint32_t data) { ---- build_firesim/patchbase/simulation.h 2026-08-02 06:12:46.916218880 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h 2026-08-02 06:06:05.805273470 -0700 -@@ -130,6 +130,10 @@ - */ - bool do_zero_out_dram = false; - -+ /// Diagnostic: round-trip a pattern through the LoadMem widget after -+ /// loading, to prove the target DRAM path independently of the DUT. -+ bool do_memcheck = false; -+ - /** - * If set, read the presence register, check it, and exit the simulation - * cleanly ---- build_firesim/patchbase/simulation.cc 2026-08-02 06:12:46.916218880 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc 2026-08-02 06:06:05.805273470 -0700 -@@ -24,6 +24,9 @@ - if (arg.find("+loadmem=") == 0) { - load_mem_path = arg.c_str() + 9; - } -+ if (arg.find("+memcheck") == 0) { -+ do_memcheck = true; -+ } - if (arg.find("+zero-out-dram") == 0) { - do_zero_out_dram = true; - } -@@ -189,6 +192,45 @@ - if (!load_mem_path.empty()) { - loadmem->load_mem_from_file(load_mem_path); - } -+ -+ // Diagnostic: write a known pattern through the LoadMem widget and read it -+ // back through the same widget. This isolates "the target DRAM path works" -+ // from any question about the design under test -- if this round-trip -+ // fails, no memory-touching target can possibly pass. -+ if (do_memcheck) { -+ const size_t chunk_bits = loadmem->get_mem_data_chunk() * 8; -+ mpz_t wr, rd; -+ mpz_init(wr); -+ mpz_init(rd); -+ unsigned failures = 0; -+ const size_t probes[] = {0x0, 0x40, 0x1000, 0x40000, 0xF0000}; -+ for (size_t addr : probes) { -+ // A value that is distinct per address and not all-ones/all-zeros. -+ mpz_set_ui(wr, 0); -+ mpz_setbit(wr, 0); -+ mpz_mul_2exp(wr, wr, chunk_bits / 2); -+ mpz_add_ui(wr, wr, (unsigned long)(0xA5A50000u | (addr & 0xFFFF))); -+ loadmem->write_mem(addr, wr); -+ loadmem->read_mem(addr, rd); -+ const bool ok = (mpz_cmp(wr, rd) == 0); -+ gmp_fprintf(stderr, -+ "[memcheck] addr=0x%zx wrote=0x%Zx read=0x%Zx %s\n", -+ addr, -+ wr, -+ rd, -+ ok ? "OK" : "MISMATCH"); -+ if (!ok) { -+ failures++; -+ } -+ } -+ mpz_clear(wr); -+ mpz_clear(rd); -+ fprintf(stderr, -+ "[memcheck] %s (%u/%zu probes failed)\n", -+ failures ? "TARGET DRAM PATH IS BROKEN" : "target DRAM path OK", -+ failures, -+ sizeof(probes) / sizeof(probes[0])); -+ } - } else { - if (do_zero_out_dram || !load_mem_path.empty()) { - fprintf(stderr, diff --git a/hw/syn/firesim/patches/0004-vitis-host-mem-base-address.patch b/hw/syn/firesim/patches/0004-vitis-host-mem-base-address.patch deleted file mode 100644 index 273dbe940e..0000000000 --- a/hw/syn/firesim/patches/0004-vitis-host-mem-base-address.patch +++ /dev/null @@ -1,66 +0,0 @@ -Fix the FireSim Vitis shim hardcoding the U250 DDR base address in RTL. - -VitisShim added a literal 0x4000000000 -- the base of DDR[0] on a U250 -- to -every host_mem_0 address, in the generated RTL rather than in any register. -On an HBM-only part such as the Alveo U55C, host_mem_0 binds to HBM[0] at -0x0 with a 512 MB (29-bit) aperture, so every access landed far outside the -bound pseudo-channel: reads returned a fixed bus default independent of -address and writes were dropped, leaving the target DRAM path silently dead -while the MMIO/control plane still worked. - -Replaced with a documented constant in VitisConstants so the base tracks -whatever --connectivity.sp binds host_mem_0 to. A proper upstream fix makes -this a Config field keyed off the platform; it is a constant here because -this tree targets one card, and the code says so. - -Validated under hw_emu, which exercises the same block design without -place-and-route: - - pre-fix (control): +memcheck 5/5 probes fail, address-independent constant - with fix : +memcheck 0/5 probes fail, every probe reads back what - was written - -The regenerated RTL was independently confirmed to contain zero occurrences -of the old constant, down from two. - ---- build_firesim/firesim/sim/midas/src/main/scala/midas/platform/VitisShim.scala.orig 2026-08-02 07:11:46.303359225 -0700 -+++ build_firesim/firesim/sim/midas/src/main/scala/midas/platform/VitisShim.scala 2026-08-02 07:11:46.331359159 -0700 -@@ -21,6 +21,25 @@ - - /** The hardcoded TCL variable name used to specify the simulator's frequency */ - val frequencyVariableName = "frequency" -+ -+ /** Base address added to every host_mem_0 access. -+ * -+ * The shim drives absolute device addresses, so this must match the base of -+ * whatever memory resource `--connectivity.sp` binds host_mem_0 to. -+ * -+ * U250 / U280, DDR[0] -> 0x4000000000 -+ * U55C, HBM[0] (HBM-only) -> 0x0, with a 512 MB (29-bit) aperture -+ * -+ * Upstream hardcoded the U250 DDR base inline. On an HBM part every access -+ * then lands far outside the bound pseudo-channel: reads return a fixed bus -+ * default independent of address and writes are dropped, so the target DRAM -+ * path is silently dead while the control plane still works. -+ * -+ * A proper upstream fix makes this a Config field keyed off the platform -+ * rather than a constant; it is a constant here because this tree targets -+ * one card. -+ */ -+ val hostMemBaseAddress = 0x0L - } - - class VitisShim(implicit p: Parameters) extends PlatformShim { -@@ -89,8 +108,10 @@ - host_mem_cdc.io.s_axi.drivenByStandardAXI4(top.module.mem(0), hostClock, hostSyncReset) - host_mem_cdc.io.s_axi_aclk := hostClock - host_mem_cdc.io.s_axi_aresetn := (!hostSyncReset).asAsyncReset -- host_mem_cdc.io.s_axi.araddr := 0x4000000000L.U(addrWidth) + top.module.mem(0).ar.bits.addr -- host_mem_cdc.io.s_axi.awaddr := 0x4000000000L.U(addrWidth) + top.module.mem(0).aw.bits.addr -+ host_mem_cdc.io.s_axi.araddr := -+ VitisConstants.hostMemBaseAddress.U(addrWidth) + top.module.mem(0).ar.bits.addr -+ host_mem_cdc.io.s_axi.awaddr := -+ VitisConstants.hostMemBaseAddress.U(addrWidth) + top.module.mem(0).aw.bits.addr - host_mem_cdc.io.s_axi.arcache.foreach { _ := AXI4Parameters.CACHE_MODIFIABLE } - host_mem_cdc.io.s_axi.awcache.foreach { _ := AXI4Parameters.CACHE_MODIFIABLE } - diff --git a/hw/syn/firesim/patches/0005-inflight-target-cycle.patch b/hw/syn/firesim/patches/0005-inflight-target-cycle.patch deleted file mode 100644 index 6b8f8b34f1..0000000000 --- a/hw/syn/firesim/patches/0005-inflight-target-cycle.patch +++ /dev/null @@ -1,54 +0,0 @@ -Correct the target-cycle count across the counter's origin. - -The clock bridge counts target cycles *started*, not completed: under FAME-1 -the target accepts the clock token for cycle N+1 and only then stalls on the -channels it needs to finish it, so exactly one cycle is in flight whenever the -target has started at least one. (The clock-token channel is wired to the -target combinationally -- SimWrapper.genClockChannel -- so a token that has -fired has been accepted, not buffered.) - -That in-flight cycle is present at both samples taken by -0001-rebase-target-cycle-counter.patch and cancels -- except across the -counter's origin, where the start sample sees a target that has yet to start -its first cycle. The result was a count that depended on whether the bitstream -had been run before: 14 cycles on the first run after programming and 13 on -every run after it, for a test that provably steps the target exactly 13 times -(TestAdder4: target_reset(5) plus 8 single steps). A fresh metasim is always -the origin case, so upstream reports the +1 on every metasim run. - -Corrects for the in-flight cycle explicitly: cycles completed at a sample are -"started - in flight", and in flight is 0 exactly when the counter reads 0 and -1 otherwise. Validated on the vitis metasim: Adder4 14 -> 13, matching the -counted steps and the card's steady state; PointerChaser 1625 -> 1624, result -still bit-exact at 0x498. Host-side only, no bitstream rebuild needed. - -This deliberately diverges from upstream, which reports the +1. - ---- build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h.orig -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h -@@ -152,14 +152,23 @@ - uint64_t start_tcycle = 0; - uint64_t end_tcycle = 0; - -- /// Target cycles consumed by *this* run. -+ /// Target cycles completed by *this* run. -+ /// - /// The counter lives in fabric and persists while the bitstream stays - /// programmed, so it must be rebased against the start sample -- host - /// cycles are already handled this way. -+ /// -+ /// It counts cycles *started*, so one cycle is in flight at any sample taken -+ /// after the target's first. That cancels between the two samples, except at -+ /// the counter's origin, where the start sample sees none. -+ /// - /// Saturates rather than wrapping: a device reprogrammed mid-run can leave - /// end < start, and an unsigned wrap would report a nonsense duration. - uint64_t elapsed_tcycles() const { -- return (end_tcycle >= start_tcycle) ? (end_tcycle - start_tcycle) : 0; -+ if (end_tcycle <= start_tcycle) { -+ return 0; -+ } -+ return end_tcycle - start_tcycle - (start_tcycle == 0 ? 1 : 0); - } - }; - diff --git a/hw/syn/firesim/patches/0006-package-design-sources.patch b/hw/syn/firesim/patches/0006-package-design-sources.patch deleted file mode 100644 index 060d7cc9b6..0000000000 --- a/hw/syn/firesim/patches/0006-package-design-sources.patch +++ /dev/null @@ -1,112 +0,0 @@ -Carry design sources that simulation resolves by library search into the kernel. - -Vortex is handed to Verilator as a handful of packages plus a wrapper, with the -remaining modules resolved by library search over the include paths. Vivado has -no equivalent, so without this it synthesizes a design whose submodules are all -missing. - -The sources cannot simply be added to the packaging project. ipx::package_project -infers bus interfaces from everything it imports, and a design carrying -SystemVerilog interfaces has no IP-XACT representation, so packaging fails on -ports it cannot express. They are instead attached to the core after packaging, -where the inference pass never sees them. - -Attaching alone is not enough either. A file referenced outside the component -root does not travel with the IP, leaving the linker to find the instantiation -but not the module, so each file is copied into the component and registered -relatively. It is registered in both the synthesis and behavioural-simulation -file groups: the first builds bitstreams, the second builds hardware emulation, -and a file present in only one is invisible to the other flow. - -The kernel top is named explicitly as well. Vivado infers it from the source set, -and a design large enough to need this hook is large enough for that inference to -settle on one of its internal modules, which then fails to package. - -Absent VORTEX_VIVADO_SRCS the flow is unchanged. - ---- build_firesim/firesim/platforms/vitis/cl_firesim/scripts/package_kernel.tcl.orig -+++ build_firesim/firesim/platforms/vitis/cl_firesim/scripts/package_kernel.tcl -@@ -46,6 +46,45 @@ - $path_to_hdl/FireSim-generated.post-processed.sv \ - ] - -+# The design under test may resolve most of its RTL by library search during -+# simulation, which Vivado has no equivalent for. When a flattened source list is -+# supplied, every file in it is added explicitly and its search paths recorded. -+if {[info exists ::env(VORTEX_VIVADO_SRCS)]} { -+ set vortex_list $::env(VORTEX_VIVADO_SRCS) -+ if {![file exists $vortex_list]} { -+ puts "ERROR: VORTEX_VIVADO_SRCS points at a missing file: $vortex_list" -+ exit 1 -+ } -+ set fh [open $vortex_list r] -+ set vortex_files {} -+ set vortex_incdirs {} -+ while {[gets $fh line] >= 0} { -+ set line [string trim $line] -+ if {$line eq ""} { continue } -+ if {[string match "+incdir+*" $line]} { -+ lappend vortex_incdirs [string range $line 8 end] -+ } elseif {[string match "+*" $line]} { -+ continue -+ } else { -+ lappend vortex_files $line -+ } -+ } -+ close $fh -+ if {[llength $vortex_files] == 0} { -+ puts "ERROR: no sources listed in $vortex_list" -+ exit 1 -+ } -+ puts "collected [llength $vortex_files] design sources from $vortex_list" -+ # Deliberately not added to this project. ipx::package_project infers bus -+ # interfaces from every file it imports, and a design carrying SystemVerilog -+ # interfaces cannot be expressed in IP-XACT -- packaging fails on ports it -+ # has no way to represent. The files are attached to the packaged core -+ # further down instead, so they ship with the IP without being parsed here. -+ if {[llength $vortex_incdirs] > 0} { -+ set_property include_dirs $vortex_incdirs [get_filesets sources_1] -+ } -+} -+ - set_property USED_IN_SYNTHESIS false [get_files $impl_xdc_path] - set_property USED_IN_IMPLEMENTATION true [get_files $impl_xdc_path] - -@@ -102,6 +141,37 @@ - set_property sdx_kernel_type rtl $core - set_property supported_families { } $core - set_property auto_family_support_level level_2 $core -+# Deliver the design sources with the packaged core. They are added to the -+# synthesis file group directly rather than through the project, so synthesis -+# sees them while the packager's interface inference never does. -+if {[info exists vortex_files] && [llength $vortex_files] > 0} { -+ # Both groups: synthesis builds the bitstream, behavioural simulation builds -+ # hardware emulation, and a file registered in only one is invisible to the -+ # other flow. -+ set target_fgs [list \ -+ [ipx::get_file_groups xilinx_anylanguagesynthesis -of_objects $core] \ -+ [ipx::get_file_groups xilinx_anylanguagebehavioralsimulation -of_objects $core]] -+ # Copied into the core rather than referenced where they lie: a path outside -+ # the component root does not survive IP delivery, so the linker would find -+ # the instantiation but not the module. -+ set vortex_dst $path_to_packaged/src -+ file mkdir $vortex_dst -+ foreach vf $vortex_files { -+ set base [file tail $vf] -+ file copy -force $vf $vortex_dst/$base -+ foreach fg $target_fgs { -+ set added [ipx::add_file src/$base $fg] -+ if {[string match "*.vh" $base] || [string match "*.svh" $base]} { -+ set_property type verilogSource $added -+ set_property is_include true $added -+ } elseif {[string match "*.sv" $base]} { -+ set_property type systemVerilogSource $added -+ } -+ } -+ } -+ puts "attached [llength $vortex_files] design sources to the packaged core" -+} -+ - ipx::create_xgui_files $core - ipx::update_checksums $core - ipx::check_integrity -kernel $core diff --git a/hw/syn/firesim/patches/0007-vitis-target-dram-aperture.patch b/hw/syn/firesim/patches/0007-vitis-target-dram-aperture.patch deleted file mode 100644 index 4739bfa6a5..0000000000 --- a/hw/syn/firesim/patches/0007-vitis-target-dram-aperture.patch +++ /dev/null @@ -1,157 +0,0 @@ ---- build_firesim/firesim/sim/midas/src/main/cc/simif_vitis.cc.orig -+++ build_firesim/firesim/sim/midas/src/main/cc/simif_vitis.cc -@@ -1,4 +1,8 @@ -+#include - #include -+#include -+#include -+#include - - #include - #include -@@ -12,6 +16,7 @@ - #include "experimental/xrt_device.h" - #include "experimental/xrt_ip.h" - #include "experimental/xrt_kernel.h" -+#include "experimental/xrt_xclbin.h" - - class simif_vitis_t final : public simif_t, public FPGAManagedStreamIO { - public: -@@ -48,10 +53,61 @@ - xrt::bo fpga_mem_0; - }; - --/// Size of the single device-memory window backing host_mem_0. --/// One U55C HBM pseudo-channel is 512 MB; a U250 DDR bank is larger, but a --/// single channel is all this simif supports either way. --constexpr size_t host_mem_0_size_bytes = 512ULL * 1024 * 1024; -+/// Size of the device-memory window backing host_mem_0. -+/// -+/// This must cover the target's whole address map, not merely the part of it a -+/// given test happens to touch. A 32-bit target places its stack at the top of -+/// 4 GiB and its program image well above any buffer it allocates, so anything -+/// smaller leaves those accesses unserved: writes are dropped and reads return -+/// an address-independent value, with no error anywhere to say so. -+/// -+/// One U55C HBM pseudo-channel is 512 MB, so 4 GiB spans eight of them and -+/// `--connectivity.sp` must bind host_mem_0 to a matching range (HOST_MEM_SP in -+/// the Vitis kernel Makefile). The two are a pair: widening one alone silently -+/// truncates. -+constexpr size_t host_mem_0_size_bytes = 4ULL * 1024 * 1024 * 1024; -+ -+/// Allocates the device-memory window backing host_mem_0, choosing a memory -+/// region that can hold all of it. Tries the largest usable regions first so a -+/// range group is preferred over any single pseudo-channel inside it. -+static xrt::bo allocate_host_mem(const xrt::device &device, -+ const std::string &binary_file, -+ size_t bytes) { -+ xrt::xclbin xclbin(binary_file); -+ const auto mems = xclbin.get_mems(); -+ -+ std::vector candidates; -+ for (size_t i = 0; i < mems.size(); ++i) { -+ if (mems[i].get_used()) { -+ candidates.push_back(i); -+ } -+ } -+ std::stable_sort(candidates.begin(), candidates.end(), -+ [&mems](size_t a, size_t b) { -+ return mems[a].get_size_kb() > mems[b].get_size_kb(); -+ }); -+ -+ for (size_t idx : candidates) { -+ try { -+ xrt::bo bo(device, bytes, xrt::bo::flags::device_only, idx); -+ std::cerr << "[vitis] host_mem_0 backed by memory index " << idx << " (" -+ << mems[idx].get_tag() << ")" << std::endl; -+ return bo; -+ } catch (const std::exception &) { -+ // Too small, or not allocatable: try the next region. -+ } -+ } -+ -+ std::cerr << "[vitis] no memory region can back a 0x" << std::hex << bytes -+ << std::dec << "-byte host_mem_0 window. Topology:" << std::endl; -+ for (size_t i = 0; i < mems.size(); ++i) { -+ std::cerr << "[vitis] [" << i << "] " << mems[i].get_tag() << " " -+ << mems[i].get_size_kb() << " KB" -+ << (mems[i].get_used() ? " (used)" : "") << std::endl; -+ } -+ throw std::runtime_error("host_mem_0 window does not fit any memory region; " -+ "widen --connectivity.sp (HOST_MEM_SP)"); -+} - - simif_vitis_t::simif_vitis_t(const TargetConfig &config, - const std::vector &args) -@@ -92,12 +148,15 @@ - kernel_handle = xrt::ip(device_handle, uuid, "firesim"); - - // Intialize FPGA-DRAM regions. -- // The final argument is the bank index for the DRAM channel, found with -- // xclbinutil: https://xilinx.github.io/XRT/master/html/xclbintools.html -- fpga_mem_0 = xrt::bo(device_handle, -- host_mem_0_size_bytes, -- xrt::bo::flags::device_only, -- 0); -+ // -+ // The last argument is a memory index into the xclbin's topology. Binding -+ // host_mem_0 to a range of pseudo-channels makes the region a memory *group* -+ // whose index is not knowable from the port name, and xrt::ip -- unlike -+ // xrt::kernel -- exposes no group_id() to ask for it. So try the indices the -+ // xclbin reports until one can back the whole window, and fail loudly with -+ // the topology printed if none can. Quietly settling for a smaller region is -+ // the exact failure this window size exists to prevent. -+ fpga_mem_0 = allocate_host_mem(device_handle, binary_file, host_mem_0_size_bytes); - - // NOTE: the kernel's host_mem_0_offset register (package_kernel.tcl, s_axi_lite - // 0x010) is IP-XACT metadata only -- it does not appear in the generated RTL, ---- build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc.orig -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc -@@ -203,7 +203,20 @@ - mpz_init(wr); - mpz_init(rd); - unsigned failures = 0; -- const size_t probes[] = {0x0, 0x40, 0x1000, 0x40000, 0xF0000}; -+ // Probes must span the target's whole address map, not just its bottom. -+ // An earlier list stopped below 1 MB and so passed against a window that -+ // served only its first eighth, while the target's program image and -+ // stack -- which sit at 0x80000000 and just under 4 GiB -- landed -+ // outside it. Proving the path works says nothing about how far it -+ // reaches; these last three are what test the reach. -+ const size_t probes[] = {0x0, -+ 0x40, -+ 0x1000, -+ 0x40000, -+ 0xF0000, -+ 0x20000000, -+ 0x80000000, -+ 0xFFFF0000}; - for (size_t addr : probes) { - // A value that is distinct per address and not all-ones/all-zeros. - mpz_set_ui(wr, 0); ---- build_firesim/firesim/platforms/vitis/cl_firesim/Makefile.orig -+++ build_firesim/firesim/platforms/vitis/cl_firesim/Makefile -@@ -50,7 +50,13 @@ - # Memory resource backing the simulator's host_mem_0 port. Upstream hardcodes - # DDR[0]; HBM-only cards (Alveo U55C) expose no DDR bank at all, so this must - # be overridable per platform. --HOST_MEM_SP ?= HBM[0] -+# -+# The range must span the target's whole address map: a 32-bit target needs -+# 4 GiB, which on a U55C is eight 512 MB HBM pseudo-channels. It is bound -+# together with host_mem_0_size_bytes in simif_vitis.cc -- the port reaches -+# what this binds, and the driver backs what that allocates, so widening -+# either alone silently truncates. -+HOST_MEM_SP ?= HBM[0:7] - - # Provide's the desired simulator frequency. This is generated from the clock - # provided to the kernel using an internal MMCM. -@@ -118,7 +124,7 @@ - VPP_MSG_FLAGS = --message-rules $(MSG_RULES) --log_dir $(LOG_SUBDIR) --report_dir $(REPORT_SUBDIR) - VPP_COMMON_ARGS = \ - --platform $(DEVICE) \ -- --temp_dir $(TEMP_SUBDIR) \ -+ --temp_dir vpp_temp.$(XSA) \ - $(VPP_OTHER_FLAGS) \ - $(VPP_MSG_FLAGS) - diff --git a/hw/syn/firesim/patches/0008-driver-teardown-before-exit.patch b/hw/syn/firesim/patches/0008-driver-teardown-before-exit.patch deleted file mode 100644 index 9ae0b9aa21..0000000000 --- a/hw/syn/firesim/patches/0008-driver-teardown-before-exit.patch +++ /dev/null @@ -1,18 +0,0 @@ ---- build_firesim/firesim/sim/midas/src/main/cc/entry.cc.orig -+++ build_firesim/firesim/sim/midas/src/main/cc/entry.cc -@@ -73,5 +73,14 @@ - simulation = create_simulation(simif, *widget_registry_ptr, args); - - // Run the simulation with the given implementation. -- return simif.run(*simulation); -+ const int rc = simif.run(*simulation); -+ -+ // Tear these down here rather than leaving them to static destruction. The -+ // platform layer holds handles into libraries with statics of their own -- -+ // XRT's device registry among them -- and by the time exit handlers run those -+ // may already be gone, so destroying the interface then walks freed state. -+ // Order mirrors construction. Nothing below may use `simif`. -+ simulation.reset(); -+ simulator.reset(); -+ return rc; - } diff --git a/hw/syn/firesim/patches/README b/hw/syn/firesim/patches/README deleted file mode 100644 index 4e696af09c..0000000000 --- a/hw/syn/firesim/patches/README +++ /dev/null @@ -1,104 +0,0 @@ -Patches against upstream FireSim (tag 1.21.0) required to build and run a -FireSim simulator on the Alveo U55C under Vitis/Vivado 2024.2. - -The paths in these patches are relative to the Vortex repository root, where -the FireSim clone sits at build_firesim/firesim/. Applying them from the root -of a FireSim checkout therefore strips two components, not one: - - cd - patch -p2 < 0001-rebase-target-cycle-counter.patch - patch -p2 < 0002-vitis-u55c-build-fixes.patch - patch -p2 < 0003-vitis-memcheck-and-buffer-lifetime.patch - patch -p2 < 0004-vitis-host-mem-base-address.patch - patch -p2 < 0005-inflight-target-cycle.patch - patch -p2 < 0006-package-design-sources.patch - patch -p2 < 0007-vitis-target-dram-aperture.patch - patch -p2 < 0008-driver-teardown-before-exit.patch - -Apply in order: 0003 and 0005 both touch simulation.{h,cc} and assume 0001 is -already in. Applying the series in order to a pristine 1.21.0 checkout -reproduces this tree's clone byte-for-byte. - -0001-rebase-target-cycle-counter.patch - Correctness fix. The target-cycle counter lives in FPGA fabric and - persists while the bitstream stays programmed, but simulation_t reported - it as an absolute value. Every driver invocation against a resident - xclbin therefore inherited the previous run's count -- observed as - 13, 26, 39, ... cycles of drift across successive runs of the same test, - with FMR and Effective Target Frequency corrupted alongside it. Host - cycles were already rebased in the same function; target cycles now are - too. Host-side only, no bitstream rebuild needed. - -0002-vitis-u55c-build-fixes.patch - Build-flow fixes. Upstream's Vitis flow targets Vitis 2022.1 and a U250; - neither assumption holds here. See the patch header for the four - independent failures and their causes. - -0003-vitis-memcheck-and-buffer-lifetime.patch - Correctness fix plus the diagnostic that settled the FASED-on-HBM - question. The xrt::bo backing host_mem_0 was a constructor local, so its - device allocation was freed before the simulation ran. +memcheck - round-trips a pattern through the LoadMem widget before target execution, - isolating the target DRAM path from the design under test; on the U55C it - reports an address-independent constant, unchanged by zeroing DRAM -- - i.e. the HBM memory path never accesses memory, while the MMIO/control - plane is fully functional. See - docs/proposals/firesim_vitis_host_mem_offset.md. - -0004-vitis-host-mem-base-address.patch - The fix that makes FASED work on an HBM part. VitisShim added a literal - 0x4000000000 (U250 DDR[0] base) to every host_mem_0 address in RTL. On the - U55C, host_mem_0 binds to HBM[0] at 0x0 with a 512 MB aperture, so every - access landed outside the window: reads returned a fixed bus default - independent of address, writes were dropped, and the target DRAM path was - dead while the control plane still worked. Now a documented constant in - VitisConstants. Validated with +memcheck under hw_emu: 5/5 probes fail - before, 0/5 after. - -0005-inflight-target-cycle.patch - Correctness fix on top of 0001. The clock bridge counts target cycles - *started*, so one cycle is always in flight; it cancels between the two - samples except across the counter's origin, leaving the first run after - programming one cycle longer than every run after it (14 vs 13 for a test - that provably steps 13 times). A fresh metasim is always the origin case, - so upstream reports the +1 on every metasim run. See - docs/proposals/firesim_inflight_target_cycle.md. - -0006-package-design-sources.patch - Build-flow fix. A design whose RTL is resolved by library search in - simulation loses its submodules under Vivado, which has no equivalent. - VORTEX_VIVADO_SRCS names a flattened source list to add to the packaged - kernel; absent it the flow is unchanged. - -0007-vitis-target-dram-aperture.patch - Correctness fix. host_mem_0 was bound to a single HBM pseudo-channel and - backed by a 512 MB allocation, while a 32-bit target spans 4 GB -- its - program image sits at 0x80000000 and its stack just under 4 GB. Uploading a - kernel therefore wrote outside the served window and the target fetched - memory that was never written, with no error anywhere to say so. Widened to - HBM[0:7] plus a matching 4 GiB window; the two are a pair and each carries a - comment saying widening one alone truncates silently. The buffer is now - allocated from whichever memory region can hold the window rather than from - bank 0 -- a range binding appears as a memory *group* (MBG[0:7], index 39 on - this platform) whose index cannot be derived from the port name, and - xrt::ip exposes no group_id(). +memcheck gains probes at 0x20000000, - 0x80000000 and 0xFFFF0000; it passed throughout this defect because it only - probed below 1 MB, which is how the bug reached the card. No Chisel change: - HostMemChannelKey.size is already 16 GiB, so the shim's address path was - never the constraint. See docs/proposals/firesim_target_memory_aperture.md. - -0008-driver-teardown-before-exit.patch - Correctness fix. `simulator` is a global std::unique_ptr, so the platform - interface was destroyed during static destruction -- and on Vitis that runs - xrt::device::~device() after XRT's own statics are gone, segfaulting in the - device registry's deleter. Upstream's comment says the global "must exist - for dpi.cc calls in emulation, must be global to avoid deletion", but a - global unique_ptr *is* deleted at exit; nothing references it once entry() - returns, on either platform. Now torn down explicitly at the end of entry(). - Never observed before because no run had ever reached normal process exit -- - every earlier one timed out. - -All are candidates for upstreaming: only the HOST_MEM_SP default is -U55C-specific, and it is parameterized rather than hardcoded. 0005 changes a -reported number relative to upstream; the patch header records why the new -number is the correct one. diff --git a/hw/syn/xilinx/platforms/v80.toml b/hw/syn/xilinx/platforms/v80.toml index ae66e2c1fc..92b8e0a6a1 100644 --- a/hw/syn/xilinx/platforms/v80.toml +++ b/hw/syn/xilinx/platforms/v80.toml @@ -41,4 +41,4 @@ kernel_mhz = 300 [firesim] platform = "aved" -host_mem_tag = "MEM" +host_mem_sp = "MEM" diff --git a/sim/firesim/Makefile b/sim/firesim/Makefile index 83cc74033a..b6bb7a4833 100644 --- a/sim/firesim/Makefile +++ b/sim/firesim/Makefile @@ -8,10 +8,18 @@ PROJECT := firesim SIMIF_DIR := $(FIRESIM_PATH)/sim/midas/src/main/cc -PLATFORM ?= vitis DESIGN ?= VortexTarget TARGET_CONFIG ?= VortexConfig -PLATFORM_CONFIG ?= Default$(shell echo $(PLATFORM) | sed 's/^./\U&/')Config + +# The card's FireSim platform and platform config come from its descriptor, the +# same ones hw/syn/firesim elaborated the target with. +FIRESIM_CARD ?= u55c +FIRESIM_MK := $(DESTDIR)/firesim_platform.mk +FIRESIM_GEN_RC := $(shell mkdir -p $(DESTDIR) && python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk -o $(FIRESIM_MK) 2> $(FIRESIM_MK).err; echo $$?) +ifneq ($(FIRESIM_GEN_RC),0) +$(error $(shell cat $(FIRESIM_MK).err)) +endif +include $(FIRESIM_MK) # Golden Gate writes the generated header here; the driver objects are built # alongside it so a rebuild of the target invalidates them. @@ -27,10 +35,14 @@ DRIVER_SRCS := $(SRC_DIR)/firesim_sim.cpp # -fPIC is required for every object that lands in the shared library, so it # has to reach the bridge and platform objects too, not just ours. SIM_CXXFLAGS := -O2 -fPIC -std=c++20 -I$(SRC_DIR) \ - -idirafter $(CONDA_PREFIX)/include -idirafter /usr/include \ - -idirafter $(XILINX_XRT)/include -SIM_LDFLAGS := -shared -L$(CONDA_PREFIX)/lib -L$(XILINX_XRT)/lib \ - -luuid -lxrt_coreutil -ldl -lz + -idirafter $(CONDA_PREFIX)/include -idirafter /usr/include +SIM_LDFLAGS := -shared -L$(CONDA_PREFIX)/lib -ldl -lz + +# The platform layer's own library: XRT for vitis cards. +ifeq ($(PLATFORM), vitis) +SIM_CXXFLAGS += -idirafter $(XILINX_XRT)/include +SIM_LDFLAGS += -L$(XILINX_XRT)/lib -luuid -lxrt_coreutil +endif .PHONY: all lib clean check-gen From 62e42ecc34c6dcfb933168a2887b52905d3df657 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 21 Sep 2026 17:49:54 -0700 Subject: [PATCH 146/336] firesim: run on the Alveo V80 through the fork's aved platform - The runtime names a FireSim run's image and card with the variables of the card's native driver: XCLBIN_PATH on vitis cards; VRT_VBIN_PATH and VRT_DEVICE_BDF on the V80, with the card discovered on the bus when the BDF is unset and an unset image an error (the V80 has no metasimulation). The transport takes the platform's plusargs rather than one image path. - V80 discovery moves to sw/common/v80_pcie.h, shared by the aved and firesim backends. - sim/firesim links VRT for the V80, built by the host toolchain; the runtime and run-firesim take FIRESIM_CARD and resolve its platform from the descriptor. - The firesim backend gains host_mem_pull/push, no-ops because its host regions are this process's memory. It had been missing them since the HAL added the pair, so the backend did not compile. - The generator reads descriptors with gen_config.py's TOML reader: FireSim's conda environment is Python 3.10, before tomllib. - FIRESIM_REV moves to the fork commit that adds the aved platform. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit a30a0f201befb6b301e2e71235c852390751e19b) --- VERSION | 2 +- docs/designs/firesim_integration.md | 26 +++++++++++++ hw/scripts/gen_platform.py | 15 +++++--- hw/syn/xilinx/platforms/v80.toml | 1 + sim/firesim/Makefile | 11 +++++- sim/firesim/firesim_sim.cpp | 10 +++-- sim/firesim/firesim_sim.h | 9 +++-- sw/common/v80_pcie.h | 54 ++++++++++++++++++++++++++ sw/runtime/aved/vortex.cpp | 36 +---------------- sw/runtime/firesim/Makefile | 13 ++++++- sw/runtime/firesim/vortex.cpp | 60 +++++++++++++++++++++++------ tests/regression/common.mk | 13 ++++++- 12 files changed, 186 insertions(+), 64 deletions(-) create mode 100644 sw/common/v80_pcie.h diff --git a/VERSION b/VERSION index 177bead6ef..f1c3ef40e3 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=4d4b0f9521313f3043114460c338b368b8506391 +FIRESIM_REV=ec0d464c11278a20e62fc1c7c307358a03f2e321 diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index d5abee0f2f..a3885cd9d6 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -255,6 +255,32 @@ overrides as the other synthesis flows do; extensions pull in their sources through `hw/syn/extensions.mk`, and a changed `CONFIGS` invalidates the flattened tree. +### 4.0 Cards and platforms + +| card | FireSim platform | image | run with | +|---|---|---|---| +| `u55c` | `vitis` (XRT, `v++`) | `platforms/vitis/cl_/bitstream/build_dir./firesim.xclbin` | `XCLBIN_PATH` | +| `v80` | `aved` (VRT, `slashkit`) | `platforms/aved/cl_/bitstream/build_dir.hw/firesim.vbin` | `VRT_VBIN_PATH`, `VRT_DEVICE_BDF` (discovered when unset) | + +A FireSim run names its image and card with the variables of the card's native +driver. On the V80 the Xilinx tools are the ones `slashkit` links with +(`/opt/xilinx/2025.2`), so the floating-point cores must be generated in that +environment too; and the static shell must be in flash, as for the native +`aved` backend. + +The `aved` platform (`AvedShim`) differs from `vitis` in three ways that matter +to anyone reading its traces: + +- **No MMCM.** The shell programs the kernel clock to the frequency the link + records (`FREQUENCY`), and the simulator runs on it directly. +- **Control registers at a 16-byte stride.** The compute shell resolves AXI-Lite + in 16-byte blocks, and FireSim's registers have read side effects (LoadMem + `R_DATA` dequeues), so the shim gives each register its own block and answers + the padding words itself. The driver scales addresses by 4. +- **A transport gate at open.** The driver reads SimulationMaster's fingerprint, + writes and reads back a probe, and checks the memory port is bound to the HBM + VNOC at the elaborated base, before any widget touches the device. + ### 4.1 Two source lists, and why Verilator resolves Vortex by **library search**, so `sources` names only the diff --git a/hw/scripts/gen_platform.py b/hw/scripts/gen_platform.py index c0e06810cf..daf200e542 100644 --- a/hw/scripts/gen_platform.py +++ b/hw/scripts/gen_platform.py @@ -32,12 +32,17 @@ import fnmatch import os import sys -import tomllib from dataclasses import dataclass, field from typing import List, Optional -PLATFORM_DIR = os.path.normpath( - os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "syn", "xilinx", "platforms")) +_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) +PLATFORM_DIR = os.path.join(_ROOT, "hw", "syn", "xilinx", "platforms") + +# The configuration generator's TOML reader: these builds run under whatever +# python3 the tool environment provides (FireSim's conda env is 3.10, before +# tomllib), and descriptors stay within the subset it reads. +sys.path.insert(0, os.path.join(_ROOT, "ci")) +from gen_config import TinyTOML # noqa: E402 FLOWS = ("xrt", "aved") @@ -97,8 +102,8 @@ def _require(d: dict, key: str, path: str, ctx: str): def load_card(path: str) -> Card: - with open(path, "rb") as f: - t = tomllib.load(f) + with open(path, "r", encoding="utf-8") as f: + t = TinyTOML().parse(f.read()) card = _require(t, "card", path, "card") mem = _require(t, "memory", path, "memory") host = t.get("host", {}) diff --git a/hw/syn/xilinx/platforms/v80.toml b/hw/syn/xilinx/platforms/v80.toml index 92b8e0a6a1..0004272258 100644 --- a/hw/syn/xilinx/platforms/v80.toml +++ b/hw/syn/xilinx/platforms/v80.toml @@ -41,4 +41,5 @@ kernel_mhz = 300 [firesim] platform = "aved" +device = "xcv80-lsva4737-2MHP-e-S" host_mem_sp = "MEM" diff --git a/sim/firesim/Makefile b/sim/firesim/Makefile index b6bb7a4833..77be235d7d 100644 --- a/sim/firesim/Makefile +++ b/sim/firesim/Makefile @@ -38,10 +38,16 @@ SIM_CXXFLAGS := -O2 -fPIC -std=c++20 -I$(SRC_DIR) \ -idirafter $(CONDA_PREFIX)/include -idirafter /usr/include SIM_LDFLAGS := -shared -L$(CONDA_PREFIX)/lib -ldl -lz -# The platform layer's own library: XRT for vitis cards. +# The platform layer's own library: XRT for vitis cards, VRT for the V80. VRT +# is built against the host glibc and libstdc++, so its platform layer is +# compiled by the host toolchain, and its headers need jsoncpp's and libxml2's. ifeq ($(PLATFORM), vitis) SIM_CXXFLAGS += -idirafter $(XILINX_XRT)/include SIM_LDFLAGS += -L$(XILINX_XRT)/lib -luuid -lxrt_coreutil +else ifeq ($(PLATFORM), aved) +SIM_CXX := /usr/bin/g++ +SIM_CXXFLAGS += $(shell pkg-config --cflags jsoncpp libxml-2.0) +SIM_LDFLAGS += -L/usr/lib/x86_64-linux-gnu -lvrt endif .PHONY: all lib clean check-gen @@ -76,7 +82,8 @@ lib: check-gen $(PIC_DIR)/FireSim-generated.const.h OUT_DIR=$(DESTDIR) \ DRIVER="$(DRIVER_SRCS)" \ CXXFLAGS="$(SIM_CXXFLAGS)" \ - LDFLAGS="$(SIM_LDFLAGS)" + LDFLAGS="$(SIM_LDFLAGS)" \ + $(if $(SIM_CXX),CXX=$(SIM_CXX)) mv -f $(DESTDIR)/lib$(PROJECT).so.tmp-$(PLATFORM) $(DESTDIR)/lib$(PROJECT).so clean: diff --git a/sim/firesim/firesim_sim.cpp b/sim/firesim/firesim_sim.cpp index 9604c3a01e..29b7dd395b 100644 --- a/sim/firesim/firesim_sim.cpp +++ b/sim/firesim/firesim_sim.cpp @@ -365,12 +365,12 @@ class firesim_sim::Impl { g_channel = nullptr; } - int init(const char *bitstream) { + int init(const std::vector &plusargs) { g_channel = &channel_; argv_storage_.emplace_back("firesim"); - if (bitstream != nullptr && *bitstream != '\0') { - argv_storage_.emplace_back(std::string("+binary_file=") + bitstream); + for (const auto &arg : plusargs) { + argv_storage_.push_back(arg); } for (auto &s : argv_storage_) { argv_.push_back(const_cast(s.c_str())); @@ -562,7 +562,9 @@ void firesim_sim::report_core_counters() { firesim_sim::~firesim_sim() { delete impl_; } -int firesim_sim::init(const char *bitstream) { return impl_->init(bitstream); } +int firesim_sim::init(const std::vector &plusargs) { + return impl_->init(plusargs); +} int firesim_sim::dcr_write(uint32_t addr, uint32_t value) { return impl_->submit(label("dcr_write addr=0x%x data=0x%x", addr, value), [addr, value] { diff --git a/sim/firesim/firesim_sim.h b/sim/firesim/firesim_sim.h index cd3bd1d54f..d7f7b67577 100644 --- a/sim/firesim/firesim_sim.h +++ b/sim/firesim/firesim_sim.h @@ -14,6 +14,8 @@ #pragma once #include +#include +#include namespace vortex { @@ -32,9 +34,10 @@ class firesim_sim { virtual ~firesim_sim(); // Starts the simulator thread and holds the target in reset until it is - // ready to accept requests. `bitstream` selects the FPGA image and is - // ignored by a metasimulation. - int init(const char* bitstream); + // ready to accept requests. `plusargs` go to the platform layer as given: + // the image (+binary_file=...) and whatever else the platform needs to find + // its card; a metasimulation takes none. + int init(const std::vector& plusargs); int mem_write(uint64_t addr, uint64_t size, const void* value); diff --git a/sw/common/v80_pcie.h b/sw/common/v80_pcie.h new file mode 100644 index 0000000000..2fb7b7e79a --- /dev/null +++ b/sw/common/v80_pcie.h @@ -0,0 +1,54 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +namespace vortex { + +// Finds an AMD Alveo V80 the way XRT enumerates by index: scans sysfs for the +// SLASH control function (10ee:50c1) and returns its bus:device in the "01:00" +// form VRT takes. Empty when no card is on the bus. +inline std::string discover_v80_bdf() { + DIR* d = opendir("/sys/bus/pci/devices"); + if (d == nullptr) { + return ""; + } + std::string found; + while (struct dirent* e = readdir(d)) { + const std::string name = e->d_name; // "0000:01:00.1" + if (name.size() < 12 || name[0] == '.') { + continue; + } + auto read_hex = [&](const char* leaf) -> unsigned { + std::ifstream f("/sys/bus/pci/devices/" + name + "/" + leaf); + unsigned v = 0; + f >> std::hex >> v; + return v; + }; + if (read_hex("vendor") == 0x10ee && read_hex("device") == 0x50c1) { + found = name.substr(0, name.rfind('.')); // drop the function + if (found.rfind("0000:", 0) == 0) { + found = found.substr(5); // "01:00" form + } + break; + } + } + closedir(d); + return found; +} + +} // namespace vortex diff --git a/sw/runtime/aved/vortex.cpp b/sw/runtime/aved/vortex.cpp index 08d431c41a..2be4583213 100644 --- a/sw/runtime/aved/vortex.cpp +++ b/sw/runtime/aved/vortex.cpp @@ -58,6 +58,7 @@ #include #include #include +#include using namespace vortex; @@ -196,46 +197,13 @@ class vx_device { #endif } - // Auto-discover the card the way XRT enumerates by index: scan sysfs for - // the SLASH control function (10ee:50c1) and derive its BDF. Makes - // VRT_DEVICE_BDF optional -- the hardcoded default below is only the last - // resort when no card is on the bus (e.g. a sim run with no env set). - static std::string discover_bdf() { - DIR* d = opendir("/sys/bus/pci/devices"); - if (d == nullptr) { - return ""; - } - std::string found; - while (struct dirent* e = readdir(d)) { - const std::string name = e->d_name; // "0000:01:00.1" - if (name.size() < 12 || name[0] == '.') { - continue; - } - auto read_hex = [&](const char* leaf) -> unsigned { - std::ifstream f("/sys/bus/pci/devices/" + name + "/" + leaf); - unsigned v = 0; - f >> std::hex >> v; - return v; - }; - if (read_hex("vendor") == 0x10ee && read_hex("device") == 0x50c1) { - found = name.substr(0, name.rfind('.')); // drop the function - if (found.rfind("0000:", 0) == 0) { - found = found.substr(5); // "01:00" form - } - break; - } - } - closedir(d); - return found; - } - int init() { // An empty value is treated as unset: the test harness exports these // unconditionally, so they arrive empty rather than absent. const char* bdf = getenv("VRT_DEVICE_BDF"); std::string bdf_storage; if (bdf == nullptr || bdf[0] == '\0') { - bdf_storage = discover_bdf(); + bdf_storage = discover_v80_bdf(); if (!bdf_storage.empty()) { fprintf(stderr, "[VXDRV] auto-discovered V80 at %s (set " "VRT_DEVICE_BDF to override)\n", bdf_storage.c_str()); diff --git a/sw/runtime/firesim/Makefile b/sw/runtime/firesim/Makefile index e7dbbac418..9a24d761ac 100644 --- a/sw/runtime/firesim/Makefile +++ b/sw/runtime/firesim/Makefile @@ -15,6 +15,17 @@ CXXFLAGS += -fPIC # Add external configuration CXXFLAGS += $(CONFIGS) +# The card's FireSim platform, from its descriptor: it decides which variables +# name the image and the card (those of the card's native driver). +FIRESIM_CARD ?= u55c +FIRESIM_PLATFORM := $(shell python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk | sed -n 's/^PLATFORM := //p') +ifeq ($(FIRESIM_PLATFORM),) +$(error cannot resolve the FireSim platform of card '$(FIRESIM_CARD)') +endif +ifeq ($(FIRESIM_PLATFORM), aved) +CXXFLAGS += -DFIRESIM_PLATFORM_AVED +endif + LDFLAGS += -shared -pthread # Find libfiresim.so siblings at runtime in the same dir libvortex-firesim.so lives in. LDFLAGS += -Wl,-rpath,'$$ORIGIN' @@ -39,7 +50,7 @@ all: $(DESTDIR)/$(PROJECT) driver: $(DESTDIR)/libfiresim.so $(DESTDIR)/libfiresim.so: force - DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib FIRESIM_CARD=$(FIRESIM_CARD) $(DESTDIR)/$(PROJECT): $(SRCS) $(DESTDIR)/libfiresim.so $(CONFIG_FILE) $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ diff --git a/sw/runtime/firesim/vortex.cpp b/sw/runtime/firesim/vortex.cpp index 6c31af7458..dd2dcc13f8 100644 --- a/sw/runtime/firesim/vortex.cpp +++ b/sw/runtime/firesim/vortex.cpp @@ -30,11 +30,14 @@ #include #include #include +#include #include #include #include #include +#include +#include #include using namespace vortex; @@ -52,18 +55,41 @@ class vx_device { } int init() { - // The bitstream is selected by the environment so that the same runtime - // serves a metasimulation and an FPGA without a rebuild. XCLBIN_PATH is - // the same variable the xrt driver reads: both consume a Vitis image, and - // a caller switching between them should not have to switch variable names - // as well. - // - // Unset falls through as empty rather than to a default filename, unlike - // the xrt driver. An empty path tells the simulator to omit +binary_file, - // which is what metasimulation needs -- it has no image to load, and naming - // one it cannot open would turn a working configuration into a failure. - const char* bitstream = getenv("XCLBIN_PATH"); - return sim_.init(bitstream != nullptr ? bitstream : ""); + // The image and card are named by the same variables the card's native + // driver reads, so switching between a native and a FireSim run does not + // mean switching variable names as well. + std::vector plusargs; +#ifdef FIRESIM_PLATFORM_AVED + // As the aved driver: VRT_VBIN_PATH and VRT_DEVICE_BDF, with the card + // discovered on the bus when the BDF is unset. There is no metasimulation + // on this platform, so an image is required. + const char* vbin = getenv("VRT_VBIN_PATH"); + if (vbin == nullptr || vbin[0] == '\0') { + fprintf(stderr, "[VXDRV] VRT_VBIN_PATH must name the FireSim .vbin\n"); + return -1; + } + const char* bdf_env = getenv("VRT_DEVICE_BDF"); + std::string bdf = (bdf_env != nullptr) ? bdf_env : ""; + if (bdf.empty()) { + bdf = discover_v80_bdf(); + if (bdf.empty()) { + fprintf(stderr, "[VXDRV] no V80 found on the bus; set VRT_DEVICE_BDF\n"); + return -1; + } + } + plusargs.push_back(std::string("+binary_file=") + vbin); + plusargs.push_back("+bdf=" + bdf); +#else + // As the xrt driver: XCLBIN_PATH. Unset falls through as no image rather + // than a default filename: an empty path is what metasimulation needs -- + // it has no image to load, and naming one it cannot open would turn a + // working configuration into a failure. + const char* xclbin = getenv("XCLBIN_PATH"); + if (xclbin != nullptr && xclbin[0] != '\0') { + plusargs.push_back(std::string("+binary_file=") + xclbin); + } +#endif + return sim_.init(plusargs); } // ----- CP register channel ----- @@ -109,6 +135,16 @@ class vx_device { return 0; } + // Host regions are this process's memory, which the CP model reads and + // writes directly: coherent, so there is nothing to sync. + int host_mem_pull(uint64_t /*cp_addr*/) { + return 0; + } + + int host_mem_push(uint64_t /*cp_addr*/) { + return 0; + } + private: // If `addr` falls in a registered host region, return it as a host // pointer (cp_addr == the pointer); otherwise nullptr → device memory. diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 25ffaf4462..bfe4fe387f 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -242,9 +242,18 @@ run-rtlsim: $(PROJECT) kernel.vxbin $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/rtlsim DESTDIR=$(VORTEX_RT_LIB) LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=rtlsim ./$(PROJECT) $(OPTS) +# FireSim takes its card from the descriptor (FIRESIM_CARD) and names the image +# and card with the variables of the card's native driver. +FIRESIM_CARD ?= u55c +ifneq ($(filter run-firesim,$(MAKECMDGOALS)),) +FIRESIM_PLATFORM := $(shell python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk | sed -n 's/^PLATFORM := //p') +endif + run-firesim: $(PROJECT) kernel.vxbin - $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) -ifeq ($(TARGET), hw) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) FIRESIM_CARD=$(FIRESIM_CARD) +ifeq ($(FIRESIM_PLATFORM), aved) + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/firesim.vbin LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(TARGET), hw) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) else XCL_EMULATION_MODE=$(TARGET) EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) From 8038bbc02277eda56e5c0f5cdbb6bb2e70c63b4c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 21 Sep 2026 22:38:52 -0700 Subject: [PATCH 147/336] firesim: run the async-BRAM pre-opt hook from the bitstream target bitstream passes VORTEX_PRE_OPT_TCL and TOOL_DIR to the card's platform build, as kernel-env does, and FIRESIM_REV moves to the fork commit that honors the hook on both the Vitis and aved platforms. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 0908a74c53bfbe95f6846d542be202467d8cc7ca) --- VERSION | 2 +- hw/syn/firesim/Makefile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f1c3ef40e3..bba7107113 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=ec0d464c11278a20e62fc1c7c307358a03f2e321 +FIRESIM_REV=dbff316d22edfd82cbf2d58958df45ec50585670 diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 5a263d1d69..ea80f15347 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -259,6 +259,7 @@ kernel-env: bitstream: vivado-sources sources xilinx-ips stage $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ + VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl TOOL_DIR=$(SCRIPT_DIR) \ $(MAKE) -C $(FIRESIM_PATH)/platforms/$(PLATFORM)/cl_$(QUINTUPLET) bitstream \ DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) From 5b7a71450174ca509f7e0638f8f528759232c225 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 22 Sep 2026 02:47:55 -0700 Subject: [PATCH 148/336] firesim: pin the fork commit that repackages vendor cores cleanly Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit b65229f81b12aa18af538eff26bc9a2862e560c3) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bba7107113..fd73291743 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=dbff316d22edfd82cbf2d58958df45ec50585670 +FIRESIM_REV=5538841908212378e9f27dbc03403d8eada44d52 From 2af3800fae4eeb49331a127cec8f3f903b7bc9d8 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 22 Sep 2026 03:20:35 -0700 Subject: [PATCH 149/336] firesim: pin the fork commit that gates V80 host clocks in fabric Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit f807e927ab8d2a0c966720e4878081f3494bc4e2) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fd73291743..1ef5182277 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=5538841908212378e9f27dbc03403d8eada44d52 +FIRESIM_REV=c82359a76fe4616a303e193e44a7c6ac33762480 From 94cca1ee26be37afc26ff5ddf89cd427c319597d Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 22 Sep 2026 18:42:02 -0700 Subject: [PATCH 150/336] docs: correct the V80 setup, shell and FireSim platform docs - xilinx_slash_setup: install ami rather than excluding it. PF0 carries the AMC, so without it the shell cannot be flashed and the board cannot be recovered; 'PF0 NOT READY' is an incomplete install, not a cosmetic warning. Adds the missing-root-port recovery, which no rescan can fix. - fpga_afu_shell, command_processor: the shell itself lives in afu/common and is shared by the xrt and aved kernel tops; only the top differs. - aved_driver_architecture: point at the card descriptor rather than the deleted platforms.mk, and drop the CP software reset from open directions -- q_reset_pulse is wired (see the same document's own section 3.5). - firesim_integration: the V80's fabric clock gates, and the two things a platform build must do for Vortex (the async-BRAM pre-opt hook and the vendor-core packaging). Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit fcb6f23f90829e1a7999e06a62af88d4d7b9140f) --- docs/designs/aved_driver_architecture.md | 17 ++++----- docs/designs/command_processor.md | 3 +- docs/designs/firesim_integration.md | 33 +++++++++++++++++ docs/designs/fpga_afu_shell.md | 22 +++++++----- docs/xilinx_slash_setup.md | 45 ++++++++++++++++++++---- 5 files changed, 95 insertions(+), 25 deletions(-) diff --git a/docs/designs/aved_driver_architecture.md b/docs/designs/aved_driver_architecture.md index 9f8715e495..748a9ef2be 100644 --- a/docs/designs/aved_driver_architecture.md +++ b/docs/designs/aved_driver_architecture.md @@ -258,7 +258,7 @@ refused with `CTL_RESET_ERROR`, correctly, because that read really is still outstanding; it reads like a reset bug and is not one. This happened on 2026-08-27 and cost a bitstream and three reboots. The cause -was `platforms.mk` pinning `MEM_TAG` but not `HOST_TAG`, leaving the Makefile's +was the card description pinning `MEM_TAG` but not `HOST_TAG`, leaving the Makefile's `HOST_TAG ?= HOST` to win. Both are pinned now, and `HOST_TAG=HOST` is refused at Makefile parse time rather than after two and a half hours of Vivado. Before blaming a hardware symptom on RTL, diff the build's generated `config.cfg` @@ -391,15 +391,11 @@ threads. `xrt` and `aved` both guard it. ## 8. Open directions -1. **A software reset for the CP.** `q_reset_pulse` is decoded in the regfile - and discarded in `VX_cp_core`. Wiring it would remove the §3.5 workaround - and make a hung kernel recoverable without reconfiguring the partition — - today a kernel that never completes leaves the CP unusable. -2. **Unify the two sync engines** (§4.3) behind one publish/refresh interface +1. **Unify the two sync engines** (§4.3) behind one publish/refresh interface with a pluggable transport. -3. **Drop the VRT-internal reach** once VRT exposes the simulated memory +2. **Drop the VRT-internal reach** once VRT exposes the simulated memory windows and the ZMQ server through public API. -4. **Gate the build on timing closure.** The V80 flow packages a bitstream +3. **Gate the build on timing closure.** The V80 flow packages a bitstream that does not meet timing without saying so. --- @@ -425,5 +421,6 @@ loopback (`minimal -l`) remains the sim-side check of the CP DMA path. Simulated host buffers are drawn from VRT's DDR window, not its HBM window, so they cannot collide with device memory. The authoritative settings and -their rationale are in -[`platforms.mk`](../../hw/syn/xilinx/aved/platforms.mk). +their rationale are in the card descriptor +[`v80.toml`](../../hw/syn/xilinx/platforms/v80.toml), which both the `aved` and +`xrt` flows read through [`gen_platform.py`](../../hw/scripts/gen_platform.py). diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index e366c9794c..2283d89f5a 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -698,7 +698,8 @@ host-side instead. They become usable once queues get independent rings. ### 9.1 FPGA backends -- **XRT** ([`hw/rtl/afu/xrt/VX_afu_wrap.sv`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv)) — +- **XRT and AVED** ([`hw/rtl/afu/common/VX_afu_wrap.sv`](../../hw/rtl/afu/common/VX_afu_wrap.sv), + wrapped by `afu/xrt/vortex_afu.v` and `afu/aved/vortex_afu.v`) — AXI-Lite is split on bit 12: `0x000–0x0FFF` → a minimal `VX_afu_ctrl` ap_ctrl stub + SCOPE; `0x1000–0x1FFF` → the CP regfile. The legacy launch FSM, DCR path, and dev-caps were **removed**; CP is the sole diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index a3885cd9d6..a97b1bc4e2 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -280,6 +280,39 @@ to anyone reading its traces: - **A transport gate at open.** The driver reads SimulationMaster's fingerprint, writes and reads back a probe, and checks the memory port is bound to the HBM VNOC at the elaborated base, before any widget touches the device. +- **Clock gates built from fabric.** FireSim gates the target clock with a + `BUFGCE` per clock domain, and the compute shell's reconfigurable partition + contains no `BUFGCE` site — only `BUFG_FABRIC`. Implementation fails at + `opt_design` with `DRC HDPR-18` ("requires resource type 'BUFGCE_HDIO' which + is not ranged by 'pblock_slash'"), and the partition is fixed by the static + shell, so no link option can widen it. `FabricClockGates` (set by + `AvedConfig`) makes Golden Gate emit `FireSimFabricClockGate` instead: the + enable passes through a latch that is transparent only while the clock is low + — so it can change the output only while the clock is low — and the gated + clock is put back on a global net by a `BUFG_FABRIC`. It synthesizes to + `LDCE` + `LUT2` + `BUFG_FABRIC`. Simulation keeps the `BUFGCE` model, and + other platforms keep `BUFGCE` itself. + +### 4.0.1 What the platform build must do for Vortex + +Both cards share two requirements that FireSim's own targets do not have, so +they are passed in from `hw/syn/firesim`: + +- **`VORTEX_PRE_OPT_TCL` — the async-BRAM patch.** `VX_async_ram_patch` leaves + `VX_placeholder` black boxes wherever an asynchronous-read RAM has to live in + block RAM, and a post-synthesis script fills them from the netlist. Without it + `opt_design` aborts with `DRC INBB-3`, but only once a configuration is large + enough to push those RAMs out of LUTRAM — so small builds pass and large ones + do not. `vitis` attaches the hook with `--xp vivado_prop:...OPT_DESIGN.TCL.PRE`; + `aved` attaches it from a pre-synthesis hook, the only place slashkit lets a + step property be set. +- **`VORTEX_XILINX_IP_DIR` — the vendor cores.** The FPU and RTU instantiate + Xilinx floating-point IP (`xil_fma`, `xil_fdiv`, …), which must be generated + for the card's part and attached to the packaged kernel's file groups; read + into the packaging project they are dropped as unreferenced (`IP_Flow + 19-3833`). They are copied into the kernel, each into a fresh directory — + Tcl's `file copy` onto an existing directory nests the source inside it, which + silently leaves a repackaged kernel pointing at the previous run's cores. ### 4.1 Two source lists, and why diff --git a/docs/designs/fpga_afu_shell.md b/docs/designs/fpga_afu_shell.md index c4c9760d71..4556c9a93d 100644 --- a/docs/designs/fpga_afu_shell.md +++ b/docs/designs/fpga_afu_shell.md @@ -1,9 +1,12 @@ -# AFU Shell (XRT / OPAE) — Design +# AFU Shell (XRT / AVED / OPAE) — Design **Scope:** the FPGA acceleration-function-unit (AFU) shells that adapt -Vortex to a host platform — the Xilinx/XRT shell -([`hw/rtl/afu/xrt/`](../../hw/rtl/afu/xrt/)) and the Intel/OPAE shell -([`hw/rtl/afu/opae/`](../../hw/rtl/afu/opae/)). This document covers the +Vortex to a host platform. The shell proper is shared +([`hw/rtl/afu/common/`](../../hw/rtl/afu/common/)); what differs per platform +is the kernel top that wraps it — Xilinx/XRT +([`hw/rtl/afu/xrt/`](../../hw/rtl/afu/xrt/)), AMD/AVED for the Alveo V80 +([`hw/rtl/afu/aved/`](../../hw/rtl/afu/aved/)) — plus the Intel/OPAE shell +([`hw/rtl/afu/opae/`](../../hw/rtl/afu/opae/)), which is separate. This document covers the **shell structure** (platform glue, memory adaptation, control/reset/ discovery contracts, and the XRT↔OPAE asymmetries). The Command Processor integration these shells host is documented in @@ -17,14 +20,17 @@ DMA-command engine have been removed. --- -## 1. XRT shell +## 1. XRT and AVED shells + +The two share every file below except the kernel top: `afu/xrt/vortex_afu.v` +is the Vitis RTL-kernel top and `afu/aved/vortex_afu.v` the SLASH one. | File | Role | |---|---| | [`vortex_afu.v`](../../hw/rtl/afu/xrt/vortex_afu.v) | Vitis RTL-kernel top — thin wrapper instantiating `VX_afu_wrap`. | -| [`vortex_afu.vh`](../../hw/rtl/afu/xrt/vortex_afu.vh) | Defines/macros (`GEN_AXI_MEM`, `GEN_AXI_HOST`, the bit-12 window). | +| [`vortex_afu.vh`](../../hw/rtl/afu/common/vortex_afu.vh) | Defines/macros (`GEN_AXI_MEM`, `GEN_AXI_HOST`, the bit-12 window). | | [`VX_afu_wrap.sv`](../../hw/rtl/afu/common/VX_afu_wrap.sv) | The real shell (~1,200 LOC): AXI-Lite bit-12 demux, `VX_cp_core`, `m_axi_host`, the Vortex instance and the device-memory path shared with the CP. | -| [`VX_afu_ctrl.sv`](../../hw/rtl/afu/xrt/VX_afu_ctrl.sv) | Slimmed AXI-Lite slave (~322 LOC): `ap_ctrl` stub at 0x00 + a SCOPE serial register pair + SCOPE watchdog. | +| [`VX_afu_ctrl.sv`](../../hw/rtl/afu/common/VX_afu_ctrl.sv) | Slimmed AXI-Lite slave (~322 LOC): `ap_ctrl` stub at 0x00 + a SCOPE serial register pair + SCOPE watchdog. | | [`VX_afu_axil_demux.sv`](../../hw/rtl/afu/common/VX_afu_axil_demux.sv) | AXI-Lite demux splitting the control space on `addr[12]`, one outstanding transaction per direction. | | [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing, including a write burst under way ahead of its AW. | | [`VX_afu_req_gate.sv`](../../hw/rtl/afu/common/VX_afu_req_gate.sv) / [`VX_afu_wr_gate.sv`](../../hw/rtl/afu/common/VX_afu_wr_gate.sv) | Withhold new reads / whole new writes while the reset sequencer quiesces, never withdrawing an offer. | @@ -78,7 +84,7 @@ DMA-command engine have been removed. one before accepting the other. xrtsim aborts on a request outside the issuing port's share of memory. - **Interrupt.** The AFU `interrupt` pin is driven from `cp_interrupt` - ([`:335`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv#L335)). + ([`:335`](../../hw/rtl/afu/common/VX_afu_wrap.sv#L335)). --- diff --git a/docs/xilinx_slash_setup.md b/docs/xilinx_slash_setup.md index 03d02918a8..2d85a35a4f 100644 --- a/docs/xilinx_slash_setup.md +++ b/docs/xilinx_slash_setup.md @@ -97,12 +97,15 @@ Artifacts land in `deb/`. ```bash cd deb -sudo apt install --no-install-recommends \ - $(ls -1 *.deb | grep -v '^ami_' | sed 's|^|./|') +sudo apt install --no-install-recommends $(ls -1 *.deb | sed 's|^|./|') ``` -`ami` is built but deliberately left out: nothing in the `slash` dependency -chain needs it, and it is the component that heartbeats the AMC over GCQ. +**Install `ami` too.** Nothing in the `slash` dependency chain pulls it in, but +the card is not fully usable without it: `ami.ko` binds PF0, and the AMC behind +PF0 is how the static shell is written to flash (§5) and how the board is +recovered. Skipping it leaves PF0 unbound at every boot and the board reporting +`NOT READY`. Its package registers the module with DKMS, so — like `slash` — it +rebuilds on kernel updates and loads automatically. If a previous **source** install exists, remove its leftovers first or they win: files under `/etc/systemd/system/` override the packaged units, and @@ -146,8 +149,18 @@ systemctl is-active vrtd v80-smi list ``` -`v80-smi list` reports **PF0 NOT READY** when `ami` is absent. That is a -readiness *report*, not a requirement: PF1, PF2 and VRTD are what matter. +All three functions should bind: PF0 to `ami`, PF1 to `slash_qdma`, PF2 to +`slash_ctl`. + +```bash +lsmod | grep -E '^ami|^slash' +for f in 0 1 2; do basename "$(readlink /sys/bus/pci/devices/0000:01:00.$f/driver)"; done +``` + +`v80-smi list` reporting **PF0 NOT READY** means `ami` is missing. Running +kernels does work in that state — PF1, PF2 and VRTD carry the data and control +paths — but flashing the shell and recovering the board do not, so treat it as +an incomplete install rather than a cosmetic warning. --- @@ -290,6 +303,26 @@ from the static shell in flash. On a board that has never had it programmed, PF0-only is expected — complete §5 first. `slash.ko` binds *only* PF1/PF2, so until they appear the driver has nothing to attach to. +**No V80 on the bus at all, and no root port.** `lspci -d 10ee:` prints +nothing and `ls /sys/bus/pci/devices/0000:00:01.1` does not exist. AMD firmware +only instantiates a root port whose link is up at POST, so a card that was +wedged — mid-program, or by a host crash — disappears entirely and the bus +numbers shift down. **No rescan can bring it back**: there is no parent bridge +to scan behind, and `v80-smi --jtag` cannot help either, because it resolves the +device through VRTD, which only knows cards that are on PCIe. + +Recover over JTAG, which needs no root and reads no flash: + +```bash +bash hw/syn/xilinx/aved/tools/jtag_load_shell.sh # loads the _nofpt PDI +sudo reboot # POST now sees a trained link +``` + +Check `0000:00:01.1` before attempting anything rescan-based. Unload `slash` +and `ami` first if they are loaded — a link transition under a live driver is +what turns this into a host reset. A card whose **both** flash partitions hold +a good shell (§5) boots on its own and does not need this. + **BARs unassigned / `enable=0` / config space reads `0xff`.** The Versal can finish booting from OSPI *after* the host has enumerated PCIe, leaving the kernel's view frozen from a failed enumeration. Symptom in `dmesg`: From 84f8f6d0b90388d1ba7dcaff953b7129da446e41 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 23 Sep 2026 09:32:28 -0700 Subject: [PATCH 151/336] docs: the host resets are PCIe errors, not CPU faults A driver reading the card's BAR while the link is down raises an uncorrectable PCIe error, and APEI firmware-first answers with a sync flood that resets the host before Linux logs anything -- leaving a BERT record that names a processor cache error. Documents the signature, the giveaway log lines, and the three fixes: patched drivers, NBIO SyncFlood Generation disabled, and both flash partitions written. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 3fcb861844984027e1c8cc75f5b72a32d6960db3) --- docs/xilinx_slash_setup.md | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/xilinx_slash_setup.md b/docs/xilinx_slash_setup.md index 2d85a35a4f..ebbcca6102 100644 --- a/docs/xilinx_slash_setup.md +++ b/docs/xilinx_slash_setup.md @@ -303,6 +303,49 @@ from the static shell in flash. On a board that has never had it programmed, PF0-only is expected — complete §5 first. `slash.ko` binds *only* PF1/PF2, so until they appear the driver has nothing to attach to. +**The whole host resets when the card is touched.** The machine reboots with +no panic and no log; the next boot prints a fatal ACPI **BERT** record naming a +processor cache error (`Check Information 0x602001f`), often on a different +core each time. It is not a CPU fault. + +``` +x86/amd: Previous system reset reason [0x80000800]: a software sync flood event occurred +GHES: APEI firmware first mode is enabled by APEI bit +``` + +The chain is: a driver reads the card's BAR while the link is down -> that read +is an unsupported request or completion timeout, an uncorrectable PCIe error -> +firmware owns error handling (APEI firmware-first) and answers with a **sync +flood**, resetting the host before Linux can log anything. Firmware then +records it in the only vocabulary BERT gives it, which is why it masquerades as +a CPU defect. + +The reader is usually AMI's logging thread: `dump_amc_log()` reads every 500 ms +and, before the fix, did so unconditionally -- so it covered every bus reset, +every reconfiguration and every wedged card. Symptoms line up with a card event +0-80 s earlier, and the giveaway in the log is: + +``` +ami 0000:01:00.0: ERROR : Logging thread error - invalid PCI data read! +eqdma_cpm5_hw_error_process: Global Err Reg(0x248) = 0xffffffff +``` + +Three things fix it, and all three are worth doing: + +1. **Patched drivers.** `dump_amc_log()` must return when the service is halted + or `pci_device_is_present()` is false, and `ami` must implement + `reset_prepare`/`reset_done` so a reset quiesces its threads. The reset path + must use `pci_reset_bus()`, not a bare `pci_bridge_secondary_bus_reset()`, + or those callbacks are never delivered. +2. **BIOS: `NBIO SyncFlood Generation` = Disabled** (Advanced -> AMD CBS -> + NBIO Common Options), with `Enable AER Cap` and EDPC/DPC enabled. This stops + *any* unanswered access from resetting the host, and is the only way to see + the real PCIe error -- with firmware-first the evidence is destroyed by the + reset. +3. **Both flash partitions written (§5).** `Shell: unknown` makes the daemon + reset the board on every image load: one boot logged 115 of them. Each is a + window for the failure above. + **No V80 on the bus at all, and no root port.** `lspci -d 10ee:` prints nothing and `ls /sys/bus/pci/devices/0000:00:01.1` does not exist. AMD firmware only instantiates a root port whose link is up at POST, so a card that was From a13ae866fe7a9d270a3052be8b695bd6efaac4be Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 23 Sep 2026 14:52:11 -0700 Subject: [PATCH 152/336] firesim: add a metasim target The port had no simulation rung: hw/syn/firesim built a bitstream and nothing else, so the only way to exercise it was to program a card. The Chisel already expected this -- VortexTarget's blackbox deliberately adds no path because "the simulator [is supplied] through its own flags" -- but the flags were never wired up. Three things were in the way: - The flattened tree is generated with -DVIVADO -DSYNTHESIS. -DVIVADO swaps the soft FPU for Xilinx floating_point IP, which is .xci with no Verilog to elaborate, and -DSYNTHESIS gates the behavioural memories out in favour of BRAM inference. Simulation needs neither, so it gets its own flattening under its own stamp. - hw/syn/common.mk exports VERILATOR as a bare path to Vortex's toolchain, and FireSim declares `VERILATOR ?= verilator --main --timing --cc --exe`. A `?=` yields to the environment, so the inherited value stripped every flag and Verilator exited asking for --cc. Drop it from the sub-make's environment rather than restating FireSim's flags here. - run-metasim must not depend on metasim: `stage` copies the harness sources unconditionally, so every run relinked the 460 MB model first. With FIRESIM_REV moved past the metasim teardown fix, the V80 target now builds and passes in Verilator: reset, DCR write and the busy checks, 34 cycles, no FPGA involved. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 304691d5eb118950969714d440171018a855da0b) --- VERSION | 2 +- hw/syn/firesim/Makefile | 93 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 1ef5182277..8797dd22d8 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=c82359a76fe4616a303e193e44a7c6ac33762480 +FIRESIM_REV=ed04b53963052f493fd7c7c95316f3f23d9321e4 diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index ea80f15347..821fbdd43c 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -53,10 +53,14 @@ QUINTUPLET := $(PLATFORM)-midasexamples-VortexTarget-VortexConfig-$(PLATFORM_CON VIVADO_SRCS_DIR := $(BUILD_DIR)/vivado_srcs VIVADO_SRCS_LIST := $(BUILD_DIR)/vivado_srcs.f +# The same tree flattened for simulation instead. gen_sources.sh writes a +# directory, not a stamp, so the rule records one to hang the dependency on. SRCS_DIR := $(BUILD_DIR)/vortex_srcs SRCS_LIST := $(BUILD_DIR)/vortex_srcs.f +SIM_SRCS_STAMP := $(SRCS_DIR)/.sources.stamp -.PHONY: stage sources vivado-sources xilinx-ips kernel-env bitstream check-firesim clean-staged +.PHONY: stage sources sim-sources vivado-sources xilinx-ips kernel-env bitstream \ + metasim run-metasim check-firesim clean-staged stage: check-firesim @mkdir -p $(TARGET_DEST) $(CC_DEST) @@ -159,6 +163,23 @@ CONFIG_STAMP := $(BUILD_DIR)/gen_cflags.stamp $(shell mkdir -p $(BUILD_DIR) && echo '$(GEN_CFLAGS)' > $(CONFIG_STAMP).tmp && \ (cmp -s $(CONFIG_STAMP).tmp $(CONFIG_STAMP) && rm -f $(CONFIG_STAMP).tmp || mv -f $(CONFIG_STAMP).tmp $(CONFIG_STAMP))) +# The metasimulator needs the opposite of both synthesis defines, so it gets its +# own flattening rather than reusing the Vivado one. +# +# -DVIVADO selects the Xilinx floating_point cores (xil_fma, xil_fdiv) over the +# soft FPU. Those are generated .xci IP with no Verilog to elaborate, so a +# metasim built from Vivado-flattened sources fails on the missing modules. +# +# -DSYNTHESIS gates VX_dp_ram/VX_sp_ram onto their BRAM-inference branch; the +# behavioural models are what a simulator wants, and they carry the read/write +# collision semantics the inference branch leaves to the tool. +SIM_DEFINES := -DNDEBUG +SIM_GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) $(SIM_DEFINES) $(RTL_INCLUDE) + +SIM_CONFIG_STAMP := $(BUILD_DIR)/sim_cflags.stamp +$(shell mkdir -p $(BUILD_DIR) && echo '$(SIM_GEN_CFLAGS)' > $(SIM_CONFIG_STAMP).tmp && \ + (cmp -s $(SIM_CONFIG_STAMP).tmp $(SIM_CONFIG_STAMP) && rm -f $(SIM_CONFIG_STAMP).tmp || mv -f $(SIM_CONFIG_STAMP).tmp $(SIM_CONFIG_STAMP))) + # Only the packages and the wrapper are named; every other Vortex module is # resolved by library search over RTL_INCLUDE. Handing the whole tree over as a # flat list instead makes Verilator stop treating the macro-computed width @@ -173,10 +194,10 @@ EXT_PKG_ORDER := VX_tcu_pkg.sv VX_dxa_pkg.sv VX_rtu_pkg.sv VX_om_pkg.sv \ EXT_PKG_NAMES := $(filter $(notdir $(RTL_PKGS)),$(EXT_PKG_ORDER)) EXPLICIT_NAMES := VX_gpu_pkg.sv VX_fpu_pkg.sv VX_tlb_pkg.sv VX_trace_pkg.sv \ $(EXT_PKG_NAMES) VX_firesim_wrap.sv -EXPLICIT_SRCS := $(addprefix $(VIVADO_SRCS_DIR)/,$(EXPLICIT_NAMES)) +EXPLICIT_SRCS := $(addprefix $(SRCS_DIR)/,$(EXPLICIT_NAMES)) sources: $(SRCS_LIST) -$(SRCS_LIST): $(VIVADO_SRCS_LIST) +$(SRCS_LIST): $(SIM_SRCS_STAMP) @mkdir -p $(BUILD_DIR) @printf '%s\n' $(EXPLICIT_SRCS) > $@ @echo "wrote $@ ($(words $(EXPLICIT_SRCS)) explicit sources; rest via library search)" @@ -229,6 +250,37 @@ $(VIVADO_SRCS_LIST): $(RTL_SRCS) fi @echo "wrote $(VIVADO_SRCS_LIST) ($$(grep -cv '^+' $(VIVADO_SRCS_LIST)) files)" +# The same flattening under the simulation defines. Verilator finds the modules +# by library search over this directory, so only the tree has to be complete -- +# the file list the search is seeded with is $(SRCS_LIST) above. +SIM_RTL_SRCS := $(filter-out $(CONFIG_STAMP),$(RTL_SRCS)) $(SIM_CONFIG_STAMP) + +sim-sources: $(SIM_SRCS_STAMP) +$(SIM_SRCS_STAMP): $(SIM_RTL_SRCS) + @mkdir -p $(BUILD_DIR) + cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(SIM_GEN_CFLAGS) \ + -TVX_firesim_wrap -Cvortex_srcs -Ovortex_srcs_all.f + @# The same zero-memory-port check the Vivado flattening makes: here it would + @# surface as a zero-width AXI port rather than an hours-later elaboration + @# failure, but it is the same defect and cheaper to catch at the source. + @if grep -qE 'localparam +VX_MEM_PORTS *= *0 *;' $(SRCS_DIR)/VX_gpu_pkg.sv; then \ + echo "ERROR: generated sources resolve VX_MEM_PORTS to 0 (no memory interface)"; \ + echo " pass only XLEN and the TCU/FPU selectors; VX_config.vh computes the rest"; \ + exit 1; \ + fi + @# The vendor FP branch is a generate-if on a localparam, not an ifdef, so the + @# instantiation survives preprocessing either way. What matters is that the + @# localparam resolves to 0 so the branch is dead -- Verilator then prunes it, + @# as the rtlsim flow has always relied on. Checking for the instantiation text + @# instead reports every build as broken. + @if grep -qE 'localparam +VENDOR_OK *= *1 *;' $(SRCS_DIR)/VX_fma_unit.sv; then \ + echo "ERROR: simulation sources select the Xilinx FP IP (VENDOR_OK=1)"; \ + echo " SIM_DEFINES must not carry -DVIVADO or -DQUARTUS"; \ + exit 1; \ + fi + touch $@ + @echo "wrote $(SRCS_DIR) ($$(ls $(SRCS_DIR) | wc -l) files)" + # Vendor floating-point cores. On Vivado the FPU and RTU select Xilinx's # floating_point IP (xil_fma, xil_fdiv, ...) instead of the soft core, and the # flattened sources only instantiate them, so the kernel packaging must be handed @@ -256,7 +308,10 @@ kernel-env: # One command from sources to the card's image. Expects the FireSim environment # (its env.sh) and the Xilinx tools for the card's platform in the environment. -bitstream: vivado-sources sources xilinx-ips stage +# $(SRCS_LIST) is deliberately not a prerequisite: the recipe hands synthesis the +# Vivado-flattened list, and pulling the simulation one in would flatten the tree +# a second time under the opposite defines for nothing. +bitstream: vivado-sources xilinx-ips stage $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl TOOL_DIR=$(SCRIPT_DIR) \ @@ -266,6 +321,36 @@ bitstream: vivado-sources sources xilinx-ips stage FREQUENCY ?= 60.0 STRATEGY ?= TIMING +# The metasimulator: the FireSim-transformed design in Verilator, driven through +# the DPI emul simif. No FPGA and no Vivado, so it runs the same target the card +# runs without needing either -- which is what makes it the step before silicon. +# +# Vortex is supplied here rather than inlined into the generated wrapper (see +# VortexTarget.scala): the packages and the wrapper are named and everything else +# resolves by library search over the flattened tree. Naming the whole tree +# instead makes Verilator stop folding the macro-computed widths into constants. +# +# The list is collapsed onto one line because it reaches Verilator through a +# shell, which would otherwise split it on the newlines. +# hw/syn/common.mk exports VERILATOR as a bare path to Vortex's own toolchain, +# and FireSim declares `VERILATOR ?= verilator --main --timing --cc --exe`. A +# `?=` yields to the environment, so the inherited value silently strips every +# flag and Verilator exits asking for --cc. Drop it from the environment instead +# of restating FireSim's flags here, which would rot against its Makefrag. The +# metasim then builds with the Verilator on PATH, which FireSim's env.sh pins. +metasim: sources stage + env -u VERILATOR $(MAKE) -C $(FIRESIM_PATH)/sim verilator $(FIRESIM_TARGET_ARGS) \ + EXTRA_VERILATOR_FLAGS="-Wno-fatal -y $(SRCS_DIR) +libext+.sv+.v $$(tr '\n' ' ' < $(SRCS_LIST))" + +# Runs it. ARGS reaches the harness; the metasim needs no image, since the design +# is the simulator. +# +# Deliberately not dependent on `metasim`: `stage` copies the harness sources +# unconditionally, so every invocation would bump their mtimes and relink the +# 460 MB model before running it. Build with `make metasim` first. +run-metasim: + env -u VERILATOR $(MAKE) -C $(FIRESIM_PATH)/sim run-verilator $(FIRESIM_TARGET_ARGS) ARGS="$(ARGS)" + # The checkout must contain the pinned fork commit. Being ahead of it is normal # while developing the fork; being behind it, or on unrelated history, builds # against FireSim changes this tree does not expect. From 6f1dc472bf0f04620ee81300318d0aace74b664a Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 23 Sep 2026 16:02:52 -0700 Subject: [PATCH 153/336] firesim: let the bitstream target build the platform's sim image cl_firesim takes TARGET=hw|sim and builds each into build_dir.$(TARGET), but the bitstream target never passed it, so the only reachable image was the bitstream and the sim rung could not be built through the normal flow. With it, the V80 target's demo runs the same image, runtime and control path under the platform's simulator and passes in 87952 cycles -- the same test that hangs on the card at Q_SEQNUM=26. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 8c06587dffb34b2f5b71fbf68884c89054740047) --- hw/syn/firesim/Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 821fbdd43c..8fb18d7bc9 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -315,12 +315,18 @@ bitstream: vivado-sources xilinx-ips stage $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl TOOL_DIR=$(SCRIPT_DIR) \ - $(MAKE) -C $(FIRESIM_PATH)/platforms/$(PLATFORM)/cl_$(QUINTUPLET) bitstream \ + $(MAKE) -C $(FIRESIM_PATH)/platforms/$(PLATFORM)/cl_$(QUINTUPLET) bitstream TARGET=$(TARGET) \ DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) FREQUENCY ?= 60.0 STRATEGY ?= TIMING +# Which image the card's platform builds. `hw` is the bitstream; `sim` is the +# model its own simulator runs, which exercises the same driver and the same +# control path without a card. The platform builds each into build_dir.$(TARGET), +# which is what the run's FPGA_BIN_DIR has to point at. +TARGET ?= hw + # The metasimulator: the FireSim-transformed design in Verilator, driven through # the DPI emul simif. No FPGA and no Vivado, so it runs the same target the card # runs without needing either -- which is what makes it the step before silicon. From ac2ca3072b11319b6b775726fb0f7fb4a988823a Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 23 Sep 2026 19:30:58 -0700 Subject: [PATCH 154/336] firesim: reach each platform's simulation image without leaking TARGET The previous commit passed TARGET straight through to the platform, which is right for aved and wrong for vitis: vitis sets `bitstream: TARGET = hw` as a target-specific variable, and a command-line TARGET beats a target-specific one, so TARGET=sim would have reached v++ as `--target sim` -- not a Vitis build target at all. It also has its own `sim` goal for hw_emu, which nothing here could reach. Name the two goals per platform and select between them with `sim-image`, so callers ask for an image rather than knowing how each platform spells it. aved resolves to `bitstream TARGET=hw|sim`, vitis to `bitstream`|`sim`. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 5b43ca4b0d8f85db2a3a2981dc91aab5e50066e3) --- hw/syn/firesim/Makefile | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 8fb18d7bc9..74d7388122 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -59,7 +59,7 @@ SRCS_DIR := $(BUILD_DIR)/vortex_srcs SRCS_LIST := $(BUILD_DIR)/vortex_srcs.f SIM_SRCS_STAMP := $(SRCS_DIR)/.sources.stamp -.PHONY: stage sources sim-sources vivado-sources xilinx-ips kernel-env bitstream \ +.PHONY: stage sources sim-sources vivado-sources xilinx-ips kernel-env bitstream sim-image \ metasim run-metasim check-firesim clean-staged stage: check-firesim @@ -315,17 +315,33 @@ bitstream: vivado-sources xilinx-ips stage $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl TOOL_DIR=$(SCRIPT_DIR) \ - $(MAKE) -C $(FIRESIM_PATH)/platforms/$(PLATFORM)/cl_$(QUINTUPLET) bitstream TARGET=$(TARGET) \ + $(MAKE) -C $(FIRESIM_PATH)/platforms/$(PLATFORM)/cl_$(QUINTUPLET) $(PLATFORM_IMAGE_GOAL) \ DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) +# The same image built for the platform's own simulator instead of the fabric. +sim-image: + $(MAKE) bitstream PLATFORM_IMAGE_GOAL='$(PLATFORM_SIM_GOAL)' + FREQUENCY ?= 60.0 STRATEGY ?= TIMING -# Which image the card's platform builds. `hw` is the bitstream; `sim` is the -# model its own simulator runs, which exercises the same driver and the same -# control path without a card. The platform builds each into build_dir.$(TARGET), -# which is what the run's FPGA_BIN_DIR has to point at. -TARGET ?= hw +# How each platform names its two images, kept here so the difference does not +# leak into the caller. aved builds both from one goal and selects with TARGET, +# putting each in build_dir.$(TARGET); vitis has a separate `sim` goal that +# selects hw_emu. +# +# Passing TARGET through to vitis would be worse than useless: its Makefile sets +# `bitstream: TARGET = hw` as a target-specific variable, and a command-line +# TARGET beats a target-specific one, so TARGET=sim would reach v++ as +# `--target sim`, which is not a Vitis build target at all. +ifeq ($(PLATFORM),aved) +PLATFORM_HW_GOAL := bitstream TARGET=hw +PLATFORM_SIM_GOAL := bitstream TARGET=sim +else +PLATFORM_HW_GOAL := bitstream +PLATFORM_SIM_GOAL := sim +endif +PLATFORM_IMAGE_GOAL ?= $(PLATFORM_HW_GOAL) # The metasimulator: the FireSim-transformed design in Verilator, driven through # the DPI emul simif. No FPGA and no Vivado, so it runs the same target the card From ebff0d8c99c002cd8cb9fdd87cd58a9436a69981 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 24 Sep 2026 14:33:33 -0700 Subject: [PATCH 155/336] firesim: add TARGET=avedsim, the in-process metasimulator on aved The FireSim runtime had a card path per platform and nothing below it: on aved the only way to run a Vortex test through FireSim was to program a V80 or build the platform's xsim image. avedsim links the Verilated FireSim-transformed target with the emulated host interface (simif_emul) into libfiresim.so, so the runtime loads it exactly as it loads the card's library. firesim_emul_main.cpp supplies the loop Verilator's --main would generate; the emul top's initial block reaches entry() through the DPI and the driver runs on the thread simif_emul spawns. At NW=NT=4 demo passes in 22290 target cycles -- the same count as the U55C hw_emu and aved sim runs -- and raycast in 1588037. Fixed on the way: - The runtime's config stamp had no prerequisite, so it was written once and never compared again: a changed CONFIGS kept linking the runtime built for the previous one. Every other runtime forces it. - The PIC driver objects shared one directory across platforms, so make could link one card's objects, built against its generated widths, into the other card's library. - The emul rule removes its model before rebuilding: the Vortex sources reach the sub-make as flags, so its own rule calls the model current after a config change. - TARGET reaches the runtime build, and run-firesim on aved branches on it. hw_emu runs get an xrt.ini (next commit). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e44c3e3b1a28405f2e0267d035daaa83805423b5) --- sim/firesim/Makefile | 60 ++++++++++++++++++++++++++++++- sim/firesim/firesim_emul_main.cpp | 30 ++++++++++++++++ sim/firesim/firesim_sim.cpp | 18 ++++++++-- sw/runtime/firesim/Makefile | 25 ++++++++++--- sw/runtime/firesim/vortex.cpp | 9 +++-- tests/regression/common.mk | 11 ++++-- 6 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 sim/firesim/firesim_emul_main.cpp diff --git a/sim/firesim/Makefile b/sim/firesim/Makefile index 77be235d7d..7a9f5b2d7f 100644 --- a/sim/firesim/Makefile +++ b/sim/firesim/Makefile @@ -65,13 +65,69 @@ check-gen: # objects current and link them into a shared library, which fails late with # "final link failed: bad value". The position-independent build gets its own # object directory, seeded with the generated headers it needs. -PIC_DIR := $(DESTDIR)/firesim_pic +# Per platform: the objects embed that platform's generated widths, and make +# judges them by timestamp, so a directory shared between cards would link one +# card's objects into the other's library. +PIC_DIR := $(DESTDIR)/firesim_pic_$(PLATFORM) $(PIC_DIR)/FireSim-generated.const.h: $(GEN_DIR)/FireSim-generated.const.h @mkdir -p $(PIC_DIR) @cp -f $(GEN_DIR)/FireSim-generated.const.* $(PIC_DIR)/ 2>/dev/null || true @cp -f $(GEN_DIR)/FireSim-generated.const.h $(PIC_DIR)/ +ifeq ($(TARGET), avedsim) + +# The metasimulator as the library: the FireSim-transformed target Verilated, +# with the emulated host interface (simif_emul) in place of the card's, linked +# into lib$(PROJECT).so so the runtime loads it exactly as it loads the card's. +# firesim_emul_main.cpp supplies the loop that --main would otherwise generate. +# +# The Vortex RTL comes from hw/syn/firesim's simulation flattening, the same +# sources the standalone metasim builds from; the generated FireSim RTL and +# headers come from the elaborated target. +SYN_DIR := $(ROOT_DIR)/hw/syn/firesim +SYN_BUILD_DIR ?= $(SYN_DIR) +SIM_SRCS_LIST := $(SYN_BUILD_DIR)/vortex_srcs.f +SIM_SRCS_DIR := $(SYN_BUILD_DIR)/vortex_srcs +ELAB_DIR := $(FIRESIM_PATH)/sim/generated-src/$(PLATFORM)/$(PLATFORM)-midasexamples-$(DESIGN)-$(TARGET_CONFIG)-$(PLATFORM_CONFIG) +EMUL_DIR := $(DESTDIR)/firesim_emul +EMUL_NAME := firesim_emul + +EMUL_SRCS := $(SRC_DIR)/firesim_sim.cpp $(SRC_DIR)/firesim_emul_main.cpp + +.PHONY: emul-sources +emul-sources: + $(MAKE) -C $(SYN_DIR) sources SYN_BUILD_DIR=$(SYN_BUILD_DIR) FIRESIM_CARD=$(FIRESIM_CARD) + +# Host toolchain, as for the aved platform library: the runtime that loads this +# is built with it. -fPIC reaches the Verilated objects too, through -CFLAGS. +# hw/syn/common.mk exports VERILATOR as a bare path and FireSim's `?=` would +# yield to it, so it is named here in full. +# +# The sub-make's own rule does not know the Vortex sources are inputs -- they +# arrive as flags -- so it would call the model current after a config change. +# This rule does know, so once it decides to rebuild, the old model goes. +$(EMUL_DIR)/V$(EMUL_NAME): $(EMUL_SRCS) $(SIM_SRCS_LIST) $(ELAB_DIR)/FireSim-generated.sv + rm -f $@ + $(MAKE) -C $(SIMIF_DIR) verilator \ + PLATFORM=$(PLATFORM) \ + DRIVER_NAME=$(EMUL_NAME) \ + GEN_FILE_BASENAME=FireSim-generated \ + GEN_DIR=$(ELAB_DIR) \ + OUT_DIR=$(EMUL_DIR) \ + DRIVER="$(EMUL_SRCS)" \ + CXX=/usr/bin/g++ \ + VERILATOR="verilator --timing --cc --exe" \ + VERILATOR_FLAGS="-Wno-fatal -y $(SIM_SRCS_DIR) +libext+.sv+.v $$(tr '\n' ' ' < $(SIM_SRCS_LIST))" \ + CXXFLAGS="-O2 -fPIC -std=c++20 -DFIRESIM_EMUL -I$(SRC_DIR)" \ + LDFLAGS="-shared" + +lib: emul-sources + $(MAKE) $(EMUL_DIR)/V$(EMUL_NAME) + cp -f $(EMUL_DIR)/V$(EMUL_NAME) $(DESTDIR)/lib$(PROJECT).so + +else + # DRIVER_NAME is chosen so the sub-make emits lib$(PROJECT).so directly. lib: check-gen $(PIC_DIR)/FireSim-generated.const.h $(MAKE) -C $(SIMIF_DIR) driver \ @@ -86,6 +142,8 @@ lib: check-gen $(PIC_DIR)/FireSim-generated.const.h $(if $(SIM_CXX),CXX=$(SIM_CXX)) mv -f $(DESTDIR)/lib$(PROJECT).so.tmp-$(PLATFORM) $(DESTDIR)/lib$(PROJECT).so +endif + clean: rm -f $(DESTDIR)/lib$(PROJECT).so rm -rf $(GEN_DIR)/bridges $(GEN_DIR)/core $(GEN_DIR)/*.o diff --git a/sim/firesim/firesim_emul_main.cpp b/sim/firesim/firesim_emul_main.cpp new file mode 100644 index 0000000000..a8261cc0bd --- /dev/null +++ b/sim/firesim/firesim_emul_main.cpp @@ -0,0 +1,30 @@ +// The metasimulator's main loop, as a function the runtime calls on its +// simulator thread rather than as a process entry point. +// +// This is Verilator's --main loop for a --timing model. The emul top's initial +// block calls simulator_entry() through the DPI, which reaches entry() and +// spawns the driver thread, exactly as the standalone metasim does. The loop +// ends when the top $finishes, which the DPI tick triggers once the driver has +// returned -- that is, once the runtime has shut the simulation down. + +#include + +#include "verilated.h" +#include "Vemul.h" + +int firesim_emul_run(int argc, char **argv) { + const std::unique_ptr contextp{new VerilatedContext}; + contextp->commandArgs(argc, argv); + const std::unique_ptr topp{new Vemul{contextp.get()}}; + + while (!contextp->gotFinish()) { + topp->eval(); + if (!topp->eventsPending()) { + break; + } + contextp->time(topp->nextTimeSlot()); + } + + topp->final(); + return 0; +} diff --git a/sim/firesim/firesim_sim.cpp b/sim/firesim/firesim_sim.cpp index 29b7dd395b..574b2789c4 100644 --- a/sim/firesim/firesim_sim.cpp +++ b/sim/firesim/firesim_sim.cpp @@ -40,6 +40,12 @@ // Provided by the simulator's entry translation unit. extern int entry(int argc, char **argv); +#ifdef FIRESIM_EMUL +// Provided by firesim_emul_main.cpp: runs the Verilated metasimulator, whose +// initial block reaches entry() through the DPI. +extern int firesim_emul_run(int argc, char **argv); +#endif + namespace { bool trace_enabled() { @@ -376,8 +382,16 @@ class firesim_sim::Impl { argv_.push_back(const_cast(s.c_str())); } - thread_ = std::thread( - [this] { entry(static_cast(argv_.size()), argv_.data()); }); + // On a card this thread is the driver. Under the metasimulator it is the + // simulator's own loop instead: entry() is then reached from a Verilog + // initial block, and the driver runs on the thread simif_emul spawns. + thread_ = std::thread([this] { +#ifdef FIRESIM_EMUL + firesim_emul_run(static_cast(argv_.size()), argv_.data()); +#else + entry(static_cast(argv_.size()), argv_.data()); +#endif + }); FSIM_TRACE("waiting for the simulator thread to come up"); { diff --git a/sw/runtime/firesim/Makefile b/sw/runtime/firesim/Makefile index 9a24d761ac..8406f65067 100644 --- a/sw/runtime/firesim/Makefile +++ b/sw/runtime/firesim/Makefile @@ -26,6 +26,14 @@ ifeq ($(FIRESIM_PLATFORM), aved) CXXFLAGS += -DFIRESIM_PLATFORM_AVED endif +# TARGET=avedsim runs the metasimulator in-process in place of the card: the +# same FireSim-transformed target in Verilator, reached through the emulated +# host interface, with no image and no VRT. It is the aved counterpart of the +# native driver's avedsim. +ifeq ($(TARGET), avedsim) +CXXFLAGS += -DFIRESIM_EMUL +endif + LDFLAGS += -shared -pthread # Find libfiresim.so siblings at runtime in the same dir libvortex-firesim.so lives in. LDFLAGS += -Wl,-rpath,'$$ORIGIN' @@ -50,14 +58,23 @@ all: $(DESTDIR)/$(PROJECT) driver: $(DESTDIR)/libfiresim.so $(DESTDIR)/libfiresim.so: force - DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib FIRESIM_CARD=$(FIRESIM_CARD) + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) $(DESTDIR)/$(PROJECT): $(SRCS) $(DESTDIR)/libfiresim.so $(CONFIG_FILE) $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ -$(CONFIG_FILE): - echo "$(CXXFLAGS)" > $@.tmp - @if ! cmp -s $@.tmp $@; then mv $@.tmp $@; else rm $@.tmp; fi +# Forced, as in every other runtime: without a prerequisite the stamp is written +# once and never compared again, so a changed CONFIGS keeps linking the runtime +# built for the previous one. +$(CONFIG_FILE): force + @mkdir -p $(@D) + @TMP=$@.tmp.$$$$ ; \ + printf '%s\n' "$(CXXFLAGS)" > $$TMP ; \ + if ! cmp -s $$TMP $@ 2>/dev/null; then \ + mv -f $$TMP $@ ; \ + else \ + rm -f $$TMP ; \ + fi clean-driver: DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim clean diff --git a/sw/runtime/firesim/vortex.cpp b/sw/runtime/firesim/vortex.cpp index dd2dcc13f8..f633fd87c4 100644 --- a/sw/runtime/firesim/vortex.cpp +++ b/sw/runtime/firesim/vortex.cpp @@ -59,10 +59,13 @@ class vx_device { // driver reads, so switching between a native and a FireSim run does not // mean switching variable names as well. std::vector plusargs; -#ifdef FIRESIM_PLATFORM_AVED +#if defined(FIRESIM_EMUL) + // The metasimulator is the design itself: there is no image to load and no + // card to find. +#elif defined(FIRESIM_PLATFORM_AVED) // As the aved driver: VRT_VBIN_PATH and VRT_DEVICE_BDF, with the card - // discovered on the bus when the BDF is unset. There is no metasimulation - // on this platform, so an image is required. + // discovered on the bus when the BDF is unset. An image is required: the + // card and the platform's simulator both run one. const char* vbin = getenv("VRT_VBIN_PATH"); if (vbin == nullptr || vbin[0] == '\0') { fprintf(stderr, "[VXDRV] VRT_VBIN_PATH must name the FireSim .vbin\n"); diff --git a/tests/regression/common.mk b/tests/regression/common.mk index bfe4fe387f..b55ca86c1a 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -249,14 +249,21 @@ ifneq ($(filter run-firesim,$(MAKECMDGOALS)),) FIRESIM_PLATFORM := $(shell python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk | sed -n 's/^PLATFORM := //p') endif +# On aved, TARGET=avedsim runs the metasimulator in-process and needs no image; +# hw and sim both run a .vbin, the card's or the platform simulator's, named by +# FPGA_BIN_DIR. run-firesim: $(PROJECT) kernel.vxbin - $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) FIRESIM_CARD=$(FIRESIM_CARD) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) ifeq ($(FIRESIM_PLATFORM), aved) +ifeq ($(TARGET), avedsim) + LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/firesim.vbin LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +endif else ifeq ($(TARGET), hw) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) else - XCL_EMULATION_MODE=$(TARGET) EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) + XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) endif run-opae: $(PROJECT) kernel.vxbin From 54bbb06276c31b1f0603e9744a0d292dc5ec7cc5 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 24 Sep 2026 14:33:33 -0700 Subject: [PATCH 156/336] firesim: fix the platform simulation images Four defects kept the aved sim and vitis hw_emu images from building or running correctly: - They were flattened with the synthesis defines only. Under VIVADO, VX_platform.vh enables ASYNC_BRAM_PATCH unless SIMULATION is defined, leaving every async RAM's read address behind an empty VX_placeholder black box that only the opt_design hook rewires. A behavioural simulation never runs opt_design, so those RAMs never latched an address. sim-image now flattens separately (vivado_srcs_sim) with -DSIMULATION, as the native xrt flow does for hw_emu. - gen_sources.sh lists packages in find(1) order, which is directory order. Synthesis reorders on its own; xsim compiles the list as given, and under SIMULATION VX_trace_pkg imports VX_gpu_pkg, so hw_emu failed on "'VX_gpu_pkg' is not declared". The list now carries the dependency order the metasim already used. - BUILD_DIR could not be overridden: a command-line variable reaches every sub-make, and FireSim's own Makefile uses BUILD_DIR to find its elaborator jar. The knob is now SYN_BUILD_DIR. - hw_emu linked with -g, so XRT logged a waveform of the whole shell on every run (~800 MB for demo). sim-image clears VPP_DEBUG (FIRESIM_REV moves past the fork commit adding it), and runs get an xrt.ini with debug_mode=off. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d4c0304fd5bbc7c4d13236cd20e0cd5e21b2f210) --- VERSION | 2 +- hw/syn/firesim/Makefile | 52 +++++++++++++++++++++++++++++++------- sw/runtime/firesim/xrt.ini | 7 +++++ 3 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 sw/runtime/firesim/xrt.ini diff --git a/VERSION b/VERSION index 8797dd22d8..242eb697b3 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=ed04b53963052f493fd7c7c95316f3f23d9321e4 +FIRESIM_REV=cf3159264f0f5f779e6d00bfbd6a2a1228f8a5df diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 74d7388122..69656135f2 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -31,7 +31,12 @@ CC_DEST := $(FIRESIM_PATH)/sim/src/main/cc/midasexamples RTL_DIR := $(VORTEX_HOME)/hw/rtl SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts -BUILD_DIR ?= $(ROOT_DIR)/hw/syn/firesim +# Overridden as SYN_BUILD_DIR, never as BUILD_DIR. A command-line variable +# reaches every sub-make, and FireSim's own Makefile uses BUILD_DIR to find its +# elaborator jar: overriding it here pointed FireSim at this directory and the +# elaboration died on "Could not find or load main class". +SYN_BUILD_DIR ?= $(ROOT_DIR)/hw/syn/firesim +BUILD_DIR := $(SYN_BUILD_DIR) FIRESIM_CARD ?= u55c PLATFORM_GEN := python3 $(SCRIPT_DIR)/gen_platform.py --card=$(FIRESIM_CARD) @@ -50,8 +55,21 @@ QUINTUPLET := $(PLATFORM)-midasexamples-VortexTarget-VortexConfig-$(PLATFORM_CON # Flattened Vortex sources for the BlackBox. Golden Gate takes a file list, not # include paths, so the tree is copied into one directory with the macros already # inlined (-P) and emitted in elaboration order: packages, interfaces, modules. -VIVADO_SRCS_DIR := $(BUILD_DIR)/vivado_srcs -VIVADO_SRCS_LIST := $(BUILD_DIR)/vivado_srcs.f +# +# The platform's simulation image (sim-image) gets its own flattening, under +# -DSIMULATION. Under VIVADO, VX_platform.vh enables ASYNC_BRAM_PATCH unless +# SIMULATION is defined, and that patch leaves every async RAM's read address +# behind an empty VX_placeholder black box which only the opt_design hook +# rewires. A behavioural simulation never runs opt_design, so without the +# define those RAMs never latch an address and read back X -- the same reason +# the native xrt flow adds -DSIMULATION for hw_emu. +IMAGE_KIND ?= hw +ifeq ($(IMAGE_KIND), sim) +IMAGE_SUFFIX := _sim +endif +VIVADO_SRCS_NAME := vivado_srcs$(IMAGE_SUFFIX) +VIVADO_SRCS_DIR := $(BUILD_DIR)/$(VIVADO_SRCS_NAME) +VIVADO_SRCS_LIST := $(BUILD_DIR)/$(VIVADO_SRCS_NAME).f # The same tree flattened for simulation instead. gen_sources.sh writes a # directory, not a stamp, so the rule records one to hang the dependency on. @@ -106,6 +124,9 @@ TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) # design lands in distributed RAM. A U55C has 600960 LUTs usable as memory and # this design asks for 753744 of them without it. SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG +ifeq ($(IMAGE_KIND), sim) +SYNTH_DEFINES += -DSIMULATION +endif # This wrapper exposes a single AXI memory port, so give Vortex a single # platform memory bank to match. Left at the default of 2, Vortex builds two @@ -159,7 +180,7 @@ GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFI # Defines arrive on the command line, which no file timestamp reflects, so a # changed CONFIGS would otherwise reuse the previous flattened tree. The stamp is # rewritten only when the flags differ, which keeps an unchanged build current. -CONFIG_STAMP := $(BUILD_DIR)/gen_cflags.stamp +CONFIG_STAMP := $(BUILD_DIR)/gen_cflags$(IMAGE_SUFFIX).stamp $(shell mkdir -p $(BUILD_DIR) && echo '$(GEN_CFLAGS)' > $(CONFIG_STAMP).tmp && \ (cmp -s $(CONFIG_STAMP).tmp $(CONFIG_STAMP) && rm -f $(CONFIG_STAMP).tmp || mv -f $(CONFIG_STAMP).tmp $(CONFIG_STAMP))) @@ -192,9 +213,11 @@ $(shell mkdir -p $(BUILD_DIR) && echo '$(SIM_GEN_CFLAGS)' > $(SIM_CONFIG_STAMP). EXT_PKG_ORDER := VX_tcu_pkg.sv VX_dxa_pkg.sv VX_rtu_pkg.sv VX_om_pkg.sv \ VX_raster_pkg.sv VX_tex_pkg.sv EXT_PKG_NAMES := $(filter $(notdir $(RTL_PKGS)),$(EXT_PKG_ORDER)) -EXPLICIT_NAMES := VX_gpu_pkg.sv VX_fpu_pkg.sv VX_tlb_pkg.sv VX_trace_pkg.sv \ - $(EXT_PKG_NAMES) VX_firesim_wrap.sv +PKG_ORDER := VX_gpu_pkg.sv VX_fpu_pkg.sv VX_tlb_pkg.sv VX_trace_pkg.sv $(EXT_PKG_NAMES) +EXPLICIT_NAMES := $(PKG_ORDER) VX_firesim_wrap.sv EXPLICIT_SRCS := $(addprefix $(SRCS_DIR)/,$(EXPLICIT_NAMES)) +empty := +space := $(empty) $(empty) sources: $(SRCS_LIST) $(SRCS_LIST): $(SIM_SRCS_STAMP) @@ -234,12 +257,21 @@ vivado-sources: $(VIVADO_SRCS_LIST) $(VIVADO_SRCS_LIST): $(RTL_SRCS) @mkdir -p $(BUILD_DIR) cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) \ - -TVX_firesim_wrap -Cvivado_srcs -Ovivado_srcs.f + -TVX_firesim_wrap -C$(VIVADO_SRCS_NAME) -O$(VIVADO_SRCS_NAME).f @for f in $(AFU_SHELL_MODULES); do \ rm -f $(VIVADO_SRCS_DIR)/$$f; \ grep -v "/$$f$$" $(VIVADO_SRCS_LIST) > $(VIVADO_SRCS_LIST).tmp && \ mv $(VIVADO_SRCS_LIST).tmp $(VIVADO_SRCS_LIST); \ done + @# gen_sources.sh lists packages in find(1) order, which is directory order, + @# not dependency order. Synthesis reorders on its own; a simulator compiles + @# the list as given. Under -DSIMULATION VX_trace_pkg imports VX_gpu_pkg, and + @# with trace first xsim failed on "'VX_gpu_pkg' is not declared". Put the + @# packages in the dependency order the metasim list already uses. + @{ grep '^+' $(VIVADO_SRCS_LIST); \ + for p in $(PKG_ORDER); do grep "/$$p$$" $(VIVADO_SRCS_LIST); done; \ + grep -v '^+' $(VIVADO_SRCS_LIST) | grep -vE '/($(subst $(space),|,$(PKG_ORDER)))$$'; \ + } > $(VIVADO_SRCS_LIST).tmp && mv $(VIVADO_SRCS_LIST).tmp $(VIVADO_SRCS_LIST) @# A design with no memory ports elaborates as zero-sized arrays deep inside @# Vivado, hours from here. Defining the config scalars instead of letting @# VX_config.vh compute them is enough to cause it, so check the result. @@ -320,7 +352,7 @@ bitstream: vivado-sources xilinx-ips stage # The same image built for the platform's own simulator instead of the fabric. sim-image: - $(MAKE) bitstream PLATFORM_IMAGE_GOAL='$(PLATFORM_SIM_GOAL)' + $(MAKE) bitstream PLATFORM_IMAGE_GOAL='$(PLATFORM_SIM_GOAL)' IMAGE_KIND=sim FREQUENCY ?= 60.0 STRATEGY ?= TIMING @@ -339,7 +371,9 @@ PLATFORM_HW_GOAL := bitstream TARGET=hw PLATFORM_SIM_GOAL := bitstream TARGET=sim else PLATFORM_HW_GOAL := bitstream -PLATFORM_SIM_GOAL := sim +# Without debug: a debug hw_emu kernel logs a waveform of the whole shell on +# every run (~800 MB for demo, tens of GB for raycast) and runs far slower. +PLATFORM_SIM_GOAL := sim VPP_DEBUG= endif PLATFORM_IMAGE_GOAL ?= $(PLATFORM_HW_GOAL) diff --git a/sw/runtime/firesim/xrt.ini b/sw/runtime/firesim/xrt.ini new file mode 100644 index 0000000000..53a4f44ee4 --- /dev/null +++ b/sw/runtime/firesim/xrt.ini @@ -0,0 +1,7 @@ +# XRT settings for FireSim hardware emulation (TARGET=hw_emu). +# +# Without this XRT logs every signal of the emulated shell and target to a +# waveform database: 797 MB for a 22k-cycle demo, and the dominant cost of the +# run. Set debug_mode=batch to get the waveform back when it is wanted. +[Emulation] +debug_mode=off From b7f6cbd1557c8606e0de724f7c998bbb8d1a9f24 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 25 Sep 2026 04:31:49 -0700 Subject: [PATCH 157/336] firesim: let avedsim take Verilator runtime arguments VORTEX_FIRESIM_EMUL_ARGS appends Verilator runtime arguments to the in-process metasimulator, and it reports what the context applied. Chiefly for "+verilator+rand+reset+2 +verilator+seed+N": registers without a reset then start from random values instead of zero, the closest a 2-state simulator comes to a 4-state one's X on an uninitialised flop. demo and raycast pass unchanged under seeds 1 and 99, to the cycle. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3496e258b28253471afaeedffd49be1096219897) --- sim/firesim/firesim_emul_main.cpp | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/sim/firesim/firesim_emul_main.cpp b/sim/firesim/firesim_emul_main.cpp index a8261cc0bd..96b7740a47 100644 --- a/sim/firesim/firesim_emul_main.cpp +++ b/sim/firesim/firesim_emul_main.cpp @@ -7,14 +7,39 @@ // ends when the top $finishes, which the DPI tick triggers once the driver has // returned -- that is, once the runtime has shut the simulation down. +#include #include +#include +#include +#include #include "verilated.h" #include "Vemul.h" int firesim_emul_run(int argc, char **argv) { + // VORTEX_FIRESIM_EMUL_ARGS appends Verilator runtime arguments, e.g. + // "+verilator+rand+reset+2 +verilator+seed+7" to start every register that + // has no reset from a random value instead of zero -- the closest a 2-state + // simulator gets to a 4-state one's X on an uninitialised flop. + std::vector extra; + if (const char *env = getenv("VORTEX_FIRESIM_EMUL_ARGS")) { + std::istringstream in(env); + for (std::string tok; in >> tok;) { + extra.push_back(tok); + } + } + std::vector args(argv, argv + argc); + for (const auto &tok : extra) { + args.push_back(tok.c_str()); + } + const std::unique_ptr contextp{new VerilatedContext}; - contextp->commandArgs(argc, argv); + contextp->commandArgs(static_cast(args.size()), args.data()); + if (!extra.empty()) { + fprintf(stderr, "[emul] extra args:%s -> randReset=%d seed=%d\n", + [&] { std::string s; for (auto &t : extra) s += " " + t; return s; }().c_str(), + contextp->randReset(), contextp->randSeed()); + } const std::unique_ptr topp{new Vemul{contextp.get()}}; while (!contextp->gotFinish()) { From 4f7120d4d22ddbd4fdf4b2cdcd0947d8441af80c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 25 Sep 2026 16:20:36 -0700 Subject: [PATCH 158/336] firesim: report a stuck launch and end it instead of waiting forever A launch waited for busy to rise and then to fall with no bound on either, and the stall report counted only busy cycles, so a target that never started printed nothing and was eventually killed mid-run. Now: - a start that never raises busy is reported, with the core counters, once the stall budget passes; - the stall report also dumps the FASED counters (reads, read beats, writes, response errors), which say whether a request the core waits on left the target, came back, or came back an error; - VORTEX_FIRESIM_TIMEOUT_S bounds a launch in wall-clock time: on expiry it reports, completes the launch and stops sending the target anything, so the application exits through its own teardown rather than a kill; - a counter dump stops at the first unanswered DCR read instead of spending the full poll budget on every counter. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 581e8f6464378af7db27d139d10092199ee3848b) --- sim/firesim/firesim_sim.cpp | 125 +++++++++++++++++++++++++++++++++--- sim/firesim/firesim_sim.h | 12 ++++ 2 files changed, 128 insertions(+), 9 deletions(-) diff --git a/sim/firesim/firesim_sim.cpp b/sim/firesim/firesim_sim.cpp index 574b2789c4..2f8c8ab2b8 100644 --- a/sim/firesim/firesim_sim.cpp +++ b/sim/firesim/firesim_sim.cpp @@ -14,6 +14,7 @@ #include "firesim_sim.h" #include "bridges/clock.h" +#include "bridges/fased_memory_timing_model.h" #include #include @@ -127,9 +128,10 @@ vortex_simulation_t *g_sim = nullptr; // not thread-safe and must never be touched by the caller. class vortex_simulation_t final : public simulation_t { public: - vortex_simulation_t(widget_registry_t ®istry, + vortex_simulation_t(simif_t &simif, widget_registry_t ®istry, const std::vector &args) - : simulation_t(registry, args), + : simulation_t(registry, args), simif(simif), + fased(registry.get_bridges()), peek_poke(registry.get_widget()), loadmem(registry.get_widget()), clock(registry.get_widget()) {} @@ -175,6 +177,11 @@ class vortex_simulation_t final : public simulation_t { peek_poke.step(n, true); } + // The DRAM timing models sit between the target's memory port and the + // card's memory. Their request and error counters say whether a request the + // core is waiting on ever left the target, came back, or came back an error. + simif_t &simif; + std::vector fased; peek_poke_t &peek_poke; loadmem_t &loadmem; // True target cycles elapsed. step() requests cycles; this reports how many @@ -191,8 +198,7 @@ std::unique_ptr create_simulation(simif_t &simif, widget_registry_t ®istry, const std::vector &args) { - (void)simif; - return std::make_unique(registry, args); + return std::make_unique(simif, registry, args); } namespace { @@ -455,7 +461,31 @@ class firesim_sim::Impl { firesim_sim::firesim_sim() : impl_(new Impl()), step_batch_(kStepBatchFloor), busy_cycles_(0), - stall_reported_(false) {} + stall_reported_(false), awaiting_busy_(false), idle_cycles_(0), + abort_phase_(0), launch_started_s_(0) {} + +// Wall-clock seconds a launch may take before the runtime gives up on it; 0 +// (the default) never does. Meant for a card run under an outer timeout: this +// fires first, so the run ends with a report and a normal exit instead of a +// kill. +static double launch_timeout_s() { + static const double value = [] { + if (const char *env = getenv("VORTEX_FIRESIM_TIMEOUT_S")) { + const double v = atof(env); + if (v > 0) { + return v; + } + } + return 0.0; + }(); + return value; +} + +static double wall_now_s() { + return std::chrono::duration( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} // Target cycles of continuous busy after which the target is assumed wedged and // its bus activity is reported. Well above any kernel this platform runs today @@ -486,6 +516,23 @@ void firesim_sim::report_progress(const char *why) { // than anything the design did. fprintf(stderr, "[firesim] %s: requested=%llu tcycle=%llu\n", why, (unsigned long long)busy_cycles_, (unsigned long long)tcycle); + std::string mem; + impl_->submit("read memory model counters", [&mem] { + static const char *const kRegs[] = { + "totalReads", "totalReadBeats", "totalWrites", "totalWriteBeats", + "rrespError", "brespError", "readMaxReqs", "writeMaxReqs"}; + for (size_t i = 0; i < g_sim->fased.size(); ++i) { + const AddressMap &map = g_sim->fased[i]->get_addr_map(); + mem += " [" + std::to_string(i) + "]"; + for (const char *reg : kRegs) { + if (map.r_reg_exists(reg)) { + mem += std::string(" ") + reg + "=" + + std::to_string(g_sim->simif.read(map.r_addr(reg))); + } + } + } + }); + fprintf(stderr, "[firesim] memory model:%s\n", mem.c_str()); report_core_counters(); } @@ -532,10 +579,11 @@ void firesim_sim::report_core_counters() { (kClassCore << 22) | ((c.csr - kMpmBase) << 16) | 0 /* core 0 */; uint32_t value = 0; if (this->dcr_read(kDcrMpmValue, tag, &value) != 0) { - line += " "; - line += c.name; - line += "=?"; - continue; + // A target that ignores one DCR read ignores them all, and each costs + // the full poll budget; the rest would only delay the report. + fprintf(stderr, "[firesim] core counters:%s %s=? (DCR path not " + "answering; remaining counters skipped)\n", line.c_str(), c.name); + return; } all_zero = all_zero && (value == 0); line += " "; @@ -581,6 +629,9 @@ int firesim_sim::init(const std::vector &plusargs) { } int firesim_sim::dcr_write(uint32_t addr, uint32_t value) { + if (abort_phase_ != 0) { + return 0; + } return impl_->submit(label("dcr_write addr=0x%x data=0x%x", addr, value), [addr, value] { auto &pp = g_sim->peek_poke; pp.poke("ctrl_dcr_req_valid", 1, true); @@ -594,6 +645,10 @@ int firesim_sim::dcr_write(uint32_t addr, uint32_t value) { } int firesim_sim::dcr_read(uint32_t addr, uint32_t tag, uint32_t *value) { + if (abort_phase_ != 0) { + *value = 0; + return -1; + } uint32_t sampled = 0; bool answered = false; int ret = impl_->submit(label("dcr_read addr=0x%x", addr), [addr, tag, &sampled, &answered] { @@ -672,6 +727,10 @@ static void prefill_region() { int firesim_sim::start() { impl_->submit("prefill", [] { prefill_region(); }); + awaiting_busy_ = true; + idle_cycles_ = 0; + abort_phase_ = 0; + launch_started_s_ = wall_now_s(); return impl_->submit("start", [] { auto &pp = g_sim->peek_poke; pp.poke("ctrl_start", 1, true); @@ -682,6 +741,29 @@ int firesim_sim::start() { } int firesim_sim::is_busy(bool *busy) { + // After the watchdog has given up the target is no longer advanced at all: + // report busy once, which is what a launch still waiting for its rising edge + // needs, then idle, so every launch state reaches its end. + if (abort_phase_ != 0) { + *busy = (abort_phase_ == 1); + abort_phase_ = 2; + return 0; + } + if (launch_timeout_s() > 0 && launch_started_s_ > 0 && + wall_now_s() - launch_started_s_ > launch_timeout_s()) { + fprintf(stderr, + "[firesim] launch exceeded VORTEX_FIRESIM_TIMEOUT_S=%.0f s " + "(busy %s); giving up on it\n", + launch_timeout_s(), awaiting_busy_ ? "never rose" : "never fell"); + report_progress("launch watchdog"); + launch_started_s_ = 0; + abort_phase_ = awaiting_busy_ ? 1 : 2; + awaiting_busy_ = false; + *busy = (abort_phase_ == 1); + abort_phase_ = 2; + return 0; + } + uint32_t sampled = 0; // Advance before sampling: the target only moves while it is being asked to, // so a caller that merely polls without stepping would spin forever. @@ -699,6 +781,10 @@ int firesim_sim::is_busy(bool *busy) { // grow towards the ceiling and give a long kernel the throughput it needs. // Going idle drops back to the floor, ready for the next launch. if (*busy) { + if (awaiting_busy_) { + awaiting_busy_ = false; + stall_reported_ = false; + } const uint32_t ceiling = step_batch_ceiling(); step_batch_ = (batch >= ceiling / 4) ? ceiling : batch * 4; busy_cycles_ += batch; @@ -711,6 +797,15 @@ int firesim_sim::is_busy(bool *busy) { stall_reported_ = true; report_progress("target still busy"); } + } else if (awaiting_busy_) { + // Still waiting for the launch to take. A kernel is busy from its first + // fetch, so a start that has not raised busy within the stall budget never + // reached the core; say so once, with the counters, rather than poll on. + idle_cycles_ += batch; + if (!stall_reported_ && idle_cycles_ >= stall_report_cycles()) { + stall_reported_ = true; + report_progress("target never went busy after start"); + } } else { // Report on a healthy completion too, not only on a stall. Without this the // only counter dumps come from wedged runs, leaving nothing to compare a @@ -724,6 +819,7 @@ int firesim_sim::is_busy(bool *busy) { step_batch_ = std::min(kStepBatchFloor, step_batch_ceiling()); busy_cycles_ = 0; stall_reported_ = false; + launch_started_s_ = 0; } return 0; } @@ -746,7 +842,14 @@ int firesim_sim::ready_wait(int64_t timeout_ms) { } } +// Once a launch has been abandoned the target is left where it stopped, and +// its memory path may be the very thing that is stuck. Nothing more is sent to +// it, so the application runs to its own exit -- and reports its failure -- +// instead of blocking on a transfer that will never complete. int firesim_sim::mem_write(uint64_t addr, uint64_t size, const void *value) { + if (abort_phase_ != 0) { + return 0; + } dump_upload(addr, size, value); return impl_->submit( label("mem_write addr=0x%lx size=%lu", (unsigned long)addr, (unsigned long)size), @@ -754,6 +857,10 @@ int firesim_sim::mem_write(uint64_t addr, uint64_t size, const void *value) { } int firesim_sim::mem_read(uint64_t addr, uint64_t size, void *value) { + if (abort_phase_ != 0) { + memset(value, 0, size); + return 0; + } return impl_->submit( label("mem_read addr=0x%lx size=%lu", (unsigned long)addr, (unsigned long)size), [addr, size, value] { access_mem(addr, size, value, false); }); diff --git a/sim/firesim/firesim_sim.h b/sim/firesim/firesim_sim.h index d7f7b67577..de68098014 100644 --- a/sim/firesim/firesim_sim.h +++ b/sim/firesim/firesim_sim.h @@ -84,6 +84,18 @@ class firesim_sim { // been reported. Both reset when the target goes idle. uint64_t busy_cycles_; bool stall_reported_; + + // A launch waits for `busy` to rise and then to fall, and nothing bounds + // either wait. These track the launch in flight so a target that never + // starts is reported instead of polled forever in silence. + bool awaiting_busy_; + uint64_t idle_cycles_; + + // Set when the launch watchdog (VORTEX_FIRESIM_TIMEOUT_S) gives up. The + // launch is then completed -- busy once, then idle -- so the application + // finishes through its own teardown rather than being killed mid-run. + int abort_phase_; + double launch_started_s_; }; } From bb7ee975f56b88ea02a1bc03f2be612466f5cd2f Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 25 Sep 2026 16:20:36 -0700 Subject: [PATCH 159/336] firesim: use the soft RTU FMA and divider on aved On aved the simulator's clock gates become clock enables in synthesis, and that conversion cannot enter a vendor core: xil_fma and xil_fdiv are synthesized out of context behind a kept hierarchy, so they would stay on a logic-driven clock, which the platform build rejects. The RTU is the only unit that reaches for them by default, so on aved it takes the soft FMA and divider -- the RTL the metasimulator already runs -- with the soft divider's latency. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e783814cb84f87e8ac272e80b79453e9258f8f35) --- hw/syn/firesim/Makefile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 69656135f2..32972ff642 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -162,6 +162,18 @@ PLATFORM_DEFINES := -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 # checked, not just the overflow avoided. PLATFORM_DEFINES += -DPLATFORM_MEMORY_ID_WIDTH=4 +# On aved the simulator's clock gates are turned into clock enables by +# synthesis (the V80's reconfigurable partition has no BUFGCE, and a fabric gate +# is not safe; see FireSimFabricClockGate). That conversion cannot enter a +# vendor core: each is synthesized out of context behind a kept hierarchy, so +# it would stay on a logic-driven clock and the platform build rejects it. The +# RTU is the only unit that reaches for them by default, so it uses the soft +# FMA and divider here -- the same RTL the metasimulator runs -- with the soft +# divider's latency in place of xil_fdiv's. +ifeq ($(PLATFORM), aved) +PLATFORM_DEFINES += -DVX_CFG_RTU_USE_DSP=0 -DVX_CFG_RTU_FDIV_LATENCY=17 +endif + # Vortex's performance counters are the only view inside the core that does not # require re-elaborating the design: at a stall they say whether warps are still # resident, whether they are waiting at a barrier, and which pipeline is holding From 500f172aafafa3288689e63ff5fcc7b28988840f Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 25 Sep 2026 16:20:41 -0700 Subject: [PATCH 160/336] firesim: pin the fork commit that runs the V80 on clock enables Picks up: the aved simif setting and verifying the kernel clock at open; the V80 clock gates turned into clock enables, with a build check that none is left on a logic-driven clock; and the metasimulator's memory model able to reorder, interleave and delay reads. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 873531f06afdfdebd01f4de7899f85a0273275da) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 242eb697b3..ff5fd9b4a6 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=cf3159264f0f5f779e6d00bfbd6a2a1228f8a5df +FIRESIM_REV=0000d293c1e899dcc308eba9d2877654ca889260 From b63abf629cf9a020fa762c565400f3971b716785 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 25 Sep 2026 19:02:21 -0700 Subject: [PATCH 161/336] configure: regenerate the templates when a version pin changes A generated template is skipped when it is newer than both its source and the configure stamp, and the stamp's signature held the configure arguments but not the version pins (VORTEX_VERSION, TOOLCHAIN_REV, GEM5_REV, FIRESIM_REV) substituted into the same templates. A bumped pin therefore never reached an existing build tree: config.mk kept the old FIRESIM_REV through three fork bumps. The pins now join the signature. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0479535910d8778fe67754ea1246337c301b6922) --- configure | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configure b/configure index a585284055..0a37713f77 100755 --- a/configure +++ b/configure @@ -211,9 +211,12 @@ THIRD_PARTY_DIR=$SOURCE_DIR/third_party # load version pins (VORTEX_VERSION, TOOLCHAIN_REV, GEM5_REV, FIRESIM_REV) source "$SOURCE_DIR/VERSION" -# stamp file tracks config parameter changes; touched only when params change +# stamp file tracks config parameter changes; touched only when params change. +# The version pins are substituted into the same templates, so they are part of +# the signature: without them a bumped pin never reaches an existing build tree. CONFIG_STAMP="$CURRENT_DIR/.config.stamp" CONFIG_SIG="XLEN=$XLEN TOOLDIR=$TOOLDIR OSVERSION=$OSVERSION PREFIX=$PREFIX PROFILE=$PROFILE SOURCE_DIR=$SOURCE_DIR BUILDDIR=$CURRENT_DIR" +CONFIG_SIG="$CONFIG_SIG VORTEX_VERSION=$VORTEX_VERSION TOOLCHAIN_REV=$TOOLCHAIN_REV GEM5_REV=$GEM5_REV FIRESIM_REV=$FIRESIM_REV" if [ ! -f "$CONFIG_STAMP" ] || [ "$(cat "$CONFIG_STAMP" 2>/dev/null)" != "$CONFIG_SIG" ]; then echo "$CONFIG_SIG" > "$CONFIG_STAMP" fi From 88a458afd086040b1b8e74b5e37f6bcee5b66600 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 25 Sep 2026 19:02:21 -0700 Subject: [PATCH 162/336] firesim: build and run it with the native Xilinx flows' tokens FireSim runs the same RTL on the same cards as the xrt and aved drivers, but had its own vocabulary: FIRESIM_CARD, bitstream/sim-image goals, SYN_BUILD_DIR, PERF_DEFINES, images inside the FireSim fork, and FIRESIM_CARD again at run time. It now takes what the native flows take. Build (hw/syn/firesim): - PLATFORM= selects an XRT card and TARGET=hw|hw_emu; without PLATFORM the card is the aved one for DEVICE (default the V80) and TARGET=hw|sim, resolved through the same descriptor lookup as hw/syn/xilinx/{xrt,aved}. - PREFIX, XLEN, CONFIGS and PERF=1 as there. The image is copied to __/bin with firesim.card naming its card (and emconfig.json for hw_emu); everything generated lives under that directory. - Nothing given on the command line reaches FireSim's own makes (MAKEOVERRIDES is cleared): FireSim uses PLATFORM, TARGET and BUILD_DIR for its own purposes, and TARGET once reached v++ as `--target sim`. The descriptor's FireSim platform is FIRESIM_PLATFORM, never PLATFORM. - `stage` copies only what changed, so FireSim's elaboration is not redone on every call; `elaborate` runs Golden Gate alone for the runtime. Run (run-firesim, blackbox.sh --driver=firesim): - The native drivers' targets: xrtsim, hw_emu, hw on an XRT card; avedsim, sim, hw on an aved card. xrtsim is new -- the in-process metasimulator that avedsim already was, built for the XRT card's platform -- and is the default, as the xrt driver defaults to its simulator. - sw/runtime/firesim/card.mk decides the card identically for the runtime build, run-firesim and blackbox.sh's own driver build: from FPGA_BIN_DIR/firesim.card for an image target, the U55C or V80 for the metasimulator (FIRESIM_CARD overrides). Each card's metasimulator gets its own model and flattening directory. Safety and environment: - ci/v80_preflight.sh runs before every aved TARGET=hw: the card's functions and root port on the bus, v80-smi healthy, no other VRT user, and a 60 s cool-down since the last design write -- the conditions every host reset so far followed. It finds the card and port from sysfs. - ci/firesim_env.sh adds FireSim's conda environment, the host compiler and linker VRT needs, and Vivado's simulator libraries on top of the card's Xilinx tools, and warns about a stale /opt VRT on the library path. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4c9955bca654de0dcddc79bd1c041b612adcd3be) Integration (phase 2): the perf-counter note names VX_csr_mpm.sv, where phase 2 moved the PERF_ENABLE gate; the blackbox help takes this commit's per-driver target list. --- ci/blackbox.sh | 6 +- ci/firesim_env.sh | 65 ++++++++++ ci/v80_preflight.sh | 66 ++++++++++ hw/scripts/gen_platform.py | 2 +- hw/syn/firesim/Makefile | 241 +++++++++++++++++++++++++----------- sim/firesim/Makefile | 41 +++--- sw/runtime/firesim/Makefile | 23 ++-- sw/runtime/firesim/card.mk | 39 ++++++ tests/regression/common.mk | 25 ++-- 9 files changed, 392 insertions(+), 116 deletions(-) create mode 100644 ci/firesim_env.sh create mode 100755 ci/v80_preflight.sh create mode 100644 sw/runtime/firesim/card.mk diff --git a/ci/blackbox.sh b/ci/blackbox.sh index c541d04c14..6ae40c9cb7 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -26,8 +26,10 @@ show_help() { show_usage echo " where" - echo "--driver: gpu, simx, rtlsim, opae, xrt, aved" - echo "--target: xrt -- xrtsim (default), hw_emu, hw; firesim -- hw, hw_emu; aved -- avedsim (default), sim, hw" + echo "--driver: gpu, simx, rtlsim, opae, xrt, aved, firesim" + echo "--target: xrt: xrtsim (default), hw_emu, hw -- aved: avedsim (default), sim, hw" + echo " firesim: the targets of the card's native driver; xrtsim (default) or" + echo " avedsim need no image, the others run FPGA_BIN_DIR's (hw/syn/firesim)" echo "--app: any subfolder test under regression, graphics, mpi, opencl, or hip" echo "--perf: 0=disable, 1=core, 3=icache, 4=dcache, 5=l2cache, 6=l3cache, 7=mem," echo " 11=tcu, 12=raster, 13=tex, 14=om, 15=rtu, 16=dxa" diff --git a/ci/firesim_env.sh b/ci/firesim_env.sh new file mode 100644 index 0000000000..a9cae0f7f0 --- /dev/null +++ b/ci/firesim_env.sh @@ -0,0 +1,65 @@ +# Environment for building and running Vortex through FireSim. Source it from +# the build directory, after setting up the card's Xilinx tools exactly as the +# native flow for that card needs them (Vitis and XRT for an XRT card; Vivado +# and VRT for an aved card): +# +# source ci/firesim_env.sh +# +# It adds only what FireSim needs on top of those: +# - FireSim's conda environment, through its env.sh (conda must be on PATH); +# - the host compiler and linker, which VRT and the aved simulator build +# need: conda's cannot link against the host glibc those libraries use; +# - Vivado's simulator libraries, which an aved TARGET=sim run loads. +# +# FIRESIM_PATH the FireSim fork checkout (default: config.mk's) +# FIRESIM_ENV_SH FireSim's env.sh, if its conda environment lives in another +# checkout (default: $FIRESIM_PATH/env.sh) + +_fs_fail() { echo "firesim_env: $*" >&2; return 1; } + +_fs_main() { + if [ -z "${FIRESIM_PATH:-}" ] && [ -f config.mk ]; then + FIRESIM_PATH=$(make -s -f config.mk -f - <<<'print-fs: ; @echo $(FIRESIM_PATH)' print-fs 2>/dev/null) + fi + [ -n "${FIRESIM_PATH:-}" ] || { _fs_fail "set FIRESIM_PATH, or source this from a configured build directory"; return 1; } + [ -d "$FIRESIM_PATH/sim" ] || { _fs_fail "no FireSim checkout at FIRESIM_PATH=$FIRESIM_PATH"; return 1; } + export FIRESIM_PATH + + command -v conda >/dev/null 2>&1 || { _fs_fail "conda is not on PATH (FireSim's env.sh needs it)"; return 1; } + local env_sh=${FIRESIM_ENV_SH:-$FIRESIM_PATH/env.sh} + [ -f "$env_sh" ] || { _fs_fail "FireSim's env.sh not found at $env_sh (set FIRESIM_ENV_SH)"; return 1; } + # env.sh reads variables it has not set, so it cannot run under set -u. + local had_u=0; case $- in *u*) had_u=1; set +u ;; esac + source "$env_sh" >/dev/null 2>&1 + [ $had_u = 1 ] && set -u + # env.sh may repoint FIRESIM_PATH at its own checkout; keep the one asked for. + export FIRESIM_PATH + + # The host compiler, and the host linker ahead of conda's under the plain + # name `ld` that CMake-driven builds invoke. + export CC=/usr/bin/gcc CXX=/usr/bin/g++ + local hostld=${XDG_CACHE_HOME:-$HOME/.cache}/vortex-firesim/hostld + mkdir -p "$hostld" && ln -sf /usr/bin/ld "$hostld/ld" + case ":$PATH:" in *":$hostld:"*) ;; *) export PATH="$hostld:$PATH" ;; esac + + if [ -n "${XILINX_VIVADO:-}" ]; then + case ":${LD_LIBRARY_PATH:-}:" in + *":$XILINX_VIVADO/lib/lnx64.o:"*) ;; + *) export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$XILINX_VIVADO/lib/lnx64.o" ;; + esac + else + echo "firesim_env: XILINX_VIVADO is not set; set up the card's Xilinx tools before building" >&2 + fi + + # A stale VRT ahead of the packaged one on the library path runs every V80 + # image at the shell's boot clock instead of the frequency it was built for. + case ":${LD_LIBRARY_PATH:-}:" in + *":/opt/xilinx/slash/lib:"*) + echo "firesim_env: warning: /opt/xilinx/slash/lib is on LD_LIBRARY_PATH; make sure it is the same VRT as the installed vrtd" >&2 ;; + esac + + echo "firesim_env: FIRESIM_PATH=$FIRESIM_PATH" +} + +_fs_main +unset -f _fs_main _fs_fail diff --git a/ci/v80_preflight.sh b/ci/v80_preflight.sh new file mode 100755 index 0000000000..bbd9f7f89a --- /dev/null +++ b/ci/v80_preflight.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Refuses to let a run program an Alveo V80 that is not in a state to take it. +# +# Every host reset seen with this card followed one of these: a design write to +# a card that was off the bus or unhealthy, a write while another process still +# had it open, or two writes back to back. Run before anything that programs the +# card (run-firesim TARGET=hw does); it exits non-zero, saying why, instead. +# +# VRT_DEVICE_BDF the card to check (bus:dev, e.g. 01:00); found on the bus if unset +# V80_COOLDOWN_S seconds to wait after the last design write (default 60) + +set -u +COOLDOWN=${V80_COOLDOWN_S:-60} + +fail() { echo "v80_preflight: $*" >&2; exit 3; } + +# The card: PF0 of a V80 (10ee:50b4), or the one named. +BDF=${VRT_DEVICE_BDF:-} +if [ -z "$BDF" ]; then + BDF=$(lspci -D -d 10ee:50b4 2>/dev/null | head -1 | cut -d' ' -f1) + [ -n "$BDF" ] || fail "no V80 (10ee:50b4) on the PCIe bus; if it dropped off, the host needs a cold power cycle" + BDF=${BDF%.*} +fi +case "$BDF" in *:*:*) ;; *) BDF="0000:$BDF" ;; esac + +# All three physical functions, and the port above them. +NFUNC=$(lspci -D -s "$BDF" 2>/dev/null | wc -l) +[ "$NFUNC" -eq 3 ] || fail "$BDF shows $NFUNC of its 3 functions on the bus" +PORT=$(basename "$(dirname "$(readlink -f /sys/bus/pci/devices/$BDF.0)")") +[ -e "/sys/bus/pci/devices/$PORT" ] || fail "the root port above $BDF ($PORT) is not present" + +# The board as the shell and vrtd see it. +SHORT=${BDF#0000:} +LIST=$(timeout 20 v80-smi list 2>&1) +echo "$LIST" | grep -q "Board 0000:$SHORT OK (PF0: OK) (PF1: OK) (PF2: OK) (VRTD: OK)" \ + || fail "v80-smi does not report $BDF healthy: +$LIST" + +# Nobody else holding a VRT session: a design write under a live user is what +# takes the card down. +USERS="" +for m in /proc/[0-9]*/maps; do + pid=${m#/proc/}; pid=${pid%/maps} + [ "$pid" = "$$" ] && continue + grep -q 'libvrt\.so' "$m" 2>/dev/null && USERS="$USERS $pid" +done +[ -z "$USERS" ] || fail "processes still hold VRT sessions:$(ps -o pid=,etime=,args= -p ${USERS// /,} | sed 's/^/\n /')" + +# A cool-down since the last design write. The journal is the only record of +# it; a user who cannot read vrtd's entries there cannot have the wait judged, +# which is reported rather than passed off as "no recent write". +JOURNAL=$(journalctl -t vrtd --since "-$((COOLDOWN + 5))s" -o short-unix --no-pager 2>&1) +if echo "$JOURNAL" | grep -qiE "insufficient permissions|no journal files"; then + echo "v80_preflight: cannot read the vrtd journal; not enforcing the ${COOLDOWN}s cool-down" >&2 +else + LAST=$(echo "$JOURNAL" | grep "Design write completed" | tail -1 | cut -d' ' -f1 | cut -d. -f1) + if [ -n "$LAST" ]; then + WAIT=$(( LAST + COOLDOWN - $(date +%s) )) + if [ "$WAIT" -gt 0 ]; then + echo "v80_preflight: last design write was $(( COOLDOWN - WAIT ))s ago; waiting ${WAIT}s" + sleep "$WAIT" + fi + fi +fi + +echo "v80_preflight: $BDF ready (port $PORT)" diff --git a/hw/scripts/gen_platform.py b/hw/scripts/gen_platform.py index daf200e542..551242c0c5 100644 --- a/hw/scripts/gen_platform.py +++ b/hw/scripts/gen_platform.py @@ -265,7 +265,7 @@ def emit_firesim_mk(c: Card) -> str: fs = _firesim(c) lines = [f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", f"FIRESIM_CARD := {c.name}", - f"PLATFORM := {fs['platform']}", + f"FIRESIM_PLATFORM := {fs['platform']}", f"PLATFORM_CONFIG := {firesim_config_class(c)}", f"FPGA_PART := {c.part}", f"HOST_MEM_SP := {fs['host_mem_sp']}"] diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 32972ff642..fc66caa0f7 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -1,13 +1,19 @@ -# FireSim platform backend. +# FireSim image build for the Xilinx cards, driven exactly as the native flows: +# +# make PLATFORM= [TARGET=hw|hw_emu] an XRT card, as hw/syn/xilinx/xrt +# make [DEVICE=] [TARGET=hw|sim] an aved card, as hw/syn/xilinx/aved +# +# with PREFIX, XLEN, CONFIGS and PERF=1 as there too. The image lands in +# $(BUILD_DIR)/bin as firesim.xclbin or firesim.vbin, beside firesim.card, which +# names the card for run-firesim; the runtime's xrtsim/avedsim targets need no +# image at all. # # FireSim itself is the vortexgpgpu/firesim fork, consumed through FIRESIM_PATH # and pinned by FIRESIM_REV in VERSION: every change Vortex needs in FireSim is a # commit there, and the checkout must contain the pinned one. The Chisel targets, -# harnesses and the card's platform config live here and are staged into it. -# -# The card comes from its descriptor (hw/syn/xilinx/platforms/.toml): -# FIRESIM_CARD selects it, and the descriptor supplies the FireSim platform, the -# FPGA part and device, the host-memory binding and its base address. +# harnesses and the card's platform config live here and are staged into it. The +# card's descriptor (hw/syn/xilinx/platforms/.toml) supplies the FireSim +# platform, the FPGA part and device, the host-memory binding and its base. ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk @@ -31,40 +37,97 @@ CC_DEST := $(FIRESIM_PATH)/sim/src/main/cc/midasexamples RTL_DIR := $(VORTEX_HOME)/hw/rtl SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts -# Overridden as SYN_BUILD_DIR, never as BUILD_DIR. A command-line variable -# reaches every sub-make, and FireSim's own Makefile uses BUILD_DIR to find its -# elaborator jar: overriding it here pointed FireSim at this directory and the -# elaboration died on "Could not find or load main class". -SYN_BUILD_DIR ?= $(ROOT_DIR)/hw/syn/firesim -BUILD_DIR := $(SYN_BUILD_DIR) +XLEN ?= 32 +PREFIX ?= build$(XLEN) + +# Nothing given here on the command line reaches FireSim's own makes. A +# command-line variable propagates to every sub-make, and FireSim uses several +# of these names for its own purposes: PLATFORM is its platform (vitis/aved), +# not an XSA; TARGET reached v++ as `--target sim`; BUILD_DIR locates its +# elaborator jar. Each sub-make below is handed exactly what it needs instead. +MAKEOVERRIDES := + +# The card, named as the native flow for its kind names it. The runtime has an +# internal need to point the flattening elsewhere (FIRESIM_CARD, FIRESIM_WORK_DIR); +# those are not user knobs. +ifneq ($(FIRESIM_CARD),) +PLATFORM_GEN_ARGS := --card=$(FIRESIM_CARD) +else ifneq ($(PLATFORM),) +XSA := $(strip $(patsubst %.xpfm,%,$(notdir $(PLATFORM)))) +DEV_ARCH := $(shell platforminfo -p $(PLATFORM) 2>/dev/null | grep 'FPGA Family' | sed 's/.*://' | sed '/ai_engine/d' | sed 's/^[[:space:]]*//') +PLATFORM_GEN_ARGS := --flow=xrt --xsa=$(XSA) --family='$(DEV_ARCH)' +else +DEVICE ?= xcv80-lsva4737-2MHP-e-S +PLATFORM_GEN_ARGS := --flow=aved --device=$(DEVICE) +endif +PLATFORM_GEN := python3 $(SCRIPT_DIR)/gen_platform.py $(PLATFORM_GEN_ARGS) + +# The card is resolved before BUILD_DIR exists, because BUILD_DIR is named after +# it. $(shell) discards exit codes, so an empty answer means the generator +# failed, and it is asked again for its message. +FIRESIM_CARD := $(shell $(PLATFORM_GEN) --emit=card 2>/dev/null) +ifeq ($(FIRESIM_CARD),) +$(error $(shell $(PLATFORM_GEN) --emit=card 2>&1 >/dev/null)) +endif +FIRESIM_PLATFORM := $(shell python3 $(SCRIPT_DIR)/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk 2>&1 | sed -n 's/^FIRESIM_PLATFORM := //p') +ifeq ($(FIRESIM_PLATFORM),) +$(error $(shell python3 $(SCRIPT_DIR)/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk 2>&1 >/dev/null)) +endif -FIRESIM_CARD ?= u55c -PLATFORM_GEN := python3 $(SCRIPT_DIR)/gen_platform.py --card=$(FIRESIM_CARD) -FIRESIM_MK := $(BUILD_DIR)/firesim_platform.mk -FIRESIM_GEN_RC := $(shell mkdir -p $(BUILD_DIR) && $(PLATFORM_GEN) --emit=firesim-mk -o $(FIRESIM_MK) 2> $(FIRESIM_MK).err; echo $$?) -ifneq ($(FIRESIM_GEN_RC),0) -$(error $(shell cat $(FIRESIM_MK).err)) +# The image each card can take, as its native flow names it. +TARGET ?= hw +ifeq ($(FIRESIM_PLATFORM), aved) +VALID_TARGETS := hw sim +else +VALID_TARGETS := hw hw_emu +endif +ifeq ($(FIRESIM_WORK_DIR),) +ifeq ($(filter $(TARGET),$(VALID_TARGETS)),) +$(error TARGET=$(TARGET) is not an image target for card $(FIRESIM_CARD): use one of $(VALID_TARGETS) \ + (xrtsim and avedsim are run targets and need no image)) endif +endif + +# Named as the native flows name theirs, so FPGA_BIN_DIR means the same thing. +ifeq ($(FIRESIM_PLATFORM), aved) +BUILD_DIR := $(abspath $(PREFIX)_aved_$(TARGET)) +else +BUILD_DIR := $(abspath $(PREFIX)_$(if $(XSA),$(XSA),$(FIRESIM_CARD))_$(TARGET)) +endif +# The runtime's metasimulator reuses the simulation flattening from here, in a +# directory of its own. +ifneq ($(FIRESIM_WORK_DIR),) +BUILD_DIR := $(abspath $(FIRESIM_WORK_DIR)) +endif +BIN_DIR := $(BUILD_DIR)/bin + +FIRESIM_MK := $(BUILD_DIR)/firesim_platform.mk +$(shell mkdir -p $(BUILD_DIR) && $(PLATFORM_GEN) --emit=firesim-mk -o $(FIRESIM_MK)) include $(FIRESIM_MK) +# A user-named XSA is the one to build for; the descriptor's is the default. +ifneq ($(XSA),) +DEVICE := $(XSA) +endif + # The FireSim flow's make variables for this card's build. -FIRESIM_TARGET_ARGS := PLATFORM=$(PLATFORM) TARGET_PROJECT=midasexamples \ +FIRESIM_TARGET_ARGS := PLATFORM=$(FIRESIM_PLATFORM) TARGET_PROJECT=midasexamples \ DESIGN=VortexTarget TARGET_CONFIG=VortexConfig PLATFORM_CONFIG=$(PLATFORM_CONFIG) -QUINTUPLET := $(PLATFORM)-midasexamples-VortexTarget-VortexConfig-$(PLATFORM_CONFIG) +QUINTUPLET := $(FIRESIM_PLATFORM)-midasexamples-VortexTarget-VortexConfig-$(PLATFORM_CONFIG) # Flattened Vortex sources for the BlackBox. Golden Gate takes a file list, not # include paths, so the tree is copied into one directory with the macros already # inlined (-P) and emitted in elaboration order: packages, interfaces, modules. # -# The platform's simulation image (sim-image) gets its own flattening, under +# A simulation image (TARGET=sim or hw_emu) gets its own flattening, under # -DSIMULATION. Under VIVADO, VX_platform.vh enables ASYNC_BRAM_PATCH unless # SIMULATION is defined, and that patch leaves every async RAM's read address # behind an empty VX_placeholder black box which only the opt_design hook # rewires. A behavioural simulation never runs opt_design, so without the # define those RAMs never latch an address and read back X -- the same reason # the native xrt flow adds -DSIMULATION for hw_emu. -IMAGE_KIND ?= hw -ifeq ($(IMAGE_KIND), sim) +ifneq ($(filter $(TARGET),sim hw_emu),) +IMAGE_KIND := sim IMAGE_SUFFIX := _sim endif VIVADO_SRCS_NAME := vivado_srcs$(IMAGE_SUFFIX) @@ -77,23 +140,36 @@ SRCS_DIR := $(BUILD_DIR)/vortex_srcs SRCS_LIST := $(BUILD_DIR)/vortex_srcs.f SIM_SRCS_STAMP := $(SRCS_DIR)/.sources.stamp -.PHONY: stage sources sim-sources vivado-sources xilinx-ips kernel-env bitstream sim-image \ +.PHONY: all help clean elaborate stage sources sim-sources vivado-sources xilinx-ips kernel-env image \ metasim run-metasim check-firesim clean-staged +.DEFAULT_GOAL := all + +help: + @echo "FireSim image build (same tokens as hw/syn/xilinx/xrt and hw/syn/xilinx/aved):" + @echo " make PLATFORM= [TARGET=hw|hw_emu] XRT card" + @echo " make [DEVICE=] [TARGET=hw|sim] aved card (default: the V80)" + @echo " options: PREFIX= XLEN=32|64 CONFIGS=\"-DVX_CFG_...\" PERF=1" + @echo " image: __/bin/firesim.{xclbin,vbin}" + @echo " run: make -C tests/regression/ run-firesim TARGET= FPGA_BIN_DIR=" + @echo " (TARGET=xrtsim|avedsim need no image)" + @echo " developer goals: metasim run-metasim sources clean" + +# Copies only what changed: FireSim's make judges its elaboration by these +# files' timestamps, so rewriting identical files would redo Golden Gate (and +# relink the metasimulator) on every run. stage: check-firesim @mkdir -p $(TARGET_DEST) $(CC_DEST) @for f in $(SCALA_SRCS); do \ - cp -f "$$f" $(TARGET_DEST)/ || exit 1; \ - echo "staged $$(basename $$f)"; \ + cmp -s "$$f" $(TARGET_DEST)/$$(basename $$f) || { cp -f "$$f" $(TARGET_DEST)/ && echo "staged $$(basename $$f)"; } || exit 1; \ done @for f in $(CC_SRCS); do \ - cp -f "$$f" $(CC_DEST)/ || exit 1; \ - echo "staged $$(basename $$f)"; \ + cmp -s "$$f" $(CC_DEST)/$$(basename $$f) || { cp -f "$$f" $(CC_DEST)/ && echo "staged $$(basename $$f)"; } || exit 1; \ done - @$(PLATFORM_GEN) --emit=scala -o $(TARGET_DEST)/$(PLATFORM_CONFIG).scala - @echo "staged $(PLATFORM_CONFIG).scala ($(FIRESIM_CARD))" - -XLEN ?= 32 + @$(PLATFORM_GEN) --emit=scala -o $(BUILD_DIR)/$(PLATFORM_CONFIG).scala + @cmp -s $(BUILD_DIR)/$(PLATFORM_CONFIG).scala $(TARGET_DEST)/$(PLATFORM_CONFIG).scala || { \ + cp -f $(BUILD_DIR)/$(PLATFORM_CONFIG).scala $(TARGET_DEST)/ && \ + echo "staged $(PLATFORM_CONFIG).scala ($(FIRESIM_CARD))"; } # Architecture overrides, as the other synthesis flows take them, e.g. # CONFIGS="-DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TEX_ENABLE". XCONFIGS is the fully @@ -170,7 +246,7 @@ PLATFORM_DEFINES += -DPLATFORM_MEMORY_ID_WIDTH=4 # RTU is the only unit that reaches for them by default, so it uses the soft # FMA and divider here -- the same RTL the metasimulator runs -- with the soft # divider's latency in place of xil_fdiv's. -ifeq ($(PLATFORM), aved) +ifeq ($(FIRESIM_PLATFORM), aved) PLATFORM_DEFINES += -DVX_CFG_RTU_USE_DSP=0 -DVX_CFG_RTU_FDIV_LATENCY=17 endif @@ -181,11 +257,12 @@ endif # it never retired looks exactly like a core with nothing left to do. They are # read over the DCR response path, which both the harness and the driver already # drive, but VX_csr_mpm.sv gates them behind PERF_ENABLE. Off by default because -# they cost area; build with PERF_DEFINES=-DPERF_ENABLE to read them, and note -# that without it every counter reads zero, which is indistinguishable from an -# idle core rather than an absent instrument. -PERF_DEFINES ?= -PLATFORM_DEFINES += $(PERF_DEFINES) +# they cost area; build with PERF=1 to read them, as in the other synthesis +# flows, and note that without it every counter reads zero, which is +# indistinguishable from an idle core rather than an absent instrument. +ifdef PERF +PLATFORM_DEFINES += -DPERF_ENABLE +endif GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) $(SYNTH_DEFINES) $(RTL_INCLUDE) @@ -350,44 +427,63 @@ kernel-env: @echo 'export VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl' @echo 'export TOOL_DIR=$(SCRIPT_DIR)' -# One command from sources to the card's image. Expects the FireSim environment -# (its env.sh) and the Xilinx tools for the card's platform in the environment. -# $(SRCS_LIST) is deliberately not a prerequisite: the recipe hands synthesis the -# Vivado-flattened list, and pulling the simulation one in would flatten the tree -# a second time under the opposite defines for nothing. -bitstream: vivado-sources xilinx-ips stage - $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) - VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ - VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl TOOL_DIR=$(SCRIPT_DIR) \ - $(MAKE) -C $(FIRESIM_PATH)/platforms/$(PLATFORM)/cl_$(QUINTUPLET) $(PLATFORM_IMAGE_GOAL) \ - DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) - -# The same image built for the platform's own simulator instead of the fabric. -sim-image: - $(MAKE) bitstream PLATFORM_IMAGE_GOAL='$(PLATFORM_SIM_GOAL)' IMAGE_KIND=sim - FREQUENCY ?= 60.0 STRATEGY ?= TIMING -# How each platform names its two images, kept here so the difference does not -# leak into the caller. aved builds both from one goal and selects with TARGET, -# putting each in build_dir.$(TARGET); vitis has a separate `sim` goal that -# selects hw_emu. -# -# Passing TARGET through to vitis would be worse than useless: its Makefile sets -# `bitstream: TARGET = hw` as a target-specific variable, and a command-line -# TARGET beats a target-specific one, so TARGET=sim would reach v++ as -# `--target sim`, which is not a Vitis build target at all. -ifeq ($(PLATFORM),aved) -PLATFORM_HW_GOAL := bitstream TARGET=hw -PLATFORM_SIM_GOAL := bitstream TARGET=sim -else -PLATFORM_HW_GOAL := bitstream +# How the card's FireSim platform spells this TARGET, and where it leaves the +# image. aved builds both images from one goal and selects with its own TARGET, +# putting each in build_dir.$(TARGET); vitis has a separate `sim` goal for +# hw_emu, and must never see TARGET at all -- its Makefile sets +# `bitstream: TARGET = hw` as a target-specific variable, which a passed-in +# TARGET would beat, sending `--target sim` to v++. +PLATFORM_BUILD_DIR := $(FIRESIM_PATH)/platforms/$(FIRESIM_PLATFORM)/cl_$(QUINTUPLET) +ifeq ($(FIRESIM_PLATFORM), aved) +PLATFORM_IMAGE_GOAL := bitstream TARGET=$(TARGET) +PLATFORM_IMAGE := $(PLATFORM_BUILD_DIR)/bitstream/build_dir.$(TARGET)/firesim.vbin +IMAGE_NAME := firesim.vbin +else ifeq ($(TARGET), hw_emu) # Without debug: a debug hw_emu kernel logs a waveform of the whole shell on # every run (~800 MB for demo, tens of GB for raycast) and runs far slower. -PLATFORM_SIM_GOAL := sim VPP_DEBUG= +PLATFORM_IMAGE_GOAL := sim VPP_DEBUG= +PLATFORM_IMAGE := $(PLATFORM_BUILD_DIR)/simulation/build_dir.$(DEVICE)/firesim.xclbin +IMAGE_NAME := firesim.xclbin +else +PLATFORM_IMAGE_GOAL := bitstream +PLATFORM_IMAGE := $(PLATFORM_BUILD_DIR)/bitstream/build_dir.$(DEVICE)/firesim.xclbin +IMAGE_NAME := firesim.xclbin +endif + +# One command from sources to the card's image, collected into $(BIN_DIR) with +# what running it needs beside it. Expects the FireSim environment +# (ci/firesim_env.sh) and the Xilinx tools for the card's platform. +all: image + @mkdir -p $(BIN_DIR) + cp -f $(PLATFORM_IMAGE) $(BIN_DIR)/$(IMAGE_NAME) + @echo $(FIRESIM_CARD) > $(BIN_DIR)/firesim.card +ifeq ($(TARGET), hw_emu) + emconfigutil --platform $(DEVICE) --od $(BIN_DIR) endif -PLATFORM_IMAGE_GOAL ?= $(PLATFORM_HW_GOAL) + @echo "image: $(abspath $(BIN_DIR))/$(IMAGE_NAME) (card $(FIRESIM_CARD), TARGET=$(TARGET))" + +# The FireSim platform's own build. $(SRCS_LIST) is deliberately not a +# prerequisite: the recipe hands synthesis the Vivado-flattened list, and pulling +# the simulation one in would flatten the tree a second time under the opposite +# defines for nothing. +image: vivado-sources xilinx-ips stage + $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) + VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ + VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl TOOL_DIR=$(SCRIPT_DIR) \ + $(MAKE) -C $(PLATFORM_BUILD_DIR) $(PLATFORM_IMAGE_GOAL) \ + DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) + +# Golden Gate's elaboration of the card's target on its own, for the runtime's +# metasimulator (xrtsim/avedsim), which needs the generated RTL and headers but +# no image. An image build elaborates as part of replace-rtl. +elaborate: stage + $(MAKE) -C $(FIRESIM_PATH)/sim verilog $(FIRESIM_TARGET_ARGS) + +clean: + rm -rf $(BUILD_DIR) # The metasimulator: the FireSim-transformed design in Verilator, driven through # the DPI emul simif. No FPGA and no Vivado, so it runs the same target the card @@ -413,9 +509,8 @@ metasim: sources stage # Runs it. ARGS reaches the harness; the metasim needs no image, since the design # is the simulator. # -# Deliberately not dependent on `metasim`: `stage` copies the harness sources -# unconditionally, so every invocation would bump their mtimes and relink the -# 460 MB model before running it. Build with `make metasim` first. +# Not dependent on `metasim`, so running never rebuilds the 460 MB model behind +# the caller's back. Build with `make metasim` first. run-metasim: env -u VERILATOR $(MAKE) -C $(FIRESIM_PATH)/sim run-verilator $(FIRESIM_TARGET_ARGS) ARGS="$(ARGS)" diff --git a/sim/firesim/Makefile b/sim/firesim/Makefile index 7a9f5b2d7f..eaa3d7cde0 100644 --- a/sim/firesim/Makefile +++ b/sim/firesim/Makefile @@ -11,8 +11,14 @@ SIMIF_DIR := $(FIRESIM_PATH)/sim/midas/src/main/cc DESIGN ?= VortexTarget TARGET_CONFIG ?= VortexConfig +# Nothing given on this make's command line reaches FireSim's makes: FireSim +# uses PLATFORM and TARGET for its own purposes. Each sub-make is handed what it +# needs instead. +MAKEOVERRIDES := + # The card's FireSim platform and platform config come from its descriptor, the -# same ones hw/syn/firesim elaborated the target with. +# same ones hw/syn/firesim elaborated the target with. The runtime names the +# card (from the image, or the metasimulator's default). FIRESIM_CARD ?= u55c FIRESIM_MK := $(DESTDIR)/firesim_platform.mk FIRESIM_GEN_RC := $(shell mkdir -p $(DESTDIR) && python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk -o $(FIRESIM_MK) 2> $(FIRESIM_MK).err; echo $$?) @@ -23,7 +29,7 @@ include $(FIRESIM_MK) # Golden Gate writes the generated header here; the driver objects are built # alongside it so a rebuild of the target invalidates them. -GEN_DIR := $(FIRESIM_PATH)/sim/output/$(PLATFORM)/$(PLATFORM)-midasexamples-$(DESIGN)-$(TARGET_CONFIG)-$(PLATFORM_CONFIG)/build +GEN_DIR := $(FIRESIM_PATH)/sim/output/$(FIRESIM_PLATFORM)/$(FIRESIM_PLATFORM)-midasexamples-$(DESIGN)-$(TARGET_CONFIG)-$(PLATFORM_CONFIG)/build # The driver is built as a shared library rather than FireSim's usual # executable, because the Vortex runtime loads it. Upstream's driver.mk only @@ -41,10 +47,10 @@ SIM_LDFLAGS := -shared -L$(CONDA_PREFIX)/lib -ldl -lz # The platform layer's own library: XRT for vitis cards, VRT for the V80. VRT # is built against the host glibc and libstdc++, so its platform layer is # compiled by the host toolchain, and its headers need jsoncpp's and libxml2's. -ifeq ($(PLATFORM), vitis) +ifeq ($(FIRESIM_PLATFORM), vitis) SIM_CXXFLAGS += -idirafter $(XILINX_XRT)/include SIM_LDFLAGS += -L$(XILINX_XRT)/lib -luuid -lxrt_coreutil -else ifeq ($(PLATFORM), aved) +else ifeq ($(FIRESIM_PLATFORM), aved) SIM_CXX := /usr/bin/g++ SIM_CXXFLAGS += $(shell pkg-config --cflags jsoncpp libxml-2.0) SIM_LDFLAGS += -L/usr/lib/x86_64-linux-gnu -lvrt @@ -68,16 +74,16 @@ check-gen: # Per platform: the objects embed that platform's generated widths, and make # judges them by timestamp, so a directory shared between cards would link one # card's objects into the other's library. -PIC_DIR := $(DESTDIR)/firesim_pic_$(PLATFORM) +PIC_DIR := $(DESTDIR)/firesim_pic_$(FIRESIM_PLATFORM) $(PIC_DIR)/FireSim-generated.const.h: $(GEN_DIR)/FireSim-generated.const.h @mkdir -p $(PIC_DIR) @cp -f $(GEN_DIR)/FireSim-generated.const.* $(PIC_DIR)/ 2>/dev/null || true @cp -f $(GEN_DIR)/FireSim-generated.const.h $(PIC_DIR)/ -ifeq ($(TARGET), avedsim) +ifneq ($(filter $(TARGET),xrtsim avedsim),) -# The metasimulator as the library: the FireSim-transformed target Verilated, +# The metasimulator as the library, for either kind of card: the FireSim-transformed target Verilated, # with the emulated host interface (simif_emul) in place of the card's, linked # into lib$(PROJECT).so so the runtime loads it exactly as it loads the card's. # firesim_emul_main.cpp supplies the loop that --main would otherwise generate. @@ -86,18 +92,21 @@ ifeq ($(TARGET), avedsim) # sources the standalone metasim builds from; the generated FireSim RTL and # headers come from the elaborated target. SYN_DIR := $(ROOT_DIR)/hw/syn/firesim -SYN_BUILD_DIR ?= $(SYN_DIR) -SIM_SRCS_LIST := $(SYN_BUILD_DIR)/vortex_srcs.f -SIM_SRCS_DIR := $(SYN_BUILD_DIR)/vortex_srcs -ELAB_DIR := $(FIRESIM_PATH)/sim/generated-src/$(PLATFORM)/$(PLATFORM)-midasexamples-$(DESIGN)-$(TARGET_CONFIG)-$(PLATFORM_CONFIG) -EMUL_DIR := $(DESTDIR)/firesim_emul +EMUL_DIR := $(DESTDIR)/firesim_emul_$(FIRESIM_CARD) +WORK_DIR := $(EMUL_DIR)/work +SIM_SRCS_LIST := $(WORK_DIR)/vortex_srcs.f +SIM_SRCS_DIR := $(WORK_DIR)/vortex_srcs +ELAB_DIR := $(FIRESIM_PATH)/sim/generated-src/$(FIRESIM_PLATFORM)/$(FIRESIM_PLATFORM)-midasexamples-$(DESIGN)-$(TARGET_CONFIG)-$(PLATFORM_CONFIG) EMUL_NAME := firesim_emul EMUL_SRCS := $(SRC_DIR)/firesim_sim.cpp $(SRC_DIR)/firesim_emul_main.cpp .PHONY: emul-sources +# The simulation flattening of the Vortex RTL for CONFIGS, and Golden Gate's +# elaboration of the card's target; both are no-ops when current. emul-sources: - $(MAKE) -C $(SYN_DIR) sources SYN_BUILD_DIR=$(SYN_BUILD_DIR) FIRESIM_CARD=$(FIRESIM_CARD) + $(MAKE) -C $(SYN_DIR) sources elaborate FIRESIM_CARD=$(FIRESIM_CARD) FIRESIM_WORK_DIR=$(WORK_DIR) \ + XLEN=$(XLEN) CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) # Host toolchain, as for the aved platform library: the runtime that loads this # is built with it. -fPIC reaches the Verilated objects too, through -CFLAGS. @@ -110,7 +119,7 @@ emul-sources: $(EMUL_DIR)/V$(EMUL_NAME): $(EMUL_SRCS) $(SIM_SRCS_LIST) $(ELAB_DIR)/FireSim-generated.sv rm -f $@ $(MAKE) -C $(SIMIF_DIR) verilator \ - PLATFORM=$(PLATFORM) \ + PLATFORM=$(FIRESIM_PLATFORM) \ DRIVER_NAME=$(EMUL_NAME) \ GEN_FILE_BASENAME=FireSim-generated \ GEN_DIR=$(ELAB_DIR) \ @@ -131,7 +140,7 @@ else # DRIVER_NAME is chosen so the sub-make emits lib$(PROJECT).so directly. lib: check-gen $(PIC_DIR)/FireSim-generated.const.h $(MAKE) -C $(SIMIF_DIR) driver \ - MAIN=$(PLATFORM) PLATFORM=$(PLATFORM) \ + MAIN=$(FIRESIM_PLATFORM) PLATFORM=$(FIRESIM_PLATFORM) \ DRIVER_NAME=lib$(PROJECT).so.tmp \ GEN_FILE_BASENAME=FireSim-generated \ GEN_DIR=$(PIC_DIR) \ @@ -140,7 +149,7 @@ lib: check-gen $(PIC_DIR)/FireSim-generated.const.h CXXFLAGS="$(SIM_CXXFLAGS)" \ LDFLAGS="$(SIM_LDFLAGS)" \ $(if $(SIM_CXX),CXX=$(SIM_CXX)) - mv -f $(DESTDIR)/lib$(PROJECT).so.tmp-$(PLATFORM) $(DESTDIR)/lib$(PROJECT).so + mv -f $(DESTDIR)/lib$(PROJECT).so.tmp-$(FIRESIM_PLATFORM) $(DESTDIR)/lib$(PROJECT).so endif diff --git a/sw/runtime/firesim/Makefile b/sw/runtime/firesim/Makefile index 8406f65067..d604055085 100644 --- a/sw/runtime/firesim/Makefile +++ b/sw/runtime/firesim/Makefile @@ -15,22 +15,18 @@ CXXFLAGS += -fPIC # Add external configuration CXXFLAGS += $(CONFIGS) -# The card's FireSim platform, from its descriptor: it decides which variables -# name the image and the card (those of the card's native driver). -FIRESIM_CARD ?= u55c -FIRESIM_PLATFORM := $(shell python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk | sed -n 's/^PLATFORM := //p') -ifeq ($(FIRESIM_PLATFORM),) -$(error cannot resolve the FireSim platform of card '$(FIRESIM_CARD)') -endif +# The card, and its FireSim platform, which decides the variables that name +# the image and the card (those of the card's native driver). +include $(SRC_DIR)/card.mk ifeq ($(FIRESIM_PLATFORM), aved) CXXFLAGS += -DFIRESIM_PLATFORM_AVED endif -# TARGET=avedsim runs the metasimulator in-process in place of the card: the -# same FireSim-transformed target in Verilator, reached through the emulated -# host interface, with no image and no VRT. It is the aved counterpart of the -# native driver's avedsim. -ifeq ($(TARGET), avedsim) +# TARGET=xrtsim or avedsim runs the metasimulator in-process in place of the +# card: the same FireSim-transformed target in Verilator, reached through the +# emulated host interface, with no image, no XRT and no VRT -- the counterpart +# of the native drivers' xrtsim and avedsim. +ifneq ($(filter $(TARGET),xrtsim avedsim),) CXXFLAGS += -DFIRESIM_EMUL endif @@ -58,7 +54,8 @@ all: $(DESTDIR)/$(PROJECT) driver: $(DESTDIR)/libfiresim.so $(DESTDIR)/libfiresim.so: force - DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) \ + CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) $(DESTDIR)/$(PROJECT): $(SRCS) $(DESTDIR)/libfiresim.so $(CONFIG_FILE) $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ diff --git a/sw/runtime/firesim/card.mk b/sw/runtime/firesim/card.mk new file mode 100644 index 0000000000..ffb1b3c34f --- /dev/null +++ b/sw/runtime/firesim/card.mk @@ -0,0 +1,39 @@ +# Which card a FireSim run is for, decided the same way wherever it is asked -- +# the runtime build, run-firesim, blackbox.sh's driver build -- from what the +# caller already has: TARGET and, for an image target, FPGA_BIN_DIR. +# +# TARGET=hw|hw_emu|sim the card hw/syn/firesim recorded beside the image +# ($(FPGA_BIN_DIR)/firesim.card) +# TARGET=xrtsim|avedsim no image; the U55C, or the V80 (FIRESIM_CARD overrides) +# TARGET empty xrtsim, or avedsim when FIRESIM_CARD names an aved card +# +# Sets FIRESIM_CARD and FIRESIM_PLATFORM (vitis or aved), and TARGET when empty. + +FIRESIM_GEN := python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py +firesim_platform_of = $(shell $(FIRESIM_GEN) --card=$(1) --emit=firesim-mk 2>/dev/null | sed -n 's/^FIRESIM_PLATFORM := //p') + +ifeq ($(strip $(TARGET)),) +FIRESIM_CARD ?= u55c +override TARGET := $(if $(filter aved,$(call firesim_platform_of,$(FIRESIM_CARD))),avedsim,xrtsim) +else ifneq ($(filter $(TARGET),xrtsim avedsim),) +FIRESIM_CARD ?= $(if $(filter avedsim,$(TARGET)),v80,u55c) +else ifeq ($(filter $(TARGET),hw hw_emu sim),) +$(error TARGET=$(TARGET) is not a FireSim target: use xrtsim, hw_emu or hw on an XRT card, \ + avedsim, sim or hw on an aved card) +else +FIRESIM_CARD ?= $(shell cat $(FPGA_BIN_DIR)/firesim.card 2>/dev/null) +ifeq ($(FIRESIM_CARD),) +$(error TARGET=$(TARGET) runs an image: set FPGA_BIN_DIR to the bin directory hw/syn/firesim built \ + (it holds firesim.card), or use TARGET=xrtsim|avedsim) +endif +endif + +FIRESIM_PLATFORM := $(call firesim_platform_of,$(FIRESIM_CARD)) +ifeq ($(FIRESIM_PLATFORM),) +$(error cannot resolve the FireSim platform of card '$(FIRESIM_CARD)') +endif + +FIRESIM_TARGETS := $(if $(filter aved,$(FIRESIM_PLATFORM)),avedsim sim hw,xrtsim hw_emu hw) +ifeq ($(filter $(TARGET),$(FIRESIM_TARGETS)),) +$(error TARGET=$(TARGET) is not a FireSim target for card $(FIRESIM_CARD): use one of $(FIRESIM_TARGETS)) +endif diff --git a/tests/regression/common.mk b/tests/regression/common.mk index b55ca86c1a..d50656be64 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -242,24 +242,27 @@ run-rtlsim: $(PROJECT) kernel.vxbin $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/rtlsim DESTDIR=$(VORTEX_RT_LIB) LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=rtlsim ./$(PROJECT) $(OPTS) -# FireSim takes its card from the descriptor (FIRESIM_CARD) and names the image -# and card with the variables of the card's native driver. -FIRESIM_CARD ?= u55c +# FireSim takes the same targets as the card's native driver: xrtsim, hw_emu +# and hw on an XRT card; avedsim, sim and hw on an aved card. An image target +# runs FPGA_BIN_DIR's firesim.xclbin or firesim.vbin; the card is decided in +# sw/runtime/firesim/card.mk, and named to the run with its native driver's +# variables. With no TARGET given it runs the in-process metasimulator. ifneq ($(filter run-firesim,$(MAKECMDGOALS)),) -FIRESIM_PLATFORM := $(shell python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk | sed -n 's/^PLATFORM := //p') +ifeq ($(origin TARGET),file) +override TARGET := +endif +include $(VORTEX_HOME)/sw/runtime/firesim/card.mk endif -# On aved, TARGET=avedsim runs the metasimulator in-process and needs no image; -# hw and sim both run a .vbin, the card's or the platform simulator's, named by -# FPGA_BIN_DIR. run-firesim: $(PROJECT) kernel.vxbin $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) -ifeq ($(FIRESIM_PLATFORM), aved) -ifeq ($(TARGET), avedsim) +ifneq ($(filter $(TARGET),xrtsim avedsim),) LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) -else - VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/firesim.vbin LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(FIRESIM_PLATFORM), aved) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh endif + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/firesim.vbin LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) else ifeq ($(TARGET), hw) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) else From 85c894980d1b626adae5eb092967fa79c5b9d808 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 25 Sep 2026 19:02:31 -0700 Subject: [PATCH 163/336] firesim: a signal stops a run instead of killing it Killing a FireSim run mid-launch is what preceded every host reset on the V80, and Ctrl-C or `timeout` did exactly that. The runtime now takes the first SIGINT or SIGTERM (where the application has no handler of its own) through the launch watchdog's path: the launch in flight is reported with the memory-model and core counters and completed, nothing more is sent to the target, and the application tears down normally. A second signal kills. A stopped run must never read as a pass -- an early test stopped before the launch and both raycast and FireSim printed PASSED. The stop is now sticky, FireSim's run reports failure, and after the application's teardown the process exits 128+signal (124 for the watchdog), whatever the application printed. Counter reads made by the stop's own report still reach the target. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 6c51263491f38d3fddfb8e16822669f572781606) --- sim/firesim/firesim_sim.cpp | 113 +++++++++++++++++++++++++++++++----- sim/firesim/firesim_sim.h | 17 +++++- 2 files changed, 114 insertions(+), 16 deletions(-) diff --git a/sim/firesim/firesim_sim.cpp b/sim/firesim/firesim_sim.cpp index 2f8c8ab2b8..ef73e27bb0 100644 --- a/sim/firesim/firesim_sim.cpp +++ b/sim/firesim/firesim_sim.cpp @@ -17,6 +17,8 @@ #include "bridges/fased_memory_timing_model.h" #include +#include +#include #include #include #include @@ -28,6 +30,7 @@ #include #include #include +#include #include #include @@ -123,6 +126,12 @@ class vortex_simulation_t; channel_t *g_channel = nullptr; vortex_simulation_t *g_sim = nullptr; +// The exit status a stopped run ends with (128+signal, or 124 for the launch +// watchdog), 0 while it has not been stopped. Once set, the simulation reports +// failure and the process exits with it, whatever the application prints: a +// run cut short must never read as a pass. +std::atomic g_stop_status{0}; + // A simulation whose run loop services requests instead of executing a fixed // script. Everything the target does happens on this thread: the bridges are // not thread-safe and must never be touched by the caller. @@ -154,7 +163,7 @@ class vortex_simulation_t final : public simulation_t { lock, [] { return g_channel->shutdown || !g_channel->pending.empty(); }); if (g_channel->shutdown && g_channel->pending.empty()) { g_sim = nullptr; - return 0; + return g_stop_status.load() != 0 ? 1 : 0; } req = std::move(g_channel->pending.front().fn); req_label = std::move(g_channel->pending.front().label); @@ -462,7 +471,9 @@ class firesim_sim::Impl { firesim_sim::firesim_sim() : impl_(new Impl()), step_batch_(kStepBatchFloor), busy_cycles_(0), stall_reported_(false), awaiting_busy_(false), idle_cycles_(0), - abort_phase_(0), launch_started_s_(0) {} + abort_phase_(0), launch_started_s_(0), stopped_(false), reporting_(false) { + install_stop_handlers(); +} // Wall-clock seconds a launch may take before the runtime gives up on it; 0 // (the default) never does. Meant for a card run under an outer timeout: this @@ -481,6 +492,63 @@ static double launch_timeout_s() { return value; } +// A first SIGINT or SIGTERM asks the run to stop instead of killing it: the +// launch in flight is reported and completed, nothing more is sent to the +// target, and the application finishes through its own teardown. Killing a +// FireSim run mid-launch is what preceded every host reset on the V80, so +// Ctrl-C and `timeout -s INT` take this path. A second signal kills as usual. +namespace { +std::atomic g_stop_signal{0}; + +extern "C" void on_stop_signal(int sig) { + if (g_stop_signal.exchange(sig) != 0) { + signal(sig, SIG_DFL); + raise(sig); + } +} +} // namespace + +// After the application's own teardown, a stopped run exits with its status. +// Output is flushed first; _exit skips any later atexit handlers. +extern "C" void exit_with_stop_status() { + const int status = g_stop_status.load(); + if (status != 0) { + fflush(nullptr); + fprintf(stderr, "[firesim] the run was stopped; its results are not valid (exit %d)\n", + status); + _exit(status); + } +} + +// Only where the application has not installed its own handler. +void firesim_sim::install_stop_handlers() { + static bool registered = false; + if (!registered) { + atexit(exit_with_stop_status); + registered = true; + } + for (int sig : {SIGINT, SIGTERM}) { + struct sigaction old_action; + if (sigaction(sig, nullptr, &old_action) == 0 && old_action.sa_handler == SIG_DFL) { + struct sigaction action = {}; + action.sa_handler = on_stop_signal; + sigemptyset(&action.sa_mask); + sigaction(sig, &action, nullptr); + } + } +} + +bool firesim_sim::stopped() { + if (!stopped_ && g_stop_signal.load() != 0) { + fprintf(stderr, "[firesim] %s received: stopping the run (send it again to kill)\n", + g_stop_signal.load() == SIGINT ? "SIGINT" : "SIGTERM"); + stopped_ = true; + int expected = 0; + g_stop_status.compare_exchange_strong(expected, 128 + g_stop_signal.load()); + } + return stopped_; +} + static double wall_now_s() { return std::chrono::duration( std::chrono::steady_clock::now().time_since_epoch()) @@ -504,6 +572,13 @@ static uint64_t stall_report_cycles() { } void firesim_sim::report_progress(const char *why) { + // The report's own reads must reach the target even when it is written + // because the run is stopping. + struct reporting_scope { + bool &flag; + explicit reporting_scope(bool &f) : flag(f) { flag = true; } + ~reporting_scope() { flag = false; } + } scope(reporting_); uint64_t tcycle = 0; const int ret = impl_->submit("read target clock", [&tcycle] { tcycle = g_sim->clock.tcycle(); }); @@ -629,7 +704,7 @@ int firesim_sim::init(const std::vector &plusargs) { } int firesim_sim::dcr_write(uint32_t addr, uint32_t value) { - if (abort_phase_ != 0) { + if (!reporting_ && stopped()) { return 0; } return impl_->submit(label("dcr_write addr=0x%x data=0x%x", addr, value), [addr, value] { @@ -645,7 +720,7 @@ int firesim_sim::dcr_write(uint32_t addr, uint32_t value) { } int firesim_sim::dcr_read(uint32_t addr, uint32_t tag, uint32_t *value) { - if (abort_phase_ != 0) { + if (!reporting_ && stopped()) { *value = 0; return -1; } @@ -726,6 +801,12 @@ static void prefill_region() { } int firesim_sim::start() { + if (stopped()) { + // Complete the launch without touching the target: busy once, then idle. + abort_phase_ = 1; + launch_started_s_ = 0; + return 0; + } impl_->submit("prefill", [] { prefill_region(); }); awaiting_busy_ = true; idle_cycles_ = 0; @@ -749,13 +830,19 @@ int firesim_sim::is_busy(bool *busy) { abort_phase_ = 2; return 0; } - if (launch_timeout_s() > 0 && launch_started_s_ > 0 && - wall_now_s() - launch_started_s_ > launch_timeout_s()) { - fprintf(stderr, - "[firesim] launch exceeded VORTEX_FIRESIM_TIMEOUT_S=%.0f s " - "(busy %s); giving up on it\n", - launch_timeout_s(), awaiting_busy_ ? "never rose" : "never fell"); - report_progress("launch watchdog"); + const bool timed_out = launch_timeout_s() > 0 && launch_started_s_ > 0 && + wall_now_s() - launch_started_s_ > launch_timeout_s(); + if (launch_started_s_ > 0 && (timed_out || stopped())) { + if (timed_out) { + fprintf(stderr, + "[firesim] launch exceeded VORTEX_FIRESIM_TIMEOUT_S=%.0f s " + "(busy %s); giving up on it\n", + launch_timeout_s(), awaiting_busy_ ? "never rose" : "never fell"); + } + report_progress(timed_out ? "launch watchdog" : "stopped"); + stopped_ = true; + int expected = 0; + g_stop_status.compare_exchange_strong(expected, timed_out ? 124 : 128 + SIGINT); launch_started_s_ = 0; abort_phase_ = awaiting_busy_ ? 1 : 2; awaiting_busy_ = false; @@ -847,7 +934,7 @@ int firesim_sim::ready_wait(int64_t timeout_ms) { // it, so the application runs to its own exit -- and reports its failure -- // instead of blocking on a transfer that will never complete. int firesim_sim::mem_write(uint64_t addr, uint64_t size, const void *value) { - if (abort_phase_ != 0) { + if (!reporting_ && stopped()) { return 0; } dump_upload(addr, size, value); @@ -857,7 +944,7 @@ int firesim_sim::mem_write(uint64_t addr, uint64_t size, const void *value) { } int firesim_sim::mem_read(uint64_t addr, uint64_t size, void *value) { - if (abort_phase_ != 0) { + if (!reporting_ && stopped()) { memset(value, 0, size); return 0; } diff --git a/sim/firesim/firesim_sim.h b/sim/firesim/firesim_sim.h index de68098014..423ce92b0b 100644 --- a/sim/firesim/firesim_sim.h +++ b/sim/firesim/firesim_sim.h @@ -91,11 +91,22 @@ class firesim_sim { bool awaiting_busy_; uint64_t idle_cycles_; - // Set when the launch watchdog (VORTEX_FIRESIM_TIMEOUT_S) gives up. The - // launch is then completed -- busy once, then idle -- so the application - // finishes through its own teardown rather than being killed mid-run. + // Set when the launch watchdog (VORTEX_FIRESIM_TIMEOUT_S) or a stop signal + // gives up on a launch. The launch is then completed -- busy once, then idle + // -- so the application finishes through its own teardown rather than being + // killed mid-run. int abort_phase_; double launch_started_s_; + + // Set once the run is being stopped -- by a signal or by the watchdog -- and + // never cleared: from then on nothing more is sent to the target. + bool stopped_; + bool stopped(); + + // True while a report reads counters, which must reach the target even as + // the run stops. + bool reporting_; + static void install_stop_handlers(); }; } From 09e03ff965d6c9997524bf8de8aff07e93d2d069 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 25 Sep 2026 19:02:31 -0700 Subject: [PATCH 164/336] docs: FireSim is driven like the card's native flow docs/simulation.md: the FireSim section is the target matrix per card, the build and run commands with the native tokens, the V80 pre-flight, what a stopped run looks like, and one table of the runtime's debugging variables. docs/designs/firesim_integration.md: the build section follows the new interface and goal list; the card table gives run targets and how the card is found; the V80 clock gates are described as they now are (gated-clock conversion, checked before opt_design) with why the fabric gate went; and the aved RTU's soft FP units are noted beside the vendor cores. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d7216725a058e3bd8984ba38ff0bc605ac71fb45) --- docs/designs/firesim_integration.md | 71 ++++++++++++++++++++--------- docs/simulation.md | 44 +++++++++++++++++- 2 files changed, 91 insertions(+), 24 deletions(-) diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index a97b1bc4e2..39958e0494 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -217,8 +217,8 @@ new bitstream; it is a known follow-up rather than a driver-side workaround. Vortex's own performance counters are the only view inside the core that does not require re-elaborating the design, and they are read over the DCR path. -They are gated behind `PERF_ENABLE` (see `PERF_DEFINES` in -`hw/syn/firesim/Makefile`); **without it every counter reads zero, which is +They are gated behind `PERF_ENABLE` (build the image with `PERF=1`, as in the +other synthesis flows); **without it every counter reads zero, which is indistinguishable from an idle core rather than an absent instrument.** The driver says so explicitly rather than leaving it to be inferred. @@ -226,28 +226,39 @@ driver says so explicitly rather than leaving it to be inferred. ## 4. Build flow -One command builds a card's image, for the card named by `FIRESIM_CARD` -(default `u55c`), from a configured build tree: +The image build takes the same tokens as the card's native flow +(`hw/syn/xilinx/xrt` or `hw/syn/xilinx/aved`), from a configured build tree: ```bash -source $FIRESIM_PATH/env.sh # FireSim's conda environment -source /opt/xilinx/2024.2/Vitis/settings64.sh # plus XRT's setup.sh -make -C hw/syn/firesim bitstream FIRESIM_CARD=u55c \ - CONFIGS="-DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TEX_ENABLE" FREQUENCY=60.0 +source ci/firesim_env.sh # after the card's Xilinx tools (Vitis+XRT, or Vivado+VRT) +make -C hw/syn/firesim PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 TARGET=hw_emu # XRT card +make -C hw/syn/firesim TARGET=hw PERF=1 CONFIGS="-DVX_CFG_NUM_WARPS=4" # aved card ``` -`bitstream` runs, in order: +`PLATFORM=` selects an XRT card and `TARGET` is `hw` or `hw_emu`; without +`PLATFORM` the card is the aved one for `DEVICE` (default the V80) and `TARGET` +is `hw` or `sim`. The image is copied to `__/bin` +(`firesim.xclbin` or `firesim.vbin`, plus `emconfig.json` for `hw_emu`) beside +`firesim.card`, which names the card for `run-firesim`; everything the build +generates lives under that directory. Nothing given on the command line +reaches FireSim's own makes, which use `PLATFORM`, `TARGET` and `BUILD_DIR` for +their own purposes. + +The default goal runs, in order: ``` -vivado-sources flatten Vortex with macros inlined (-P) -sources Verilator source list (packages + wrapper) +vivado-sources flatten Vortex with macros inlined (-P); -DSIMULATION for sim/hw_emu xilinx-ips generate the Xilinx floating-point cores for the card's part stage copy the Chisel/C++ targets and the card's platform config - class into $(FIRESIM_PATH) + class into $(FIRESIM_PATH) (only what changed) replace-rtl Golden Gate elaboration (in $(FIRESIM_PATH)/sim) -bitstream synthesis + place-and-route (in the platform's cl_ dir) +image the platform's own build in its cl_ dir: bitstream, + or the hw_emu/sim image ``` +`sources`, `elaborate`, `metasim` and `run-metasim` are developer goals; the +runtime uses the first two to build the `xrtsim`/`avedsim` metasimulator. + The card's FireSim platform, part, device, host-memory binding and base address come from its descriptor in `hw/syn/xilinx/platforms/`; the platform config class (`U55CPlatformConfig`) is generated from it. `CONFIGS` takes architecture @@ -257,10 +268,18 @@ flattened tree. ### 4.0 Cards and platforms -| card | FireSim platform | image | run with | +| card | FireSim platform | run targets | run with | |---|---|---|---| -| `u55c` | `vitis` (XRT, `v++`) | `platforms/vitis/cl_/bitstream/build_dir./firesim.xclbin` | `XCLBIN_PATH` | -| `v80` | `aved` (VRT, `slashkit`) | `platforms/aved/cl_/bitstream/build_dir.hw/firesim.vbin` | `VRT_VBIN_PATH`, `VRT_DEVICE_BDF` (discovered when unset) | +| `u55c` | `vitis` (XRT, `v++`) | `xrtsim`, `hw_emu`, `hw` | `XCLBIN_PATH` | +| `v80` | `aved` (VRT, `slashkit`) | `avedsim`, `sim`, `hw` | `VRT_VBIN_PATH`, `VRT_DEVICE_BDF` (discovered when unset) | + +`run-firesim` (and `blackbox.sh --driver=firesim`) takes the card from +`$(FPGA_BIN_DIR)/firesim.card` for an image target; `xrtsim` and `avedsim` need +no image and default to the U55C and the V80 (`FIRESIM_CARD` overrides). With +no `TARGET` it runs `xrtsim`, as the xrt driver defaults to its simulator. On +the V80, `TARGET=hw` runs `ci/v80_preflight.sh` first, and a first `SIGINT` or +`SIGTERM` stops any FireSim run cleanly instead of killing it mid-launch; the +run is then reported `FAILED` and exits 130 or 143 (124 for the launch watchdog). A FireSim run names its image and card with the variables of the card's native driver. On the V80 the Xilinx tools are the ones `slashkit` links with @@ -286,12 +305,17 @@ to anyone reading its traces: `opt_design` with `DRC HDPR-18` ("requires resource type 'BUFGCE_HDIO' which is not ranged by 'pblock_slash'"), and the partition is fixed by the static shell, so no link option can widen it. `FabricClockGates` (set by - `AvedConfig`) makes Golden Gate emit `FireSimFabricClockGate` instead: the - enable passes through a latch that is transparent only while the clock is low - — so it can change the output only while the clock is low — and the gated - clock is put back on a global net by a `BUFG_FABRIC`. It synthesizes to - `LDCE` + `LUT2` + `BUFG_FABRIC`. Simulation keeps the `BUFGCE` model, and - other platforms keep `BUFGCE` itself. + `AvedConfig`) makes Golden Gate emit `FireSimFabricClockGate` instead, which + in hardware is not a gate at all: a plain AND marked `gated_clock`, which + synthesis's gated-clock conversion (turned on for the kernel's run by the + platform build) moves onto the clock-enable pin of every register, RAM and + DSP it drives, all clocked from the ungated clock — `BUFGCE`'s semantics + with no derived clock left. A pre-opt check fails the build if any kernel + clock is still driven by logic. An earlier latch + `BUFG_FABRIC` gate put the + target on a clock ~5 ns behind the host's, with its latch-to-AND step + untimed, and on the card the target started but its memory model never + completed a read. Simulation keeps the `BUFGCE` model, and other platforms + keep `BUFGCE` itself. ### 4.0.1 What the platform build must do for Vortex @@ -313,6 +337,9 @@ they are passed in from `hw/syn/firesim`: 19-3833`). They are copied into the kernel, each into a fresh directory — Tcl's `file copy` onto an existing directory nests the source inside it, which silently leaves a repackaged kernel pointing at the previous run's cores. + On `aved` the conversion above cannot enter these out-of-context cores, so + there the RTU uses the soft FMA and divider (`VX_CFG_RTU_USE_DSP=0`) and no + vendor core is instantiated. ### 4.1 Two source lists, and why diff --git a/docs/simulation.md b/docs/simulation.md index 7803c072b5..3238deb02a 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -28,11 +28,51 @@ running orders of magnitude faster than Verilator. For scale, `sgemm -n32` takes ~1228 ms on rtlsim and ~8 ms on an Alveo U55C, and `sgemm -n1024` — over a billion instructions — completes in ~85 s. +FireSim is driven exactly like the card's native flow; only the driver name +differs. It takes the same targets as the card's native driver: + +| card | targets | image built by | +|---|---|---| +| XRT (e.g. U55C) | `xrtsim` (default), `hw_emu`, `hw` | `make -C hw/syn/firesim PLATFORM= TARGET=hw_emu\|hw` | +| aved (V80) | `avedsim` (default), `sim`, `hw` | `make -C hw/syn/firesim TARGET=sim\|hw` | + +`xrtsim` and `avedsim` run the FireSim-transformed design in Verilator inside +the runtime and need no image or Vivado. The image builds take `PREFIX`, +`XLEN`, `CONFIGS` and `PERF=1` as `hw/syn/xilinx/{xrt,aved}` do, and leave the +image in `__/bin` together with `firesim.card`, which +tells the run which card it is for. From the build directory: + ```bash -export FPGA_BIN_DIR= -./ci/blackbox.sh --driver=firesim --target=hw --app=sgemm --args="-n1024" +source ci/firesim_env.sh # after the card's Xilinx tools; see the script + +# no image needed +./ci/blackbox.sh --driver=firesim --target=avedsim --app=demo + +# build an image, then run on it +CONFIGS="-DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4" make -C hw/syn/firesim TARGET=hw PERF=1 +FPGA_BIN_DIR=$PWD/hw/syn/firesim/build32_aved_hw/bin \ + ./ci/blackbox.sh --driver=firesim --target=hw --warps=4 --threads=4 --app=demo ``` +On the V80, `TARGET=hw` runs `ci/v80_preflight.sh` first: it refuses to +program a card that is off the bus, unhealthy or still in use, and waits out a +cool-down since the last programming. A first `Ctrl-C` (or `SIGTERM`) stops a +FireSim run cleanly — the launch in flight is reported and completed, and the +application tears down normally — and a second one kills it. A stopped run +never reads as a pass: FireSim reports it `FAILED` and the process exits 130 +(143 for `SIGTERM`, 124 when `VORTEX_FIRESIM_TIMEOUT_S` gave up). + +Debugging knobs, all read from the environment by the runtime: + +| variable | effect | +|---|---| +| `VORTEX_FIRESIM_TIMEOUT_S` | give up on a launch after this many seconds, with a report | +| `VORTEX_FIRESIM_STALL_CYCLES` | report (once) after this many busy target cycles; default 500000 | +| `VORTEX_FIRESIM_REPORT_ON_IDLE` | also report the counters when a launch completes | +| `VORTEX_FIRESIM_TRACE` | time-stamp every request to the simulator thread | +| `VORTEX_FIRESIM_EMUL_ARGS` | extra Verilator arguments for `xrtsim`/`avedsim`, e.g. `+verilator+rand+reset+2` | +| `FIRESIM_MM_REORDER`, `FIRESIM_MM_LATENCY` | `xrtsim`/`avedsim` host memory: reorder (1), also interleave (2) and delay reads | + The architecture, build flow and design invariants are documented in [designs/firesim_integration.md](designs/firesim_integration.md); the toolchain component is [section 12 of the toolchain From b663f0a588f364fe45dabdfe8b60c2b686fea164 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 25 Sep 2026 19:02:31 -0700 Subject: [PATCH 165/336] firesim: pin the fork commit that repackages a changed design Picks up the platforms' kernel packaging depending on the design under test, and aved packaging each TARGET separately. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a6b2504a1488a3d62b865b1901fceec8a36d17c3) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ff5fd9b4a6..8f00b14dfe 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=0000d293c1e899dcc308eba9d2877654ca889260 +FIRESIM_REV=b36112f98f31e04d183e9aaed70fbfbe4c2fe191 From 83f31b0978bd846b5714d3d7c28fec5a1a21dc98 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 17:46:58 -0700 Subject: [PATCH 166/336] firesim: thread every socket through one physical copy (FAME5=1) FireSim's model multithreading (Golden Gate FAME-5) simulates N identical instances with one physical copy of their logic and N copies of their state. It works on FIRRTL, so Vortex -- a Verilog BlackBox until now -- has to reach Golden Gate as FIRRTL. FAME5=1 does that and threads every socket: - RTL, all under VX_CFG_FAME5: core_id and socket_id become ports (the CORE_ID/SOCKET_ID parameters made each instance a different module); VX_socket_top is the socket with plain ports, since sv2v inlines any module with interface ports; VX_cluster instantiates it; the async-BRAM patch stays out, its placeholders being a Vivado-hook matter. - hw/syn/firesim/fame5: sv2v, then Yosys flattening all but VX_socket_top, then repairs of Yosys's FIRRTL writer (signed and wide shifts, concatenated connects, signed zero tests, nesting beyond the parser's stack), VX_dp_ram and VX_sp_ram regenerated as FIRRTL memories with their exact semantics, spliced into VortexTarget's placeholder, every socket annotated. - The build: FAME5=1 on the existing goals, its own build directory, emul library and FireSim target config (VortexFame5Config); no Vortex sources for Verilator or packaging, since Vortex is in the generated RTL. FAME5_THREADS=0 builds the same FIRRTL unthreaded, for A/B. Checked: demo on 4 cores (NW=NT=4) in avedsim gives instrs=9776, cycles=15541, PASSED, identically through the existing flow, FAME5=1 FAME5_THREADS=0, and FAME5=1 (4 threads). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 35d3e8a533bfa8320e348b3718583479e75d42ee) Integration (phase 2): the VX_CFG_FAME5 hooks in the Vortex RTL (socket and core ids as ports, VX_socket_top, the async-BRAM patch switch) are not replayed. 803697377 removes them again with the FAME-5 lane, restoring every RTL file, so replaying them would only add and then drop simulator-only changes to the hardware. The FAME5=1 flow is incomplete until then. --- hw/syn/firesim/Makefile | 81 +++++- hw/syn/firesim/fame5/annotate_threads.py | 33 +++ hw/syn/firesim/fame5/fix_firrtl.py | 271 ++++++++++++++++++ hw/syn/firesim/fame5/gen_rams.py | 184 ++++++++++++ hw/syn/firesim/fame5/splice.py | 68 +++++ hw/syn/firesim/fame5/to_firrtl.sh | 37 +++ .../firesim/src/main/scala/VortexTarget.scala | 5 + sim/firesim/Makefile | 17 +- sw/runtime/firesim/Makefile | 3 +- tests/regression/common.mk | 2 +- 10 files changed, 691 insertions(+), 10 deletions(-) create mode 100644 hw/syn/firesim/fame5/annotate_threads.py create mode 100644 hw/syn/firesim/fame5/fix_firrtl.py create mode 100644 hw/syn/firesim/fame5/gen_rams.py create mode 100644 hw/syn/firesim/fame5/splice.py create mode 100755 hw/syn/firesim/fame5/to_firrtl.sh diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index fc66caa0f7..8bbfb5d738 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -88,11 +88,18 @@ $(error TARGET=$(TARGET) is not an image target for card $(FIRESIM_CARD): use on endif endif +# FAME5=1: FireSim model multithreading. Every socket is time-multiplexed +# through one physical copy (Golden Gate's FAME-5), which needs Vortex as FIRRTL +# rather than as a Verilog BlackBox; see fame5/ and +# docs/designs/firesim_integration.md. Its images never share a directory with +# the unthreaded ones. +FAME5_SUFFIX := $(if $(FAME5),_fame5) + # Named as the native flows name theirs, so FPGA_BIN_DIR means the same thing. ifeq ($(FIRESIM_PLATFORM), aved) -BUILD_DIR := $(abspath $(PREFIX)_aved_$(TARGET)) +BUILD_DIR := $(abspath $(PREFIX)_aved_$(TARGET)$(FAME5_SUFFIX)) else -BUILD_DIR := $(abspath $(PREFIX)_$(if $(XSA),$(XSA),$(FIRESIM_CARD))_$(TARGET)) +BUILD_DIR := $(abspath $(PREFIX)_$(if $(XSA),$(XSA),$(FIRESIM_CARD))_$(TARGET)$(FAME5_SUFFIX)) endif # The runtime's metasimulator reuses the simulation flattening from here, in a # directory of its own. @@ -110,10 +117,15 @@ ifneq ($(XSA),) DEVICE := $(XSA) endif +# FireSim keys its generated RTL by design and configs only, so a FAME-5 build +# names its own target config (identical to VortexConfig) to keep its output +# apart from the unthreaded one's. +VORTEX_TARGET_CONFIG := $(if $(FAME5),VortexFame5Config,VortexConfig) + # The FireSim flow's make variables for this card's build. FIRESIM_TARGET_ARGS := PLATFORM=$(FIRESIM_PLATFORM) TARGET_PROJECT=midasexamples \ - DESIGN=VortexTarget TARGET_CONFIG=VortexConfig PLATFORM_CONFIG=$(PLATFORM_CONFIG) -QUINTUPLET := $(FIRESIM_PLATFORM)-midasexamples-VortexTarget-VortexConfig-$(PLATFORM_CONFIG) + DESIGN=VortexTarget TARGET_CONFIG=$(VORTEX_TARGET_CONFIG) PLATFORM_CONFIG=$(PLATFORM_CONFIG) +QUINTUPLET := $(FIRESIM_PLATFORM)-midasexamples-VortexTarget-$(VORTEX_TARGET_CONFIG)-$(PLATFORM_CONFIG) # Flattened Vortex sources for the BlackBox. Golden Gate takes a file list, not # include paths, so the tree is copied into one directory with the macros already @@ -130,6 +142,15 @@ ifneq ($(filter $(TARGET),sim hw_emu),) IMAGE_KIND := sim IMAGE_SUFFIX := _sim endif +# FAME-5 converts one flattening to FIRRTL for every target, image or not. It is +# the synthesis one: SIMULATION would bring in simulation-only tasks ($fflush, +# $write) that have no FIRRTL form. VX_CFG_FAME5 keeps the async-BRAM patch out +# of it (VX_platform.vh), whose placeholders only a Vivado hook resolves; the RAM +# leaves are regenerated as FIRRTL memories instead. +ifdef FAME5 +IMAGE_KIND := +IMAGE_SUFFIX := +endif VIVADO_SRCS_NAME := vivado_srcs$(IMAGE_SUFFIX) VIVADO_SRCS_DIR := $(BUILD_DIR)/$(VIVADO_SRCS_NAME) VIVADO_SRCS_LIST := $(BUILD_DIR)/$(VIVADO_SRCS_NAME).f @@ -263,6 +284,9 @@ endif ifdef PERF PLATFORM_DEFINES += -DPERF_ENABLE endif +ifdef FAME5 +PLATFORM_DEFINES += -DVX_CFG_FAME5 +endif GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) $(SYNTH_DEFINES) $(RTL_INCLUDE) @@ -469,6 +493,7 @@ endif # prerequisite: the recipe hands synthesis the Vivado-flattened list, and pulling # the simulation one in would flatten the tree a second time under the opposite # defines for nothing. +ifndef FAME5 image: vivado-sources xilinx-ips stage $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ @@ -481,6 +506,54 @@ image: vivado-sources xilinx-ips stage # no image. An image build elaborates as part of replace-rtl. elaborate: stage $(MAKE) -C $(FIRESIM_PATH)/sim verilog $(FIRESIM_TARGET_ARGS) +else +# Under FAME-5 Vortex is part of the generated RTL, so the image is handed no +# Vortex sources, no vendor cores and no pre-opt hook: there is nothing left +# outside FireSim-generated.sv for them to apply to. +image: elaborate + $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) + TOOL_DIR=$(SCRIPT_DIR) $(MAKE) -C $(PLATFORM_BUILD_DIR) $(PLATFORM_IMAGE_GOAL) \ + DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) + +# Vortex as FIRRTL: sv2v, then Yosys flattening everything but VX_socket_top +# (the unit threaded), with the RAM leaves regenerated as FIRRTL memories +# (fame5/to_firrtl.sh). +FAME5_DIR := $(BUILD_DIR)/fame5 +FAME5_V := $(FAME5_DIR)/vortex.v +FAME5_FIR := $(FAME5_DIR)/vortex.fir +FAME5_SCRIPTS := $(wildcard $(SRC_DIR)/fame5/*) +SV2V ?= sv2v + +$(FAME5_V): $(VIVADO_SRCS_LIST) + @mkdir -p $(FAME5_DIR) + $(SV2V) $$(grep '^+define+' $(VIVADO_SRCS_LIST) | sed 's/^+define+/-D/') --top=VX_firesim_wrap \ + $$(grep -v '^+' $(VIVADO_SRCS_LIST)) -w $@ + +$(FAME5_FIR): $(FAME5_V) $(FAME5_SCRIPTS) + bash $(SRC_DIR)/fame5/to_firrtl.sh $(FAME5_V) VX_firesim_wrap $@ VX_socket_top VX_firesim_wrap \ + "blackbox VX_dp_ram VX_sp_ram" + +# The generator's circuit names VX_firesim_wrap as an extmodule; it is replaced +# by the converted Vortex, and every socket is annotated for threading, before +# Golden Gate runs. The circuit is regenerated each time, since the splice edits +# it in place. FAME5_THREADS=0 skips the annotations: the same FIRRTL design, +# unthreaded, which must agree with the threaded one cycle for cycle. +# Yosys writes long nested cat() chains, and the FIRRTL parser recurses once per +# level: the 8 MB thread stack FireSim sets overflows on a converted Vortex. +# FireSim only sets its default when the variable is unset. +export JAVA_TOOL_OPTIONS ?= -Xmx48G -Xss1G -Djava.io.tmpdir=$(FIRESIM_PATH)/sim/.java_tmp + +GEN_FIR := $(FIRESIM_PATH)/sim/generated-src/$(FIRESIM_PLATFORM)/$(QUINTUPLET)/firesim.midasexamples.VortexTarget.$(VORTEX_TARGET_CONFIG).fir +GEN_ANNO := $(GEN_FIR:.fir=.anno.json) +elaborate: stage $(FAME5_FIR) + rm -f $(GEN_FIR) + $(MAKE) -C $(FIRESIM_PATH)/sim $(GEN_FIR) $(FIRESIM_TARGET_ARGS) + python3 $(SRC_DIR)/fame5/splice.py $(GEN_FIR) $(FAME5_FIR) +ifneq ($(FAME5_THREADS),0) + python3 $(SRC_DIR)/fame5/annotate_threads.py $(GEN_FIR) $(GEN_ANNO) VX_socket_top +endif + $(MAKE) -C $(FIRESIM_PATH)/sim verilog $(FIRESIM_TARGET_ARGS) +endif clean: rm -rf $(BUILD_DIR) diff --git a/hw/syn/firesim/fame5/annotate_threads.py b/hw/syn/firesim/fame5/annotate_threads.py new file mode 100644 index 0000000000..052e1e4270 --- /dev/null +++ b/hw/syn/firesim/fame5/annotate_threads.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Annotate every instance of a module for FAME-5 threading, in the anno file. + +usage: annotate_threads.py + +Finds each `inst of ` in and appends a +FirrtlEnableModelMultiThreadingAnnotation targeting it -- what Chisel's +EnableModelMultiThreadingAnnotation would emit, for designs with no Chisel. +""" +import json +import re +import sys + +fir, anno, mod = sys.argv[1:] +circuit, parent, targets = None, None, [] +for line in open(fir): + m = re.match(r"circuit (\S+?)\s*:", line) + if m: + circuit = m.group(1) + m = re.match(r" (?:ext)?module (\S+?)\s*:", line) + if m: + parent = m.group(1) + m = re.match(rf"\s+inst (\S+) of {re.escape(mod)}\b", line) + if m: + targets.append(f"~{circuit}|{parent}/{m.group(1)}:{mod}") +if not targets: + sys.exit(f"annotate_threads: no instance of {mod}") +annos = json.load(open(anno)) +annos = [a for a in annos if not (a["class"].endswith("FirrtlEnableModelMultiThreadingAnnotation") + and a["target"] in targets)] +annos += [{"class": "midas.targetutils.FirrtlEnableModelMultiThreadingAnnotation", "target": t} for t in targets] +json.dump(annos, open(anno, "w"), indent=2) +print(f"annotate_threads: {len(targets)} instances of {mod}") diff --git a/hw/syn/firesim/fame5/fix_firrtl.py b/hw/syn/firesim/fame5/fix_firrtl.py new file mode 100644 index 0000000000..943ba378a4 --- /dev/null +++ b/hw/syn/firesim/fame5/fix_firrtl.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Repair Yosys write_firrtl's output. + +usage: fix_firrtl.py + +1. Instance outputs driving split wires. Yosys emits + cat(a, cat(b, c)) <= inst.port + which FIRRTL does not allow; each piece gets its slice instead: + c <= bits(inst.port, wc-1, 0); b <= bits(..., wc+wb-1, wc); ... + +2. $shift cells with a signed amount, below. + +5. Zero tests of signed values. Yosys's logic ops test each operand with + neq(x, UInt<1>("h0")), even when x is asSInt(...), which FIRRTL rejects as + a signed/unsigned comparison. x != 0 does not depend on the sign, so the + asSInt is dropped (likewise for eq). + +4. Deep expressions. The FIRRTL parser recurses once per nesting level, and + Yosys writes long cat()/mux() chains (thousands deep for wide arbiters); + any subexpression deeper than MAX_DEPTH is hoisted into a node. + +3. Left shifts. Yosys writes bits(dshl(a, min(b, 2^19-1)), hi, 0): correct, but + the intermediate is ~2^19 bits wide, beyond Verilator's constant limit. Only + bits hi..0 are kept, so the clamp is lowered to the first 2^k-1 > hi. + +Yosys emits mux(B < 0, bits(dshl(A, ), 0, W), dshr(A, B)) +which is not FIRRTL (`<`), has bits() bounds reversed, and shifts left by B +rather than -B. $shift semantics are: Y = A >> B, and for B < 0, Y = A << -B. +This rewrites each occurrence to + + mux(lt(asSInt(B), SInt<1>(0)), + bits(dshl(pad(A, W), ), W-1, 0), + bits(pad(dshr(A, B), W), W-1, 0)) + +which needs no width but W, taken from the original bits(..., 0, W). +""" +import sys + +sys.setrecursionlimit(100000) # expressions nest thousands deep + + +def match_paren(s, i): + """s[i] == '('; return the index of its matching ')'.""" + depth = 0 + for j in range(i, len(s)): + if s[j] == "(": + depth += 1 + elif s[j] == ")": + depth -= 1 + if depth == 0: + return j + raise ValueError("unbalanced") + + +def split_args(s): + """Split a comma-separated argument list at top level.""" + out, depth, cur = [], 0, "" + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if ch == "," and depth == 0: + out.append(cur.strip()) + cur = "" + else: + cur += ch + out.append(cur.strip()) + return out + + +def fix_line(line): + count = 0 + start = 0 + while True: + i = line.find("mux(", start) + if i < 0: + return line, count + j = match_paren(line, i + 3) + args = split_args(line[i + 4:j]) + if len(args) == 3 and args[0].endswith(" < 0"): + b = args[0][: -len(" < 0")] + left, right = args[1], args[2] + # left: bits(dshl(A, S), 0, W) + inner = split_args(left[len("bits("):-1]) + dshl = inner[0] + w = int(inner[2]) + a = split_args(dshl[len("dshl("):-1])[0] + assert right.startswith("dshr(") + c = w.bit_length() + mag = f"asUInt(neg(asSInt({b})))" + amt = f"bits(mux(gt({mag}, UInt<{c}>({w})), UInt<{c}>({w}), {mag}), {c - 1}, 0)" + new = (f"mux(lt(asSInt({b}), SInt<1>(0)), " + f"bits(dshl(pad({a}, {w}), {amt}), {w - 1}, 0), " + f"bits(pad(dshr({a}, {b}), {w}), {w - 1}, 0))") + line = line[:i] + new + line[j + 1:] + count += 1 + start = i + len(new) + else: + start = i + 4 + + +import re + +BIG = "UInt<19>(524287)" +MAX_DEPTH = 48 +LITERAL = re.compile(r"^[US]Int<\d+>\(") + + +def parse(e): + """Expression -> (text, children) tree; literals and references are leaves.""" + e = e.strip() + m = re.match(r"^([A-Za-z_][A-Za-z0-9_]*)\(", e) + if m and not LITERAL.match(e) and match_paren(e, len(m.group(1))) == len(e) - 1: + return (m.group(1), [parse(a) for a in split_args(e[len(m.group(1)) + 1:-1])]) + return (e, None) + + +def emit(t): + op, kids = t + return op if kids is None else f"{op}({', '.join(emit(k) for k in kids)})" + + +def hoist(t, nodes, ns, indent): + """Rewrite t so no expression exceeds MAX_DEPTH; returns (tree, depth).""" + op, kids = t + if kids is None: + return t, 0 + new, depth = [], 0 + for k in kids: + k2, d = hoist(k, nodes, ns, indent) + if d >= MAX_DEPTH: + name = f"_fx_{ns[0]}" + ns[0] += 1 + nodes.append(f"{indent}node {name} = {emit(k2)}\n") + k2, d = (name, None), 0 + new.append(k2) + depth = max(depth, d) + return (op, new), depth + 1 + + +ZERO = re.compile(r'^UInt(<\d+>)?\(("h0+"|0)\)$') + + +def fix_zero_tests(line): + count, start = 0, 0 + while True: + m = re.compile(r"\b(n?eq)\(asSInt\(").search(line, start) + if not m: + return line, count + i = m.start() + j = match_paren(line, m.end(1)) + args = split_args(line[m.end(1) + 1:j]) + if len(args) == 2 and ZERO.match(args[1]) and args[0].startswith("asSInt("): + inner = args[0][len("asSInt("):-1] + new = f"{m.group(1)}({inner}, {args[1]})" + line = line[:i] + new + line[j + 1:] + count += 1 + start = i + len(new) + else: + start = m.end() + + +def fix_deep(line, ns): + m = re.match(r"(\s+)(node \S+ = |\S+ <= )(.*?)(\s*@\[.*)?$", line.rstrip("\n")) + if not m: + return None + indent, head, rhs, info = m.group(1), m.group(2), m.group(3), m.group(4) or "" + d = cur = 0 + for c in rhs: + if c == "(": + cur += 1 + d = max(d, cur) + elif c == ")": + cur -= 1 + if d <= MAX_DEPTH: + return None + nodes = [] + t, _ = hoist(parse(rhs), nodes, ns, indent) + return nodes + [f"{indent}{head}{emit(t)}{info}\n"] + + +def fix_dshl(line): + count, start = 0, 0 + while True: + i = line.find("bits(dshl(", start) + if i < 0: + return line, count + j = match_paren(line, i + 4) + args = split_args(line[i + 5:j]) + if len(args) == 3 and BIG in args[0]: + hi = int(args[1]) + d = args[0] + dargs = split_args(d[len("dshl("):-1]) + amt = dargs[1] + m = re.fullmatch(r"asUInt\(mux\(gt\((.*), UInt<19>\(524287\)\), UInt<19>\(524287\), bits\((.*), 18, 0\)\)\)", amt) + if m and m.group(1) == m.group(2): + b = m.group(1) + k = (hi + 1).bit_length() + mx = (1 << k) - 1 + amt = f"asUInt(mux(gt({b}, UInt<{k}>({mx})), UInt<{k}>({mx}), bits(pad({b}, {k}), {k - 1}, 0)))" + new = f"bits(dshl({dargs[0]}, {amt}), {args[1]}, {args[2]})" + line = line[:i] + new + line[j + 1:] + count += 1 + start = i + len(new) + continue + start = i + 5 + + +DECL = re.compile(r"\s+(?:input|output|wire|reg) (\S+?)\s*: UInt<(\d+)>") + + +def cat_leaves(expr): + """cat(x, cat(y, z)) -> [x, y, z] (most significant first).""" + expr = expr.strip() + if not expr.startswith("cat("): + return [expr] + out = [] + for a in split_args(expr[4:match_paren(expr, 3)]): + out += cat_leaves(a) + return out + + +def fix_cat_lhs(line, widths): + m = re.match(r"(\s+)(cat\(.*\)) <= (\S+)(\s*@.*)?$", line.rstrip("\n")) + if not m: + return None + indent, lhs, rhs, info = m.group(1), m.group(2), m.group(3), m.group(4) or "" + out, lo = [], 0 + for leaf in reversed(cat_leaves(lhs)): + if leaf not in widths: + sys.exit(f"fix_firrtl: no width for {leaf} in: {line.strip()[:120]}") + w = widths[leaf] + out.append(f"{indent}{leaf} <= bits({rhs}, {lo + w - 1}, {lo}){info}\n") + lo += w + return out + + +total = cats = shls = zeros = 0 +hoisted = [0] +widths = {} +with open(sys.argv[1]) as f, open(sys.argv[2], "w") as o: + for line in f: + if line.startswith(" module ") or line.startswith(" extmodule "): + widths = {} + d = DECL.match(line) + if d: + widths[d.group(1)] = int(d.group(2)) + if line.lstrip().startswith("cat("): + fixed = fix_cat_lhs(line, widths) + if fixed: + o.writelines(fixed) + cats += 1 + continue + if " < 0" in line: + line, n = fix_line(line) + total += n + if BIG in line: + line, n = fix_dshl(line) + shls += n + if "asSInt(" in line and "eq(asSInt(" in line: + line, n = fix_zero_tests(line) + zeros += n + if line.count("(") > MAX_DEPTH: + deep = fix_deep(line, hoisted) + if deep: + o.writelines(deep) + continue + o.write(line) +print(f"fix_firrtl: rewrote {total} signed shifts, {cats} concatenated connects, {shls} left shifts, " + f"{zeros} signed zero tests, hoisted {hoisted[0]} deep subexpressions") diff --git a/hw/syn/firesim/fame5/gen_rams.py b/hw/syn/firesim/fame5/gen_rams.py new file mode 100644 index 0000000000..dcc3944bb4 --- /dev/null +++ b/hw/syn/firesim/fame5/gen_rams.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Replace VX_dp_ram/VX_sp_ram extmodules in a Yosys FIRRTL file with FIRRTL memories. + +usage: gen_rams.py + +Yosys keeps VX_dp_ram as a black box (its FIRRTL writer cannot emit clocked +read ports or byte enables) and writes one extmodule per instance, named +VX_dp_ram_, with that instance's parameters. Each is replaced here by +a module of the same name built on a FIRRTL `mem` whose data type is a vector +of WRENW lanes (so byte enables are write masks), reproducing VX_dp_ram's +simulation model exactly: + + OUT_REG=1, RDW_MODE=W raddr registered on read, rdata = ram[raddr_r] + -> 1-cycle read of the next raddr_r, plus a bypass + of a same-cycle write to that address. + OUT_REG=1, RDW_MODE=R rdata_r <= ram[raddr] on read (value before write) + -> writes delayed one cycle through a buffer that + reads forward from; output held when not reading. + OUT_REG=0, RDW_MODE=W rdata = ram[raddr], combinational. + OUT_REG=0, RDW_MODE=R combinational, with the one-cycle old-data bypass. + +The bypasses make the result independent of the memory's read-under-write +behaviour, which FAME-5's threaded memories do not preserve. Every OUT_REG=1 +memory has read latency 1, so it stays block-RAM-mappable after threading. +""" +import re +import sys + +DEFAULTS = dict(DATAW=1, SIZE=1, WRENW=1, OUT_REG=0, RDW_MODE="W", RESET_RAM=0, INIT_ENABLE=0) + + +def log2up(n): + return max(1, (n - 1).bit_length()) + + +def lanes(expr, n, w): + return [f"bits({expr}, {i * w + w - 1}, {i * w})" for i in range(n)] + + +def cat(items): + """cat(items[n-1], ..., items[0]) -- items[0] is the least significant.""" + out = items[0] + for it in items[1:]: + out = f"cat({it}, {out})" + return out + + +def gen(name, prm, sp): + p = dict(DEFAULTS) + p.update(prm) + for k in ("RESET_RAM", "INIT_ENABLE"): + if int(p[k]): + sys.exit(f"gen_rams: {name}: {k} is not supported") + dw, size, nw = int(p["DATAW"]), int(p["SIZE"]), int(p["WRENW"]) + oreg, mode = int(p["OUT_REG"]), p["RDW_MODE"].strip('"') + aw, lw = log2up(size), dw // nw + assert lw * nw == dw + L = [] + a = L.append + a(f" module {name}:") + addrs = (("input", "addr", aw),) if sp else (("input", "waddr", aw), ("input", "raddr", aw)) + for d, n, w in (("input", "clk", None), ("input", "reset", 1), ("input", "read", 1), ("input", "write", 1), + ("input", "wren", nw), ("input", "wdata", dw)) + addrs + (("output", "rdata", dw),): + a(f" {d} {n}: {'Clock' if w is None else f'UInt<{w}>'}") + a("") + if sp: + # VX_sp_ram: one address for both; its no-change mode is read-first + # with the output kept when the cycle also writes. + a(" node waddr = addr") + a(" node raddr = addr") + if mode == "N": + assert oreg, f"{name}: RDW_MODE N needs OUT_REG" + mode = "N" + rden = "and(read, not(write))" if mode == "N" else "read" + if mode == "N": + mode = "R" + readers = ["r", "r2"] if (oreg == 0 and mode == "R") else ["r"] + a(" mem ram:") + a(f" data-type => UInt<{lw}>[{nw}]") + a(f" depth => {size}") + a(f" read-latency => {1 if oreg else 0}") + a(" write-latency => 1") + for r in readers: + a(f" reader => {r}") + a(" writer => w") + a(" read-under-write => undefined") + a("") + for r in readers: + a(f" ram.{r}.clk <= clk") + a(f" ram.{r}.en <= UInt<1>(1)") + a(" ram.w.clk <= clk") + mask = (lambda src: [f"bits({src}, {i}, {i})" for i in range(nw)]) if nw > 1 else (lambda src: ["UInt<1>(1)"]) + + def write_port(en, addr, data, wren): + a(f" ram.w.en <= {en}") + a(f" ram.w.addr <= {addr}") + for i, (d, m) in enumerate(zip(lanes(data, nw, lw), mask(wren))): + a(f" ram.w.data[{i}] <= {d}") + a(f" ram.w.mask[{i}] <= {m}") + + def merge(base, data, hit, wren): + """base with the lanes of data enabled by (hit & wren) substituted.""" + m = mask(wren) + out = [] + for i, (b, d) in enumerate(zip(lanes(base, nw, lw), lanes(data, nw, lw))): + out.append(f"mux(and({hit}, {m[i]}), {d}, {b})") + return cat(out) + + mem_out = lambda r: cat([f"ram.{r}.data[{i}]" for i in range(nw)]) + + if oreg and mode == "W": + a(f" reg raddr_r: UInt<{aw}>, clk") + a(" node next_addr = mux(read, raddr, raddr_r)") + a(" raddr_r <= next_addr") + a(" ram.r.addr <= next_addr") + write_port("write", "waddr", "wdata", "wren") + a(" reg hit_r: UInt<1>, clk") + a(" hit_r <= and(write, eq(waddr, next_addr))") + a(f" reg wdata_r: UInt<{dw}>, clk") + a(" wdata_r <= wdata") + a(f" reg wren_r: UInt<{nw}>, clk") + a(" wren_r <= wren") + a(f" rdata <= {merge(mem_out('r'), 'wdata_r', 'hit_r', 'wren_r')}") + elif oreg and mode == "R": + a(" reg wb_v: UInt<1>, clk") + a(f" reg wb_addr: UInt<{aw}>, clk") + a(f" reg wb_data: UInt<{dw}>, clk") + a(f" reg wb_en: UInt<{nw}>, clk") + a(" wb_v <= write") + a(" wb_addr <= waddr") + a(" wb_data <= wdata") + a(" wb_en <= wren") + write_port("wb_v", "wb_addr", "wb_data", "wb_en") + a(" ram.r.addr <= raddr") + a(" reg hit_r: UInt<1>, clk") + a(" hit_r <= and(wb_v, eq(wb_addr, raddr))") + a(f" reg fwd_data: UInt<{dw}>, clk") + a(" fwd_data <= wb_data") + a(f" reg fwd_en: UInt<{nw}>, clk") + a(" fwd_en <= wb_en") + a(" reg read_r: UInt<1>, clk") + a(f" read_r <= {rden}") + a(f" reg hold: UInt<{dw}>, clk") + a(f" node val = mux(read_r, {merge(mem_out('r'), 'fwd_data', 'hit_r', 'fwd_en')}, hold)") + a(" hold <= val") + a(" rdata <= val") + elif not oreg and mode == "W": + a(" ram.r.addr <= raddr") + write_port("write", "waddr", "wdata", "wren") + a(f" rdata <= {mem_out('r')}") + else: + a(" ram.r.addr <= raddr") + a(" ram.r2.addr <= waddr") + write_port("write", "waddr", "wdata", "wren") + a(" reg prev_write: UInt<1>, clk with: (reset => (reset, UInt<1>(0)))") + a(f" reg prev_data: UInt<{dw}>, clk with: (reset => (reset, UInt<{dw}>(0)))") + a(f" reg prev_waddr: UInt<{aw}>, clk with: (reset => (reset, UInt<{aw}>(0)))") + a(" prev_write <= write") + a(f" prev_data <= {mem_out('r2')}") + a(" prev_waddr <= waddr") + a(f" rdata <= mux(and(prev_write, eq(prev_waddr, raddr)), prev_data, {mem_out('r')})") + a("") + return L + + +src = open(sys.argv[1]).read().splitlines() +out, i, n = [], 0, 0 +while i < len(src): + m = re.match(r" extmodule (VX_[ds]p_ram\S*?)\s*:", src[i]) + if not m: + out.append(src[i]) + i += 1 + continue + name, j, prm = m.group(1), i + 1, {} + while j < len(src) and not re.match(r" (ext)?module ", src[j]): + pm = re.match(r"\s+parameter (\w+) = (.*?)\s*$", src[j]) + if pm: + prm[pm.group(1)] = pm.group(2) + j += 1 + out += gen(name, prm, name.startswith("VX_sp_ram")) + n += 1 + i = j +open(sys.argv[2], "w").write("\n".join(out) + "\n") +print(f"gen_rams: {n} VX_dp_ram/VX_sp_ram instances") diff --git a/hw/syn/firesim/fame5/splice.py b/hw/syn/firesim/fame5/splice.py new file mode 100644 index 0000000000..b865d7d9f8 --- /dev/null +++ b/hw/syn/firesim/fame5/splice.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Replace placeholder extmodules in a Chisel-emitted .fir with FIRRTL modules. + +usage: splice.py ... + +Each is a Yosys circuit whose main module is M; all its modules +are inserted. Chisel +emits one extmodule per BlackBox instance (M, M_1, M_2, ...), all with +`defname = M`. Every one of them is removed, the first is replaced by M's +body, and every `inst ... of ` is pointed at M, so that the +wrappers around them become identical and dedup to a single module. +""" +import re +import sys + + +def module_of(path): + """(main module name, every module of the file) -- the main one replaces the placeholders.""" + lines = open(path).read().splitlines() + name = next(re.match(r"circuit (\S+?)\s*:", l).group(1) for l in lines if l.startswith("circuit ")) + start = next(i for i, l in enumerate(lines) if re.match(r" (ext)?module \S+\s*:", l)) + return name, lines[start:] + + +def stanzas(circuit): + """Split into (header, [(kind, name, lines)]).""" + head, out, cur = [], [], None + for l in circuit: + m = re.match(r" (extmodule|module) (\S+?)\s*:", l) + if m: + cur = (m.group(1), m.group(2), [l]) + out.append(cur) + elif cur is None: + head.append(l) + else: + cur[2].append(l) + return head, out + + +def splice(circuit, name, body): + head, mods = stanzas(circuit) + placeholders = { + n for k, n, ls in mods + if k == "extmodule" and any(re.match(rf"\s+defname = {re.escape(name)}\s*$", l) for l in ls) + } + if not placeholders: + sys.exit(f"splice: no extmodule with defname {name}") + out, placed = list(head), False + for k, n, ls in mods: + if n in placeholders: + if not placed: + out += body + [""] + placed = True + continue + for l in ls: + m = re.match(r"(\s+inst \S+ of )(\S+)(.*)$", l) + if m and m.group(2) in placeholders: + l = m.group(1) + name + m.group(3) + out.append(l) + return out, len(placeholders) + + +circuit = open(sys.argv[1]).read().splitlines() +for m in sys.argv[2:]: + name, body = module_of(m) + circuit, n = splice(circuit, name, body) + print(f"splice: {name} <- {m} ({n} placeholders)") +open(sys.argv[1], "w").write("\n".join(circuit) + "\n") diff --git a/hw/syn/firesim/fame5/to_firrtl.sh b/hw/syn/firesim/fame5/to_firrtl.sh new file mode 100755 index 0000000000..ab101d5358 --- /dev/null +++ b/hw/syn/firesim/fame5/to_firrtl.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# usage: yosys2firrtl.sh [] [] +# Verilog (sv2v output) -> FIRRTL, flattened except for . +# +# Only the boundary of the unit to be threaded has to survive: FAME-5 rewrites +# that unit's insides regardless, and everything around it is one model. The +# kept module must be instantiated without parameter overrides: Yosys's FIRRTL +# writer cannot name an instance of a parameterized ($paramod) module, and +# identical instances are what threading needs in the first place. +set -euo pipefail +IN=$1 TOP=$2 OUT=$3 KEEP=$4 NEWTOP=${5:-$2} PRE=${6:-} +Y=${YOSYS:-yosys} +W=$(mktemp -d) +trap 'rm -rf "$W"' EXIT +REN="" +[ "$NEWTOP" != "$TOP" ] && REN="rename $TOP $NEWTOP" +$Y -q -l "$W/y.log" -p " + read_verilog -sv $IN + $PRE + hierarchy -check -top $TOP + setattr -set hdlname \"blackbox\" t:\$paramod* + setattr -mod -set keep_hierarchy 1 $KEEP + proc -norom; flatten; opt -nosdff -nodffe + memory -nomap; memory_nordff; opt -full; opt -nosdff -nodffe + dffunmap; opt_clean -purge + splitnets -driver; opt_clean -purge + splitnets $TOP; opt_clean -purge + $REN + write_firrtl $W/raw.fir" || { tail -5 "$W/y.log"; exit 1; } +# The writer types clocks as UInt<1> and converts each use with asClock(); +# (Above: the non-threaded top is split to single bits -- Yosys reuses bits of a +# wire to drive other bits of the same wire, a loop at FIRRTL's word level.) +# every clock here is a port named clk, which Chisel sees as a Clock. +python3 "$(dirname "$0")/fix_firrtl.py" "$W/raw.fir" "$W/fixed0.fir" +python3 "$(dirname "$0")/gen_rams.py" "$W/fixed0.fir" "$W/fixed.fir" +sed -e 's/^\( input clk: \)UInt<1>/\1Clock/' -e 's/asClock(clk)/clk/g' "$W/fixed.fir" > "$OUT" +echo "yosys2firrtl: $(grep -c '^ module ' "$OUT") modules, $(grep -c ' inst ' "$OUT") instances -> $OUT" diff --git a/hw/syn/firesim/src/main/scala/VortexTarget.scala b/hw/syn/firesim/src/main/scala/VortexTarget.scala index c70edd00bc..a9a72765f6 100644 --- a/hw/syn/firesim/src/main/scala/VortexTarget.scala +++ b/hw/syn/firesim/src/main/scala/VortexTarget.scala @@ -226,6 +226,11 @@ class VortexConfig NastiParameters(dataBits = 512, addrBits = 32, idBits = 4) }) +/** VortexConfig under another name, for FAME5=1 builds: FireSim keys its generated RTL by design and configs, and the + * threaded design must not share a directory with the unthreaded one. + */ +class VortexFame5Config extends Config(new VortexConfig) + class VortexTarget(implicit val p: Parameters) extends RawModule { val clock = RationalClockBridge().io.clocks.head val reset = WireInit(false.B) diff --git a/sim/firesim/Makefile b/sim/firesim/Makefile index eaa3d7cde0..1ceb30c0e8 100644 --- a/sim/firesim/Makefile +++ b/sim/firesim/Makefile @@ -9,7 +9,7 @@ PROJECT := firesim SIMIF_DIR := $(FIRESIM_PATH)/sim/midas/src/main/cc DESIGN ?= VortexTarget -TARGET_CONFIG ?= VortexConfig +TARGET_CONFIG ?= $(if $(FAME5),VortexFame5Config,VortexConfig) # Nothing given on this make's command line reaches FireSim's makes: FireSim # uses PLATFORM and TARGET for its own purposes. Each sub-make is handed what it @@ -92,7 +92,7 @@ ifneq ($(filter $(TARGET),xrtsim avedsim),) # sources the standalone metasim builds from; the generated FireSim RTL and # headers come from the elaborated target. SYN_DIR := $(ROOT_DIR)/hw/syn/firesim -EMUL_DIR := $(DESTDIR)/firesim_emul_$(FIRESIM_CARD) +EMUL_DIR := $(DESTDIR)/firesim_emul_$(FIRESIM_CARD)$(if $(FAME5),_fame5) WORK_DIR := $(EMUL_DIR)/work SIM_SRCS_LIST := $(WORK_DIR)/vortex_srcs.f SIM_SRCS_DIR := $(WORK_DIR)/vortex_srcs @@ -101,12 +101,21 @@ EMUL_NAME := firesim_emul EMUL_SRCS := $(SRC_DIR)/firesim_sim.cpp $(SRC_DIR)/firesim_emul_main.cpp +# Under FAME5 the generated RTL already contains Vortex (converted to FIRRTL and +# threaded by Golden Gate), so Verilator is given no Vortex sources. +ifdef FAME5 +EMUL_VERILATOR_FLAGS := -Wno-fatal +else +EMUL_VERILATOR_FLAGS := -Wno-fatal -y $(SIM_SRCS_DIR) +libext+.sv+.v $$(tr '\n' ' ' < $(SIM_SRCS_LIST)) +endif + .PHONY: emul-sources # The simulation flattening of the Vortex RTL for CONFIGS, and Golden Gate's # elaboration of the card's target; both are no-ops when current. emul-sources: $(MAKE) -C $(SYN_DIR) sources elaborate FIRESIM_CARD=$(FIRESIM_CARD) FIRESIM_WORK_DIR=$(WORK_DIR) \ - XLEN=$(XLEN) CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) + XLEN=$(XLEN) CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) $(if $(FAME5),FAME5=$(FAME5)) \ + $(if $(FAME5_THREADS),FAME5_THREADS=$(FAME5_THREADS)) # Host toolchain, as for the aved platform library: the runtime that loads this # is built with it. -fPIC reaches the Verilated objects too, through -CFLAGS. @@ -127,7 +136,7 @@ $(EMUL_DIR)/V$(EMUL_NAME): $(EMUL_SRCS) $(SIM_SRCS_LIST) $(ELAB_DIR)/FireSim-gen DRIVER="$(EMUL_SRCS)" \ CXX=/usr/bin/g++ \ VERILATOR="verilator --timing --cc --exe" \ - VERILATOR_FLAGS="-Wno-fatal -y $(SIM_SRCS_DIR) +libext+.sv+.v $$(tr '\n' ' ' < $(SIM_SRCS_LIST))" \ + VERILATOR_FLAGS="$(EMUL_VERILATOR_FLAGS)" \ CXXFLAGS="-O2 -fPIC -std=c++20 -DFIRESIM_EMUL -I$(SRC_DIR)" \ LDFLAGS="-shared" diff --git a/sw/runtime/firesim/Makefile b/sw/runtime/firesim/Makefile index d604055085..8eac1c64a7 100644 --- a/sw/runtime/firesim/Makefile +++ b/sw/runtime/firesim/Makefile @@ -55,7 +55,8 @@ driver: $(DESTDIR)/libfiresim.so $(DESTDIR)/libfiresim.so: force DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) \ - CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) + CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) $(if $(FAME5),FAME5=$(FAME5)) \ + $(if $(FAME5_THREADS),FAME5_THREADS=$(FAME5_THREADS)) $(DESTDIR)/$(PROJECT): $(SRCS) $(DESTDIR)/libfiresim.so $(CONFIG_FILE) $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ diff --git a/tests/regression/common.mk b/tests/regression/common.mk index d50656be64..0e46b3444f 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -189,7 +189,7 @@ kernel.vxbin: kernel.elf $(VORTEX_KN_PATH)/lib$(KERNEL_LIB).a: $(MAKE) -C $(VORTEX_KN_PATH) -RUNTIME_ARGS = CONFIGS="$(CONFIGS)" $(if $(DEBUG),DEBUG=$(DEBUG)) $(if $(PERF),PERF=$(PERF)) $(if $(SCOPE),SCOPE=$(SCOPE)) $(if $(SAIF),SAIF=$(SAIF)) +RUNTIME_ARGS = CONFIGS="$(CONFIGS)" $(if $(DEBUG),DEBUG=$(DEBUG)) $(if $(PERF),PERF=$(PERF)) $(if $(FAME5),FAME5=$(FAME5)) $(if $(FAME5_THREADS),FAME5_THREADS=$(FAME5_THREADS)) $(if $(SCOPE),SCOPE=$(SCOPE)) $(if $(SAIF),SAIF=$(SAIF)) # FORCE, not a bare rule: a rule with no prerequisites runs only when its target # is missing, so once this library exists make never re-enters the sub-make that From f500734264318b66c4066a5e3868d09278757250 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sat, 26 Sep 2026 18:37:44 -0700 Subject: [PATCH 167/336] firesim/fame5: repair FIRRTL in linear time; keep work files by the output fix_firrtl.py re-scanned each statement for every rewrite, which is quadratic in line length; a 64-core conversion has lines of over a million characters nested thousands deep and had not finished after 27 minutes. It now tokenizes each statement once into a tree, rewrites bottom-up and prints iteratively. Output is unchanged (lockstep RAM bench and the 4-core demo match cycle for cycle); the 64-core repair takes about a minute. to_firrtl.sh wrote its intermediates to a mktemp directory it then deleted. They now go to .work beside the output and are kept, since the Yosys log and the unrepaired FIRRTL are what a later failure is debugged from. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7f308ffd68470e325ec0aaf60996f526cefeb977) --- hw/syn/firesim/fame5/fix_firrtl.py | 392 +++++++++++++---------------- hw/syn/firesim/fame5/to_firrtl.sh | 6 +- 2 files changed, 178 insertions(+), 220 deletions(-) diff --git a/hw/syn/firesim/fame5/fix_firrtl.py b/hw/syn/firesim/fame5/fix_firrtl.py index 943ba378a4..de8788d857 100644 --- a/hw/syn/firesim/fame5/fix_firrtl.py +++ b/hw/syn/firesim/fame5/fix_firrtl.py @@ -1,224 +1,181 @@ #!/usr/bin/env python3 -"""Repair Yosys write_firrtl's output. +"""Repair Yosys write_firrtl's output so the FIRRTL compiler and Verilator take it. usage: fix_firrtl.py +Each rewrite keeps the circuit's meaning; they work on expression trees parsed +in one linear pass (a converted Vortex has lines of over a million characters, +nested thousands deep). + 1. Instance outputs driving split wires. Yosys emits cat(a, cat(b, c)) <= inst.port - which FIRRTL does not allow; each piece gets its slice instead: - c <= bits(inst.port, wc-1, 0); b <= bits(..., wc+wb-1, wc); ... - -2. $shift cells with a signed amount, below. - -5. Zero tests of signed values. Yosys's logic ops test each operand with - neq(x, UInt<1>("h0")), even when x is asSInt(...), which FIRRTL rejects as - a signed/unsigned comparison. x != 0 does not depend on the sign, so the - asSInt is dropped (likewise for eq). - -4. Deep expressions. The FIRRTL parser recurses once per nesting level, and - Yosys writes long cat()/mux() chains (thousands deep for wide arbiters); - any subexpression deeper than MAX_DEPTH is hoisted into a node. - -3. Left shifts. Yosys writes bits(dshl(a, min(b, 2^19-1)), hi, 0): correct, but - the intermediate is ~2^19 bits wide, beyond Verilator's constant limit. Only - bits hi..0 are kept, so the clamp is lowered to the first 2^k-1 > hi. - -Yosys emits mux(B < 0, bits(dshl(A, ), 0, W), dshr(A, B)) -which is not FIRRTL (`<`), has bits() bounds reversed, and shifts left by B -rather than -B. $shift semantics are: Y = A >> B, and for B < 0, Y = A << -B. -This rewrites each occurrence to - - mux(lt(asSInt(B), SInt<1>(0)), - bits(dshl(pad(A, W), ), W-1, 0), - bits(pad(dshr(A, B), W), W-1, 0)) - -which needs no width but W, taken from the original bits(..., 0, W). + which FIRRTL does not allow; each piece gets its slice instead. + +2. Shifts by a signed amount ($shift, B signed). Yosys emits + mux(B < 0, bits(dshl(A, ), 0, W), dshr(A, B)) + which is not FIRRTL (`<`), has bits() bounds reversed, and shifts left by B + rather than -B. $shift is Y = A >> B, and Y = A << -B for B < 0, so it becomes + mux(lt(asSInt(B), SInt<1>(0)), + bits(dshl(pad(A, W), min(-B, W)), W-1, 0), + bits(pad(dshr(A, B), W), W-1, 0)) + +3. Left shifts. Yosys writes bits(dshl(a, min(b, 2^19-1)), hi, lo): correct, + but the intermediate is ~2^19 bits wide, beyond Verilator's constant limit. + Only bits hi..lo are kept, so the clamp becomes the first 2^k-1 > hi. + +4. Zero tests of signed values: neq(asSInt(x), 0) is rejected as a + signed/unsigned comparison; x != 0 does not depend on the sign, so the + asSInt goes (likewise for eq). + +5. Deep expressions. The FIRRTL parser recurses once per nesting level, and + Yosys writes cat()/mux() chains thousands deep for wide arbiters; any + subexpression deeper than MAX_DEPTH is hoisted into a node. """ +import re import sys -sys.setrecursionlimit(100000) # expressions nest thousands deep - - -def match_paren(s, i): - """s[i] == '('; return the index of its matching ')'.""" - depth = 0 - for j in range(i, len(s)): - if s[j] == "(": - depth += 1 - elif s[j] == ")": - depth -= 1 - if depth == 0: - return j - raise ValueError("unbalanced") - - -def split_args(s): - """Split a comma-separated argument list at top level.""" - out, depth, cur = [], 0, "" - for ch in s: - if ch == "(": - depth += 1 - elif ch == ")": - depth -= 1 - if ch == "," and depth == 0: - out.append(cur.strip()) - cur = "" - else: - cur += ch - out.append(cur.strip()) - return out - - -def fix_line(line): - count = 0 - start = 0 - while True: - i = line.find("mux(", start) - if i < 0: - return line, count - j = match_paren(line, i + 3) - args = split_args(line[i + 4:j]) - if len(args) == 3 and args[0].endswith(" < 0"): - b = args[0][: -len(" < 0")] - left, right = args[1], args[2] - # left: bits(dshl(A, S), 0, W) - inner = split_args(left[len("bits("):-1]) - dshl = inner[0] - w = int(inner[2]) - a = split_args(dshl[len("dshl("):-1])[0] - assert right.startswith("dshr(") - c = w.bit_length() - mag = f"asUInt(neg(asSInt({b})))" - amt = f"bits(mux(gt({mag}, UInt<{c}>({w})), UInt<{c}>({w}), {mag}), {c - 1}, 0)" - new = (f"mux(lt(asSInt({b}), SInt<1>(0)), " - f"bits(dshl(pad({a}, {w}), {amt}), {w - 1}, 0), " - f"bits(pad(dshr({a}, {b}), {w}), {w - 1}, 0))") - line = line[:i] + new + line[j + 1:] - count += 1 - start = i + len(new) - else: - start = i + 4 - +sys.setrecursionlimit(100000) # transforms recurse over trees thousands deep -import re - -BIG = "UInt<19>(524287)" MAX_DEPTH = 48 -LITERAL = re.compile(r"^[US]Int<\d+>\(") - +BIG = "UInt<19>(524287)" -def parse(e): - """Expression -> (text, children) tree; literals and references are leaves.""" - e = e.strip() - m = re.match(r"^([A-Za-z_][A-Za-z0-9_]*)\(", e) - if m and not LITERAL.match(e) and match_paren(e, len(m.group(1))) == len(e) - 1: - return (m.group(1), [parse(a) for a in split_args(e[len(m.group(1)) + 1:-1])]) - return (e, None) +TOKEN = re.compile(r"\s*(?:(?P[US]Int(?:<\d+>)?\([^()]*\))" + r"|(?P[A-Za-z_]\w*)\(" + r"|(?P\))|(?P,)|(?P<)" + r"|(?P[^\s(),<]+))") +ZERO = re.compile(r'^UInt(<\d+>)?\(("h0+"|0)\)$') +DECL = re.compile(r"\s+(?:input|output|wire|reg) (\S+?)\s*: UInt<(\d+)>") +STMT = re.compile(r"(\s+)(node \S+ = |\S+ <= )(.*?)(\s*@\[.*)?$") + + +class Node: + """An operation: op(args...). Leaves (references, literals, integers) are str.""" + __slots__ = ("op", "args") + + def __init__(self, op, args): + self.op, self.args = op, args + + +def parse(text): + """One linear pass over text; `B < 0` (only in rewrite 2) parses as Node('<').""" + stack = [Node("", [])] + items = [[]] # tokens of the argument being read, per open call + pos = 0 + while pos < len(text): + m = TOKEN.match(text, pos) + if not m or m.end() == pos: + if text[pos:].strip() == "": + break + raise ValueError(f"cannot tokenize at: {text[pos:pos + 60]}") + pos = m.end() + kind = m.lastgroup + if kind == "call": + node = Node(m.group("call"), []) + items[-1].append(node) + stack.append(node) + items.append([]) + elif kind in ("comma", "close"): + arg = items[-1] + if arg: + stack[-1].args.append(arg[0] if len(arg) == 1 else Node("<", [arg[0], arg[2]])) + items[-1] = [] + if kind == "close": + stack.pop() + items.pop() + elif kind == "lt": + items[-1].append("<") + else: + items[-1].append(m.group(kind)) + arg = items[0] + return arg[0] if len(arg) == 1 else Node("<", [arg[0], arg[2]]) def emit(t): - op, kids = t - return op if kids is None else f"{op}({', '.join(emit(k) for k in kids)})" - - -def hoist(t, nodes, ns, indent): - """Rewrite t so no expression exceeds MAX_DEPTH; returns (tree, depth).""" - op, kids = t - if kids is None: + """Iterative, so an unhoisted tree of any depth can be printed.""" + out, work = [], [t] + while work: + x = work.pop() + if isinstance(x, str): + out.append(x) + elif isinstance(x, Node): + if x.op == "<": + work += [x.args[1], " < ", x.args[0]] + continue + work.append(")") + for i, a in enumerate(reversed(x.args)): + work.append(a) + if i < len(x.args) - 1: + work.append(", ") + work.append(f"{x.op}(") + return "".join(out) + + +def lit(w, v): + return f"UInt<{w}>({v})" + + +counts = dict(shift=0, shl=0, zero=0, hoist=0, cat=0) + + +def rewrite(t): + """Rewrites 2-4, bottom-up.""" + if isinstance(t, str): + return t + t.args = [rewrite(a) for a in t.args] + a = t.args + # 2. mux(B < 0, bits(dshl(A, S), 0, W), dshr(A, B)) + if t.op == "mux" and len(a) == 3 and isinstance(a[0], Node) and a[0].op == "<": + b, w = a[0].args[0], int(a[1].args[2]) + A = a[1].args[0].args[0] + c = w.bit_length() + mag = Node("asUInt", [Node("neg", [Node("asSInt", [b])])]) + amt = Node("bits", [Node("mux", [Node("gt", [mag, lit(c, w)]), lit(c, w), mag]), str(c - 1), "0"]) + counts["shift"] += 1 + return Node("mux", [Node("lt", [Node("asSInt", [b]), "SInt<1>(0)"]), + Node("bits", [Node("dshl", [Node("pad", [A, str(w)]), amt]), str(w - 1), "0"]), + Node("bits", [Node("pad", [Node("dshr", [A, b]), str(w)]), str(w - 1), "0"])]) + # 3. bits(dshl(A, asUInt(mux(gt(b, BIG), BIG, bits(b, 18, 0)))), hi, lo) + if (t.op == "bits" and len(a) == 3 and isinstance(a[0], Node) and a[0].op == "dshl" + and isinstance(a[0].args[1], Node) and a[0].args[1].op == "asUInt"): + mx = a[0].args[1].args[0] + if (isinstance(mx, Node) and mx.op == "mux" and mx.args[1] == BIG + and isinstance(mx.args[0], Node) and mx.args[0].op == "gt" and mx.args[0].args[1] == BIG): + b = mx.args[0].args[0] + k = (int(a[1]) + 1).bit_length() + m = (1 << k) - 1 + a[0].args[1] = Node("asUInt", [Node("mux", [Node("gt", [b, lit(k, m)]), lit(k, m), + Node("bits", [Node("pad", [b, str(k)]), str(k - 1), "0"])])]) + counts["shl"] += 1 + # 4. (n)eq(asSInt(x), 0) + if (t.op in ("eq", "neq") and len(a) == 2 and isinstance(a[0], Node) and a[0].op == "asSInt" + and isinstance(a[1], str) and ZERO.match(a[1])): + t.args[0] = a[0].args[0] + counts["zero"] += 1 + return t + + +def hoist(t, nodes, indent): + """5. Returns (tree, depth) with every deeper subexpression moved to a node.""" + if isinstance(t, str): return t, 0 - new, depth = [], 0 - for k in kids: - k2, d = hoist(k, nodes, ns, indent) + depth = 0 + for i, k in enumerate(t.args): + k2, d = hoist(k, nodes, indent) if d >= MAX_DEPTH: - name = f"_fx_{ns[0]}" - ns[0] += 1 + name = f"_fx_{counts['hoist']}" + counts["hoist"] += 1 nodes.append(f"{indent}node {name} = {emit(k2)}\n") - k2, d = (name, None), 0 - new.append(k2) + k2, d = name, 0 + t.args[i] = k2 depth = max(depth, d) - return (op, new), depth + 1 - - -ZERO = re.compile(r'^UInt(<\d+>)?\(("h0+"|0)\)$') - - -def fix_zero_tests(line): - count, start = 0, 0 - while True: - m = re.compile(r"\b(n?eq)\(asSInt\(").search(line, start) - if not m: - return line, count - i = m.start() - j = match_paren(line, m.end(1)) - args = split_args(line[m.end(1) + 1:j]) - if len(args) == 2 and ZERO.match(args[1]) and args[0].startswith("asSInt("): - inner = args[0][len("asSInt("):-1] - new = f"{m.group(1)}({inner}, {args[1]})" - line = line[:i] + new + line[j + 1:] - count += 1 - start = i + len(new) - else: - start = m.end() + return t, depth + 1 -def fix_deep(line, ns): - m = re.match(r"(\s+)(node \S+ = |\S+ <= )(.*?)(\s*@\[.*)?$", line.rstrip("\n")) - if not m: - return None - indent, head, rhs, info = m.group(1), m.group(2), m.group(3), m.group(4) or "" - d = cur = 0 - for c in rhs: - if c == "(": - cur += 1 - d = max(d, cur) - elif c == ")": - cur -= 1 - if d <= MAX_DEPTH: - return None - nodes = [] - t, _ = hoist(parse(rhs), nodes, ns, indent) - return nodes + [f"{indent}{head}{emit(t)}{info}\n"] - - -def fix_dshl(line): - count, start = 0, 0 - while True: - i = line.find("bits(dshl(", start) - if i < 0: - return line, count - j = match_paren(line, i + 4) - args = split_args(line[i + 5:j]) - if len(args) == 3 and BIG in args[0]: - hi = int(args[1]) - d = args[0] - dargs = split_args(d[len("dshl("):-1]) - amt = dargs[1] - m = re.fullmatch(r"asUInt\(mux\(gt\((.*), UInt<19>\(524287\)\), UInt<19>\(524287\), bits\((.*), 18, 0\)\)\)", amt) - if m and m.group(1) == m.group(2): - b = m.group(1) - k = (hi + 1).bit_length() - mx = (1 << k) - 1 - amt = f"asUInt(mux(gt({b}, UInt<{k}>({mx})), UInt<{k}>({mx}), bits(pad({b}, {k}), {k - 1}, 0)))" - new = f"bits(dshl({dargs[0]}, {amt}), {args[1]}, {args[2]})" - line = line[:i] + new + line[j + 1:] - count += 1 - start = i + len(new) - continue - start = i + 5 - - -DECL = re.compile(r"\s+(?:input|output|wire|reg) (\S+?)\s*: UInt<(\d+)>") - - -def cat_leaves(expr): +def cat_leaves(t): """cat(x, cat(y, z)) -> [x, y, z] (most significant first).""" - expr = expr.strip() - if not expr.startswith("cat("): - return [expr] - out = [] - for a in split_args(expr[4:match_paren(expr, 3)]): - out += cat_leaves(a) - return out + if isinstance(t, Node) and t.op == "cat": + return [l for a in t.args for l in cat_leaves(a)] + return [emit(t)] def fix_cat_lhs(line, widths): @@ -227,17 +184,21 @@ def fix_cat_lhs(line, widths): return None indent, lhs, rhs, info = m.group(1), m.group(2), m.group(3), m.group(4) or "" out, lo = [], 0 - for leaf in reversed(cat_leaves(lhs)): + for leaf in reversed(cat_leaves(parse(lhs))): if leaf not in widths: sys.exit(f"fix_firrtl: no width for {leaf} in: {line.strip()[:120]}") w = widths[leaf] out.append(f"{indent}{leaf} <= bits({rhs}, {lo + w - 1}, {lo}){info}\n") lo += w + counts["cat"] += 1 return out -total = cats = shls = zeros = 0 -hoisted = [0] +def needs_work(line): + return (" < 0" in line or BIG in line or "eq(asSInt(" in line + or line.count("(") > MAX_DEPTH) + + widths = {} with open(sys.argv[1]) as f, open(sys.argv[2], "w") as o: for line in f: @@ -250,22 +211,17 @@ def fix_cat_lhs(line, widths): fixed = fix_cat_lhs(line, widths) if fixed: o.writelines(fixed) - cats += 1 continue - if " < 0" in line: - line, n = fix_line(line) - total += n - if BIG in line: - line, n = fix_dshl(line) - shls += n - if "asSInt(" in line and "eq(asSInt(" in line: - line, n = fix_zero_tests(line) - zeros += n - if line.count("(") > MAX_DEPTH: - deep = fix_deep(line, hoisted) - if deep: - o.writelines(deep) + if needs_work(line): + m = STMT.match(line.rstrip("\n")) + if m: + indent, head, rhs, info = m.group(1), m.group(2), m.group(3), m.group(4) or "" + nodes = [] + t, _ = hoist(rewrite(parse(rhs)), nodes, indent) + o.writelines(nodes) + o.write(f"{indent}{head}{emit(t)}{info}\n") continue o.write(line) -print(f"fix_firrtl: rewrote {total} signed shifts, {cats} concatenated connects, {shls} left shifts, " - f"{zeros} signed zero tests, hoisted {hoisted[0]} deep subexpressions") +print(f"fix_firrtl: rewrote {counts['shift']} signed shifts, {counts['cat']} concatenated connects, " + f"{counts['shl']} left shifts, {counts['zero']} signed zero tests, " + f"hoisted {counts['hoist']} deep subexpressions") diff --git a/hw/syn/firesim/fame5/to_firrtl.sh b/hw/syn/firesim/fame5/to_firrtl.sh index ab101d5358..c9dea0328b 100755 --- a/hw/syn/firesim/fame5/to_firrtl.sh +++ b/hw/syn/firesim/fame5/to_firrtl.sh @@ -10,8 +10,10 @@ set -euo pipefail IN=$1 TOP=$2 OUT=$3 KEEP=$4 NEWTOP=${5:-$2} PRE=${6:-} Y=${YOSYS:-yosys} -W=$(mktemp -d) -trap 'rm -rf "$W"' EXIT +# Intermediate files live beside the output (kept: the Yosys log and the +# unrepaired FIRRTL are what one reads when a later step rejects the result). +W=$(dirname "$(realpath -m "$OUT")")/$(basename "$OUT" .fir).work +rm -rf "$W" && mkdir -p "$W" REN="" [ "$NEWTOP" != "$TOP" ] && REN="rename $TOP $NEWTOP" $Y -q -l "$W/y.log" -p " From cf4fddedb775ebb919745506bbb4ae587659490c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sat, 26 Sep 2026 18:37:44 -0700 Subject: [PATCH 168/336] firesim: peek without the 0.1 s stability wait after a blocking step A blocking peek waits for the step to finish and then up to 0.1 s for PRECISE_PEEKABLE before reading. The driver only peeks after a blocking step, so the first wait is already satisfied, and it never checks the stability result, so the second changes nothing but time. A FAME-5 design never raises PRECISE_PEEKABLE, so every peek cost the full 0.1 s; dcr_read peeks once per target cycle, making each read about 20 s and the 64-core V80 demo never finish. Peeking non-blocking returns the same value immediately. Follow-up: find why FAME-5 outputs are never precisely peekable. On the V80, 64 cores NW=NT=16 FAME5=1: demo -n1 passes in 7 s (315.7 kHz, FMR 190); default 1M points passes in 16 s (3.46M cycles, 377.8 kHz). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 012cc80cf5456eb4a68477b39e8e2ce88df79485) --- sim/firesim/firesim_sim.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sim/firesim/firesim_sim.cpp b/sim/firesim/firesim_sim.cpp index ef73e27bb0..f84f6e2e6d 100644 --- a/sim/firesim/firesim_sim.cpp +++ b/sim/firesim/firesim_sim.cpp @@ -186,6 +186,14 @@ class vortex_simulation_t final : public simulation_t { peek_poke.step(n, true); } + // Only ever called after a blocking step, which has already waited for the + // step to finish. A blocking peek would additionally wait up to 0.1 s for + // PRECISE_PEEKABLE, which a FAME-5 design never raises, and then return the + // same value: 0.1 s lost per peek, and dcr_read peeks once per target cycle. + uint32_t peek(const char *name) { + return peek_poke.peek(name, false); + } + // The DRAM timing models sit between the target's memory port and the // card's memory. Their request and error counters say whether a request the // core is waiting on ever left the target, came back, or came back an error. @@ -752,8 +760,8 @@ int firesim_sim::dcr_read(uint32_t addr, uint32_t tag, uint32_t *value) { const uint32_t kPollRounds = dcr_poll_rounds(); for (uint32_t i = 0; i < kPollRounds; ++i) { g_sim->step(1); - if (pp.peek("ctrl_dcr_rsp_valid", true) != 0) { - sampled = pp.peek("ctrl_dcr_rsp_data", true); + if (g_sim->peek("ctrl_dcr_rsp_valid") != 0) { + sampled = g_sim->peek("ctrl_dcr_rsp_data"); answered = true; break; } @@ -857,7 +865,7 @@ int firesim_sim::is_busy(bool *busy) { const uint32_t batch = step_batch_; int ret = impl_->submit(label("step %u + busy", batch), [&sampled, batch] { g_sim->step(batch); - sampled = g_sim->peek_poke.peek("ctrl_busy", true); + sampled = g_sim->peek("ctrl_busy"); }); if (ret != 0) { return ret; From 371c2aaf28cb541d3eccb9fdce2c309be6ffaf37 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sat, 26 Sep 2026 22:50:58 -0700 Subject: [PATCH 169/336] firesim: runtime plusargs and a wave-tracing metasimulator build Two observability gaps, both previously unreachable without editing code: VORTEX_FIRESIM_ARGS forwards extra plusargs to the FireSim driver on every vehicle. Chiefly FASED's runtime settings, the only way to change the modelled memory timing without a rebuild: +mm_readLatency_0=100 (with +mm_useHardwareDefaultRuntimeSettings_0 covering the unset registers) takes the 4-core FAME-5 demo from 15541 to 47371 cycles, same 9776 instructions. EMUL_DEBUG=1 builds the metasimulator with wave tracing (--trace +define+DEBUG), arming the emul top's +waveformfile/+dump-start/+max-cycles plusargs, passed at run time through VORTEX_FIRESIM_EMUL_ARGS. Its own build directory: the model rule does not depend on flags, so sharing one would hand back whichever build ran first. The custom emul main now calls traceEverOn before time 0, which Verilator's generated --main would have emitted and $dumpvars aborts without. Traced and untraced runs agree cycle-for-cycle (15541); the dump is ~380 KB per host cycle on the 4-core FAME-5 model, so the window plusargs are not optional. Co-Authored-By: Claude Fable 5 (cherry picked from commit f86852fee13c0db91b2c41d150ee8f94b68826c7) --- sim/firesim/Makefile | 10 ++++++++++ sim/firesim/firesim_emul_main.cpp | 3 +++ sw/runtime/firesim/vortex.cpp | 11 +++++++++++ 3 files changed, 24 insertions(+) diff --git a/sim/firesim/Makefile b/sim/firesim/Makefile index 1ceb30c0e8..2faab70733 100644 --- a/sim/firesim/Makefile +++ b/sim/firesim/Makefile @@ -109,6 +109,16 @@ else EMUL_VERILATOR_FLAGS := -Wno-fatal -y $(SIM_SRCS_DIR) +libext+.sv+.v $$(tr '\n' ' ' < $(SIM_SRCS_LIST)) endif +# EMUL_DEBUG=1: a wave-tracing metasimulator. --trace Verilates the dump hooks +# and +define+DEBUG arms the emul top's +waveformfile/+dump-start plusargs +# (FireSim's top.sv), which VORTEX_FIRESIM_EMUL_ARGS passes at run time. Its own +# build directory: the model rule does not depend on flags, so sharing one would +# hand back whichever build ran first. +ifdef EMUL_DEBUG +EMUL_VERILATOR_FLAGS += --trace --trace-structs +define+DEBUG +EMUL_DIR := $(EMUL_DIR)_dbg +endif + .PHONY: emul-sources # The simulation flattening of the Vortex RTL for CONFIGS, and Golden Gate's # elaboration of the card's target; both are no-ops when current. diff --git a/sim/firesim/firesim_emul_main.cpp b/sim/firesim/firesim_emul_main.cpp index 96b7740a47..8993266acf 100644 --- a/sim/firesim/firesim_emul_main.cpp +++ b/sim/firesim/firesim_emul_main.cpp @@ -34,6 +34,9 @@ int firesim_emul_run(int argc, char **argv) { } const std::unique_ptr contextp{new VerilatedContext}; + // Required before time 0 for an EMUL_DEBUG build's $dumpvars to arm; a + // no-op in a build without tracing. (Verilator's own --main emits it.) + contextp->traceEverOn(true); contextp->commandArgs(static_cast(args.size()), args.data()); if (!extra.empty()) { fprintf(stderr, "[emul] extra args:%s -> randReset=%d seed=%d\n", diff --git a/sw/runtime/firesim/vortex.cpp b/sw/runtime/firesim/vortex.cpp index f633fd87c4..363710fda0 100644 --- a/sw/runtime/firesim/vortex.cpp +++ b/sw/runtime/firesim/vortex.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -92,6 +93,16 @@ class vx_device { plusargs.push_back(std::string("+binary_file=") + xclbin); } #endif + // Extra plusargs for FireSim's bridges, card and metasimulator alike -- + // chiefly FASED's runtime settings (+mm_readLatency_0=... with + // +mm_useHardwareDefaultRuntimeSettings_0 covering the rest), which are + // the only way to change the memory timing without a rebuild. + if (const char* extra = getenv("VORTEX_FIRESIM_ARGS")) { + std::istringstream in(extra); + for (std::string tok; in >> tok;) { + plusargs.push_back(tok); + } + } return sim_.init(plusargs); } From b8371e38dea241036fbb9182c9c367796f3e4912 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sat, 26 Sep 2026 22:50:58 -0700 Subject: [PATCH 170/336] docs: rewrite firesim_integration.md around the questions it must answer The document predated FAME-5, the V80 port and the in-process metasimulator, and drew its architecture in ASCII. Rewritten as answers to the questions a reader arrives with -- what is on the FPGA, how the RTL is synthesized (both lanes), which module FAME-5 time-multiplexes and how, how timing is modelled, how the memory system works and is configured, and how to get waves, RTL messages and counters out of each vehicle -- with six SVG figures (docs/assets/img/firesim_*.svg) replacing every text diagram, per the repo's figure convention. Everything stated was verified against the source or measured this cycle: the 64-core NW=NT=16 FAME-5 V80 image (utilization, timing, FMR, demo and sgemm results), the FASED runtime-settings path, and the EMUL_DEBUG wave flow including the VCD hierarchy and its size. The stale "metasimulation is dead" note is gone: xrtsim/avedsim are the supported metasim path. Co-Authored-By: Claude Fable 5 (cherry picked from commit fbfc870a856caea43189db1d61e71e407f426ada) --- docs/assets/img/firesim_build_flow.svg | 123 ++ docs/assets/img/firesim_fame5_threading.svg | 135 +++ docs/assets/img/firesim_fpga_architecture.svg | 180 +++ docs/assets/img/firesim_memory_system.svg | 134 +++ docs/assets/img/firesim_system_stack.svg | 124 ++ docs/assets/img/firesim_timing_model.svg | 123 ++ docs/designs/firesim_integration.md | 1045 ++++++++++------- 7 files changed, 1465 insertions(+), 399 deletions(-) create mode 100644 docs/assets/img/firesim_build_flow.svg create mode 100644 docs/assets/img/firesim_fame5_threading.svg create mode 100644 docs/assets/img/firesim_fpga_architecture.svg create mode 100644 docs/assets/img/firesim_memory_system.svg create mode 100644 docs/assets/img/firesim_system_stack.svg create mode 100644 docs/assets/img/firesim_timing_model.svg diff --git a/docs/assets/img/firesim_build_flow.svg b/docs/assets/img/firesim_build_flow.svg new file mode 100644 index 0000000000..2ba1638343 --- /dev/null +++ b/docs/assets/img/firesim_build_flow.svg @@ -0,0 +1,123 @@ + + + + + + + + + + Build flow — how Vortex RTL becomes a FireSim image + Left: the default FAME-1 build, where Vortex stays Verilog and Golden Gate never sees inside it. Right: FAME5=1, where Vortex is converted to FIRRTL so Golden Gate can thread the sockets. Both run from hw/syn/firesim/Makefile with the native flows' tokens. + + + + Common front end — the configured tree, flattened + make -C hw/syn/firesim [PLATFORM=<xsa> | DEVICE=<part>] TARGET=hw|sim|hw_emu [FAME5=1] [PERF=1] CONFIGS="-DVX_CFG_…" + gen_sources.sh -P preprocesses hw/rtl into one self-contained directory (macros inlined), top VX_firesim_wrap: + vivado_srcs/ + .f ← -DSYNTHESIS -DVIVADO -DNDEBUG (+ -DSIMULATION for TARGET=sim/hw_emu, never under FAME5) + vortex_srcs/ (metasim) ← -DNDEBUG only: behavioural RAMs, soft FPU + Platform defines both lanes share: VX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 · PLATFORM_MEMORY_ID_WIDTH=4 (paired with idBits) · [PERF_ENABLE] + + + + Default: FAME-1, Vortex as a Verilog BlackBox + + + stage → replace-rtl — Golden Gate on the harness only + VortexTarget.scala elaborates: VortexDUT holds VX_firesim_wrap as an opaque BlackBox; + Golden Gate applies FAME-1 at its boundary, inserts bridges and token channels. + → FireSim-generated.sv (harness; Vortex NOT inside) + + + xilinx-ips — vendor floating-point cores + xil_fma, xil_fdiv, … generated for FPGA_PART; the flattened tree only instantiates them. + (On aved the RTU keeps the soft FMA/divider: the clock-enable conversion cannot enter an OOC core.) + + + Platform build — the card's own flow + vitis: v++ link · aved: slashkit. The kernel is packaged from three inputs: + VORTEX_VIVADO_SRCS = the flattened tree VORTEX_XILINX_IP_DIR = the vendor cores + FireSim-generated.sv = the harness VORTEX_PRE_OPT_TCL = async-BRAM patch hook + Vivado synthesis + place-and-route at FREQUENCY (default 60 MHz), STRATEGY=TIMING; + on aved, FireSim's clock gates become clock enables (no BUFGCE in the partition). + + + Image — build32_<aved|xsa>_<hw|sim|hw_emu>/bin/ + firesim.vbin | firesim.xclbin + firesim.card + + + Metasimulator (no image) — TARGET=xrtsim / avedsim + `elaborate` runs Golden Gate alone; sim/firesim/Makefile Verilates + FireSim-generated.sv + the vortex_srcs tree (library search) into libfiresim.so, + with simif_emul as the host interface — the design itself is the simulator. + EMUL_DEBUG=1 adds --trace +define+DEBUG → +waveformfile=… writes a VCD + FAME5=1 uses the generated RTL only (Vortex is already inside it) + Same target, same tokens as the card: the step before silicon. + + + + FAME5=1: Vortex converted to FIRRTL, sockets threaded + + + sv2v — SystemVerilog → Verilog-2005 + fame5/vortex.v (interfaces inlined; VX_socket_top keeps a plain-wire boundary) + + + to_firrtl.sh — Yosys → FIRRTL (fame5/) + Flattens everything except VX_socket_top (the threaded unit, instantiated without + parameters); VX_dp_ram / VX_sp_ram become black boxes that gen_rams.py regenerates + as FIRRTL memories (clocked reads, byte enables); fix_firrtl.py repairs what Yosys + emits that the FIRRTL compiler rejects (signed shifts, deep nesting, cat-LHS, …). + → fame5/vortex.fir (~384 MB at 64 cores NW=NT=16, ~4 min) + + + splice + annotate + Golden Gate + splice.py replaces the harness FIR's VX_firesim_wrap extmodule with the converted Vortex; + annotate_threads.py adds one FirrtlEnableModelMultiThreadingAnnotation per socket + instance (FAME5_THREADS=0 skips: same design, unthreaded, must match cycle-for-cycle). + Golden Gate then applies FAME-1 + FAME-5: one threaded copy, N-deep state. + → FireSim-generated.sv (Vortex inside; 229 MB at 64 cores, ~12 min / 36 GB heap) + + + Platform build — handed nothing but the generated RTL + No Vortex sources, no vendor IP, no pre-opt hook: nothing outside FireSim-generated.sv + is left for them to apply to. Same Vivado flow, FREQUENCY, STRATEGY. + + + Image — build32_aved_hw_fame5/bin/firesim.vbin + _fame5 directories throughout; VortexFame5Config keys the generated RTL apart + + RTL preconditions (all under `ifdef VX_CFG_FAME5): CORE_ID parameter → core_id/socket_id + ports so all sockets elaborate identically; ASYNC_BRAM_PATCH off; no VM, no graphics. + + + + + + + + + + + + + + Run either lane's image with the same command: FPGA_BIN_DIR=<bin> ./ci/blackbox.sh --driver=firesim --target=hw --app=… (FAME5=1 for a threaded build). + The Makefile guards the classic failure modes: SYNTHESIS missing from the flattened tree (every RAM lands in LUTRAM), VX_MEM_PORTS resolving to 0, a changed CONFIGS reusing a stale flatten. + diff --git a/docs/assets/img/firesim_fame5_threading.svg b/docs/assets/img/firesim_fame5_threading.svg new file mode 100644 index 0000000000..83836d4af9 --- /dev/null +++ b/docs/assets/img/firesim_fame5_threading.svg @@ -0,0 +1,135 @@ + + + + + + + + + + FAME-5 model multithreading — 64 sockets through one physical copy + Golden Gate threads instances of exactly one module; the annotation names it, the pass rewrites it. Nothing about Vortex's architecture changes — the same 64 sockets exist in target time, they just take turns using the silicon. + + + + Target view (unchanged) + 64 identical instances of VX_socket_top, + each with its own token channels: + + + g_sockets[0] : VX_socket_top + socket_id=0 · dcr/mem/kmu/gbar/busy channels + + g_sockets[1] : VX_socket_top + socket_id=1 + + g_sockets[2] : VX_socket_top + socket_id=2 + ⋮ + + g_sockets[63] : VX_socket_top + socket_id=63 + + Identical is a hard requirement: same module, + same parameters, same ports. CORE_ID was a + parameter (64 different modules); under + VX_CFG_FAME5 + it is the socket_id port, so all + 64 elaborate to one module. + + + + Static mux / demux + Inserted by Golden Gate at the + threaded copy's boundary. + A free-running thread counter + selects whose channels connect + this host cycle: + t = 0,1,2,…,63,0,… + one thread per host cycle, + fixed round-robin + + + + Physical view — VX_socket_top_threaded (built once) + + + All combinational logic, shared + Decode, issue, ALU/FPU lanes, LSU, cache control, DSPs — one copy serves + every socket in turn. This is where the 5.5× area win comes from: + 64 × 221K LUTs (unthreaded, would not fit) → 719K LUTs threaded + Logic that would be per-socket state machines is unchanged — it just + computes on whichever socket's state is selected. + + + Every register → 64-deep RAM + Read thread t's state, compute, + write it back — same host cycle, + so it maps to LUTRAM + (distributed, comb-read): + RAM64X1S ≈ 1 LUT / state bit + 311K LUTRAM in the 64-core image + + + Every SRAM → 64× deeper + Thread index becomes the top + address bits: + addr' = {t, addr} + Read latency 0 or 1 only; latency-1 + keeps them BRAM/URAM-mappable. + 1,679 BRAM36 + 97 URAM at 64 cores + + Not threaded (single copy, outside the sockets): the cluster's 64-way memory arbiter, KMU, DCR, global barrier, + VX_mem_to_axi, every FireSim bridge and FASED. A BlackBox inside the threaded module would be silently shared + across threads — wrong results, no diagnostic — which is why the RAM leaves are FIRRTL memories, not black boxes. + + + + + + + + + + + Cost: host cycles per target cycle + + One target cycle needs every thread to take its turn — at least 64 host cycles. A cross-thread combinational dependency (a socket's output feeding + another's input in the same target cycle) forces a second rotation, so the observed FMR lands between N and 2N plus bridge stalls: + + + + + t=0 + t=1 + t=2 + … + t=63 + arbiter/bridges + t=0 + … + + ←——————————— one target cycle ≈ 64–130 host cycles ———————————→ + + Measured, 64 cores NW=NT=16 on the V80 at 60 MHz: FMR 130–190 → 320–460 kHz of target time. + The unthreaded design would not fit at any speed; threading trades 64× the silicon for ~1/130 of the clock. + + Sanity anchor: FAME5_THREADS=0 builds the identical FIRRTL design unthreaded. It must — and does — agree with the threaded build + cycle for cycle (4-core demo: instrs=9776, cycles=15541 on baseline, threaded, and unthreaded-FIRRTL builds alike). + diff --git a/docs/assets/img/firesim_fpga_architecture.svg b/docs/assets/img/firesim_fpga_architecture.svg new file mode 100644 index 0000000000..84d3abb5a7 --- /dev/null +++ b/docs/assets/img/firesim_fpga_architecture.svg @@ -0,0 +1,180 @@ + + + + + + + + + + + + + A Vortex instance on the FPGA — V80 (AVED shell), FAME-5 build + Everything inside pblock_slash is FireSim-generated.sv, one reconfigurable module. Solid orange = memory data; solid grey = control (MMIO); dashed green = clocks. The U55C build is the same picture with the Vitis shell around it and HBM[0:7] as the memory binding. + + + + xcv80-lsva4737-2MHP-e-S + + + + AVED static region (in flash) + + + PCIe Gen5 ×8 + QDMA + BAR0 → AXI-Lite control network + (the HOST slave bridge is unused: + its reads never complete) + + + Base logic + uuid_rom · SMBus RPU · firewall · + clock/reset: programs the kernel clock + to the linked frequency — no MMCM, + FREQUENCY=60 → 60.000 MHz + + + dfx_decoupler + isolates the RM during reprogramming + + + HBM VNOC (NoC) + Routes the RM's one wide master to the + HBM stacks. The MEM segment maps + 32 GB at + 0x40_0000_0000 (HOST_MEM_SP=MEM) + covering the target's whole 32-bit map + + + HBM2e — 32 GB + Functional target memory: every byte the + GPU reads or writes lives here; FASED + only decides *when* answers return + + + + Reconfigurable partition pblock_slash — RM “slash_firesim” = FireSim-generated.sv + + + + AvedShim / SimulationMaster + Fingerprint 0x46697265 (“Fire”), transport gate at open; every FireSim MMIO register gets its own 16-byte AXI-Lite block + (the shell resolves AXI-Lite in 16 B units and LoadMem's R_DATA has read side effects), driver scales addresses ×4; + host-memory base check: the memory port must be bound at the elaborated base before any widget is touched + + + + PeekPoke bridge + The driver's hands: pokes + ctrl_dcr_req_*, ctrl_start + peeks + ctrl_busy, ctrl_dcr_rsp_* + step(n) = advance n target cycles + + + LoadMem widget + Bulk target-memory access from + the host, beside the timing model: + kernel images, buffers, readback. + Chunk unit = 32-bit words + + + Clock bridge + Generates the target clock token + stream; clockmodule counts + tcycle (target) / hcycle (host) + their ratio is FMR + + + FASED memory timing model + Latency-bandwidth pipe, per-run settings + readLatency=30 writeLatency=30 + maxReads=16 maxWrites=16 + functional half: ingress/egress queues, + reorder buffer, host-memory master + + + + FAMETop — the FAME-1 hub (SimWrapper) + Every wire between a bridge and the target is a token channel (ready/valid + payload). One target cycle fires exactly when every input + channel holds a token and every output channel has space; otherwise the target clock is gated and the design waits, cycle-accurately. + This is what host-decoupling means: the target cannot outrun the host, and the host cannot observe a half-advanced cycle. + Under FAME-5 the hub also carries one channel set per socket, multiplexed round-robin into the single threaded copy (next figure). + Clock gating on the V80 is a clock-enable conversion (FireSimFabricClockGate): the partition has no BUFGCE site. + + + + Target — VortexDUT ▸ VX_firesim_wrap ▸ Vortex + Control surface: DCR request/response + start/busy (poked by PeekPoke). Memory: one AXI4 port, 512-bit data, 32-bit address, 4-bit id. + + + Single-copy logic (built once) + KMU (CTA dispatch) · DCR bus · global barrier unit · + cluster: 64-way socket↔memory arbiter · VX_mem_to_axi + (tag buffer: 16 outstanding reads, 4-bit AXI id) + Everything here sees 64 sockets and does not know + they share one physical copy. + + + g_sockets — 64 × VX_socket_top (NW=NT=16 shown) + Each socket: 1 core + I$/D$/LMEM, socket_id as a port. + FAME5=1 → one physical copy, VX_socket_top_threaded: + registers become 64-deep LUTRAM, SRAMs 64× deeper, + one socket advances per host cycle, round-robin. + unthreaded: ~14.1M LUTs (5.5× the V80) — threaded: 719K LUTs, routed at 60 MHz + + Routed 64-core NW=NT=16 image, whole device: 974K LUTs (37.9%) · 565K FFs (11%) · 311K LUTRAM (24%) · 1,683 BRAM36 (45%) · + 105 URAM (5.5%) · 64 DSP · WNS +0.33 ns at 60 MHz + + + Control path: PCIe BAR0 → AXI-Lite → shim register blocks → widget MMIO. Memory data path: FASED / LoadMem host-memory master → HBM VNOC → HBM. + Host clock: free-running 60 MHz from the shell. Target clock: the host clock, enabled only on token-complete cycles — the target has no clock of its own. + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/img/firesim_memory_system.svg b/docs/assets/img/firesim_memory_system.svg new file mode 100644 index 0000000000..7e2d6ee5b0 --- /dev/null +++ b/docs/assets/img/firesim_memory_system.svg @@ -0,0 +1,134 @@ + + + + + + + + + + + + + The memory system — functional data in HBM, latency from a model + Nothing on this path is a DRAM: the “GPU memory” the kernel sees is the card's HBM behind a timing model that makes it look like whatever the +mm_ settings say. Determinism follows — congestion on the real HBM stalls target time instead of changing results. + + + + Vortex (target time) + + + Cores → L1 → (L2/L3 if enabled) + 64-core FAME-5 build: 64 sockets' misses + meet at the cluster arbiter + + + VX_mem_to_axi + Tag buffer: UUID-wide upstream tags → + 16 outstanding reads, 4-bit AXI id + (paired: PLATFORM_MEMORY_ID_WIDTH=4 ↔ idBits=4) + + + m_axi_mem_0 — one AXI4 port + 512-bit data · 32-bit addr · INCR bursts + NUM_BANKS=1: the wrapper presents one port, + so Vortex builds one + + + + FASED bridge (FireSim-generated) + + + Timing model — latency-bandwidth pipe + Runtime registers, set per run, no rebuild: + readLatency = 30 · writeLatency = 30 (target cycles) + readMaxReqs / writeMaxReqs ≤ maxReads/Writes = 16 + Set via VORTEX_FIRESIM_ARGS plusargs: + +mm_readLatency_0=100 +mm_writeLatency_0=100 + +mm_useHardwareDefaultRuntimeSettings_0 (covers the rest) + + + Functional model + Ingress/egress queues, per-id reorder buffer; reads and writes + go to host memory when accepted, responses are held until the + timing model's release cycle. Instrumented: + totalReads/Writes(Beats) · readMaxReqs · rrespError · … + printed by the driver on every stall report and watchdog + + + LoadMem widget — the host's side door + Kernel upload, buffer copy, readback: same host-memory master, + no target time passes. Chunk unit is 32-bit words. + + + + Shell binding (per card, from its descriptor) + + + V80 (aved) — hw/syn/xilinx/platforms/v80.toml + host_mem_sp = "MEM" → HBM VNOC, 32 GB window + base = 0x40_0000_0000 (WithHostMemBase) + The whole 32-bit target map fits the window; the AFU + rebases every access. HBM0 (1 GB) would truncate it — + the stack sits just under 4 GB. HOST (QDMA slave + bridge) never completes reads: refused by the descriptor. + + + U55C (vitis) + HOST_MEM_SP = HBM[0:7] → 8 × 512 MB = 4 GiB + base = 0x0 + A narrower binding silently serves only part of the map: + kernel uploads land outside it, reads return junk. + + + One host memory channel, 64-bit + Host-side AXI: beatBytes=8 at the host clock. This bounds + *simulation speed* under memory traffic, never target + timing: a starved FASED gates the target clock and FMR + rises; the kernel's cycle count is unchanged. + + + + + + + + + + + What the model does — and does not — capture + + The default pipe is a flat-latency, fixed-parallelism abstraction. Its ceiling is easy to compute and easy to hit: + peak read bandwidth ≈ maxReads × 64 B / readLatency = 16 × 64 / 30 ≈ 34 B per target cycle — for the whole GPU + demo at 64 cores (memory-bound, IPC 0.18) and sgemm 1024³ (14.7 FLOP/cycle against ~2000 available) both sit on this ceiling. Raising it means + raising maxReads (compile time, area) or the modelled latencies (runtime), not a faster card. + + There is no DRAM structure in the default model: no banks, rows, refresh, or contention. FASED has DDR3 FIFO/FR-FCFS models with those (compile-time + choice, WithDDR3FIFOMAS), and an optional tags-only LLC model; neither is enabled for Vortex today. HBM timing does not exist in FASED at all. + + Configuration knobs, by where they live: + image build: FREQUENCY=60 STRATEGY=TIMING DEVICE=<part> HOST_MEM_SP=<tag> (hw/syn/firesim → platform build) + elaboration: NastiParameters(dataBits=512, addrBits=32, idBits=4) · BaseParams(maxReads/maxWrites=16) · platform toml [firesim] + per run: VORTEX_FIRESIM_ARGS="+mm_…_0=…" — the only knob that needs no rebuild + + Address-width warning: the wrapper takes its AXI width from VX_CFG_MEM_ADDR_WIDTH (48 at XLEN=64) while the Scala side pins addrBits=32; + a disagreement is padded or truncated with no diagnostic. XLEN=64 on FireSim needs both sides changed together. + idBits is bounded both ways: 1<<idBits wraps Scala's Int at 32 (silently wrong responses); per-id state made idBits=8 cost 751K LUTs on a U55C. 4 matches what Vortex drives. + diff --git a/docs/assets/img/firesim_system_stack.svg b/docs/assets/img/firesim_system_stack.svg new file mode 100644 index 0000000000..9649df93e2 --- /dev/null +++ b/docs/assets/img/firesim_system_stack.svg @@ -0,0 +1,124 @@ + + + + + + + + + + FireSim + Vortex — software and hardware stack + One stack, two vehicles: the card (left of the red cut: PCIe) and the in-process metasimulator (right: the same design, Verilated). Every layer above the cut is identical in both. + + + + Host process (one per run) + + + Application — tests/regression/<app> + vx_dev_open / vx_mem_alloc / vx_copy_to_dev / vx_start / vx_ready_wait, linked against libvortex.so + + + Runtime HAL — sw/runtime/firesim/vortex.cpp → libvortex-firesim.so + Driver ABI + software command processor (cmd_processor.cpp); kernel upload, DCR writes, launch, flush + VRT_VBIN_PATH / XCLBIN_PATH · VORTEX_FIRESIM_ARGS → extra plusargs (+mm_…) + + + Transport — sim/firesim/firesim_sim.cpp (in libfiresim.so) + Owns the simulator thread; every bridge call is a labelled request marshalled onto it. + Adaptive step batching, DCR single-step polling, launch watchdog, stall reports + + + FireSim driver core — $(FIRESIM_PATH)/sim/midas/src/main/cc + Widget registry from FireSim-generated.const.h: + peek_poke_t · loadmem_t · clockmodule_t · FASEDMemoryTimingModel (driver half) + simif_t: MMIO read/write + bulk memory, one implementation per vehicle + + + simif_aved (VRT) + V80: vrtd programs the PDI, + MMIO over PCIe BAR + + + simif_vitis (XRT) + U55C: xclbin load, + MMIO + DMA through XRT + + + + Metasimulator (TARGET=xrtsim / avedsim) — no card, no image + + + Same application · same runtime HAL · same transport + Built with -DFIRESIM_EMUL; libfiresim.so is the Verilated model itself + + + simif_emul — the emulated host interface + MMIO and DMA become DPI calls into the Verilated top (top.sv / emul); + firesim_emul_main.cpp runs Verilator's event loop on the simulator thread. + VORTEX_FIRESIM_EMUL_ARGS → +waveformfile=… +max-cycles=… (EMUL_DEBUG=1 build) + + + Vemul — the whole FPGA payload, Verilated + FireSim-generated.sv + (unthreaded builds) the flattened Vortex tree, + bit-identical to what the card synthesizes. Same tokens, same schedule: + what passes here passes on silicon, cycle for cycle. + FAME5=1: Vortex is already inside FireSim-generated.sv (converted to + FIRRTL and threaded), so Verilator is handed no Vortex sources at all. + Build: sw/runtime/firesim → sim/firesim/Makefile (EMUL_DIR per card, + _fame5 and _dbg variants); run: blackbox.sh --driver=firesim + --target=avedsim|xrtsim [FAME5=1] [EMUL_DEBUG=1] + + + + PCIe — MMIO (control) + memory writes; everything below is on the FPGA + + + + FPGA (V80 shown; U55C differs only in the shell) + + + Shell — AVED static region + AvedShim / SimulationMaster + PCIe endpoint, AXI-Lite control (16 B blocks), host clock, HBM NoC — see the FPGA architecture figure + + + FPGATop — FireSim's generated harness (FireSim-generated.sv) + Bridges: PeekPoke (ctrl_* ports) · LoadMem (bulk memory) · ClockBridge · FASED (memory timing) + Token channels host-decouple everything inside; the target advances only when stepped. + + + Target — VortexDUT ▸ VX_firesim_wrap ▸ Vortex + The design under test: DCR + start/busy control surface, one 512-bit AXI4 memory port. + FAME5=1: the sockets inside are time-multiplexed through one physical copy. + + + + + + + + + + + + + + diff --git a/docs/assets/img/firesim_timing_model.svg b/docs/assets/img/firesim_timing_model.svg new file mode 100644 index 0000000000..39da862a54 --- /dev/null +++ b/docs/assets/img/firesim_timing_model.svg @@ -0,0 +1,123 @@ + + + + + + + + + + How simulation timing is modelled + Three layers, strictly separated: token exchange decides *when a target cycle may happen*; timing models decide *what latency the target observes*; the driver decides *how many cycles to ask for*. Wall-clock speed never leaks into target time. + + + + 1 — Host decoupling (FAME-1): the target clock is a resource, not a fact + + + PeekPoke + tokens on demand (step) + + FASED + one token per mem beat + + Clock bridge + the cycle budget itself + + + + + + + + + + + + token channels (ready/valid queues) + + + FAMETop hub + target + Fires one target cycle when EVERY input + channel holds a token and every output + has space. Otherwise the clock enable + stays low: the target is frozen mid-state, + not observing time. ○ = the missing token + that is gating the cycle right now. + + + Why this is exact + The target cannot advance past a bridge that has not answered, and a bridge + cannot answer early: however slow the host is — PCIe hiccups, a paused driver, + Verilator instead of silicon — the target sees the identical cycle sequence. + Metasim and FPGA runs agree cycle-for-cycle for this reason. + Two clocks, never conflated: + tcycle — target cycles advanced (clockmodule; Vortex PERF: cycles) + hcycle — host cycles spent simulating them + FMR = hcycle/tcycle; target freq = host_freq / FMR + 64-core FAME-5 on the V80: 60 MHz / FMR 130–190 ≈ 320–460 kHz of target time + + + + 2 — Observed latency: FASED's timing model, in target cycles + + The memory the GPU sees is split in two. The functional half reads/writes real HBM through the host memory port. The timing half is a latency-bandwidth + pipe that schedules, in target time, when each response is released. If HBM is slow, target time simply stops until the data is there — the GPU never notices. + + + target time → + + + T: AR accepted + + T+30: R data released + + readLatency = 30 target cycles (runtime register, +mm_readLatency_0=…) + + ≤16 reads in flight + + Bandwidth ceiling of this model: 16 outstanding × 64 B / 30 cycles ≈ 34 B per target cycle for the whole GPU — sgemm at 64 cores runs into exactly this. + Counters (totalReads, readMaxReqs, …) are read from the model over MMIO and printed by the driver on every stall report. + + + + 3 — The driver's schedule: adaptive step batching (firesim_sim.cpp) + + The target only moves when stepped, and each poll costs a thread handoff plus an MMIO round trip. The driver grows the batch ×4 from a floor of 16 target + cycles up to a ceiling (default 100,000; VORTEX_FIRESIM_STEP_BATCH), and drops back to the floor whenever the target goes idle: + + + + 16 + 64 + 256 + 1024 + 4096 + …×4… + 100000 (ceiling) + busy sampled after each batch + + + Consequences worth knowing: + • “Target Cycles Emulated” is quantized to this schedule — Vortex's own PERF: cycles counter is the cycle authority. + • A short kernel is only *observed* to finish at the next poll; the floor of 16 exists so a launch's busy edge is never stepped over. + • DCR reads poll one cycle at a time (dcr_rsp_valid is a one-cycle pulse), bounded by VORTEX_FIRESIM_DCR_POLL_ROUNDS — slow, but exact. + diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index 39958e0494..a86e733393 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -1,49 +1,111 @@ # FireSim integration — host-decoupled FPGA simulation of Vortex This document describes how Vortex runs as a [FireSim](https://fires.im/) -target on a Xilinx Alveo U55C. It covers three things: - -1. The **execution model** — what host-decoupled simulation means for a - driver, and why the runtime *steps* the target rather than waiting on it. -2. The **integration architecture** — the Chisel target, the RTL wrapper, the - transport layer, and how a kernel launch reaches the card. -3. The **build flow** — elaboration through place-and-route, and the - invariants that make the difference between a design that fits and one - that silently does not. - -FireSim itself is an external dependency, developed in a clone of -[`vortexgpgpu/firesim`](https://github.com/vortexgpgpu/firesim) (branch -`vortex_3.x`, based on upstream 1.21.0) at `~/dev/firesim`, and consumed by -the Vortex build through `$(FIRESIM_PATH)`, which `config.mk` defaults to -`$(TOOLDIR)/firesim` — the same source/tool split POCL, Mesa and LLVM use. -Vortex's own changes to FireSim are small bug fixes and platform plumbing -rather than divergence, pinned by `FIRESIM_REV` in `VERSION`; see §6. +target: a multi-core GPU simulated on an FPGA at cycle exactness, on the +Alveo U55C (Vitis/XRT) and the Alveo V80 (AVED/VRT). It answers, in order: +| § | question | +|---|---| +| [1](#1-system-architecture) | What is on the FPGA, what is on the host, and how do they meet? | +| [2](#2-how-vortex-rtl-is-synthesized) | How is the Vortex RTL synthesized — with and without FAME-5? | +| [3](#3-fame-5-many-sockets-one-physical-copy) | Which modules are time-multiplexed ("replicated") and how does that work? | +| [4](#4-how-simulation-timing-is-modelled) | How is simulation timing modelled? | +| [5](#5-the-memory-system) | How is memory modelled and configured, and how is the shell configured? | +| [6](#6-observability) | How do you get wave traces, `$display`, `$error`, asserts, counters? | +| [7](#7-the-transport-layer) | What does the driver actually do, and why? | +| [8](#8-running) | How do you run it? | +| [9](#9-measured-results) | What has been measured? | +| [10](#10-relationship-to-upstream-firesim) | How does the FireSim fork relate to upstream? | +| [11](#11-design-invariants) | What must never be broken? | +| [12](#12-known-limitations) | What doesn't work (yet)? | + +FireSim itself is an external dependency: the +[`vortexgpgpu/firesim`](https://github.com/vortexgpgpu/firesim) fork, consumed +through `$(FIRESIM_PATH)` (default `$(TOOLDIR)/firesim`) and pinned by +`FIRESIM_REV` in `VERSION` — see [§10](#10-relationship-to-upstream-firesim). For running kernels on hardware generally, see -[`fpga_setup.md`](../fpga_setup.md); for the other simulation vehicles, see +[`fpga_setup.md`](../fpga_setup.md); for the other simulation vehicles, [`simulation.md`](../simulation.md). --- -## 1. Execution model - -### 1.1 What "host-decoupled" means - -FireSim does not emulate Vortex at wall-clock speed and then let the host -observe it. It compiles the design through **Golden Gate** (MIDAS) into a -FAME-1 transformed model in which *every* target clock edge is explicitly -scheduled by the host. The FPGA holds the transformed design; the host driver -decides how many target cycles to advance and when. - -The practical consequence, and the thing that shapes the entire driver: **the -target does not run unless the host tells it to.** There is no free-running -clock to poll. A driver that issues an MMIO read and waits will wait forever, -because nothing advanced the design between the request and the read. - -This is the single most important difference from the XRT flow, where the -kernel clock free-runs and the host merely observes. - -### 1.2 Target cycles vs. host cycles +## 1. System architecture + +![FireSim + Vortex system stack](../assets/img/firesim_system_stack.svg) + +### 1.1 The layering + +Everything above the PCIe boundary except the FireSim driver core lives in +this repository — deliberately, so the FireSim checkout stays close to +upstream. From the application down: + +- **Application** (`tests/regression/`): the ordinary `vx_*` API; it + cannot tell FireSim from any other backend. +- **Runtime HAL** (`sw/runtime/firesim/vortex.cpp` → + `libvortex-firesim.so`): the driver ABI plus the software command + processor; kernel upload, DCR configuration, launch, flush. +- **Transport** (`sim/firesim/firesim_sim.cpp`, in `libfiresim.so`): owns the + simulator thread and marshals every operation onto it; stepping policy, + watchdogs, diagnostics ([§7](#7-the-transport-layer)). +- **FireSim driver core** (`$(FIRESIM_PATH)/sim/midas/src/main/cc`): the + widgets generated alongside the hardware — `peek_poke_t`, `loadmem_t`, + `clockmodule_t`, FASED's driver half — over a `simif_t` that is + `simif_aved` (VRT) on the V80, `simif_vitis` (XRT) on the U55C, and + `simif_emul` in the metasimulator. +- **FPGA payload** (`FireSim-generated.sv`): the shell shim, FireSim's + bridges, the FAME hub, and the target — Vortex behind + `hw/rtl/afu/firesim/VX_firesim_wrap.sv`. + +The **metasimulator** (`TARGET=xrtsim`/`avedsim`) is the same stack with the +bottom swapped: `libfiresim.so` *is* the Verilated FPGA payload, reached +through `simif_emul` over DPI, and needs no card, image, XRT or VRT. It runs +the same tokens on the same generated design, which is why it agrees with +silicon cycle for cycle and is the step before every bitstream. + +### 1.2 What is on the FPGA + +![A Vortex instance + FireSim shell on the FPGA](../assets/img/firesim_fpga_architecture.svg) + +On the V80, the AVED static region (permanently in flash) carries the PCIe +endpoint, the AXI-Lite control network, the clock/reset logic and the HBM +NoC. The reconfigurable partition holds one module — `FireSim-generated.sv` — +containing: + +- **`AvedShim` / SimulationMaster**: the control-plane adapter. The compute + shell resolves AXI-Lite in 16-byte blocks and FireSim registers have read + side effects (LoadMem's `R_DATA` dequeues), so every register gets its own + block and the shim answers the padding words itself; the driver scales + addresses by 4. At open, the driver checks the `0x46697265` ("Fire") + fingerprint, writes and reads back a probe, and verifies the memory port is + bound at the elaborated base before any widget is touched. +- **Bridges** (the host-visible half of the simulation): PeekPoke (the + target's DCR/start/busy surface), LoadMem (bulk target memory), the clock + bridge (`tcycle`/`hcycle`), and FASED (the memory model, + [§5](#5-the-memory-system)). +- **The FAME hub** (`FAMETop`): token channels between the bridges and the + target; the mechanism of host decoupling ([§4](#4-how-simulation-timing-is-modelled)). +- **The target**: `VortexDUT` ▸ `VX_firesim_wrap` ▸ Vortex. + +Clocking on the V80 has one platform quirk worth knowing: FireSim gates the +target clock, the partition has no `BUFGCE` site, and a fabric clock gate is +not safe. `AvedConfig` therefore selects `FireSimFabricClockGate` — a plain +AND marked `gated_clock` that synthesis's gated-clock conversion moves onto +the clock-enable pin of every register, RAM and DSP it drives. `BUFGCE` +semantics, no derived clock; a pre-opt check fails the build if any kernel +clock is still driven by logic. (This is also why vendor OOC IP cannot sit +inside the kernel on aved: the conversion cannot enter it, so the RTU uses +the soft FMA/divider there — `VX_CFG_RTU_USE_DSP=0`.) + +### 1.3 Host-decoupled execution + +FireSim does not emulate Vortex at wall-clock speed and let the host observe +it. Golden Gate (MIDAS) compiles the design into a FAME-1 transformed model +in which *every* target clock edge is explicitly scheduled. The practical +consequence, and the thing that shapes the entire driver: **the target does +not run unless the host tells it to.** There is no free-running clock to +poll; a driver that issues an MMIO read and waits, waits forever. This is the +single most important difference from the XRT flow, where the kernel clock +free-runs and the host merely observes. Two clocks matter and must not be conflated: @@ -52,432 +114,617 @@ Two clocks matter and must not be conflated: | **target cycles** | cycles the Vortex design advanced | `clockmodule_t::tcycle()`, and Vortex's own `PERF: cycles=` | | **host cycles** | cycles the FPGA spent simulating them | `clockmodule_t::hcycle()` | -Their ratio is **FMR** (FPGA-to-Model Ratio) — how many host cycles it costs -to simulate one target cycle. FMR is the honest measure of simulation -efficiency; the effective target frequency is `host_freq / FMR`. - -**Do not quote FireSim's `Target Cycles Emulated` as a kernel cycle count.** -It reports cycles the *driver stepped*, not cycles the kernel took, and the -driver steps in growing batches (§3.3). A kernel is only observed to have -finished at the first poll *after* it does, so the figure is quantized to the -batch schedule and can overstate a short kernel by up to 4×. Two workloads -that finish inside the same inter-poll interval report the identical number. -Vortex's `PERF: cycles=` counter is exact and is what every measurement in -this repository uses. +Their ratio is **FMR** (FPGA-to-Model Ratio); the effective target frequency +is `host_freq / FMR`. **Do not quote FireSim's `Target Cycles Emulated` as a +kernel cycle count** — it reports cycles the *driver stepped*, quantized by +the batching schedule ([§4.3](#43-the-drivers-schedule)), and can overstate a +short kernel by up to 4×. Vortex's `PERF: cycles=` counter is exact and is +what every measurement in this repository uses. --- -## 2. Integration architecture +## 2. How Vortex RTL is synthesized -### 2.1 Layering +![Build flow](../assets/img/firesim_build_flow.svg) -``` - tests/regression/ application - │ vx_* API - sw/runtime/firesim/vortex.cpp runtime HAL (driver ABI, XCLBIN_PATH) - │ firesim_sim class - sim/firesim/firesim_sim.cpp transport (request queue, stepping) - │ peek_poke / loadmem bridges - $(FIRESIM_PATH)/sim/… FireSim driver (simif_vitis, widgets) - │ XRT - ─────────────────────────────── PCIe ─────────────────────────────── - FireSim-generated.sv FAME-1 model (Golden Gate) - │ ctrl_* / AXI - hw/rtl/afu/firesim/VX_firesim_wrap.sv AFU wrapper - │ - Vortex the design under test -``` +There are two build lanes. In the **default (FAME-1) lane** Vortex stays +Verilog: Golden Gate sees it as an opaque BlackBox, transforms only its +boundary, and the card's platform flow synthesizes the Verilog directly. In +the **FAME-5 lane** (`FAME5=1`) Vortex is converted to FIRRTL so Golden Gate +can transform *inside* it — that is what enables the socket +time-multiplexing of [§3](#3-fame-5-many-sockets-one-physical-copy). Both +are driven from `hw/syn/firesim/Makefile` with the same tokens as the native +flows: -Everything above the PCIe line except the FireSim driver lives in this -repository; that is deliberate, so the FireSim checkout stays close to -upstream. - -### 2.2 The Chisel target — `hw/syn/firesim/src/main/scala/VortexTarget.scala` +```bash +source ci/firesim_env.sh # after the card's Xilinx environment +# U55C (XRT): TARGET=hw | hw_emu +make -C hw/syn/firesim PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 TARGET=hw +# V80 (AVED): TARGET=hw | sim +make -C hw/syn/firesim TARGET=hw PERF=1 CONFIGS="-DVX_CFG_NUM_WARPS=16" +# V80, 64 cores through one physical socket: +make -C hw/syn/firesim FAME5=1 TARGET=hw CONFIGS="-DVX_CFG_NUM_CORES=64 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_THREADS=16" +``` -`VortexDUT` instantiates Vortex as an opaque Verilog **BlackBox**. Golden Gate -cannot see inside it, which is the point: the FAME transform channelizes the -BlackBox's boundary signals and leaves its internals alone. +The image lands in `__[_fame5]/bin` as +`firesim.vbin` or `firesim.xclbin`, beside `firesim.card`, which names the +card for `run-firesim`. `PREFIX`, `XLEN`, `CONFIGS`, `PERF=1` work as in the +other synthesis flows; extensions pull their sources in through +`hw/syn/extensions.mk`. Nothing given on this command line reaches FireSim's +own makes (`MAKEOVERRIDES` is cleared): FireSim uses `PLATFORM`, `TARGET` and +`BUILD_DIR` for its own purposes, and each sub-make is handed exactly what it +needs. -`VortexConfig` sets the memory interface the target presents: +### 2.1 The flattened trees -```scala -NastiParameters(dataBits = 512, addrBits = 32, idBits = 4) -``` +Golden Gate and Vivado take a file list, not include paths, so +`gen_sources.sh -P` preprocesses `hw/rtl` into one self-contained directory +with the macros already inlined, top `VX_firesim_wrap`. Two flattenings +exist because simulation and synthesis need opposite defines: -`idBits` is **paired with `PLATFORM_MEMORY_ID_WIDTH` in -`hw/syn/firesim/Makefile`** and neither may be changed alone — one shapes the -IO the BlackBox is instantiated with, the other shapes the module's own ports, -and Verilog pads or truncates a disagreement with no diagnostic. - -It is also bounded from both sides, which is not obvious: - -- **Too large and it overflows.** FASED sizes its read-response egress with - `1 << idBits` in Scala `Int` arithmetic. At 32 that shift wraps to 1, which - both selects the untranslated egress branch and gives it a single queue for - every AXI id. Responses then leave carrying the requesting transaction's id - and another transaction's data. Elaboration succeeds and nothing warns — the - generated design is simply smaller and wrong. -- **Too large and it does not fit.** The response deinterleaver and the id-yank - queues carry per-id state, so model area scales with the id space. At 8 that - is 750,955 LUTs as memory against the 600,960 a U55C has. At 4 it is 51,678. - -4 is what Vortex actually drives: `VX_mem_to_axi`'s tag buffer is engaged -because `UUID_WIDTH` puts the upstream tag past any AXI id width, so reads -carry `CLOG2(TAG_BUFFER_SIZE) = 4` bits. It also matches FASED's `maxReads` -of 16. - -### 2.3 The RTL wrapper — `hw/rtl/afu/firesim/VX_firesim_wrap.sv` - -A thin AFU presenting Vortex as: - -- one AXI4 memory port (`m_axi_mem_0_*`), -- a DCR request/response pair for configuration, -- `start` / `busy` for launch and completion. - -`VX_CFG_PLATFORM_MEMORY_NUM_BANKS=1` is set for this wrapper. Left at the -default of 2, Vortex builds two internal memory ports and `VX_mem_to_axi` -arbitrates them onto the one bank, recovering the originating port from the -*returned* AXI id (`rsp_xbar_sel_in = m_axi_rid[0 +: NUM_PORTS_IN_BITS]`). -That makes correctness depend on the memory model echoing `rid` bit-exact — an -assumption real AXI satisfies but a timing model need not. One port removes the -dependency entirely and matches what the wrapper presents. - -### 2.4 Memory model - -Target DRAM is served by **FASED**, FireSim's parameterized memory timing -model, backed by host DRAM through the LoadMem widget. The Vitis platform -binds `host_mem_0` across HBM[0:7] with a 4 GiB window — a 32-bit target spans -4 GB, with its program image at `0x80000000` and its stack just under 4 GB, so -a narrower binding puts kernel uploads outside the served window and the target -fetches memory nothing ever wrote, with no error anywhere. +| flattening | defines | consumed by | +|---|---|---| +| `vivado_srcs/` | `-DSYNTHESIS -DVIVADO -DNDEBUG` (+ `-DSIMULATION` for `TARGET=sim`/`hw_emu`; never under FAME5) | platform synthesis; sv2v under FAME5 | +| `vortex_srcs/` | `-DNDEBUG` only | the Verilator metasimulator | + +The invariants here have each cost hours when broken: + +- **`-DSYNTHESIS` is not optional.** `VX_dp_ram`/`VX_sp_ram` gate all + block-RAM inference behind it, and because the tree is *preprocessed*, the + BRAM branch is not inactive without it — it is deleted. Every memory then + lands in distributed RAM and place-and-route fails hours later at ~750K + LUTs-as-memory with nothing pointing at the cause. +- **Verilator gets a library search, not a flat list.** Handing it every + file makes it stop treating the macro-computed width parameters as + constants; only the packages (in dependency order) and the wrapper are + named, the rest resolves via `-y`. +- **A changed `CONFIGS` must invalidate the flatten.** Defines arrive on the + command line, which no file timestamp reflects; a stamp file rewritten only + when the flags differ hangs the dependency. +- **Only overrides and type selectors are passed, never the resolved + config.** Passing the whole resolved `XCONFIGS` replaces `VX_config.vh`'s + computations with report values, which sets the memory port count to zero; + both flatten rules grep the result for `VX_MEM_PORTS = 0` and fail fast. + +### 2.2 The default lane: Vortex as a BlackBox + +`hw/syn/firesim/src/main/scala/VortexTarget.scala` instantiates Vortex as a +Chisel BlackBox. Golden Gate cannot see inside it, which is the point: the +FAME-1 transform channelizes the boundary signals — the AXI memory port, the +DCR/start/busy control surface — and leaves the internals alone. `image` +then runs, in order: `vivado-sources`, `xilinx-ips` (the `xil_fma`/`xil_fdiv` +floating-point cores for the card's part), `stage` (copy the Scala targets, +harnesses and the generated platform config into the fork — only what +changed, since FireSim's make judges elaboration by these timestamps), +`replace-rtl` (Golden Gate), and the platform's own build (`v++` on vitis, +slashkit on aved), handed: + +- `VORTEX_VIVADO_SRCS` — the flattened tree, packaged into the kernel; +- `VORTEX_XILINX_IP_DIR` — the vendor cores, copied in fresh directories + (Tcl's `file copy` onto an existing directory nests instead of replacing, + which silently reuses the previous run's cores); +- `VORTEX_PRE_OPT_TCL` — the async-BRAM patch hook. `VX_async_ram_patch` + leaves `VX_placeholder` black boxes wherever an asynchronous-read RAM must + live in block RAM; without the hook, `opt_design` aborts with `DRC INBB-3` + — but only once a configuration is large enough to push those RAMs out of + LUTRAM, so small builds pass and large ones do not; +- `FREQUENCY` (default 60 MHz) and `STRATEGY` (default `TIMING`). + +### 2.3 The FAME-5 lane: Vortex as FIRRTL + +Golden Gate's threading transform works on FIRRTL, so `FAME5=1` converts the +whole flattened tree (`hw/syn/firesim/fame5/`): + +1. **sv2v** lowers the SystemVerilog to Verilog-2005. sv2v inlines every + module with interface ports into its parent — which would erase the + boundary of the unit to be threaded. `VX_socket_top` + (`hw/rtl/VX_socket_top.sv`) exists for exactly this reason: a + parameterless wrapper around `VX_socket` whose ports are plain wires, so + it survives conversion as one module. +2. **`to_firrtl.sh` (Yosys)** flattens everything except `VX_socket_top` and + writes FIRRTL. `VX_dp_ram`/`VX_sp_ram` are kept as black boxes that + `gen_rams.py` regenerates as native FIRRTL memories — Yosys cannot express + clocked read ports or byte enables, and a real BlackBox inside the + threaded module would be silently shared across threads + ([§3](#3-fame-5-many-sockets-one-physical-copy)). The script also carries + the Yosys workarounds this conversion needed: `proc -norom`, + `memory_nordff` for small clocked-read arrays, `opt -nosdff -nodffe` plus + `dffunmap`, and `splitnets` against word-level false loops. +3. **`fix_firrtl.py`** repairs what Yosys emits that the FIRRTL compiler or + Verilator reject: signed-shift lowering, 2¹⁹-bit `dshl` intermediates, + `cat()` on the left-hand side, signed zero tests, and expression nesting + thousands of levels deep (hoisted into nodes — the FIRRTL parser recurses + per level). It runs in one linear pass; a 64-core conversion has lines + over a million characters long. +4. **`splice.py`** replaces the `VX_firesim_wrap` extmodule in Golden Gate's + own elaborated circuit with the converted Vortex, and + **`annotate_threads.py`** appends one + `FirrtlEnableModelMultiThreadingAnnotation` per socket instance + (`FAME5_THREADS=0` skips this: the identical design, unthreaded — the + cycle-exactness anchor). +5. **Golden Gate** then applies FAME-1 and FAME-5 together and emits a + `FireSim-generated.sv` that *contains* Vortex. The platform build is + handed no Vortex sources, no vendor IP and no pre-opt hook — there is + nothing left outside the generated file for them to apply to. + +The RTL preconditions, all under `` `ifdef VX_CFG_FAME5 ``: per-core +`CORE_ID` parameters become `core_id`/`socket_id` *ports* (a parameter would +make 64 different modules; threading needs one), `ASYNC_BRAM_PATCH` is +disabled (its placeholders only a Vivado hook resolves), and `VX_socket_top` +statically rejects VM and graphics configurations it does not carry ports +for. + +At 64 cores NW=NT=16: conversion ≈ 4 min (384 MB of FIRRTL), Golden Gate +≈ 12 min at ≈ 36 GB heap (`JAVA_TOOL_OPTIONS` raises `-Xmx` and, for the +parser's recursion, `-Xss`), generated Verilog 229 MB. --- -## 3. The transport layer — `sim/firesim/firesim_sim.cpp` +## 3. FAME-5: many sockets, one physical copy + +![FAME-5 threading](../assets/img/firesim_fame5_threading.svg) + +The 64-core question is an area question. One NW=NT=16 core synthesizes to +~221K LUTs; 64 of them need ~14.1M against the V80's 2.57M. FAME-5 ("model +multithreading") removes the duplication instead of shrinking the core: all +64 sockets **take turns using one physical copy of the socket logic**, one +socket per host cycle, round-robin. + +**What is threaded.** Exactly one module type per design (a Golden Gate +restriction), chosen by annotation — there is no automatic detection, with +or without Chisel. Vortex threads `VX_socket_top`: one core plus its I$, D$ +and local memory. Everything outside it — the cluster's socket↔memory +arbiter, KMU, DCR, global barrier, `VX_mem_to_axi`, all bridges and FASED — +is built once and does not know the sockets share silicon. + +**What the transform does.** The target-visible design is unchanged: 64 +instances, each with its own token channels. Golden Gate replaces them with +one `VX_socket_top_threaded` plus a static mux/demux driven by a free-running +thread counter, and transforms the state: + +- **every register becomes an N-deep, combinationally-read memory** (read + thread *t*'s state, compute, write back — same host cycle), which Vivado + maps to LUTRAM at ~1 LUT per state bit; +- **every SRAM becomes N× deeper**, the thread index forming the top address + bits; read latency 0 or 1 only, and latency-1 keeps BRAM/URAM mapping. + +**The requirements that shaped the RTL work:** + +- instances must be *identical* — same module, same parameters. Hence + `core_id`/`socket_id` as ports and the parameterless wrapper; +- **no BlackBox inside the threaded module.** It would be instantiated once + and silently shared by all threads: wrong results, no diagnostic (proven + on a prototype). Hence the FIRRTL regeneration of the RAM leaves; +- a module with a single instance is skipped, and only one module type can + be threaded. + +**The cost.** One target cycle takes at least N host cycles (the rotation), +up to ~2N when a cross-thread combinational dependency forces a second pass, +plus bridge stalls: the 64-core V80 build measures FMR 130–190, i.e. +320–460 kHz of target time at the 60 MHz host clock. Per-thread state is +initialized during host reset, so target reset behaves identically in every +thread. + +**The correctness anchor.** `FAME5_THREADS=0` builds the same +FIRRTL-converted design with no threading. The three builds — native +Verilog baseline, unthreaded FIRRTL, threaded — agree exactly (4-core demo: +`instrs=9776, cycles=15541` in all three). -The largest single piece of the integration, and where host-decoupling shows -up as real complexity. - -### 3.1 Threading +--- -FireSim's bridges are **not thread-safe** and must only be touched by the -thread running the simulation loop. The transport therefore owns a simulator -thread and marshals every operation onto it as a closure: +## 4. How simulation timing is modelled -``` -caller ──push labelled_request_t──> queue ──> simulator thread ──> bridges - <──────── condition variable ──────── -``` +![Timing model](../assets/img/firesim_timing_model.svg) -Each request carries a label so a trace attributes a stall to an operation -rather than showing a column of identical entries. +Three layers, strictly separated: -### 3.2 Memory access — the chunk-unit invariant +### 4.1 Token exchange decides *when a cycle may happen* -`access_mem` moves data through LoadMem one host beat at a time, with -read-modify-write on unaligned edges. +Every wire between a bridge and the target is a token channel. The FAME hub +fires one target cycle exactly when every input channel holds a token and +every output channel has space; otherwise the target clock stays gated and +the design is frozen mid-state, not observing time. However slow the host +is — PCIe latency, a paused driver, Verilator instead of silicon — the target +sees the identical cycle sequence. This is why metasim and FPGA agree cycle +for cycle, and why wall-clock speed never contaminates a measurement. -**LoadMem's chunk is counted in 32-bit words, not bytes.** `read_mem` and -`write_mem` push exactly that many *words* through the data register. Treating -it as a byte count uploads a quarter of each beat and advances the address by a -quarter of a beat — storing two live bytes per eight and dropping every write -that does not land on a beat boundary. A kernel image uploaded that way is -fetched as a mixture of written and stale memory, and nothing reports an error. +### 4.2 Timing models decide *what latency the target observes* -Whole-beat writes bypass the read-modify-write path. +Latency the GPU experiences is not the card's memory latency — it is whatever +the FASED timing model schedules, in target cycles ([§5](#5-the-memory-system)). +A memory response that takes 400 host cycles to fetch from HBM still arrives, +in target time, exactly `readLatency` cycles after its request; the missing +host cycles simply never existed for the target. -### 3.3 Advancing the target — adaptive step batching +### 4.3 The driver's schedule -`is_busy()` must both catch the launch edge and give a long kernel throughput. -These pull in opposite directions: a small step never misses `busy` asserting -but costs an MMIO round trip per few cycles; a large step is efficient but can -step past a short kernel entirely. +The driver decides *how many cycles to ask for*. `is_busy()` must both catch +the launch edge (busy is a level; a batch stepped past a short kernel would +wait forever on a rise that already happened) and give long kernels +throughput (each poll costs a thread handoff plus an MMIO round trip). The +batch therefore grows ×4 from a floor of 16 target cycles to a ceiling +(default 100,000, `VORTEX_FIRESIM_STEP_BATCH`), dropping back to the floor +when the target goes idle. This schedule is what quantizes +`Target Cycles Emulated` (§1.3). DCR reads are the exception: `dcr_rsp_valid` +is a one-cycle pulse, so the poll advances one target cycle at a time, +bounded by `VORTEX_FIRESIM_DCR_POLL_ROUNDS` ([§7](#7-the-transport-layer)). -The driver grows the batch: floor of 16 target cycles, ×4 per poll up to a -ceiling, dropping back to the floor when the target goes idle. Cumulative poll -boundaries therefore land at 16, 80, 336, 1360, 5456, 21840, … +--- -This is also the origin of the `Target Cycles Emulated` quantization in §1.2 — -the counter is sampled at the poll that *observes* completion, not at the -busy→idle transition itself. +## 5. The memory system -### 3.4 DCR reads must single-step +![Memory system](../assets/img/firesim_memory_system.svg) -`dcr_rsp_valid` is a one-cycle pulse passed straight out of the core, so a -batched step lands past it and the read reports unanswered on a target that -replied. `dcr_read` therefore advances one target cycle per iteration, and its -bound is expressed in cycles rather than polls. +### 5.1 The path -The cost is one MMIO round trip per target cycle — tenths of a second for a -cache flush on the card, minutes under emulation. Batched polling would need -the response **latched at the wrapper**, which is a target-side change and a -new bitstream; it is a known follow-up rather than a driver-side workaround. +Vortex presents **one AXI4 memory port** (512-bit data, 32-bit address, +4-bit id, INCR bursts at full width). `VX_firesim_wrap` is built with +`VX_CFG_PLATFORM_MEMORY_NUM_BANKS=1` so Vortex builds exactly one internal +port: at the default of two, `VX_mem_to_axi` recovers the originating port +from the *returned* AXI id, which makes correctness depend on the memory +model echoing `rid` bit-exact — an assumption real AXI satisfies but a timing +model need not. -### 3.5 Diagnostics +That port terminates in **FASED**, FireSim's memory timing model, which is +two halves: -Vortex's own performance counters are the only view inside the core that does -not require re-elaborating the design, and they are read over the DCR path. -They are gated behind `PERF_ENABLE` (build the image with `PERF=1`, as in the -other synthesis flows); **without it every counter reads zero, which is -indistinguishable from an idle core rather than an absent instrument.** The -driver says so explicitly rather than leaving it to be inferred. +- **Functional**: data lives in the card's real memory, reached over the + shell's host-memory port. Reads and writes are performed when accepted; + responses are held in egress queues (with a per-id reorder buffer) until + released. +- **Timing**: a latency-bandwidth pipe decides the release cycle. Runtime + registers, defaults: `readLatency=30`, `writeLatency=30` target cycles, + ≤16 reads and ≤16 writes in flight. ---- +The **LoadMem widget** is the host's side door onto the same functional +memory — kernel upload, buffer copies, readback — consuming no target time. -## 4. Build flow +### 5.2 Configuring the timing model per run -The image build takes the same tokens as the card's native flow -(`hw/syn/xilinx/xrt` or `hw/syn/xilinx/aved`), from a configured build tree: +FASED's registers are set by plusargs, which the runtime forwards from +`VORTEX_FIRESIM_ARGS` (card and metasim alike). Any register set explicitly +requires the rest to be covered by the hardware-defaults plusarg: ```bash -source ci/firesim_env.sh # after the card's Xilinx tools (Vitis+XRT, or Vivado+VRT) -make -C hw/syn/firesim PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 TARGET=hw_emu # XRT card -make -C hw/syn/firesim TARGET=hw PERF=1 CONFIGS="-DVX_CFG_NUM_WARPS=4" # aved card +VORTEX_FIRESIM_ARGS="+mm_readLatency_0=100 +mm_writeLatency_0=100 +mm_useHardwareDefaultRuntimeSettings_0" \ + ./ci/blackbox.sh --driver=firesim --target=avedsim --app=demo ... ``` -`PLATFORM=` selects an XRT card and `TARGET` is `hw` or `hw_emu`; without -`PLATFORM` the card is the aved one for `DEVICE` (default the V80) and `TARGET` -is `hw` or `sim`. The image is copied to `__/bin` -(`firesim.xclbin` or `firesim.vbin`, plus `emconfig.json` for `hw_emu`) beside -`firesim.card`, which names the card for `run-firesim`; everything the build -generates lives under that directory. Nothing given on the command line -reaches FireSim's own makes, which use `PLATFORM`, `TARGET` and `BUILD_DIR` for -their own purposes. +Validated: the 4-core FAME-5 demo runs 15,541 cycles at the default 30-cycle +latency and 47,371 at 100/100 — same 9,776 instructions, `PASSED` both ways. +This is the only memory knob that needs no rebuild. + +The model's ceiling is worth computing before trusting a bandwidth-bound +number: `maxReads × 64 B / readLatency` ≈ 34 B per target cycle for the +whole GPU at the defaults. The 64-core `sgemm` sits exactly on it. Raising +the ceiling means `maxReads` (compile-time, and the egress area scales with +it) or lower modelled latency (runtime). There is no DRAM structure — banks, +rows, refresh — in the default pipe; FASED has DDR3 FIFO/FR-FCFS models and a +tags-only LLC model as compile-time alternatives (`WithDDR3FIFOMAS`, +`WithLLCModel`), none enabled for Vortex today, and no HBM timing model at +all. + +FASED's counters (`totalReads`, `totalWrites`, beat counts, `readMaxReqs`, +response errors) are read over MMIO and printed by the driver on every stall +report and watchdog — the first place to look when a kernel starves. + +### 5.3 Configuring the shell + +Everything card-specific comes from the platform descriptor +(`hw/syn/xilinx/platforms/.toml`), whose `[firesim]` section names the +FireSim platform, the part, and the host-memory binding; `gen_platform.py` +generates from it both the make fragment and the platform config class +(`V80PlatformConfig` = `WithHostMemBase(0x40_0000_0000)` + `DefaultAvedConfig`), +so one file governs elaboration and link alike. + +| | V80 (aved) | U55C (vitis) | +|---|---|---| +| binding | `host_mem_sp = "MEM"`: the HBM VNOC, **32 GB at 0x40_0000_0000** | `HOST_MEM_SP = HBM[0:7]`: 8 × 512 MB = 4 GiB at 0 | +| why | the whole 32-bit target map in one window; HBM0 (1 GB) would truncate it — the stack sits just under 4 GB; `HOST` (QDMA slave bridge) never completes reads and is refused by the descriptor | the card has no DDR bank; a narrower binding silently serves part of the map and kernel uploads land outside it | +| host channel | one, 64-bit, at the host clock | one, 64-bit | +| image-build knobs | `FREQUENCY` (60), `STRATEGY` (`TIMING`), `DEVICE` | + `PLATFORM=` | + +The host channel's width bounds *simulation speed* under memory traffic, +never target timing: a starved FASED gates the target clock, FMR rises, and +the kernel's cycle count is unchanged. + +Two width couplings must move together or nothing warns: + +- **`idBits` (VortexTarget.scala) ↔ `PLATFORM_MEMORY_ID_WIDTH` + (hw/syn/firesim/Makefile).** One shapes the IO the BlackBox is instantiated + with, the other the module's ports; Verilog pads or truncates a + disagreement silently. The value is bounded both ways: at 32, + `1 << idBits` wraps Scala's `Int` and every AXI id shares one response + queue (silently wrong data); at 8, per-id egress state cost 750,955 LUTs + against a U55C's 600,960. 4 matches what Vortex drives + (`CLOG2(TAG_BUFFER_SIZE)`) and FASED's `maxReads` of 16. +- **`addrBits` ↔ `VX_CFG_MEM_ADDR_WIDTH`.** The Scala side pins 32 while the + wrapper follows XLEN (48 at XLEN=64): XLEN=64 on FireSim needs both changed + together or addresses truncate silently. -The default goal runs, in order: +--- -``` -vivado-sources flatten Vortex with macros inlined (-P); -DSIMULATION for sim/hw_emu -xilinx-ips generate the Xilinx floating-point cores for the card's part -stage copy the Chisel/C++ targets and the card's platform config - class into $(FIRESIM_PATH) (only what changed) -replace-rtl Golden Gate elaboration (in $(FIRESIM_PATH)/sim) -image the platform's own build in its cl_ dir: bitstream, - or the hw_emu/sim image -``` +## 6. Observability -`sources`, `elaborate`, `metasim` and `run-metasim` are developer goals; the -runtime uses the first two to build the `xrtsim`/`avedsim` metasimulator. +### 6.1 Wave traces -The card's FireSim platform, part, device, host-memory binding and base address -come from its descriptor in `hw/syn/xilinx/platforms/`; the platform config -class (`U55CPlatformConfig`) is generated from it. `CONFIGS` takes architecture -overrides as the other synthesis flows do; extensions pull in their sources -through `hw/syn/extensions.mk`, and a changed `CONFIGS` invalidates the -flattened tree. +**Metasimulator (the primary path).** Build the metasim library with tracing +and pass the dump plusargs at run time: -### 4.0 Cards and platforms +```bash +EMUL_DEBUG=1 FAME5=1 \ +VORTEX_FIRESIM_EMUL_ARGS="+waveformfile=$PWD/demo.vcd +max-cycles=60000" \ + ./ci/blackbox.sh --driver=firesim --target=avedsim --cores=4 --warps=4 --threads=4 --app=demo +``` -| card | FireSim platform | run targets | run with | -|---|---|---|---| -| `u55c` | `vitis` (XRT, `v++`) | `xrtsim`, `hw_emu`, `hw` | `XCLBIN_PATH` | -| `v80` | `aved` (VRT, `slashkit`) | `avedsim`, `sim`, `hw` | `VRT_VBIN_PATH`, `VRT_DEVICE_BDF` (discovered when unset) | - -`run-firesim` (and `blackbox.sh --driver=firesim`) takes the card from -`$(FPGA_BIN_DIR)/firesim.card` for an image target; `xrtsim` and `avedsim` need -no image and default to the U55C and the V80 (`FIRESIM_CARD` overrides). With -no `TARGET` it runs `xrtsim`, as the xrt driver defaults to its simulator. On -the V80, `TARGET=hw` runs `ci/v80_preflight.sh` first, and a first `SIGINT` or -`SIGTERM` stops any FireSim run cleanly instead of killing it mid-launch; the -run is then reported `FAILED` and exits 130 or 143 (124 for the launch watchdog). - -A FireSim run names its image and card with the variables of the card's native -driver. On the V80 the Xilinx tools are the ones `slashkit` links with -(`/opt/xilinx/2025.2`), so the floating-point cores must be generated in that -environment too; and the static shell must be in flash, as for the native -`aved` backend. - -The `aved` platform (`AvedShim`) differs from `vitis` in three ways that matter -to anyone reading its traces: - -- **No MMCM.** The shell programs the kernel clock to the frequency the link - records (`FREQUENCY`), and the simulator runs on it directly. -- **Control registers at a 16-byte stride.** The compute shell resolves AXI-Lite - in 16-byte blocks, and FireSim's registers have read side effects (LoadMem - `R_DATA` dequeues), so the shim gives each register its own block and answers - the padding words itself. The driver scales addresses by 4. -- **A transport gate at open.** The driver reads SimulationMaster's fingerprint, - writes and reads back a probe, and checks the memory port is bound to the HBM - VNOC at the elaborated base, before any widget touches the device. -- **Clock gates built from fabric.** FireSim gates the target clock with a - `BUFGCE` per clock domain, and the compute shell's reconfigurable partition - contains no `BUFGCE` site — only `BUFG_FABRIC`. Implementation fails at - `opt_design` with `DRC HDPR-18` ("requires resource type 'BUFGCE_HDIO' which - is not ranged by 'pblock_slash'"), and the partition is fixed by the static - shell, so no link option can widen it. `FabricClockGates` (set by - `AvedConfig`) makes Golden Gate emit `FireSimFabricClockGate` instead, which - in hardware is not a gate at all: a plain AND marked `gated_clock`, which - synthesis's gated-clock conversion (turned on for the kernel's run by the - platform build) moves onto the clock-enable pin of every register, RAM and - DSP it drives, all clocked from the ungated clock — `BUFGCE`'s semantics - with no derived clock left. A pre-opt check fails the build if any kernel - clock is still driven by logic. An earlier latch + `BUFG_FABRIC` gate put the - target on a clock ~5 ns behind the host's, with its latch-to-AND step - untimed, and on the card the target started but its memory model never - completed a read. Simulation keeps the `BUFGCE` model, and other platforms - keep `BUFGCE` itself. - -### 4.0.1 What the platform build must do for Vortex - -Both cards share two requirements that FireSim's own targets do not have, so -they are passed in from `hw/syn/firesim`: - -- **`VORTEX_PRE_OPT_TCL` — the async-BRAM patch.** `VX_async_ram_patch` leaves - `VX_placeholder` black boxes wherever an asynchronous-read RAM has to live in - block RAM, and a post-synthesis script fills them from the netlist. Without it - `opt_design` aborts with `DRC INBB-3`, but only once a configuration is large - enough to push those RAMs out of LUTRAM — so small builds pass and large ones - do not. `vitis` attaches the hook with `--xp vivado_prop:...OPT_DESIGN.TCL.PRE`; - `aved` attaches it from a pre-synthesis hook, the only place slashkit lets a - step property be set. -- **`VORTEX_XILINX_IP_DIR` — the vendor cores.** The FPU and RTU instantiate - Xilinx floating-point IP (`xil_fma`, `xil_fdiv`, …), which must be generated - for the card's part and attached to the packaged kernel's file groups; read - into the packaging project they are dropped as unreferenced (`IP_Flow - 19-3833`). They are copied into the kernel, each into a fresh directory — - Tcl's `file copy` onto an existing directory nests the source inside it, which - silently leaves a repackaged kernel pointing at the previous run's cores. - On `aved` the conversion above cannot enter these out-of-context cores, so - there the RTU uses the soft FMA and divider (`VX_CFG_RTU_USE_DSP=0`) and no - vendor core is instantiated. - -### 4.1 Two source lists, and why - -Verilator resolves Vortex by **library search**, so `sources` names only the -packages and the wrapper — handing it the whole tree as a flat list makes it -stop treating the macro-computed width parameters as constants. Vivado has no -such search and must be handed every file, so `vivado-sources` flattens the -tree into one directory with macros already inlined. The constant-parameter -failure that rules flattening out for Verilator is an elaboration behaviour, -not a synthesis one. - -### 4.2 Synthesis invariants - -**`SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG` is not optional.** -`VX_dp_ram` and `VX_sp_ram` gate all block-RAM inference behind `` `ifdef -SYNTHESIS ``, and `gen_sources.sh -P` *preprocesses* into the flattened tree — -so without it the BRAM branch is not merely inactive, it is deleted before -Vivado sees it and every memory in the design lands in distributed RAM. The -symptom is a place-and-route failure hours later at ~750k LUTs-as-memory, with -nothing pointing at the cause. - -**The Makefile is a prerequisite of the flatten rule.** Without it, changing -any define does not invalidate the flattened tree and the next build silently -reuses the previous elaboration. Note it must be `$(firstword $(MAKEFILE_LIST))` -— `lastword` resolves to `build/config.mk`. - -**Guard the elaboration, not just the build.** `v++` runs synthesis into -place-and-route without stopping, so an over-utilized design surfaces only -after the placer has been tried. Two cheap checks catch the failure modes -above before hours are spent: that FASED elaborated a per-id response path -(`ReorderBuffer` present, multi-bit `MultiQueue` selectors), and that the -post-synthesis utilization report fits. Any such watchdog **must be -freshness-qualified** against the current run's timestamp — an unqualified -lookup reads the previous build's report and kills a healthy build. +`EMUL_DEBUG=1` (sim/firesim/Makefile) Verilates with `--trace +--trace-structs +define+DEBUG` into its own build directory (`…_dbg`) — the +model rule does not depend on flags, so debug and normal builds must not +share one. The plusargs are FireSim's emul-top contract (`top.sv`): +`+waveformfile=` opens the VCD, and `+max-cycles=` is **required** — +dumping runs from `+dump-start` (default 0) for `max-cycles` host clocks and +`$dumpoff`s at the end, so without it the window is empty. The run itself +continues to completion after the window closes. `VORTEX_FIRESIM_EMUL_ARGS` +also takes ordinary Verilator runtime args (e.g. `+verilator+rand+reset+2` +to randomize uninitialized state). + +**Size the window before running.** The dump covers every signal of the +FPGA payload: the 4-core FAME-5 demo above writes ~23 GB for a 60,000 +host-cycle window (~380 KB per host cycle). Place `+dump-start` just before +the region of interest and keep `+max-cycles` in the low tens of thousands; +the simulation itself runs to completion after the window closes, unperturbed +(same 15,541 cycles traced and untraced). + +**Reading the waves.** The hierarchy is +`TOP.emul.FPGATop.sim.target.…` — `emul` is the DPI testbench, `FPGATop` +the FPGA payload, `sim.target` the FAME hub's interior. Orientation: + +- everything toggles on the **host clock**; one target cycle is one firing + of the hub, so anchor on the clock channel or `ctrl_busy` rather than + counting clock edges; +- the target lives under `FPGATop.sim.target…`; under FAME-5 the interesting + state is in `VX_socket_top_threaded`, where a former register is an + N-entry array indexed by the thread counter, and `_fx_*` signals are + expression nodes hoisted by the FIRRTL conversion; +- the bridges' AXI (FASED) and MMIO surfaces are at the top level — the + fastest way to see whether a hang is the target or the transport. + +**U55C `hw_emu`** uses XSIM, not Verilator: waveform capture is governed by +`sw/runtime/firesim/xrt.ini` (`debug_mode=batch` writes the `.wdb`; it is +~800 MB for a demo run and the dominant cost, so it is off by default). + +**On the card: none.** There is no ILA in these images (AutoILA is off on +the V80 — Versal has no `ila` core — and not wired on the U55C flow). On-card +visibility is the counters and driver diagnostics below; anything deeper +moves to the metasimulator with the same image configuration. + +### 6.2 `$display`, `$error`, assertions + +All of Vortex's messages and assertions live behind `SIMULATION` +(`hw/rtl/VX_platform.vh`: `STATIC_ASSERT`, `ASSERT`/`RUNTIME_ASSERT`, +`ERROR`, and the `DBG_TRACE_*` machinery). Where that define exists in this +flow: + +| vehicle | `SIMULATION`? | consequence | +|---|---|---| +| `TARGET=sim` (V80 XSIM image), `TARGET=hw_emu` (U55C) | **yes** | `$display`/`$error`/asserts fire in the platform simulator's log | +| metasim `xrtsim`/`avedsim`, unthreaded | no (`-DNDEBUG` only) | compiled out — the metasim flattening carries neither synthesis nor simulation defines, and adding `-DSIMULATION` there is untested (a few library modules would also want the `dpi_trace` objects the metasim does not link) | +| any `FAME5=1` build | **structurally impossible** | sv2v/Yosys have no FIRRTL form for `$fflush`/`$write`; the conversion is always made from the synthesis flattening | +| `TARGET=hw` | no (`SYNTHESIS`) | compiled out, as in every synthesis flow | + +Golden Gate's own synthesized printf/assert machinery (`SynthPrints`, +`SynthAsserts` — both enabled in the platform config) applies only to +printf/stop nodes authored in Chisel or present in the FIRRTL; Vortex arrives +either as a BlackBox or as Yosys-converted FIRRTL containing neither, so it +contributes nothing here. In short: **for RTL-level messages and asserts, +use `TARGET=sim`/`hw_emu`, or rtlsim outside FireSim entirely; on `hw` and +under FAME-5, instrument via counters and the driver.** + +### 6.3 Performance counters and driver diagnostics + +Vortex's performance counters are the only view inside the core that needs no +re-elaboration. They are read over the DCR path and gated behind +`PERF_ENABLE`: build the image with `PERF=1`, and note that **without it +every counter reads zero, which is indistinguishable from an idle core**. +The driver prints, on every stall report, watchdog expiry and (optionally) +completion: FASED's request/beat/error counters, the core's scheduler and +pipeline-stall counters, and the D-cache counters — enough to separate "a +stalled pipeline" from "a memory request that never returned". + +The transport's own knobs and reports: + +| env var | effect | +|---|---| +| `VORTEX_FIRESIM_TRACE=1` | timestamped log of every request the transport services | +| `VORTEX_FIRESIM_ARGS` | extra plusargs to the FireSim driver (`+mm_*`, §5.2) | +| `VORTEX_FIRESIM_EMUL_ARGS` | Verilator runtime args, metasim only (waves, rand-reset) | +| `VORTEX_FIRESIM_TIMEOUT_S` | launch watchdog, wall seconds; expiry prints the full counter report and exits 124 | +| `VORTEX_FIRESIM_STALL_CYCLES` | target-cycles-busy threshold for the wedge report | +| `VORTEX_FIRESIM_STEP_BATCH` | batching ceiling (§4.3) | +| `VORTEX_FIRESIM_DCR_POLL_ROUNDS` | DCR read poll bound, in target cycles | +| `VORTEX_FIRESIM_PREFILL`, `VORTEX_FIRESIM_DUMP_UPLOADS`, `VORTEX_FIRESIM_VERIFY_WRITES`, `VORTEX_FIRESIM_REPORT_ON_IDLE` | debugging aids: pattern-stamp a region before launch; dump every upload to files; read back every write; counter dump on healthy completion | + +### 6.4 Choosing a vehicle + +| need | vehicle | +|---|---| +| waves of the exact FPGA payload | metasim + `EMUL_DEBUG=1` | +| Vortex `$display`/asserts | `TARGET=sim` / `hw_emu` (or rtlsim) | +| speed with full fidelity | the card | +| memory-timing experiments without rebuilds | any vehicle + `VORTEX_FIRESIM_ARGS` | --- -## 5. Measured results +## 7. The transport layer + +`sim/firesim/firesim_sim.cpp` is the largest single piece of the +integration, and where host-decoupling shows up as real complexity. + +- **Threading.** FireSim's bridges are not thread-safe and are only touched + by the simulator thread; every operation is a labelled closure marshalled + through a queue (the label is what makes a trace attribute a stall to an + operation). In the metasimulator the same thread runs Verilator's event + loop, and `entry()` is reached from the emul top's initial block. +- **Memory access.** `access_mem` moves data through LoadMem one host beat + at a time with read–modify–write on unaligned edges. **LoadMem's chunk is + counted in 32-bit words, not bytes**; treating it as bytes stores two live + bytes per eight and drops every unaligned write — a kernel image uploaded + that way is fetched as a mixture of written and stale memory with no error + anywhere. Whole-beat writes bypass the RMW path. +- **Stepping** is §4.3's adaptive batching; **DCR reads** single-step + because the response valid is a one-cycle pulse. Batched DCR polling would + need the response latched in the wrapper — a target-side change and a new + bitstream, noted as a follow-up. +- **Peeks after a blocking step are non-blocking.** A blocking peek waits up + to 0.1 s for the `PRECISE_PEEKABLE` flag, which a FAME-5 design never + raises, then returns the same value; after a blocking step the wait adds + nothing but time. Left blocking, every DCR poll cost 0.1 s and a 64-core + readback took 20 s per register. (Why FAME-5 outputs never assert + precise-peekable is an open follow-up.) +- **Watchdogs.** A launch that exceeds `VORTEX_FIRESIM_TIMEOUT_S` or a + busy-level that exceeds `VORTEX_FIRESIM_STALL_CYCLES` produces the full + counter report (§6.3) and a clean abort — on this vehicle, "hung" must be + distinguishable from "slow" without a reboot. -U55C, kernel clock constrained at 60 MHz, `STRATEGY=TIMING`. Configuration: -1 cluster / 1 core / 4 warps / 4 threads, SIMD 4, L1 only, no A or C -extension, no TCU/DXA/RTU/TEX/RASTER/OM, `FPU_USE_DSP=0`. +--- -### 5.1 Utilization (kernel only) +## 8. Running -| resource | used | available | % | +```bash +# metasimulator (no image, no card): +FAME5=1 ./ci/blackbox.sh --driver=firesim --target=avedsim --cores=64 --warps=16 --threads=16 --app=demo +# the card: +FAME5=1 FPGA_BIN_DIR=$PWD/build32_aved_hw_fame5/bin \ + ./ci/blackbox.sh --driver=firesim --target=hw --cores=64 --warps=16 --threads=16 --app=demo +``` + +| card | FireSim platform | run targets | image / card named by | |---|---|---|---| -| CLB LUTs | 131,578 | 1,303,680 | 10.09 | -| — as logic | 79,900 | 1,303,680 | 6.13 | -| — as memory | 51,678 | 600,960 | 8.60 | -| CLB registers | 62,004 | 2,607,360 | 2.38 | -| Block RAM tiles | 109 | 2,016 | 5.41 | -| URAM | 8 | 960 | 0.83 | -| DSP | 24 | 9,024 | 0.27 | +| `u55c` | `vitis` (XRT) | `xrtsim`, `hw_emu`, `hw` | `XCLBIN_PATH` | +| `v80` | `aved` (VRT) | `avedsim`, `sim`, `hw` | `VRT_VBIN_PATH`, `VRT_DEVICE_BDF` (discovered when unset) | + +`run-firesim` takes the card from `$(FPGA_BIN_DIR)/firesim.card`; the +metasim targets need no image and default to the card of their platform +(`FIRESIM_CARD` overrides). The architecture flags (`--cores`, `--warps`, +`--threads`, `CONFIGS=`) must match the image's — they configure the runtime +and, in the metasim, the model build. On the V80, `TARGET=hw` runs +`ci/v80_preflight.sh` first (link state, cooldown between PDI loads), and a +first `SIGINT`/`SIGTERM` stops the run cleanly (`FAILED`, exit 130/143; 124 +is the launch watchdog). A FireSim run uses the card's native driver +environment — on the V80 that means the slashkit-linked Xilinx tools and the +static shell in flash, as for the native `aved` backend. -DSP usage is near zero because this configuration builds the FPU from LUTs by -choice (`FPU_USE_DSP=0`), not for want of DSPs. +--- -### 5.2 Timing +## 9. Measured results -Kernel clock `clk_out1_firesim_clocking` closes at **WNS +5.283 ns** over -548,101 endpoints with zero failing, implying Fmax ≈ 87.8 MHz against the -60 MHz constraint. The design-level WNS of +0.003 ns belongs to the Vitis -shell's 450 MHz `hbm_aclk`, **not** to Vortex — reading the global number as -the kernel's inverts the conclusion. +### 9.1 U55C, default lane (FAME-1) -### 5.3 Kernels +1 cluster / 1 core / 4 warps / 4 threads, L1 only, `FPU_USE_DSP=0`, kernel +clock 60 MHz. Kernel-only utilization: 131,578 LUTs (10.1%), 62,004 FFs, +109 BRAM tiles, 8 URAM, 24 DSP. The kernel clock closes at WNS +5.283 ns +(Fmax ≈ 87.8 MHz); the design-level WNS of +0.003 ns belongs to the Vitis +shell's 450 MHz `hbm_aclk`, not to Vortex. `sgemm -n1024`: +instrs=1,187,250,192, cycles=3,229,839,120, 84.6 s of kernel time (~153× +faster than rtlsim on `sgemm -n32`). Instruction counts match rtlsim +exactly; cycle counts differ −1.3%…+7.1% — the FASED timing model, not the +core: FPGA and hw_emu agree exactly. -| app | args | instrs | cycles | IPC | -|---|---|---|---|---| -| demo | -n64 | 2,444 | 6,429 | 0.380 | -| vecadd | -n1024 | 6,160 | 18,837 | 0.327 | -| sgemm | -n32 | 48,400 | 115,490 | 0.419 | -| sgemm | -n1024 | 1,187,250,192 | 3,229,839,120 | 0.368 | +### 9.2 V80, FAME-5, 64 cores NW=NT=16 -`sgemm -n1024` completes in **84.6 s** of kernel time. Instruction counts match -rtlsim exactly; cycle counts differ by −1.3% to +7.1%, which is the FASED -memory timing model rather than the core — FPGA and hw_emu cycle counts agree -exactly. +Image: 6 h 33 m end to end (synthesis ≈ 87 min, routing ≈ 2.5 h — ~300K +routing overlaps at the first iteration, from the SLR2 concentration below), +**timing met at 60 MHz**: WNS +0.329 ns, WHS +0.010 ns, zero failing +endpoints. -For scale: `sgemm -n32` takes 1228 ms on rtlsim and 8 ms on the card, ~153×. +| resource | used | available | % | +|---|---|---|---| +| CLB LUTs | 974,424 | 2,574,208 | 37.9 | +| — as logic | 662,530 | 2,574,208 | 25.7 | +| — as memory (LUTRAM) | 311,894 | 1,287,104 | 24.2 | +| Registers | 564,836 | 5,148,416 | 11.0 | +| Block RAM tiles | 1,682.5 | 3,741 | 45.0 | +| URAM | 105 | 1,925 | 5.5 | +| DSP | 64 | 10,848 | 0.6 | + +The threaded socket is 719K LUTs / 258K LUTRAM / 1,679 BRAM — one copy in +place of an unthreadable ~14.1M LUTs. FireSim's own token infrastructure +carries 224K FFs (more than the socket's 183K), and placement concentrates: +SLR2 holds 91% of the FFs at 79% slice occupancy, SLR1 71% of its BRAM — the +route-time driver, and the first place to look before raising `FREQUENCY`. + +On the card: + +| app | instrs | cycles | IPC | target kHz | FMR | wall | +|---|---|---|---|---|---|---| +| demo `-n1` (16K pts) | 45,012 | 7,884 | 5.71 | 316 | 190 | 7 s | +| demo (1M pts) | 625,664 | 3,460,483 | 0.18 | 378 | 159 | 16 s | +| sgemm `-n1024` (1024³) | 296,816,640 | 145,714,145 | 2.04 | 462 | 130 | 323 s | + +Both memory-bound results sit on the timing model's ~34 B/cycle ceiling +(§5.2). Cycle-exactness across builds: 4-core demo identical +(`instrs=9776, cycles=15541`) on the native baseline, the unthreaded-FIRRTL +build (`FAME5_THREADS=0`) and the threaded build. --- -## 6. Relationship to upstream FireSim - -The FireSim checkout is a **git clone of the fork**, `vortexgpgpu/firesim` -branch `vortex_3.x`, based on upstream tag 1.21.0. Every change Vortex needs in -FireSim is a commit there, and `VERSION` pins the one this tree is built and -tested against (`FIRESIM_REV`). `make stage` refuses a checkout that does not -contain the pinned commit; being ahead of it is normal while developing the -fork. - -The fork's changes are bug fixes and platform plumbing rather than divergence: - -| area | why | -|---|---| -| Vitis platform | host-memory base as a config field (`HostMemBaseAddressKey`), checked by the driver against the memory the binding resolved to; 4 GiB DRAM aperture; part read from the platform; vendor IP cores packaged with the kernel | -| driver lifetime | tear the platform down at the end of `entry()`; a global `unique_ptr` is destroyed during static destruction, after XRT's own statics, and segfaults in the device registry | -| driver build | the Vitis driver is compiled by the host toolchain: XRT needs the host glibc, and conda's start files do not link against glibc 2.34+ | -| cycle accounting | `end_tcycle` is read absolute and never rebased, so it accumulates across runs and FMR divides a per-run quantity by a cumulative one | -| build | Vitis 2024.2 fixes; package the flattened Vortex sources into the kernel | +## 10. Relationship to upstream FireSim -All are candidates for upstreaming. +The FireSim checkout is a clone of the `vortexgpgpu/firesim` fork (branch +`vortex_3.x`, based on upstream 1.21.0), consumed through `FIRESIM_PATH` and +pinned by `FIRESIM_REV` in `VERSION`. `make stage` refuses a checkout that +does not contain the pinned commit; being ahead of it is normal while +developing the fork. The fork's changes are platform plumbing and bug fixes +rather than divergence — the AVED platform (shim, register blocks, fabric +clock gates, host-memory base as a config field), driver lifetime and +toolchain fixes, cycle-accounting fixes, and kernel packaging — all +candidates for upstreaming. Everything Vortex-specific (targets, wrapper, +transport, FAME-5 conversion) lives in this repository and is staged in at +build time. --- -## 7. Design invariants - -1. **`idBits` and `PLATFORM_MEMORY_ID_WIDTH` move together.** A disagreement is - padded or truncated silently. Raising either requires re-checking - utilization, not merely avoiding the `1 << idBits` overflow. -2. **`SYNTHESIS` must reach the flattened tree.** Its absence is invisible - until place-and-route and costs hours. -3. **Vortex's `PERF:` counters are the cycle authority.** FireSim's - `Target Cycles Emulated` is quantized by the step schedule. -4. **The target only advances when stepped.** Any new driver operation that - waits on target state must step, and must know whether the signal it waits - on is a pulse or a level. -5. **All-zero performance counters mean the instrument is absent** - (`PERF_ENABLE` off), not that the core is idle. +## 11. Design invariants + +1. **The target only advances when stepped.** Any new driver operation that + waits on target state must step, and must know whether the signal it + waits on is a pulse or a level. +2. **`idBits` ↔ `PLATFORM_MEMORY_ID_WIDTH` move together**, and `addrBits` ↔ + `VX_CFG_MEM_ADDR_WIDTH`; disagreements pad or truncate silently. +3. **`SYNTHESIS` must reach the flattened tree** — its absence surfaces + hours later in place-and-route. +4. **Vortex's `PERF:` counters are the cycle authority**, never + `Target Cycles Emulated`. +5. **FAME-5 threads one module type, and its instances must be identical** — + IDs travel as ports, never parameters. +6. **No BlackBox inside the threaded module** — it would be shared across + threads silently. +7. **All-zero performance counters mean `PERF_ENABLE` is off**, not that the + core is idle. +8. **LoadMem's chunk unit is 32-bit words**, not bytes. +9. **`FAME5_THREADS=0` must stay cycle-identical to the threaded build** — + it is the anchor that separates conversion bugs from threading bugs. --- -## 8. Known limitations - -- **Metasimulation is dead.** FireSim's Verilator metasim builds and starts, - then deadlocks before target cycle 0 (65 threads in `futex_wait`, 0% CPU, no - output past `entering simif.run`). It is deliberately not fixed: the card - supersedes every purpose it served. Anyone reaching for it should expect this - rather than read the silence as a slow build. -- **DCR reads are slow** by one MMIO round trip per target cycle (§3.4). -- **Regression coverage is partial.** The kernels in §5.3 plus `conv3`, - `dogfood`, `dotproduct` and `mstress` pass on the card; the suite has not - been swept exhaustively, and a large share of it requires extensions this - configuration disables. -- **The conda environment is not relocatable** as built. It records its own - prefix in every script shebang and in package metadata, so the tree must - stay where `build-setup.sh` created it, or be re-packaged with `conda-pack` - and unpacked with `conda-unpack` at the new location. See - [`../building_toolchain.md`](../building_toolchain.md) (section: FireSim). +## 12. Known limitations + +- **No RTL `$display`/asserts on `hw` or under FAME-5** (§6.2); the metasim + compiles them out too (no DPI linked). RTL-message debugging belongs to + `TARGET=sim`/`hw_emu` or rtlsim. +- **DCR reads single-step** — one MMIO round trip per target cycle; batched + polling needs a latched response in the wrapper (target-side follow-up). +- **FAME-5 excludes VM and graphics**: `VX_socket_top` does not carry their + ports and statically rejects those configurations. +- **XLEN=64 is not wired through FireSim**: `addrBits` is pinned at 32 + (§5.3). +- **Metasim speed falls with thread count**: the 64-thread FAME-5 model runs + at a few tens of target cycles per second under Verilator — use it for + correctness and waves, the card for anything long. +- **One host memory channel** bounds simulation speed under heavy memory + traffic (never target timing). +- **The default memory model is a flat latency-bandwidth pipe** — no banks, + rows, refresh or HBM timing (§5.2). +- **The conda environment is not relocatable** as built; re-package with + `conda-pack` to move it (see + [`../building_toolchain.md`](../building_toolchain.md)). From 52e5aee087e7b4067eb5442b1a98af34f122c2ca Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 27 Sep 2026 01:52:40 -0700 Subject: [PATCH 171/336] docs: state firesim_integration.md's contents declaratively The section index was phrased as reader questions and two headings as How-questions, which reads as a tutorial rather than a specification. Descriptive titles throughout; anchors updated. Co-Authored-By: Claude Fable 5 (cherry picked from commit 1efb15a9c70421f6abeb32d7b9fdd90ca2446aba) --- docs/designs/firesim_integration.md | 42 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index a86e733393..2b39e43c7e 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -1,23 +1,23 @@ # FireSim integration — host-decoupled FPGA simulation of Vortex -This document describes how Vortex runs as a [FireSim](https://fires.im/) -target: a multi-core GPU simulated on an FPGA at cycle exactness, on the -Alveo U55C (Vitis/XRT) and the Alveo V80 (AVED/VRT). It answers, in order: +This document specifies how Vortex runs as a [FireSim](https://fires.im/) +target: a multi-core GPU simulated on an FPGA with cycle exactness, on the +Alveo U55C (Vitis/XRT) and the Alveo V80 (AVED/VRT). -| § | question | +| § | contents | |---|---| -| [1](#1-system-architecture) | What is on the FPGA, what is on the host, and how do they meet? | -| [2](#2-how-vortex-rtl-is-synthesized) | How is the Vortex RTL synthesized — with and without FAME-5? | -| [3](#3-fame-5-many-sockets-one-physical-copy) | Which modules are time-multiplexed ("replicated") and how does that work? | -| [4](#4-how-simulation-timing-is-modelled) | How is simulation timing modelled? | -| [5](#5-the-memory-system) | How is memory modelled and configured, and how is the shell configured? | -| [6](#6-observability) | How do you get wave traces, `$display`, `$error`, asserts, counters? | -| [7](#7-the-transport-layer) | What does the driver actually do, and why? | -| [8](#8-running) | How do you run it? | -| [9](#9-measured-results) | What has been measured? | -| [10](#10-relationship-to-upstream-firesim) | How does the FireSim fork relate to upstream? | -| [11](#11-design-invariants) | What must never be broken? | -| [12](#12-known-limitations) | What doesn't work (yet)? | +| [1](#1-system-architecture) | System architecture: host stack, FPGA payload, execution model | +| [2](#2-synthesis-of-the-vortex-rtl) | Synthesis of the Vortex RTL, with and without FAME-5 | +| [3](#3-fame-5-many-sockets-one-physical-copy) | FAME-5 socket time-multiplexing | +| [4](#4-the-simulation-timing-model) | The simulation timing model | +| [5](#5-the-memory-system) | The memory system and its configuration; shell configuration | +| [6](#6-observability) | Observability: wave traces, RTL messages and assertions, counters | +| [7](#7-the-transport-layer) | The transport layer | +| [8](#8-running) | Run targets and commands | +| [9](#9-measured-results) | Measured results | +| [10](#10-relationship-to-upstream-firesim) | Relationship to upstream FireSim | +| [11](#11-design-invariants) | Design invariants | +| [12](#12-known-limitations) | Known limitations | FireSim itself is an external dependency: the [`vortexgpgpu/firesim`](https://github.com/vortexgpgpu/firesim) fork, consumed @@ -62,7 +62,7 @@ through `simif_emul` over DPI, and needs no card, image, XRT or VRT. It runs the same tokens on the same generated design, which is why it agrees with silicon cycle for cycle and is the step before every bitstream. -### 1.2 What is on the FPGA +### 1.2 The FPGA payload ![A Vortex instance + FireSim shell on the FPGA](../assets/img/firesim_fpga_architecture.svg) @@ -83,7 +83,7 @@ containing: bridge (`tcycle`/`hcycle`), and FASED (the memory model, [§5](#5-the-memory-system)). - **The FAME hub** (`FAMETop`): token channels between the bridges and the - target; the mechanism of host decoupling ([§4](#4-how-simulation-timing-is-modelled)). + target; the mechanism of host decoupling ([§4](#4-the-simulation-timing-model)). - **The target**: `VortexDUT` ▸ `VX_firesim_wrap` ▸ Vortex. Clocking on the V80 has one platform quirk worth knowing: FireSim gates the @@ -123,7 +123,7 @@ what every measurement in this repository uses. --- -## 2. How Vortex RTL is synthesized +## 2. Synthesis of the Vortex RTL ![Build flow](../assets/img/firesim_build_flow.svg) @@ -264,7 +264,7 @@ parser's recursion, `-Xss`), generated Verilog 229 MB. ![FAME-5 threading](../assets/img/firesim_fame5_threading.svg) -The 64-core question is an area question. One NW=NT=16 core synthesizes to +The obstacle to a 64-core build is area. One NW=NT=16 core synthesizes to ~221K LUTs; 64 of them need ~14.1M against the V80's 2.57M. FAME-5 ("model multithreading") removes the duplication instead of shrinking the core: all 64 sockets **take turns using one physical copy of the socket logic**, one @@ -312,7 +312,7 @@ Verilog baseline, unthreaded FIRRTL, threaded — agree exactly (4-core demo: --- -## 4. How simulation timing is modelled +## 4. The simulation timing model ![Timing model](../assets/img/firesim_timing_model.svg) From 829ead5c44c6c1423344f676c89828f7d78eee2c Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 17:47:08 -0700 Subject: [PATCH 172/336] firesim: FAME-5 socket threading integration and conversion flow RTL: give VX_socket_top a flat port boundary and per-socket identity as ports so all sockets elaborate identically for Golden Gate threading (FAME5-gated; no behavior change for other platforms). Flow: selective FIRRTL export with iterative memory mapping (export_firrtl.py), identifier shortening and locator stripping to keep full-GPU FIRRTL under the 2 GiB reader limit (shorten_fir.py, 9x), and the to_firrtl.sh pipeline updates that wire them in. Co-Authored-By: Claude Fable 5 (cherry picked from commit 41b59d5a5adb6e9c98ff6b2119bb9d58b20fe16d) Integration (phase 2): the RTL half (VX_socket_top's flat boundary, FAME5-gated in VX_cluster) is not replayed; 803697377 removes the FAME-5 RTL hooks again. The conversion-flow scripts are taken. Co-Authored-By: Claude Opus 5.5 --- hw/syn/firesim/fame5/export_firrtl.py | 56 ++++++++++++++++++++ hw/syn/firesim/fame5/shorten_fir.py | 75 +++++++++++++++++++++++++++ hw/syn/firesim/fame5/to_firrtl.sh | 14 ++++- 3 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 hw/syn/firesim/fame5/export_firrtl.py create mode 100644 hw/syn/firesim/fame5/shorten_fir.py diff --git a/hw/syn/firesim/fame5/export_firrtl.py b/hw/syn/firesim/fame5/export_firrtl.py new file mode 100644 index 0000000000..4551e6152c --- /dev/null +++ b/hw/syn/firesim/fame5/export_firrtl.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""FIRRTL export with selective memory lowering. + +usage: export_firrtl.py + +FIRRTL write ports carry one enable and one uniform mask, so write_firrtl +rejects the occasional $mem whose ports encode read-modify-write, per-field +or shifting stores ('Complex write enable on port N on memory M'). Every +memory it can express must stay a memory -- FAME-5 threading turns those +into dense deep LUTRAM/BRAM -- so only the memories the writer itself names +are memory_map'ed to FF logic, and the export is retried until it goes +through. The rejected arrays are register-class storage in every flow +(async reads, a few entries deep); this just makes that explicit. +""" + +import os +import re +import subprocess +import sys + +GLOB = re.compile(r"[][*?]") +ERR = re.compile(r"Complex write enable on port \d+ on memory (\S+?)\.?\s*$", re.M) + +il, out, log = sys.argv[1:4] +yosys = os.environ.get("YOSYS", "yosys") + +mapped = [] # selections accumulated across attempts +with open(log, "w") as lf: + for attempt in range(30): + script = f"read_rtlil {il}\n" + if mapped: + script += "memory_map " + " ".join(mapped) + "\n" + script += f"write_firrtl {out}\n" + r = subprocess.run([yosys, "-q", "-p", script], capture_output=True, text=True) + lf.write(f"--- attempt {attempt}: mapped={mapped}\n{r.stdout}{r.stderr}\n") + lf.flush() + if r.returncode == 0: + sys.exit(0) + m = ERR.search(r.stdout + r.stderr) + if not m: + lf.write("unrecognized failure\n") + print(f"export_firrtl: unrecognized yosys failure (see {log})", file=sys.stderr) + sys.exit(1) + path = m.group(1) + module, tail = path.split(".", 1)[0], path.rsplit(".", 1)[-1] + if GLOB.search(module + tail): + # fall back to the exact path with glob metachars neutralized + sel = [f"{module}/c:" + GLOB.sub("?", path.split('.', 1)[1])] + else: + # map every instance of this array kind in one step + sel = [f"{module}/c:*.{tail}", f"{module}/c:{tail}"] + print(f"export_firrtl: memory_map {' '.join(sel)} ({path})") + mapped += sel + +print(f"export_firrtl: did not converge in 30 attempts (see {log})", file=sys.stderr) +sys.exit(1) diff --git a/hw/syn/firesim/fame5/shorten_fir.py b/hw/syn/firesim/fame5/shorten_fir.py new file mode 100644 index 0000000000..01123c4020 --- /dev/null +++ b/hw/syn/firesim/fame5/shorten_fir.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Shorten internal identifiers in Yosys-written FIRRTL. + +usage: shorten_fir.py + +Flattening leaves every wire of the top region carrying its full +hierarchical path as its name, and splitnets multiplies those wires per +bit: the identifiers become most of the file's bytes, and a full-GPU +export exceeds the FIRRTL reader's 2 GiB input limit. This pass renames +module-local identifiers (wire/node/reg/cmem/smem/mem/inst names) longer +than a threshold to sequential short names, per module. Module names and +port names are never touched, so instantiation, splicing and annotation +are unaffected. Renaming is semantics-preserving: FIRRTL identifiers are +local to their module. +""" + +import re +import sys + +THRESHOLD = 8 # keep short names for readability; rename only long ones + +DECL = re.compile( + r"^\s*(?:wire|node|reg|cmem|smem|inst)\s+([A-Za-z_][A-Za-z0-9_$]*)" +) +MEM = re.compile(r"^\s*mem\s+([A-Za-z_][A-Za-z0-9_$]*)\s*:") +MODULE = re.compile(r"^ (?:ext)?module\s+(\S+?)\s*:") +TOKEN = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*") + +src, dst = sys.argv[1], sys.argv[2] + +# Pass 1: per module, map long declared names to short ones. +maps = {} # module -> {long: short} +mod = None +with open(src, "r", errors="strict") as fh: + for line in fh: + m = MODULE.match(line) + if m: + mod = m.group(1) + maps[mod] = {} + continue + if mod is None: + continue + d = DECL.match(line) or MEM.match(line) + if d: + name = d.group(1) + if len(name) > THRESHOLD and name not in maps[mod]: + maps[mod][name] = f"sn{len(maps[mod])}" + +# Safety: no existing short identifier may collide with generated ones. +collide = re.compile(r"^sn\d+$") +for mod_name, mp in maps.items(): + for long_name in mp: + assert not collide.match(long_name), f"collision risk: {long_name}" + +LOCATOR = re.compile(r"\s*@\[[^\]]*\]") + +def rename_line(line, mp): + line = LOCATOR.sub("", line) + return TOKEN.sub(lambda m: mp.get(m.group(0), m.group(0)), line) + +# Pass 2: rewrite. +mod = None +mp = {} +with open(src, "r", errors="strict") as fh, open(dst, "w") as out: + for line in fh: + m = MODULE.match(line) + if m: + mod = m.group(1) + mp = maps.get(mod, {}) + out.write(LOCATOR.sub("", line)) + continue + out.write(rename_line(line, mp)) + +total = sum(len(m) for m in maps.values()) +print(f"shorten_fir: renamed {total} identifiers across {len(maps)} modules") diff --git a/hw/syn/firesim/fame5/to_firrtl.sh b/hw/syn/firesim/fame5/to_firrtl.sh index c9dea0328b..98916834ff 100755 --- a/hw/syn/firesim/fame5/to_firrtl.sh +++ b/hw/syn/firesim/fame5/to_firrtl.sh @@ -28,7 +28,19 @@ $Y -q -l "$W/y.log" -p " splitnets -driver; opt_clean -purge splitnets $TOP; opt_clean -purge $REN - write_firrtl $W/raw.fir" || { tail -5 "$W/y.log"; exit 1; } + write_rtlil $W/design.il" || { tail -5 "$W/y.log"; exit 1; } +# FIRRTL memories carry one enable and one uniform mask per write port, so a +# handful of RTL arrays - read-modify-write DCR files, shift queues, per-field +# struct writes - collect into \$mem cells write_firrtl rejects ('Complex +# write enable'). Everything it can express stays a memory (threading turns +# those into dense deep LUTRAM); only the rejected ones are memory_map'ed to +# the FF logic every other flow infers for them anyway. The writer names each +# offender, so export simply iterates until it goes through. +YOSYS=$Y python3 "$(dirname "$0")/export_firrtl.py" "$W/design.il" "$W/raw0.fir" "$W/export.log" || { tail -5 "$W/export.log"; exit 1; } +# Flattened + bit-split identifiers are most of the file's bytes; a full-GPU +# export otherwise exceeds the FIRRTL reader's 2 GiB input limit. +python3 "$(dirname "$0")/shorten_fir.py" "$W/raw0.fir" "$W/raw.fir" +rm -f "$W/raw0.fir" # The writer types clocks as UInt<1> and converts each use with asClock(); # (Above: the non-threaded top is split to single bits -- Yosys reuses bits of a # wire to drive other bits of the same wire, a loop at FIRRTL's word level.) From a84044a1c9d66d43e94eba6411e3c8bed8fffce8 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 9 Oct 2026 10:36:33 -0700 Subject: [PATCH 173/336] rtlsim: AXI_BUS mode with a latency-bandwidth-pipe memory model AXI_BUS=1 builds rtlsim around rtlsim_axi_shim: Vortex behind its AXI4 adapter (Vortex_axi), with the parameters the FPGA platform wrappers pass, so the adapter is the same logic the platforms run. Memory timing comes from sim/common/axi_latency_pipe.h, a cycle-exact model of FireSim FASED's Latency Bandwidth Pipe: fixed read/write latency (release one cycle early into a one-entry response stage), bounded outstanding reads and writes, AW and last-W pairing as in FASED's SplitTransactionModel. Defaults are FASED's (30/30 cycles, 16/16 outstanding); AXI_MEM_READ_LATENCY, AXI_MEM_WRITE_LATENCY, AXI_MEM_MAX_READS and AXI_MEM_MAX_WRITES override them. This gives a Verilator reference with the same memory timing as a FireSim image, so the two can be compared cycle for cycle. The default build is unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c06ecbad5100089df2c005d03e03a64a2a93a292) Integration (phase 2): phase 2's command-processor DMA port into the LLC is not on Vortex_axi, so under AXI_BUS the CP DMA model is compiled out and cp_llc_client() reports false (the CP writes memory directly, as on the FPGA); the default rtlsim build keeps the port. This is the resolution the foldsim branch carries at its tip. --- sim/common/axi_latency_pipe.h | 189 +++++++++++++++++++++++++++++++++ sim/rtlsim/Makefile | 7 ++ sim/rtlsim/processor.cpp | 190 +++++++++++++++++++++++++++++++++- sim/rtlsim/rtlsim_axi_shim.sv | 144 ++++++++++++++++++++++++++ 4 files changed, 525 insertions(+), 5 deletions(-) create mode 100644 sim/common/axi_latency_pipe.h create mode 100644 sim/rtlsim/rtlsim_axi_shim.sv diff --git a/sim/common/axi_latency_pipe.h b/sim/common/axi_latency_pipe.h new file mode 100644 index 0000000000..c7591af18b --- /dev/null +++ b/sim/common/axi_latency_pipe.h @@ -0,0 +1,189 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +namespace vortex { + +// Timing of one AXI4 memory port as a latency-bandwidth pipe: a fixed latency +// per transaction and a bound on the reads and writes in flight, one beat per +// cycle on each response channel. Cycle for cycle this is FireSim FASED's +// "Latency Bandwidth Pipe" (LatencyBandwidthPipe.scala over TimingModel.scala's +// SplitTransactionModel and AXI4Releaser); the defaults are FASED's hardware +// defaults. Every output is a function of registered state only, so the caller +// samples the port's request side once per cycle, before the clock edge, and +// calls step() after it. Timing only: responses leave in the order their +// requests were accepted, so the caller keeps the data alongside. +class AxiLatencyPipe { +public: + struct Config { + uint32_t read_latency = 30; + uint32_t write_latency = 30; + uint32_t max_reads = 16; + uint32_t max_writes = 16; + }; + + struct Inputs { + bool ar_valid = false; + uint32_t ar_id = 0; + uint32_t ar_len = 0; + bool aw_valid = false; + uint32_t aw_id = 0; + bool w_valid = false; + bool w_last = false; + bool r_ready = false; + bool b_ready = false; + }; + + AxiLatencyPipe() : cfg_() {} + explicit AxiLatencyPipe(const Config& cfg) : cfg_(cfg) {} + + void reset() { + *this = AxiLatencyPipe(cfg_); + } + + bool ar_ready() const { return pending_reads_ < cfg_.max_reads; } + bool aw_ready() const { return pending_aw_ < cfg_.max_writes; } + bool w_ready() const { return pending_w_ < cfg_.max_writes; } + + bool r_valid() const { return cur_read_.has_value(); } + uint32_t r_id() const { return cur_read_->id; } + bool r_last() const { return cur_read_->beat == cur_read_->len; } + uint32_t r_beat() const { return cur_read_->beat; } + + bool b_valid() const { return cur_write_.has_value(); } + uint32_t b_id() const { return cur_write_->id; } + + // Advances one cycle on the request-side values sampled before the edge. + void step(const Inputs& in) { + bool ar_fire = in.ar_valid && this->ar_ready(); + bool aw_fire = in.aw_valid && this->aw_ready(); + bool wlast_fire = in.w_valid && in.w_last && this->w_ready(); + bool r_fire = this->r_valid() && in.r_ready; + bool rlast_fire = r_fire && this->r_last(); + bool b_fire = this->b_valid() && in.b_ready; + + // Reads: latency pipe (a flow queue) into a one-entry pipe stage that + // drives R. Release is one cycle early, since that stage adds the cycle. + { + std::optional arrived; + if (ar_fire) { + arrived = Read{cycle_ + cfg_.read_latency - 1, in.ar_id, in.ar_len, 0}; + } + Read* head = !read_pipe_.empty() ? &read_pipe_.front() : (arrived ? &*arrived : nullptr); + bool release = head && head->release <= cycle_; + bool stage_ready = !cur_read_ || rlast_fire; + if (r_fire && !rlast_fire) { + ++cur_read_->beat; + } else if (rlast_fire) { + cur_read_.reset(); + } + if (release && stage_ready) { + cur_read_ = *head; + if (!read_pipe_.empty()) { + read_pipe_.pop_front(); + } else { + arrived.reset(); + } + } + if (arrived) { + read_pipe_.push_back(*arrived); + } + } + + // Writes: an AW and its last W beat, in either order, form one write; its + // latency counts from the cycle both have arrived. + { + bool new_write = (pending_w_ > pending_aw_ && aw_fire) + || (pending_w_ < pending_aw_ && wlast_fire) + || (aw_fire && wlast_fire); + std::optional aw_arrived; + if (aw_fire) { + aw_arrived = in.aw_id; + } + std::optional formed; + if (new_write) { + uint32_t id; + if (!aw_queue_.empty()) { + id = aw_queue_.front(); + aw_queue_.pop_front(); + } else { + id = *aw_arrived; + aw_arrived.reset(); + } + formed = Write{cycle_ + cfg_.write_latency - 1, id}; + } + if (aw_arrived) { + aw_queue_.push_back(*aw_arrived); + } + Write* head = !write_pipe_.empty() ? &write_pipe_.front() : (formed ? &*formed : nullptr); + bool release = head && head->release <= cycle_; + bool stage_ready = !cur_write_ || b_fire; + if (b_fire) { + cur_write_.reset(); + } + if (release && stage_ready) { + cur_write_ = *head; + if (!write_pipe_.empty()) { + write_pipe_.pop_front(); + } else { + formed.reset(); + } + } + if (formed) { + write_pipe_.push_back(*formed); + } + } + + pending_reads_ = count(pending_reads_, ar_fire, rlast_fire); + pending_aw_ = count(pending_aw_, aw_fire, b_fire); + pending_w_ = count(pending_w_, wlast_fire, b_fire); + ++cycle_; + } + +private: + struct Read { + uint64_t release; + uint32_t id; + uint32_t len; + uint32_t beat; + }; + + struct Write { + uint64_t release; + uint32_t id; + }; + + static uint32_t count(uint32_t value, bool inc, bool dec) { + if (inc && !dec) return value + 1; + if (dec && !inc && value != 0) return value - 1; + return value; + } + + Config cfg_; + uint64_t cycle_ = 0; + uint32_t pending_reads_ = 0; + uint32_t pending_aw_ = 0; + uint32_t pending_w_ = 0; + std::deque read_pipe_; + std::optional cur_read_; + std::deque aw_queue_; + std::deque write_pipe_; + std::optional cur_write_; +}; + +} // namespace vortex diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index ed85464742..9ece4088e8 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -133,7 +133,14 @@ SRCS = $(SW_COMMON_DIR)/util.cpp $(SIM_COMMON_DIR)/mem.cpp $(SW_COMMON_DIR)/soft SRCS += $(DPI_DIR)/dpi_util.cpp $(DPI_DIR)/dpi_float.cpp SRCS += $(SRC_DIR)/processor.cpp +# AXI_BUS=1 simulates Vortex behind its AXI4 adapter, as the FPGA platforms +# instantiate it, against a latency-bandwidth-pipe memory model. +ifdef AXI_BUS +TOP = rtlsim_axi_shim +CXXFLAGS += -DAXI_BUS +else TOP = rtlsim_shim +endif VL_FLAGS = --exe VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index 327a131bb3..ac4602049e 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -14,7 +14,14 @@ #include #include "processor.h" +#ifdef AXI_BUS +#include "Vrtlsim_axi_shim.h" +#include +typedef Vrtlsim_axi_shim Device; +#else #include "Vrtlsim_shim.h" +typedef Vrtlsim_shim Device; +#endif #ifdef VCD_OUTPUT #include @@ -44,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +60,23 @@ #include +// AXI_BUS memory timing; the defaults are FASED's. +#ifndef AXI_MEM_READ_LATENCY +#define AXI_MEM_READ_LATENCY 30 +#endif + +#ifndef AXI_MEM_WRITE_LATENCY +#define AXI_MEM_WRITE_LATENCY 30 +#endif + +#ifndef AXI_MEM_MAX_READS +#define AXI_MEM_MAX_READS 16 +#endif + +#ifndef AXI_MEM_MAX_WRITES +#define AXI_MEM_MAX_WRITES 16 +#endif + #ifndef TRACE_START_TIME #define TRACE_START_TIME 0ull #endif @@ -86,8 +111,10 @@ typedef uint64_t Word; using namespace vortex; +#ifndef AXI_BUS static uint64_t g_mem_done_seq = 0; static uint32_t g_mem_bank_addr_width = (VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS)); +#endif static uint64_t timestamp = 0; @@ -111,7 +138,11 @@ bool sim_trace_enabled() { class Processor::Impl { public: +#ifdef AXI_BUS + Impl() { +#else Impl() : dram_sim_(DramSim::platform_config()) { +#endif // restart sim time so a re-created model registers at time zero timestamp = 0; @@ -133,7 +164,7 @@ class Processor::Impl { context_->assertOn(false); // create device instance - device_ = new Vrtlsim_shim(context_); + device_ = new Device(context_); #ifdef VCD_OUTPUT context_->traceEverOn(true); @@ -265,6 +296,7 @@ class Processor::Impl { } } +#ifndef AXI_BUS void cp_mem_write(uint64_t addr, const uint8_t* src, uint64_t size, bool noalloc) { while (size != 0) { uint64_t offset = addr % CP_MEM_DATA_SIZE; @@ -318,16 +350,20 @@ class Processor::Impl { cp_rd_completed_ = 0; } } +#endif private: void reset() { this->mem_bus_reset(); this->dcr_bus_reset(); +#ifndef AXI_BUS this->cp_mem_reset(); +#endif print_bufs_.clear(); +#ifndef AXI_BUS for (auto& reqs : pending_mem_reqs_) { reqs.clear(); } @@ -336,6 +372,7 @@ class Processor::Impl { std::queue empty; std::swap(dram_queue_[b], empty); } +#endif device_->start = 0; device_->reset = 1; @@ -358,11 +395,32 @@ class Processor::Impl { this->eval(); } + #ifdef AXI_BUS + this->axi_drive(); + #else for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { device_->mem_req_ready[b] = 1; } + #endif } +#ifdef AXI_BUS + void tick() { + device_->clk = 0; + this->eval(); + + this->axi_sample(); + + device_->clk = 1; + this->eval(); + + this->axi_step(); + + #ifndef NDEBUG + fflush(stdout); + #endif + } +#else void tick() { device_->clk = 0; this->eval(); @@ -396,6 +454,7 @@ class Processor::Impl { fflush(stdout); #endif } +#endif void eval() { device_->eval(); @@ -412,6 +471,102 @@ class Processor::Impl { ++timestamp; } +#ifdef AXI_BUS + static constexpr uint32_t AXI_BEAT_SIZE = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + + void mem_bus_reset() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + AxiLatencyPipe::Config cfg; + cfg.read_latency = AXI_MEM_READ_LATENCY; + cfg.write_latency = AXI_MEM_WRITE_LATENCY; + cfg.max_reads = AXI_MEM_MAX_READS; + cfg.max_writes = AXI_MEM_MAX_WRITES; + axi_mem_[b] = AxiLatencyPipe(cfg); + axi_rd_data_[b].clear(); + axi_wr_addr_[b].clear(); + axi_in_[b] = AxiLatencyPipe::Inputs(); + } + this->axi_drive(); + } + + // Samples each port's request side before the clock edge and performs the + // accepted transactions on memory: writes first, then reads. + void axi_sample() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& mem = axi_mem_[b]; + auto& in = axi_in_[b]; + in.ar_valid = device_->m_axi_arvalid[b]; + in.ar_id = device_->m_axi_arid[b]; + in.ar_len = device_->m_axi_arlen[b]; + in.aw_valid = device_->m_axi_awvalid[b]; + in.aw_id = device_->m_axi_awid[b]; + in.w_valid = device_->m_axi_wvalid[b]; + in.w_last = device_->m_axi_wlast[b]; + in.r_ready = device_->m_axi_rready[b]; + in.b_ready = device_->m_axi_bready[b]; + + if (in.aw_valid && mem.aw_ready()) { + axi_wr_addr_[b].push_back(uint64_t(device_->m_axi_awaddr[b])); + } + if (in.w_valid && mem.w_ready()) { + // a write beat never runs ahead of its address on this port + assert(!axi_wr_addr_[b].empty()); + uint64_t addr = axi_wr_addr_[b].front(); + auto data = VDataCast::get(device_->m_axi_wdata[b]); + uint64_t strb = device_->m_axi_wstrb[b]; + for (uint32_t i = 0; i < AXI_BEAT_SIZE; ++i) { + if ((strb >> i) & 0x1) { + (*ram_)[addr + i] = data[i]; + } + } + if (in.w_last) { + axi_wr_addr_[b].pop_front(); + } else { + axi_wr_addr_[b].front() += AXI_BEAT_SIZE; + } + } + if (in.ar_valid && mem.ar_ready()) { + uint64_t addr = uint64_t(device_->m_axi_araddr[b]); + std::vector data((in.ar_len + 1) * AXI_BEAT_SIZE); + ram_->read(data.data(), addr, data.size()); + axi_rd_data_[b].push_back(std::move(data)); + } + axi_rlast_fire_[b] = mem.r_valid() && in.r_ready && mem.r_last(); + } + } + + void axi_step() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + if (axi_rlast_fire_[b]) { + axi_rd_data_[b].pop_front(); + } + axi_mem_[b].step(axi_in_[b]); + } + this->axi_drive(); + } + + void axi_drive() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& mem = axi_mem_[b]; + device_->m_axi_arready[b] = mem.ar_ready(); + device_->m_axi_awready[b] = mem.aw_ready(); + device_->m_axi_wready[b] = mem.w_ready(); + device_->m_axi_rvalid[b] = mem.r_valid(); + device_->m_axi_rresp[b] = 0; + if (mem.r_valid()) { + device_->m_axi_rid[b] = mem.r_id(); + device_->m_axi_rlast[b] = mem.r_last(); + memcpy(VDataCast::get(device_->m_axi_rdata[b]), + axi_rd_data_[b].front().data() + mem.r_beat() * AXI_BEAT_SIZE, AXI_BEAT_SIZE); + } + device_->m_axi_bvalid[b] = mem.b_valid(); + device_->m_axi_bresp[b] = 0; + if (mem.b_valid()) { + device_->m_axi_bid[b] = mem.b_id(); + } + } + } +#else void mem_bus_reset() { for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { device_->mem_req_ready[b] = 0; @@ -510,12 +665,14 @@ class Processor::Impl { } } } +#endif void dcr_bus_reset() { device_->dcr_req_valid = 0; device_->dcr_req_rw = 0; } +#ifndef AXI_BUS void cp_mem_reset() { cp_reqs_.clear(); cp_free_slots_.clear(); @@ -560,6 +717,7 @@ class Processor::Impl { device_->cp_mem_req_tag = req.tag; memcpy(VDataCast::get(device_->cp_mem_req_data), req.data.data(), CP_MEM_DATA_SIZE); } +#endif void wait(uint32_t cycles) { for (int i = 0; i < cycles; ++i) { @@ -569,8 +727,16 @@ class Processor::Impl { private: + std::unordered_map print_bufs_; + +#ifdef AXI_BUS + std::array axi_mem_; + std::array axi_in_; + std::array axi_rlast_fire_; + std::array>, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_rd_data_; + std::array, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_wr_addr_; +#else typedef struct { - Vrtlsim_shim* device; std::array data; uint64_t addr; uint64_t bank_addr; @@ -580,8 +746,6 @@ class Processor::Impl { uint64_t done_seq; } mem_req_t; - std::unordered_map print_bufs_; - typedef struct { std::array data; uint64_t addr; @@ -610,9 +774,10 @@ class Processor::Impl { std::array mem_rd_rsp_ready_; DramSim dram_sim_; +#endif VerilatedContext* context_; - Vrtlsim_shim* device_; + Device* device_; RAM* ram_; @@ -651,15 +816,30 @@ int Processor::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { } void Processor::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { +#ifdef AXI_BUS + // Vortex_axi ties the CP DMA port off, so cp_llc_client() is false + __unused(addr, src, size, noalloc); + std::abort(); +#else impl_->cp_mem_write(addr, static_cast(src), size, noalloc); +#endif } void Processor::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { +#ifdef AXI_BUS + __unused(addr, dst, size); + std::abort(); +#else impl_->cp_mem_read(addr, static_cast(dst), size); +#endif } bool Processor::cp_llc_client() const { +#ifdef AXI_BUS + return false; +#else return VX_CFG_CP_ON_LLC != 0; +#endif } uint64_t Processor::cp_llc_size() const { diff --git a/sim/rtlsim/rtlsim_axi_shim.sv b/sim/rtlsim/rtlsim_axi_shim.sv new file mode 100644 index 0000000000..b4dac97b34 --- /dev/null +++ b/sim/rtlsim/rtlsim_axi_shim.sv @@ -0,0 +1,144 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifndef PLATFORM_MEMORY_ID_WIDTH +`define PLATFORM_MEMORY_ID_WIDTH 32 +`endif + +// Vortex behind its AXI4 memory adapter (Vortex_axi), as the FPGA platforms +// see it, with the AXI channels a memory model needs. The parameters are the +// ones the platform wrappers pass, so the adapter is the same logic they run. +module rtlsim_axi_shim import VX_gpu_pkg::*; #( + parameter AXI_DATA_WIDTH = `VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8, + parameter AXI_ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH, + parameter AXI_TID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, + parameter AXI_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + output wire m_axi_awvalid [AXI_NUM_BANKS], + input wire m_axi_awready [AXI_NUM_BANKS], + output wire [AXI_ADDR_WIDTH-1:0] m_axi_awaddr [AXI_NUM_BANKS], + output wire [AXI_TID_WIDTH-1:0] m_axi_awid [AXI_NUM_BANKS], + output wire [7:0] m_axi_awlen [AXI_NUM_BANKS], + + output wire m_axi_wvalid [AXI_NUM_BANKS], + input wire m_axi_wready [AXI_NUM_BANKS], + output wire [AXI_DATA_WIDTH-1:0] m_axi_wdata [AXI_NUM_BANKS], + output wire [AXI_DATA_WIDTH/8-1:0] m_axi_wstrb [AXI_NUM_BANKS], + output wire m_axi_wlast [AXI_NUM_BANKS], + + input wire m_axi_bvalid [AXI_NUM_BANKS], + output wire m_axi_bready [AXI_NUM_BANKS], + input wire [AXI_TID_WIDTH-1:0] m_axi_bid [AXI_NUM_BANKS], + input wire [1:0] m_axi_bresp [AXI_NUM_BANKS], + + output wire m_axi_arvalid [AXI_NUM_BANKS], + input wire m_axi_arready [AXI_NUM_BANKS], + output wire [AXI_ADDR_WIDTH-1:0] m_axi_araddr [AXI_NUM_BANKS], + output wire [AXI_TID_WIDTH-1:0] m_axi_arid [AXI_NUM_BANKS], + output wire [7:0] m_axi_arlen [AXI_NUM_BANKS], + + input wire m_axi_rvalid [AXI_NUM_BANKS], + output wire m_axi_rready [AXI_NUM_BANKS], + input wire [AXI_DATA_WIDTH-1:0] m_axi_rdata [AXI_NUM_BANKS], + input wire m_axi_rlast [AXI_NUM_BANKS], + input wire [AXI_TID_WIDTH-1:0] m_axi_rid [AXI_NUM_BANKS], + input wire [1:0] m_axi_rresp [AXI_NUM_BANKS], + + input wire dcr_req_valid, + input wire dcr_req_rw, + input wire [VX_DCR_ADDR_WIDTH-1:0] dcr_req_addr, + input wire [VX_DCR_DATA_WIDTH-1:0] dcr_req_data, + + output wire dcr_rsp_valid, + output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + + input wire start, + output wire busy +); + `SCOPE_IO_SWITCH (1); + + Vortex_axi #( + .AXI_DATA_WIDTH (AXI_DATA_WIDTH), + .AXI_ADDR_WIDTH (AXI_ADDR_WIDTH), + .AXI_TID_WIDTH (AXI_TID_WIDTH), + .AXI_NUM_BANKS (AXI_NUM_BANKS) + ) vortex_axi ( + `SCOPE_IO_BIND (0) + + .clk (clk), + .reset (reset), + + .m_axi_awvalid (m_axi_awvalid), + .m_axi_awready (m_axi_awready), + .m_axi_awaddr (m_axi_awaddr), + .m_axi_awid (m_axi_awid), + .m_axi_awlen (m_axi_awlen), + `UNUSED_PIN (m_axi_awsize), + `UNUSED_PIN (m_axi_awburst), + `UNUSED_PIN (m_axi_awlock), + `UNUSED_PIN (m_axi_awcache), + `UNUSED_PIN (m_axi_awprot), + `UNUSED_PIN (m_axi_awqos), + `UNUSED_PIN (m_axi_awregion), + + .m_axi_wvalid (m_axi_wvalid), + .m_axi_wready (m_axi_wready), + .m_axi_wdata (m_axi_wdata), + .m_axi_wstrb (m_axi_wstrb), + .m_axi_wlast (m_axi_wlast), + + .m_axi_bvalid (m_axi_bvalid), + .m_axi_bready (m_axi_bready), + .m_axi_bid (m_axi_bid), + .m_axi_bresp (m_axi_bresp), + + .m_axi_arvalid (m_axi_arvalid), + .m_axi_arready (m_axi_arready), + .m_axi_araddr (m_axi_araddr), + .m_axi_arid (m_axi_arid), + .m_axi_arlen (m_axi_arlen), + `UNUSED_PIN (m_axi_arsize), + `UNUSED_PIN (m_axi_arburst), + `UNUSED_PIN (m_axi_arlock), + `UNUSED_PIN (m_axi_arcache), + `UNUSED_PIN (m_axi_arprot), + `UNUSED_PIN (m_axi_arqos), + `UNUSED_PIN (m_axi_arregion), + + .m_axi_rvalid (m_axi_rvalid), + .m_axi_rready (m_axi_rready), + .m_axi_rdata (m_axi_rdata), + .m_axi_rlast (m_axi_rlast), + .m_axi_rid (m_axi_rid), + .m_axi_rresp (m_axi_rresp), + + .dcr_req_valid (dcr_req_valid), + .dcr_req_rw (dcr_req_rw), + .dcr_req_addr (dcr_req_addr), + .dcr_req_data (dcr_req_data), + + .dcr_rsp_valid (dcr_rsp_valid), + .dcr_rsp_data (dcr_rsp_data), + + .start (start), + .busy (busy) + ); + +endmodule From ac1b20f459966fdc4ad04065f4053e62b0caf374 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 9 Oct 2026 10:36:49 -0700 Subject: [PATCH 174/336] firesim: clear generated source directories before regenerating them gen_sources.sh adds to its output directory without clearing it. A build directory reused across configurations (the metasimulator's emul directory is one per card) kept sources only the previous configuration needed, and the file list picked them up: after a TEX-enabled build, a 4-core build without TEX failed in sv2v with `could not find "TEX_STAGE_BITS" in package "VX_gpu_pkg"`. Both the Vivado and the simulation flattening now start from an empty directory. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 30630754d2747ba2e24531d088820b329e264a2e) --- hw/syn/firesim/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 8bbfb5d738..26c85b3567 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -369,6 +369,9 @@ RTL_SRCS := $(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh') vivado-sources: $(VIVADO_SRCS_LIST) $(VIVADO_SRCS_LIST): $(RTL_SRCS) @mkdir -p $(BUILD_DIR) + @# gen_sources.sh adds to its directory without clearing it, so a source the + @# previous configuration needed (a TEX file, say) would stay and be listed. + rm -rf $(VIVADO_SRCS_DIR) cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) \ -TVX_firesim_wrap -C$(VIVADO_SRCS_NAME) -O$(VIVADO_SRCS_NAME).f @for f in $(AFU_SHELL_MODULES); do \ @@ -403,6 +406,8 @@ SIM_RTL_SRCS := $(filter-out $(CONFIG_STAMP),$(RTL_SRCS)) $(SIM_CONFIG_STAMP) sim-sources: $(SIM_SRCS_STAMP) $(SIM_SRCS_STAMP): $(SIM_RTL_SRCS) @mkdir -p $(BUILD_DIR) + @# Cleared for the same reason as the Vivado flattening above. + rm -rf $(SRCS_DIR) cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(SIM_GEN_CFLAGS) \ -TVX_firesim_wrap -Cvortex_srcs -Ovortex_srcs_all.f @# The same zero-memory-port check the Vivado flattening makes: here it would From 0c27a52c4cd2e97ba5b02b442e22c37b9a56d1cd Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 9 Oct 2026 10:37:03 -0700 Subject: [PATCH 175/336] firesim: nested multi-class threading flow (maxsim) FAME5=1 threaded only VX_socket_top. The flow now keeps and annotates any set of module classes, nested, for Golden Gate's multi-class folding. - FAME5_CLASSES (default VX_socket_top): modules kept by the conversion and annotated for threading, including every __p parameterization. A classes.txt stamp reruns the conversion when the list changes. - FAME5_LIFT: fame5/lift_boundaries.py (pyslang) gives a unit with SV interface ports a plain-port wrapper __lift, rewriting its instantiation sites, since sv2v inlines any module with interface ports. Port directions come from modports or the drivers; per-instance parameters are dropped (drop=P, debug-only) or promoted to ports (port=P, recursively). Lifted sources go to $(FAME5_DIR)/lift; RTL is untouched. - fame5/name_kept.py: kept parameterized modules get public names (__p); Yosys write_firrtl otherwise names their instances by hdlname, which is "blackbox" for the RAM leaves. - to_firrtl.sh: split kept modules' ports (FAME5_SPLIT_KEPT_PORTS=0 to disable), so a unit reading back its own wide output is no combinational loop across the model boundary. - gen_rams.py: optional FAME5_NATIVE_RAMS (plain latency-1 read-first RAMs for Golden Gate's held threaded memories) and FAME5_FAMERAM (one FameRam black box per registered read-first RAM, fame5/FameRam.v, attached by fame5/annotate_blackboxes.py), so threaded RAMs stay BRAM/URAM-mappable. Validated on 4 cores (NW=NT=4): IW=1 with sockets + FMA lanes, and IW=4 with sockets + issue slice, ALU int/muldiv, LSU slice/scheduler and FPU classes; demo and sgemm cycle-exact against unthreaded builds, in metasimulation and on V80 silicon. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4908beecd2f217e40d60dafd970ac9a674556b0b) --- hw/syn/firesim/Makefile | 48 +- hw/syn/firesim/fame5/FameRam.v | 98 ++++ hw/syn/firesim/fame5/annotate_blackboxes.py | 23 + hw/syn/firesim/fame5/annotate_threads.py | 10 +- hw/syn/firesim/fame5/gen_rams.py | 51 +- hw/syn/firesim/fame5/lift_boundaries.py | 534 ++++++++++++++++++++ hw/syn/firesim/fame5/name_kept.py | 54 ++ hw/syn/firesim/fame5/to_firrtl.sh | 8 + 8 files changed, 814 insertions(+), 12 deletions(-) create mode 100644 hw/syn/firesim/fame5/FameRam.v create mode 100644 hw/syn/firesim/fame5/annotate_blackboxes.py create mode 100644 hw/syn/firesim/fame5/lift_boundaries.py create mode 100644 hw/syn/firesim/fame5/name_kept.py diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 26c85b3567..2269c8f737 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -529,13 +529,48 @@ FAME5_FIR := $(FAME5_DIR)/vortex.fir FAME5_SCRIPTS := $(wildcard $(SRC_DIR)/fame5/*) SV2V ?= sv2v -$(FAME5_V): $(VIVADO_SRCS_LIST) +# FAME5_LIFT: units given plain-port boundaries before sv2v (fame5/lift_boundaries.py, +# which needs pyslang: LIFT_PYTHON), so that interface-port units such as +# VX_issue_slice survive conversion and can be FAME5_CLASSES (as __lift). +# The RTL is not changed: the pass rewrites a copy of the staged sources. +# Each entry is [:drop=,...] (parameters that only label instances). +FAME5_LIFT ?= +LIFT_PYTHON ?= python3 +FAME5_LIFT_STAMP := $(FAME5_DIR)/lift.txt +FAME5_SRCS_LIST := $(if $(strip $(FAME5_LIFT)),$(FAME5_DIR)/lift/srcs.f,$(VIVADO_SRCS_LIST)) + +$(FAME5_LIFT_STAMP): FORCE @mkdir -p $(FAME5_DIR) - $(SV2V) $$(grep '^+define+' $(VIVADO_SRCS_LIST) | sed 's/^+define+/-D/') --top=VX_firesim_wrap \ - $$(grep -v '^+' $(VIVADO_SRCS_LIST)) -w $@ + @echo "$(FAME5_LIFT)" | cmp -s - $@ || echo "$(FAME5_LIFT)" > $@ -$(FAME5_FIR): $(FAME5_V) $(FAME5_SCRIPTS) - bash $(SRC_DIR)/fame5/to_firrtl.sh $(FAME5_V) VX_firesim_wrap $@ VX_socket_top VX_firesim_wrap \ +$(FAME5_DIR)/lift/srcs.f: $(VIVADO_SRCS_LIST) $(SRC_DIR)/fame5/lift_boundaries.py $(FAME5_LIFT_STAMP) + rm -rf $(FAME5_DIR)/lift + $(LIFT_PYTHON) $(SRC_DIR)/fame5/lift_boundaries.py --srcs $(VIVADO_SRCS_LIST) --top VX_firesim_wrap \ + --out $(FAME5_DIR)/lift $(foreach u,$(FAME5_LIFT),--unit '$(u)') + +$(FAME5_V): $(FAME5_SRCS_LIST) + @mkdir -p $(FAME5_DIR) + $(SV2V) $$(grep '^+define+' $(FAME5_SRCS_LIST) | sed 's/^+define+/-D/') --top=VX_firesim_wrap \ + $$(grep -v '^+' $(FAME5_SRCS_LIST)) -w $@ + +# FAME5_CLASSES: the modules threaded as classes, each kept as hierarchy by the +# conversion together with every parameterization of it. Units inside a socket +# (e.g. VX_fma_unit, the FPU lanes) become nested classes, extracted from every +# socket and threaded on their own; FAME5_FOLD (Golden Gate) sets how many +# physical copies each class gets. +FAME5_CLASSES ?= VX_socket_top +FAME5_KEEP := $(foreach c,$(FAME5_CLASSES),$(c) $$paramod*\$(c) $$paramod*\$(c)\*) +FAME5_CLASSES_STAMP := $(FAME5_DIR)/classes.txt + +# Rewritten only when the class list changes, so the conversion reruns then. +.PHONY: FORCE +FORCE: +$(FAME5_CLASSES_STAMP): FORCE + @mkdir -p $(FAME5_DIR) + @echo "$(FAME5_CLASSES)" | cmp -s - $@ || echo "$(FAME5_CLASSES)" > $@ + +$(FAME5_FIR): $(FAME5_V) $(FAME5_SCRIPTS) $(FAME5_CLASSES_STAMP) + bash $(SRC_DIR)/fame5/to_firrtl.sh $(FAME5_V) VX_firesim_wrap $@ '$(FAME5_KEEP)' VX_firesim_wrap \ "blackbox VX_dp_ram VX_sp_ram" # The generator's circuit names VX_firesim_wrap as an extmodule; it is replaced @@ -554,8 +589,9 @@ elaborate: stage $(FAME5_FIR) rm -f $(GEN_FIR) $(MAKE) -C $(FIRESIM_PATH)/sim $(GEN_FIR) $(FIRESIM_TARGET_ARGS) python3 $(SRC_DIR)/fame5/splice.py $(GEN_FIR) $(FAME5_FIR) + python3 $(SRC_DIR)/fame5/annotate_blackboxes.py $(GEN_FIR) $(GEN_ANNO) ifneq ($(FAME5_THREADS),0) - python3 $(SRC_DIR)/fame5/annotate_threads.py $(GEN_FIR) $(GEN_ANNO) VX_socket_top + for c in $(FAME5_CLASSES); do python3 $(SRC_DIR)/fame5/annotate_threads.py $(GEN_FIR) $(GEN_ANNO) $$c || exit 1; done endif $(MAKE) -C $(FIRESIM_PATH)/sim verilog $(FIRESIM_TARGET_ARGS) endif diff --git a/hw/syn/firesim/fame5/FameRam.v b/hw/syn/firesim/fame5/FameRam.v new file mode 100644 index 0000000000..743104e83c --- /dev/null +++ b/hw/syn/firesim/fame5/FameRam.v @@ -0,0 +1,98 @@ +// FameRam: the registered, read-first RAM of VX_dp_ram / VX_sp_ram (OUT_REG=1, +// RDW_MODE="R") as one memory primitive Golden Gate can thread. +// +// Semantics per thread, at its target clock edge (clk is the model's gated +// target clock): byte-lane writes of wdata under wren; when read, the output +// register takes ram[raddr] as it was before this edge's write; when not +// read, the output holds. +// +// Unthreaded (NTHREADS == 1) this is the plain inferred RAM. Threaded, the +// memory is NTHREADS times deeper with the thread index as its low address +// bits, so it stays one RAM (BRAM/URAM-mappable, byte-write enables intact) +// instead of one LUTRAM per byte lane. The shared output register is copied, +// one host cycle after the edge, into a per-thread buffer when that thread +// read; rdata is that thread's entry. threadIdx names the thread whose target +// edge is next (the write stage under pipelined multithreading). +module FameRam #( + parameter DATAW = 32, + parameter SIZE = 64, + parameter WRENW = 1, + parameter ADDRW = 6, + parameter NTHREADS = 1, + parameter TIDXW = 1, + parameter RAM_STYLE = "auto" +) ( + input wire clk, + input wire reset, + input wire read, + input wire write, + input wire [WRENW-1:0] wren, + input wire [ADDRW-1:0] waddr, + input wire [ADDRW-1:0] raddr, + input wire [DATAW-1:0] wdata, + output wire [DATAW-1:0] rdata, + input wire hostClock, + input wire hostReset, + input wire [15:0] threadIdx +); + localparam LANEW = DATAW / WRENW; + localparam DEPTH = SIZE * NTHREADS; + localparam TW = (NTHREADS > 1) ? TIDXW : 0; + + (* ram_style = RAM_STYLE *) reg [DATAW-1:0] ram [0:DEPTH-1]; + reg [DATAW-1:0] dout; + + wire [ADDRW+TW-1:0] wa; + wire [ADDRW+TW-1:0] ra; + generate + if (NTHREADS > 1) begin : g_taddr + assign wa = {waddr, threadIdx[TIDXW-1:0]}; + assign ra = {raddr, threadIdx[TIDXW-1:0]}; + end else begin : g_addr + assign wa = waddr; + assign ra = raddr; + end + endgenerate + + integer i; + always @(posedge clk) begin + for (i = 0; i < WRENW; i = i + 1) begin + if (write && wren[i]) + ram[wa][i*LANEW +: LANEW] <= wdata[i*LANEW +: LANEW]; + end + if (read) + dout <= ram[ra]; + end + + generate + if (NTHREADS > 1) begin : g_threaded + // Edge of the gated target clock, seen in the host domain. + reg tgl; + reg tgl_d; + reg rd_e; + reg [TIDXW-1:0] t_e; + always @(posedge clk) begin + tgl <= ~tgl; + rd_e <= read; + t_e <= threadIdx[TIDXW-1:0]; + end + always @(posedge hostClock) + tgl_d <= tgl; + wire edge_seen = tgl ^ tgl_d; + + (* ram_style = "distributed" *) reg [DATAW-1:0] rbuf [0:NTHREADS-1]; + always @(posedge hostClock) begin + if (edge_seen && rd_e) + rbuf[t_e] <= dout; + end + assign rdata = rbuf[threadIdx[TIDXW-1:0]]; + + initial begin + tgl = 1'b0; + tgl_d = 1'b0; + end + end else begin : g_plain + assign rdata = dout; + end + endgenerate +endmodule diff --git a/hw/syn/firesim/fame5/annotate_blackboxes.py b/hw/syn/firesim/fame5/annotate_blackboxes.py new file mode 100644 index 0000000000..f57e960429 --- /dev/null +++ b/hw/syn/firesim/fame5/annotate_blackboxes.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Point Golden Gate at the Verilog of the FIRRTL black boxes the flow emits. + +usage: annotate_blackboxes.py + +Adds a BlackBoxPathAnno for every FameRam extmodule (gen_rams with +FAME5_FAMERAM=1), so FameRam.v is copied next to the generated design and +appended to it, threaded or not. +""" +import json +import os +import re +import sys + +fir, anno = sys.argv[1], sys.argv[2] +src = open(fir).read() +circuit = re.search(r"^circuit (\S+?)\s*:", src, re.M).group(1) +names = re.findall(r"^ extmodule (\S+?)\s*:\n(?: .*\n)*? defname = FameRam$", src, re.M) +path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "FameRam.v") +annos = json.load(open(anno)) +annos += [{"class": "firrtl.transforms.BlackBoxPathAnno", "target": f"{circuit}.{n}", "path": path} for n in names] +json.dump(annos, open(anno, "w"), indent=1) +print(f"annotate_blackboxes: {len(names)} FameRam black boxes") diff --git a/hw/syn/firesim/fame5/annotate_threads.py b/hw/syn/firesim/fame5/annotate_threads.py index 052e1e4270..cdefc0d335 100644 --- a/hw/syn/firesim/fame5/annotate_threads.py +++ b/hw/syn/firesim/fame5/annotate_threads.py @@ -3,9 +3,11 @@ usage: annotate_threads.py -Finds each `inst of ` in and appends a -FirrtlEnableModelMultiThreadingAnnotation targeting it -- what Chisel's +Finds each `inst of ` in -- or of any +parameterization of it, `__p` (fame5/name_kept.py) -- and appends a +FirrtlEnableModelMultiThreadingAnnotation targeting it: what Chisel's EnableModelMultiThreadingAnnotation would emit, for designs with no Chisel. +Each parameterization is its own module, so it becomes its own threaded class. """ import json import re @@ -20,9 +22,9 @@ m = re.match(r" (?:ext)?module (\S+?)\s*:", line) if m: parent = m.group(1) - m = re.match(rf"\s+inst (\S+) of {re.escape(mod)}\b", line) + m = re.match(rf"\s+inst (\S+) of ({re.escape(mod)}(?:__p\d+)?)\s*(?:@|$)", line) if m: - targets.append(f"~{circuit}|{parent}/{m.group(1)}:{mod}") + targets.append(f"~{circuit}|{parent}/{m.group(1)}:{m.group(2)}") if not targets: sys.exit(f"annotate_threads: no instance of {mod}") annos = json.load(open(anno)) diff --git a/hw/syn/firesim/fame5/gen_rams.py b/hw/syn/firesim/fame5/gen_rams.py index dcc3944bb4..dd9f2f4386 100644 --- a/hw/syn/firesim/fame5/gen_rams.py +++ b/hw/syn/firesim/fame5/gen_rams.py @@ -23,6 +23,7 @@ behaviour, which FAME-5's threaded memories do not preserve. Every OUT_REG=1 memory has read latency 1, so it stays block-RAM-mappable after threading. """ +import os import re import sys @@ -71,10 +72,52 @@ def gen(name, prm, sp): if mode == "N": assert oreg, f"{name}: RDW_MODE N needs OUT_REG" mode = "N" + orig_mode = mode rden = "and(read, not(write))" if mode == "N" else "read" if mode == "N": mode = "R" readers = ["r", "r2"] if (oreg == 0 and mode == "R") else ["r"] + # FAME5_NATIVE_RAMS=1: a registered read-first RAM is a plain latency-1 + # memory with read-under-write = old, read enable = rden, and output hold + # left to the memory -- no write buffer, forwarding or hold registers. + # Requires a Golden Gate whose threaded memories hold per-thread read data + # while the read port is disabled. + # (No-change mode is excluded: its exclusive read/write enables would let + # InferReadWrite merge the ports into one read-write port.) + native_r = oreg and orig_mode == "R" and os.environ.get("FAME5_NATIVE_RAMS") == "1" + # FAME5_FAMERAM=1: the same RAM as one FameRam black box (FameRam.v), which + # Golden Gate threads as a single memory -- byte lanes kept together, so it + # stays BRAM/URAM-mappable instead of becoming one LUTRAM per lane. + if oreg and orig_mode == "R" and os.environ.get("FAME5_FAMERAM") == "1": + style = "block" if dw * size >= 4096 else "auto" + a(f" inst ram of FameRam_{name}") + a(" ram.clk <= clk") + a(" ram.reset <= reset") + a(" ram.read <= read") + a(" ram.write <= write") + a(" ram.wren <= wren") + a(" ram.waddr <= waddr") + a(" ram.raddr <= raddr") + a(" ram.wdata <= wdata") + a(" rdata <= ram.rdata") + # Host-side ports, tied off here; Golden Gate reconnects them when it + # threads the RAM. Every FameRam extmodule has the same port list. + a(" ram.hostClock <= clk") + a(" ram.hostReset <= UInt<1>(0)") + a(" ram.threadIdx <= UInt<16>(0)") + a("") + a(f" extmodule FameRam_{name}:") + for d, n, w in (("input", "clk", None), ("input", "reset", 1), ("input", "read", 1), ("input", "write", 1), + ("input", "wren", nw), ("input", "waddr", aw), ("input", "raddr", aw), + ("input", "wdata", dw), ("output", "rdata", dw), + ("input", "hostClock", None), ("input", "hostReset", 1), ("input", "threadIdx", 16)): + a(f" {d} {n}: {'Clock' if w is None else f'UInt<{w}>'}") + a(" defname = FameRam") + for k, v in (("DATAW", dw), ("SIZE", size), ("WRENW", nw), ("ADDRW", aw), ("NTHREADS", 1), ("TIDXW", 1)): + a(f" parameter {k} = {v}") + a(f' parameter RAM_STYLE = "{style}"') + a("") + return L a(" mem ram:") a(f" data-type => UInt<{lw}>[{nw}]") a(f" depth => {size}") @@ -83,11 +126,11 @@ def gen(name, prm, sp): for r in readers: a(f" reader => {r}") a(" writer => w") - a(" read-under-write => undefined") + a(f" read-under-write => {'old' if native_r else 'undefined'}") a("") for r in readers: a(f" ram.{r}.clk <= clk") - a(f" ram.{r}.en <= UInt<1>(1)") + a(f" ram.{r}.en <= {rden if native_r else 'UInt<1>(1)'}") a(" ram.w.clk <= clk") mask = (lambda src: [f"bits({src}, {i}, {i})" for i in range(nw)]) if nw > 1 else (lambda src: ["UInt<1>(1)"]) @@ -121,6 +164,10 @@ def merge(base, data, hit, wren): a(f" reg wren_r: UInt<{nw}>, clk") a(" wren_r <= wren") a(f" rdata <= {merge(mem_out('r'), 'wdata_r', 'hit_r', 'wren_r')}") + elif native_r: + a(" ram.r.addr <= raddr") + write_port("write", "waddr", "wdata", "wren") + a(f" rdata <= {mem_out('r')}") elif oreg and mode == "R": a(" reg wb_v: UInt<1>, clk") a(f" reg wb_addr: UInt<{aw}>, clk") diff --git a/hw/syn/firesim/fame5/lift_boundaries.py b/hw/syn/firesim/fame5/lift_boundaries.py new file mode 100644 index 0000000000..ab3931f0f9 --- /dev/null +++ b/hw/syn/firesim/fame5/lift_boundaries.py @@ -0,0 +1,534 @@ +#!/usr/bin/env python3 +"""Interface boundary lifting: make interface-port units FAME-5 fold classes. + +sv2v inlines every module that has SystemVerilog interface ports, so units such +as VX_issue_slice or VX_cluster never reach the FIRRTL as modules and cannot be +threaded. This pass gives each requested unit a boundary made of plain wires, +without touching the RTL: it rewrites a copy of the (preprocessed) sources. + +For each unit U: + * every elaborated instance must have the same parameter values (parameters + listed as droppable -- debug labels such as ISSUE_ID -- are left at their + defaults); + * a parameter-free wrapper U__lift is generated: each interface port becomes + one plain port per signal (per array element, packed), with the direction + taken from the port's modport or, for a bare interface port, from where + the signal is driven (inside the unit -> output); the wrapper instantiates + the interfaces with their elaborated parameters and U with its elaborated + parameter values; + * every instantiation site of U is replaced by glue wires/assigns between + the parent's interfaces and an instance of U__lift. +Behaviour is unchanged: the glue is pure wiring. + +Parameters that are functional instance IDs (e.g. VX_cluster's CLUSTER_ID) +are promoted to 32-bit input ports (port=P): the parameter becomes a port of U, +every child parameter override whose expression uses it is promoted the same +way down the hierarchy, and every affected instantiation passes the value as a +port connection instead (the old default where it was not overridden). Uses in +constant contexts (widths, generate conditions) make the re-elaboration fail, +so an invalid promotion cannot go through silently. + +usage: lift_boundaries.py --srcs --top --out + --unit U[:drop=P1,P2][;port=P3] [--unit ...] +""" +import argparse +import collections +import os +import re +import shutil +import sys + +import pyslang +from pyslang import ast, analysis + +SIG_KINDS = (ast.SymbolKind.Variable, ast.SymbolKind.Net) + + +def read_srcs(path): + defines, incdirs, files = [], [], [] + for line in open(path): + line = line.strip() + if line.startswith("+define+"): + defines.append(line[len("+define+"):]) + elif line.startswith("+incdir+"): + incdirs.append(line[len("+incdir+"):]) + elif line: + files.append(line) + return defines, incdirs, files + + +def q(a): + return "'" + a.replace("'", "'\\''") + "'" if any(c in a for c in " '\"") else a + + +def elaborate(defines, incdirs, files, top): + d = pyslang.driver.Driver() + d.addStandardArgs() + args = ["slang", "--top", top, "--allow-hierarchical-const", "--ignore-unknown-modules"] + args += [f"-D{x}" for x in defines] + [f"-I{x}" for x in incdirs] + files + if not (d.parseCommandLine(" ".join(q(a) for a in args), pyslang.driver.CommandLineOptions()) + and d.processOptions() and d.parseAllSources()): + sys.exit("lift: slang could not parse the sources") + comp = d.createCompilation() + used = set() + + def visit(sym): + if sym.kind == ast.SymbolKind.Instance: + used.add(sym.definition.name) + return ast.VisitAction.Advance + for t in comp.getRoot().topInstances: + t.visit(visit) + sm = comp.sourceManager + + def unused_only(fname): + # A file that only defines modules outside the elaborated hierarchy (as + # sv2v --top ignores them): errors there cannot affect the design. + names = re.findall(r"^\s*(module|interface|package)\s+(\w+)", open(fname).read(), re.M) + return bool(names) and all(k == "module" and n not in used for k, n in names) + errs = [x for x in comp.getAllDiagnostics() if x.isError() + and not unused_only(str(sm.getFileName(x.location)))] + if errs: + x = errs[0] + sys.exit(f"lift: {len(errs)} elaboration errors (first: {x.code} at " + f"{sm.getFileName(x.location)}:{sm.getLineNumber(x.location)})") + am = analysis.AnalysisManager() + am.analyze(comp) + return d, comp, am + + +def vec(w): + return "" if w == 1 else f"[{w - 1}:0] " + + +def lit(v): + s = str(v) + return s + + +class Unit: + def __init__(self, spec): + name, _, opts = spec.partition(":") + self.name = name + self.drop = set() + self.ports = [] # parameters promoted to input ports (functional instance IDs) + for o in filter(None, opts.split(";")): + k, _, v = o.partition("=") + if k == "drop": + self.drop |= set(filter(None, v.split(","))) + elif k == "port": + self.ports += [x for x in v.split(",") if x] + self.insts = [] + + +def params_of(inst, drop=()): + """Overridable value and type parameters of an elaborated instance, as SV text.""" + out = collections.OrderedDict() + for m in inst.body: + if m.name in drop: + continue + if m.kind == ast.SymbolKind.Parameter and not m.isLocalParam: + out[m.name] = lit(m.value) + elif m.kind == ast.SymbolKind.TypeParameter and not m.isLocalParam: + t = str(m.targetType.type) + if not re.match(r"^[A-Za-z_][\w$]*(::[A-Za-z_][\w$]*)*$", t) and not re.match(r"^(logic|bit)( ?\[\d+:\d+\])*$", t): + sys.exit(f"lift: type parameter {inst.hierarchicalPath}.{m.name} = '{t}' has no name to pass") + out[m.name] = t + return out + + +def iface_elements(conn): + target, modport = conn.ifaceConn + if target.kind == ast.SymbolKind.InstanceArray: + return list(target.elements), modport + return [target], modport + + +def signal_dirs(unit_inst, elem, modport_name, am): + """[(signal, 'input'|'output', width)] from the unit's point of view.""" + out = [] + mp = None + if modport_name: + mp = next((m for m in elem.body if m.kind == ast.SymbolKind.Modport and m.name == modport_name), None) + mp_dirs = {} + if mp is not None: + for p in mp: + mp_dirs[p.name] = "input" if str(p.direction).endswith("In") else "output" + upath = unit_inst.hierarchicalPath + for m in elem.body: + if m.kind not in SIG_KINDS: + continue + if m.type.isUnpackedArray: + sys.exit(f"lift: unpacked interface signal {elem.hierarchicalPath}.{m.name} not supported") + if mp is not None: + if m.name not in mp_dirs: + continue + d = mp_dirs[m.name] + else: + drv = am.getDrivers(m) + inside = any(dv.containingSymbol.hierarchicalPath == upath + or dv.containingSymbol.hierarchicalPath.startswith(upath + ".") + or dv.containingSymbol.hierarchicalPath.startswith(upath + "[") for dv in drv) + d = "output" if inside else "input" + out.append((m.name, d, m.type.bitWidth)) + return out + + +def port_range(port): + """Declared interface-array range as text ('[0:3]'), or '' for a single interface.""" + r = port.declaredRange + if not r: + return "", 1 + rr = r[0] + lo, hi = rr.left, rr.right + return f"[{lo}:{hi}]", abs(hi - lo) + 1 + + +def build_wrapper(u, rep, am): + ports, body, inner_conns = [], [], [] + shape = [] # per port: ('plain', name, dir, w) or ('iface', name, rng, n, [(sig, dir, w)]) + for c in rep.portConnections: + p = c.port + if p.kind == ast.SymbolKind.InterfacePort: + elems, mp = iface_elements(c) + rng, n = port_range(p) + sigs = signal_dirs(rep, elems[0], p.modport, am) + idef = p.interfaceDef.name + iparams = params_of(elems[0]) + pa = "" + if iparams: + pa = " #(" + ", ".join(f".{k}({v})" for k, v in iparams.items()) + ")" + body.append(f" {idef}{pa} {p.name} {rng}();".replace(" ();", "();")) + for s, d, w in sigs: + pn = f"{p.name}__{s}" + ports.append(f" {d} wire {('[' + str(n - 1) + ':0]') if rng else ''}{vec(w)}{pn}") + for k in range(n): + ref = f"{p.name}[{k}].{s}" if rng else f"{p.name}.{s}" + pk = f"{pn}[{k}]" if rng else pn + body.append(f" assign {ref} = {pk};" if d == "input" else f" assign {pk} = {ref};") + inner_conns.append(f" .{p.name} ({p.name})") + shape.append(("iface", p.name, rng, n, sigs)) + else: + d = str(p.direction).split(".")[-1].lower() + d = {"in": "input", "out": "output", "inout": "inout"}.get(d, d) + if p.type.isUnpackedArray: + sys.exit(f"lift: unpacked port {u.name}.{p.name} not supported") + w = p.type.bitWidth + ports.append(f" {d} wire {vec(w)}{p.name}") + inner_conns.append(f" .{p.name} ({p.name})") + shape.append(("plain", p.name, d, w)) + for pp in u.ports: + ports.insert(0, f" input wire [31:0] {pp}") + inner_conns.insert(0, f" .{pp} ({pp})") + shape.insert(0, ("promoted", pp)) + prm = params_of(rep, u.drop | set(u.ports)) + pa = (" #(\n" + ",\n".join(f" .{k} ({v})" for k, v in prm.items()) + "\n )") if prm else "" + text = (f"// Generated by lift_boundaries.py: {u.name} with plain ports (FAME-5 fold boundary).\n" + f"module {u.name}__lift (\n" + ",\n".join(ports) + "\n);\n" + "\n".join(body) + "\n\n" + f" {u.name}{pa} inner (\n" + ",\n".join(inner_conns) + "\n );\nendmodule\n") + return text, shape + + +SLICE = re.compile(r"^(.*)\[(.+)\+:(.+)\]$") + + +def elem_expr(e, k, is_array): + e = e.strip() + if not is_array: + return e + m = SLICE.match(e) + if m: + return f"{m.group(1)}[({m.group(2).strip()}) + {k}]" + if re.match(r"^[A-Za-z_]\w*$", e): + return f"{e}[{k}]" + if k == 0: + return e # a single element bound to a one-element array port + sys.exit(f"lift: cannot index connection expression '{e}'") + + +def rewrite_site(u, site_syntax, shape, uid): + """Replacement text for one instantiation statement.""" + inst = site_syntax.instances[0] + iname = str(inst.decl.name).strip() + conns = {} + for c in inst.connections: + if not hasattr(c, "name"): + continue # separator token + nm = str(c.name).strip() + ex = str(c.expr).strip() if c.expr is not None else "" + conns[nm] = ex + overrides = site_overrides(site_syntax) + pre = f"__lift{uid}" + decl, glue, pc = [], [], [] + for kind, *rest in shape: + if kind == "promoted": + pp = rest[0] + if pp not in overrides: + sys.exit(f"lift: {u.name} instance {iname}: promoted parameter {pp} is not overridden at the site") + pc.append(f" .{pp} ({overrides[pp]})") + continue + if kind == "plain": + name, d, w = rest + pc.append(f" .{name} ({conns.get(name, '')})") + continue + name, rng, n, sigs = rest + e = conns.get(name) + if e is None: + sys.exit(f"lift: {u.name} instance {iname}: interface port {name} not connected by name") + for s, d, w in sigs: + wn = f"{pre}_{name}__{s}" + decl.append(f"wire {('[' + str(n - 1) + ':0]') if rng else ''}{vec(w)}{wn};") + for k in range(n): + ek = elem_expr(e, k, bool(rng)) + wk = f"{wn}[{k}]" if rng else wn + glue.append(f"assign {wk} = {ek}.{s};" if d == "input" else f"assign {ek}.{s} = {wk};") + pc.append(f" .{name}__{s} ({wn})") + return ("// lift_boundaries: " + u.name + " -> " + u.name + "__lift\n " + + "\n ".join(decl + glue) + "\n " + + f"{u.name}__lift {iname} (\n" + ",\n".join(pc) + "\n );") + + +def site_overrides(site): + """{param: expr text} of a HierarchyInstantiationSyntax's #(...) named overrides.""" + out = {} + pva = site.parameters + if pva is None: + return out + for it in pva.parameters: + if hasattr(it, "name") and hasattr(it, "expr"): + out[str(it.name).strip()] = str(it.expr).strip() if it.expr is not None else "" + return out + + +def _remove_item(text_len_getter, node, sm): + """(start, end) covering a separated-list item and one adjacent comma.""" + return node.sourceRange.start.offset, node.sourceRange.end.offset + + +class Promoter: + """Parameter-to-port promotion across definitions (see module docstring).""" + + def __init__(self, comp, sm, lifted_sites): + self.comp, self.sm = comp, sm + self.lifted_sites = lifted_sites # site keys rewritten by lifting: skip + self.done = {} # (def, param) -> default text + self.edits = collections.defaultdict(list) + self.site_conv = collections.defaultdict(set) # site key -> {param} converted + self.insts = collections.defaultdict(list) + for t in comp.getRoot().topInstances: + t.visit(self._collect) + + def _collect(self, sym): + if sym.kind == ast.SymbolKind.Instance: + self.insts[sym.definition.name].append(sym) + return ast.VisitAction.Advance + + def key(self, node): + r = node.sourceRange + return (self.sm.getFileName(r.start), r.start.offset) + + def promote(self, defname, param): + if (defname, param) in self.done: + return + rep = self.insts[defname][0] + dsyn = rep.definition.syntax + hdr = dsyn.header + f = self.sm.getFileName(dsyn.sourceRange.start) + text = open(f).read() + default = None + target = None + for it in hdr.parameters.declarations: + if not hasattr(it, "declarators"): + continue + names = [str(dd.name).strip() for dd in it.declarators if hasattr(dd, "name")] + if param in names: + if len(names) != 1: + sys.exit(f"lift: {defname}.{param} shares a declaration; split it first") + dd = [dd for dd in it.declarators if hasattr(dd, "name")][0] + default = str(dd.initializer.expr).strip() if dd.initializer is not None else "0" + target = it + if target is None: + sys.exit(f"lift: no parameter {param} in {defname}") + self.done[(defname, param)] = default + a, b = target.sourceRange.start.offset, target.sourceRange.end.offset + self.edits[f].append(_with_comma(text, a, b)) + op = hdr.ports.openParen.location.offset + 1 + self.edits[f].append((op, op, f"\n input wire [31:0] {param},")) + # children whose overrides use the promoted value + for ch in rep.body: + pass + for cinst in self._children(rep): + site = cinst.syntax.parent + ov = site_overrides(site) + for q, ex in ov.items(): + if re.search(rf"\b{re.escape(param)}\b", ex): + self.promote(cinst.definition.name, q) + + def _children(self, inst): + out = [] + def v(sym): + if sym is inst: + return ast.VisitAction.Advance + if sym.kind == ast.SymbolKind.Instance: + out.append(sym) + return ast.VisitAction.Skip + return ast.VisitAction.Advance + inst.visit(v) + return out + + def convert_sites(self): + """Every instantiation of a promoted (def, param): override -> port connection.""" + for (defname, param), default in self.done.items(): + for inst in self.insts[defname]: + site = inst.syntax.parent + k = self.key(site) + if k in self.lifted_sites or param in self.site_conv[k]: + continue + self.site_conv[k].add(param) + f = k[0] + text = open(f).read() + ov_items = {} + if site.parameters is not None: + for it in site.parameters.parameters: + if hasattr(it, "name"): + ov_items[str(it.name).strip()] = it + if param in ov_items: + it = ov_items[param] + ex = str(it.expr).strip() + a, b = it.sourceRange.start.offset, it.sourceRange.end.offset + self.edits[f].append(_with_comma(text, a, b)) + else: + ex = default + inst_syn = site.instances[0] + op = inst_syn.openParen.location.offset + 1 + self.edits[f].append((op, op, f"\n .{param} ({ex}),")) + + +def _with_comma(text, a, b): + """Deletion range for a list item at [a, b) plus its separating comma.""" + j = b + while j < len(text) and text[j] in " \t\r\n": + j += 1 + if j < len(text) and text[j] == ",": + return (a, j + 1, "") + i = a - 1 + while i >= 0 and text[i] in " \t\r\n": + i -= 1 + if i >= 0 and text[i] == ",": + return (i, b, "") + return (a, b, "") + + +def apply_edits(text, edits): + """Apply (start, end, repl) edits; overlapping pure deletions are merged.""" + edits = sorted(edits, key=lambda e: (e[0], e[1])) + merged = [] + for s, e, t in edits: + if merged and s < merged[-1][1]: + ps, pe, pt = merged[-1] + if t or pt: + sys.exit(f"lift: overlapping edits at {s}") + merged[-1] = (ps, max(pe, e), "") + else: + merged.append((s, e, t)) + for s, e, t in reversed(merged): + text = text[:s] + t + text[e:] + return text + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--srcs", required=True) + ap.add_argument("--top", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--unit", action="append", required=True) + a = ap.parse_args() + a.out = os.path.abspath(a.out) + defines, incdirs, files = read_srcs(a.srcs) + d, comp, am = elaborate(defines, incdirs, files, a.top) + sm = comp.sourceManager + units = {u.name: u for u in (Unit(s) for s in a.unit)} + + def visit(sym): + if sym.kind == ast.SymbolKind.Instance and sym.definition.name in units: + units[sym.definition.name].insts.append(sym) + return ast.VisitAction.Advance + # Only the elaborated hierarchy: slang also builds placeholder instances of + # uninstantiated definitions, which must not count. + for t in comp.getRoot().topInstances: + t.visit(visit) + + edits = collections.defaultdict(list) # file -> [(start, end, text)] + wrappers = {} + uid = 0 + lifted_sites = set() + for u in units.values(): + if not u.insts: + sys.exit(f"lift: no instance of {u.name}") + skip = u.drop | set(u.ports) + groups = collections.Counter(tuple(params_of(i, skip).items()) for i in u.insts) + if len(groups) > 1: + keys = list(params_of(u.insts[0], skip).keys()) + diff = [k for k in keys if len({dict(g)[k] for g in groups}) > 1] + sys.exit(f"lift: instances of {u.name} differ in {diff}; drop them if they are labels") + text, shape = build_wrapper(u, u.insts[0], am) + wrappers[u.name] = text + seen = set() + for i in u.insts: + site = i.syntax.parent # HierarchyInstantiationSyntax + rng = site.sourceRange + key = (sm.getFileName(rng.start), rng.start.offset) + if key in seen: + continue + seen.add(key) + lifted_sites.add(key) + if len(site.instances) != 1: + sys.exit(f"lift: {u.name}: multi-instance statement at {key}") + edits[key[0]].append((rng.start.offset, rng.end.offset, rewrite_site(u, site, shape, uid))) + uid += 1 + print(f"lift: {u.name}: {len(u.insts)} instances, {len(seen)} sites, " + f"{sum(1 for s in shape if s[0] == 'iface')} interface ports, " + f"{sum(len(s[4]) for s in shape if s[0] == 'iface')} lifted signals") + + pr = Promoter(comp, sm, lifted_sites) + for u in units.values(): + for pp in u.ports: + pr.promote(u.name, pp) + pr.convert_sites() + for f, es in pr.edits.items(): + edits[f].extend(es) + if pr.done: + print("lift: promoted " + ", ".join(f"{d}.{p}" for d, p in pr.done)) + + os.makedirs(os.path.join(a.out, "src"), exist_ok=True) + out_files = [] + by_real = {os.path.realpath(str(k)): k for k in edits} + applied, matched = 0, set() + for f in files: + dst = os.path.join(a.out, "src", os.path.basename(f)) + key = by_real.get(os.path.realpath(f)) + if key is not None: + open(dst, "w").write(apply_edits(open(f).read(), edits[key])) + applied += 1 + matched.add(key) + else: + shutil.copyfile(f, dst) + out_files.append(dst) + for name, text in wrappers.items(): + p = os.path.join(a.out, "src", f"{name}__lift.sv") + open(p, "w").write(text) + out_files.append(p) + with open(os.path.join(a.out, "srcs.f"), "w") as o: + for x in defines: + o.write(f"+define+{x}\n") + for x in incdirs: + o.write(f"+incdir+{x}\n") + o.write(f"+incdir+{os.path.join(a.out, 'src')}\n") + for x in out_files: + o.write(x + "\n") + print(f"lift: wrote {len(out_files)} files ({applied} of {len(edits)} rewritten) to {a.out}") + if applied != len(edits): + sys.exit(f"lift: edits for unmatched files: {sorted(set(edits) - matched)}") + + +if __name__ == "__main__": + main() diff --git a/hw/syn/firesim/fame5/name_kept.py b/hw/syn/firesim/fame5/name_kept.py new file mode 100644 index 0000000000..a991cc9b35 --- /dev/null +++ b/hw/syn/firesim/fame5/name_kept.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Give kept parameterized modules public names in an RTLIL design. + +usage: name_kept.py (rewritten in place) + +After flattening, the only modules left are the top, the kept units (FAME-5 +classes) and the black boxes. A kept unit that was instantiated with +parameters is named $paramod...\\, and Yosys write_firrtl names its +instances by the cell's hdlname attribute rather than the module, so they +would not refer to the module it emits. Renaming each non-blackbox $paramod +module to \\__p (module and every cell of that type) makes the +writer use the module's own name. +""" + +import re +import sys + +il = sys.argv[1] +text = open(il).read() + +mods = [] +attrs = [] +for line in text.splitlines(): + s = line.strip() + if s.startswith("attribute "): + attrs.append(s) + continue + if s.startswith("module "): + name = s.split(None, 1)[1] + if name.startswith("$paramod") and not any(a.startswith("attribute \\blackbox") for a in attrs): + mods.append(name) + attrs = [] + +if mods: + ren = {} + count = {} + for m in mods: + base = re.sub(r"[^A-Za-z0-9_]", "_", m.split("\\")[1]) if "\\" in m else "mod" + n = count.get(base, 0) + count[base] = n + 1 + ren[m] = f"\\{base}__p{n}" + out = [] + for line in text.splitlines(keepends=True): + s = line.lstrip() + if s.startswith("module ") or s.startswith("cell "): + parts = line.split(" ") + for i, p in enumerate(parts): + key = p.rstrip("\n") + if key in ren: + parts[i] = ren[key] + ("\n" if p.endswith("\n") else "") + line = " ".join(parts) + out.append(line) + open(il, "w").write("".join(out)) +print(f"name_kept: {len(mods)} kept parameterized modules renamed") diff --git a/hw/syn/firesim/fame5/to_firrtl.sh b/hw/syn/firesim/fame5/to_firrtl.sh index 98916834ff..8d07c0091f 100755 --- a/hw/syn/firesim/fame5/to_firrtl.sh +++ b/hw/syn/firesim/fame5/to_firrtl.sh @@ -16,6 +16,12 @@ W=$(dirname "$(realpath -m "$OUT")")/$(basename "$OUT" .fir).work rm -rf "$W" && mkdir -p "$W" REN="" [ "$NEWTOP" != "$TOP" ] && REN="rename $TOP $NEWTOP" +SPLITKEPT="splitnets -ports -driver A:keep_hierarchy; opt_clean -purge" +[ "${FAME5_SPLIT_KEPT_PORTS:-1}" = 0 ] && SPLITKEPT="" +# Kept modules' ports are split at driver boundaries, as the top's are: a unit +# that computes some bits of a wide output from other bits of it (an LSU +# request built from its own address field) would otherwise read its own +# output port, which FIRRTL's word-level loop check rejects. $Y -q -l "$W/y.log" -p " read_verilog -sv $IN $PRE @@ -26,9 +32,11 @@ $Y -q -l "$W/y.log" -p " memory -nomap; memory_nordff; opt -full; opt -nosdff -nodffe dffunmap; opt_clean -purge splitnets -driver; opt_clean -purge + $SPLITKEPT splitnets $TOP; opt_clean -purge $REN write_rtlil $W/design.il" || { tail -5 "$W/y.log"; exit 1; } +python3 "$(dirname "$0")/name_kept.py" "$W/design.il" # FIRRTL memories carry one enable and one uniform mask per write port, so a # handful of RTL arrays - read-modify-write DCR files, shift queues, per-field # struct writes - collect into \$mem cells write_firrtl rejects ('Complex From 59abe5f676d86abf7d22132a105fe764c0972ebe Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 9 Oct 2026 10:37:08 -0700 Subject: [PATCH 176/336] tests: run-firesim for the HIP and OpenCL suites The regression suite already runs on FireSim (tests/regression/common.mk); HIP and OpenCL tests had no run-firesim target. They take the same card selection (sw/runtime/firesim/card.mk) and the same image conventions: xrtsim/avedsim run the metasimulator in-process, aved TARGET=hw runs FPGA_BIN_DIR's firesim.vbin after the V80 preflight, XRT runs firesim.xclbin. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8bd3ce1667badc50f373fe93ce2f5f3f26db0feb) --- tests/hip/common.mk | 27 ++++++++++++++++++++++++++- tests/opencl/common.mk | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/hip/common.mk b/tests/hip/common.mk index 77c80baa6e..07b530afe1 100644 --- a/tests/hip/common.mk +++ b/tests/hip/common.mk @@ -169,9 +169,34 @@ else $(HIP_OCL_ENV) SCOPE_JSON_PATH=$(VORTEX_RT_LIB)/scope.json LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) VORTEX_DRIVER=xrt ./$(PROJECT) $(OPTS) endif +# FireSim takes the same targets as the card's native driver; the card and its +# variables are decided in sw/runtime/firesim/card.mk (convention established +# in tests/regression/common.mk). +ifneq ($(filter run-firesim,$(MAKECMDGOALS)),) +ifeq ($(origin TARGET),file) +override TARGET := +endif +include $(VORTEX_HOME)/sw/runtime/firesim/card.mk +endif + +run-firesim: $(PROJECT) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) +ifneq ($(filter $(TARGET),xrtsim avedsim),) + $(HIP_OCL_ENV) LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(FIRESIM_PLATFORM), aved) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh +endif + $(HIP_OCL_ENV) VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/firesim.vbin LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(TARGET), hw) + $(HIP_OCL_ENV) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else + $(HIP_OCL_ENV) XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +endif + clean: rm -f $(PROJECT) *.o *.vxbin *.dump *.ll *.log *.spv common.h -.PHONY: all run-simx run-rtlsim run-opae run-xrt clean +.PHONY: all run-simx run-rtlsim run-opae run-xrt run-firesim clean FORCE: diff --git a/tests/opencl/common.mk b/tests/opencl/common.mk index 84774d3e2d..39c3823cfd 100644 --- a/tests/opencl/common.mk +++ b/tests/opencl/common.mk @@ -152,6 +152,31 @@ else SCOPE_JSON_PATH=$(VORTEX_RT_LIB)/scope.json LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=xrt ./$(PROJECT) $(OPTS) endif +# FireSim takes the same targets as the card's native driver; the card and its +# variables are decided in sw/runtime/firesim/card.mk (convention established +# in tests/regression/common.mk). +ifneq ($(filter run-firesim,$(MAKECMDGOALS)),) +ifeq ($(origin TARGET),file) +override TARGET := +endif +include $(VORTEX_HOME)/sw/runtime/firesim/card.mk +endif + +run-firesim: $(PROJECT) $(KERNEL_SRCS) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) +ifneq ($(filter $(TARGET),xrtsim avedsim),) + LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(FIRESIM_PLATFORM), aved) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh +endif + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/firesim.vbin LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(TARGET), hw) + XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else + XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +endif + .depend: $(SRCS) $(CXX) $(CXXFLAGS) -MM $^ > .depend; From a8a75d439a38b851536689385560efe287e1cae9 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 9 Oct 2026 10:37:08 -0700 Subject: [PATCH 177/336] firesim: V80 simulator clock default 250 MHz Other cards keep 60 MHz. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 078d86f235608fd4374dafb0a60bb41469a56540) --- hw/syn/firesim/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 2269c8f737..8028b6d8ae 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -456,7 +456,12 @@ kernel-env: @echo 'export VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl' @echo 'export TOOL_DIR=$(SCRIPT_DIR)' +# Simulator clock (MHz): the image is timed at it and the shell programs it. +ifeq ($(FIRESIM_PLATFORM), aved) +FREQUENCY ?= 250.0 +else FREQUENCY ?= 60.0 +endif STRATEGY ?= TIMING # How the card's FireSim platform spells this TARGET, and where it leaves the From 9ed83c37fed8eabe6be93ec31f2a2c59118ea529 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 9 Oct 2026 17:05:53 -0700 Subject: [PATCH 178/336] firesim: lift_boundaries accepts use-before-declare VX_tcu_core references wmma_sp_meta before declaring it, which Vivado and Verilator accept and slang rejects by default (UsedBeforeDeclared), so any TCU configuration failed boundary lifting. Elaborate with --allow-use-before-declare, as the other tools effectively do. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b74133a76a5bb33659f58f6def448c8026defbba) --- hw/syn/firesim/fame5/lift_boundaries.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hw/syn/firesim/fame5/lift_boundaries.py b/hw/syn/firesim/fame5/lift_boundaries.py index ab3931f0f9..8f7de58149 100644 --- a/hw/syn/firesim/fame5/lift_boundaries.py +++ b/hw/syn/firesim/fame5/lift_boundaries.py @@ -64,7 +64,10 @@ def q(a): def elaborate(defines, incdirs, files, top): d = pyslang.driver.Driver() d.addStandardArgs() - args = ["slang", "--top", top, "--allow-hierarchical-const", "--ignore-unknown-modules"] + # The Vortex RTL references some nets before their declaration (VX_tcu_core), + # which Vivado and Verilator accept; slang makes it an error unless allowed. + args = ["slang", "--top", top, "--allow-hierarchical-const", "--allow-use-before-declare", + "--ignore-unknown-modules"] args += [f"-D{x}" for x in defines] + [f"-I{x}" for x in incdirs] + files if not (d.parseCommandLine(" ".join(q(a) for a in args), pyslang.driver.CommandLineOptions()) and d.processOptions() and d.parseAllSources()): From 836bc8ed88697be278767759865e15ae76f349a8 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 9 Oct 2026 17:05:53 -0700 Subject: [PATCH 179/336] firesim: split kept-module wires that drive their own bits Yosys reuses a bit of a wire to drive other bits of it; in a TCU build the write-enable vectors of tcu_unit.agu's response memory replicate one enable bit. That is acyclic per bit, but FIRRTL checks loops per wire, so Golden Gate rejected VX_socket_top with four "Combinational loop detected" errors. The top is already split to single bits wholesale; inside kept modules, fame5/selfref_wires.py now tags just the wires connected to their own bits and `splitnets a:fame5_selfref` splits those, leaving everything else as is. Validated on 4 cores NT=NW=16 IW=4 with A, TCU (TFR fp16 sparse wgmma), DXA and L2: demo passes in metasimulation, cycle-exact against Verilator. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 9080a292151bc958d7d92fafe162cc5ce9e69aa6) --- hw/syn/firesim/fame5/selfref_wires.py | 42 +++++++++++++++++++++++++++ hw/syn/firesim/fame5/to_firrtl.sh | 8 +++++ 2 files changed, 50 insertions(+) create mode 100644 hw/syn/firesim/fame5/selfref_wires.py diff --git a/hw/syn/firesim/fame5/selfref_wires.py b/hw/syn/firesim/fame5/selfref_wires.py new file mode 100644 index 0000000000..1e27f401c6 --- /dev/null +++ b/hw/syn/firesim/fame5/selfref_wires.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Tag the wires whose bits are driven by other bits of the same wire. + +usage: selfref_wires.py (rewritten in place; prints the count) + +Yosys reuses a bit of a wire to drive other bits of it (a write-enable vector +built by replicating one enable bit, say). That is acyclic bit by bit, but +FIRRTL checks loops per wire, so the exported module fails the combinational +loop check. Each such wire gets the attribute fame5_selfref, for +`splitnets a:fame5_selfref` to split it into single bits; nothing else is +touched. +""" + +import re +import sys + +il = sys.argv[1] +lines = open(il).read().splitlines(keepends=True) + +selfref = set() # (module, wire) +mod = None +for line in lines: + if line.startswith("module "): + mod = line.split()[1] + elif line.startswith(" connect "): + m = re.match(r" connect (\\\S+|\$\S+)(?: \[[0-9:]+\])?\s+(.*)", line) + if m and m.group(1) in re.findall(r"(\\\S+|\$\S+)", m.group(2)): + selfref.add((mod, m.group(1))) + +if selfref: + out = [] + mod = None + for line in lines: + if line.startswith("module "): + mod = line.split()[1] + elif line.startswith(" wire "): + name = line.split()[-1] + if (mod, name) in selfref: + out.append(" attribute \\fame5_selfref 1\n") + out.append(line) + open(il, "w").write("".join(out)) +print(len(selfref)) diff --git a/hw/syn/firesim/fame5/to_firrtl.sh b/hw/syn/firesim/fame5/to_firrtl.sh index 8d07c0091f..64e5a7d48c 100755 --- a/hw/syn/firesim/fame5/to_firrtl.sh +++ b/hw/syn/firesim/fame5/to_firrtl.sh @@ -37,6 +37,14 @@ $Y -q -l "$W/y.log" -p " $REN write_rtlil $W/design.il" || { tail -5 "$W/y.log"; exit 1; } python3 "$(dirname "$0")/name_kept.py" "$W/design.il" +# Inside kept modules (only the top is split wholesale) the same bit reuse +# shows up as a wire feeding its own bits; split just those wires. +if [ "$(python3 "$(dirname "$0")/selfref_wires.py" "$W/design.il")" != 0 ]; then + $Y -q -l "$W/y_selfref.log" -p " + read_rtlil $W/design.il + splitnets a:fame5_selfref; opt_clean -purge + write_rtlil $W/design.il" || { tail -5 "$W/y_selfref.log"; exit 1; } +fi # FIRRTL memories carry one enable and one uniform mask per write port, so a # handful of RTL arrays - read-modify-write DCR files, shift queues, per-field # struct writes - collect into \$mem cells write_firrtl rejects ('Complex From 417cba4a50fc2db3ce48564ae50b60f8ea6561fd Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 9 Oct 2026 18:00:53 -0700 Subject: [PATCH 180/336] firesim: V80 simulator clock default 143 MHz, the measured Fmax A build targeted at 250 MHz (4 cores, NW=NT=4, IW=4, sockets plus six threaded unit classes) routed with WNS -2.974 ns at 4.000 ns: a 6.974 ns critical path, 143.4 MHz. The default is now that measured figure. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 219e82e9521495fb4ec7ef575c2d3e031ab182e4) --- hw/syn/firesim/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 8028b6d8ae..21656c50a0 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -457,8 +457,10 @@ kernel-env: @echo 'export TOOL_DIR=$(SCRIPT_DIR)' # Simulator clock (MHz): the image is timed at it and the shell programs it. +# V80: the Fmax a 250 MHz-targeted build reached (4 cores, NW=NT=4, IW=4, seven +# threaded classes: routed WNS -2.974 ns at 4 ns, i.e. 143.4 MHz). ifeq ($(FIRESIM_PLATFORM), aved) -FREQUENCY ?= 250.0 +FREQUENCY ?= 143.0 else FREQUENCY ?= 60.0 endif From 9344816698cbcfdbdbc1a26af90fc6e62c86d73c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 11 Oct 2026 20:06:30 -0700 Subject: [PATCH 181/336] maxsim: threaded FPGA simulation of Vortex on the V80, compiled by CIRCT Every class of identical units is time-multiplexed through one physical copy, cycle-exact against the RTL, with no Chisel, Golden Gate, Yosys or sv2v anywhere in the flow: - hw/syn/maxsim: circt-verilog imports Vortex; maxsim-extract-models cuts each level into its own logic and the class instances; maxsim-compose threads the classes and joins every model through one token fabric; firtool writes it out; the V80's native IP packaging and slashkit link build the image. lift_boundaries.py gives interface-port units plain ports and turns instance IDs into ports on the flow's copy of the sources, so Vortex's RTL is not changed; the copy's async-read RAM patch (placeholders for a Vivado hook) is replaced by the RAM it stands for. The flow builds the STD FPU and the RTU without vendor cores. - hw/rtl/afu/maxsim: the kernel. maxsim_mem times the AXI4 port as AxiLatencyPipe does, sending requests to HBM at once; maxsim_ctrl is the AXI4-Lite register block and the target's control endpoint (command queue for DCR writes and reads, start, reset; cycle, memory-wait and per-class counters). Registers avoid 0x40-0xBF, which the V80 shell's MSI-X table answers. - sw/runtime/maxsim (--driver=maxsim): the software command processor over VRT registers and QDMA on the card, or over the same kernel in Verilator (sim/maxsim, TARGET=avedsim). V80, s2w4i2 (2 cores, NW=NT=4, IW=2): demo, sgemm and sgemm n=128 PASS with cycle counts identical to Verilator; 138k routed LUTs (FireSim image 219k), 149.8 MHz; 8 host cycles per target cycle; sgemm n=128 in 196 ms, 50x SimX. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 60b965f28a27d6970c62238fd958897fddd4309b) Integration (phase 2): the linker-config emitter reads the descriptor's single memory tag (phase 2's descriptor schema); CHANGELOG keeps both entries. --- CHANGELOG.md | 3 + ci/blackbox.sh | 4 +- docs/assets/img/maxsim_architecture.svg | 105 +++++ docs/designs/maxsim.md | 240 +++++++++++ docs/simulation.md | 40 +- hw/rtl/afu/maxsim/maxsim_afu.sv | 271 ++++++++++++ hw/rtl/afu/maxsim/maxsim_ctrl.sv | 288 +++++++++++++ hw/rtl/afu/maxsim/maxsim_mem.sv | 395 +++++++++++++++++ hw/rtl/afu/maxsim/maxsim_regs.vh | 57 +++ hw/rtl/afu/maxsim/maxsim_token.sv | 76 ++++ hw/scripts/gen_platform.py | 20 +- hw/syn/maxsim/Makefile | 164 +++++++ hw/syn/maxsim/VX_async_ram_patch.sv | 76 ++++ hw/syn/maxsim/lift_boundaries.py | 540 ++++++++++++++++++++++++ hw/syn/maxsim/maxsim.py | 145 +++++++ hw/syn/xilinx/aved/package_ip.tcl | 27 +- sim/maxsim/Makefile | 53 +++ sim/maxsim/maxsim_emul.cpp | 285 +++++++++++++ sim/maxsim/maxsim_emul.h | 45 ++ sw/common/maxsim_regs.h | 55 +++ sw/runtime/maxsim/Makefile | 69 +++ sw/runtime/maxsim/vortex.cpp | 537 +++++++++++++++++++++++ tests/regression/common.mk | 11 + 23 files changed, 3486 insertions(+), 20 deletions(-) create mode 100644 docs/assets/img/maxsim_architecture.svg create mode 100644 docs/designs/maxsim.md create mode 100644 hw/rtl/afu/maxsim/maxsim_afu.sv create mode 100644 hw/rtl/afu/maxsim/maxsim_ctrl.sv create mode 100644 hw/rtl/afu/maxsim/maxsim_mem.sv create mode 100644 hw/rtl/afu/maxsim/maxsim_regs.vh create mode 100644 hw/rtl/afu/maxsim/maxsim_token.sv create mode 100644 hw/syn/maxsim/Makefile create mode 100644 hw/syn/maxsim/VX_async_ram_patch.sv create mode 100644 hw/syn/maxsim/lift_boundaries.py create mode 100644 hw/syn/maxsim/maxsim.py create mode 100644 sim/maxsim/Makefile create mode 100644 sim/maxsim/maxsim_emul.cpp create mode 100644 sim/maxsim/maxsim_emul.h create mode 100644 sw/common/maxsim_regs.h create mode 100644 sw/runtime/maxsim/Makefile create mode 100644 sw/runtime/maxsim/vortex.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index f2e579169a..8950a3d61f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, of at least half the LLC write around it. A launch behind DMA writes waits for an L1-only flush token sent on the slot. The runtime sees the new `DMA_LLC_CLIENT` CP capability. *Why:* rtlsim at 2 clusters + L2 + L3: dogfood -30%, multikernel -42.7%; dogfood -64% with an 8 MB L3. +- **maxsim: threaded FPGA simulation of Vortex on the Alveo V80, compiled by CIRCT.** Every class of identical units (sockets, issue slices, ALUs, FPUs, load-store units) is time-multiplexed through one physical copy, cycle-exact against the RTL, with no Chisel, Golden Gate, Yosys or sv2v: `circt-verilog` imports Vortex, CIRCT's `maxsim-extract-models` / `maxsim-compose` cut it into threaded models joined by one token fabric ([hw/syn/maxsim/maxsim.py](hw/syn/maxsim/maxsim.py)), and the V80's native flow packages it with a latency-bandwidth memory endpoint and a command-queue control block ([hw/rtl/afu/maxsim/](hw/rtl/afu/maxsim/)). `--driver=maxsim` runs it on the card (`TARGET=hw`, VRT + QDMA) or in-process in Verilator (`avedsim`, the default). Vortex's RTL is unchanged: units get plain-port boundaries and instance IDs as ports from a source-to-source pass on the flow's copy ([hw/syn/maxsim/lift_boundaries.py](hw/syn/maxsim/lift_boundaries.py)). See [docs/designs/maxsim.md](docs/designs/maxsim.md). + *Why:* the simulated GPU is no longer bounded by the card's size, and the shell around it is ~8k LUTs instead of FireSim's ~93k — s2w4i2 (2 cores, NW=NT=4, IW=2) routes in 138k LUTs vs 219k and runs sgemm n=128 in 196 ms on V80: 50× SimX on the same host, 1.7× the FireSim image, cycle counts identical to Verilator. + *Limits:* V80 only; one memory port, single-beat transactions; classes are named (`MAXSIM_CLASSES`), not found; kernel register offsets 0x40–0xBF are unusable on the V80 shell (its PCIe MSI-X table answers them). ### Changed diff --git a/ci/blackbox.sh b/ci/blackbox.sh index 6ae40c9cb7..d0e9fa32c6 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -26,7 +26,7 @@ show_help() { show_usage echo " where" - echo "--driver: gpu, simx, rtlsim, opae, xrt, aved, firesim" + echo "--driver: gpu, simx, rtlsim, opae, xrt, aved, firesim, maxsim" echo "--target: xrt: xrtsim (default), hw_emu, hw -- aved: avedsim (default), sim, hw" echo " firesim: the targets of the card's native driver; xrtsim (default) or" echo " avedsim need no image, the others run FPGA_BIN_DIR's (hw/syn/firesim)" @@ -98,7 +98,7 @@ parse_args() { set_driver_path() { case $DRIVER in gpu) DRIVER_PATH="" ;; - simx|rtlsim|opae|xrt|aved|firesim) DRIVER_PATH="$ROOT_DIR/sw/runtime/$DRIVER" ;; + simx|rtlsim|opae|xrt|aved|firesim|maxsim) DRIVER_PATH="$ROOT_DIR/sw/runtime/$DRIVER" ;; *) echo "Invalid driver: $DRIVER"; exit 1 ;; esac } diff --git a/docs/assets/img/maxsim_architecture.svg b/docs/assets/img/maxsim_architecture.svg new file mode 100644 index 0000000000..7ba7d389bf --- /dev/null +++ b/docs/assets/img/maxsim_architecture.svg @@ -0,0 +1,105 @@ + + + + + + + + + + Compile: one IR from SystemVerilog to the kernel (hw/syn/maxsim/maxsim.py) + + circt-verilog + Vortex, flat but for + the classes + + maxsim-extract-models + each level a netlist of its own + logic and the class instances + + model cleanups + share-adders, low-bits + narrowing, const-prop + + maxsim-compose + classes threaded (FAME-5), + channels in one token fabric + + firtool, Vivado + maxsim_sim.sv, packaged + and linked as on the V80 + + + + + + Run: the kernel (hw/rtl/afu/maxsim) and its host + + maxsim_afu + + + maxsim_sim (generated) + every channel: two slots per producer thread, a count per reader + + rest of Vortex + 1 thread + + socket + 1 copy, N threads + + issue, ALU, LSU, FPU + 1 copy each, N x M threads + + + + token fabric: registers, or memories read at the reader's thread + a thread finishes its target cycle once each input has a token and each output was taken; + state in LUTRAM per thread, written with the clock enable of that finish + + + maxsim_ctrl + AXI4-Lite registers + command queue: DCR, + start, reset; busy, + cycle counters + + maxsim_mem + latency-bandwidth pipe + in target cycles; + requests to HBM at once, + a cycle waits for data + + + Endpoints offer and take tokens with valid and ready from their own state, so they connect to the + simulator's channels directly. The target runs free: one token per channel per target cycle. + Shown for one memory port and Vortex's DCR, start and busy (the target VX_firesim_wrap). + + + host: --driver=maxsim + the runtime's command + processor model, in software; + VRT on the V80, or the kernel + in Verilator (avedsim) + + HBM + target address + base; + host DMA by QDMA + + AXI-Lite + + AXI4 + + QDMA + diff --git a/docs/designs/maxsim.md b/docs/designs/maxsim.md new file mode 100644 index 0000000000..d343cf4d96 --- /dev/null +++ b/docs/designs/maxsim.md @@ -0,0 +1,240 @@ +# maxsim — threaded FPGA simulation of Vortex, compiled by CIRCT + +maxsim runs Vortex on the Alveo V80 as a host-decoupled, cycle-exact +simulator in which every class of identical units — sockets, issue slices, +ALUs, FPUs, load-store units — is time-multiplexed through **one physical +copy** (FAME-5 model multithreading). A GPU many times the card's size fits +on it, and runs orders of magnitude faster than Verilator. + +| § | contents | +|---|---| +| [1](#1-overview) | Overview and how to run it | +| [2](#2-the-compiler) | The compiler: netlists, threading, composition | +| [3](#3-the-simulator) | The simulator: threads, channels, the token fabric | +| [4](#4-the-kernel) | The kernel: memory and control endpoints | +| [5](#5-the-runtime) | The runtime and its register contract | +| [6](#6-validation) | Validation | +| [7](#7-measured-results) | Measured results | +| [8](#8-invariants-and-limitations) | Invariants and limitations | + +--- + +## 1. Overview + +![maxsim architecture](../assets/img/maxsim_architecture.svg) + +The whole flow is one IR from SystemVerilog to Verilog: `circt-verilog` +imports Vortex, a few CIRCT passes cut it into models and compose the +simulator, `firtool` writes it out, and the card's native flow (IP-XACT +packaging, `slashkit link`) builds the image. There is no Chisel, Golden +Gate, Yosys or sv2v anywhere in it, and no FireSim checkout. + +```bash +export MAXSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift VX_lsu_scheduler__lift VX_fpu_std" +export MAXSIM_LIFT="VX_socket:drop=INSTANCE_ID;port=SOCKET_ID VX_issue_slice:drop=ISSUE_ID VX_alu_int:port=BLOCK_IDX VX_alu_muldiv VX_lsu_slice:port=CORE_ID VX_lsu_scheduler" +CONFIGS="-DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_ISSUE_WIDTH=2" +CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=maxsim --app=sgemm # Verilator +make -C hw/syn/maxsim CONFIGS="$CONFIGS" # the image +FPGA_BIN_DIR=$PWD/hw/syn/maxsim/build32_maxsim_hw/bin CONFIGS="$CONFIGS" \ + ./ci/blackbox.sh --driver=maxsim --target=hw --app=sgemm # the V80 +``` + +`MAXSIM_CLASSES` names the modules threaded as classes; a class may contain +others (the socket contains the issue slices). `MAXSIM_LIFT` gives +interface-port units a plain-port boundary first, so they can be classes +(`hw/syn/maxsim/lift_boundaries.py`, a source-to-source pass on pyslang; the +RTL is not changed). `port=P` turns an instance-ID parameter into an input +port, and every parameter derived from it below, so that all instances of the +class elaborate to one module: the socket's `SOCKET_ID` carries every core's +`CORE_ID` with it. The tools are found through `CIRCT_BIN`. + +The flow reads Vortex as source, so it builds the configuration whose units +are source: the STD FPU (`-DVX_CFG_FPU_TYPE_STD`) and the RTU without vendor +cores. Vivado's asynchronous-read RAM patch (placeholder cells a +post-synthesis hook rewires) is replaced in the flow's copy of the sources by +the RAM it stands for (`hw/syn/maxsim/VX_async_ram_patch.sv`). + +--- + +## 2. The compiler + +`hw/syn/maxsim/maxsim.py` drives four steps, all in one MLIR module of the +HW, Comb and Seq dialects: + +1. **Import and flatten.** `circt-verilog` elaborates the target + (`VX_firesim_wrap`: Vortex with its AXI4 port, DCR, start and busy as plain + ports) with `--detect-memories`; everything but the class modules is + inlined. +2. **Cut netlists, innermost first** (`maxsim-extract-models{netlist}`). A + module with class instances becomes `_model` — its own logic, with + ports for what it exchanges with the instances — and `` becomes a + netlist of the same ports, instantiating `_model` beside the class + instances, wired as before. Instances of the module elsewhere stay valid, + so the socket's netlist is cut out of the top in turn. Bits of a model + output that are always the same constant stay in the parent, and an input + driven only by the parent's inputs and constants, or exactly by another + model's output, is wired to those directly. +3. **Clean up the models**: `seq-reg-const-prop`, `comb-share-adders` + (`mux(c, a+b, a-b)` → one adder), `comb-low-bits-narrowing` (adders whose + operands have known-zero low bits), canonicalization. These are what a + threaded model's area is sensitive to; ExportVerilog emits sign-extended + multiplies as `$signed` so Vivado maps them to single DSPs. +4. **Compose** (`maxsim-compose`). The top netlist's nested netlists are + inlined, every instance of a model becomes one thread of that model's + single threaded copy (§3), and every wire between them a channel of the + token fabric. The result is `maxsim_sim`: `hostClock`, `hostReset`, a + channel per target port (`_valid`, `_ready`, `_bits`) and a + `profile_finish` bit per class. + +`maxsim.py` also writes `maxsim_sim.json` (the ports, and each class's thread +count) and `maxsim_sim.vh` (the class count, for the kernel). + +--- + +## 3. The simulator + +**Threads.** A model of N instances is one module that simulates them in turn +(`maxsim-fame5`'s threader): every register becomes an N-entry memory read at +the current thread, every memory N times deeper. A thread finishes its +target cycle when each input channel has its token and each output's token +was taken; its state is then written, on the host clock with that finish as +the enable — no gated clocks. Outputs that depend on no input fire eagerly, +the next cycle's value as soon as the thread finishes. The thread index +advances every host cycle and is a port of the module, so the fabric reads it +rather than keeping its own counter. + +**Channels.** Every output of a model, and every input of the target, is a +*producer*; every input of a model, and every output of the target, a +*consumer* that reads bits of one or more producers, for each of its threads. +Each producer thread has two token slots and a count of tokens written; each +reader of it a count of tokens read. A producer is ready while every reader +of the thread has read all but one of its tokens; a consumer's token is valid +when each producer it reads has one for it. This is the two-entry, no +flow-through channel of FireSim's LI-BDN simulators, with fan-out, so the +target's behaviour does not depend on the host's schedule. + +**Storage.** A producer read only by threaded consumers that read the same +bits of it for every thread keeps its slots in a memory addressed by the +reader's thread and slot (LUTRAM); any other, in registers. A target port +that is exactly one port of a model of one thread is wired to the shell +directly, since the shell's valid and ready depend on its own state alone. + +--- + +## 4. The kernel + +`hw/rtl/afu/maxsim/maxsim_afu.sv` has the native V80 kernel's ports but for +the host-memory master: `s_axi_control` and `m_axi_mem_0` (linked to the +`MEM` HBM aperture). It instantiates `maxsim_sim` and two endpoints, both +models of the target's surroundings that take one token per channel per +target cycle (`maxsim_token.sv`): + +- **`maxsim_mem`** is the target's AXI4 port, timed as a latency-bandwidth + pipe — cycle for cycle `sim/common/axi_latency_pipe.h`: a fixed latency per + transaction, bounded reads and writes in flight, one beat per cycle, + responses in request order. A request goes to HBM (`base` + target address) + the cycle the target makes it; a target cycle that should present a + response memory has not returned yet does not finish until it has, and a + write is acknowledged to the target only after HBM acknowledged it. +- **`maxsim_ctrl`** is the AXI4-Lite register block and the endpoint of the + target's reset, DCR, start and busy. The target runs free: the endpoint + gives it a token every target cycle. Host commands are queued and each is + applied for one target cycle (a reset for its count). Kernel reset holds the + target in reset for 64 cycles. + +--- + +## 5. The runtime + +`sw/runtime/maxsim/vortex.cpp` (`--driver=maxsim`) is a transport under the +runtime's software command-processor model, as the FireSim backend is: the +model's DCR writes and reads, start and busy become commands and status reads +of the control block, and its device-memory accesses DMA at the target's +addresses. `TARGET=hw` uses VRT — the kernel's registers through +`vrt::Kernel`, device memory through a QDMA queue — and `TARGET=avedsim` +the same kernel in Verilator (`sim/maxsim`), with a sparse device memory and +each register access costing the kernel cycles a PCIe access would. + +The register map is `sw/common/maxsim_regs.h` (`hw/rtl/afu/maxsim/maxsim_regs.vh` +for the RTL): + +| offset | register | +|---|---| +| `0x00`, `0x04` | ID `"MXSM"`, version | +| `0x08` | control: bit 0 the target runs | +| `0x0C` | status: busy (anything queued, running, or in flight), queue full, DCR response ready, memory error | +| `0x10`, `0x14`, `0x18` | command address, data, and the command: DCR write, DCR read, start, reset | +| `0x1C` | the last DCR read's response | +| `0x20`–`0x2C` | target cycles, kernel-clock cycles | +| `0x30`, `0x38` | kernel-clock cycles a target cycle waited for HBM; the number of classes | +| `0xC0`–`0xD4` | memory model: device base, read and write latency, most reads and writes in flight | +| `0x100 + 8c` | per class, threads finished | + +`0x40`–`0xBF` is empty: on the V80 the PCIe block keeps its MSI-X table at +that offset of the BAR the first kernel is mapped at, and answers those +accesses itself (§8). The runtime refuses an image whose `VERSION` differs +from its own map. + +`busy` covers queued commands, a start the target has not yet answered with +busy, and transactions the memory model has in flight, so a host that sees +the target idle has every effect of what it asked for. + +--- + +## 6. Validation + +- **Token co-simulation** (`maxsim_sim` against the RTL it was compiled from, + in Verilator): random target inputs, every channel stalled at random, every + output token compared. s2w4i2: 3,000 target cycles, 51,000 tokens, 0 + differences. +- **The memory endpoint against `AxiLatencyPipe`**, in Verilator: random + requests and device-memory latency, every target cycle's outputs and every + read's data compared; five latency and limit settings, 0 differences. +- **Applications** in the in-process simulator and on the V80 (§7). +- **CIRCT lit tests** for every pass (`test/Transforms/maxsim-*.mlir`). + +--- + +## 7. Measured results + +Configuration s2w4i2 (2 cores, 4 warps, 4 threads, issue width 2), seven +classes: one rest-of-Vortex model, the socket (2 threads) and six unit +classes (4 threads each). + +| | where | demo cycles | sgemm cycles | sgemm n=128 cycles | host / target cycle | +|---|---|---:|---:|---:|---:| +| maxsim, threads in turn, in-process | Verilator | 8,702 | 415,454 | | 12.06 | +| maxsim, ready scheduling, in-process | Verilator | 8,702 | 415,454 | | 8.00 | +| maxsim, threads in turn | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 or 12.00 | +| maxsim, ready scheduling | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 | +| rtlsim, AXI4 with `AxiLatencyPipe` | Verilator | 8,710 | | | | + +On the V80 the kernel closes at 149.8 MHz (run at 143 MHz) in 138k routed +LUTs, against 219k for the FireSim image of the same configuration. +sgemm n=128 takes 196 ms with ready scheduling: 50× SimX on the same host, +1.7× the FireSim image. + +Instruction counts equal the FireSim images' and rtlsim's. Cycle counts are +1.5–6 % below the FireSim images', which modelled memory with FASED: its +latency-bandwidth pipe has the same settings, but its channel mechanics add +cycles `AxiLatencyPipe` does not; maxsim agrees with `AxiLatencyPipe` +(rtlsim) instead. + +--- + +## 8. Invariants and limitations + +1. **Valid and ready of an endpoint depend on its state alone.** The + simulator wires target ports to the shell directly; a combinational path + from a channel's ready to its valid would close a loop. +2. **Every model ticks on its first finish.** A threaded model's target clock + is enabled from the first finish after reset, so token k of every channel + is target cycle k. +3. **The memory endpoint takes single-beat transactions**, as Vortex issues + them; a burst sets the status register's memory error. +4. **One memory port**: the target is built with one platform memory bank. +5. **Classes are named, not found**: `MAXSIM_CLASSES` lists them. +6. **No kernel register in `0x40`–`0xBF`.** The V80 shell configures the + PCIe block's MSI-X table internally, at offset `0x40` of the BAR whose + offset 0 is the first kernel; reads there return the table's entries and + writes stick in it, without reaching the kernel. diff --git a/docs/simulation.md b/docs/simulation.md index 3238deb02a..cde48cf8d6 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -83,6 +83,44 @@ One caveat worth knowing when reading its output: FireSim's kernel took, and is quantized by the driver's polling schedule. Use Vortex's own `PERF: cycles=` line, which is exact. +### Threaded FPGA Simulation (maxsim) + +maxsim compiles Vortex into a simulator for the V80 with CIRCT: each class of +identical units (sockets, issue slices, ALUs, ...) is threaded through one +physical copy, so a GPU far larger than the card fits on it, still cycle-exact +against the RTL. Its kernel carries a small control block and a +latency-bandwidth memory model; the target runs free, and the runtime talks to +it like the native V80 driver does (AXI4-Lite registers, QDMA for memory). + +| target | what runs | image built by | +|---|---|---| +| `avedsim` (default) | the same kernel in Verilator, in-process | nothing to build | +| `hw` | the image on the V80 | `make -C hw/syn/maxsim` | + +```bash +# the classes to thread, and the units to give plain-port boundaries first +export MAXSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift VX_lsu_scheduler__lift VX_fpu_std" +export MAXSIM_LIFT="VX_socket:drop=INSTANCE_ID;port=SOCKET_ID VX_issue_slice:drop=ISSUE_ID VX_alu_int:port=BLOCK_IDX VX_alu_muldiv VX_lsu_slice:port=CORE_ID VX_lsu_scheduler" +CONFIGS="-DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_ISSUE_WIDTH=2" + +# in Verilator, no image needed +CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=maxsim --app=sgemm + +# build the image, then run on the V80 +make -C hw/syn/maxsim CONFIGS="$CONFIGS" +FPGA_BIN_DIR=$PWD/hw/syn/maxsim/build32_maxsim_hw/bin \ + CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=maxsim --target=hw --app=sgemm +``` + +The flow needs `circt-verilog`, `circt-opt` and `firtool` from a CIRCT build +with the maxsim passes (`CIRCT_BIN=`, or `PATH`), and pyslang for the +boundary lifting (`LIFT_PYTHON`). The memory model's settings are runtime +knobs: `MAXSIM_MEM_READ_LATENCY`, `MAXSIM_MEM_WRITE_LATENCY` (target cycles, +30 by default) and `MAXSIM_MEM_MAX_READS`, `MAXSIM_MEM_MAX_WRITES` (16). At +exit the runtime reports the target and kernel-clock cycles the simulator ran +and how busy each threaded class was. The design is +[designs/maxsim.md](designs/maxsim.md). + ### How to Test (using `blackbox.sh`) Running tests under specific drivers (rtlsim,simx,fpga) is done using the script named `blackbox.sh` located in the `ci` folder. Running command `./ci/blackbox.sh --help` from the Vortex root directory will display the following command line arguments for `blackbox.sh`: @@ -93,7 +131,7 @@ Running tests under specific drivers (rtlsim,simx,fpga) is done using the script - *Threads* - used to specify the number of threads (smallest unit of computation) within a configuration. - *L2cache* - used to enable the shared l2cache among the Vortex cores. - *L3cache* - used to enable the shared l3cache among the Vortex clusters. -- *Driver* - used to specify which driver to run the Vortex simulation (either rtlsim, opae, xrt, simx, firesim). +- *Driver* - used to specify which driver to run the Vortex simulation (either rtlsim, opae, xrt, simx, firesim, maxsim). - *Debug* - used to enable debug mode for the Vortex simulation. - *Perf* - used to enable the detailed performance counters within the Vortex simulation. - *App* - used to specify which test/benchmark to run in the Vortex simulation. The main choices are vecadd, sgemm, basic, demo, and dogfood. Other tests/benchmarks are located in the `/benchmarks/opencl` folder though not all of them work wit the current version of Vortex. diff --git a/hw/rtl/afu/maxsim/maxsim_afu.sv b/hw/rtl/afu/maxsim/maxsim_afu.sv new file mode 100644 index 0000000000..bf67ce4ac9 --- /dev/null +++ b/hw/rtl/afu/maxsim/maxsim_afu.sv @@ -0,0 +1,271 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "maxsim_sim.vh" + +// The maxsim kernel: the simulator CIRCT composes of Vortex (maxsim_sim, of +// the target VX_firesim_wrap), the memory endpoint on its AXI4 port and the +// control endpoint on its DCR, start and busy signals. Its ports are the +// native V80 kernel's, but for the host-memory master it does not need. +module maxsim_afu #( + parameter C_S_AXI_CONTROL_ADDR_WIDTH = 16, + parameter C_S_AXI_CONTROL_DATA_WIDTH = 32, + parameter C_M_AXI_MEM_ID_WIDTH = 1, + parameter C_M_AXI_MEM_DATA_WIDTH = 512, + parameter C_M_AXI_MEM_ADDR_WIDTH = 64, + // The target's AXI4 port. + parameter TARGET_ID_W = 4, + parameter TARGET_ADDR_W = 64, + parameter DCR_ADDR_W = 12, + parameter DCR_DATA_W = 32 +) ( + input wire ap_clk, + input wire ap_rst_n, + + output wire m_axi_mem_0_awvalid, + input wire m_axi_mem_0_awready, + output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_0_awaddr, + output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_0_awid, + output wire [7:0] m_axi_mem_0_awlen, + output wire m_axi_mem_0_wvalid, + input wire m_axi_mem_0_wready, + output wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_0_wdata, + output wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_0_wstrb, + output wire m_axi_mem_0_wlast, + output wire m_axi_mem_0_arvalid, + input wire m_axi_mem_0_arready, + output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_0_araddr, + output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_0_arid, + output wire [7:0] m_axi_mem_0_arlen, + input wire m_axi_mem_0_rvalid, + output wire m_axi_mem_0_rready, + input wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_0_rdata, + input wire m_axi_mem_0_rlast, + input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_0_rid, + input wire [1:0] m_axi_mem_0_rresp, + input wire m_axi_mem_0_bvalid, + output wire m_axi_mem_0_bready, + input wire [1:0] m_axi_mem_0_bresp, + input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_0_bid, + + input wire s_axi_control_awvalid, + output wire s_axi_control_awready, + input wire [C_S_AXI_CONTROL_ADDR_WIDTH-1:0] s_axi_control_awaddr, + input wire s_axi_control_wvalid, + output wire s_axi_control_wready, + input wire [C_S_AXI_CONTROL_DATA_WIDTH-1:0] s_axi_control_wdata, + input wire [C_S_AXI_CONTROL_DATA_WIDTH/8-1:0] s_axi_control_wstrb, + input wire s_axi_control_arvalid, + output wire s_axi_control_arready, + input wire [C_S_AXI_CONTROL_ADDR_WIDTH-1:0] s_axi_control_araddr, + output wire s_axi_control_rvalid, + input wire s_axi_control_rready, + output wire [C_S_AXI_CONTROL_DATA_WIDTH-1:0] s_axi_control_rdata, + output wire [1:0] s_axi_control_rresp, + output wire s_axi_control_bvalid, + input wire s_axi_control_bready, + output wire [1:0] s_axi_control_bresp +); + localparam DATA_W = C_M_AXI_MEM_DATA_WIDTH; + + // A registered reset, so it reaches the whole kernel in one cycle's time. + reg reset; + always @(posedge ap_clk) + reset <= ~ap_rst_n; + + // ---- The simulator's channels, named after the target's ports ---- +`define MAXSIM_CH(name, w) \ + wire name``_valid, name``_ready; \ + wire [w-1:0] name``_bits; + + `MAXSIM_CH(reset, 1) + `MAXSIM_CH(m_axi_mem_0_awready, 1) + `MAXSIM_CH(m_axi_mem_0_wready, 1) + `MAXSIM_CH(m_axi_mem_0_arready, 1) + `MAXSIM_CH(m_axi_mem_0_rvalid, 1) + `MAXSIM_CH(m_axi_mem_0_rdata, DATA_W) + `MAXSIM_CH(m_axi_mem_0_rlast, 1) + `MAXSIM_CH(m_axi_mem_0_rid, TARGET_ID_W) + `MAXSIM_CH(m_axi_mem_0_rresp, 2) + `MAXSIM_CH(m_axi_mem_0_bvalid, 1) + `MAXSIM_CH(m_axi_mem_0_bresp, 2) + `MAXSIM_CH(m_axi_mem_0_bid, TARGET_ID_W) + `MAXSIM_CH(dcr_req_valid, 1) + `MAXSIM_CH(dcr_req_rw, 1) + `MAXSIM_CH(dcr_req_addr, DCR_ADDR_W) + `MAXSIM_CH(dcr_req_data, DCR_DATA_W) + `MAXSIM_CH(start, 1) + `MAXSIM_CH(m_axi_mem_0_awvalid, 1) + `MAXSIM_CH(m_axi_mem_0_awaddr, TARGET_ADDR_W) + `MAXSIM_CH(m_axi_mem_0_awid, TARGET_ID_W) + `MAXSIM_CH(m_axi_mem_0_awlen, 8) + `MAXSIM_CH(m_axi_mem_0_wvalid, 1) + `MAXSIM_CH(m_axi_mem_0_wdata, DATA_W) + `MAXSIM_CH(m_axi_mem_0_wstrb, DATA_W/8) + `MAXSIM_CH(m_axi_mem_0_wlast, 1) + `MAXSIM_CH(m_axi_mem_0_arvalid, 1) + `MAXSIM_CH(m_axi_mem_0_araddr, TARGET_ADDR_W) + `MAXSIM_CH(m_axi_mem_0_arid, TARGET_ID_W) + `MAXSIM_CH(m_axi_mem_0_arlen, 8) + `MAXSIM_CH(m_axi_mem_0_rready, 1) + `MAXSIM_CH(m_axi_mem_0_bready, 1) + `MAXSIM_CH(dcr_rsp_valid, 1) + `MAXSIM_CH(dcr_rsp_data, DCR_DATA_W) + `MAXSIM_CH(busy, 1) +`undef MAXSIM_CH + +`define MAXSIM_IN(name) \ + .name``_valid (name``_valid), .name``_ready (name``_ready), .name``_bits (name``_bits) + + wire [`MAXSIM_NUM_CLASSES-1:0] profile_finish /*verilator public_flat_rd*/; + + maxsim_sim sim ( + .hostClock (ap_clk), + .hostReset (reset), + .profile_finish (profile_finish), + `MAXSIM_IN(reset), + `MAXSIM_IN(m_axi_mem_0_awready), + `MAXSIM_IN(m_axi_mem_0_wready), + `MAXSIM_IN(m_axi_mem_0_arready), + `MAXSIM_IN(m_axi_mem_0_rvalid), + `MAXSIM_IN(m_axi_mem_0_rdata), + `MAXSIM_IN(m_axi_mem_0_rlast), + `MAXSIM_IN(m_axi_mem_0_rid), + `MAXSIM_IN(m_axi_mem_0_rresp), + `MAXSIM_IN(m_axi_mem_0_bvalid), + `MAXSIM_IN(m_axi_mem_0_bresp), + `MAXSIM_IN(m_axi_mem_0_bid), + `MAXSIM_IN(dcr_req_valid), + `MAXSIM_IN(dcr_req_rw), + `MAXSIM_IN(dcr_req_addr), + `MAXSIM_IN(dcr_req_data), + `MAXSIM_IN(start), + `MAXSIM_IN(m_axi_mem_0_awvalid), + `MAXSIM_IN(m_axi_mem_0_awaddr), + `MAXSIM_IN(m_axi_mem_0_awid), + `MAXSIM_IN(m_axi_mem_0_awlen), + `MAXSIM_IN(m_axi_mem_0_wvalid), + `MAXSIM_IN(m_axi_mem_0_wdata), + `MAXSIM_IN(m_axi_mem_0_wstrb), + `MAXSIM_IN(m_axi_mem_0_wlast), + `MAXSIM_IN(m_axi_mem_0_arvalid), + `MAXSIM_IN(m_axi_mem_0_araddr), + `MAXSIM_IN(m_axi_mem_0_arid), + `MAXSIM_IN(m_axi_mem_0_arlen), + `MAXSIM_IN(m_axi_mem_0_rready), + `MAXSIM_IN(m_axi_mem_0_bready), + `MAXSIM_IN(dcr_rsp_valid), + `MAXSIM_IN(dcr_rsp_data), + `MAXSIM_IN(busy) + ); + +`define MAXSIM_MEM(name) \ + .name``_valid (m_axi_mem_0_``name``_valid), .name``_ready (m_axi_mem_0_``name``_ready), \ + .name``_bits (m_axi_mem_0_``name``_bits) + + wire [31:0] mem_read_latency, mem_write_latency; + wire [7:0] mem_max_reads, mem_max_writes; + wire [63:0] mem_base; + wire mem_idle, mem_error, mem_stall; + + maxsim_mem #( + .ID_W (TARGET_ID_W), + .ADDR_W (TARGET_ADDR_W), + .DATA_W (DATA_W) + ) mem ( + .clk (ap_clk), + .reset (reset), + .read_latency (mem_read_latency), + .write_latency (mem_write_latency), + .max_reads (mem_max_reads), + .max_writes (mem_max_writes), + .base (mem_base), + .idle (mem_idle), + .error (mem_error), + .stall (mem_stall), + `MAXSIM_MEM(awvalid), `MAXSIM_MEM(awaddr), `MAXSIM_MEM(awid), `MAXSIM_MEM(awlen), + `MAXSIM_MEM(wvalid), `MAXSIM_MEM(wdata), `MAXSIM_MEM(wstrb), `MAXSIM_MEM(wlast), + `MAXSIM_MEM(arvalid), `MAXSIM_MEM(araddr), `MAXSIM_MEM(arid), `MAXSIM_MEM(arlen), + `MAXSIM_MEM(rready), `MAXSIM_MEM(bready), + `MAXSIM_MEM(awready), `MAXSIM_MEM(wready), `MAXSIM_MEM(arready), + `MAXSIM_MEM(rvalid), `MAXSIM_MEM(rdata), `MAXSIM_MEM(rlast), `MAXSIM_MEM(rid), `MAXSIM_MEM(rresp), + `MAXSIM_MEM(bvalid), `MAXSIM_MEM(bresp), `MAXSIM_MEM(bid), + .m_awvalid (m_axi_mem_0_awvalid), + .m_awready (m_axi_mem_0_awready), + .m_awaddr (m_axi_mem_0_awaddr), + .m_awlen (m_axi_mem_0_awlen), + .m_wvalid (m_axi_mem_0_wvalid), + .m_wready (m_axi_mem_0_wready), + .m_wdata (m_axi_mem_0_wdata), + .m_wstrb (m_axi_mem_0_wstrb), + .m_wlast (m_axi_mem_0_wlast), + .m_bvalid (m_axi_mem_0_bvalid), + .m_bready (m_axi_mem_0_bready), + .m_arvalid (m_axi_mem_0_arvalid), + .m_arready (m_axi_mem_0_arready), + .m_araddr (m_axi_mem_0_araddr), + .m_arlen (m_axi_mem_0_arlen), + .m_rvalid (m_axi_mem_0_rvalid), + .m_rready (m_axi_mem_0_rready), + .m_rdata (m_axi_mem_0_rdata) + ); + assign m_axi_mem_0_awid = '0; + assign m_axi_mem_0_arid = '0; + + maxsim_ctrl #( + .DCR_ADDR_W (DCR_ADDR_W), + .DCR_DATA_W (DCR_DATA_W), + .NUM_CLASSES (`MAXSIM_NUM_CLASSES) + ) ctrl ( + .clk (ap_clk), + .reset (reset), + .s_awvalid (s_axi_control_awvalid), + .s_awready (s_axi_control_awready), + .s_awaddr (s_axi_control_awaddr[11:0]), + .s_wvalid (s_axi_control_wvalid), + .s_wready (s_axi_control_wready), + .s_wdata (s_axi_control_wdata), + .s_bvalid (s_axi_control_bvalid), + .s_bready (s_axi_control_bready), + .s_arvalid (s_axi_control_arvalid), + .s_arready (s_axi_control_arready), + .s_araddr (s_axi_control_araddr[11:0]), + .s_rvalid (s_axi_control_rvalid), + .s_rready (s_axi_control_rready), + .s_rdata (s_axi_control_rdata), + .mem_read_latency (mem_read_latency), + .mem_write_latency (mem_write_latency), + .mem_max_reads (mem_max_reads), + .mem_max_writes (mem_max_writes), + .mem_base (mem_base), + .mem_idle (mem_idle), + .mem_error (mem_error), + .mem_stall (mem_stall), + .profile_finish (profile_finish), + `MAXSIM_IN(reset), + `MAXSIM_IN(dcr_req_valid), + `MAXSIM_IN(dcr_req_rw), + `MAXSIM_IN(dcr_req_addr), + `MAXSIM_IN(dcr_req_data), + `MAXSIM_IN(start), + `MAXSIM_IN(dcr_rsp_valid), + `MAXSIM_IN(dcr_rsp_data), + `MAXSIM_IN(busy) + ); + assign s_axi_control_rresp = 2'b00; + assign s_axi_control_bresp = 2'b00; + +`undef MAXSIM_IN +`undef MAXSIM_MEM + +endmodule diff --git a/hw/rtl/afu/maxsim/maxsim_ctrl.sv b/hw/rtl/afu/maxsim/maxsim_ctrl.sv new file mode 100644 index 0000000000..21e28a52e1 --- /dev/null +++ b/hw/rtl/afu/maxsim/maxsim_ctrl.sv @@ -0,0 +1,288 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "maxsim_regs.vh" + +// The host's view of a maxsim simulator: an AXI4-Lite register block, and the +// endpoint that drives the target's control inputs from it. +// +// The target runs free: the endpoint gives it a token every cycle. The host +// queues commands -- a DCR write or read, a start, a reset -- and each is +// applied to the target for one cycle (a reset for its count of cycles), in +// order, at the first cycle the queue reaches it. A DCR read's response is +// kept for the host; `busy` covers everything queued or in flight, so a host +// that sees the target idle has every effect of what it asked for. +// +// See maxsim_regs.vh for the register map. +module maxsim_ctrl #( + parameter DCR_ADDR_W = 12, + parameter DCR_DATA_W = 32, + parameter QUEUE = 16, + parameter RESET_CYCLES = 64, + parameter MEM_DEPTH = 16, + parameter NUM_CLASSES = 1 +) ( + input wire clk, + input wire reset, + + input wire s_awvalid, + output wire s_awready, + input wire [11:0] s_awaddr, + input wire s_wvalid, + output wire s_wready, + input wire [31:0] s_wdata, + output reg s_bvalid, + input wire s_bready, + input wire s_arvalid, + output wire s_arready, + input wire [11:0] s_araddr, + output reg s_rvalid, + input wire s_rready, + output reg [31:0] s_rdata, + + // The memory endpoint's settings and state. + output reg [31:0] mem_read_latency, + output reg [31:0] mem_write_latency, + output reg [7:0] mem_max_reads, + output reg [7:0] mem_max_writes, + output reg [63:0] mem_base, + input wire mem_idle, + input wire mem_error, + input wire mem_stall, + // Which classes of the simulator's models finish a thread this cycle. + input wire [NUM_CLASSES-1:0] profile_finish, + + // Target inputs, as channels of the simulator: the endpoint's outputs. + output wire reset_valid, input wire reset_ready, output wire reset_bits, + output wire dcr_req_valid_valid, input wire dcr_req_valid_ready, output wire dcr_req_valid_bits, + output wire dcr_req_rw_valid, input wire dcr_req_rw_ready, output wire dcr_req_rw_bits, + output wire dcr_req_addr_valid, input wire dcr_req_addr_ready, output wire [DCR_ADDR_W-1:0] dcr_req_addr_bits, + output wire dcr_req_data_valid, input wire dcr_req_data_ready, output wire [DCR_DATA_W-1:0] dcr_req_data_bits, + output wire start_valid, input wire start_ready, output wire start_bits, + + // Target outputs: the endpoint's inputs. + input wire dcr_rsp_valid_valid, output wire dcr_rsp_valid_ready, input wire dcr_rsp_valid_bits, + input wire dcr_rsp_data_valid, output wire dcr_rsp_data_ready, input wire [DCR_DATA_W-1:0] dcr_rsp_data_bits, + input wire busy_valid, output wire busy_ready, input wire busy_bits +); + localparam QW = $clog2(QUEUE); + localparam QC = $clog2(QUEUE + 1); + + // ---- Commands ---- + reg [2:0] q_op [QUEUE]; + reg [DCR_ADDR_W-1:0] q_addr [QUEUE]; + reg [DCR_DATA_W-1:0] q_data [QUEUE]; + reg [QW-1:0] q_head, q_tail; + reg [QC-1:0] q_count; + reg [DCR_ADDR_W-1:0] cmd_addr; + reg [DCR_DATA_W-1:0] cmd_data; + + reg run; + reg [31:0] reset_count; + reg start_pending; + reg [15:0] start_wait; + reg target_busy; + reg rsp_valid; + reg [DCR_DATA_W-1:0] rsp_data; + reg [63:0] target_cycles, host_cycles, mem_stalls; + reg [47:0] finishes [NUM_CLASSES]; + + wire q_empty = (q_count == 0); + wire [2:0] op = q_empty ? `MAXSIM_CMD_NONE : q_op[q_head]; + + // ---- The target's control inputs this cycle ---- + wire in_reset = (reset_count != 0) | (op == `MAXSIM_CMD_RESET); + wire dcr_valid = ~in_reset & ((op == `MAXSIM_CMD_DCR_WRITE) | (op == `MAXSIM_CMD_DCR_READ)); + wire dcr_rw = (op == `MAXSIM_CMD_DCR_WRITE); + wire do_start = ~in_reset & (op == `MAXSIM_CMD_START); + + wire finish; + wire [5:0] done; + wire [2:0] have; + maxsim_token_out o_reset (clk, reset, reset_valid, reset_ready, run, finish, done[0]); + maxsim_token_out o_dvalid (clk, reset, dcr_req_valid_valid, dcr_req_valid_ready, run, finish, done[1]); + maxsim_token_out o_drw (clk, reset, dcr_req_rw_valid, dcr_req_rw_ready, run, finish, done[2]); + maxsim_token_out o_daddr (clk, reset, dcr_req_addr_valid, dcr_req_addr_ready, run, finish, done[3]); + maxsim_token_out o_ddata (clk, reset, dcr_req_data_valid, dcr_req_data_ready, run, finish, done[4]); + maxsim_token_out o_start (clk, reset, start_valid, start_ready, run, finish, done[5]); + assign reset_bits = in_reset; + assign dcr_req_valid_bits = dcr_valid; + assign dcr_req_rw_bits = dcr_rw; + assign dcr_req_addr_bits = q_addr[q_head]; + assign dcr_req_data_bits = q_data[q_head]; + assign start_bits = do_start; + + wire rsp_v, busy; + wire [DCR_DATA_W-1:0] rsp_d; + maxsim_token_in #(1) t_rspv (clk, reset, dcr_rsp_valid_valid, dcr_rsp_valid_ready, dcr_rsp_valid_bits, finish, have[0], rsp_v); + maxsim_token_in #(DCR_DATA_W) t_rspd (clk, reset, dcr_rsp_data_valid, dcr_rsp_data_ready, dcr_rsp_data_bits, finish, have[1], rsp_d); + maxsim_token_in #(1) t_busy (clk, reset, busy_valid, busy_ready, busy_bits, finish, have[2], busy); + + assign finish = run & (&have) & (&done); + + // ---- The host's registers ---- + reg aw_got, w_got, ar_got; + reg [11:0] aw_addr, ar_addr; + reg [31:0] w_data; + assign s_awready = ~aw_got; + assign s_wready = ~w_got; + assign s_arready = ~ar_got & ~s_rvalid; + + wire wr = aw_got & w_got & ~s_bvalid; + wire [11:0] wa = {aw_addr[11:2], 2'b00}; + wire push = wr & (wa == `MAXSIM_REG_CMD) & (q_count != QC'(QUEUE)); + wire pop = finish & ~q_empty & (reset_count == 0); + + wire status_busy = ~q_empty | start_pending | target_busy | ~mem_idle | (reset_count != 0); + + always @(posedge clk) begin + if (reset) begin + q_head <= 0; + q_tail <= 0; + q_count <= 0; + run <= 1'b1; + reset_count <= RESET_CYCLES; + start_pending <= 1'b0; + target_busy <= 1'b0; + rsp_valid <= 1'b0; + target_cycles <= 0; + host_cycles <= 0; + mem_stalls <= 0; + for (integer c = 0; c < NUM_CLASSES; ++c) + finishes[c] <= 0; + mem_read_latency <= 32'd30; + mem_write_latency <= 32'd30; + mem_max_reads <= 8'(MEM_DEPTH); + mem_max_writes <= 8'(MEM_DEPTH); + mem_base <= 64'h40_0000_0000; + aw_got <= 1'b0; + w_got <= 1'b0; + ar_got <= 1'b0; + s_bvalid <= 1'b0; + s_rvalid <= 1'b0; + end else begin + host_cycles <= host_cycles + 64'd1; + if (mem_stall) + mem_stalls <= mem_stalls + 64'd1; + for (integer c = 0; c < NUM_CLASSES; ++c) + if (profile_finish[c]) + finishes[c] <= finishes[c] + 48'd1; + + // One target cycle of the endpoint. + if (finish) begin + target_cycles <= target_cycles + 64'd1; + if (reset_count != 0) + reset_count <= reset_count - 32'd1; + else if (op == `MAXSIM_CMD_RESET) + reset_count <= q_data[q_head] - 32'd1; + if (rsp_v) begin + rsp_valid <= 1'b1; + rsp_data <= rsp_d; + end + target_busy <= busy; + // A start counts as busy until the target says it is, or plainly + // never will. + if (do_start) begin + start_pending <= 1'b1; + start_wait <= 16'hffff; + end else if (start_pending) begin + if (busy || start_wait == 0) + start_pending <= 1'b0; + start_wait <= start_wait - 16'd1; + end + end + if (pop) + q_head <= q_head + 1'b1; + if (push) begin + q_op[q_tail] <= w_data[2:0]; + q_addr[q_tail] <= cmd_addr; + q_data[q_tail] <= cmd_data; + q_tail <= q_tail + 1'b1; + // A read's response is the next one. + if (w_data[2:0] == `MAXSIM_CMD_DCR_READ) + rsp_valid <= 1'b0; + end + q_count <= q_count + QC'(push) - QC'(pop); + + // AXI4-Lite: a write once both its address and data are in. + if (s_awvalid & s_awready) begin + aw_got <= 1'b1; + aw_addr <= s_awaddr; + end + if (s_wvalid & s_wready) begin + w_got <= 1'b1; + w_data <= s_wdata; + end + if (wr) begin + aw_got <= 1'b0; + w_got <= 1'b0; + s_bvalid <= 1'b1; + case (wa) + `MAXSIM_REG_CTRL: run <= w_data[0]; + `MAXSIM_REG_CMD_ADDR: cmd_addr <= DCR_ADDR_W'(w_data); + `MAXSIM_REG_CMD_DATA: cmd_data <= DCR_DATA_W'(w_data); + `MAXSIM_REG_MEM_BASE_LO: mem_base[31:0] <= w_data; + `MAXSIM_REG_MEM_BASE_HI: mem_base[63:32] <= w_data; + `MAXSIM_REG_MEM_RLAT: mem_read_latency <= (w_data == 0) ? 32'd1 : w_data; + `MAXSIM_REG_MEM_WLAT: mem_write_latency <= (w_data == 0) ? 32'd1 : w_data; + `MAXSIM_REG_MEM_MAXR: mem_max_reads <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); + `MAXSIM_REG_MEM_MAXW: mem_max_writes <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); + default:; + endcase + end + if (s_bvalid & s_bready) + s_bvalid <= 1'b0; + + if (s_arvalid & s_arready) begin + ar_got <= 1'b1; + ar_addr <= s_araddr; + end + if (ar_got) begin + ar_got <= 1'b0; + s_rvalid <= 1'b1; + case ({ar_addr[11:2], 2'b00}) + `MAXSIM_REG_ID: s_rdata <= `MAXSIM_ID; + `MAXSIM_REG_VERSION: s_rdata <= `MAXSIM_VERSION; + `MAXSIM_REG_CTRL: s_rdata <= {31'b0, run}; + `MAXSIM_REG_STATUS: s_rdata <= {27'b0, mem_error, rsp_valid, + (q_count == QC'(QUEUE)), status_busy}; + `MAXSIM_REG_CMD_ADDR: s_rdata <= 32'(cmd_addr); + `MAXSIM_REG_CMD_DATA: s_rdata <= 32'(cmd_data); + `MAXSIM_REG_DCR_RSP: s_rdata <= 32'(rsp_data); + `MAXSIM_REG_TCYCLES_LO: s_rdata <= target_cycles[31:0]; + `MAXSIM_REG_TCYCLES_HI: s_rdata <= target_cycles[63:32]; + `MAXSIM_REG_HCYCLES_LO: s_rdata <= host_cycles[31:0]; + `MAXSIM_REG_HCYCLES_HI: s_rdata <= host_cycles[63:32]; + `MAXSIM_REG_MEM_BASE_LO: s_rdata <= mem_base[31:0]; + `MAXSIM_REG_MEM_BASE_HI: s_rdata <= mem_base[63:32]; + `MAXSIM_REG_MEM_RLAT: s_rdata <= mem_read_latency; + `MAXSIM_REG_MEM_WLAT: s_rdata <= mem_write_latency; + `MAXSIM_REG_MEM_MAXR: s_rdata <= 32'(mem_max_reads); + `MAXSIM_REG_MEM_MAXW: s_rdata <= 32'(mem_max_writes); + `MAXSIM_REG_MSTALL_LO: s_rdata <= mem_stalls[31:0]; + `MAXSIM_REG_MSTALL_HI: s_rdata <= mem_stalls[63:32]; + `MAXSIM_REG_NUM_CLASSES: s_rdata <= 32'(NUM_CLASSES); + default: s_rdata <= 32'd0; + endcase + if (ar_addr >= `MAXSIM_REG_FINISH_BASE + && ar_addr < `MAXSIM_REG_FINISH_BASE + 12'(8 * NUM_CLASSES)) begin + s_rdata <= ar_addr[2] ? 32'(finishes[(ar_addr - `MAXSIM_REG_FINISH_BASE) >> 3] >> 32) + : finishes[(ar_addr - `MAXSIM_REG_FINISH_BASE) >> 3][31:0]; + end + end + if (s_rvalid & s_rready) + s_rvalid <= 1'b0; + end + end + +endmodule diff --git a/hw/rtl/afu/maxsim/maxsim_mem.sv b/hw/rtl/afu/maxsim/maxsim_mem.sv new file mode 100644 index 0000000000..849100dce7 --- /dev/null +++ b/hw/rtl/afu/maxsim/maxsim_mem.sv @@ -0,0 +1,395 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The target's AXI4 memory port, as a maxsim endpoint. +// +// Timing is a latency-bandwidth pipe, cycle for cycle the model of +// sim/common/axi_latency_pipe.h (FireSim FASED's "Latency Bandwidth Pipe"): a +// fixed latency per transaction, at most `max_reads` reads and `max_writes` +// writes in flight, one beat per cycle on each response channel, responses in +// the order their requests were accepted. Every output is a function of the +// pipe's state, so all of them are offered at the start of a target cycle. +// +// Data lives in device memory at `base` plus the target's address. A request +// goes to memory the cycle the target makes it; the response is offered to +// the target when the pipe releases it, and a target cycle that would offer +// one memory has not returned yet simply does not finish until it has. A +// write is acknowledged to the target only once memory has acknowledged it, +// so any later request of the target sees it. +// +// Single-beat transactions only, as Vortex issues them: a longer burst sets +// `error`. +module maxsim_mem #( + parameter ID_W = 4, + parameter ADDR_W = 64, + parameter DATA_W = 512, + parameter DEPTH = 16 +) ( + input wire clk, + input wire reset, + + input wire [31:0] read_latency, + input wire [31:0] write_latency, + input wire [7:0] max_reads, + input wire [7:0] max_writes, + input wire [63:0] base, + // No transaction in flight at the target. + output wire idle, + output reg error, + // A host cycle a target cycle waits for memory. + output wire stall, + + // Target outputs, as channels of the simulator: the endpoint's inputs. + input wire awvalid_valid, output wire awvalid_ready, input wire awvalid_bits, + input wire awaddr_valid, output wire awaddr_ready, input wire [ADDR_W-1:0] awaddr_bits, + input wire awid_valid, output wire awid_ready, input wire [ID_W-1:0] awid_bits, + input wire awlen_valid, output wire awlen_ready, input wire [7:0] awlen_bits, + input wire wvalid_valid, output wire wvalid_ready, input wire wvalid_bits, + input wire wdata_valid, output wire wdata_ready, input wire [DATA_W-1:0] wdata_bits, + input wire wstrb_valid, output wire wstrb_ready, input wire [DATA_W/8-1:0] wstrb_bits, + input wire wlast_valid, output wire wlast_ready, input wire wlast_bits, + input wire arvalid_valid, output wire arvalid_ready, input wire arvalid_bits, + input wire araddr_valid, output wire araddr_ready, input wire [ADDR_W-1:0] araddr_bits, + input wire arid_valid, output wire arid_ready, input wire [ID_W-1:0] arid_bits, + input wire arlen_valid, output wire arlen_ready, input wire [7:0] arlen_bits, + input wire rready_valid, output wire rready_ready, input wire rready_bits, + input wire bready_valid, output wire bready_ready, input wire bready_bits, + + // Target inputs: the endpoint's outputs. + output wire awready_valid, input wire awready_ready, output wire awready_bits, + output wire wready_valid, input wire wready_ready, output wire wready_bits, + output wire arready_valid, input wire arready_ready, output wire arready_bits, + output wire rvalid_valid, input wire rvalid_ready, output wire rvalid_bits, + output wire rdata_valid, input wire rdata_ready, output wire [DATA_W-1:0] rdata_bits, + output wire rlast_valid, input wire rlast_ready, output wire rlast_bits, + output wire rid_valid, input wire rid_ready, output wire [ID_W-1:0] rid_bits, + output wire rresp_valid, input wire rresp_ready, output wire [1:0] rresp_bits, + output wire bvalid_valid, input wire bvalid_ready, output wire bvalid_bits, + output wire bresp_valid, input wire bresp_ready, output wire [1:0] bresp_bits, + output wire bid_valid, input wire bid_ready, output wire [ID_W-1:0] bid_bits, + + // Device memory. + output wire m_awvalid, + input wire m_awready, + output wire [63:0] m_awaddr, + output wire [7:0] m_awlen, + output wire m_wvalid, + input wire m_wready, + output wire [DATA_W-1:0] m_wdata, + output wire [DATA_W/8-1:0] m_wstrb, + output wire m_wlast, + input wire m_bvalid, + output wire m_bready, + output wire m_arvalid, + input wire m_arready, + output wire [63:0] m_araddr, + output wire [7:0] m_arlen, + input wire m_rvalid, + output wire m_rready, + input wire [DATA_W-1:0] m_rdata +); + localparam CW = $clog2(DEPTH + 1); + localparam PW = $clog2(DEPTH); + + wire finish; + + // ---- The target's request side, one token per channel per cycle ---- + wire awvalid, wvalid, wlast, arvalid, rready, bready; + wire [ADDR_W-1:0] awaddr, araddr; + wire [ID_W-1:0] awid, arid; + wire [7:0] awlen, arlen; + wire [DATA_W-1:0] wdata; + wire [DATA_W/8-1:0] wstrb; + wire [13:0] have; + + maxsim_token_in #(1) t_awvalid (clk, reset, awvalid_valid, awvalid_ready, awvalid_bits, finish, have[0], awvalid); + maxsim_token_in #(ADDR_W) t_awaddr (clk, reset, awaddr_valid, awaddr_ready, awaddr_bits, finish, have[1], awaddr); + maxsim_token_in #(ID_W) t_awid (clk, reset, awid_valid, awid_ready, awid_bits, finish, have[2], awid); + maxsim_token_in #(8) t_awlen (clk, reset, awlen_valid, awlen_ready, awlen_bits, finish, have[3], awlen); + maxsim_token_in #(1) t_wvalid (clk, reset, wvalid_valid, wvalid_ready, wvalid_bits, finish, have[4], wvalid); + maxsim_token_in #(DATA_W) t_wdata (clk, reset, wdata_valid, wdata_ready, wdata_bits, finish, have[5], wdata); + maxsim_token_in #(DATA_W/8) t_wstrb (clk, reset, wstrb_valid, wstrb_ready, wstrb_bits, finish, have[6], wstrb); + maxsim_token_in #(1) t_wlast (clk, reset, wlast_valid, wlast_ready, wlast_bits, finish, have[7], wlast); + maxsim_token_in #(1) t_arvalid (clk, reset, arvalid_valid, arvalid_ready, arvalid_bits, finish, have[8], arvalid); + maxsim_token_in #(ADDR_W) t_araddr (clk, reset, araddr_valid, araddr_ready, araddr_bits, finish, have[9], araddr); + maxsim_token_in #(ID_W) t_arid (clk, reset, arid_valid, arid_ready, arid_bits, finish, have[10], arid); + maxsim_token_in #(8) t_arlen (clk, reset, arlen_valid, arlen_ready, arlen_bits, finish, have[11], arlen); + maxsim_token_in #(1) t_rready (clk, reset, rready_valid, rready_ready, rready_bits, finish, have[12], rready); + maxsim_token_in #(1) t_bready (clk, reset, bready_valid, bready_ready, bready_bits, finish, have[13], bready); + + // ---- The pipe's state ---- + reg [31:0] cycle; + reg [CW-1:0] pending_reads, pending_aw, pending_w; + + // Reads accepted and not yet driving R, and the one that is. + reg [31:0] rp_release [DEPTH]; + reg [ID_W-1:0] rp_id [DEPTH]; + reg [PW-1:0] rp_head, rp_tail; + reg [CW-1:0] rp_count; + reg cur_r_valid; + reg [ID_W-1:0] cur_r_id; + + // AW ids waiting for their write data, writes formed and not yet driving + // B, and the one that is. + reg [ID_W-1:0] awq_id [DEPTH]; + reg [PW-1:0] awq_head, awq_tail; + reg [CW-1:0] awq_count; + reg [31:0] wp_release [DEPTH]; + reg [ID_W-1:0] wp_id [DEPTH]; + reg [PW-1:0] wp_head, wp_tail; + reg [CW-1:0] wp_count; + reg cur_w_valid; + reg [ID_W-1:0] cur_w_id; + + // Memory's side: read data in order, and writes acknowledged. + reg [DATA_W-1:0] rd_data [DEPTH]; + reg [PW-1:0] rd_head, rd_tail; + reg [CW-1:0] rd_count; + reg [7:0] b_acked, b_given; + + // ---- The pipe's outputs: functions of its state ---- + wire ar_ready = (pending_reads < CW'(max_reads)); + wire aw_ready = (pending_aw < CW'(max_writes)); + wire w_ready = (pending_w < CW'(max_writes)); + // R needs its data back from memory, and B its write acknowledged. + wire ok = (~cur_r_valid | (rd_count != 0)) & (~cur_w_valid | (b_acked != b_given)); + wire [10:0] done; + + maxsim_token_out o_awready (clk, reset, awready_valid, awready_ready, ok, finish, done[0]); + maxsim_token_out o_wready (clk, reset, wready_valid, wready_ready, ok, finish, done[1]); + maxsim_token_out o_arready (clk, reset, arready_valid, arready_ready, ok, finish, done[2]); + maxsim_token_out o_rvalid (clk, reset, rvalid_valid, rvalid_ready, ok, finish, done[3]); + maxsim_token_out o_rdata (clk, reset, rdata_valid, rdata_ready, ok, finish, done[4]); + maxsim_token_out o_rlast (clk, reset, rlast_valid, rlast_ready, ok, finish, done[5]); + maxsim_token_out o_rid (clk, reset, rid_valid, rid_ready, ok, finish, done[6]); + maxsim_token_out o_rresp (clk, reset, rresp_valid, rresp_ready, ok, finish, done[7]); + maxsim_token_out o_bvalid (clk, reset, bvalid_valid, bvalid_ready, ok, finish, done[8]); + maxsim_token_out o_bresp (clk, reset, bresp_valid, bresp_ready, ok, finish, done[9]); + maxsim_token_out o_bid (clk, reset, bid_valid, bid_ready, ok, finish, done[10]); + + assign awready_bits = aw_ready; + assign wready_bits = w_ready; + assign arready_bits = ar_ready; + assign rvalid_bits = cur_r_valid; + assign rdata_bits = rd_data[rd_head]; + assign rlast_bits = 1'b1; + assign rid_bits = cur_r_id; + assign rresp_bits = 2'b00; + assign bvalid_bits = cur_w_valid; + assign bresp_bits = 2'b00; + assign bid_bits = cur_w_id; + + assign finish = (&have) & (&done); + assign stall = ~ok; + assign idle = (pending_reads == 0) & (pending_aw == 0) & (pending_w == 0); + + // ---- One target cycle of the pipe, on the request side's tokens ---- + wire ar_fire = arvalid & ar_ready; + wire aw_fire = awvalid & aw_ready; + wire w_fire = wvalid & w_ready; + wire wlast_fire = w_fire & wlast; + wire r_fire = cur_r_valid & rready; + wire b_fire = cur_w_valid & bready; + + // Reads: the latency pipe, into the stage driving R; a read released the + // cycle it arrives skips the pipe. + wire [31:0] ar_release = cycle + read_latency - 32'd1; + wire rp_empty = (rp_count == 0); + wire r_head = ~rp_empty | ar_fire; + wire [31:0] r_head_rel = rp_empty ? ar_release : rp_release[rp_head]; + wire [ID_W-1:0] r_head_id = rp_empty ? arid : rp_id[rp_head]; + wire r_release = r_head & ($signed(cycle - r_head_rel) >= 0); + wire r_take = r_release & (~cur_r_valid | r_fire); + wire rp_push = ar_fire & ~(r_take & rp_empty); + wire rp_pop = r_take & ~rp_empty; + + // Writes: an AW and its last W beat, in either order, form one write, its + // latency counted from the cycle both are in. + wire new_write = ((pending_w > pending_aw) & aw_fire) + | ((pending_w < pending_aw) & wlast_fire) + | (aw_fire & wlast_fire); + wire awq_empty = (awq_count == 0); + wire [ID_W-1:0] w_form_id = awq_empty ? awid : awq_id[awq_head]; + wire awq_pop = new_write & ~awq_empty; + wire awq_push = aw_fire & ~(new_write & awq_empty); + wire [31:0] w_release = cycle + write_latency - 32'd1; + wire wp_empty = (wp_count == 0); + wire w_head = ~wp_empty | new_write; + wire [31:0] w_head_rel = wp_empty ? w_release : wp_release[wp_head]; + wire [ID_W-1:0] w_head_id = wp_empty ? w_form_id : wp_id[wp_head]; + wire w_rel = w_head & ($signed(cycle - w_head_rel) >= 0); + wire w_take = w_rel & (~cur_w_valid | b_fire); + wire wp_push = new_write & ~(w_take & wp_empty); + wire wp_pop = w_take & ~wp_empty; + + function automatic [CW-1:0] count(input [CW-1:0] value, input inc, input dec); + count = (inc & ~dec) ? value + 1'b1 : ((dec & ~inc & (value != 0)) ? value - 1'b1 : value); + endfunction + + always @(posedge clk) begin + if (reset) begin + cycle <= 0; + pending_reads <= 0; + pending_aw <= 0; + pending_w <= 0; + rp_head <= 0; + rp_tail <= 0; + rp_count <= 0; + cur_r_valid <= 1'b0; + awq_head <= 0; + awq_tail <= 0; + awq_count <= 0; + wp_head <= 0; + wp_tail <= 0; + wp_count <= 0; + cur_w_valid <= 1'b0; + error <= 1'b0; + end else if (finish) begin + cycle <= cycle + 32'd1; + if (rp_push) begin + rp_release[rp_tail] <= ar_release; + rp_id[rp_tail] <= arid; + rp_tail <= rp_tail + 1'b1; + end + if (rp_pop) + rp_head <= rp_head + 1'b1; + rp_count <= rp_count + CW'(rp_push) - CW'(rp_pop); + if (r_take) begin + cur_r_valid <= 1'b1; + cur_r_id <= r_head_id; + end else if (r_fire) begin + cur_r_valid <= 1'b0; + end + + if (awq_push) begin + awq_id[awq_tail] <= awid; + awq_tail <= awq_tail + 1'b1; + end + if (awq_pop) + awq_head <= awq_head + 1'b1; + awq_count <= awq_count + CW'(awq_push) - CW'(awq_pop); + if (wp_push) begin + wp_release[wp_tail] <= w_release; + wp_id[wp_tail] <= w_form_id; + wp_tail <= wp_tail + 1'b1; + end + if (wp_pop) + wp_head <= wp_head + 1'b1; + wp_count <= wp_count + CW'(wp_push) - CW'(wp_pop); + if (w_take) begin + cur_w_valid <= 1'b1; + cur_w_id <= w_head_id; + end else if (b_fire) begin + cur_w_valid <= 1'b0; + end + + pending_reads <= count(pending_reads, ar_fire, r_fire); + pending_aw <= count(pending_aw, aw_fire, b_fire); + pending_w <= count(pending_w, wlast_fire, b_fire); + if ((ar_fire && arlen != 0) || (aw_fire && awlen != 0)) + error <= 1'b1; + end + end + + // ---- Memory: requests as the target makes them, in order ---- + reg [63:0] arq_addr [DEPTH]; + reg [PW-1:0] arq_head, arq_tail; + reg [CW-1:0] arq_count; + reg [63:0] awm_addr [DEPTH]; + reg [PW-1:0] awm_head, awm_tail; + reg [CW-1:0] awm_count; + reg [DATA_W-1:0] wm_data [DEPTH]; + reg [DATA_W/8-1:0] wm_strb [DEPTH]; + reg [PW-1:0] wm_head, wm_tail; + reg [CW-1:0] wm_count; + + wire arq_push = finish & ar_fire; + wire arq_pop = m_arvalid & m_arready; + wire awm_push = finish & aw_fire; + wire awm_pop = m_awvalid & m_awready; + wire wm_push = finish & w_fire; + wire wm_pop = m_wvalid & m_wready; + wire rd_push = m_rvalid & m_rready; + wire rd_pop = finish & r_fire; + + assign m_arvalid = (arq_count != 0); + assign m_araddr = arq_addr[arq_head]; + assign m_arlen = 8'd0; + assign m_rready = 1'b1; + assign m_awvalid = (awm_count != 0); + assign m_awaddr = awm_addr[awm_head]; + assign m_awlen = 8'd0; + assign m_wvalid = (wm_count != 0); + assign m_wdata = wm_data[wm_head]; + assign m_wstrb = wm_strb[wm_head]; + assign m_wlast = 1'b1; + assign m_bready = 1'b1; + + always @(posedge clk) begin + if (reset) begin + arq_head <= 0; + arq_tail <= 0; + arq_count <= 0; + awm_head <= 0; + awm_tail <= 0; + awm_count <= 0; + wm_head <= 0; + wm_tail <= 0; + wm_count <= 0; + rd_head <= 0; + rd_tail <= 0; + rd_count <= 0; + b_acked <= 0; + b_given <= 0; + end else begin + if (arq_push) begin + arq_addr[arq_tail] <= base + 64'(araddr); + arq_tail <= arq_tail + 1'b1; + end + if (arq_pop) + arq_head <= arq_head + 1'b1; + arq_count <= arq_count + CW'(arq_push) - CW'(arq_pop); + + if (awm_push) begin + awm_addr[awm_tail] <= base + 64'(awaddr); + awm_tail <= awm_tail + 1'b1; + end + if (awm_pop) + awm_head <= awm_head + 1'b1; + awm_count <= awm_count + CW'(awm_push) - CW'(awm_pop); + + if (wm_push) begin + wm_data[wm_tail] <= wdata; + wm_strb[wm_tail] <= wstrb; + wm_tail <= wm_tail + 1'b1; + end + if (wm_pop) + wm_head <= wm_head + 1'b1; + wm_count <= wm_count + CW'(wm_push) - CW'(wm_pop); + + if (rd_push) begin + rd_data[rd_tail] <= m_rdata; + rd_tail <= rd_tail + 1'b1; + end + if (rd_pop) + rd_head <= rd_head + 1'b1; + rd_count <= rd_count + CW'(rd_push) - CW'(rd_pop); + + if (m_bvalid) + b_acked <= b_acked + 8'd1; + if (finish & b_fire) + b_given <= b_given + 8'd1; + end + end + +endmodule diff --git a/hw/rtl/afu/maxsim/maxsim_regs.vh b/hw/rtl/afu/maxsim/maxsim_regs.vh new file mode 100644 index 0000000000..6e9def4e42 --- /dev/null +++ b/hw/rtl/afu/maxsim/maxsim_regs.vh @@ -0,0 +1,57 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The maxsim simulator's register map, as sw/common/maxsim_regs.h gives it to +// the runtime. Byte offsets on s_axi_control; unlisted offsets read zero. +// +// 0x040-0x0BF is left empty: on the V80 the PCIe block's MSI-X table sits at +// that offset of the BAR the first kernel is mapped at, and answers those +// accesses itself. + +`ifndef MAXSIM_REGS_VH +`define MAXSIM_REGS_VH + +`define MAXSIM_ID 32'h4D58534D +`define MAXSIM_VERSION 32'd2 + +`define MAXSIM_REG_ID 12'h000 +`define MAXSIM_REG_VERSION 12'h004 +`define MAXSIM_REG_CTRL 12'h008 +`define MAXSIM_REG_STATUS 12'h00C +`define MAXSIM_REG_CMD_ADDR 12'h010 +`define MAXSIM_REG_CMD_DATA 12'h014 +`define MAXSIM_REG_CMD 12'h018 +`define MAXSIM_REG_DCR_RSP 12'h01C +`define MAXSIM_REG_TCYCLES_LO 12'h020 +`define MAXSIM_REG_TCYCLES_HI 12'h024 +`define MAXSIM_REG_HCYCLES_LO 12'h028 +`define MAXSIM_REG_HCYCLES_HI 12'h02C +`define MAXSIM_REG_MSTALL_LO 12'h030 +`define MAXSIM_REG_MSTALL_HI 12'h034 +`define MAXSIM_REG_NUM_CLASSES 12'h038 +`define MAXSIM_REG_MEM_BASE_LO 12'h0C0 +`define MAXSIM_REG_MEM_BASE_HI 12'h0C4 +`define MAXSIM_REG_MEM_RLAT 12'h0C8 +`define MAXSIM_REG_MEM_WLAT 12'h0CC +`define MAXSIM_REG_MEM_MAXR 12'h0D0 +`define MAXSIM_REG_MEM_MAXW 12'h0D4 +// Per class c: the host cycles a thread of it finished its target cycle. +`define MAXSIM_REG_FINISH_BASE 12'h100 + +`define MAXSIM_CMD_NONE 3'd0 +`define MAXSIM_CMD_DCR_WRITE 3'd1 +`define MAXSIM_CMD_DCR_READ 3'd2 +`define MAXSIM_CMD_START 3'd3 +`define MAXSIM_CMD_RESET 3'd4 + +`endif // MAXSIM_REGS_VH diff --git a/hw/rtl/afu/maxsim/maxsim_token.sv b/hw/rtl/afu/maxsim/maxsim_token.sv new file mode 100644 index 0000000000..531f9fb63d --- /dev/null +++ b/hw/rtl/afu/maxsim/maxsim_token.sv @@ -0,0 +1,76 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The token ends of a maxsim endpoint: a model of the target's surroundings +// that takes one token on each input channel and gives one on each output +// channel per target cycle, finishing the cycle once it has them all. +// +// Valid and ready depend on the endpoint's state alone, never on the +// simulator's handshake, so the simulator may connect them to its models +// combinationally. + +// An input channel: the token is kept until the endpoint finishes its cycle. +module maxsim_token_in #( + parameter W = 1 +) ( + input wire clk, + input wire reset, + input wire valid, + output wire ready, + input wire [W-1:0] bits, + input wire finish, + output wire have, + output wire [W-1:0] data +); + reg got; + reg [W-1:0] held; + + assign ready = ~got; + assign have = got | valid; + assign data = got ? held : bits; + + always @(posedge clk) begin + if (reset || finish) begin + got <= 1'b0; + end else if (valid) begin + got <= 1'b1; + end + if (~got) begin + held <= bits; + end + end +endmodule + +// An output channel: the token is offered once `ok` until it is taken. +module maxsim_token_out ( + input wire clk, + input wire reset, + output wire valid, + input wire ready, + input wire ok, + input wire finish, + output wire done +); + reg fired; + + assign valid = ~fired & ok; + assign done = fired | (ok & ready); + + always @(posedge clk) begin + if (reset || finish) begin + fired <= 1'b0; + end else if (valid & ready) begin + fired <= 1'b1; + end + end +endmodule diff --git a/hw/scripts/gen_platform.py b/hw/scripts/gen_platform.py index 551242c0c5..579906267d 100644 --- a/hw/scripts/gen_platform.py +++ b/hw/scripts/gen_platform.py @@ -218,20 +218,23 @@ def emit_mk(c: Card) -> str: return "\n".join(lines) + "\n" -def emit_cfg(c: Card, mem_tag: str, host_tag: str, kernel_mhz: int) -> str: +def emit_cfg(c: Card, mem_tag: str, host_tag: str, kernel_mhz: int, + kernel: str = "vortex_afu") -> str: + """The linker config of `kernel`; one with no host-memory port has no host tag.""" if c.flow != "aved": raise DescriptorError(f"card '{c.name}' is an {c.flow} card; cfg is for the aved flow") - check_host_tag(c, host_tag) - inst = KERNEL_INSTANCE["aved"] + if host_tag: + check_host_tag(c, host_tag) + inst = KERNEL_INSTANCE["aved"] if kernel == "vortex_afu" else f"{kernel}_0" return "\n".join([ f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", "[connectivity]", "# Bind to the compute-only static shell. The service shell adds the DCMAC", "# ethernet stack the AFU has no use for.", "shell=compute", - f"nk=vortex_afu:1:{inst}", + f"nk={kernel}:1:{inst}", f"sp={inst}.m_axi_mem_0:{mem_tag}", - f"sp={inst}.m_axi_host:{host_tag}", + ] + ([f"sp={inst}.m_axi_host:{host_tag}"] if host_tag else []) + [ "", "# Kernel clock, in Hz rather than via the linker's --clock-hz flag, whose", "# name and help text disagree on the unit.", @@ -305,6 +308,8 @@ def main(argv: Optional[List[str]] = None) -> int: ap.add_argument("--mem-tag", default="") ap.add_argument("--host-tag", default="") ap.add_argument("--kernel-mhz", type=int, default=0) + ap.add_argument("--kernel", default="vortex_afu", help="kernel the cfg links (aved flow)") + ap.add_argument("--no-host", action="store_true", help="the kernel has no host-memory port") ap.add_argument("--platform-dir", default=PLATFORM_DIR) ap.add_argument("-o", "--output", default="-") a = ap.parse_args(argv) @@ -317,8 +322,9 @@ def main(argv: Optional[List[str]] = None) -> int: if a.emit == "mk": out = emit_mk(c) elif a.emit == "cfg": - out = emit_cfg(c, a.mem_tag or c.mem_tag, a.host_tag or c.host_tag, - a.kernel_mhz or c.kernel_mhz) + out = emit_cfg(c, a.mem_tag or c.mem_tag, + "" if a.no_host else (a.host_tag or c.host_tag), + a.kernel_mhz or c.kernel_mhz, a.kernel) elif a.emit == "firesim-mk": out = emit_firesim_mk(c) elif a.emit == "scala": diff --git a/hw/syn/maxsim/Makefile b/hw/syn/maxsim/Makefile new file mode 100644 index 0000000000..e97f24fc99 --- /dev/null +++ b/hw/syn/maxsim/Makefile @@ -0,0 +1,164 @@ +# maxsim image build for the V80, driven as the native aved flow +# (hw/syn/xilinx/aved): +# +# make [DEVICE=] [PREFIX=...] [CONFIGS="-DVX_CFG_..."] [PERF=1] +# [MAXSIM_CLASSES="..."] [MAXSIM_LIFT="..."] [CIRCT_BIN=] +# +# CIRCT compiles Vortex into a simulator (maxsim.py): each class threaded +# through one physical copy (FAME-5), the rest of the target a model of its +# own, and every channel between them in one token fabric. The kernel +# (hw/rtl/afu/maxsim) adds the memory and control endpoints around it. The +# image lands in $(BUILD_DIR)/bin/maxsim.vbin; run it with +# FPGA_BIN_DIR= ./ci/blackbox.sh --driver=maxsim --target=hw ... +# The runtime's avedsim target simulates the same generated design in-process. + +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk +include $(VORTEX_HOME)/hw/syn/common.mk + +SRC_DIR := $(VORTEX_HOME)/hw/syn/maxsim +RTL_DIR := $(VORTEX_HOME)/hw/rtl +SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts +MAXSIM_RTL_DIR := $(RTL_DIR)/afu/maxsim +VIVADO := $(XILINX_VIVADO)/bin/vivado +SLASHKIT ?= slashkit +# Parallel link runs; each Vivado of a large simulator needs tens of GB. +SLASHKIT_JOBS ?= + +XLEN ?= 32 +PREFIX ?= build$(XLEN) +TARGET ?= hw +DEVICE ?= xcv80-lsva4737-2MHP-e-S + +ifneq ($(TARGET), hw) +$(error TARGET must be hw: the runtime's avedsim target simulates the generated design itself) +endif + +BUILD_DIR := $(abspath $(PREFIX)_maxsim_$(TARGET)) +# The runtime's in-process simulator generates the design in a directory of +# its own. +ifneq ($(MAXSIM_WORK_DIR),) +BUILD_DIR := $(abspath $(MAXSIM_WORK_DIR)) +endif +BIN_DIR := $(BUILD_DIR)/bin +GEN_DIR := $(BUILD_DIR)/gen + +# The classes threaded by FAME-5, and the units given plain-port boundaries +# first so they can be classes (lift_boundaries.py; [:drop=P,...][;port=P,...]). +MAXSIM_CLASSES ?= VX_socket__lift +MAXSIM_LIFT ?= VX_socket:drop=INSTANCE_ID;port=SOCKET_ID +# Where circt-verilog, circt-opt and firtool are (PATH by default). +CIRCT_BIN ?= +LIFT_PYTHON ?= python3 +# The kernel clock (MHz): the image is timed at it and the shell programs it. +MAXSIM_FREQ ?= 143 + +# The target: Vortex with its AXI4 port, DCR, start and busy as plain ports. +TARGET_TOP := VX_firesim_wrap + +# The same tree the native synthesis flows elaborate. +CONFIGS ?= +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs \ + -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm \ + -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(RTL_DIR)/afu/firesim +SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG +# One memory port, as the memory endpoint models; 4 AXI id bits, what Vortex +# drives with its tag buffer engaged. The FPU and RTU use Vortex's own +# datapaths (the STD FPU, the RTU without vendor cores): a vendor netlist is not +# source the flow can read, nor state it can thread. +PLATFORM_DEFINES := -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DPLATFORM_MEMORY_ID_WIDTH=4 \ + -DVX_CFG_RTU_USE_DSP=0 -DVX_CFG_RTU_FDIV_LATENCY=17 \ + -DVX_CFG_FPU_TYPE_STD +ifdef PERF +PLATFORM_DEFINES += -DPERF_ENABLE +endif +TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(shell python3 $(VORTEX_HOME)/ci/gen_config.py \ + --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN) $(SYNTH_DEFINES) $(PLATFORM_DEFINES)')) +GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) \ + $(SYNTH_DEFINES) $(RTL_INCLUDE) + +# Rewritten only when a setting changes, so an unchanged build stays current. +define stamp +$(shell mkdir -p $(BUILD_DIR) && echo '$(2)' > $(1).tmp && \ + (cmp -s $(1).tmp $(1) && rm -f $(1).tmp || mv -f $(1).tmp $(1))) +endef +CONFIG_STAMP := $(BUILD_DIR)/cflags.stamp +CLASSES_STAMP := $(BUILD_DIR)/classes.stamp +$(call stamp,$(CONFIG_STAMP),$(GEN_CFLAGS)) +$(call stamp,$(CLASSES_STAMP),$(MAXSIM_CLASSES) | $(MAXSIM_LIFT)) + +SRCS_NAME := vortex_srcs +SRCS_DIR := $(BUILD_DIR)/$(SRCS_NAME) +SRCS_LIST := $(BUILD_DIR)/$(SRCS_NAME).f +LIFT_LIST := $(BUILD_DIR)/lift/srcs.f +TARGET_SRCS := $(if $(strip $(MAXSIM_LIFT)),$(LIFT_LIST),$(SRCS_LIST)) +GEN_SV := $(GEN_DIR)/maxsim_sim.sv +KERNEL_SRCS := $(BUILD_DIR)/kernel_srcs.f +IP_DIR := $(BUILD_DIR)/maxsim_afu_ip +CFG := $(BUILD_DIR)/config.cfg +VBIN := $(BIN_DIR)/maxsim.vbin + +RTL_SRCS := $(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh') \ + $(wildcard $(ROOT_DIR)/hw/VX_config.vh) $(wildcard $(ROOT_DIR)/hw/VX_types.vh) \ + $(firstword $(MAKEFILE_LIST)) $(CONFIG_STAMP) +# The AFU shells of the native flows are second tops, and the maxsim kernel is +# compiled apart from the target. +SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv) $(wildcard $(MAXSIM_RTL_DIR)/*.sv)) + +.PHONY: all sources generate package-ip clean FORCE + +all: $(VBIN) + +# The flow threads memories itself, so an asynchronous-read RAM stays one +# (VX_async_ram_patch.sv) instead of becoming placeholders for a Vivado hook. +sources: $(SRCS_LIST) +$(SRCS_LIST): $(RTL_SRCS) $(SRC_DIR)/VX_async_ram_patch.sv + rm -rf $(SRCS_DIR) + cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) -T$(TARGET_TOP) -C$(SRCS_NAME) \ + -O$(SRCS_NAME).f + cp $(SRC_DIR)/VX_async_ram_patch.sv $(SRCS_DIR)/VX_async_ram_patch.sv + @for f in $(SHELL_MODULES); do \ + rm -f $(SRCS_DIR)/$$f; \ + grep -v "/$$f$$" $@ > $@.tmp && mv $@.tmp $@; \ + done + @if grep -qE 'localparam +VX_MEM_PORTS *= *0 *;' $(SRCS_DIR)/VX_gpu_pkg.sv; then \ + echo "ERROR: generated sources resolve VX_MEM_PORTS to 0 (no memory interface)"; exit 1; \ + fi + +$(LIFT_LIST): $(SRCS_LIST) $(SRC_DIR)/lift_boundaries.py $(CLASSES_STAMP) + rm -rf $(BUILD_DIR)/lift + $(LIFT_PYTHON) $(SRC_DIR)/lift_boundaries.py --srcs $(SRCS_LIST) --top $(TARGET_TOP) \ + --out $(BUILD_DIR)/lift $(foreach u,$(MAXSIM_LIFT),--unit '$(u)') + +generate: $(GEN_SV) +$(GEN_SV): $(TARGET_SRCS) $(SRC_DIR)/maxsim.py $(CLASSES_STAMP) + python3 $(SRC_DIR)/maxsim.py --srcs $(TARGET_SRCS) --top $(TARGET_TOP) --out $(GEN_DIR) \ + $(if $(CIRCT_BIN),--circt-bin $(CIRCT_BIN)) $(MAXSIM_CLASSES) + +$(KERNEL_SRCS): $(GEN_SV) + @{ echo "+incdir+$(MAXSIM_RTL_DIR)"; echo "+incdir+$(GEN_DIR)"; \ + echo "$(GEN_SV)"; \ + for f in maxsim_token.sv maxsim_mem.sv maxsim_ctrl.sv maxsim_afu.sv; do echo "$(MAXSIM_RTL_DIR)/$$f"; done; \ + } > $@ + +package-ip: $(IP_DIR)/component.xml +$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(wildcard $(MAXSIM_RTL_DIR)/*) + cd $(BUILD_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -nojournal -nolog \ + -source $(VORTEX_HOME)/hw/syn/xilinx/aved/package_ip.tcl \ + -tclargs maxsim_afu $(KERNEL_SRCS) . $(DEVICE) no-vendor-ip + +$(CFG): FORCE + @python3 $(SCRIPT_DIR)/gen_platform.py --flow=aved --device=$(DEVICE) --emit=cfg --kernel=maxsim_afu \ + --no-host --kernel-mhz=$(MAXSIM_FREQ) -o $@.tmp + @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ + +FORCE: + +$(VBIN): $(IP_DIR)/component.xml $(CFG) + mkdir -p $(BIN_DIR) + TOOL_DIR=$(SCRIPT_DIR) $(SLASHKIT) link -c $(CFG) -p hw -o $@ -k $(IP_DIR)/component.xml \ + --vivado $(VIVADO) $(if $(SLASHKIT_JOBS),--jobs $(SLASHKIT_JOBS)) + @echo "image: $@" + +clean: + rm -rf $(BUILD_DIR) diff --git a/hw/syn/maxsim/VX_async_ram_patch.sv b/hw/syn/maxsim/VX_async_ram_patch.sv new file mode 100644 index 0000000000..43ca5617a4 --- /dev/null +++ b/hw/syn/maxsim/VX_async_ram_patch.sv @@ -0,0 +1,76 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The maxsim flow's VX_async_ram_patch: the RAM it stands for, read +// asynchronously. +// +// On Vivado, hw/rtl/libs/VX_async_ram_patch realizes an asynchronous-read RAM +// on a synchronous BRAM through placeholder cells that a post-synthesis hook +// rewires; with or without RADDR_REG it reads the word at raddr. The maxsim +// flow threads its memories itself and cannot run that hook, so it replaces the +// staged copy of the module with this one: same parameters and ports, same +// contents and reads. Plain SystemVerilog, as the staged sources are already +// preprocessed. +module VX_async_ram_patch #( + parameter DATAW = 1, + parameter SIZE = 1, + parameter WRENW = 1, + parameter DUAL_PORT = 0, + parameter FORCE_BRAM = 0, + parameter RADDR_REG = 0, + parameter RADDR_RESET = 0, + parameter WRITE_FIRST = 0, + parameter INIT_ENABLE = 0, + parameter INIT_FILE = "", + parameter [DATAW-1:0] INIT_VALUE = 0, + parameter ADDRW = (((SIZE) > 1) ? $clog2(SIZE) : 1) +) ( + input wire clk, + input wire reset, + input wire read, + input wire write, + input wire [WRENW-1:0] wren, + input wire [ADDRW-1:0] waddr, + input wire [DATAW-1:0] wdata, + input wire [ADDRW-1:0] raddr, + output wire [DATAW-1:0] rdata +); + localparam WSELW = DATAW / WRENW; + + reg [DATAW-1:0] ram [0:SIZE-1]; + + if (INIT_ENABLE != 0) begin : g_init + if (INIT_FILE != "") begin : g_file + initial $readmemh(INIT_FILE, ram); + end else begin : g_value + initial begin + for (integer i = 0; i < SIZE; ++i) begin + ram[i] = INIT_VALUE; + end + end + end + end + + always @(posedge clk) begin + if (write) begin + for (integer i = 0; i < WRENW; ++i) begin + if (wren[i]) begin + ram[waddr][i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; + end + end + end + end + + assign rdata = ram[raddr]; + +endmodule diff --git a/hw/syn/maxsim/lift_boundaries.py b/hw/syn/maxsim/lift_boundaries.py new file mode 100644 index 0000000000..43b8950879 --- /dev/null +++ b/hw/syn/maxsim/lift_boundaries.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python3 +"""Interface boundary lifting: make interface-port units maxsim classes. + +A class is threaded and wired into the token fabric port by port, so it must be +a module whose ports are plain signals; units such as VX_issue_slice or +VX_cluster have SystemVerilog interface ports instead. This pass gives each +requested unit a boundary made of plain wires, without touching the RTL: it +rewrites a copy of the (preprocessed) sources. + +For each unit U: + * every elaborated instance must have the same parameter values (parameters + listed as droppable -- debug labels such as ISSUE_ID -- are left at their + defaults); + * a parameter-free wrapper U__lift is generated: each interface port becomes + one plain port per signal (per array element, packed), with the direction + taken from the port's modport or, for a bare interface port, from where + the signal is driven (inside the unit -> output); the wrapper instantiates + the interfaces with their elaborated parameters and U with its elaborated + parameter values; + * every instantiation site of U is replaced by glue wires/assigns between + the parent's interfaces and an instance of U__lift. +Behaviour is unchanged: the glue is pure wiring. + +Parameters that are functional instance IDs (e.g. VX_cluster's CLUSTER_ID) +are promoted to 32-bit input ports (port=P): the parameter becomes a port of U, +every child parameter override whose expression uses it is promoted the same +way down the hierarchy, and every affected instantiation passes the value as a +port connection instead (the old default where it was not overridden). Uses in +constant contexts (widths, generate conditions) make the re-elaboration fail, +so an invalid promotion cannot go through silently. + +usage: lift_boundaries.py --srcs --top --out + --unit U[:drop=P1,P2][;port=P3] [--unit ...] +""" +import argparse +import collections +import os +import re +import shutil +import sys + +import pyslang +from pyslang import ast, analysis + +SIG_KINDS = (ast.SymbolKind.Variable, ast.SymbolKind.Net) + + +def read_srcs(path): + defines, incdirs, files = [], [], [] + for line in open(path): + line = line.strip() + if line.startswith("+define+"): + defines.append(line[len("+define+"):]) + elif line.startswith("+incdir+"): + incdirs.append(line[len("+incdir+"):]) + elif line: + files.append(line) + return defines, incdirs, files + + +def q(a): + return "'" + a.replace("'", "'\\''") + "'" if any(c in a for c in " '\"") else a + + +def elaborate(defines, incdirs, files, top): + d = pyslang.driver.Driver() + d.addStandardArgs() + # The Vortex RTL references some nets before their declaration (VX_tcu_core), + # which Vivado and Verilator accept; slang makes it an error unless allowed. + args = ["slang", "--top", top, "--allow-hierarchical-const", "--allow-use-before-declare", + "--ignore-unknown-modules"] + args += [f"-D{x}" for x in defines] + [f"-I{x}" for x in incdirs] + files + if not (d.parseCommandLine(" ".join(q(a) for a in args), pyslang.driver.CommandLineOptions()) + and d.processOptions() and d.parseAllSources()): + sys.exit("lift: slang could not parse the sources") + comp = d.createCompilation() + used = set() + + def visit(sym): + if sym.kind == ast.SymbolKind.Instance: + used.add(sym.definition.name) + return ast.VisitAction.Advance + for t in comp.getRoot().topInstances: + t.visit(visit) + sm = comp.sourceManager + + def unused_only(fname): + # A file that only defines modules outside the elaborated hierarchy: + # errors there cannot affect the design. + names = re.findall(r"^\s*(module|interface|package)\s+(\w+)", open(fname).read(), re.M) + return bool(names) and all(k == "module" and n not in used for k, n in names) + errs = [x for x in comp.getAllDiagnostics() if x.isError() + and not unused_only(str(sm.getFileName(x.location)))] + if errs: + x = errs[0] + sys.exit(f"lift: {len(errs)} elaboration errors (first: {x.code} at " + f"{sm.getFileName(x.location)}:{sm.getLineNumber(x.location)})") + am = analysis.AnalysisManager() + am.analyze(comp) + return d, comp, am + + +def vec(w): + return "" if w == 1 else f"[{w - 1}:0] " + + +def lit(v): + s = str(v) + return s + + +class Unit: + def __init__(self, spec): + name, _, opts = spec.partition(":") + self.name = name + self.drop = set() + self.ports = [] # parameters promoted to input ports (functional instance IDs) + for o in filter(None, opts.split(";")): + k, _, v = o.partition("=") + if k == "drop": + self.drop |= set(filter(None, v.split(","))) + elif k == "port": + self.ports += [x for x in v.split(",") if x] + else: + sys.exit(f"lift: unknown option '{o}' for {name} (U[:drop=P,...][;port=P,...])") + self.insts = [] + + +def params_of(inst, drop=()): + """Overridable value and type parameters of an elaborated instance, as SV text.""" + out = collections.OrderedDict() + for m in inst.body: + if m.name in drop: + continue + if m.kind == ast.SymbolKind.Parameter and not m.isLocalParam: + out[m.name] = lit(m.value) + elif m.kind == ast.SymbolKind.TypeParameter and not m.isLocalParam: + t = str(m.targetType.type) + if not re.match(r"^[A-Za-z_][\w$]*(::[A-Za-z_][\w$]*)*$", t) and not re.match(r"^(logic|bit)( ?\[\d+:\d+\])*$", t): + sys.exit(f"lift: type parameter {inst.hierarchicalPath}.{m.name} = '{t}' has no name to pass") + out[m.name] = t + return out + + +def iface_elements(conn): + target, modport = conn.ifaceConn + if target.kind == ast.SymbolKind.InstanceArray: + return list(target.elements), modport + return [target], modport + + +def signal_dirs(unit_inst, elem, modport_name, am): + """[(signal, 'input'|'output', width)] from the unit's point of view.""" + out = [] + mp = None + if modport_name: + mp = next((m for m in elem.body if m.kind == ast.SymbolKind.Modport and m.name == modport_name), None) + mp_dirs = {} + if mp is not None: + for p in mp: + mp_dirs[p.name] = "input" if str(p.direction).endswith("In") else "output" + upath = unit_inst.hierarchicalPath + for m in elem.body: + if m.kind not in SIG_KINDS: + continue + if m.type.isUnpackedArray: + sys.exit(f"lift: unpacked interface signal {elem.hierarchicalPath}.{m.name} not supported") + if mp is not None: + if m.name not in mp_dirs: + continue + d = mp_dirs[m.name] + else: + drv = am.getDrivers(m) + inside = any(dv.containingSymbol.hierarchicalPath == upath + or dv.containingSymbol.hierarchicalPath.startswith(upath + ".") + or dv.containingSymbol.hierarchicalPath.startswith(upath + "[") for dv in drv) + d = "output" if inside else "input" + out.append((m.name, d, m.type.bitWidth)) + return out + + +def port_range(port): + """Declared interface-array range as text ('[0:3]'), or '' for a single interface.""" + r = port.declaredRange + if not r: + return "", 1 + rr = r[0] + lo, hi = rr.left, rr.right + return f"[{lo}:{hi}]", abs(hi - lo) + 1 + + +def build_wrapper(u, rep, am): + ports, body, inner_conns = [], [], [] + shape = [] # per port: ('plain', name, dir, w) or ('iface', name, rng, n, [(sig, dir, w)]) + for c in rep.portConnections: + p = c.port + if p.kind == ast.SymbolKind.InterfacePort: + elems, mp = iface_elements(c) + rng, n = port_range(p) + sigs = signal_dirs(rep, elems[0], p.modport, am) + idef = p.interfaceDef.name + iparams = params_of(elems[0]) + pa = "" + if iparams: + pa = " #(" + ", ".join(f".{k}({v})" for k, v in iparams.items()) + ")" + body.append(f" {idef}{pa} {p.name} {rng}();".replace(" ();", "();")) + for s, d, w in sigs: + pn = f"{p.name}__{s}" + ports.append(f" {d} wire {('[' + str(n - 1) + ':0]') if rng else ''}{vec(w)}{pn}") + for k in range(n): + ref = f"{p.name}[{k}].{s}" if rng else f"{p.name}.{s}" + pk = f"{pn}[{k}]" if rng else pn + body.append(f" assign {ref} = {pk};" if d == "input" else f" assign {pk} = {ref};") + inner_conns.append(f" .{p.name} ({p.name})") + shape.append(("iface", p.name, rng, n, sigs)) + else: + d = str(p.direction).split(".")[-1].lower() + d = {"in": "input", "out": "output", "inout": "inout"}.get(d, d) + if p.type.isUnpackedArray: + sys.exit(f"lift: unpacked port {u.name}.{p.name} not supported") + w = p.type.bitWidth + ports.append(f" {d} wire {vec(w)}{p.name}") + inner_conns.append(f" .{p.name} ({p.name})") + shape.append(("plain", p.name, d, w)) + for pp in u.ports: + ports.insert(0, f" input wire [31:0] {pp}") + inner_conns.insert(0, f" .{pp} ({pp})") + shape.insert(0, ("promoted", pp)) + prm = params_of(rep, u.drop | set(u.ports)) + pa = (" #(\n" + ",\n".join(f" .{k} ({v})" for k, v in prm.items()) + "\n )") if prm else "" + text = (f"// Generated by lift_boundaries.py: {u.name} with plain ports (FAME-5 fold boundary).\n" + f"module {u.name}__lift (\n" + ",\n".join(ports) + "\n);\n" + "\n".join(body) + "\n\n" + f" {u.name}{pa} inner (\n" + ",\n".join(inner_conns) + "\n );\nendmodule\n") + return text, shape + + +SLICE = re.compile(r"^(.*)\[(.+)\+:(.+)\]$") + + +def elem_expr(e, k, is_array): + e = e.strip() + if not is_array: + return e + m = SLICE.match(e) + if m: + return f"{m.group(1)}[({m.group(2).strip()}) + {k}]" + if re.match(r"^[A-Za-z_]\w*$", e): + return f"{e}[{k}]" + if k == 0: + return e # a single element bound to a one-element array port + sys.exit(f"lift: cannot index connection expression '{e}'") + + +def rewrite_site(u, site_syntax, shape, uid): + """Replacement text for one instantiation statement.""" + inst = site_syntax.instances[0] + iname = str(inst.decl.name).strip() + conns = {} + for c in inst.connections: + if not hasattr(c, "name"): + continue # separator token + nm = str(c.name).strip() + ex = str(c.expr).strip() if c.expr is not None else "" + conns[nm] = ex + overrides = site_overrides(site_syntax) + pre = f"__lift{uid}" + decl, glue, pc = [], [], [] + for kind, *rest in shape: + if kind == "promoted": + pp = rest[0] + if pp not in overrides: + sys.exit(f"lift: {u.name} instance {iname}: promoted parameter {pp} is not overridden at the site") + pc.append(f" .{pp} ({overrides[pp]})") + continue + if kind == "plain": + name, d, w = rest + pc.append(f" .{name} ({conns.get(name, '')})") + continue + name, rng, n, sigs = rest + e = conns.get(name) + if e is None: + sys.exit(f"lift: {u.name} instance {iname}: interface port {name} not connected by name") + for s, d, w in sigs: + wn = f"{pre}_{name}__{s}" + decl.append(f"wire {('[' + str(n - 1) + ':0]') if rng else ''}{vec(w)}{wn};") + for k in range(n): + ek = elem_expr(e, k, bool(rng)) + wk = f"{wn}[{k}]" if rng else wn + glue.append(f"assign {wk} = {ek}.{s};" if d == "input" else f"assign {ek}.{s} = {wk};") + pc.append(f" .{name}__{s} ({wn})") + return ("// lift_boundaries: " + u.name + " -> " + u.name + "__lift\n " + + "\n ".join(decl + glue) + "\n " + + f"{u.name}__lift {iname} (\n" + ",\n".join(pc) + "\n );") + + +def site_overrides(site): + """{param: expr text} of a HierarchyInstantiationSyntax's #(...) named overrides.""" + out = {} + pva = site.parameters + if pva is None: + return out + for it in pva.parameters: + if hasattr(it, "name") and hasattr(it, "expr"): + out[str(it.name).strip()] = str(it.expr).strip() if it.expr is not None else "" + return out + + +def _remove_item(text_len_getter, node, sm): + """(start, end) covering a separated-list item and one adjacent comma.""" + return node.sourceRange.start.offset, node.sourceRange.end.offset + + +class Promoter: + """Parameter-to-port promotion across definitions (see module docstring).""" + + def __init__(self, comp, sm, lifted_sites): + self.comp, self.sm = comp, sm + self.lifted_sites = lifted_sites # site keys rewritten by lifting: skip + self.done = {} # (def, param) -> default text + self.edits = collections.defaultdict(list) + self.site_conv = collections.defaultdict(set) # site key -> {param} converted + self.insts = collections.defaultdict(list) + for t in comp.getRoot().topInstances: + t.visit(self._collect) + + def _collect(self, sym): + if sym.kind == ast.SymbolKind.Instance: + self.insts[sym.definition.name].append(sym) + return ast.VisitAction.Advance + + def key(self, node): + r = node.sourceRange + return (self.sm.getFileName(r.start), r.start.offset) + + def promote(self, defname, param): + if (defname, param) in self.done: + return + rep = self.insts[defname][0] + dsyn = rep.definition.syntax + hdr = dsyn.header + f = self.sm.getFileName(dsyn.sourceRange.start) + text = open(f).read() + default = None + target = None + for it in hdr.parameters.declarations: + if not hasattr(it, "declarators"): + continue + names = [str(dd.name).strip() for dd in it.declarators if hasattr(dd, "name")] + if param in names: + if len(names) != 1: + sys.exit(f"lift: {defname}.{param} shares a declaration; split it first") + dd = [dd for dd in it.declarators if hasattr(dd, "name")][0] + default = str(dd.initializer.expr).strip() if dd.initializer is not None else "0" + target = it + if target is None: + sys.exit(f"lift: no parameter {param} in {defname}") + self.done[(defname, param)] = default + a, b = target.sourceRange.start.offset, target.sourceRange.end.offset + self.edits[f].append(_with_comma(text, a, b)) + op = hdr.ports.openParen.location.offset + 1 + self.edits[f].append((op, op, f"\n input wire [31:0] {param},")) + # children whose overrides use the promoted value + for ch in rep.body: + pass + for cinst in self._children(rep): + site = cinst.syntax.parent + ov = site_overrides(site) + for q, ex in ov.items(): + if re.search(rf"\b{re.escape(param)}\b", ex): + self.promote(cinst.definition.name, q) + + def _children(self, inst): + out = [] + def v(sym): + if sym is inst: + return ast.VisitAction.Advance + if sym.kind == ast.SymbolKind.Instance: + out.append(sym) + return ast.VisitAction.Skip + return ast.VisitAction.Advance + inst.visit(v) + return out + + def convert_sites(self): + """Every instantiation of a promoted (def, param): override -> port connection.""" + for (defname, param), default in self.done.items(): + for inst in self.insts[defname]: + site = inst.syntax.parent + k = self.key(site) + if k in self.lifted_sites or param in self.site_conv[k]: + continue + self.site_conv[k].add(param) + f = k[0] + text = open(f).read() + ov_items = {} + if site.parameters is not None: + for it in site.parameters.parameters: + if hasattr(it, "name"): + ov_items[str(it.name).strip()] = it + if param in ov_items: + it = ov_items[param] + ex = str(it.expr).strip() + a, b = it.sourceRange.start.offset, it.sourceRange.end.offset + self.edits[f].append(_with_comma(text, a, b)) + else: + ex = default + inst_syn = site.instances[0] + op = inst_syn.openParen.location.offset + 1 + self.edits[f].append((op, op, f"\n .{param} ({ex}),")) + + +def _with_comma(text, a, b): + """Deletion range for a list item at [a, b) plus its separating comma.""" + j = b + while j < len(text) and text[j] in " \t\r\n": + j += 1 + if j < len(text) and text[j] == ",": + return (a, j + 1, "") + i = a - 1 + while i >= 0 and text[i] in " \t\r\n": + i -= 1 + if i >= 0 and text[i] == ",": + return (i, b, "") + return (a, b, "") + + +def apply_edits(text, edits): + """Apply (start, end, repl) edits; overlapping pure deletions are merged.""" + edits = sorted(edits, key=lambda e: (e[0], e[1])) + merged = [] + for s, e, t in edits: + if merged and s < merged[-1][1]: + ps, pe, pt = merged[-1] + if t or pt: + sys.exit(f"lift: overlapping edits at {s}") + merged[-1] = (ps, max(pe, e), "") + else: + merged.append((s, e, t)) + for s, e, t in reversed(merged): + text = text[:s] + t + text[e:] + return text + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--srcs", required=True) + ap.add_argument("--top", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--unit", action="append", required=True) + a = ap.parse_args() + a.out = os.path.abspath(a.out) + defines, incdirs, files = read_srcs(a.srcs) + d, comp, am = elaborate(defines, incdirs, files, a.top) + sm = comp.sourceManager + units = {u.name: u for u in (Unit(s) for s in a.unit)} + + def visit(sym): + if sym.kind == ast.SymbolKind.Instance and sym.definition.name in units: + units[sym.definition.name].insts.append(sym) + return ast.VisitAction.Advance + # Only the elaborated hierarchy: slang also builds placeholder instances of + # uninstantiated definitions, which must not count. + for t in comp.getRoot().topInstances: + t.visit(visit) + + edits = collections.defaultdict(list) # file -> [(start, end, text)] + wrappers = {} + uid = 0 + lifted_sites = set() + for u in units.values(): + if not u.insts: + sys.exit(f"lift: no instance of {u.name}") + skip = u.drop | set(u.ports) + groups = collections.Counter(tuple(params_of(i, skip).items()) for i in u.insts) + if len(groups) > 1: + keys = list(params_of(u.insts[0], skip).keys()) + diff = [k for k in keys if len({dict(g)[k] for g in groups}) > 1] + sys.exit(f"lift: instances of {u.name} differ in {diff}; drop them if they are labels") + text, shape = build_wrapper(u, u.insts[0], am) + wrappers[u.name] = text + seen = set() + for i in u.insts: + site = i.syntax.parent # HierarchyInstantiationSyntax + rng = site.sourceRange + key = (sm.getFileName(rng.start), rng.start.offset) + if key in seen: + continue + seen.add(key) + lifted_sites.add(key) + if len(site.instances) != 1: + sys.exit(f"lift: {u.name}: multi-instance statement at {key}") + edits[key[0]].append((rng.start.offset, rng.end.offset, rewrite_site(u, site, shape, uid))) + uid += 1 + print(f"lift: {u.name}: {len(u.insts)} instances, {len(seen)} sites, " + f"{sum(1 for s in shape if s[0] == 'iface')} interface ports, " + f"{sum(len(s[4]) for s in shape if s[0] == 'iface')} lifted signals") + + pr = Promoter(comp, sm, lifted_sites) + for u in units.values(): + for pp in u.ports: + pr.promote(u.name, pp) + pr.convert_sites() + for f, es in pr.edits.items(): + edits[f].extend(es) + if pr.done: + print("lift: promoted " + ", ".join(f"{d}.{p}" for d, p in pr.done)) + + os.makedirs(os.path.join(a.out, "src"), exist_ok=True) + out_files = [] + by_real = {os.path.realpath(str(k)): k for k in edits} + applied, matched = 0, set() + for f in files: + dst = os.path.join(a.out, "src", os.path.basename(f)) + key = by_real.get(os.path.realpath(f)) + if key is not None: + open(dst, "w").write(apply_edits(open(f).read(), edits[key])) + applied += 1 + matched.add(key) + else: + shutil.copyfile(f, dst) + out_files.append(dst) + for name, text in wrappers.items(): + p = os.path.join(a.out, "src", f"{name}__lift.sv") + open(p, "w").write(text) + out_files.append(p) + with open(os.path.join(a.out, "srcs.f"), "w") as o: + for x in defines: + o.write(f"+define+{x}\n") + for x in incdirs: + o.write(f"+incdir+{x}\n") + o.write(f"+incdir+{os.path.join(a.out, 'src')}\n") + for x in out_files: + o.write(x + "\n") + print(f"lift: wrote {len(out_files)} files ({applied} of {len(edits)} rewritten) to {a.out}") + if applied != len(edits): + sys.exit(f"lift: edits for unmatched files: {sorted(set(edits) - matched)}") + + +if __name__ == "__main__": + main() diff --git a/hw/syn/maxsim/maxsim.py b/hw/syn/maxsim/maxsim.py new file mode 100644 index 0000000000..4389bd5cdc --- /dev/null +++ b/hw/syn/maxsim/maxsim.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Compile a target to a maxsim simulator with CIRCT. + +usage: maxsim.py --srcs --top --out [--circt-bin ] + [ ...] + +The target is imported with circt-verilog and flattened, but for the classes: +the modules simulated as models of their own, each class's instances threaded +through one copy (FAME-5). A class may contain others. Each module with class +instances, innermost first, is cut into a netlist of its own logic and the +models (maxsim-extract-models); the target's netlist is then composed into the +simulator (maxsim-compose): one threaded module per class, the rest of the +target as a model of one thread, and a fabric of token channels between them. + +Writes /maxsim_sim.sv, the module `maxsim_sim` with a channel per target +port (`_valid`, `_ready`, `_bits`) and a `profile_finish` bit per class; +/maxsim_sim.json, its port list and the classes' thread counts, in the +order of those bits; and /maxsim_sim.vh, their number for the kernel. +""" +import argparse +import json +import os +import re +import subprocess +import sys + +# Passes each model gets before it is threaded: register constants, shared +# adders and narrowed low bits matter for area once every register of a model +# is replicated per thread. +OPTIMIZE = ['hw.module(seq-reg-const-prop)', 'hw.module(comb-share-adders)', 'canonicalize', + 'hw.module(comb-low-bits-narrowing)', 'canonicalize', 'cse'] + + +def run(cmd, **kw): + print('+', ' '.join(cmd), file=sys.stderr) + return subprocess.run(cmd, check=True, text=True, **kw) + + +def publish(src, public): + """The design with only the `public` modules public.""" + s = re.sub(r'hw\.module @', 'hw.module private @', src) + for m in public: + s, n = re.subn(r'hw\.module private @' + re.escape(m) + r'\(', f'hw.module @{m}(', s, count=1) + if not n: + sys.exit(f'no module {m} in the design') + return s + + +def instantiated(text, module, names): + """The modules of `names` the module `module` instantiates.""" + m = re.search(r'hw\.module (?:private )?@' + re.escape(module) + r'\(.*?\n \}\n', text, re.S) + return {n for n in re.findall(r'hw\.instance "[^"]*" @(\w+)\(', m.group(0)) if n in names} + + +def signature(text, module): + """A module's ports: [(direction, name, width)].""" + m = re.search(r'hw\.module (?:private )?@' + re.escape(module) + r'\((.*?)\) (?:attributes .*?)?\{', + text, re.S) + ports = [] + for d, n, t in re.findall(r'(in|out) %?(\w+) : ([^,)]+)', m.group(1)): + w = re.fullmatch(r'i(\d+)', t.strip()) + ports.append((d, n, int(w.group(1)) if w else 0)) + return ports + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + ap.add_argument('--srcs', required=True, help='source list: +define+, +incdir+ and files') + ap.add_argument('--top', required=True) + ap.add_argument('--out', required=True) + ap.add_argument('--circt-bin', default='') + ap.add_argument('classes', nargs='+') + a = ap.parse_args() + tool = lambda t: os.path.join(a.circt_bin, t) if a.circt_bin else t + os.makedirs(a.out, exist_ok=True) + out = lambda f: os.path.join(a.out, f) + + defines, incdirs, files = [], [], [] + for line in open(a.srcs).read().split('\n'): + if line.startswith('+define+'): + defines.append('-D' + line[len('+define+'):]) + elif line.startswith('+incdir+'): + incdirs.append('-I' + line[len('+incdir+'):]) + elif line.strip(): + files.append(line.strip()) + design = out('design.mlir') + run([tool('circt-verilog'), '--ir-hw', f'--top={a.top}', '--allow-use-before-declare', + '--allow-hierarchical-const', '--ignore-uninstantiated-modules', '--detect-memories', + '--preserve-unpacked-ports', *defines, *incdirs, *files, '-o', design]) + + # Flat but for the classes. + flat = out('flat.mlir') + with open(flat + '.in', 'w') as f: + f.write(publish(open(design).read(), [a.top] + a.classes)) + run([tool('circt-opt'), flat + '.in', '--symbol-dce', '--hw-flatten-modules', '--symbol-dce', + '--canonicalize', '--cse', '-o', flat]) + os.remove(flat + '.in') + text = open(flat).read() + + # Each module with class instances, innermost first, becomes a netlist. + classes = set(a.classes) + children = {m: instantiated(text, m, classes) for m in [a.top] + a.classes} + order, seen = [], set() + + def visit(m): + if m in seen: + return + seen.add(m) + for c in sorted(children[m]): + visit(c) + if children[m]: + order.append(m) + visit(a.top) + unused = classes - seen + if unused: + sys.exit(f'classes not in {a.top}: {" ".join(sorted(unused))}') + passes = [f'maxsim-extract-models{{module={m} models={",".join(sorted(children[m]))} netlist}}' + for m in order] + pipeline = passes + OPTIMIZE + [f'maxsim-compose{{module={a.top} sim=maxsim_sim}}', + 'canonicalize', 'cse'] + sim = out('maxsim_sim.mlir') + run([tool('circt-opt'), flat, f'--pass-pipeline=builtin.module({",".join(pipeline)})', '-o', sim]) + sim_text = open(sim).read() + run([tool('firtool'), '--format=mlir', sim, '-o', out('maxsim_sim.sv')]) + + threads = re.search(r'maxsim\.threads = \[(.*?)\]', sim_text).group(1) + info = { + 'top': a.top, + 'threads': dict(t.strip('"').split('=') for t in re.findall(r'"[^"]*"', threads)), + 'ports': [{'dir': d, 'name': n, 'width': w} for d, n, w in signature(sim_text, 'maxsim_sim')], + } + for name, n in info['threads'].items(): + info['threads'][name] = int(n) + with open(out('maxsim_sim.json'), 'w') as f: + json.dump(info, f, indent=1) + # The kernel's view of it: how many classes report their finishes, bit c + # of `profile_finish` for the c-th class of the JSON's `threads`. + with open(out('maxsim_sim.vh'), 'w') as f: + f.write('// Generated by maxsim.py. Do not edit.\n' + f'`define MAXSIM_NUM_CLASSES {len(info["threads"])}\n') + print(f'maxsim: {", ".join(f"{k} x{v}" for k, v in info["threads"].items())}', file=sys.stderr) + + +if __name__ == '__main__': + main() diff --git a/hw/syn/xilinx/aved/package_ip.tcl b/hw/syn/xilinx/aved/package_ip.tcl index 0e6bdb33cc..b1b9bb0ed6 100644 --- a/hw/syn/xilinx/aved/package_ip.tcl +++ b/hw/syn/xilinx/aved/package_ip.tcl @@ -14,9 +14,9 @@ # Package the AFU as an IP-XACT component. The V80 linker consumes the # component.xml directly, so there is no intermediate object container. -if { $::argc != 4 } { - puts "ERROR: Program \"$::argv0\" requires 4 arguments!\n" - puts "Usage: $::argv0 \n" +if { $::argc != 4 && $::argc != 5 } { + puts "ERROR: Program \"$::argv0\" requires 4 or 5 arguments!\n" + puts "Usage: $::argv0 \[no-vendor-ip\]\n" exit 1 } @@ -24,6 +24,9 @@ set krnl_name [lindex $::argv 0] set vcs_file [lindex $::argv 1] set build_dir [lindex $::argv 2] set part [lindex $::argv 3] +# A kernel whose RTL instantiates no vendor floating-point core (a maxsim +# simulator, whose FPU is soft logic) is packaged without them. +set vendor_ip [expr {$::argc != 5 || [lindex $::argv 4] ne "no-vendor-ip"}] set tool_dir $::env(TOOL_DIR) @@ -36,9 +39,11 @@ set path_to_packaged "${build_dir}/${krnl_name}_ip" # and RTL elaboration fails at link time, after the IP looks fine. # xilinx_ip_gen.tcl opens and closes its own in-memory project, so it has to # run before the packaging project below. -set argv [list ${build_dir}/ip $part] -set argc 2 -source ${tool_dir}/xilinx_ip_gen.tcl +if {$vendor_ip} { + set argv [list ${build_dir}/ip $part] + set argc 2 + source ${tool_dir}/xilinx_ip_gen.tcl +} create_project -force kernel_pack $path_to_tmp_project -part $part @@ -65,11 +70,13 @@ foreach line [split $data "\n"] { # Pull in the floating-point cores generated above so they are packaged # alongside the RTL. The set matches the XRT flow's package_kernel.tcl. -set ip_files [list] -foreach ip {xil_fdiv xil_fma xil_fsqrt xil_fmul xil_fadd} { - lappend ip_files [file normalize "${build_dir}/ip/${ip}/${ip}.xci"] +if {$vendor_ip} { + set ip_files [list] + foreach ip {xil_fdiv xil_fma xil_fsqrt xil_fmul xil_fadd} { + lappend ip_files [file normalize "${build_dir}/ip/${ip}/${ip}.xci"] + } + add_files -verbose -norecurse -fileset [get_filesets sources_1] $ip_files } -add_files -verbose -norecurse -fileset [get_filesets sources_1] $ip_files set_property top $krnl_name [current_fileset] update_compile_order -fileset sources_1 diff --git a/sim/maxsim/Makefile b/sim/maxsim/Makefile new file mode 100644 index 0000000000..234702954f --- /dev/null +++ b/sim/maxsim/Makefile @@ -0,0 +1,53 @@ +include ../common.mk + +# The maxsim kernel in Verilator, as a library the runtime's avedsim target +# links: the same simulator hw/syn/maxsim generates for the image, so a run +# here is the image's run, cycle for cycle, but for device memory's timing. + +DESTDIR ?= $(CURDIR) + +SRC_DIR := $(VORTEX_HOME)/sim/maxsim +MAXSIM_RTL_DIR := $(RTL_DIR)/afu/maxsim +WORK_DIR := $(DESTDIR)/maxsim_emul +GEN_SV := $(WORK_DIR)/gen/maxsim_sim.sv + +CONFIG_FILE := $(DESTDIR)/maxsim_emul_config.stamp + +CXXFLAGS += -std=c++17 -fPIC -O2 -DNDEBUG -I$(SRC_DIR) +LDFLAGS += -shared -pthread + +VL_FLAGS += -O3 -Wno-fatal -Wno-lint -Wno-style -Wno-MULTIDRIVEN --x-initial 0 --x-assign 0 +VL_FLAGS += -I$(MAXSIM_RTL_DIR) -I$(WORK_DIR)/gen +THREADS ?= $(shell python3 -c 'import multiprocessing as mp; print(min(32, mp.cpu_count()))') +VL_FLAGS += -j $(THREADS) + +TOP := maxsim_afu +RTL_SRCS := $(GEN_SV) $(MAXSIM_RTL_DIR)/maxsim_token.sv $(MAXSIM_RTL_DIR)/maxsim_mem.sv \ + $(MAXSIM_RTL_DIR)/maxsim_ctrl.sv $(MAXSIM_RTL_DIR)/maxsim_afu.sv +SRCS := $(SRC_DIR)/maxsim_emul.cpp + +PROJECT := libmaxsim_emul.so + +.PHONY: all lib force clean + +all: lib +lib: $(DESTDIR)/$(PROJECT) + +# The simulator, generated for these CONFIGS by the image's own flow. +$(GEN_SV): force + $(MAKE) -C $(ROOT_DIR)/hw/syn/maxsim generate TARGET=hw MAXSIM_WORK_DIR=$(WORK_DIR) CONFIGS='$(CONFIGS)' \ + $(if $(PERF),PERF=$(PERF)) $(if $(MAXSIM_CLASSES),MAXSIM_CLASSES='$(MAXSIM_CLASSES)') \ + $(if $(MAXSIM_LIFT),MAXSIM_LIFT='$(MAXSIM_LIFT)') $(if $(CIRCT_BIN),CIRCT_BIN=$(CIRCT_BIN)) \ + $(if $(LIFT_PYTHON),LIFT_PYTHON=$(LIFT_PYTHON)) + +$(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(wildcard $(MAXSIM_RTL_DIR)/*) $(CONFIG_FILE) + $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(RTL_SRCS) --top-module $(TOP) \ + $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + +$(CONFIG_FILE): force + @mkdir -p $(@D) + @printf '%s\n' "$(VL_FLAGS)" "$(CXXFLAGS)" "$(CONFIGS)" > $@.tmp + @if ! cmp -s $@.tmp $@; then mv $@.tmp $@; else rm $@.tmp; fi + +clean: + rm -rf $(DESTDIR)/$(PROJECT).obj_dir $(DESTDIR)/$(PROJECT) $(WORK_DIR) diff --git a/sim/maxsim/maxsim_emul.cpp b/sim/maxsim/maxsim_emul.cpp new file mode 100644 index 0000000000..56adc06747 --- /dev/null +++ b/sim/maxsim/maxsim_emul.cpp @@ -0,0 +1,285 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "maxsim_emul.h" + +#include "Vmaxsim_afu.h" +#include "Vmaxsim_afu___024root.h" +#include "verilated.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace vortex; + +double sc_time_stamp() { return 0; } + +namespace { + +uint64_t env_or(const char* name, uint64_t value) { + const char* s = getenv(name); + return (s != nullptr && s[0] != '\0') ? strtoull(s, nullptr, 0) : value; +} + +constexpr uint64_t kPage = 4096; +constexpr uint32_t kBeat = 64; + +// Device memory: sparse, zero where never written. +class memory_t { +public: + void write(uint64_t addr, const uint8_t* data, uint64_t size, const uint8_t* strobe = nullptr) { + for (uint64_t i = 0; i < size; ++i) { + if (strobe == nullptr || strobe[i]) + page(addr + i)[(addr + i) % kPage] = data[i]; + } + } + void read(uint64_t addr, uint8_t* data, uint64_t size) { + for (uint64_t i = 0; i < size; ++i) { + auto it = pages_.find((addr + i) / kPage); + data[i] = (it == pages_.end()) ? 0 : it->second[(addr + i) % kPage]; + } + } + +private: + std::array& page(uint64_t addr) { + auto& p = pages_[addr / kPage]; + return p; + } + std::unordered_map> pages_; +}; + +} // namespace + +class maxsim_emul::Impl { +public: + Impl() + : mem_latency_(env_or("MAXSIM_EMUL_MEM_LATENCY", 100)), + mmio_cycles_(env_or("MAXSIM_EMUL_MMIO_CYCLES", 250)) {} + + ~Impl() { + if (top_) + top_->final(); + if (trace_ != nullptr) + fclose(trace_); + } + + int init() { + // MAXSIM_EMUL_TRACE=: which classes finish a thread, each kernel + // cycle, one byte per cycle of up to 8 classes. + if (const char* path = getenv("MAXSIM_EMUL_TRACE")) + trace_ = fopen(path, "wb"); + context_ = std::make_unique(); + top_ = std::make_unique(context_.get()); + top_->ap_rst_n = 0; + idle_master(); + for (int i = 0; i < 16; ++i) + tick(); + top_->ap_rst_n = 1; + tick(); + return 0; + } + + int reg_write(uint32_t offset, uint32_t value) { + std::lock_guard lock(mutex_); + run(mmio_cycles_); + top_->s_axi_control_awvalid = 1; + top_->s_axi_control_awaddr = offset; + top_->s_axi_control_wvalid = 1; + top_->s_axi_control_wdata = value; + top_->s_axi_control_wstrb = 0xf; + top_->s_axi_control_bready = 1; + bool aw = false, w = false, b = false; + for (int i = 0; i < 1000 && !b; ++i) { + top_->ap_clk = 0; + top_->eval(); + bool aw_fire = top_->s_axi_control_awvalid && top_->s_axi_control_awready; + bool w_fire = top_->s_axi_control_wvalid && top_->s_axi_control_wready; + b = top_->s_axi_control_bvalid; + edge(); + aw |= aw_fire; + w |= w_fire; + top_->s_axi_control_awvalid = !aw; + top_->s_axi_control_wvalid = !w; + } + top_->s_axi_control_bready = 0; + return b ? 0 : -1; + } + + int reg_read(uint32_t offset, uint32_t* value) { + std::lock_guard lock(mutex_); + run(mmio_cycles_); + top_->s_axi_control_arvalid = 1; + top_->s_axi_control_araddr = offset; + top_->s_axi_control_rready = 1; + bool r = false; + for (int i = 0; i < 1000 && !r; ++i) { + top_->ap_clk = 0; + top_->eval(); + bool ar_fire = top_->s_axi_control_arvalid && top_->s_axi_control_arready; + r = top_->s_axi_control_rvalid; + if (r) + *value = top_->s_axi_control_rdata; + edge(); + if (ar_fire) + top_->s_axi_control_arvalid = 0; + } + top_->s_axi_control_rready = 0; + return r ? 0 : -1; + } + + int mem_write(uint64_t addr, const void* data, uint64_t size) { + std::lock_guard lock(mutex_); + memory_.write(addr, static_cast(data), size); + return 0; + } + + int mem_read(uint64_t addr, void* data, uint64_t size) { + std::lock_guard lock(mutex_); + memory_.read(addr, static_cast(data), size); + return 0; + } + +private: + struct request_t { + uint64_t addr; + uint64_t at; + }; + + void idle_master() { + top_->s_axi_control_awvalid = 0; + top_->s_axi_control_wvalid = 0; + top_->s_axi_control_bready = 0; + top_->s_axi_control_arvalid = 0; + top_->s_axi_control_rready = 0; + } + + void run(uint64_t cycles) { + for (uint64_t i = 0; i < cycles; ++i) + tick(); + } + + void tick() { + top_->ap_clk = 0; + top_->eval(); + edge(); + } + + // The memory's side of the kernel's AXI4 port: every request accepted, and + // answered in order once its latency has passed. + void drive_memory() { + top_->m_axi_mem_0_awready = 1; + top_->m_axi_mem_0_wready = 1; + top_->m_axi_mem_0_arready = 1; + bool r = !reads_.empty() && reads_.front().at <= cycle_; + top_->m_axi_mem_0_rvalid = r; + if (r) { + uint8_t beat[kBeat]; + memory_.read(reads_.front().addr, beat, kBeat); + std::memcpy(top_->m_axi_mem_0_rdata.data(), beat, kBeat); + } + top_->m_axi_mem_0_rlast = 1; + top_->m_axi_mem_0_rid = 0; + top_->m_axi_mem_0_rresp = 0; + top_->m_axi_mem_0_bvalid = !writes_.empty() && writes_.front() <= cycle_; + top_->m_axi_mem_0_bid = 0; + top_->m_axi_mem_0_bresp = 0; + } + + // The rising edge: the handshakes as the kernel saw them before it, then + // the edge itself. + void edge() { + drive_memory(); + top_->eval(); + bool ar = top_->m_axi_mem_0_arvalid; + uint64_t araddr = top_->m_axi_mem_0_araddr; + bool aw = top_->m_axi_mem_0_awvalid; + uint64_t awaddr = top_->m_axi_mem_0_awaddr; + bool w = top_->m_axi_mem_0_wvalid; + uint8_t wdata[kBeat], strobe[kBeat]; + if (w) { + std::memcpy(wdata, top_->m_axi_mem_0_wdata.data(), kBeat); + uint64_t s = top_->m_axi_mem_0_wstrb; + for (uint32_t i = 0; i < kBeat; ++i) + strobe[i] = (s >> i) & 1; + } + bool r_fire = top_->m_axi_mem_0_rvalid && top_->m_axi_mem_0_rready; + bool b_fire = top_->m_axi_mem_0_bvalid && top_->m_axi_mem_0_bready; + + if (trace_ != nullptr) + fputc(top_->rootp->maxsim_afu__DOT__profile_finish & 0xff, trace_); + top_->ap_clk = 1; + top_->eval(); + ++cycle_; + + if (ar) + reads_.push_back({araddr, cycle_ + mem_latency_}); + if (r_fire) + reads_.pop_front(); + if (aw) + aw_.push_back(awaddr); + if (w) { + std::array beat; + std::memcpy(beat.data(), wdata, kBeat); + std::memcpy(beat.data() + kBeat, strobe, kBeat); + w_.push_back(beat); + } + while (!aw_.empty() && !w_.empty()) { + memory_.write(aw_.front(), w_.front().data(), kBeat, w_.front().data() + kBeat); + aw_.pop_front(); + w_.pop_front(); + writes_.push_back(cycle_ + mem_latency_); + } + if (b_fire) + writes_.pop_front(); + } + + std::unique_ptr context_; + std::unique_ptr top_; + std::mutex mutex_; + memory_t memory_; + uint64_t cycle_ = 0; + uint64_t mem_latency_, mmio_cycles_; + FILE* trace_ = nullptr; + std::deque reads_; + std::deque aw_; + std::deque> w_; + std::deque writes_; +}; + +maxsim_emul::maxsim_emul() : impl_(new Impl()) {} + +maxsim_emul::~maxsim_emul() { delete impl_; } + +int maxsim_emul::init() { return impl_->init(); } + +int maxsim_emul::reg_write(uint32_t offset, uint32_t value) { + return impl_->reg_write(offset, value); +} + +int maxsim_emul::reg_read(uint32_t offset, uint32_t* value) { + return impl_->reg_read(offset, value); +} + +int maxsim_emul::mem_write(uint64_t addr, const void* data, uint64_t size) { + return impl_->mem_write(addr, data, size); +} + +int maxsim_emul::mem_read(uint64_t addr, void* data, uint64_t size) { + return impl_->mem_read(addr, data, size); +} diff --git a/sim/maxsim/maxsim_emul.h b/sim/maxsim/maxsim_emul.h new file mode 100644 index 0000000000..f5b0adc522 --- /dev/null +++ b/sim/maxsim/maxsim_emul.h @@ -0,0 +1,45 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace vortex { + +// The maxsim kernel (hw/rtl/afu/maxsim) simulated in-process, as the +// runtime's avedsim target reaches it: its control registers, and the device +// memory behind its AXI4 port. The kernel clock runs only while the host +// touches a register, each access costing the kernel cycles a register access +// over PCIe would (MAXSIM_EMUL_MMIO_CYCLES, 250 by default), so a host that +// polls a register is what lets the target run. +class maxsim_emul { +public: + maxsim_emul(); + ~maxsim_emul(); + + int init(); + + int reg_write(uint32_t offset, uint32_t value); + int reg_read(uint32_t offset, uint32_t* value); + + // Device memory, by device address. + int mem_write(uint64_t addr, const void* data, uint64_t size); + int mem_read(uint64_t addr, void* data, uint64_t size); + +private: + class Impl; + Impl* impl_; +}; + +} // namespace vortex diff --git a/sw/common/maxsim_regs.h b/sw/common/maxsim_regs.h new file mode 100644 index 0000000000..55532d0e65 --- /dev/null +++ b/sw/common/maxsim_regs.h @@ -0,0 +1,55 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The maxsim simulator's register map (hw/rtl/afu/maxsim/maxsim_regs.vh). +// Byte offsets on the kernel's AXI4-Lite control port. 0x040-0x0BF is left +// empty: on the V80 the PCIe block's MSI-X table sits at that offset of the BAR +// the first kernel is mapped at, and answers those accesses itself. + +#pragma once + +#define MAXSIM_ID 0x4D58534Du +#define MAXSIM_VERSION 2u + +#define MAXSIM_REG_ID 0x000u +#define MAXSIM_REG_VERSION 0x004u +#define MAXSIM_REG_CTRL 0x008u // bit 0: the target runs +#define MAXSIM_REG_STATUS 0x00Cu +#define MAXSIM_REG_CMD_ADDR 0x010u // DCR address of the next command +#define MAXSIM_REG_CMD_DATA 0x014u // DCR data, or reset cycles +#define MAXSIM_REG_CMD 0x018u // write: queue a MAXSIM_CMD_* +#define MAXSIM_REG_DCR_RSP 0x01Cu // the last DCR read's response +#define MAXSIM_REG_TCYCLES_LO 0x020u // target cycles since the kernel's reset +#define MAXSIM_REG_TCYCLES_HI 0x024u +#define MAXSIM_REG_HCYCLES_LO 0x028u // host (kernel clock) cycles +#define MAXSIM_REG_HCYCLES_HI 0x02Cu +#define MAXSIM_REG_MSTALL_LO 0x030u // host cycles waiting for device memory +#define MAXSIM_REG_MSTALL_HI 0x034u +#define MAXSIM_REG_NUM_CLASSES 0x038u // classes of threaded models +#define MAXSIM_REG_MEM_BASE_LO 0x0C0u // device address of target address 0 +#define MAXSIM_REG_MEM_BASE_HI 0x0C4u +#define MAXSIM_REG_MEM_RLAT 0x0C8u // read latency, target cycles +#define MAXSIM_REG_MEM_WLAT 0x0CCu // write latency, target cycles +#define MAXSIM_REG_MEM_MAXR 0x0D0u // most reads in flight +#define MAXSIM_REG_MEM_MAXW 0x0D4u // most writes in flight +#define MAXSIM_REG_FINISH(c) (0x100u + 8u * (c)) // threads of class c finished, lo/hi + +#define MAXSIM_STATUS_BUSY 0x1u // queued, running, or memory in flight +#define MAXSIM_STATUS_FULL 0x2u // the command queue is full +#define MAXSIM_STATUS_RSP 0x4u // a DCR read has its response +#define MAXSIM_STATUS_MEM_ERR 0x8u // the target made a burst + +#define MAXSIM_CMD_DCR_WRITE 1u +#define MAXSIM_CMD_DCR_READ 2u +#define MAXSIM_CMD_START 3u +#define MAXSIM_CMD_RESET 4u diff --git a/sw/runtime/maxsim/Makefile b/sw/runtime/maxsim/Makefile new file mode 100644 index 0000000000..081ad37e50 --- /dev/null +++ b/sw/runtime/maxsim/Makefile @@ -0,0 +1,69 @@ +include ../common.mk + +# TARGET=hw runs a maxsim image (hw/syn/maxsim) on the V80 through VRT; +# TARGET=avedsim simulates the same kernel in-process (sim/maxsim). +ifeq ($(strip $(TARGET)),) +override TARGET := avedsim +endif + +DESTDIR ?= $(CURDIR)/.. + +CONFIG_FILE := $(DESTDIR)/maxsim_rt_config.stamp +SRC_DIR := $(VORTEX_HOME)/sw/runtime/maxsim + +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -I$(INC_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) -I$(RT_COMMON_DIR) +CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) +CXXFLAGS += -fPIC +CXXFLAGS += $(CONFIGS) + +LDFLAGS += -shared -pthread -Wl,-rpath,'$$ORIGIN' + +SRCS := $(SRC_DIR)/vortex.cpp $(RT_COMMON_DIR)/utils.cpp $(SIM_COMMON_DIR)/cmd_processor.cpp + +ifeq ($(TARGET), avedsim) +CXXFLAGS += -DMAXSIM_EMUL -I$(SIM_DIR)/maxsim +LDFLAGS += -L$(DESTDIR) -lmaxsim_emul +EMUL := $(DESTDIR)/libmaxsim_emul.so +else ifeq ($(TARGET), hw) +# VRT's public headers include and from +# versioned subdirectories. +CXXFLAGS += $(if $(VRT_HOME),-I$(VRT_HOME)/include) $(shell pkg-config --cflags jsoncpp libxml-2.0 2>/dev/null) +LDFLAGS += $(if $(VRT_HOME),-L$(VRT_HOME)/lib) -lvrt -lvrtdpp +else +$(error TARGET must be hw or avedsim (got '$(TARGET)')) +endif + +ifdef DEBUG + CXXFLAGS += -g -O0 +else + CXXFLAGS += -O2 -DNDEBUG +endif + +PROJECT := libvortex-maxsim.so + +.PHONY: all force driver clean-driver clean-runtime clean + +all: $(DESTDIR)/$(PROJECT) + +driver: $(EMUL) + +$(DESTDIR)/libmaxsim_emul.so: force + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/maxsim lib CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) + +$(DESTDIR)/$(PROJECT): $(SRCS) $(EMUL) $(CONFIG_FILE) + $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ + +$(CONFIG_FILE): force + @mkdir -p $(@D) + @TMP=$@.tmp.$$$$ ; \ + printf '%s\n' "$(CXXFLAGS)" "$(TARGET)" > $$TMP ; \ + if ! cmp -s $$TMP $@ 2>/dev/null; then mv -f $$TMP $@ ; else rm -f $$TMP ; fi + +clean-driver: + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/maxsim clean + +clean-runtime: + rm -f $(DESTDIR)/$(PROJECT) + +clean: clean-driver clean-runtime diff --git a/sw/runtime/maxsim/vortex.cpp b/sw/runtime/maxsim/vortex.cpp new file mode 100644 index 0000000000..2f3236b14a --- /dev/null +++ b/sw/runtime/maxsim/vortex.cpp @@ -0,0 +1,537 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ============================================================================ +// maxsim backend — a pure transport HAL (see callbacks.h), like the firesim +// backend: the target carries no Command Processor, so the regfile surface +// comes from the functional CommandProcessor model, and its hooks reach the +// maxsim simulator. Device memory is the simulator's, in device memory at +// MAXSIM_REG_MEM_BASE; the target's DCR, start and busy are its command queue +// and status (sw/common/maxsim_regs.h). +// +// TARGET=hw runs the image on a V80 through VRT: registers over AXI4-Lite, +// device memory over a QDMA queue. TARGET=avedsim simulates the same kernel +// in-process (sim/maxsim). +// ============================================================================ + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef MAXSIM_EMUL +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +using namespace vortex; + +namespace { + +// Where target address 0 is in device memory: the start of HBM, which the +// kernel's memory port reaches through the MEM tag (hw/syn/xilinx/platforms/ +// v80.toml). The target's 4 GB lie in HBM0..HBM7, which nothing else uses. +constexpr uint64_t kMemBase = 0x4000000000ull; + +#ifdef MAXSIM_EMUL + +class transport_t { +public: + int init() { return sim_.init(); } + int reg_write(uint32_t off, uint32_t v) { return sim_.reg_write(off, v); } + int reg_read(uint32_t off, uint32_t* v) { return sim_.reg_read(off, v); } + int mem_write(uint64_t addr, const void* src, uint64_t n) { + return sim_.mem_write(kMemBase + addr, src, n); + } + int mem_read(uint64_t addr, void* dst, uint64_t n) { + return sim_.mem_read(kMemBase + addr, dst, n); + } + +private: + maxsim_emul sim_; +}; + +#else + +#define KERNEL_NAME "maxsim_afu_0" + +// VRT throws; nothing may cross the callbacks' C boundary. +#define VRT_TRY() try { +#define VRT_CATCH(_ret) \ + } \ + catch (const std::exception& _e) { \ + fprintf(stderr, "[VXDRV] VRT exception: %s\n", _e.what()); \ + return _ret; \ + } + +class transport_t { +public: + int init() { + const char* vbin = getenv("VRT_VBIN_PATH"); + if (vbin == nullptr || vbin[0] == '\0') { + fprintf(stderr, "[VXDRV] VRT_VBIN_PATH must name the maxsim .vbin\n"); + return -1; + } + const char* bdf_env = getenv("VRT_DEVICE_BDF"); + std::string bdf = (bdf_env != nullptr) ? bdf_env : ""; + if (bdf.empty()) { + bdf = discover_v80_bdf(); + if (bdf.empty()) { + fprintf(stderr, "[VXDRV] no V80 found on the bus; set VRT_DEVICE_BDF\n"); + return -1; + } + } + VRT_TRY() + device_ = std::make_unique(bdf, vbin, /*program=*/true); + // Clock the kernel at the rate the image was timed for. + const uint64_t timed_hz = device_->getMaxFrequency(); + device_->setFrequency(timed_hz); + const uint64_t clk_hz = device_->getFrequency(); + if (clk_hz == 0 || clk_hz > timed_hz + timed_hz / 200) { + fprintf(stderr, "[VXDRV] user clock reads %lu Hz, image is timed for %lu Hz\n", + (unsigned long)clk_hz, (unsigned long)timed_hz); + return -1; + } + printf("[VXDRV] maxsim kernel clock %lu Hz\n", (unsigned long)clk_hz); + kernel_ = std::make_unique(*device_, KERNEL_NAME); + // A QDMA queue for device memory: the buffer's own memory is the + // smallest allocation, outside the target's 4 GB, and only its queue is + // used, at the target's addresses. + session_ = std::make_unique(); + auto vdev = session_->getDeviceByBdf(bdf); + dma_ = std::make_unique(vdev.openHbmBuffer(/*region=*/8, kDmaChunk)); + fd_ = dma_->getFd(); + VRT_CATCH(-1) + // The host side of the transfers: a kernel DMA buffer, mapped. + struct slash_qdma_buf_create create = {}; + create.size = sizeof(create); + create.flags = O_CLOEXEC; + create.length = kStaging; + staging_fd_ = ioctl(fd_, SLASH_QDMA_IOCTL_BUF_CREATE, &create); + if (staging_fd_ < 0) { + fprintf(stderr, "[VXDRV] cannot create a QDMA buffer: %s\n", strerror(errno)); + return -1; + } + void* p = mmap(nullptr, kStaging, PROT_READ | PROT_WRITE, MAP_SHARED, staging_fd_, 0); + if (p == MAP_FAILED) { + fprintf(stderr, "[VXDRV] cannot map the QDMA buffer: %s\n", strerror(errno)); + return -1; + } + staging_ = static_cast(p); + return 0; + } + + ~transport_t() { + if (staging_ != nullptr) + munmap(staging_, kStaging); + if (staging_fd_ >= 0) + close(staging_fd_); + } + + int reg_write(uint32_t off, uint32_t v) { + VRT_TRY() + kernel_->write(off, v); + VRT_CATCH(-1) + return 0; + } + + int reg_read(uint32_t off, uint32_t* v) { + VRT_TRY() + *v = kernel_->read(off); + VRT_CATCH(-1) + return 0; + } + + // QDMA moves whole 4 KB pages; a partial one is read, merged and written. + int mem_write(uint64_t addr, const void* src, uint64_t n) { + std::lock_guard lock(mutex_); + auto* bytes = static_cast(src); + while (n != 0) { + const uint64_t page = addr & ~(kPage - 1); + const uint64_t off = addr - page; + const uint64_t span = std::min(kDmaChunk - off, n); + const uint64_t len = (off + span + kPage - 1) & ~(kPage - 1); + if (off != 0 || (span % kPage) != 0) { + // Only the first and last pages can be partial. + if (dma(page, staging_, kPage, false) != 0 || + (len > kPage && + dma(page + len - kPage, staging_ + len - kPage, kPage, false) != 0)) + return -1; + } + std::memcpy(staging_ + off, bytes, span); + if (dma(page, staging_, len, true) != 0) + return -1; + addr += span; + bytes += span; + n -= span; + } + return 0; + } + + int mem_read(uint64_t addr, void* dst, uint64_t n) { + std::lock_guard lock(mutex_); + auto* bytes = static_cast(dst); + while (n != 0) { + const uint64_t page = addr & ~(kPage - 1); + const uint64_t off = addr - page; + const uint64_t span = std::min(kDmaChunk - off, n); + const uint64_t len = (off + span + kPage - 1) & ~(kPage - 1); + if (dma(page, staging_, len, false) != 0) + return -1; + std::memcpy(bytes, staging_ + off, span); + addr += span; + bytes += span; + n -= span; + } + return 0; + } + +private: + static constexpr uint64_t kPage = 4096; + static constexpr uint64_t kDmaChunk = 64ull << 20; + static constexpr uint64_t kStaging = kDmaChunk + kPage; + + // `len` bytes at target address `addr`, from or to `host` in the staging + // buffer, both page aligned. + int dma(uint64_t addr, uint8_t* host, uint64_t len, bool to_device) { + struct slash_qdma_transfer xfer = {}; + xfer.size = sizeof(xfer); + xfer.count = 1; + xfer.xfers[0].qpair_index = 0; + xfer.xfers[0].direction = to_device ? SLASH_QDMA_XFER_H2C : SLASH_QDMA_XFER_C2H; + xfer.xfers[0].buf_fd = staging_fd_; + xfer.xfers[0].buf_offset = static_cast(host - staging_); + xfer.xfers[0].dev_addr = kMemBase + addr; + xfer.xfers[0].length = len; + const int n = ioctl(fd_, SLASH_QDMA_QPAIR_IOCTL_TRANSFER, &xfer); + if (n < 0 || static_cast(n) != len) { + fprintf(stderr, "[VXDRV] QDMA %s of %lu bytes at 0x%lx failed: %s\n", + to_device ? "write" : "read", (unsigned long)len, + (unsigned long)(kMemBase + addr), n < 0 ? strerror(errno) : "short"); + return -1; + } + return 0; + } + + std::unique_ptr device_; + std::unique_ptr kernel_; + std::unique_ptr session_; + std::unique_ptr dma_; + int fd_ = -1; + int staging_fd_ = -1; + uint8_t* staging_ = nullptr; + std::mutex mutex_; +}; + +#endif + +uint64_t env_or(const char* name, uint64_t value) { + const char* s = getenv(name); + return (s != nullptr && s[0] != '\0') ? strtoull(s, nullptr, 0) : value; +} + +// What the host spent on the transport: register accesses, and device-memory +// transfers, with their wall time. +struct transport_stats_t { + uint64_t reads = 0, writes = 0, dma_bytes = 0; + double reg_s = 0, dma_s = 0; +}; + +class timer_t { +public: + explicit timer_t(double& total) : total_(total), start_(std::chrono::steady_clock::now()) {} + ~timer_t() { + total_ += std::chrono::duration(std::chrono::steady_clock::now() - start_).count(); + } + +private: + double& total_; + std::chrono::steady_clock::time_point start_; +}; + +// The transport, counted. +class counted_transport_t : public transport_t { +public: + int reg_write(uint32_t off, uint32_t v) { + timer_t t(stats.reg_s); + ++stats.writes; + return transport_t::reg_write(off, v); + } + int reg_read(uint32_t off, uint32_t* v) { + timer_t t(stats.reg_s); + ++stats.reads; + return transport_t::reg_read(off, v); + } + int mem_write(uint64_t addr, const void* src, uint64_t n) { + timer_t t(stats.dma_s); + stats.dma_bytes += n; + return transport_t::mem_write(addr, src, n); + } + int mem_read(uint64_t addr, void* dst, uint64_t n) { + timer_t t(stats.dma_s); + stats.dma_bytes += n; + return transport_t::mem_read(addr, dst, n); + } + transport_stats_t stats; +}; + +} // namespace + +class vx_device { +public: + vx_device() : cp_(make_cp_hooks()) {} + + ~vx_device() { + if (open_) + report(); + for (auto& kv : host_regions_) + free(reinterpret_cast(kv.first)); + host_regions_.clear(); + } + + int init() { + if (transport_.init() != 0) + return -1; + uint32_t id = 0, version = 0; + if (transport_.reg_read(MAXSIM_REG_ID, &id) != 0 || id != MAXSIM_ID) { + fprintf(stderr, "[VXDRV] the kernel is not a maxsim simulator (id 0x%08x)\n", id); + return -1; + } + if (transport_.reg_read(MAXSIM_REG_VERSION, &version) != 0 || version != MAXSIM_VERSION) { + fprintf(stderr, "[VXDRV] the kernel's register map is version %u, this runtime's %u\n", + version, MAXSIM_VERSION); + return -1; + } + // The memory model: a latency-bandwidth pipe, FASED's defaults unless set. + const uint32_t rlat = env_or("MAXSIM_MEM_READ_LATENCY", 30); + const uint32_t wlat = env_or("MAXSIM_MEM_WRITE_LATENCY", 30); + const uint32_t maxr = env_or("MAXSIM_MEM_MAX_READS", 16); + const uint32_t maxw = env_or("MAXSIM_MEM_MAX_WRITES", 16); + if (reg_write(MAXSIM_REG_MEM_BASE_LO, uint32_t(kMemBase)) != 0 || + reg_write(MAXSIM_REG_MEM_BASE_HI, uint32_t(kMemBase >> 32)) != 0 || + reg_write(MAXSIM_REG_MEM_RLAT, rlat) != 0 || + reg_write(MAXSIM_REG_MEM_WLAT, wlat) != 0 || + reg_write(MAXSIM_REG_MEM_MAXR, maxr) != 0 || + reg_write(MAXSIM_REG_MEM_MAXW, maxw) != 0 || + reg_write(MAXSIM_REG_CTRL, 1) != 0) + return -1; + // A target left running by an earlier process is reset, as a card's + // native driver resets its device on open: Vortex counts VX_CFG_RESET_DELAY + // cycles of reset before it takes a request. + if (command(MAXSIM_CMD_RESET, 0, 64) != 0 || wait_idle() != 0) + return -1; + open_ = true; + return 0; + } + + // ----- CP register channel ----- + // No hardware CP: the functional model answers, ticked around each access. + int cp_reg_write(uint32_t off, uint32_t value) { + cp_.mmio_write(off, value); + for (int i = 0; i < 256 && cp_.busy(); ++i) + cp_.tick(); + return 0; + } + + int cp_reg_read(uint32_t off, uint32_t* value) { + for (int i = 0; i < 256 && cp_.busy(); ++i) + cp_.tick(); + *value = cp_.mmio_read(off); + return 0; + } + + // ----- CP-visible host memory ----- + // The CP model reads host memory directly; only device memory crosses the + // transport. + int host_mem_alloc(uint64_t size, void** host_ptr, uint64_t* cp_addr) { + uint64_t asize = aligned_size(size, CACHE_BLOCK_SIZE); + void* ptr = aligned_alloc(CACHE_BLOCK_SIZE, asize); + if (ptr == nullptr) + return -1; + std::lock_guard g(host_mu_); + host_regions_[reinterpret_cast(ptr)] = asize; + *host_ptr = ptr; + *cp_addr = reinterpret_cast(ptr); + return 0; + } + + int host_mem_free(uint64_t cp_addr) { + { + std::lock_guard g(host_mu_); + auto it = host_regions_.find(cp_addr); + if (it == host_regions_.end()) + return -1; + host_regions_.erase(it); + } + free(reinterpret_cast(cp_addr)); + return 0; + } + + int host_mem_pull(uint64_t /*cp_addr*/) { return 0; } + int host_mem_push(uint64_t /*cp_addr*/) { return 0; } + +private: + int reg_write(uint32_t off, uint32_t v) { return transport_.reg_write(off, v); } + + uint64_t read64(uint32_t lo) { + uint32_t l = 0, h = 0, h2 = 0; + do { + transport_.reg_read(lo + 4, &h); + transport_.reg_read(lo, &l); + transport_.reg_read(lo + 4, &h2); + } while (h != h2); + return (uint64_t(h) << 32) | l; + } + + // How fast the simulator ran since the kernel's reset: target cycles, the + // kernel clock's cycles, and those a target cycle waited for device memory. + void report() { + const uint64_t target = read64(MAXSIM_REG_TCYCLES_LO); + const uint64_t host = read64(MAXSIM_REG_HCYCLES_LO); + const uint64_t stall = read64(MAXSIM_REG_MSTALL_LO); + fprintf(stderr, + "[maxsim] target cycles=%lu host cycles=%lu host/target=%.2f memory waits=%lu\n", + (unsigned long)target, (unsigned long)host, + target ? double(host) / double(target) : 0.0, (unsigned long)stall); + // Each class finishes each of its threads once a target cycle, so its + // finishes per host cycle is how busy its one physical copy is. + uint32_t classes = 0; + transport_.reg_read(MAXSIM_REG_NUM_CLASSES, &classes); + std::string line; + for (uint32_t c = 0; c < classes && host != 0; ++c) { + const uint64_t finishes = read64(MAXSIM_REG_FINISH(c)); + char buf[64]; + snprintf(buf, sizeof(buf), " %u:%.1f%%", c, 100.0 * double(finishes) / double(host)); + line += buf; + } + if (!line.empty()) + fprintf(stderr, "[maxsim] class busy:%s\n", line.c_str()); + const auto& st = transport_.stats; + fprintf(stderr, + "[maxsim] host: %lu register reads, %lu writes in %.3f s; %lu bytes of device memory in %.3f s\n", + (unsigned long)st.reads, (unsigned long)st.writes, st.reg_s, + (unsigned long)st.dma_bytes, st.dma_s); + } + + int status(uint32_t* value) { return transport_.reg_read(MAXSIM_REG_STATUS, value); } + + // Queues a command once the queue has room. + int command(uint32_t op, uint32_t addr, uint32_t data) { + uint32_t s = 0; + do { + if (status(&s) != 0) + return -1; + } while (s & MAXSIM_STATUS_FULL); + if (reg_write(MAXSIM_REG_CMD_ADDR, addr) != 0 || reg_write(MAXSIM_REG_CMD_DATA, data) != 0 || + reg_write(MAXSIM_REG_CMD, op) != 0) + return -1; + return 0; + } + + int wait_idle() { + uint32_t s = 0; + do { + if (status(&s) != 0) + return -1; + } while (s & MAXSIM_STATUS_BUSY); + return 0; + } + + void* host_region_ptr(uint64_t addr) { + std::lock_guard g(host_mu_); + if (host_regions_.empty()) + return nullptr; + auto it = host_regions_.upper_bound(addr); + if (it == host_regions_.begin()) + return nullptr; + --it; + if (addr >= it->first && addr < it->first + it->second) + return reinterpret_cast(addr); + return nullptr; + } + + vortex::CommandProcessor::Hooks make_cp_hooks() { + vortex::CommandProcessor::Hooks h; + h.dram_read = [this](uint64_t addr, void* dst, std::size_t bytes) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(dst, hp, bytes); + return; + } + transport_.mem_read(addr, dst, bytes); + }; + h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + transport_.mem_write(addr, src, bytes); + }; + h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { + command(MAXSIM_CMD_DCR_WRITE, addr, value); + }; + h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { + if (command(MAXSIM_CMD_DCR_READ, addr, tag) != 0) + return 0; + uint32_t s = 0, value = 0; + do { + if (status(&s) != 0) + return 0; + } while (!(s & MAXSIM_STATUS_RSP)); + transport_.reg_read(MAXSIM_REG_DCR_RSP, &value); + return value; + }; + h.vortex_start = [this]() { command(MAXSIM_CMD_START, 0, 0); }; + h.vortex_busy = [this]() -> bool { + uint32_t s = 0; + if (status(&s) != 0) + return false; + return (s & MAXSIM_STATUS_BUSY) != 0; + }; +#ifdef VX_CFG_VM_ENABLE + h.mmu_fault_report = true; +#endif + return h; + } + + counted_transport_t transport_; + bool open_ = false; + vortex::CommandProcessor cp_; + std::mutex host_mu_; + std::map host_regions_; +}; + +#include diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 0e46b3444f..86d16d324c 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -269,6 +269,17 @@ else XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) endif +# maxsim: TARGET=hw runs FPGA_BIN_DIR's maxsim.vbin (hw/syn/maxsim) on the V80; +# TARGET=avedsim, the default, simulates the same kernel in-process. +run-maxsim: $(PROJECT) kernel.vxbin + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/maxsim DESTDIR=$(VORTEX_RT_LIB) TARGET=$(if $(filter hw,$(TARGET)),hw,avedsim) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/maxsim.vbin LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=maxsim ./$(PROJECT) $(OPTS) +else + LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=maxsim ./$(PROJECT) $(OPTS) +endif + run-opae: $(PROJECT) kernel.vxbin $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/opae DESTDIR=$(VORTEX_RT_LIB) SCOPE_JSON_PATH=$(VORTEX_RT_LIB)/scope.json OPAE_DRV_PATHS=$(OPAE_DRV_PATHS) LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=opae ./$(PROJECT) $(OPTS) From 04c5d428526694fb1633d3bcdee1780332b7549e Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 17:49:28 -0700 Subject: [PATCH 182/336] firesim: remove the FAME-5 lane and its RTL hooks maxsim replaces FireSim's FAME-5 lane: the FireSim backend goes back to running Vortex as an unthreaded BlackBox, and the lane's conversion flow (hw/syn/firesim/fame5: sv2v, Yosys, FIRRTL regeneration), its target config and its FAME5 make variables are gone. The lane's VX_CFG_FAME5 hooks in the RTL are removed with it, restoring every file to its upstream state: socket and core indices as ports, the VX_socket_top wrapper, and the async-BRAM patch switch were simulator-only changes to the hardware. maxsim does the same on its own copy of the sources (lift_boundaries.py port=SOCKET_ID / port=CORE_ID, and the RAM the patch stands for), so Vortex's RTL serves every flow unchanged. docs/designs/firesim_integration.md now points to docs/designs/maxsim.md for model multithreading. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 803697377ac4278eef3d4020d256d79e4135b0b0) Integration (phase 2): the Vortex RTL is left as it is. The FAME-5 hooks this commit removes were not replayed (35d3e8a53, 41b59d5a5), so the RTL already is in the upstream state it restores. --- docs/assets/img/firesim_build_flow.svg | 79 +-- docs/assets/img/firesim_fame5_threading.svg | 135 ----- docs/assets/img/firesim_fpga_architecture.svg | 34 +- docs/assets/img/firesim_memory_system.svg | 4 +- docs/assets/img/firesim_system_stack.svg | 15 +- docs/assets/img/firesim_timing_model.svg | 1 - docs/designs/firesim_integration.md | 223 ++------ hw/syn/firesim/Makefile | 116 +--- hw/syn/firesim/fame5/FameRam.v | 98 ---- hw/syn/firesim/fame5/annotate_blackboxes.py | 23 - hw/syn/firesim/fame5/annotate_threads.py | 35 -- hw/syn/firesim/fame5/export_firrtl.py | 56 -- hw/syn/firesim/fame5/fix_firrtl.py | 227 -------- hw/syn/firesim/fame5/gen_rams.py | 231 -------- hw/syn/firesim/fame5/lift_boundaries.py | 537 ------------------ hw/syn/firesim/fame5/name_kept.py | 54 -- hw/syn/firesim/fame5/selfref_wires.py | 42 -- hw/syn/firesim/fame5/shorten_fir.py | 75 --- hw/syn/firesim/fame5/splice.py | 68 --- hw/syn/firesim/fame5/to_firrtl.sh | 67 --- .../firesim/src/main/scala/VortexTarget.scala | 5 - sim/firesim/Makefile | 13 +- sim/firesim/firesim_sim.cpp | 4 +- sw/runtime/firesim/Makefile | 3 +- tests/regression/common.mk | 2 +- 25 files changed, 89 insertions(+), 2058 deletions(-) delete mode 100644 docs/assets/img/firesim_fame5_threading.svg delete mode 100644 hw/syn/firesim/fame5/FameRam.v delete mode 100644 hw/syn/firesim/fame5/annotate_blackboxes.py delete mode 100644 hw/syn/firesim/fame5/annotate_threads.py delete mode 100644 hw/syn/firesim/fame5/export_firrtl.py delete mode 100644 hw/syn/firesim/fame5/fix_firrtl.py delete mode 100644 hw/syn/firesim/fame5/gen_rams.py delete mode 100644 hw/syn/firesim/fame5/lift_boundaries.py delete mode 100644 hw/syn/firesim/fame5/name_kept.py delete mode 100644 hw/syn/firesim/fame5/selfref_wires.py delete mode 100644 hw/syn/firesim/fame5/shorten_fir.py delete mode 100644 hw/syn/firesim/fame5/splice.py delete mode 100755 hw/syn/firesim/fame5/to_firrtl.sh diff --git a/docs/assets/img/firesim_build_flow.svg b/docs/assets/img/firesim_build_flow.svg index 2ba1638343..983187e59d 100644 --- a/docs/assets/img/firesim_build_flow.svg +++ b/docs/assets/img/firesim_build_flow.svg @@ -1,4 +1,4 @@ - + @@ -21,23 +21,23 @@ .w { stroke:#44525f; stroke-width:1.4; fill:none; marker-end:url(#arr); } - + Build flow — how Vortex RTL becomes a FireSim image - Left: the default FAME-1 build, where Vortex stays Verilog and Golden Gate never sees inside it. Right: FAME5=1, where Vortex is converted to FIRRTL so Golden Gate can thread the sockets. Both run from hw/syn/firesim/Makefile with the native flows' tokens. + Vortex stays Verilog: Golden Gate never sees inside it. Runs from hw/syn/firesim/Makefile with the native flows' tokens. + Threaded images (FAME-5) are the maxsim flow, compiled by CIRCT: see maxsim.md. + + + + Front end — the configured tree, flattened + make -C hw/syn/firesim [PLATFORM=<xsa> | DEVICE=<part>] TARGET=hw|sim|hw_emu + gen_sources.sh -P preprocesses hw/rtl into one self-contained directory, top VX_firesim_wrap: + vivado_srcs/ + .f ← -DSYNTHESIS -DVIVADO -DNDEBUG (+ -DSIMULATION for TARGET=sim/hw_emu) + vortex_srcs/ (metasim) ← -DNDEBUG only: behavioural RAMs, soft FPU + Platform defines: VX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 · PLATFORM_MEMORY_ID_WIDTH=4 + (paired with idBits) · [PERF_ENABLE] - - - Common front end — the configured tree, flattened - make -C hw/syn/firesim [PLATFORM=<xsa> | DEVICE=<part>] TARGET=hw|sim|hw_emu [FAME5=1] [PERF=1] CONFIGS="-DVX_CFG_…" - gen_sources.sh -P preprocesses hw/rtl into one self-contained directory (macros inlined), top VX_firesim_wrap: - vivado_srcs/ + .f ← -DSYNTHESIS -DVIVADO -DNDEBUG (+ -DSIMULATION for TARGET=sim/hw_emu, never under FAME5) - vortex_srcs/ (metasim) ← -DNDEBUG only: behavioural RAMs, soft FPU - Platform defines both lanes share: VX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 · PLATFORM_MEMORY_ID_WIDTH=4 (paired with idBits) · [PERF_ENABLE] - - - Default: FAME-1, Vortex as a Verilog BlackBox - + FAME-1: Vortex as a Verilog BlackBox stage → replace-rtl — Golden Gate on the harness only VortexTarget.scala elaborates: VortexDUT holds VX_firesim_wrap as an opaque BlackBox; @@ -67,57 +67,16 @@ FireSim-generated.sv + the vortex_srcs tree (library search) into libfiresim.so, with simif_emul as the host interface — the design itself is the simulator. EMUL_DEBUG=1 adds --trace +define+DEBUG → +waveformfile=… writes a VCD - FAME5=1 uses the generated RTL only (Vortex is already inside it) Same target, same tokens as the card: the step before silicon. - - - FAME5=1: Vortex converted to FIRRTL, sockets threaded - - - sv2v — SystemVerilog → Verilog-2005 - fame5/vortex.v (interfaces inlined; VX_socket_top keeps a plain-wire boundary) - - - to_firrtl.sh — Yosys → FIRRTL (fame5/) - Flattens everything except VX_socket_top (the threaded unit, instantiated without - parameters); VX_dp_ram / VX_sp_ram become black boxes that gen_rams.py regenerates - as FIRRTL memories (clocked reads, byte enables); fix_firrtl.py repairs what Yosys - emits that the FIRRTL compiler rejects (signed shifts, deep nesting, cat-LHS, …). - → fame5/vortex.fir (~384 MB at 64 cores NW=NT=16, ~4 min) - - - splice + annotate + Golden Gate - splice.py replaces the harness FIR's VX_firesim_wrap extmodule with the converted Vortex; - annotate_threads.py adds one FirrtlEnableModelMultiThreadingAnnotation per socket - instance (FAME5_THREADS=0 skips: same design, unthreaded, must match cycle-for-cycle). - Golden Gate then applies FAME-1 + FAME-5: one threaded copy, N-deep state. - → FireSim-generated.sv (Vortex inside; 229 MB at 64 cores, ~12 min / 36 GB heap) - - - Platform build — handed nothing but the generated RTL - No Vortex sources, no vendor IP, no pre-opt hook: nothing outside FireSim-generated.sv - is left for them to apply to. Same Vivado flow, FREQUENCY, STRATEGY. - - - Image — build32_aved_hw_fame5/bin/firesim.vbin - _fame5 directories throughout; VortexFame5Config keys the generated RTL apart - - RTL preconditions (all under `ifdef VX_CFG_FAME5): CORE_ID parameter → core_id/socket_id - ports so all sockets elaborate identically; ASYNC_BRAM_PATCH off; no VM, no graphics. - - - + - - - - - Run either lane's image with the same command: FPGA_BIN_DIR=<bin> ./ci/blackbox.sh --driver=firesim --target=hw --app=… (FAME5=1 for a threaded build). - The Makefile guards the classic failure modes: SYNTHESIS missing from the flattened tree (every RAM lands in LUTRAM), VX_MEM_PORTS resolving to 0, a changed CONFIGS reusing a stale flatten. + Run the image: FPGA_BIN_DIR=<bin> ./ci/blackbox.sh --driver=firesim --target=hw --app=… + The Makefile guards the classic failure modes: SYNTHESIS missing from the flattened tree (every RAM + lands in LUTRAM), VX_MEM_PORTS resolving to 0, a changed CONFIGS reusing a stale flatten. diff --git a/docs/assets/img/firesim_fame5_threading.svg b/docs/assets/img/firesim_fame5_threading.svg deleted file mode 100644 index 83836d4af9..0000000000 --- a/docs/assets/img/firesim_fame5_threading.svg +++ /dev/null @@ -1,135 +0,0 @@ - - - - - - - - - - FAME-5 model multithreading — 64 sockets through one physical copy - Golden Gate threads instances of exactly one module; the annotation names it, the pass rewrites it. Nothing about Vortex's architecture changes — the same 64 sockets exist in target time, they just take turns using the silicon. - - - - Target view (unchanged) - 64 identical instances of VX_socket_top, - each with its own token channels: - - - g_sockets[0] : VX_socket_top - socket_id=0 · dcr/mem/kmu/gbar/busy channels - - g_sockets[1] : VX_socket_top - socket_id=1 - - g_sockets[2] : VX_socket_top - socket_id=2 - ⋮ - - g_sockets[63] : VX_socket_top - socket_id=63 - - Identical is a hard requirement: same module, - same parameters, same ports. CORE_ID was a - parameter (64 different modules); under - VX_CFG_FAME5 - it is the socket_id port, so all - 64 elaborate to one module. - - - - Static mux / demux - Inserted by Golden Gate at the - threaded copy's boundary. - A free-running thread counter - selects whose channels connect - this host cycle: - t = 0,1,2,…,63,0,… - one thread per host cycle, - fixed round-robin - - - - Physical view — VX_socket_top_threaded (built once) - - - All combinational logic, shared - Decode, issue, ALU/FPU lanes, LSU, cache control, DSPs — one copy serves - every socket in turn. This is where the 5.5× area win comes from: - 64 × 221K LUTs (unthreaded, would not fit) → 719K LUTs threaded - Logic that would be per-socket state machines is unchanged — it just - computes on whichever socket's state is selected. - - - Every register → 64-deep RAM - Read thread t's state, compute, - write it back — same host cycle, - so it maps to LUTRAM - (distributed, comb-read): - RAM64X1S ≈ 1 LUT / state bit - 311K LUTRAM in the 64-core image - - - Every SRAM → 64× deeper - Thread index becomes the top - address bits: - addr' = {t, addr} - Read latency 0 or 1 only; latency-1 - keeps them BRAM/URAM-mappable. - 1,679 BRAM36 + 97 URAM at 64 cores - - Not threaded (single copy, outside the sockets): the cluster's 64-way memory arbiter, KMU, DCR, global barrier, - VX_mem_to_axi, every FireSim bridge and FASED. A BlackBox inside the threaded module would be silently shared - across threads — wrong results, no diagnostic — which is why the RAM leaves are FIRRTL memories, not black boxes. - - - - - - - - - - - Cost: host cycles per target cycle - - One target cycle needs every thread to take its turn — at least 64 host cycles. A cross-thread combinational dependency (a socket's output feeding - another's input in the same target cycle) forces a second rotation, so the observed FMR lands between N and 2N plus bridge stalls: - - - - - t=0 - t=1 - t=2 - … - t=63 - arbiter/bridges - t=0 - … - - ←——————————— one target cycle ≈ 64–130 host cycles ———————————→ - - Measured, 64 cores NW=NT=16 on the V80 at 60 MHz: FMR 130–190 → 320–460 kHz of target time. - The unthreaded design would not fit at any speed; threading trades 64× the silicon for ~1/130 of the clock. - - Sanity anchor: FAME5_THREADS=0 builds the identical FIRRTL design unthreaded. It must — and does — agree with the threaded build - cycle for cycle (4-core demo: instrs=9776, cycles=15541 on baseline, threaded, and unthreaded-FIRRTL builds alike). - diff --git a/docs/assets/img/firesim_fpga_architecture.svg b/docs/assets/img/firesim_fpga_architecture.svg index 84d3abb5a7..82dce0c875 100644 --- a/docs/assets/img/firesim_fpga_architecture.svg +++ b/docs/assets/img/firesim_fpga_architecture.svg @@ -1,4 +1,4 @@ - + @@ -31,16 +31,16 @@ .wc { stroke:#3a7d44; stroke-width:1.4; fill:none; marker-end:url(#arrg); stroke-dasharray:5 3; } - - A Vortex instance on the FPGA — V80 (AVED shell), FAME-5 build + + A Vortex instance on the FPGA — V80 (AVED shell) Everything inside pblock_slash is FireSim-generated.sv, one reconfigurable module. Solid orange = memory data; solid grey = control (MMIO); dashed green = clocks. The U55C build is the same picture with the Vitis shell around it and HBM[0:7] as the memory binding. - + xcv80-lsva4737-2MHP-e-S - + AVED static region (in flash) @@ -75,7 +75,7 @@ only decides *when* answers return - + Reconfigurable partition pblock_slash — RM “slash_firesim” = FireSim-generated.sv @@ -122,36 +122,30 @@ Every wire between a bridge and the target is a token channel (ready/valid + payload). One target cycle fires exactly when every input channel holds a token and every output channel has space; otherwise the target clock is gated and the design waits, cycle-accurately. This is what host-decoupling means: the target cannot outrun the host, and the host cannot observe a half-advanced cycle. - Under FAME-5 the hub also carries one channel set per socket, multiplexed round-robin into the single threaded copy (next figure). Clock gating on the V80 is a clock-enable conversion (FireSimFabricClockGate): the partition has no BUFGCE site. - + Target — VortexDUT ▸ VX_firesim_wrap ▸ Vortex Control surface: DCR request/response + start/busy (poked by PeekPoke). Memory: one AXI4 port, 512-bit data, 32-bit address, 4-bit id. Single-copy logic (built once) KMU (CTA dispatch) · DCR bus · global barrier unit · - cluster: 64-way socket↔memory arbiter · VX_mem_to_axi + cluster: socket↔memory arbiter · VX_mem_to_axi (tag buffer: 16 outstanding reads, 4-bit AXI id) - Everything here sees 64 sockets and does not know - they share one physical copy. + Built once for the whole GPU. - g_sockets — 64 × VX_socket_top (NW=NT=16 shown) + g_sockets — N × VX_socket_top Each socket: 1 core + I$/D$/LMEM, socket_id as a port. - FAME5=1 → one physical copy, VX_socket_top_threaded: - registers become 64-deep LUTRAM, SRAMs 64× deeper, - one socket advances per host cycle, round-robin. - unthreaded: ~14.1M LUTs (5.5× the V80) — threaded: 719K LUTs, routed at 60 MHz + One physical copy per socket. Threading the sockets + through one copy (FAME-5) is the maxsim flow: maxsim.md. - Routed 64-core NW=NT=16 image, whole device: 974K LUTs (37.9%) · 565K FFs (11%) · 311K LUTRAM (24%) · 1,683 BRAM36 (45%) · - 105 URAM (5.5%) · 64 DSP · WNS +0.33 ns at 60 MHz - Control path: PCIe BAR0 → AXI-Lite → shim register blocks → widget MMIO. Memory data path: FASED / LoadMem host-memory master → HBM VNOC → HBM. - Host clock: free-running 60 MHz from the shell. Target clock: the host clock, enabled only on token-complete cycles — the target has no clock of its own. + Control path: PCIe BAR0 → AXI-Lite → shim register blocks → widget MMIO. Memory data path: FASED / LoadMem host-memory master → HBM VNOC → HBM. + Host clock: free-running 60 MHz from the shell. Target clock: the host clock, enabled only on token-complete cycles — the target has no clock of its own. diff --git a/docs/assets/img/firesim_memory_system.svg b/docs/assets/img/firesim_memory_system.svg index 7e2d6ee5b0..ea3e498c17 100644 --- a/docs/assets/img/firesim_memory_system.svg +++ b/docs/assets/img/firesim_memory_system.svg @@ -36,8 +36,8 @@ Cores → L1 → (L2/L3 if enabled) - 64-core FAME-5 build: 64 sockets' misses - meet at the cluster arbiter + the sockets' misses meet at the + cluster arbiter VX_mem_to_axi diff --git a/docs/assets/img/firesim_system_stack.svg b/docs/assets/img/firesim_system_stack.svg index 9649df93e2..1710193a79 100644 --- a/docs/assets/img/firesim_system_stack.svg +++ b/docs/assets/img/firesim_system_stack.svg @@ -63,7 +63,7 @@ MMIO + DMA through XRT - + Metasimulator (TARGET=xrtsim / avedsim) — no card, no image @@ -76,16 +76,14 @@ firesim_emul_main.cpp runs Verilator's event loop on the simulator thread. VORTEX_FIRESIM_EMUL_ARGS → +waveformfile=… +max-cycles=… (EMUL_DEBUG=1 build) - + Vemul — the whole FPGA payload, Verilated - FireSim-generated.sv + (unthreaded builds) the flattened Vortex tree, + FireSim-generated.sv + the flattened Vortex tree, bit-identical to what the card synthesizes. Same tokens, same schedule: what passes here passes on silicon, cycle for cycle. - FAME5=1: Vortex is already inside FireSim-generated.sv (converted to - FIRRTL and threaded), so Verilator is handed no Vortex sources at all. - Build: sw/runtime/firesim → sim/firesim/Makefile (EMUL_DIR per card, - _fame5 and _dbg variants); run: blackbox.sh --driver=firesim - --target=avedsim|xrtsim [FAME5=1] [EMUL_DEBUG=1] + Build: sw/runtime/firesim → sim/firesim/Makefile (EMUL_DIR per card, + and a _dbg variant); run: blackbox.sh --driver=firesim + --target=avedsim|xrtsim [EMUL_DEBUG=1] @@ -107,7 +105,6 @@ Target — VortexDUT ▸ VX_firesim_wrap ▸ Vortex The design under test: DCR + start/busy control surface, one 512-bit AXI4 memory port. - FAME5=1: the sockets inside are time-multiplexed through one physical copy. diff --git a/docs/assets/img/firesim_timing_model.svg b/docs/assets/img/firesim_timing_model.svg index 39da862a54..fc74bd7b39 100644 --- a/docs/assets/img/firesim_timing_model.svg +++ b/docs/assets/img/firesim_timing_model.svg @@ -73,7 +73,6 @@ tcycle — target cycles advanced (clockmodule; Vortex PERF: cycles) hcycle — host cycles spent simulating them FMR = hcycle/tcycle; target freq = host_freq / FMR - 64-core FAME-5 on the V80: 60 MHz / FMR 130–190 ≈ 320–460 kHz of target time diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index 2b39e43c7e..703b3e7b74 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -7,8 +7,8 @@ Alveo U55C (Vitis/XRT) and the Alveo V80 (AVED/VRT). | § | contents | |---|---| | [1](#1-system-architecture) | System architecture: host stack, FPGA payload, execution model | -| [2](#2-synthesis-of-the-vortex-rtl) | Synthesis of the Vortex RTL, with and without FAME-5 | -| [3](#3-fame-5-many-sockets-one-physical-copy) | FAME-5 socket time-multiplexing | +| [2](#2-synthesis-of-the-vortex-rtl) | Synthesis of the Vortex RTL | +| [3](#3-threaded-simulation) | Threaded simulation: maxsim | | [4](#4-the-simulation-timing-model) | The simulation timing model | | [5](#5-the-memory-system) | The memory system and its configuration; shell configuration | | [6](#6-observability) | Observability: wave traces, RTL messages and assertions, counters | @@ -127,14 +127,10 @@ what every measurement in this repository uses. ![Build flow](../assets/img/firesim_build_flow.svg) -There are two build lanes. In the **default (FAME-1) lane** Vortex stays -Verilog: Golden Gate sees it as an opaque BlackBox, transforms only its -boundary, and the card's platform flow synthesizes the Verilog directly. In -the **FAME-5 lane** (`FAME5=1`) Vortex is converted to FIRRTL so Golden Gate -can transform *inside* it — that is what enables the socket -time-multiplexing of [§3](#3-fame-5-many-sockets-one-physical-copy). Both -are driven from `hw/syn/firesim/Makefile` with the same tokens as the native -flows: +Vortex stays Verilog: Golden Gate sees it as an opaque BlackBox, transforms +only its boundary (FAME-1), and the card's platform flow synthesizes the +Verilog directly. The build is driven from `hw/syn/firesim/Makefile` with the +same tokens as the native flows: ```bash source ci/firesim_env.sh # after the card's Xilinx environment @@ -142,11 +138,9 @@ source ci/firesim_env.sh # after the card's Xilinx environment make -C hw/syn/firesim PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 TARGET=hw # V80 (AVED): TARGET=hw | sim make -C hw/syn/firesim TARGET=hw PERF=1 CONFIGS="-DVX_CFG_NUM_WARPS=16" -# V80, 64 cores through one physical socket: -make -C hw/syn/firesim FAME5=1 TARGET=hw CONFIGS="-DVX_CFG_NUM_CORES=64 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_THREADS=16" ``` -The image lands in `__[_fame5]/bin` as +The image lands in `__/bin` as `firesim.vbin` or `firesim.xclbin`, beside `firesim.card`, which names the card for `run-firesim`. `PREFIX`, `XLEN`, `CONFIGS`, `PERF=1` work as in the other synthesis flows; extensions pull their sources in through @@ -164,7 +158,7 @@ exist because simulation and synthesis need opposite defines: | flattening | defines | consumed by | |---|---|---| -| `vivado_srcs/` | `-DSYNTHESIS -DVIVADO -DNDEBUG` (+ `-DSIMULATION` for `TARGET=sim`/`hw_emu`; never under FAME5) | platform synthesis; sv2v under FAME5 | +| `vivado_srcs/` | `-DSYNTHESIS -DVIVADO -DNDEBUG` (+ `-DSIMULATION` for `TARGET=sim`/`hw_emu`) | platform synthesis | | `vortex_srcs/` | `-DNDEBUG` only | the Verilator metasimulator | The invariants here have each cost hours when broken: @@ -186,7 +180,7 @@ The invariants here have each cost hours when broken: computations with report values, which sets the memory port count to zero; both flatten rules grep the result for `VX_MEM_PORTS = 0` and fail fast. -### 2.2 The default lane: Vortex as a BlackBox +### 2.2 Vortex as a BlackBox `hw/syn/firesim/src/main/scala/VortexTarget.scala` instantiates Vortex as a Chisel BlackBox. Golden Gate cannot see inside it, which is the point: the @@ -210,105 +204,14 @@ slashkit on aved), handed: LUTRAM, so small builds pass and large ones do not; - `FREQUENCY` (default 60 MHz) and `STRATEGY` (default `TIMING`). -### 2.3 The FAME-5 lane: Vortex as FIRRTL - -Golden Gate's threading transform works on FIRRTL, so `FAME5=1` converts the -whole flattened tree (`hw/syn/firesim/fame5/`): - -1. **sv2v** lowers the SystemVerilog to Verilog-2005. sv2v inlines every - module with interface ports into its parent — which would erase the - boundary of the unit to be threaded. `VX_socket_top` - (`hw/rtl/VX_socket_top.sv`) exists for exactly this reason: a - parameterless wrapper around `VX_socket` whose ports are plain wires, so - it survives conversion as one module. -2. **`to_firrtl.sh` (Yosys)** flattens everything except `VX_socket_top` and - writes FIRRTL. `VX_dp_ram`/`VX_sp_ram` are kept as black boxes that - `gen_rams.py` regenerates as native FIRRTL memories — Yosys cannot express - clocked read ports or byte enables, and a real BlackBox inside the - threaded module would be silently shared across threads - ([§3](#3-fame-5-many-sockets-one-physical-copy)). The script also carries - the Yosys workarounds this conversion needed: `proc -norom`, - `memory_nordff` for small clocked-read arrays, `opt -nosdff -nodffe` plus - `dffunmap`, and `splitnets` against word-level false loops. -3. **`fix_firrtl.py`** repairs what Yosys emits that the FIRRTL compiler or - Verilator reject: signed-shift lowering, 2¹⁹-bit `dshl` intermediates, - `cat()` on the left-hand side, signed zero tests, and expression nesting - thousands of levels deep (hoisted into nodes — the FIRRTL parser recurses - per level). It runs in one linear pass; a 64-core conversion has lines - over a million characters long. -4. **`splice.py`** replaces the `VX_firesim_wrap` extmodule in Golden Gate's - own elaborated circuit with the converted Vortex, and - **`annotate_threads.py`** appends one - `FirrtlEnableModelMultiThreadingAnnotation` per socket instance - (`FAME5_THREADS=0` skips this: the identical design, unthreaded — the - cycle-exactness anchor). -5. **Golden Gate** then applies FAME-1 and FAME-5 together and emits a - `FireSim-generated.sv` that *contains* Vortex. The platform build is - handed no Vortex sources, no vendor IP and no pre-opt hook — there is - nothing left outside the generated file for them to apply to. - -The RTL preconditions, all under `` `ifdef VX_CFG_FAME5 ``: per-core -`CORE_ID` parameters become `core_id`/`socket_id` *ports* (a parameter would -make 64 different modules; threading needs one), `ASYNC_BRAM_PATCH` is -disabled (its placeholders only a Vivado hook resolves), and `VX_socket_top` -statically rejects VM and graphics configurations it does not carry ports -for. - -At 64 cores NW=NT=16: conversion ≈ 4 min (384 MB of FIRRTL), Golden Gate -≈ 12 min at ≈ 36 GB heap (`JAVA_TOOL_OPTIONS` raises `-Xmx` and, for the -parser's recursion, `-Xss`), generated Verilog 229 MB. - --- -## 3. FAME-5: many sockets, one physical copy - -![FAME-5 threading](../assets/img/firesim_fame5_threading.svg) - -The obstacle to a 64-core build is area. One NW=NT=16 core synthesizes to -~221K LUTs; 64 of them need ~14.1M against the V80's 2.57M. FAME-5 ("model -multithreading") removes the duplication instead of shrinking the core: all -64 sockets **take turns using one physical copy of the socket logic**, one -socket per host cycle, round-robin. - -**What is threaded.** Exactly one module type per design (a Golden Gate -restriction), chosen by annotation — there is no automatic detection, with -or without Chisel. Vortex threads `VX_socket_top`: one core plus its I$, D$ -and local memory. Everything outside it — the cluster's socket↔memory -arbiter, KMU, DCR, global barrier, `VX_mem_to_axi`, all bridges and FASED — -is built once and does not know the sockets share silicon. - -**What the transform does.** The target-visible design is unchanged: 64 -instances, each with its own token channels. Golden Gate replaces them with -one `VX_socket_top_threaded` plus a static mux/demux driven by a free-running -thread counter, and transforms the state: - -- **every register becomes an N-deep, combinationally-read memory** (read - thread *t*'s state, compute, write back — same host cycle), which Vivado - maps to LUTRAM at ~1 LUT per state bit; -- **every SRAM becomes N× deeper**, the thread index forming the top address - bits; read latency 0 or 1 only, and latency-1 keeps BRAM/URAM mapping. - -**The requirements that shaped the RTL work:** - -- instances must be *identical* — same module, same parameters. Hence - `core_id`/`socket_id` as ports and the parameterless wrapper; -- **no BlackBox inside the threaded module.** It would be instantiated once - and silently shared by all threads: wrong results, no diagnostic (proven - on a prototype). Hence the FIRRTL regeneration of the RAM leaves; -- a module with a single instance is skipped, and only one module type can - be threaded. - -**The cost.** One target cycle takes at least N host cycles (the rotation), -up to ~2N when a cross-thread combinational dependency forces a second pass, -plus bridge stalls: the 64-core V80 build measures FMR 130–190, i.e. -320–460 kHz of target time at the 60 MHz host clock. Per-thread state is -initialized during host reset, so target reset behaves identically in every -thread. - -**The correctness anchor.** `FAME5_THREADS=0` builds the same -FIRRTL-converted design with no threading. The three builds — native -Verilog baseline, unthreaded FIRRTL, threaded — agree exactly (4-core demo: -`instrs=9776, cycles=15541` in all three). +## 3. Threaded simulation + +This backend simulates Vortex unthreaded: one physical copy of every module. +Threading a GPU's identical units through one physical copy each (FAME-5) is +the maxsim flow, which compiles Vortex with CIRCT rather than Golden Gate and +has its own kernel, driver and image: see [maxsim.md](maxsim.md). --- @@ -390,13 +293,13 @@ VORTEX_FIRESIM_ARGS="+mm_readLatency_0=100 +mm_writeLatency_0=100 +mm_useHardwar ./ci/blackbox.sh --driver=firesim --target=avedsim --app=demo ... ``` -Validated: the 4-core FAME-5 demo runs 15,541 cycles at the default 30-cycle -latency and 47,371 at 100/100 — same 9,776 instructions, `PASSED` both ways. +Validated: a 4-core demo runs 15,541 cycles at the default 30-cycle latency +and 47,371 at 100/100 — same 9,776 instructions, `PASSED` both ways. This is the only memory knob that needs no rebuild. The model's ceiling is worth computing before trusting a bandwidth-bound number: `maxReads × 64 B / readLatency` ≈ 34 B per target cycle for the -whole GPU at the defaults. The 64-core `sgemm` sits exactly on it. Raising +whole GPU at the defaults. Raising the ceiling means `maxReads` (compile-time, and the egress area scales with it) or lower modelled latency (runtime). There is no DRAM structure — banks, rows, refresh — in the default pipe; FASED has DDR3 FIFO/FR-FCFS models and a @@ -452,7 +355,7 @@ Two width couplings must move together or nothing warns: and pass the dump plusargs at run time: ```bash -EMUL_DEBUG=1 FAME5=1 \ +EMUL_DEBUG=1 \ VORTEX_FIRESIM_EMUL_ARGS="+waveformfile=$PWD/demo.vcd +max-cycles=60000" \ ./ci/blackbox.sh --driver=firesim --target=avedsim --cores=4 --warps=4 --threads=4 --app=demo ``` @@ -469,7 +372,7 @@ also takes ordinary Verilator runtime args (e.g. `+verilator+rand+reset+2` to randomize uninitialized state). **Size the window before running.** The dump covers every signal of the -FPGA payload: the 4-core FAME-5 demo above writes ~23 GB for a 60,000 +FPGA payload: a 4-core demo writes ~23 GB for a 60,000 host-cycle window (~380 KB per host cycle). Place `+dump-start` just before the region of interest and keep `+max-cycles` in the low tens of thousands; the simulation itself runs to completion after the window closes, unperturbed @@ -482,10 +385,7 @@ the FPGA payload, `sim.target` the FAME hub's interior. Orientation: - everything toggles on the **host clock**; one target cycle is one firing of the hub, so anchor on the clock channel or `ctrl_busy` rather than counting clock edges; -- the target lives under `FPGATop.sim.target…`; under FAME-5 the interesting - state is in `VX_socket_top_threaded`, where a former register is an - N-entry array indexed by the thread counter, and `_fx_*` signals are - expression nodes hoisted by the FIRRTL conversion; +- the target lives under `FPGATop.sim.target…`; - the bridges' AXI (FASED) and MMIO surfaces are at the top level — the fastest way to see whether a hang is the target or the transport. @@ -508,17 +408,15 @@ flow: | vehicle | `SIMULATION`? | consequence | |---|---|---| | `TARGET=sim` (V80 XSIM image), `TARGET=hw_emu` (U55C) | **yes** | `$display`/`$error`/asserts fire in the platform simulator's log | -| metasim `xrtsim`/`avedsim`, unthreaded | no (`-DNDEBUG` only) | compiled out — the metasim flattening carries neither synthesis nor simulation defines, and adding `-DSIMULATION` there is untested (a few library modules would also want the `dpi_trace` objects the metasim does not link) | -| any `FAME5=1` build | **structurally impossible** | sv2v/Yosys have no FIRRTL form for `$fflush`/`$write`; the conversion is always made from the synthesis flattening | +| metasim `xrtsim`/`avedsim` | no (`-DNDEBUG` only) | compiled out — the metasim flattening carries neither synthesis nor simulation defines, and adding `-DSIMULATION` there is untested (a few library modules would also want the `dpi_trace` objects the metasim does not link) | | `TARGET=hw` | no (`SYNTHESIS`) | compiled out, as in every synthesis flow | Golden Gate's own synthesized printf/assert machinery (`SynthPrints`, `SynthAsserts` — both enabled in the platform config) applies only to printf/stop nodes authored in Chisel or present in the FIRRTL; Vortex arrives -either as a BlackBox or as Yosys-converted FIRRTL containing neither, so it -contributes nothing here. In short: **for RTL-level messages and asserts, -use `TARGET=sim`/`hw_emu`, or rtlsim outside FireSim entirely; on `hw` and -under FAME-5, instrument via counters and the driver.** +as a BlackBox containing neither, so it contributes nothing here. In short: +**for RTL-level messages and asserts, use `TARGET=sim`/`hw_emu`, or rtlsim +outside FireSim entirely; on `hw`, instrument via counters and the driver.** ### 6.3 Performance counters and driver diagnostics @@ -576,11 +474,8 @@ integration, and where host-decoupling shows up as real complexity. need the response latched in the wrapper — a target-side change and a new bitstream, noted as a follow-up. - **Peeks after a blocking step are non-blocking.** A blocking peek waits up - to 0.1 s for the `PRECISE_PEEKABLE` flag, which a FAME-5 design never - raises, then returns the same value; after a blocking step the wait adds - nothing but time. Left blocking, every DCR poll cost 0.1 s and a 64-core - readback took 20 s per register. (Why FAME-5 outputs never assert - precise-peekable is an open follow-up.) + to 0.1 s for the `PRECISE_PEEKABLE` flag before returning; after a blocking + step the value is already final and the wait adds nothing but time. - **Watchdogs.** A launch that exceeds `VORTEX_FIRESIM_TIMEOUT_S` or a busy-level that exceeds `VORTEX_FIRESIM_STALL_CYCLES` produces the full counter report (§6.3) and a clean abort — on this vehicle, "hung" must be @@ -592,10 +487,10 @@ integration, and where host-decoupling shows up as real complexity. ```bash # metasimulator (no image, no card): -FAME5=1 ./ci/blackbox.sh --driver=firesim --target=avedsim --cores=64 --warps=16 --threads=16 --app=demo +./ci/blackbox.sh --driver=firesim --target=avedsim --warps=4 --threads=4 --app=demo # the card: -FAME5=1 FPGA_BIN_DIR=$PWD/build32_aved_hw_fame5/bin \ - ./ci/blackbox.sh --driver=firesim --target=hw --cores=64 --warps=16 --threads=16 --app=demo +FPGA_BIN_DIR=$PWD/hw/syn/firesim/build32_aved_hw/bin \ + ./ci/blackbox.sh --driver=firesim --target=hw --warps=4 --threads=4 --app=demo ``` | card | FireSim platform | run targets | image / card named by | @@ -618,7 +513,7 @@ static shell in flash, as for the native `aved` backend. ## 9. Measured results -### 9.1 U55C, default lane (FAME-1) +### 9.1 U55C 1 cluster / 1 core / 4 warps / 4 threads, L1 only, `FPU_USE_DSP=0`, kernel clock 60 MHz. Kernel-only utilization: 131,578 LUTs (10.1%), 62,004 FFs, @@ -630,41 +525,8 @@ faster than rtlsim on `sgemm -n32`). Instruction counts match rtlsim exactly; cycle counts differ −1.3%…+7.1% — the FASED timing model, not the core: FPGA and hw_emu agree exactly. -### 9.2 V80, FAME-5, 64 cores NW=NT=16 - -Image: 6 h 33 m end to end (synthesis ≈ 87 min, routing ≈ 2.5 h — ~300K -routing overlaps at the first iteration, from the SLR2 concentration below), -**timing met at 60 MHz**: WNS +0.329 ns, WHS +0.010 ns, zero failing -endpoints. - -| resource | used | available | % | -|---|---|---|---| -| CLB LUTs | 974,424 | 2,574,208 | 37.9 | -| — as logic | 662,530 | 2,574,208 | 25.7 | -| — as memory (LUTRAM) | 311,894 | 1,287,104 | 24.2 | -| Registers | 564,836 | 5,148,416 | 11.0 | -| Block RAM tiles | 1,682.5 | 3,741 | 45.0 | -| URAM | 105 | 1,925 | 5.5 | -| DSP | 64 | 10,848 | 0.6 | - -The threaded socket is 719K LUTs / 258K LUTRAM / 1,679 BRAM — one copy in -place of an unthreadable ~14.1M LUTs. FireSim's own token infrastructure -carries 224K FFs (more than the socket's 183K), and placement concentrates: -SLR2 holds 91% of the FFs at 79% slice occupancy, SLR1 71% of its BRAM — the -route-time driver, and the first place to look before raising `FREQUENCY`. - -On the card: - -| app | instrs | cycles | IPC | target kHz | FMR | wall | -|---|---|---|---|---|---|---| -| demo `-n1` (16K pts) | 45,012 | 7,884 | 5.71 | 316 | 190 | 7 s | -| demo (1M pts) | 625,664 | 3,460,483 | 0.18 | 378 | 159 | 16 s | -| sgemm `-n1024` (1024³) | 296,816,640 | 145,714,145 | 2.04 | 462 | 130 | 323 s | - -Both memory-bound results sit on the timing model's ~34 B/cycle ceiling -(§5.2). Cycle-exactness across builds: 4-core demo identical -(`instrs=9776, cycles=15541`) on the native baseline, the unthreaded-FIRRTL -build (`FAME5_THREADS=0`) and the threaded build. +Threaded images and their results are the maxsim flow's: +[maxsim.md](maxsim.md). --- @@ -679,7 +541,7 @@ rather than divergence — the AVED platform (shim, register blocks, fabric clock gates, host-memory base as a config field), driver lifetime and toolchain fixes, cycle-accounting fixes, and kernel packaging — all candidates for upstreaming. Everything Vortex-specific (targets, wrapper, -transport, FAME-5 conversion) lives in this repository and is staged in at +transport) lives in this repository and is staged in at build time. --- @@ -695,32 +557,21 @@ build time. hours later in place-and-route. 4. **Vortex's `PERF:` counters are the cycle authority**, never `Target Cycles Emulated`. -5. **FAME-5 threads one module type, and its instances must be identical** — - IDs travel as ports, never parameters. -6. **No BlackBox inside the threaded module** — it would be shared across - threads silently. -7. **All-zero performance counters mean `PERF_ENABLE` is off**, not that the +5. **All-zero performance counters mean `PERF_ENABLE` is off**, not that the core is idle. -8. **LoadMem's chunk unit is 32-bit words**, not bytes. -9. **`FAME5_THREADS=0` must stay cycle-identical to the threaded build** — - it is the anchor that separates conversion bugs from threading bugs. +6. **LoadMem's chunk unit is 32-bit words**, not bytes. --- ## 12. Known limitations -- **No RTL `$display`/asserts on `hw` or under FAME-5** (§6.2); the metasim +- **No RTL `$display`/asserts on `hw`** (§6.2); the metasim compiles them out too (no DPI linked). RTL-message debugging belongs to `TARGET=sim`/`hw_emu` or rtlsim. - **DCR reads single-step** — one MMIO round trip per target cycle; batched polling needs a latched response in the wrapper (target-side follow-up). -- **FAME-5 excludes VM and graphics**: `VX_socket_top` does not carry their - ports and statically rejects those configurations. - **XLEN=64 is not wired through FireSim**: `addrBits` is pinned at 32 (§5.3). -- **Metasim speed falls with thread count**: the 64-thread FAME-5 model runs - at a few tens of target cycles per second under Verilator — use it for - correctness and waves, the card for anything long. - **One host memory channel** bounds simulation speed under heavy memory traffic (never target timing). - **The default memory model is a flat latency-bandwidth pipe** — no banks, diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 21656c50a0..fd051944d0 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -88,18 +88,11 @@ $(error TARGET=$(TARGET) is not an image target for card $(FIRESIM_CARD): use on endif endif -# FAME5=1: FireSim model multithreading. Every socket is time-multiplexed -# through one physical copy (Golden Gate's FAME-5), which needs Vortex as FIRRTL -# rather than as a Verilog BlackBox; see fame5/ and -# docs/designs/firesim_integration.md. Its images never share a directory with -# the unthreaded ones. -FAME5_SUFFIX := $(if $(FAME5),_fame5) - # Named as the native flows name theirs, so FPGA_BIN_DIR means the same thing. ifeq ($(FIRESIM_PLATFORM), aved) -BUILD_DIR := $(abspath $(PREFIX)_aved_$(TARGET)$(FAME5_SUFFIX)) +BUILD_DIR := $(abspath $(PREFIX)_aved_$(TARGET)) else -BUILD_DIR := $(abspath $(PREFIX)_$(if $(XSA),$(XSA),$(FIRESIM_CARD))_$(TARGET)$(FAME5_SUFFIX)) +BUILD_DIR := $(abspath $(PREFIX)_$(if $(XSA),$(XSA),$(FIRESIM_CARD))_$(TARGET)) endif # The runtime's metasimulator reuses the simulation flattening from here, in a # directory of its own. @@ -117,10 +110,7 @@ ifneq ($(XSA),) DEVICE := $(XSA) endif -# FireSim keys its generated RTL by design and configs only, so a FAME-5 build -# names its own target config (identical to VortexConfig) to keep its output -# apart from the unthreaded one's. -VORTEX_TARGET_CONFIG := $(if $(FAME5),VortexFame5Config,VortexConfig) +VORTEX_TARGET_CONFIG := VortexConfig # The FireSim flow's make variables for this card's build. FIRESIM_TARGET_ARGS := PLATFORM=$(FIRESIM_PLATFORM) TARGET_PROJECT=midasexamples \ @@ -142,15 +132,6 @@ ifneq ($(filter $(TARGET),sim hw_emu),) IMAGE_KIND := sim IMAGE_SUFFIX := _sim endif -# FAME-5 converts one flattening to FIRRTL for every target, image or not. It is -# the synthesis one: SIMULATION would bring in simulation-only tasks ($fflush, -# $write) that have no FIRRTL form. VX_CFG_FAME5 keeps the async-BRAM patch out -# of it (VX_platform.vh), whose placeholders only a Vivado hook resolves; the RAM -# leaves are regenerated as FIRRTL memories instead. -ifdef FAME5 -IMAGE_KIND := -IMAGE_SUFFIX := -endif VIVADO_SRCS_NAME := vivado_srcs$(IMAGE_SUFFIX) VIVADO_SRCS_DIR := $(BUILD_DIR)/$(VIVADO_SRCS_NAME) VIVADO_SRCS_LIST := $(BUILD_DIR)/$(VIVADO_SRCS_NAME).f @@ -284,9 +265,6 @@ endif ifdef PERF PLATFORM_DEFINES += -DPERF_ENABLE endif -ifdef FAME5 -PLATFORM_DEFINES += -DVX_CFG_FAME5 -endif GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) $(SYNTH_DEFINES) $(RTL_INCLUDE) @@ -457,8 +435,7 @@ kernel-env: @echo 'export TOOL_DIR=$(SCRIPT_DIR)' # Simulator clock (MHz): the image is timed at it and the shell programs it. -# V80: the Fmax a 250 MHz-targeted build reached (4 cores, NW=NT=4, IW=4, seven -# threaded classes: routed WNS -2.974 ns at 4 ns, i.e. 143.4 MHz). +# V80: the Fmax measured on a FireSim image of Vortex (143.4 MHz). ifeq ($(FIRESIM_PLATFORM), aved) FREQUENCY ?= 143.0 else @@ -505,7 +482,6 @@ endif # prerequisite: the recipe hands synthesis the Vivado-flattened list, and pulling # the simulation one in would flatten the tree a second time under the opposite # defines for nothing. -ifndef FAME5 image: vivado-sources xilinx-ips stage $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ @@ -518,90 +494,6 @@ image: vivado-sources xilinx-ips stage # no image. An image build elaborates as part of replace-rtl. elaborate: stage $(MAKE) -C $(FIRESIM_PATH)/sim verilog $(FIRESIM_TARGET_ARGS) -else -# Under FAME-5 Vortex is part of the generated RTL, so the image is handed no -# Vortex sources, no vendor cores and no pre-opt hook: there is nothing left -# outside FireSim-generated.sv for them to apply to. -image: elaborate - $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) - TOOL_DIR=$(SCRIPT_DIR) $(MAKE) -C $(PLATFORM_BUILD_DIR) $(PLATFORM_IMAGE_GOAL) \ - DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) - -# Vortex as FIRRTL: sv2v, then Yosys flattening everything but VX_socket_top -# (the unit threaded), with the RAM leaves regenerated as FIRRTL memories -# (fame5/to_firrtl.sh). -FAME5_DIR := $(BUILD_DIR)/fame5 -FAME5_V := $(FAME5_DIR)/vortex.v -FAME5_FIR := $(FAME5_DIR)/vortex.fir -FAME5_SCRIPTS := $(wildcard $(SRC_DIR)/fame5/*) -SV2V ?= sv2v - -# FAME5_LIFT: units given plain-port boundaries before sv2v (fame5/lift_boundaries.py, -# which needs pyslang: LIFT_PYTHON), so that interface-port units such as -# VX_issue_slice survive conversion and can be FAME5_CLASSES (as __lift). -# The RTL is not changed: the pass rewrites a copy of the staged sources. -# Each entry is [:drop=,...] (parameters that only label instances). -FAME5_LIFT ?= -LIFT_PYTHON ?= python3 -FAME5_LIFT_STAMP := $(FAME5_DIR)/lift.txt -FAME5_SRCS_LIST := $(if $(strip $(FAME5_LIFT)),$(FAME5_DIR)/lift/srcs.f,$(VIVADO_SRCS_LIST)) - -$(FAME5_LIFT_STAMP): FORCE - @mkdir -p $(FAME5_DIR) - @echo "$(FAME5_LIFT)" | cmp -s - $@ || echo "$(FAME5_LIFT)" > $@ - -$(FAME5_DIR)/lift/srcs.f: $(VIVADO_SRCS_LIST) $(SRC_DIR)/fame5/lift_boundaries.py $(FAME5_LIFT_STAMP) - rm -rf $(FAME5_DIR)/lift - $(LIFT_PYTHON) $(SRC_DIR)/fame5/lift_boundaries.py --srcs $(VIVADO_SRCS_LIST) --top VX_firesim_wrap \ - --out $(FAME5_DIR)/lift $(foreach u,$(FAME5_LIFT),--unit '$(u)') - -$(FAME5_V): $(FAME5_SRCS_LIST) - @mkdir -p $(FAME5_DIR) - $(SV2V) $$(grep '^+define+' $(FAME5_SRCS_LIST) | sed 's/^+define+/-D/') --top=VX_firesim_wrap \ - $$(grep -v '^+' $(FAME5_SRCS_LIST)) -w $@ - -# FAME5_CLASSES: the modules threaded as classes, each kept as hierarchy by the -# conversion together with every parameterization of it. Units inside a socket -# (e.g. VX_fma_unit, the FPU lanes) become nested classes, extracted from every -# socket and threaded on their own; FAME5_FOLD (Golden Gate) sets how many -# physical copies each class gets. -FAME5_CLASSES ?= VX_socket_top -FAME5_KEEP := $(foreach c,$(FAME5_CLASSES),$(c) $$paramod*\$(c) $$paramod*\$(c)\*) -FAME5_CLASSES_STAMP := $(FAME5_DIR)/classes.txt - -# Rewritten only when the class list changes, so the conversion reruns then. -.PHONY: FORCE -FORCE: -$(FAME5_CLASSES_STAMP): FORCE - @mkdir -p $(FAME5_DIR) - @echo "$(FAME5_CLASSES)" | cmp -s - $@ || echo "$(FAME5_CLASSES)" > $@ - -$(FAME5_FIR): $(FAME5_V) $(FAME5_SCRIPTS) $(FAME5_CLASSES_STAMP) - bash $(SRC_DIR)/fame5/to_firrtl.sh $(FAME5_V) VX_firesim_wrap $@ '$(FAME5_KEEP)' VX_firesim_wrap \ - "blackbox VX_dp_ram VX_sp_ram" - -# The generator's circuit names VX_firesim_wrap as an extmodule; it is replaced -# by the converted Vortex, and every socket is annotated for threading, before -# Golden Gate runs. The circuit is regenerated each time, since the splice edits -# it in place. FAME5_THREADS=0 skips the annotations: the same FIRRTL design, -# unthreaded, which must agree with the threaded one cycle for cycle. -# Yosys writes long nested cat() chains, and the FIRRTL parser recurses once per -# level: the 8 MB thread stack FireSim sets overflows on a converted Vortex. -# FireSim only sets its default when the variable is unset. -export JAVA_TOOL_OPTIONS ?= -Xmx48G -Xss1G -Djava.io.tmpdir=$(FIRESIM_PATH)/sim/.java_tmp - -GEN_FIR := $(FIRESIM_PATH)/sim/generated-src/$(FIRESIM_PLATFORM)/$(QUINTUPLET)/firesim.midasexamples.VortexTarget.$(VORTEX_TARGET_CONFIG).fir -GEN_ANNO := $(GEN_FIR:.fir=.anno.json) -elaborate: stage $(FAME5_FIR) - rm -f $(GEN_FIR) - $(MAKE) -C $(FIRESIM_PATH)/sim $(GEN_FIR) $(FIRESIM_TARGET_ARGS) - python3 $(SRC_DIR)/fame5/splice.py $(GEN_FIR) $(FAME5_FIR) - python3 $(SRC_DIR)/fame5/annotate_blackboxes.py $(GEN_FIR) $(GEN_ANNO) -ifneq ($(FAME5_THREADS),0) - for c in $(FAME5_CLASSES); do python3 $(SRC_DIR)/fame5/annotate_threads.py $(GEN_FIR) $(GEN_ANNO) $$c || exit 1; done -endif - $(MAKE) -C $(FIRESIM_PATH)/sim verilog $(FIRESIM_TARGET_ARGS) -endif clean: rm -rf $(BUILD_DIR) diff --git a/hw/syn/firesim/fame5/FameRam.v b/hw/syn/firesim/fame5/FameRam.v deleted file mode 100644 index 743104e83c..0000000000 --- a/hw/syn/firesim/fame5/FameRam.v +++ /dev/null @@ -1,98 +0,0 @@ -// FameRam: the registered, read-first RAM of VX_dp_ram / VX_sp_ram (OUT_REG=1, -// RDW_MODE="R") as one memory primitive Golden Gate can thread. -// -// Semantics per thread, at its target clock edge (clk is the model's gated -// target clock): byte-lane writes of wdata under wren; when read, the output -// register takes ram[raddr] as it was before this edge's write; when not -// read, the output holds. -// -// Unthreaded (NTHREADS == 1) this is the plain inferred RAM. Threaded, the -// memory is NTHREADS times deeper with the thread index as its low address -// bits, so it stays one RAM (BRAM/URAM-mappable, byte-write enables intact) -// instead of one LUTRAM per byte lane. The shared output register is copied, -// one host cycle after the edge, into a per-thread buffer when that thread -// read; rdata is that thread's entry. threadIdx names the thread whose target -// edge is next (the write stage under pipelined multithreading). -module FameRam #( - parameter DATAW = 32, - parameter SIZE = 64, - parameter WRENW = 1, - parameter ADDRW = 6, - parameter NTHREADS = 1, - parameter TIDXW = 1, - parameter RAM_STYLE = "auto" -) ( - input wire clk, - input wire reset, - input wire read, - input wire write, - input wire [WRENW-1:0] wren, - input wire [ADDRW-1:0] waddr, - input wire [ADDRW-1:0] raddr, - input wire [DATAW-1:0] wdata, - output wire [DATAW-1:0] rdata, - input wire hostClock, - input wire hostReset, - input wire [15:0] threadIdx -); - localparam LANEW = DATAW / WRENW; - localparam DEPTH = SIZE * NTHREADS; - localparam TW = (NTHREADS > 1) ? TIDXW : 0; - - (* ram_style = RAM_STYLE *) reg [DATAW-1:0] ram [0:DEPTH-1]; - reg [DATAW-1:0] dout; - - wire [ADDRW+TW-1:0] wa; - wire [ADDRW+TW-1:0] ra; - generate - if (NTHREADS > 1) begin : g_taddr - assign wa = {waddr, threadIdx[TIDXW-1:0]}; - assign ra = {raddr, threadIdx[TIDXW-1:0]}; - end else begin : g_addr - assign wa = waddr; - assign ra = raddr; - end - endgenerate - - integer i; - always @(posedge clk) begin - for (i = 0; i < WRENW; i = i + 1) begin - if (write && wren[i]) - ram[wa][i*LANEW +: LANEW] <= wdata[i*LANEW +: LANEW]; - end - if (read) - dout <= ram[ra]; - end - - generate - if (NTHREADS > 1) begin : g_threaded - // Edge of the gated target clock, seen in the host domain. - reg tgl; - reg tgl_d; - reg rd_e; - reg [TIDXW-1:0] t_e; - always @(posedge clk) begin - tgl <= ~tgl; - rd_e <= read; - t_e <= threadIdx[TIDXW-1:0]; - end - always @(posedge hostClock) - tgl_d <= tgl; - wire edge_seen = tgl ^ tgl_d; - - (* ram_style = "distributed" *) reg [DATAW-1:0] rbuf [0:NTHREADS-1]; - always @(posedge hostClock) begin - if (edge_seen && rd_e) - rbuf[t_e] <= dout; - end - assign rdata = rbuf[threadIdx[TIDXW-1:0]]; - - initial begin - tgl = 1'b0; - tgl_d = 1'b0; - end - end else begin : g_plain - assign rdata = dout; - end - endgenerate -endmodule diff --git a/hw/syn/firesim/fame5/annotate_blackboxes.py b/hw/syn/firesim/fame5/annotate_blackboxes.py deleted file mode 100644 index f57e960429..0000000000 --- a/hw/syn/firesim/fame5/annotate_blackboxes.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -"""Point Golden Gate at the Verilog of the FIRRTL black boxes the flow emits. - -usage: annotate_blackboxes.py - -Adds a BlackBoxPathAnno for every FameRam extmodule (gen_rams with -FAME5_FAMERAM=1), so FameRam.v is copied next to the generated design and -appended to it, threaded or not. -""" -import json -import os -import re -import sys - -fir, anno = sys.argv[1], sys.argv[2] -src = open(fir).read() -circuit = re.search(r"^circuit (\S+?)\s*:", src, re.M).group(1) -names = re.findall(r"^ extmodule (\S+?)\s*:\n(?: .*\n)*? defname = FameRam$", src, re.M) -path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "FameRam.v") -annos = json.load(open(anno)) -annos += [{"class": "firrtl.transforms.BlackBoxPathAnno", "target": f"{circuit}.{n}", "path": path} for n in names] -json.dump(annos, open(anno, "w"), indent=1) -print(f"annotate_blackboxes: {len(names)} FameRam black boxes") diff --git a/hw/syn/firesim/fame5/annotate_threads.py b/hw/syn/firesim/fame5/annotate_threads.py deleted file mode 100644 index cdefc0d335..0000000000 --- a/hw/syn/firesim/fame5/annotate_threads.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -"""Annotate every instance of a module for FAME-5 threading, in the anno file. - -usage: annotate_threads.py - -Finds each `inst of ` in -- or of any -parameterization of it, `__p` (fame5/name_kept.py) -- and appends a -FirrtlEnableModelMultiThreadingAnnotation targeting it: what Chisel's -EnableModelMultiThreadingAnnotation would emit, for designs with no Chisel. -Each parameterization is its own module, so it becomes its own threaded class. -""" -import json -import re -import sys - -fir, anno, mod = sys.argv[1:] -circuit, parent, targets = None, None, [] -for line in open(fir): - m = re.match(r"circuit (\S+?)\s*:", line) - if m: - circuit = m.group(1) - m = re.match(r" (?:ext)?module (\S+?)\s*:", line) - if m: - parent = m.group(1) - m = re.match(rf"\s+inst (\S+) of ({re.escape(mod)}(?:__p\d+)?)\s*(?:@|$)", line) - if m: - targets.append(f"~{circuit}|{parent}/{m.group(1)}:{m.group(2)}") -if not targets: - sys.exit(f"annotate_threads: no instance of {mod}") -annos = json.load(open(anno)) -annos = [a for a in annos if not (a["class"].endswith("FirrtlEnableModelMultiThreadingAnnotation") - and a["target"] in targets)] -annos += [{"class": "midas.targetutils.FirrtlEnableModelMultiThreadingAnnotation", "target": t} for t in targets] -json.dump(annos, open(anno, "w"), indent=2) -print(f"annotate_threads: {len(targets)} instances of {mod}") diff --git a/hw/syn/firesim/fame5/export_firrtl.py b/hw/syn/firesim/fame5/export_firrtl.py deleted file mode 100644 index 4551e6152c..0000000000 --- a/hw/syn/firesim/fame5/export_firrtl.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -"""FIRRTL export with selective memory lowering. - -usage: export_firrtl.py - -FIRRTL write ports carry one enable and one uniform mask, so write_firrtl -rejects the occasional $mem whose ports encode read-modify-write, per-field -or shifting stores ('Complex write enable on port N on memory M'). Every -memory it can express must stay a memory -- FAME-5 threading turns those -into dense deep LUTRAM/BRAM -- so only the memories the writer itself names -are memory_map'ed to FF logic, and the export is retried until it goes -through. The rejected arrays are register-class storage in every flow -(async reads, a few entries deep); this just makes that explicit. -""" - -import os -import re -import subprocess -import sys - -GLOB = re.compile(r"[][*?]") -ERR = re.compile(r"Complex write enable on port \d+ on memory (\S+?)\.?\s*$", re.M) - -il, out, log = sys.argv[1:4] -yosys = os.environ.get("YOSYS", "yosys") - -mapped = [] # selections accumulated across attempts -with open(log, "w") as lf: - for attempt in range(30): - script = f"read_rtlil {il}\n" - if mapped: - script += "memory_map " + " ".join(mapped) + "\n" - script += f"write_firrtl {out}\n" - r = subprocess.run([yosys, "-q", "-p", script], capture_output=True, text=True) - lf.write(f"--- attempt {attempt}: mapped={mapped}\n{r.stdout}{r.stderr}\n") - lf.flush() - if r.returncode == 0: - sys.exit(0) - m = ERR.search(r.stdout + r.stderr) - if not m: - lf.write("unrecognized failure\n") - print(f"export_firrtl: unrecognized yosys failure (see {log})", file=sys.stderr) - sys.exit(1) - path = m.group(1) - module, tail = path.split(".", 1)[0], path.rsplit(".", 1)[-1] - if GLOB.search(module + tail): - # fall back to the exact path with glob metachars neutralized - sel = [f"{module}/c:" + GLOB.sub("?", path.split('.', 1)[1])] - else: - # map every instance of this array kind in one step - sel = [f"{module}/c:*.{tail}", f"{module}/c:{tail}"] - print(f"export_firrtl: memory_map {' '.join(sel)} ({path})") - mapped += sel - -print(f"export_firrtl: did not converge in 30 attempts (see {log})", file=sys.stderr) -sys.exit(1) diff --git a/hw/syn/firesim/fame5/fix_firrtl.py b/hw/syn/firesim/fame5/fix_firrtl.py deleted file mode 100644 index de8788d857..0000000000 --- a/hw/syn/firesim/fame5/fix_firrtl.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python3 -"""Repair Yosys write_firrtl's output so the FIRRTL compiler and Verilator take it. - -usage: fix_firrtl.py - -Each rewrite keeps the circuit's meaning; they work on expression trees parsed -in one linear pass (a converted Vortex has lines of over a million characters, -nested thousands deep). - -1. Instance outputs driving split wires. Yosys emits - cat(a, cat(b, c)) <= inst.port - which FIRRTL does not allow; each piece gets its slice instead. - -2. Shifts by a signed amount ($shift, B signed). Yosys emits - mux(B < 0, bits(dshl(A, ), 0, W), dshr(A, B)) - which is not FIRRTL (`<`), has bits() bounds reversed, and shifts left by B - rather than -B. $shift is Y = A >> B, and Y = A << -B for B < 0, so it becomes - mux(lt(asSInt(B), SInt<1>(0)), - bits(dshl(pad(A, W), min(-B, W)), W-1, 0), - bits(pad(dshr(A, B), W), W-1, 0)) - -3. Left shifts. Yosys writes bits(dshl(a, min(b, 2^19-1)), hi, lo): correct, - but the intermediate is ~2^19 bits wide, beyond Verilator's constant limit. - Only bits hi..lo are kept, so the clamp becomes the first 2^k-1 > hi. - -4. Zero tests of signed values: neq(asSInt(x), 0) is rejected as a - signed/unsigned comparison; x != 0 does not depend on the sign, so the - asSInt goes (likewise for eq). - -5. Deep expressions. The FIRRTL parser recurses once per nesting level, and - Yosys writes cat()/mux() chains thousands deep for wide arbiters; any - subexpression deeper than MAX_DEPTH is hoisted into a node. -""" -import re -import sys - -sys.setrecursionlimit(100000) # transforms recurse over trees thousands deep - -MAX_DEPTH = 48 -BIG = "UInt<19>(524287)" - -TOKEN = re.compile(r"\s*(?:(?P[US]Int(?:<\d+>)?\([^()]*\))" - r"|(?P[A-Za-z_]\w*)\(" - r"|(?P\))|(?P,)|(?P<)" - r"|(?P[^\s(),<]+))") -ZERO = re.compile(r'^UInt(<\d+>)?\(("h0+"|0)\)$') -DECL = re.compile(r"\s+(?:input|output|wire|reg) (\S+?)\s*: UInt<(\d+)>") -STMT = re.compile(r"(\s+)(node \S+ = |\S+ <= )(.*?)(\s*@\[.*)?$") - - -class Node: - """An operation: op(args...). Leaves (references, literals, integers) are str.""" - __slots__ = ("op", "args") - - def __init__(self, op, args): - self.op, self.args = op, args - - -def parse(text): - """One linear pass over text; `B < 0` (only in rewrite 2) parses as Node('<').""" - stack = [Node("", [])] - items = [[]] # tokens of the argument being read, per open call - pos = 0 - while pos < len(text): - m = TOKEN.match(text, pos) - if not m or m.end() == pos: - if text[pos:].strip() == "": - break - raise ValueError(f"cannot tokenize at: {text[pos:pos + 60]}") - pos = m.end() - kind = m.lastgroup - if kind == "call": - node = Node(m.group("call"), []) - items[-1].append(node) - stack.append(node) - items.append([]) - elif kind in ("comma", "close"): - arg = items[-1] - if arg: - stack[-1].args.append(arg[0] if len(arg) == 1 else Node("<", [arg[0], arg[2]])) - items[-1] = [] - if kind == "close": - stack.pop() - items.pop() - elif kind == "lt": - items[-1].append("<") - else: - items[-1].append(m.group(kind)) - arg = items[0] - return arg[0] if len(arg) == 1 else Node("<", [arg[0], arg[2]]) - - -def emit(t): - """Iterative, so an unhoisted tree of any depth can be printed.""" - out, work = [], [t] - while work: - x = work.pop() - if isinstance(x, str): - out.append(x) - elif isinstance(x, Node): - if x.op == "<": - work += [x.args[1], " < ", x.args[0]] - continue - work.append(")") - for i, a in enumerate(reversed(x.args)): - work.append(a) - if i < len(x.args) - 1: - work.append(", ") - work.append(f"{x.op}(") - return "".join(out) - - -def lit(w, v): - return f"UInt<{w}>({v})" - - -counts = dict(shift=0, shl=0, zero=0, hoist=0, cat=0) - - -def rewrite(t): - """Rewrites 2-4, bottom-up.""" - if isinstance(t, str): - return t - t.args = [rewrite(a) for a in t.args] - a = t.args - # 2. mux(B < 0, bits(dshl(A, S), 0, W), dshr(A, B)) - if t.op == "mux" and len(a) == 3 and isinstance(a[0], Node) and a[0].op == "<": - b, w = a[0].args[0], int(a[1].args[2]) - A = a[1].args[0].args[0] - c = w.bit_length() - mag = Node("asUInt", [Node("neg", [Node("asSInt", [b])])]) - amt = Node("bits", [Node("mux", [Node("gt", [mag, lit(c, w)]), lit(c, w), mag]), str(c - 1), "0"]) - counts["shift"] += 1 - return Node("mux", [Node("lt", [Node("asSInt", [b]), "SInt<1>(0)"]), - Node("bits", [Node("dshl", [Node("pad", [A, str(w)]), amt]), str(w - 1), "0"]), - Node("bits", [Node("pad", [Node("dshr", [A, b]), str(w)]), str(w - 1), "0"])]) - # 3. bits(dshl(A, asUInt(mux(gt(b, BIG), BIG, bits(b, 18, 0)))), hi, lo) - if (t.op == "bits" and len(a) == 3 and isinstance(a[0], Node) and a[0].op == "dshl" - and isinstance(a[0].args[1], Node) and a[0].args[1].op == "asUInt"): - mx = a[0].args[1].args[0] - if (isinstance(mx, Node) and mx.op == "mux" and mx.args[1] == BIG - and isinstance(mx.args[0], Node) and mx.args[0].op == "gt" and mx.args[0].args[1] == BIG): - b = mx.args[0].args[0] - k = (int(a[1]) + 1).bit_length() - m = (1 << k) - 1 - a[0].args[1] = Node("asUInt", [Node("mux", [Node("gt", [b, lit(k, m)]), lit(k, m), - Node("bits", [Node("pad", [b, str(k)]), str(k - 1), "0"])])]) - counts["shl"] += 1 - # 4. (n)eq(asSInt(x), 0) - if (t.op in ("eq", "neq") and len(a) == 2 and isinstance(a[0], Node) and a[0].op == "asSInt" - and isinstance(a[1], str) and ZERO.match(a[1])): - t.args[0] = a[0].args[0] - counts["zero"] += 1 - return t - - -def hoist(t, nodes, indent): - """5. Returns (tree, depth) with every deeper subexpression moved to a node.""" - if isinstance(t, str): - return t, 0 - depth = 0 - for i, k in enumerate(t.args): - k2, d = hoist(k, nodes, indent) - if d >= MAX_DEPTH: - name = f"_fx_{counts['hoist']}" - counts["hoist"] += 1 - nodes.append(f"{indent}node {name} = {emit(k2)}\n") - k2, d = name, 0 - t.args[i] = k2 - depth = max(depth, d) - return t, depth + 1 - - -def cat_leaves(t): - """cat(x, cat(y, z)) -> [x, y, z] (most significant first).""" - if isinstance(t, Node) and t.op == "cat": - return [l for a in t.args for l in cat_leaves(a)] - return [emit(t)] - - -def fix_cat_lhs(line, widths): - m = re.match(r"(\s+)(cat\(.*\)) <= (\S+)(\s*@.*)?$", line.rstrip("\n")) - if not m: - return None - indent, lhs, rhs, info = m.group(1), m.group(2), m.group(3), m.group(4) or "" - out, lo = [], 0 - for leaf in reversed(cat_leaves(parse(lhs))): - if leaf not in widths: - sys.exit(f"fix_firrtl: no width for {leaf} in: {line.strip()[:120]}") - w = widths[leaf] - out.append(f"{indent}{leaf} <= bits({rhs}, {lo + w - 1}, {lo}){info}\n") - lo += w - counts["cat"] += 1 - return out - - -def needs_work(line): - return (" < 0" in line or BIG in line or "eq(asSInt(" in line - or line.count("(") > MAX_DEPTH) - - -widths = {} -with open(sys.argv[1]) as f, open(sys.argv[2], "w") as o: - for line in f: - if line.startswith(" module ") or line.startswith(" extmodule "): - widths = {} - d = DECL.match(line) - if d: - widths[d.group(1)] = int(d.group(2)) - if line.lstrip().startswith("cat("): - fixed = fix_cat_lhs(line, widths) - if fixed: - o.writelines(fixed) - continue - if needs_work(line): - m = STMT.match(line.rstrip("\n")) - if m: - indent, head, rhs, info = m.group(1), m.group(2), m.group(3), m.group(4) or "" - nodes = [] - t, _ = hoist(rewrite(parse(rhs)), nodes, indent) - o.writelines(nodes) - o.write(f"{indent}{head}{emit(t)}{info}\n") - continue - o.write(line) -print(f"fix_firrtl: rewrote {counts['shift']} signed shifts, {counts['cat']} concatenated connects, " - f"{counts['shl']} left shifts, {counts['zero']} signed zero tests, " - f"hoisted {counts['hoist']} deep subexpressions") diff --git a/hw/syn/firesim/fame5/gen_rams.py b/hw/syn/firesim/fame5/gen_rams.py deleted file mode 100644 index dd9f2f4386..0000000000 --- a/hw/syn/firesim/fame5/gen_rams.py +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python3 -"""Replace VX_dp_ram/VX_sp_ram extmodules in a Yosys FIRRTL file with FIRRTL memories. - -usage: gen_rams.py - -Yosys keeps VX_dp_ram as a black box (its FIRRTL writer cannot emit clocked -read ports or byte enables) and writes one extmodule per instance, named -VX_dp_ram_, with that instance's parameters. Each is replaced here by -a module of the same name built on a FIRRTL `mem` whose data type is a vector -of WRENW lanes (so byte enables are write masks), reproducing VX_dp_ram's -simulation model exactly: - - OUT_REG=1, RDW_MODE=W raddr registered on read, rdata = ram[raddr_r] - -> 1-cycle read of the next raddr_r, plus a bypass - of a same-cycle write to that address. - OUT_REG=1, RDW_MODE=R rdata_r <= ram[raddr] on read (value before write) - -> writes delayed one cycle through a buffer that - reads forward from; output held when not reading. - OUT_REG=0, RDW_MODE=W rdata = ram[raddr], combinational. - OUT_REG=0, RDW_MODE=R combinational, with the one-cycle old-data bypass. - -The bypasses make the result independent of the memory's read-under-write -behaviour, which FAME-5's threaded memories do not preserve. Every OUT_REG=1 -memory has read latency 1, so it stays block-RAM-mappable after threading. -""" -import os -import re -import sys - -DEFAULTS = dict(DATAW=1, SIZE=1, WRENW=1, OUT_REG=0, RDW_MODE="W", RESET_RAM=0, INIT_ENABLE=0) - - -def log2up(n): - return max(1, (n - 1).bit_length()) - - -def lanes(expr, n, w): - return [f"bits({expr}, {i * w + w - 1}, {i * w})" for i in range(n)] - - -def cat(items): - """cat(items[n-1], ..., items[0]) -- items[0] is the least significant.""" - out = items[0] - for it in items[1:]: - out = f"cat({it}, {out})" - return out - - -def gen(name, prm, sp): - p = dict(DEFAULTS) - p.update(prm) - for k in ("RESET_RAM", "INIT_ENABLE"): - if int(p[k]): - sys.exit(f"gen_rams: {name}: {k} is not supported") - dw, size, nw = int(p["DATAW"]), int(p["SIZE"]), int(p["WRENW"]) - oreg, mode = int(p["OUT_REG"]), p["RDW_MODE"].strip('"') - aw, lw = log2up(size), dw // nw - assert lw * nw == dw - L = [] - a = L.append - a(f" module {name}:") - addrs = (("input", "addr", aw),) if sp else (("input", "waddr", aw), ("input", "raddr", aw)) - for d, n, w in (("input", "clk", None), ("input", "reset", 1), ("input", "read", 1), ("input", "write", 1), - ("input", "wren", nw), ("input", "wdata", dw)) + addrs + (("output", "rdata", dw),): - a(f" {d} {n}: {'Clock' if w is None else f'UInt<{w}>'}") - a("") - if sp: - # VX_sp_ram: one address for both; its no-change mode is read-first - # with the output kept when the cycle also writes. - a(" node waddr = addr") - a(" node raddr = addr") - if mode == "N": - assert oreg, f"{name}: RDW_MODE N needs OUT_REG" - mode = "N" - orig_mode = mode - rden = "and(read, not(write))" if mode == "N" else "read" - if mode == "N": - mode = "R" - readers = ["r", "r2"] if (oreg == 0 and mode == "R") else ["r"] - # FAME5_NATIVE_RAMS=1: a registered read-first RAM is a plain latency-1 - # memory with read-under-write = old, read enable = rden, and output hold - # left to the memory -- no write buffer, forwarding or hold registers. - # Requires a Golden Gate whose threaded memories hold per-thread read data - # while the read port is disabled. - # (No-change mode is excluded: its exclusive read/write enables would let - # InferReadWrite merge the ports into one read-write port.) - native_r = oreg and orig_mode == "R" and os.environ.get("FAME5_NATIVE_RAMS") == "1" - # FAME5_FAMERAM=1: the same RAM as one FameRam black box (FameRam.v), which - # Golden Gate threads as a single memory -- byte lanes kept together, so it - # stays BRAM/URAM-mappable instead of becoming one LUTRAM per lane. - if oreg and orig_mode == "R" and os.environ.get("FAME5_FAMERAM") == "1": - style = "block" if dw * size >= 4096 else "auto" - a(f" inst ram of FameRam_{name}") - a(" ram.clk <= clk") - a(" ram.reset <= reset") - a(" ram.read <= read") - a(" ram.write <= write") - a(" ram.wren <= wren") - a(" ram.waddr <= waddr") - a(" ram.raddr <= raddr") - a(" ram.wdata <= wdata") - a(" rdata <= ram.rdata") - # Host-side ports, tied off here; Golden Gate reconnects them when it - # threads the RAM. Every FameRam extmodule has the same port list. - a(" ram.hostClock <= clk") - a(" ram.hostReset <= UInt<1>(0)") - a(" ram.threadIdx <= UInt<16>(0)") - a("") - a(f" extmodule FameRam_{name}:") - for d, n, w in (("input", "clk", None), ("input", "reset", 1), ("input", "read", 1), ("input", "write", 1), - ("input", "wren", nw), ("input", "waddr", aw), ("input", "raddr", aw), - ("input", "wdata", dw), ("output", "rdata", dw), - ("input", "hostClock", None), ("input", "hostReset", 1), ("input", "threadIdx", 16)): - a(f" {d} {n}: {'Clock' if w is None else f'UInt<{w}>'}") - a(" defname = FameRam") - for k, v in (("DATAW", dw), ("SIZE", size), ("WRENW", nw), ("ADDRW", aw), ("NTHREADS", 1), ("TIDXW", 1)): - a(f" parameter {k} = {v}") - a(f' parameter RAM_STYLE = "{style}"') - a("") - return L - a(" mem ram:") - a(f" data-type => UInt<{lw}>[{nw}]") - a(f" depth => {size}") - a(f" read-latency => {1 if oreg else 0}") - a(" write-latency => 1") - for r in readers: - a(f" reader => {r}") - a(" writer => w") - a(f" read-under-write => {'old' if native_r else 'undefined'}") - a("") - for r in readers: - a(f" ram.{r}.clk <= clk") - a(f" ram.{r}.en <= {rden if native_r else 'UInt<1>(1)'}") - a(" ram.w.clk <= clk") - mask = (lambda src: [f"bits({src}, {i}, {i})" for i in range(nw)]) if nw > 1 else (lambda src: ["UInt<1>(1)"]) - - def write_port(en, addr, data, wren): - a(f" ram.w.en <= {en}") - a(f" ram.w.addr <= {addr}") - for i, (d, m) in enumerate(zip(lanes(data, nw, lw), mask(wren))): - a(f" ram.w.data[{i}] <= {d}") - a(f" ram.w.mask[{i}] <= {m}") - - def merge(base, data, hit, wren): - """base with the lanes of data enabled by (hit & wren) substituted.""" - m = mask(wren) - out = [] - for i, (b, d) in enumerate(zip(lanes(base, nw, lw), lanes(data, nw, lw))): - out.append(f"mux(and({hit}, {m[i]}), {d}, {b})") - return cat(out) - - mem_out = lambda r: cat([f"ram.{r}.data[{i}]" for i in range(nw)]) - - if oreg and mode == "W": - a(f" reg raddr_r: UInt<{aw}>, clk") - a(" node next_addr = mux(read, raddr, raddr_r)") - a(" raddr_r <= next_addr") - a(" ram.r.addr <= next_addr") - write_port("write", "waddr", "wdata", "wren") - a(" reg hit_r: UInt<1>, clk") - a(" hit_r <= and(write, eq(waddr, next_addr))") - a(f" reg wdata_r: UInt<{dw}>, clk") - a(" wdata_r <= wdata") - a(f" reg wren_r: UInt<{nw}>, clk") - a(" wren_r <= wren") - a(f" rdata <= {merge(mem_out('r'), 'wdata_r', 'hit_r', 'wren_r')}") - elif native_r: - a(" ram.r.addr <= raddr") - write_port("write", "waddr", "wdata", "wren") - a(f" rdata <= {mem_out('r')}") - elif oreg and mode == "R": - a(" reg wb_v: UInt<1>, clk") - a(f" reg wb_addr: UInt<{aw}>, clk") - a(f" reg wb_data: UInt<{dw}>, clk") - a(f" reg wb_en: UInt<{nw}>, clk") - a(" wb_v <= write") - a(" wb_addr <= waddr") - a(" wb_data <= wdata") - a(" wb_en <= wren") - write_port("wb_v", "wb_addr", "wb_data", "wb_en") - a(" ram.r.addr <= raddr") - a(" reg hit_r: UInt<1>, clk") - a(" hit_r <= and(wb_v, eq(wb_addr, raddr))") - a(f" reg fwd_data: UInt<{dw}>, clk") - a(" fwd_data <= wb_data") - a(f" reg fwd_en: UInt<{nw}>, clk") - a(" fwd_en <= wb_en") - a(" reg read_r: UInt<1>, clk") - a(f" read_r <= {rden}") - a(f" reg hold: UInt<{dw}>, clk") - a(f" node val = mux(read_r, {merge(mem_out('r'), 'fwd_data', 'hit_r', 'fwd_en')}, hold)") - a(" hold <= val") - a(" rdata <= val") - elif not oreg and mode == "W": - a(" ram.r.addr <= raddr") - write_port("write", "waddr", "wdata", "wren") - a(f" rdata <= {mem_out('r')}") - else: - a(" ram.r.addr <= raddr") - a(" ram.r2.addr <= waddr") - write_port("write", "waddr", "wdata", "wren") - a(" reg prev_write: UInt<1>, clk with: (reset => (reset, UInt<1>(0)))") - a(f" reg prev_data: UInt<{dw}>, clk with: (reset => (reset, UInt<{dw}>(0)))") - a(f" reg prev_waddr: UInt<{aw}>, clk with: (reset => (reset, UInt<{aw}>(0)))") - a(" prev_write <= write") - a(f" prev_data <= {mem_out('r2')}") - a(" prev_waddr <= waddr") - a(f" rdata <= mux(and(prev_write, eq(prev_waddr, raddr)), prev_data, {mem_out('r')})") - a("") - return L - - -src = open(sys.argv[1]).read().splitlines() -out, i, n = [], 0, 0 -while i < len(src): - m = re.match(r" extmodule (VX_[ds]p_ram\S*?)\s*:", src[i]) - if not m: - out.append(src[i]) - i += 1 - continue - name, j, prm = m.group(1), i + 1, {} - while j < len(src) and not re.match(r" (ext)?module ", src[j]): - pm = re.match(r"\s+parameter (\w+) = (.*?)\s*$", src[j]) - if pm: - prm[pm.group(1)] = pm.group(2) - j += 1 - out += gen(name, prm, name.startswith("VX_sp_ram")) - n += 1 - i = j -open(sys.argv[2], "w").write("\n".join(out) + "\n") -print(f"gen_rams: {n} VX_dp_ram/VX_sp_ram instances") diff --git a/hw/syn/firesim/fame5/lift_boundaries.py b/hw/syn/firesim/fame5/lift_boundaries.py deleted file mode 100644 index 8f7de58149..0000000000 --- a/hw/syn/firesim/fame5/lift_boundaries.py +++ /dev/null @@ -1,537 +0,0 @@ -#!/usr/bin/env python3 -"""Interface boundary lifting: make interface-port units FAME-5 fold classes. - -sv2v inlines every module that has SystemVerilog interface ports, so units such -as VX_issue_slice or VX_cluster never reach the FIRRTL as modules and cannot be -threaded. This pass gives each requested unit a boundary made of plain wires, -without touching the RTL: it rewrites a copy of the (preprocessed) sources. - -For each unit U: - * every elaborated instance must have the same parameter values (parameters - listed as droppable -- debug labels such as ISSUE_ID -- are left at their - defaults); - * a parameter-free wrapper U__lift is generated: each interface port becomes - one plain port per signal (per array element, packed), with the direction - taken from the port's modport or, for a bare interface port, from where - the signal is driven (inside the unit -> output); the wrapper instantiates - the interfaces with their elaborated parameters and U with its elaborated - parameter values; - * every instantiation site of U is replaced by glue wires/assigns between - the parent's interfaces and an instance of U__lift. -Behaviour is unchanged: the glue is pure wiring. - -Parameters that are functional instance IDs (e.g. VX_cluster's CLUSTER_ID) -are promoted to 32-bit input ports (port=P): the parameter becomes a port of U, -every child parameter override whose expression uses it is promoted the same -way down the hierarchy, and every affected instantiation passes the value as a -port connection instead (the old default where it was not overridden). Uses in -constant contexts (widths, generate conditions) make the re-elaboration fail, -so an invalid promotion cannot go through silently. - -usage: lift_boundaries.py --srcs --top --out - --unit U[:drop=P1,P2][;port=P3] [--unit ...] -""" -import argparse -import collections -import os -import re -import shutil -import sys - -import pyslang -from pyslang import ast, analysis - -SIG_KINDS = (ast.SymbolKind.Variable, ast.SymbolKind.Net) - - -def read_srcs(path): - defines, incdirs, files = [], [], [] - for line in open(path): - line = line.strip() - if line.startswith("+define+"): - defines.append(line[len("+define+"):]) - elif line.startswith("+incdir+"): - incdirs.append(line[len("+incdir+"):]) - elif line: - files.append(line) - return defines, incdirs, files - - -def q(a): - return "'" + a.replace("'", "'\\''") + "'" if any(c in a for c in " '\"") else a - - -def elaborate(defines, incdirs, files, top): - d = pyslang.driver.Driver() - d.addStandardArgs() - # The Vortex RTL references some nets before their declaration (VX_tcu_core), - # which Vivado and Verilator accept; slang makes it an error unless allowed. - args = ["slang", "--top", top, "--allow-hierarchical-const", "--allow-use-before-declare", - "--ignore-unknown-modules"] - args += [f"-D{x}" for x in defines] + [f"-I{x}" for x in incdirs] + files - if not (d.parseCommandLine(" ".join(q(a) for a in args), pyslang.driver.CommandLineOptions()) - and d.processOptions() and d.parseAllSources()): - sys.exit("lift: slang could not parse the sources") - comp = d.createCompilation() - used = set() - - def visit(sym): - if sym.kind == ast.SymbolKind.Instance: - used.add(sym.definition.name) - return ast.VisitAction.Advance - for t in comp.getRoot().topInstances: - t.visit(visit) - sm = comp.sourceManager - - def unused_only(fname): - # A file that only defines modules outside the elaborated hierarchy (as - # sv2v --top ignores them): errors there cannot affect the design. - names = re.findall(r"^\s*(module|interface|package)\s+(\w+)", open(fname).read(), re.M) - return bool(names) and all(k == "module" and n not in used for k, n in names) - errs = [x for x in comp.getAllDiagnostics() if x.isError() - and not unused_only(str(sm.getFileName(x.location)))] - if errs: - x = errs[0] - sys.exit(f"lift: {len(errs)} elaboration errors (first: {x.code} at " - f"{sm.getFileName(x.location)}:{sm.getLineNumber(x.location)})") - am = analysis.AnalysisManager() - am.analyze(comp) - return d, comp, am - - -def vec(w): - return "" if w == 1 else f"[{w - 1}:0] " - - -def lit(v): - s = str(v) - return s - - -class Unit: - def __init__(self, spec): - name, _, opts = spec.partition(":") - self.name = name - self.drop = set() - self.ports = [] # parameters promoted to input ports (functional instance IDs) - for o in filter(None, opts.split(";")): - k, _, v = o.partition("=") - if k == "drop": - self.drop |= set(filter(None, v.split(","))) - elif k == "port": - self.ports += [x for x in v.split(",") if x] - self.insts = [] - - -def params_of(inst, drop=()): - """Overridable value and type parameters of an elaborated instance, as SV text.""" - out = collections.OrderedDict() - for m in inst.body: - if m.name in drop: - continue - if m.kind == ast.SymbolKind.Parameter and not m.isLocalParam: - out[m.name] = lit(m.value) - elif m.kind == ast.SymbolKind.TypeParameter and not m.isLocalParam: - t = str(m.targetType.type) - if not re.match(r"^[A-Za-z_][\w$]*(::[A-Za-z_][\w$]*)*$", t) and not re.match(r"^(logic|bit)( ?\[\d+:\d+\])*$", t): - sys.exit(f"lift: type parameter {inst.hierarchicalPath}.{m.name} = '{t}' has no name to pass") - out[m.name] = t - return out - - -def iface_elements(conn): - target, modport = conn.ifaceConn - if target.kind == ast.SymbolKind.InstanceArray: - return list(target.elements), modport - return [target], modport - - -def signal_dirs(unit_inst, elem, modport_name, am): - """[(signal, 'input'|'output', width)] from the unit's point of view.""" - out = [] - mp = None - if modport_name: - mp = next((m for m in elem.body if m.kind == ast.SymbolKind.Modport and m.name == modport_name), None) - mp_dirs = {} - if mp is not None: - for p in mp: - mp_dirs[p.name] = "input" if str(p.direction).endswith("In") else "output" - upath = unit_inst.hierarchicalPath - for m in elem.body: - if m.kind not in SIG_KINDS: - continue - if m.type.isUnpackedArray: - sys.exit(f"lift: unpacked interface signal {elem.hierarchicalPath}.{m.name} not supported") - if mp is not None: - if m.name not in mp_dirs: - continue - d = mp_dirs[m.name] - else: - drv = am.getDrivers(m) - inside = any(dv.containingSymbol.hierarchicalPath == upath - or dv.containingSymbol.hierarchicalPath.startswith(upath + ".") - or dv.containingSymbol.hierarchicalPath.startswith(upath + "[") for dv in drv) - d = "output" if inside else "input" - out.append((m.name, d, m.type.bitWidth)) - return out - - -def port_range(port): - """Declared interface-array range as text ('[0:3]'), or '' for a single interface.""" - r = port.declaredRange - if not r: - return "", 1 - rr = r[0] - lo, hi = rr.left, rr.right - return f"[{lo}:{hi}]", abs(hi - lo) + 1 - - -def build_wrapper(u, rep, am): - ports, body, inner_conns = [], [], [] - shape = [] # per port: ('plain', name, dir, w) or ('iface', name, rng, n, [(sig, dir, w)]) - for c in rep.portConnections: - p = c.port - if p.kind == ast.SymbolKind.InterfacePort: - elems, mp = iface_elements(c) - rng, n = port_range(p) - sigs = signal_dirs(rep, elems[0], p.modport, am) - idef = p.interfaceDef.name - iparams = params_of(elems[0]) - pa = "" - if iparams: - pa = " #(" + ", ".join(f".{k}({v})" for k, v in iparams.items()) + ")" - body.append(f" {idef}{pa} {p.name} {rng}();".replace(" ();", "();")) - for s, d, w in sigs: - pn = f"{p.name}__{s}" - ports.append(f" {d} wire {('[' + str(n - 1) + ':0]') if rng else ''}{vec(w)}{pn}") - for k in range(n): - ref = f"{p.name}[{k}].{s}" if rng else f"{p.name}.{s}" - pk = f"{pn}[{k}]" if rng else pn - body.append(f" assign {ref} = {pk};" if d == "input" else f" assign {pk} = {ref};") - inner_conns.append(f" .{p.name} ({p.name})") - shape.append(("iface", p.name, rng, n, sigs)) - else: - d = str(p.direction).split(".")[-1].lower() - d = {"in": "input", "out": "output", "inout": "inout"}.get(d, d) - if p.type.isUnpackedArray: - sys.exit(f"lift: unpacked port {u.name}.{p.name} not supported") - w = p.type.bitWidth - ports.append(f" {d} wire {vec(w)}{p.name}") - inner_conns.append(f" .{p.name} ({p.name})") - shape.append(("plain", p.name, d, w)) - for pp in u.ports: - ports.insert(0, f" input wire [31:0] {pp}") - inner_conns.insert(0, f" .{pp} ({pp})") - shape.insert(0, ("promoted", pp)) - prm = params_of(rep, u.drop | set(u.ports)) - pa = (" #(\n" + ",\n".join(f" .{k} ({v})" for k, v in prm.items()) + "\n )") if prm else "" - text = (f"// Generated by lift_boundaries.py: {u.name} with plain ports (FAME-5 fold boundary).\n" - f"module {u.name}__lift (\n" + ",\n".join(ports) + "\n);\n" + "\n".join(body) + "\n\n" - f" {u.name}{pa} inner (\n" + ",\n".join(inner_conns) + "\n );\nendmodule\n") - return text, shape - - -SLICE = re.compile(r"^(.*)\[(.+)\+:(.+)\]$") - - -def elem_expr(e, k, is_array): - e = e.strip() - if not is_array: - return e - m = SLICE.match(e) - if m: - return f"{m.group(1)}[({m.group(2).strip()}) + {k}]" - if re.match(r"^[A-Za-z_]\w*$", e): - return f"{e}[{k}]" - if k == 0: - return e # a single element bound to a one-element array port - sys.exit(f"lift: cannot index connection expression '{e}'") - - -def rewrite_site(u, site_syntax, shape, uid): - """Replacement text for one instantiation statement.""" - inst = site_syntax.instances[0] - iname = str(inst.decl.name).strip() - conns = {} - for c in inst.connections: - if not hasattr(c, "name"): - continue # separator token - nm = str(c.name).strip() - ex = str(c.expr).strip() if c.expr is not None else "" - conns[nm] = ex - overrides = site_overrides(site_syntax) - pre = f"__lift{uid}" - decl, glue, pc = [], [], [] - for kind, *rest in shape: - if kind == "promoted": - pp = rest[0] - if pp not in overrides: - sys.exit(f"lift: {u.name} instance {iname}: promoted parameter {pp} is not overridden at the site") - pc.append(f" .{pp} ({overrides[pp]})") - continue - if kind == "plain": - name, d, w = rest - pc.append(f" .{name} ({conns.get(name, '')})") - continue - name, rng, n, sigs = rest - e = conns.get(name) - if e is None: - sys.exit(f"lift: {u.name} instance {iname}: interface port {name} not connected by name") - for s, d, w in sigs: - wn = f"{pre}_{name}__{s}" - decl.append(f"wire {('[' + str(n - 1) + ':0]') if rng else ''}{vec(w)}{wn};") - for k in range(n): - ek = elem_expr(e, k, bool(rng)) - wk = f"{wn}[{k}]" if rng else wn - glue.append(f"assign {wk} = {ek}.{s};" if d == "input" else f"assign {ek}.{s} = {wk};") - pc.append(f" .{name}__{s} ({wn})") - return ("// lift_boundaries: " + u.name + " -> " + u.name + "__lift\n " - + "\n ".join(decl + glue) + "\n " - + f"{u.name}__lift {iname} (\n" + ",\n".join(pc) + "\n );") - - -def site_overrides(site): - """{param: expr text} of a HierarchyInstantiationSyntax's #(...) named overrides.""" - out = {} - pva = site.parameters - if pva is None: - return out - for it in pva.parameters: - if hasattr(it, "name") and hasattr(it, "expr"): - out[str(it.name).strip()] = str(it.expr).strip() if it.expr is not None else "" - return out - - -def _remove_item(text_len_getter, node, sm): - """(start, end) covering a separated-list item and one adjacent comma.""" - return node.sourceRange.start.offset, node.sourceRange.end.offset - - -class Promoter: - """Parameter-to-port promotion across definitions (see module docstring).""" - - def __init__(self, comp, sm, lifted_sites): - self.comp, self.sm = comp, sm - self.lifted_sites = lifted_sites # site keys rewritten by lifting: skip - self.done = {} # (def, param) -> default text - self.edits = collections.defaultdict(list) - self.site_conv = collections.defaultdict(set) # site key -> {param} converted - self.insts = collections.defaultdict(list) - for t in comp.getRoot().topInstances: - t.visit(self._collect) - - def _collect(self, sym): - if sym.kind == ast.SymbolKind.Instance: - self.insts[sym.definition.name].append(sym) - return ast.VisitAction.Advance - - def key(self, node): - r = node.sourceRange - return (self.sm.getFileName(r.start), r.start.offset) - - def promote(self, defname, param): - if (defname, param) in self.done: - return - rep = self.insts[defname][0] - dsyn = rep.definition.syntax - hdr = dsyn.header - f = self.sm.getFileName(dsyn.sourceRange.start) - text = open(f).read() - default = None - target = None - for it in hdr.parameters.declarations: - if not hasattr(it, "declarators"): - continue - names = [str(dd.name).strip() for dd in it.declarators if hasattr(dd, "name")] - if param in names: - if len(names) != 1: - sys.exit(f"lift: {defname}.{param} shares a declaration; split it first") - dd = [dd for dd in it.declarators if hasattr(dd, "name")][0] - default = str(dd.initializer.expr).strip() if dd.initializer is not None else "0" - target = it - if target is None: - sys.exit(f"lift: no parameter {param} in {defname}") - self.done[(defname, param)] = default - a, b = target.sourceRange.start.offset, target.sourceRange.end.offset - self.edits[f].append(_with_comma(text, a, b)) - op = hdr.ports.openParen.location.offset + 1 - self.edits[f].append((op, op, f"\n input wire [31:0] {param},")) - # children whose overrides use the promoted value - for ch in rep.body: - pass - for cinst in self._children(rep): - site = cinst.syntax.parent - ov = site_overrides(site) - for q, ex in ov.items(): - if re.search(rf"\b{re.escape(param)}\b", ex): - self.promote(cinst.definition.name, q) - - def _children(self, inst): - out = [] - def v(sym): - if sym is inst: - return ast.VisitAction.Advance - if sym.kind == ast.SymbolKind.Instance: - out.append(sym) - return ast.VisitAction.Skip - return ast.VisitAction.Advance - inst.visit(v) - return out - - def convert_sites(self): - """Every instantiation of a promoted (def, param): override -> port connection.""" - for (defname, param), default in self.done.items(): - for inst in self.insts[defname]: - site = inst.syntax.parent - k = self.key(site) - if k in self.lifted_sites or param in self.site_conv[k]: - continue - self.site_conv[k].add(param) - f = k[0] - text = open(f).read() - ov_items = {} - if site.parameters is not None: - for it in site.parameters.parameters: - if hasattr(it, "name"): - ov_items[str(it.name).strip()] = it - if param in ov_items: - it = ov_items[param] - ex = str(it.expr).strip() - a, b = it.sourceRange.start.offset, it.sourceRange.end.offset - self.edits[f].append(_with_comma(text, a, b)) - else: - ex = default - inst_syn = site.instances[0] - op = inst_syn.openParen.location.offset + 1 - self.edits[f].append((op, op, f"\n .{param} ({ex}),")) - - -def _with_comma(text, a, b): - """Deletion range for a list item at [a, b) plus its separating comma.""" - j = b - while j < len(text) and text[j] in " \t\r\n": - j += 1 - if j < len(text) and text[j] == ",": - return (a, j + 1, "") - i = a - 1 - while i >= 0 and text[i] in " \t\r\n": - i -= 1 - if i >= 0 and text[i] == ",": - return (i, b, "") - return (a, b, "") - - -def apply_edits(text, edits): - """Apply (start, end, repl) edits; overlapping pure deletions are merged.""" - edits = sorted(edits, key=lambda e: (e[0], e[1])) - merged = [] - for s, e, t in edits: - if merged and s < merged[-1][1]: - ps, pe, pt = merged[-1] - if t or pt: - sys.exit(f"lift: overlapping edits at {s}") - merged[-1] = (ps, max(pe, e), "") - else: - merged.append((s, e, t)) - for s, e, t in reversed(merged): - text = text[:s] + t + text[e:] - return text - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--srcs", required=True) - ap.add_argument("--top", required=True) - ap.add_argument("--out", required=True) - ap.add_argument("--unit", action="append", required=True) - a = ap.parse_args() - a.out = os.path.abspath(a.out) - defines, incdirs, files = read_srcs(a.srcs) - d, comp, am = elaborate(defines, incdirs, files, a.top) - sm = comp.sourceManager - units = {u.name: u for u in (Unit(s) for s in a.unit)} - - def visit(sym): - if sym.kind == ast.SymbolKind.Instance and sym.definition.name in units: - units[sym.definition.name].insts.append(sym) - return ast.VisitAction.Advance - # Only the elaborated hierarchy: slang also builds placeholder instances of - # uninstantiated definitions, which must not count. - for t in comp.getRoot().topInstances: - t.visit(visit) - - edits = collections.defaultdict(list) # file -> [(start, end, text)] - wrappers = {} - uid = 0 - lifted_sites = set() - for u in units.values(): - if not u.insts: - sys.exit(f"lift: no instance of {u.name}") - skip = u.drop | set(u.ports) - groups = collections.Counter(tuple(params_of(i, skip).items()) for i in u.insts) - if len(groups) > 1: - keys = list(params_of(u.insts[0], skip).keys()) - diff = [k for k in keys if len({dict(g)[k] for g in groups}) > 1] - sys.exit(f"lift: instances of {u.name} differ in {diff}; drop them if they are labels") - text, shape = build_wrapper(u, u.insts[0], am) - wrappers[u.name] = text - seen = set() - for i in u.insts: - site = i.syntax.parent # HierarchyInstantiationSyntax - rng = site.sourceRange - key = (sm.getFileName(rng.start), rng.start.offset) - if key in seen: - continue - seen.add(key) - lifted_sites.add(key) - if len(site.instances) != 1: - sys.exit(f"lift: {u.name}: multi-instance statement at {key}") - edits[key[0]].append((rng.start.offset, rng.end.offset, rewrite_site(u, site, shape, uid))) - uid += 1 - print(f"lift: {u.name}: {len(u.insts)} instances, {len(seen)} sites, " - f"{sum(1 for s in shape if s[0] == 'iface')} interface ports, " - f"{sum(len(s[4]) for s in shape if s[0] == 'iface')} lifted signals") - - pr = Promoter(comp, sm, lifted_sites) - for u in units.values(): - for pp in u.ports: - pr.promote(u.name, pp) - pr.convert_sites() - for f, es in pr.edits.items(): - edits[f].extend(es) - if pr.done: - print("lift: promoted " + ", ".join(f"{d}.{p}" for d, p in pr.done)) - - os.makedirs(os.path.join(a.out, "src"), exist_ok=True) - out_files = [] - by_real = {os.path.realpath(str(k)): k for k in edits} - applied, matched = 0, set() - for f in files: - dst = os.path.join(a.out, "src", os.path.basename(f)) - key = by_real.get(os.path.realpath(f)) - if key is not None: - open(dst, "w").write(apply_edits(open(f).read(), edits[key])) - applied += 1 - matched.add(key) - else: - shutil.copyfile(f, dst) - out_files.append(dst) - for name, text in wrappers.items(): - p = os.path.join(a.out, "src", f"{name}__lift.sv") - open(p, "w").write(text) - out_files.append(p) - with open(os.path.join(a.out, "srcs.f"), "w") as o: - for x in defines: - o.write(f"+define+{x}\n") - for x in incdirs: - o.write(f"+incdir+{x}\n") - o.write(f"+incdir+{os.path.join(a.out, 'src')}\n") - for x in out_files: - o.write(x + "\n") - print(f"lift: wrote {len(out_files)} files ({applied} of {len(edits)} rewritten) to {a.out}") - if applied != len(edits): - sys.exit(f"lift: edits for unmatched files: {sorted(set(edits) - matched)}") - - -if __name__ == "__main__": - main() diff --git a/hw/syn/firesim/fame5/name_kept.py b/hw/syn/firesim/fame5/name_kept.py deleted file mode 100644 index a991cc9b35..0000000000 --- a/hw/syn/firesim/fame5/name_kept.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -"""Give kept parameterized modules public names in an RTLIL design. - -usage: name_kept.py (rewritten in place) - -After flattening, the only modules left are the top, the kept units (FAME-5 -classes) and the black boxes. A kept unit that was instantiated with -parameters is named $paramod...\\, and Yosys write_firrtl names its -instances by the cell's hdlname attribute rather than the module, so they -would not refer to the module it emits. Renaming each non-blackbox $paramod -module to \\__p (module and every cell of that type) makes the -writer use the module's own name. -""" - -import re -import sys - -il = sys.argv[1] -text = open(il).read() - -mods = [] -attrs = [] -for line in text.splitlines(): - s = line.strip() - if s.startswith("attribute "): - attrs.append(s) - continue - if s.startswith("module "): - name = s.split(None, 1)[1] - if name.startswith("$paramod") and not any(a.startswith("attribute \\blackbox") for a in attrs): - mods.append(name) - attrs = [] - -if mods: - ren = {} - count = {} - for m in mods: - base = re.sub(r"[^A-Za-z0-9_]", "_", m.split("\\")[1]) if "\\" in m else "mod" - n = count.get(base, 0) - count[base] = n + 1 - ren[m] = f"\\{base}__p{n}" - out = [] - for line in text.splitlines(keepends=True): - s = line.lstrip() - if s.startswith("module ") or s.startswith("cell "): - parts = line.split(" ") - for i, p in enumerate(parts): - key = p.rstrip("\n") - if key in ren: - parts[i] = ren[key] + ("\n" if p.endswith("\n") else "") - line = " ".join(parts) - out.append(line) - open(il, "w").write("".join(out)) -print(f"name_kept: {len(mods)} kept parameterized modules renamed") diff --git a/hw/syn/firesim/fame5/selfref_wires.py b/hw/syn/firesim/fame5/selfref_wires.py deleted file mode 100644 index 1e27f401c6..0000000000 --- a/hw/syn/firesim/fame5/selfref_wires.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -"""Tag the wires whose bits are driven by other bits of the same wire. - -usage: selfref_wires.py (rewritten in place; prints the count) - -Yosys reuses a bit of a wire to drive other bits of it (a write-enable vector -built by replicating one enable bit, say). That is acyclic bit by bit, but -FIRRTL checks loops per wire, so the exported module fails the combinational -loop check. Each such wire gets the attribute fame5_selfref, for -`splitnets a:fame5_selfref` to split it into single bits; nothing else is -touched. -""" - -import re -import sys - -il = sys.argv[1] -lines = open(il).read().splitlines(keepends=True) - -selfref = set() # (module, wire) -mod = None -for line in lines: - if line.startswith("module "): - mod = line.split()[1] - elif line.startswith(" connect "): - m = re.match(r" connect (\\\S+|\$\S+)(?: \[[0-9:]+\])?\s+(.*)", line) - if m and m.group(1) in re.findall(r"(\\\S+|\$\S+)", m.group(2)): - selfref.add((mod, m.group(1))) - -if selfref: - out = [] - mod = None - for line in lines: - if line.startswith("module "): - mod = line.split()[1] - elif line.startswith(" wire "): - name = line.split()[-1] - if (mod, name) in selfref: - out.append(" attribute \\fame5_selfref 1\n") - out.append(line) - open(il, "w").write("".join(out)) -print(len(selfref)) diff --git a/hw/syn/firesim/fame5/shorten_fir.py b/hw/syn/firesim/fame5/shorten_fir.py deleted file mode 100644 index 01123c4020..0000000000 --- a/hw/syn/firesim/fame5/shorten_fir.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -"""Shorten internal identifiers in Yosys-written FIRRTL. - -usage: shorten_fir.py - -Flattening leaves every wire of the top region carrying its full -hierarchical path as its name, and splitnets multiplies those wires per -bit: the identifiers become most of the file's bytes, and a full-GPU -export exceeds the FIRRTL reader's 2 GiB input limit. This pass renames -module-local identifiers (wire/node/reg/cmem/smem/mem/inst names) longer -than a threshold to sequential short names, per module. Module names and -port names are never touched, so instantiation, splicing and annotation -are unaffected. Renaming is semantics-preserving: FIRRTL identifiers are -local to their module. -""" - -import re -import sys - -THRESHOLD = 8 # keep short names for readability; rename only long ones - -DECL = re.compile( - r"^\s*(?:wire|node|reg|cmem|smem|inst)\s+([A-Za-z_][A-Za-z0-9_$]*)" -) -MEM = re.compile(r"^\s*mem\s+([A-Za-z_][A-Za-z0-9_$]*)\s*:") -MODULE = re.compile(r"^ (?:ext)?module\s+(\S+?)\s*:") -TOKEN = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*") - -src, dst = sys.argv[1], sys.argv[2] - -# Pass 1: per module, map long declared names to short ones. -maps = {} # module -> {long: short} -mod = None -with open(src, "r", errors="strict") as fh: - for line in fh: - m = MODULE.match(line) - if m: - mod = m.group(1) - maps[mod] = {} - continue - if mod is None: - continue - d = DECL.match(line) or MEM.match(line) - if d: - name = d.group(1) - if len(name) > THRESHOLD and name not in maps[mod]: - maps[mod][name] = f"sn{len(maps[mod])}" - -# Safety: no existing short identifier may collide with generated ones. -collide = re.compile(r"^sn\d+$") -for mod_name, mp in maps.items(): - for long_name in mp: - assert not collide.match(long_name), f"collision risk: {long_name}" - -LOCATOR = re.compile(r"\s*@\[[^\]]*\]") - -def rename_line(line, mp): - line = LOCATOR.sub("", line) - return TOKEN.sub(lambda m: mp.get(m.group(0), m.group(0)), line) - -# Pass 2: rewrite. -mod = None -mp = {} -with open(src, "r", errors="strict") as fh, open(dst, "w") as out: - for line in fh: - m = MODULE.match(line) - if m: - mod = m.group(1) - mp = maps.get(mod, {}) - out.write(LOCATOR.sub("", line)) - continue - out.write(rename_line(line, mp)) - -total = sum(len(m) for m in maps.values()) -print(f"shorten_fir: renamed {total} identifiers across {len(maps)} modules") diff --git a/hw/syn/firesim/fame5/splice.py b/hw/syn/firesim/fame5/splice.py deleted file mode 100644 index b865d7d9f8..0000000000 --- a/hw/syn/firesim/fame5/splice.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -"""Replace placeholder extmodules in a Chisel-emitted .fir with FIRRTL modules. - -usage: splice.py ... - -Each is a Yosys circuit whose main module is M; all its modules -are inserted. Chisel -emits one extmodule per BlackBox instance (M, M_1, M_2, ...), all with -`defname = M`. Every one of them is removed, the first is replaced by M's -body, and every `inst ... of ` is pointed at M, so that the -wrappers around them become identical and dedup to a single module. -""" -import re -import sys - - -def module_of(path): - """(main module name, every module of the file) -- the main one replaces the placeholders.""" - lines = open(path).read().splitlines() - name = next(re.match(r"circuit (\S+?)\s*:", l).group(1) for l in lines if l.startswith("circuit ")) - start = next(i for i, l in enumerate(lines) if re.match(r" (ext)?module \S+\s*:", l)) - return name, lines[start:] - - -def stanzas(circuit): - """Split into (header, [(kind, name, lines)]).""" - head, out, cur = [], [], None - for l in circuit: - m = re.match(r" (extmodule|module) (\S+?)\s*:", l) - if m: - cur = (m.group(1), m.group(2), [l]) - out.append(cur) - elif cur is None: - head.append(l) - else: - cur[2].append(l) - return head, out - - -def splice(circuit, name, body): - head, mods = stanzas(circuit) - placeholders = { - n for k, n, ls in mods - if k == "extmodule" and any(re.match(rf"\s+defname = {re.escape(name)}\s*$", l) for l in ls) - } - if not placeholders: - sys.exit(f"splice: no extmodule with defname {name}") - out, placed = list(head), False - for k, n, ls in mods: - if n in placeholders: - if not placed: - out += body + [""] - placed = True - continue - for l in ls: - m = re.match(r"(\s+inst \S+ of )(\S+)(.*)$", l) - if m and m.group(2) in placeholders: - l = m.group(1) + name + m.group(3) - out.append(l) - return out, len(placeholders) - - -circuit = open(sys.argv[1]).read().splitlines() -for m in sys.argv[2:]: - name, body = module_of(m) - circuit, n = splice(circuit, name, body) - print(f"splice: {name} <- {m} ({n} placeholders)") -open(sys.argv[1], "w").write("\n".join(circuit) + "\n") diff --git a/hw/syn/firesim/fame5/to_firrtl.sh b/hw/syn/firesim/fame5/to_firrtl.sh deleted file mode 100755 index 64e5a7d48c..0000000000 --- a/hw/syn/firesim/fame5/to_firrtl.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# usage: yosys2firrtl.sh [] [] -# Verilog (sv2v output) -> FIRRTL, flattened except for . -# -# Only the boundary of the unit to be threaded has to survive: FAME-5 rewrites -# that unit's insides regardless, and everything around it is one model. The -# kept module must be instantiated without parameter overrides: Yosys's FIRRTL -# writer cannot name an instance of a parameterized ($paramod) module, and -# identical instances are what threading needs in the first place. -set -euo pipefail -IN=$1 TOP=$2 OUT=$3 KEEP=$4 NEWTOP=${5:-$2} PRE=${6:-} -Y=${YOSYS:-yosys} -# Intermediate files live beside the output (kept: the Yosys log and the -# unrepaired FIRRTL are what one reads when a later step rejects the result). -W=$(dirname "$(realpath -m "$OUT")")/$(basename "$OUT" .fir).work -rm -rf "$W" && mkdir -p "$W" -REN="" -[ "$NEWTOP" != "$TOP" ] && REN="rename $TOP $NEWTOP" -SPLITKEPT="splitnets -ports -driver A:keep_hierarchy; opt_clean -purge" -[ "${FAME5_SPLIT_KEPT_PORTS:-1}" = 0 ] && SPLITKEPT="" -# Kept modules' ports are split at driver boundaries, as the top's are: a unit -# that computes some bits of a wide output from other bits of it (an LSU -# request built from its own address field) would otherwise read its own -# output port, which FIRRTL's word-level loop check rejects. -$Y -q -l "$W/y.log" -p " - read_verilog -sv $IN - $PRE - hierarchy -check -top $TOP - setattr -set hdlname \"blackbox\" t:\$paramod* - setattr -mod -set keep_hierarchy 1 $KEEP - proc -norom; flatten; opt -nosdff -nodffe - memory -nomap; memory_nordff; opt -full; opt -nosdff -nodffe - dffunmap; opt_clean -purge - splitnets -driver; opt_clean -purge - $SPLITKEPT - splitnets $TOP; opt_clean -purge - $REN - write_rtlil $W/design.il" || { tail -5 "$W/y.log"; exit 1; } -python3 "$(dirname "$0")/name_kept.py" "$W/design.il" -# Inside kept modules (only the top is split wholesale) the same bit reuse -# shows up as a wire feeding its own bits; split just those wires. -if [ "$(python3 "$(dirname "$0")/selfref_wires.py" "$W/design.il")" != 0 ]; then - $Y -q -l "$W/y_selfref.log" -p " - read_rtlil $W/design.il - splitnets a:fame5_selfref; opt_clean -purge - write_rtlil $W/design.il" || { tail -5 "$W/y_selfref.log"; exit 1; } -fi -# FIRRTL memories carry one enable and one uniform mask per write port, so a -# handful of RTL arrays - read-modify-write DCR files, shift queues, per-field -# struct writes - collect into \$mem cells write_firrtl rejects ('Complex -# write enable'). Everything it can express stays a memory (threading turns -# those into dense deep LUTRAM); only the rejected ones are memory_map'ed to -# the FF logic every other flow infers for them anyway. The writer names each -# offender, so export simply iterates until it goes through. -YOSYS=$Y python3 "$(dirname "$0")/export_firrtl.py" "$W/design.il" "$W/raw0.fir" "$W/export.log" || { tail -5 "$W/export.log"; exit 1; } -# Flattened + bit-split identifiers are most of the file's bytes; a full-GPU -# export otherwise exceeds the FIRRTL reader's 2 GiB input limit. -python3 "$(dirname "$0")/shorten_fir.py" "$W/raw0.fir" "$W/raw.fir" -rm -f "$W/raw0.fir" -# The writer types clocks as UInt<1> and converts each use with asClock(); -# (Above: the non-threaded top is split to single bits -- Yosys reuses bits of a -# wire to drive other bits of the same wire, a loop at FIRRTL's word level.) -# every clock here is a port named clk, which Chisel sees as a Clock. -python3 "$(dirname "$0")/fix_firrtl.py" "$W/raw.fir" "$W/fixed0.fir" -python3 "$(dirname "$0")/gen_rams.py" "$W/fixed0.fir" "$W/fixed.fir" -sed -e 's/^\( input clk: \)UInt<1>/\1Clock/' -e 's/asClock(clk)/clk/g' "$W/fixed.fir" > "$OUT" -echo "yosys2firrtl: $(grep -c '^ module ' "$OUT") modules, $(grep -c ' inst ' "$OUT") instances -> $OUT" diff --git a/hw/syn/firesim/src/main/scala/VortexTarget.scala b/hw/syn/firesim/src/main/scala/VortexTarget.scala index a9a72765f6..c70edd00bc 100644 --- a/hw/syn/firesim/src/main/scala/VortexTarget.scala +++ b/hw/syn/firesim/src/main/scala/VortexTarget.scala @@ -226,11 +226,6 @@ class VortexConfig NastiParameters(dataBits = 512, addrBits = 32, idBits = 4) }) -/** VortexConfig under another name, for FAME5=1 builds: FireSim keys its generated RTL by design and configs, and the - * threaded design must not share a directory with the unthreaded one. - */ -class VortexFame5Config extends Config(new VortexConfig) - class VortexTarget(implicit val p: Parameters) extends RawModule { val clock = RationalClockBridge().io.clocks.head val reset = WireInit(false.B) diff --git a/sim/firesim/Makefile b/sim/firesim/Makefile index 2faab70733..3f473e92c0 100644 --- a/sim/firesim/Makefile +++ b/sim/firesim/Makefile @@ -9,7 +9,7 @@ PROJECT := firesim SIMIF_DIR := $(FIRESIM_PATH)/sim/midas/src/main/cc DESIGN ?= VortexTarget -TARGET_CONFIG ?= $(if $(FAME5),VortexFame5Config,VortexConfig) +TARGET_CONFIG ?= VortexConfig # Nothing given on this make's command line reaches FireSim's makes: FireSim # uses PLATFORM and TARGET for its own purposes. Each sub-make is handed what it @@ -92,7 +92,7 @@ ifneq ($(filter $(TARGET),xrtsim avedsim),) # sources the standalone metasim builds from; the generated FireSim RTL and # headers come from the elaborated target. SYN_DIR := $(ROOT_DIR)/hw/syn/firesim -EMUL_DIR := $(DESTDIR)/firesim_emul_$(FIRESIM_CARD)$(if $(FAME5),_fame5) +EMUL_DIR := $(DESTDIR)/firesim_emul_$(FIRESIM_CARD) WORK_DIR := $(EMUL_DIR)/work SIM_SRCS_LIST := $(WORK_DIR)/vortex_srcs.f SIM_SRCS_DIR := $(WORK_DIR)/vortex_srcs @@ -101,13 +101,7 @@ EMUL_NAME := firesim_emul EMUL_SRCS := $(SRC_DIR)/firesim_sim.cpp $(SRC_DIR)/firesim_emul_main.cpp -# Under FAME5 the generated RTL already contains Vortex (converted to FIRRTL and -# threaded by Golden Gate), so Verilator is given no Vortex sources. -ifdef FAME5 -EMUL_VERILATOR_FLAGS := -Wno-fatal -else EMUL_VERILATOR_FLAGS := -Wno-fatal -y $(SIM_SRCS_DIR) +libext+.sv+.v $$(tr '\n' ' ' < $(SIM_SRCS_LIST)) -endif # EMUL_DEBUG=1: a wave-tracing metasimulator. --trace Verilates the dump hooks # and +define+DEBUG arms the emul top's +waveformfile/+dump-start plusargs @@ -124,8 +118,7 @@ endif # elaboration of the card's target; both are no-ops when current. emul-sources: $(MAKE) -C $(SYN_DIR) sources elaborate FIRESIM_CARD=$(FIRESIM_CARD) FIRESIM_WORK_DIR=$(WORK_DIR) \ - XLEN=$(XLEN) CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) $(if $(FAME5),FAME5=$(FAME5)) \ - $(if $(FAME5_THREADS),FAME5_THREADS=$(FAME5_THREADS)) + XLEN=$(XLEN) CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) # Host toolchain, as for the aved platform library: the runtime that loads this # is built with it. -fPIC reaches the Verilated objects too, through -CFLAGS. diff --git a/sim/firesim/firesim_sim.cpp b/sim/firesim/firesim_sim.cpp index f84f6e2e6d..854bbae4b1 100644 --- a/sim/firesim/firesim_sim.cpp +++ b/sim/firesim/firesim_sim.cpp @@ -188,8 +188,8 @@ class vortex_simulation_t final : public simulation_t { // Only ever called after a blocking step, which has already waited for the // step to finish. A blocking peek would additionally wait up to 0.1 s for - // PRECISE_PEEKABLE, which a FAME-5 design never raises, and then return the - // same value: 0.1 s lost per peek, and dcr_read peeks once per target cycle. + // PRECISE_PEEKABLE and then return the same value: time lost per peek, and + // dcr_read peeks once per target cycle. uint32_t peek(const char *name) { return peek_poke.peek(name, false); } diff --git a/sw/runtime/firesim/Makefile b/sw/runtime/firesim/Makefile index 8eac1c64a7..d604055085 100644 --- a/sw/runtime/firesim/Makefile +++ b/sw/runtime/firesim/Makefile @@ -55,8 +55,7 @@ driver: $(DESTDIR)/libfiresim.so $(DESTDIR)/libfiresim.so: force DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) \ - CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) $(if $(FAME5),FAME5=$(FAME5)) \ - $(if $(FAME5_THREADS),FAME5_THREADS=$(FAME5_THREADS)) + CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) $(DESTDIR)/$(PROJECT): $(SRCS) $(DESTDIR)/libfiresim.so $(CONFIG_FILE) $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 86d16d324c..273140d244 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -189,7 +189,7 @@ kernel.vxbin: kernel.elf $(VORTEX_KN_PATH)/lib$(KERNEL_LIB).a: $(MAKE) -C $(VORTEX_KN_PATH) -RUNTIME_ARGS = CONFIGS="$(CONFIGS)" $(if $(DEBUG),DEBUG=$(DEBUG)) $(if $(PERF),PERF=$(PERF)) $(if $(FAME5),FAME5=$(FAME5)) $(if $(FAME5_THREADS),FAME5_THREADS=$(FAME5_THREADS)) $(if $(SCOPE),SCOPE=$(SCOPE)) $(if $(SAIF),SAIF=$(SAIF)) +RUNTIME_ARGS = CONFIGS="$(CONFIGS)" $(if $(DEBUG),DEBUG=$(DEBUG)) $(if $(PERF),PERF=$(PERF)) $(if $(SCOPE),SCOPE=$(SCOPE)) $(if $(SAIF),SAIF=$(SAIF)) # FORCE, not a bare rule: a rule with no prerequisites runs only when its target # is missing, so once this library exists make never re-enters the sub-make that From fe6f9f32112a2b3348c928961b63a431f1874a05 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 12 Oct 2026 03:06:43 -0700 Subject: [PATCH 183/336] firesim: resolve the unit types of a synthesis build as synthesis TYPE_DEFINES comes from a configuration resolved without SYNTHESIS, which selects the simulation FPU (STD) over the FPGA one (DSP). The image flow used it for synthesis too, so a FireSim image carried a different FPU from the one every native FPGA flow of the same configuration builds. Synthesis now resolves its unit types with the synthesis defines; the metasimulator keeps the simulation ones. The DSP FPU's Xilinx floating-point cores are already generated and packaged by the image flow (xilinx-ips). Not yet exercised by a FireSim image build. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e3e94aa29af0943b074d5ba10210c2f2bdb32992) --- hw/syn/firesim/Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index fd051944d0..b137e4e427 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -266,7 +266,12 @@ ifdef PERF PLATFORM_DEFINES += -DPERF_ENABLE endif -GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) $(SYNTH_DEFINES) $(RTL_INCLUDE) +# Synthesis resolves the unit types as a synthesis build: TYPE_DEFINES comes from +# a resolution without SYNTHESIS, which picks the simulation FPU (STD) over the +# FPGA one (DSP) and would hand synthesis a different design than Vortex's. +SYNTH_TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(shell python3 $(VORTEX_HOME)/ci/gen_config.py \ + --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN) $(SYNTH_DEFINES) $(PLATFORM_DEFINES)')) +GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(SYNTH_TYPE_DEFINES) $(PLATFORM_DEFINES) $(SYNTH_DEFINES) $(RTL_INCLUDE) # Defines arrive on the command line, which no file timestamp reflects, so a # changed CONFIGS would otherwise reuse the previous flattened tree. The stamp is From 1c2d22fe85872aed75cc9fde4d5e1e391732ed18 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 12 Oct 2026 05:49:24 -0700 Subject: [PATCH 184/336] maxsim: latch each target cycle's command in the control endpoint The endpoint offered its command tokens (reset, DCR valid/rw/addr/data, start) as functions of the live queue head. The simulator may take a cycle's token before the endpoint finishes the cycle; a command the host queued in between changed the bits of a token already taken, and was then popped as applied though the target never saw it. On tinygpu (1 core, 2 warps x 2 threads, no L1 caches or local memory) 16 of the 20 launch DCR writes were lost this way and the kernel never ran; on larger configurations the timing happened to hide it. The command applied in a target cycle is now taken from the queue when the previous cycle finishes, and the tokens are functions of it alone. A reset command of count 0 lasts one cycle instead of wrapping to 2^32. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit ac9a12ec183cca1c4b5e83369ceb4b8609b2c1b8) --- hw/rtl/afu/maxsim/maxsim_ctrl.sv | 41 +++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/hw/rtl/afu/maxsim/maxsim_ctrl.sv b/hw/rtl/afu/maxsim/maxsim_ctrl.sv index 21e28a52e1..491766e212 100644 --- a/hw/rtl/afu/maxsim/maxsim_ctrl.sv +++ b/hw/rtl/afu/maxsim/maxsim_ctrl.sv @@ -19,7 +19,7 @@ // The target runs free: the endpoint gives it a token every cycle. The host // queues commands -- a DCR write or read, a start, a reset -- and each is // applied to the target for one cycle (a reset for its count of cycles), in -// order, at the first cycle the queue reaches it. A DCR read's response is +// order, from the cycle after the queue reaches it. A DCR read's response is // kept for the host; `busy` covers everything queued or in flight, so a host // that sees the target idle has every effect of what it asked for. // @@ -86,6 +86,13 @@ module maxsim_ctrl #( reg [QC-1:0] q_count; reg [DCR_ADDR_W-1:0] cmd_addr; reg [DCR_DATA_W-1:0] cmd_data; + // The command applied this target cycle, taken from the queue when the + // previous cycle finished: the target's control tokens are functions of it + // alone, so they do not change between the target taking one and the + // endpoint finishing the cycle. + reg [2:0] cur_op; + reg [DCR_ADDR_W-1:0] cur_addr; + reg [DCR_DATA_W-1:0] cur_data; reg run; reg [31:0] reset_count; @@ -98,13 +105,17 @@ module maxsim_ctrl #( reg [47:0] finishes [NUM_CLASSES]; wire q_empty = (q_count == 0); - wire [2:0] op = q_empty ? `MAXSIM_CMD_NONE : q_op[q_head]; // ---- The target's control inputs this cycle ---- - wire in_reset = (reset_count != 0) | (op == `MAXSIM_CMD_RESET); - wire dcr_valid = ~in_reset & ((op == `MAXSIM_CMD_DCR_WRITE) | (op == `MAXSIM_CMD_DCR_READ)); - wire dcr_rw = (op == `MAXSIM_CMD_DCR_WRITE); - wire do_start = ~in_reset & (op == `MAXSIM_CMD_START); + wire in_reset = (reset_count != 0) | (cur_op == `MAXSIM_CMD_RESET); + wire dcr_valid = ~in_reset & ((cur_op == `MAXSIM_CMD_DCR_WRITE) | (cur_op == `MAXSIM_CMD_DCR_READ)); + wire dcr_rw = (cur_op == `MAXSIM_CMD_DCR_WRITE); + wire do_start = ~in_reset & (cur_op == `MAXSIM_CMD_START); + // Reset cycles left after this one; a reset command lasts its count of + // cycles, at least one. + wire [31:0] reset_next = (reset_count != 0) ? reset_count - 32'd1 + : ((cur_op == `MAXSIM_CMD_RESET) && (cur_data > 1)) ? 32'(cur_data) - 32'd1 + : 32'd0; wire finish; wire [5:0] done; @@ -118,8 +129,8 @@ module maxsim_ctrl #( assign reset_bits = in_reset; assign dcr_req_valid_bits = dcr_valid; assign dcr_req_rw_bits = dcr_rw; - assign dcr_req_addr_bits = q_addr[q_head]; - assign dcr_req_data_bits = q_data[q_head]; + assign dcr_req_addr_bits = cur_addr; + assign dcr_req_data_bits = cur_data; assign start_bits = do_start; wire rsp_v, busy; @@ -141,9 +152,10 @@ module maxsim_ctrl #( wire wr = aw_got & w_got & ~s_bvalid; wire [11:0] wa = {aw_addr[11:2], 2'b00}; wire push = wr & (wa == `MAXSIM_REG_CMD) & (q_count != QC'(QUEUE)); - wire pop = finish & ~q_empty & (reset_count == 0); + wire pop = finish & ~q_empty & (reset_next == 0); - wire status_busy = ~q_empty | start_pending | target_busy | ~mem_idle | (reset_count != 0); + wire status_busy = ~q_empty | (cur_op != `MAXSIM_CMD_NONE) | start_pending | target_busy + | ~mem_idle | (reset_count != 0); always @(posedge clk) begin if (reset) begin @@ -152,6 +164,7 @@ module maxsim_ctrl #( q_count <= 0; run <= 1'b1; reset_count <= RESET_CYCLES; + cur_op <= `MAXSIM_CMD_NONE; start_pending <= 1'b0; target_busy <= 1'b0; rsp_valid <= 1'b0; @@ -181,10 +194,10 @@ module maxsim_ctrl #( // One target cycle of the endpoint. if (finish) begin target_cycles <= target_cycles + 64'd1; - if (reset_count != 0) - reset_count <= reset_count - 32'd1; - else if (op == `MAXSIM_CMD_RESET) - reset_count <= q_data[q_head] - 32'd1; + reset_count <= reset_next; + cur_op <= pop ? q_op[q_head] : `MAXSIM_CMD_NONE; + cur_addr <= q_addr[q_head]; + cur_data <= q_data[q_head]; if (rsp_v) begin rsp_valid <= 1'b1; rsp_data <= rsp_d; From ae70cfc0a670519cc387da67f692abbf5a50060a Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 12 Oct 2026 05:49:24 -0700 Subject: [PATCH 185/336] maxsim: in-process kernel reads memory when it takes a request; traces The in-process kernel's memory answered a read with the data memory held when the response went out, so a write issued after the read but before its latency had passed showed up in it. It now captures the data when it takes the request, as a memory does. Debug aids for the kernel: MAXSIM_EMUL_AXI_TRACE (the memory port's requests and responses), MAXSIM_EMUL_REG_TRACE (the host's register accesses), and, in a VCD=1 build, MAXSIM_EMUL_VCD with MAXSIM_EMUL_VCD_FROM and MAXSIM_EMUL_VCD_DEPTH. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 1fbb27a0721916646c084451c5de493ccfc1aaae) --- sim/maxsim/Makefile | 3 ++ sim/maxsim/maxsim_emul.cpp | 75 ++++++++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/sim/maxsim/Makefile b/sim/maxsim/Makefile index 234702954f..2f66c89b00 100644 --- a/sim/maxsim/Makefile +++ b/sim/maxsim/Makefile @@ -18,6 +18,9 @@ LDFLAGS += -shared -pthread VL_FLAGS += -O3 -Wno-fatal -Wno-lint -Wno-style -Wno-MULTIDRIVEN --x-initial 0 --x-assign 0 VL_FLAGS += -I$(MAXSIM_RTL_DIR) -I$(WORK_DIR)/gen +ifdef VCD +VL_FLAGS += --trace --trace-structs +endif THREADS ?= $(shell python3 -c 'import multiprocessing as mp; print(min(32, mp.cpu_count()))') VL_FLAGS += -j $(THREADS) diff --git a/sim/maxsim/maxsim_emul.cpp b/sim/maxsim/maxsim_emul.cpp index 56adc06747..8280fa2182 100644 --- a/sim/maxsim/maxsim_emul.cpp +++ b/sim/maxsim/maxsim_emul.cpp @@ -16,6 +16,9 @@ #include "Vmaxsim_afu.h" #include "Vmaxsim_afu___024root.h" #include "verilated.h" +#ifdef VM_TRACE +#include "verilated_vcd_c.h" +#endif #include #include @@ -73,10 +76,18 @@ class maxsim_emul::Impl { mmio_cycles_(env_or("MAXSIM_EMUL_MMIO_CYCLES", 250)) {} ~Impl() { +#ifdef VM_TRACE + if (vcd_) + vcd_->close(); +#endif if (top_) top_->final(); if (trace_ != nullptr) fclose(trace_); + if (axi_trace_ != nullptr) + fclose(axi_trace_); + if (reg_trace_ != nullptr) + fclose(reg_trace_); } int init() { @@ -84,8 +95,27 @@ class maxsim_emul::Impl { // cycle, one byte per cycle of up to 8 classes. if (const char* path = getenv("MAXSIM_EMUL_TRACE")) trace_ = fopen(path, "wb"); + // MAXSIM_EMUL_AXI_TRACE=: the kernel's memory port, one line per + // request and response. + if (const char* path = getenv("MAXSIM_EMUL_AXI_TRACE")) + axi_trace_ = fopen(path, "w"); + // MAXSIM_EMUL_REG_TRACE=: every register access of the host. + if (const char* path = getenv("MAXSIM_EMUL_REG_TRACE")) + reg_trace_ = fopen(path, "w"); context_ = std::make_unique(); top_ = std::make_unique(context_.get()); +#ifdef VM_TRACE + // MAXSIM_EMUL_VCD=: a waveform of the kernel, in a build with --trace, + // from kernel cycle MAXSIM_EMUL_VCD_FROM on, MAXSIM_EMUL_VCD_DEPTH levels + // of hierarchy deep (2: the kernel's channels). + vcd_from_ = env_or("MAXSIM_EMUL_VCD_FROM", 0); + if (const char* path = getenv("MAXSIM_EMUL_VCD")) { + context_->traceEverOn(true); + vcd_ = std::make_unique(); + top_->trace(vcd_.get(), int(env_or("MAXSIM_EMUL_VCD_DEPTH", 99))); + vcd_->open(path); + } +#endif top_->ap_rst_n = 0; idle_master(); for (int i = 0; i < 16; ++i) @@ -118,6 +148,8 @@ class maxsim_emul::Impl { top_->s_axi_control_wvalid = !w; } top_->s_axi_control_bready = 0; + if (reg_trace_ != nullptr) + fprintf(reg_trace_, "%lu W %03x %08x\n", (unsigned long)cycle_, offset, value); return b ? 0 : -1; } @@ -140,6 +172,8 @@ class maxsim_emul::Impl { top_->s_axi_control_arvalid = 0; } top_->s_axi_control_rready = 0; + if (reg_trace_ != nullptr) + fprintf(reg_trace_, "%lu R %03x %08x\n", (unsigned long)cycle_, offset, *value); return r ? 0 : -1; } @@ -156,9 +190,11 @@ class maxsim_emul::Impl { } private: + // A read, with the data memory held when it took the request: a write + // that comes after it is not in its response. struct request_t { - uint64_t addr; uint64_t at; + std::array data; }; void idle_master() { @@ -188,11 +224,8 @@ class maxsim_emul::Impl { top_->m_axi_mem_0_arready = 1; bool r = !reads_.empty() && reads_.front().at <= cycle_; top_->m_axi_mem_0_rvalid = r; - if (r) { - uint8_t beat[kBeat]; - memory_.read(reads_.front().addr, beat, kBeat); - std::memcpy(top_->m_axi_mem_0_rdata.data(), beat, kBeat); - } + if (r) + std::memcpy(top_->m_axi_mem_0_rdata.data(), reads_.front().data.data(), kBeat); top_->m_axi_mem_0_rlast = 1; top_->m_axi_mem_0_rid = 0; top_->m_axi_mem_0_rresp = 0; @@ -225,10 +258,30 @@ class maxsim_emul::Impl { fputc(top_->rootp->maxsim_afu__DOT__profile_finish & 0xff, trace_); top_->ap_clk = 1; top_->eval(); +#ifdef VM_TRACE + if (vcd_ && cycle_ >= vcd_from_) + vcd_->dump(cycle_); +#endif ++cycle_; - if (ar) - reads_.push_back({araddr, cycle_ + mem_latency_}); + if (axi_trace_ != nullptr) { + if (ar) + fprintf(axi_trace_, "%lu AR %lx\n", (unsigned long)cycle_, (unsigned long)araddr); + if (aw) + fprintf(axi_trace_, "%lu AW %lx\n", (unsigned long)cycle_, (unsigned long)awaddr); + if (w) + fprintf(axi_trace_, "%lu W\n", (unsigned long)cycle_); + if (r_fire) + fprintf(axi_trace_, "%lu R\n", (unsigned long)cycle_); + if (b_fire) + fprintf(axi_trace_, "%lu B\n", (unsigned long)cycle_); + } + if (ar) { + request_t req; + req.at = cycle_ + mem_latency_; + memory_.read(araddr, req.data.data(), kBeat); + reads_.push_back(req); + } if (r_fire) reads_.pop_front(); if (aw) @@ -251,11 +304,17 @@ class maxsim_emul::Impl { std::unique_ptr context_; std::unique_ptr top_; +#ifdef VM_TRACE + std::unique_ptr vcd_; + uint64_t vcd_from_ = 0; +#endif std::mutex mutex_; memory_t memory_; uint64_t cycle_ = 0; uint64_t mem_latency_, mmio_cycles_; FILE* trace_ = nullptr; + FILE* axi_trace_ = nullptr; + FILE* reg_trace_ = nullptr; std::deque reads_; std::deque aw_; std::deque> w_; From 9c5125406926561d0ddd0cbb6e2d0032ffa7d57c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 12 Oct 2026 08:29:25 -0700 Subject: [PATCH 186/336] hw/syn: CIRCT replaces sv2v and Verilator's preprocessor Every synthesis flow now takes its SystemVerilog through CIRCT: - gen_sources.sh -P preprocesses with circt-verilog -E instead of verilator -E. Over the whole RTL tree (full-GPU defines) the two produce the same code; Verilator only kept lint-pragma comments. The tinygpu maxsim simulator generated from it matches the old reference in token co-simulation (0 errors). - The Yosys flow elaborates with CIRCT into one Verilog-2005 file (hw/scripts/sv_to_v.sh, replacing sv2v.sh): packed arrays and structs become vectors, and the SRAM macros (BB_MODULES) stay instances with their parameter values; yosys reads them beside their stubs in SystemVerilog mode. - CIRCT installs under $TOOLDIR/circt (toolchain_install.sh --circt, in place of --sv2v); hw/syn/common.mk exports CIRCT_VERILOG, CIRCT_OPT and FIRTOOL, and maxsim takes its CIRCT from there. The asic_gate fingerprints its baselines on the CIRCT version. Cache DUT through Yosys/ABC on ASAP7: SRAM area identical (16,307 um^2), logic area 56,934 vs 53,936 um^2 (+5.6%): CIRCT folds synchronous resets into the next value and lowers array reads to mux trees. Needs the CIRCT prebuilt (toolchain_prebuilt.sh --circt) before CI can install it. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5b4020c2bb1f09146281ad7d26b63f1f5a89146b) --- CHANGELOG.md | 1 + README.md | 2 +- ci/asic_gate.py | 2 +- ci/synth_gate.py | 27 ++++--- ci/toolchain_install.sh.in | 23 ++++-- ci/toolchain_prebuilt.sh.in | 18 +++-- docs/building_toolchain.md | 2 +- docs/designs/build_configuration_system.md | 8 +- docs/designs/continuous_integration.md | 4 +- docs/synthesis_analysis.md | 2 +- hw/scripts/gen_sources.sh | 22 +++--- hw/scripts/sv2v.sh | 80 ------------------- hw/scripts/sv_to_v.sh | 92 ++++++++++++++++++++++ hw/syn/common.mk | 12 ++- hw/syn/maxsim/Makefile | 4 +- hw/syn/xilinx/aved/Makefile | 2 +- hw/syn/xilinx/dut/common.mk | 6 +- hw/syn/xilinx/xrt/Makefile | 6 +- hw/syn/yosys/Makefile | 14 +++- hw/syn/yosys/run_synth.sh | 6 +- 20 files changed, 188 insertions(+), 145 deletions(-) delete mode 100755 hw/scripts/sv2v.sh create mode 100755 hw/scripts/sv_to_v.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 8950a3d61f..c81cc47c33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,7 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, value exactly. - **FPGA run instructions.** The docs exported `TARGET=hw`, which `blackbox.sh` ignores, so runs fell back to xrtsim; they now pass `--target=hw`. +- **CIRCT is the SystemVerilog frontend of every synthesis flow; sv2v and Verilator's preprocessor are gone.** `gen_sources.sh -P` preprocesses with `circt-verilog -E` (identical code to `verilator -E` over the whole RTL tree; only comments differ), and the Yosys flow elaborates with CIRCT into one Verilog-2005 file ([hw/scripts/sv_to_v.sh](hw/scripts/sv_to_v.sh)) instead of sv2v: packed arrays and structs become vectors, and the SRAM macros (`BB_MODULES`) stay instances with their parameter values, read in SystemVerilog mode beside their stubs. CIRCT installs under `$TOOLDIR/circt` (`toolchain_install.sh --circt`; `hw/syn/common.mk` exports `CIRCT_VERILOG`, `CIRCT_OPT`, `FIRTOOL`), and the `asic_gate` fingerprints its baselines on the CIRCT version. Verilator remains the simulator (and `--xml-only` scope metadata). On the cache DUT (ASAP7), SRAM area is identical and logic area is 5.6% higher than through sv2v: CIRCT folds synchronous resets into the next-value logic and lowers array reads to mux trees, both open items. ## [3.0] — 2026-10-07 diff --git a/README.md b/README.md index 39658e3c24..ddd3f6baa0 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ The following dependencies will be fetched prebuilt by `toolchain_install.sh`. - [SoftFloat](https://github.com/ucb-bar/berkeley-softfloat-3.git) - [Ramulator](https://github.com/CMU-SAFARI/ramulator.git) - [Yosys](https://github.com/YosysHQ/yosys) -- [Sv2v](https://github.com/zachjs/sv2v) +- [CIRCT](https://github.com/llvm/circt) - [Mesa](https://www.mesa3d.org/) - [chipStar](https://github.com/CHIP-Star/chipStar) ### Install Vortex codebase diff --git a/ci/asic_gate.py b/ci/asic_gate.py index 1a5dd579a2..84b11ec4c6 100755 --- a/ci/asic_gate.py +++ b/ci/asic_gate.py @@ -13,7 +13,7 @@ from the reports/ set the flow already produces Unlike the FPGA gate this one runs on a hosted runner as a normal pytest cell -(`check: asic_gate`, tier `full`), because yosys/sv2v/OpenSTA ship in the +(`check: asic_gate`, tier `full`), because yosys/CIRCT/OpenSTA ship in the prebuilt toolchain and ASAP7 is a 59 MB fetch. Usage (from a source checkout): diff --git a/ci/synth_gate.py b/ci/synth_gate.py index 36913c2f91..2750d27690 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -333,15 +333,15 @@ class Yosys(Tool): fallback_weight = 900 job_label = "make" - # sv2v is the parser: a bad define, a missing include or a hierarchy error - # dies there, before Yosys ever reads a line. run_synth.sh stamps `gen-ys` - # immediately after source generation and conversion have both succeeded, so - # that stamp is this flow's "past the typo risk" marker. + # CIRCT is the parser: a bad define, a missing include or a hierarchy error + # dies in sv_to_v.sh, before Yosys ever reads a line. run_synth.sh stamps + # `gen-ys` immediately after source generation and conversion have both + # succeeded, so that stamp is this flow's "past the typo risk" marker. elab_done = "TIME gen-ys" elab_label = "source conversion" phases = ( ("gen_sources.sh", "sources"), - ("sv2v.sh", "sv2v"), + ("sv_to_v.sh", "verilog"), (elab_done, "synthesis"), ("TIME yosys", "sram"), ("run_sta.tcl", "timing"), @@ -355,7 +355,8 @@ def env(self, args): "corner": os.environ.get("CORNER", "tt"), "yosys": tool_version("yosys", "-V"), "sta": tool_version("sta", "-version"), - "sv2v": tool_version("sv2v", "--version"), + "circt": tool_version("circt-verilog", "--version", + package="circt", line_prefix="CIRCT"), "xlen": args.xlen} def hash_key(self, env): @@ -370,13 +371,13 @@ def hash_key(self, env): # tool change. All three ship in one tarball pinned by TOOLCHAIN_REV, so # this makes that pin part of the baseline contract. return [env["pdk"], env["vt"], env["corner"], str(env["xlen"]), - env["yosys"], env["sta"], env["sv2v"]] + env["yosys"], env["sta"], env["circt"]] def env_warning(self, build, env): # Area and Fmax move with the Yosys/ABC version far more than Vivado's # do, so a version change is worth saying out loud on every build. for key, label in (("yosys", "Yosys"), ("sta", "OpenSTA"), - ("sv2v", "sv2v")): + ("circt", "CIRCT")): ref = build.get("baseline_env", {}).get(key) if ref and env.get(key) and ref != env[key]: # The version is in config_hash, so this build is already going @@ -441,7 +442,7 @@ def describe_baseline(self, base): TOOLS = {t.name: t for t in (Xilinx(), Yosys())} -def tool_version(name, flag): +def tool_version(name, flag, package=None, line_prefix=None): """First line of ` `, or "" if it cannot be asked. Resolved out of $TOOLDIR the way hw/syn/common.mk does, with a PATH fallback: @@ -449,7 +450,7 @@ def tool_version(name, flag): copy of yosys would make the baseline's tool env a lie. """ tooldir = os.environ.get("TOOLDIR") or os.path.expanduser("~/tools") - binary = os.path.join(tooldir, name, "bin", name) + binary = os.path.join(tooldir, package or name, "bin", name) if not os.path.exists(binary): binary = name try: @@ -460,6 +461,12 @@ def tool_version(name, flag): text = out.stdout.decode(errors="replace").strip() if not text: return "" + # A tool that prints several components' versions names the one that + # matters on a line of its own (circt-verilog: "CIRCT "). + if line_prefix: + lines = [l.strip() for l in text.splitlines() + if l.strip().startswith(line_prefix)] + return lines[0] if lines else "" # Up to the build details: yosys reports its git sha AND the g++ that built # it ("Yosys 0.57+157 (git sha1 ..., g++ 11.4 ...)"). Those identify the # binary, not the tool, and this string is part of the baseline fingerprint diff --git a/ci/toolchain_install.sh.in b/ci/toolchain_install.sh.in index ac6f49e943..a9c58dbbc8 100755 --- a/ci/toolchain_install.sh.in +++ b/ci/toolchain_install.sh.in @@ -163,12 +163,19 @@ verilator() mkdir -p $TOOLDIR && rm -rf $TOOLDIR/verilator && mv verilator $TOOLDIR rm -rf verilator verilator.tar.bz2* } -sv2v() +# CIRCT: the SystemVerilog frontend of the synthesis flows (circt-verilog, +# circt-opt, firtool), packaged in parts as verilator is. +circt() { - wget $REPOSITORY/sv2v/$OSVERSION/sv2v.tar.bz2 - tar -xvf sv2v.tar.bz2 - mkdir -p $TOOLDIR && rm -rf $TOOLDIR/sv2v && mv sv2v $TOOLDIR - rm -rf sv2v sv2v.tar.bz2 + parts=$(eval echo {a..z}) + for x in $parts + do + wget $REPOSITORY/circt/$OSVERSION/circt.tar.bz2.parta$x || break + done + cat circt.tar.bz2.parta* > circt.tar.bz2 + tar -xvf circt.tar.bz2 + mkdir -p $TOOLDIR && rm -rf $TOOLDIR/circt && mv circt $TOOLDIR + rm -rf circt circt.tar.bz2* } yosys() @@ -232,7 +239,7 @@ install_base() libc64 riscv32 riscv64 - sv2v + circt yosys sta mesa @@ -244,7 +251,7 @@ install_base() show_usage() { echo "Install Pre-built Vortex Toolchain" - echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--sv2v] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" + echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" echo " (no arguments: install all components except gem5)" echo " gem5 is opt-in: it is a large source build." } @@ -280,7 +287,7 @@ while [ "$1" != "" ]; do ;; --libc64 ) libc64 ;; - --sv2v ) sv2v + --circt ) circt ;; --yosys ) yosys ;; diff --git a/ci/toolchain_prebuilt.sh.in b/ci/toolchain_prebuilt.sh.in index a720eae002..3e7c6bb65c 100755 --- a/ci/toolchain_prebuilt.sh.in +++ b/ci/toolchain_prebuilt.sh.in @@ -124,12 +124,14 @@ verilator() rm verilator.tar.bz2 } -sv2v() +circt() { - echo "prebuilt sv2v..." - tar -C $TOOLDIR -cvjf sv2v.tar.bz2 sv2v - mkdir -p ./sv2v/$OSVERSION - mv sv2v.tar.bz2 ./sv2v/$OSVERSION + echo "prebuilt circt..." + tar -C $TOOLDIR -cvjf circt.tar.bz2 circt/bin/circt-verilog circt/bin/circt-opt circt/bin/firtool + split -b 50M circt.tar.bz2 "circt.tar.bz2.part" + mkdir -p ./circt/$OSVERSION + mv circt.tar.bz2.part* ./circt/$OSVERSION + rm circt.tar.bz2 } yosys() @@ -172,7 +174,7 @@ gem5() show_usage() { echo "Setup Pre-built Vortex Toolchain" - echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--sv2v] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" + echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" } while [ "$1" != "" ]; do @@ -199,7 +201,7 @@ while [ "$1" != "" ]; do ;; --libc64 ) libc64 ;; - --sv2v ) sv2v + --circt ) circt ;; --yosys ) yosys ;; @@ -217,7 +219,7 @@ while [ "$1" != "" ]; do libcrt64 libc32 libc64 - sv2v + circt yosys sta mesa diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index f699645215..c5177b458b 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -1171,7 +1171,7 @@ Both accept the same per-component flags: ``` --pocl --chipstar --verilator --riscv32 --riscv64 --llvm --mesa ---libcrt32 --libcrt64 --libc32 --libc64 --sv2v --yosys --sta +--libcrt32 --libcrt64 --libc32 --libc64 --circt --yosys --sta --gem5 (slim runtime — see §10) --all (every component above; SST is NOT included — see §11) ``` diff --git a/docs/designs/build_configuration_system.md b/docs/designs/build_configuration_system.md index ccc4a9a3bb..79c9523bf5 100644 --- a/docs/designs/build_configuration_system.md +++ b/docs/designs/build_configuration_system.md @@ -322,14 +322,14 @@ mix of knobs and computed results. RTL reads `` `VX_CFG_* `` and `` `VX_MEM_* `` macros directly; there is no typed-`localparam` configuration package. One was built and reverted: -- **sv2v drops wildcard re-exports.** The Yosys ASIC flow converts through - sv2v, which discards `export pkg::*` symbols — the flow failed on - `L3_CACHE_SIZE` and its siblings. +- **sv2v drops wildcard re-exports.** The Yosys ASIC flow converted through + sv2v at the time, which discards `export pkg::*` symbols — the flow failed + on `L3_CACHE_SIZE` and its siblings. (The flow now elaborates with CIRCT.) - **Package compile order is tool-divergent.** Verilator is strict, Vivado auto-sorts, VCS is order-sensitive. Backtick macros are the one representation every tool handles identically. -Do not re-attempt a package without first solving sv2v re-export. +Do not re-attempt a package without first checking re-exports in every flow. ### 9.2 Superseded directions diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index 7283e97d0b..e8c5591ea8 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -271,7 +271,7 @@ declares, recording every run's metrics in golden baselines. | | `fpga_gate` | `asic_gate` | |---|---|---| -| flow | Vivado synthesis, place and route | sv2v → Yosys → ABC → OpenSTA, on ASAP7 | +| flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | | design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | | gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | | also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | @@ -356,7 +356,7 @@ so the flow reports the signed worst slack instead. |---|---| | scheduling | longest build first, from the recorded build times, so the longest is in flight from the start; `-j` caps parallel builds and each build's tool threads are derived from it | | isolation | every build gets a unique `PREFIX`, hence its own tree and log. On the Yosys flow this is load-bearing: its Makefile caches generated sources and does not regenerate them when the include set changes, so two designs sharing a tree would synthesize the first one's sources | -| early-failure watch | the runner follows each log and announces the point past which a front-end error can no longer occur — the end of RTL elaboration for Vivado, the end of source generation and sv2v conversion for Yosys. A build that fails before it is reported as failed before synthesis, with its error lines quoted | +| early-failure watch | the runner follows each log and announces the point past which a front-end error can no longer occur — the end of RTL elaboration for Vivado, the end of source generation and CIRCT elaboration for Yosys. A build that fails before it is reported as failed before synthesis, with its error lines quoted | | resumable sessions | each build directory carries a stamp of config hash, status and metrics. `--resume` reuses a build finished for this config, lets an unfinished one continue from its checkpoint, and rebuilds one whose config changed | | progress | phase transitions as they land, with a heartbeat between; `-v` streams the raw log | diff --git a/docs/synthesis_analysis.md b/docs/synthesis_analysis.md index 607da4efe4..d497e0750f 100644 --- a/docs/synthesis_analysis.md +++ b/docs/synthesis_analysis.md @@ -434,7 +434,7 @@ Output reports are written to `build/hw/syn/yosys/tfr_fedp_VX_tcu_fedp_tfr/repor Build directory: `_/` (e.g., `test_Vortex/`). -The flow uses `sv2v` to convert SystemVerilog sources to Verilog before feeding them to Yosys. +The flow elaborates the SystemVerilog sources with CIRCT ([hw/scripts/sv_to_v.sh](../hw/scripts/sv_to_v.sh)) into one Verilog-2005 file for Yosys: packed arrays and structs become plain vectors, and the SRAM macros (`BB_MODULES`) stay instances with their parameter values, bound to their stubs. #### Per-DUT synthesis diff --git a/hw/scripts/gen_sources.sh b/hw/scripts/gen_sources.sh index 26bfcda304..a8514246cb 100755 --- a/hw/scripts/gen_sources.sh +++ b/hw/scripts/gen_sources.sh @@ -19,8 +19,9 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # every stage of a pipeline. set -eo pipefail -# Resolve verilator path at runtime, allowing it to be overridden by environment variable. -VERILATOR="${VERILATOR:-verilator}" +# The preprocessor is CIRCT's SystemVerilog frontend; hw/syn/common.mk exports +# its absolute path. +CIRCT_VERILOG="${CIRCT_VERILOG:-circt-verilog}" defines=() includes=() @@ -121,17 +122,16 @@ copy_one_file() { "$SCRIPT_DIR"/repl_params.py $params_str -T"$top_module" "$file" > "$temp_file" input="$temp_file" fi - # Trust verilator's exit status, not the output size: a missing tool, - # unresolved include, or preprocessor error returns non-zero and is caught - # here (this is the failure that otherwise resurfaced far downstream as - # sv2v "Could not find top module"). An *empty but successful* output is - # legitimate — config-gated modules (e.g. VX_decompressor.sv when RVC is + # Trust the preprocessor's exit status, not the output size: a missing + # tool, unresolved include, or preprocessor error returns non-zero and is + # caught here rather than far downstream. An *empty but successful* output + # is legitimate — config-gated modules (e.g. VX_decompressor.sv when RVC is # disabled) preprocess to nothing — so emptiness alone must not fail. - if ! "$VERILATOR" $defines_str $includes_str -E -P "$input" > "$dest"; then + if ! "$CIRCT_VERILOG" -E $defines_str $includes_str "$input" -o "$dest"; then if [ -n "$temp_file" ]; then rm -f "$temp_file"; fi - echo "gen_sources.sh: error: verilator failed to preprocess '$file'" >&2 - echo " using VERILATOR='$VERILATOR' — ensure it is reachable" \ - "(set VERILATOR / VERILATOR_PATH, see hw/syn/common.mk) and that" \ + echo "gen_sources.sh: error: circt-verilog failed to preprocess '$file'" >&2 + echo " using CIRCT_VERILOG='$CIRCT_VERILOG' — ensure it is reachable" \ + "(set CIRCT_VERILOG / CIRCT_PATH, see hw/syn/common.mk) and that" \ "the include paths and defines resolve." >&2 exit 1 fi diff --git a/hw/scripts/sv2v.sh b/hw/scripts/sv2v.sh deleted file mode 100755 index 3f2f928b8f..0000000000 --- a/hw/scripts/sv2v.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Copyright © 2019-2023 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this script uses sv2v and yosys tools to run. -# sv2v: https://github.com/zachjs/sv2v -# yosys: http://www.clifford.at/yosys/ - -# Tool binaries: prefer caller-supplied env vars (set by hw/syn/common.mk -# to absolute $(TOOLDIR)/{sv2v,yosys}/bin/...) so the build is self- -# contained. Fall back to PATH lookup for callers that don't set them. -SV2V="${SV2V:-sv2v}" -YOSYS="${YOSYS:-yosys}" - -# exit when any command fails -set -e - -source="" -includes=() -macro_args="" -output_file=out.v - -usage() { echo "$0 usage:" && grep " .)\ #" $0; exit 0; } -[ $# -eq 0 ] && usage -while getopts "t:s:o:I:D:h" arg; do - case $arg in - t) # source - top=--top=${OPTARG} - ;; - s) # source - source=${OPTARG} - ;; - o) # output-file - output_file=${OPTARG} - ;; - I) # include directory - includes+=(${OPTARG}) - ;; - D) # macro definition - macro_args="$macro_args -D${OPTARG}" - ;; - h | *) - usage - exit 0 - ;; - esac -done - -# process include paths -inc_args="" -for dir in "${includes[@]}" -do - inc_args="$inc_args -I$dir" -done - -# process source files -file_args=$source -for dir in "${includes[@]}" -do - for file in $(find $dir -maxdepth 1 -name '*.v' -o -name '*.sv' -type f) - do - echo "file: $file" - file_args="$file_args $file" - done -done - -# system-verilog to verilog conversion -echo "$SV2V $top $macro_args $inc_args $file_args -v -w $output_file" -"$SV2V" $top $macro_args $inc_args $file_args -v -w $output_file \ No newline at end of file diff --git a/hw/scripts/sv_to_v.sh b/hw/scripts/sv_to_v.sh new file mode 100755 index 0000000000..bd1cbab9b5 --- /dev/null +++ b/hw/scripts/sv_to_v.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Elaborates SystemVerilog sources with CIRCT and writes the design as one +# Verilog-2005 file, for synthesis tools with a Verilog frontend (yosys): +# no packed arrays or structs (ports, registers and logic become plain +# vectors), no local variables, and black-box modules left as instances of +# themselves with their parameter values, for the tool to bind. +# CIRCT_VERILOG, CIRCT_OPT and FIRTOOL name the tools (hw/syn/common.mk). + +CIRCT_VERILOG="${CIRCT_VERILOG:-circt-verilog}" +CIRCT_OPT="${CIRCT_OPT:-circt-opt}" +FIRTOOL="${FIRTOOL:-firtool}" + +set -eo pipefail + +top="" +source="" +includes=() +defines=() +blackboxes=() +output_file=out.v + +usage() { echo "$0 usage:" && grep " .)\ #" $0; exit 0; } +[ $# -eq 0 ] && usage +while getopts "t:s:o:I:D:b:h" arg; do + case $arg in + t) # top module + top=${OPTARG} + ;; + s) # source + source=${OPTARG} + ;; + o) # output-file + output_file=${OPTARG} + ;; + I) # include directory, whose sources are read too + includes+=("${OPTARG}") + ;; + D) # macro definition + defines+=("-D${OPTARG}") + ;; + b) # black-box modules, comma-separated + IFS=',' read -r -a names <<< "${OPTARG}" + for name in "${names[@]}"; do + [ -n "$name" ] && blackboxes+=("--blackbox=$name") + done + ;; + h | *) + usage + exit 0 + ;; + esac +done + +if [ -z "$top" ]; then + echo "sv_to_v.sh: error: no top module (-t)" >&2 + exit 1 +fi + +inc_args=() +files=() +[ -n "$source" ] && files+=("$source") +for dir in "${includes[@]}"; do + inc_args+=("-I$dir") + while IFS= read -r file; do + files+=("$file") + done < <(find "$dir" -maxdepth 1 -type f \( -name '*.v' -o -name '*.sv' \) | sort) +done + +ir="${output_file%.*}.mlir" +lowered="${output_file%.*}.lowered.mlir" +"$CIRCT_VERILOG" --ir-hw --top="$top" --detect-memories --allow-use-before-declare \ + --ignore-uninstantiated-modules "${blackboxes[@]}" "${defines[@]}" "${inc_args[@]}" \ + "${files[@]}" -o "$ir" +"$CIRCT_OPT" "$ir" --seq-lower-aggregate-regs --hw-aggregate-ports-to-int \ + --hw-convert-bitcasts --hw-aggregate-to-comb --canonicalize --cse -o "$lowered" +"$FIRTOOL" --format=mlir "$lowered" --disable-all-randomization \ + --lowering-options=noAlwaysComb,disallowLocalVariables,disallowPackedArrays,disallowPackedStructAssignments,svVarDeclStyle=reg,locationInfoStyle=none,omitVersionComment \ + -o "$output_file" diff --git a/hw/syn/common.mk b/hw/syn/common.mk index 24883c7502..84604150da 100644 --- a/hw/syn/common.mk +++ b/hw/syn/common.mk @@ -21,17 +21,21 @@ ifndef TOOLDIR $(error TOOLDIR not set — include $$(ROOT_DIR)/config.mk before hw/syn/common.mk) endif -# Tool install prefixes (overridable; default under $(TOOLDIR)). -SV2V_PATH ?= $(TOOLDIR)/sv2v +# Tool install prefixes (overridable; default under $(TOOLDIR)). CIRCT is the +# SystemVerilog frontend of every flow: its preprocessor (gen_sources.sh -P) +# and, for yosys, its Verilog-2005 export (sv_to_v.sh). +CIRCT_PATH ?= $(TOOLDIR)/circt YOSYS_PATH ?= $(TOOLDIR)/yosys STA_PATH ?= $(TOOLDIR)/sta VERILATOR_PATH ?= $(TOOLDIR)/verilator # Absolute tool binaries exported via env var so builds are self-contained # and multiple trees can coexist without a sourced env polluting PATH. -SV2V ?= $(SV2V_PATH)/bin/sv2v +CIRCT_VERILOG ?= $(CIRCT_PATH)/bin/circt-verilog +CIRCT_OPT ?= $(CIRCT_PATH)/bin/circt-opt +FIRTOOL ?= $(CIRCT_PATH)/bin/firtool YOSYS ?= $(YOSYS_PATH)/bin/yosys STA ?= $(STA_PATH)/bin/sta VERILATOR ?= $(VERILATOR_PATH)/bin/verilator -export SV2V_PATH YOSYS_PATH STA_PATH VERILATOR_PATH SV2V YOSYS STA VERILATOR +export CIRCT_PATH YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL YOSYS STA VERILATOR diff --git a/hw/syn/maxsim/Makefile b/hw/syn/maxsim/Makefile index e97f24fc99..15ebe55918 100644 --- a/hw/syn/maxsim/Makefile +++ b/hw/syn/maxsim/Makefile @@ -47,8 +47,8 @@ GEN_DIR := $(BUILD_DIR)/gen # first so they can be classes (lift_boundaries.py; [:drop=P,...][;port=P,...]). MAXSIM_CLASSES ?= VX_socket__lift MAXSIM_LIFT ?= VX_socket:drop=INSTANCE_ID;port=SOCKET_ID -# Where circt-verilog, circt-opt and firtool are (PATH by default). -CIRCT_BIN ?= +# Where circt-verilog, circt-opt and firtool are (hw/syn/common.mk's CIRCT). +CIRCT_BIN ?= $(CIRCT_PATH)/bin LIFT_PYTHON ?= python3 # The kernel clock (MHz): the image is timed at it and the shell programs it. MAXSIM_FREQ ?= 143 diff --git a/hw/syn/xilinx/aved/Makefile b/hw/syn/xilinx/aved/Makefile index 1dae65a0fc..c2ce83a999 100644 --- a/hw/syn/xilinx/aved/Makefile +++ b/hw/syn/xilinx/aved/Makefile @@ -1,6 +1,6 @@ ROOT_DIR := $(realpath ../../../..) include $(ROOT_DIR)/config.mk -# exports VERILATOR/VERILATOR_PATH, which gen_sources.sh resolves from the +# exports CIRCT_VERILOG, the preprocessor gen_sources.sh resolves from the # environment rather than PATH include $(VORTEX_HOME)/hw/syn/common.mk diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index e76070a987..49c99f3867 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -1,8 +1,8 @@ ROOT_DIR := $(realpath ../../../../..) include $(ROOT_DIR)/config.mk -# Exports VERILATOR/VERILATOR_PATH as absolute paths. gen_sources.sh resolves -# verilator from the environment, so without this the flow silently depends on -# verilator being on PATH and fails with "verilator: command not found". +# Exports CIRCT_VERILOG as an absolute path. gen_sources.sh resolves its +# preprocessor from the environment, so without this the flow silently depends +# on circt-verilog being on PATH. include $(VORTEX_HOME)/hw/syn/common.mk DEVICE ?= xcu55c-fsvh2892-2L-e diff --git a/hw/syn/xilinx/xrt/Makefile b/hw/syn/xilinx/xrt/Makefile index 9123d3a25e..32eb40b5ce 100644 --- a/hw/syn/xilinx/xrt/Makefile +++ b/hw/syn/xilinx/xrt/Makefile @@ -1,8 +1,8 @@ ROOT_DIR := $(realpath ../../../..) include $(ROOT_DIR)/config.mk -# Exports VERILATOR/VERILATOR_PATH as absolute paths. gen_sources.sh resolves -# verilator from the environment, so without this the flow silently depends on -# verilator being on PATH and fails with "verilator: command not found". +# Exports CIRCT_VERILOG as an absolute path. gen_sources.sh resolves its +# preprocessor from the environment, so without this the flow silently depends +# on circt-verilog being on PATH. include $(VORTEX_HOME)/hw/syn/common.mk ifneq ($(findstring Makefile, $(MAKEFILE_LIST)), Makefile) diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index b3c8a6c225..0fd6a01175 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -1,6 +1,6 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk -include $(ROOT_DIR)/hw/syn/common.mk # SV2V_PATH/YOSYS_PATH/STA_PATH + exported SV2V/YOSYS/STA +include $(ROOT_DIR)/hw/syn/common.mk # CIRCT/YOSYS/STA paths + exported tools DELAY_UNC ?= 0.02 DELAY_IO ?= 0.05 @@ -90,6 +90,7 @@ else BB_MODULES ?= VX_dp_ram_asic,VX_sp_ram_asic endif +comma := , SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts RTL_DIR := $(VORTEX_HOME)/hw/rtl @@ -203,9 +204,13 @@ $(BUILD_DIR): $(BUILD_DIR)/src: | $(BUILD_DIR) $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -C$(BUILD_DIR)/src -sv2v: $(BUILD_DIR)/project.v +# CIRCT elaborates the design into one Verilog-2005 file; the black-box +# modules (SRAM macros) stay instances with their parameter values, bound to +# their stubs in the sources. +verilog: $(BUILD_DIR)/project.v $(BUILD_DIR)/project.v: $(BUILD_DIR)/src - $(SCRIPT_DIR)/sv2v.sh -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -o$(BUILD_DIR)/project.v + $(SCRIPT_DIR)/sv_to_v.sh -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -b'$(BB_MODULES)' \ + -o$(BUILD_DIR)/project.v # Remove severity/control system-task lines yosys can't synthesize # ($fatal/$error/$warning/$info/$stop are used by the cvfpu sources). sed -ri '/^[[:space:]]*[$$](finish|display|fatal|error|warning|info|stop)/d' $(BUILD_DIR)/project.v @@ -213,6 +218,9 @@ $(BUILD_DIR)/project.v: $(BUILD_DIR)/src gen-sources: $(BUILD_DIR)/sources.txt $(BUILD_DIR)/sources.txt: $(BUILD_DIR)/project.v echo $(realpath $(BUILD_DIR)/project.v) > $(BUILD_DIR)/sources.txt + for m in $(subst $(comma), ,$(BB_MODULES)); do \ + grep -lE "^[[:space:]]*module[[:space:]]+$$m\b" $(BUILD_DIR)/src/*.v $(BUILD_DIR)/src/*.sv 2>/dev/null; \ + done | xargs -r realpath | sort -u >> $(BUILD_DIR)/sources.txt ifneq ($(strip $(DUT_FILELIST)),) SYN_SRC_FILE := $(abspath $(DUT_FILELIST)) diff --git a/hw/syn/yosys/run_synth.sh b/hw/syn/yosys/run_synth.sh index e0df65e607..a1713ee0d6 100755 --- a/hw/syn/yosys/run_synth.sh +++ b/hw/syn/yosys/run_synth.sh @@ -231,7 +231,9 @@ log "Writing $YS" for l in "${LIB_LIST[@]}"; do printf "read_liberty -lib %q\n" "$l"; done echo "# read sources" - for s in "${SRC_FILES[@]}"; do printf "read_verilog -defer %q\n" "$s"; done + # SystemVerilog mode: the design is Verilog-2005, which it includes, and + # the black-box stubs are SystemVerilog. + for s in "${SRC_FILES[@]}"; do printf "read_verilog -sv -defer %q\n" "$s"; done if [[ -n "$BB_MODULES" ]]; then IFS=',' read -r -a bbmods <<< "$BB_MODULES" @@ -303,7 +305,7 @@ log "Writing $YS" fi # Drop what mapping left behind before anything measures or reads the # netlist. Without it the gate reports dead cells as area, and the written - # netlist carries ~129 undriven `assign w = 'hx` wires -- sv2v function + # netlist carries undriven `assign w = 'hx` wires -- frontend function # scopes that survive as debris. OpenSTA's Verilog reader rejects those # outright once Yosys marks them `signed` (0.69 does, 0.57 did not), which # is how a missing clean read as a Yosys incompatibility. From a43dc9a9b668a3cea58c4e291be16d753ef3f91c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 12 Oct 2026 09:05:16 -0700 Subject: [PATCH 187/336] maxsim: fused classes (A+B), the LSU slice with its scheduler MAXSIM_CLASSES takes a class A+B: two modules of the same parent whose instances CIRCT pairs and simulates as one model (maxsim-fuse-models). The LSU slice and its scheduler exchange bits combinationally both ways (the slice's request, the scheduler's ready on it, the slice's ready on that), which took each thread two visits per target cycle; fused, the path is inside one model. The documented class set fuses them. tinygpu (V80): demo and sgemm -n16 pass with the same instructions and cycles; 4.00 -> 3.00 host cycles per target cycle; Fmax 162.3 -> 161.2 MHz. Area (Vivado OOC, opt_design): tinygpu 31,926 -> 31,592 LUTs, s2w4i2 166,519 -> 165,385 LUTs. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 6424c4c65b5d56e1232750bb1ac0f25216a2c2a1) --- docs/designs/maxsim.md | 14 ++++++++++++-- hw/syn/maxsim/maxsim.py | 25 ++++++++++++++++++++----- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/designs/maxsim.md b/docs/designs/maxsim.md index d343cf4d96..a478e26934 100644 --- a/docs/designs/maxsim.md +++ b/docs/designs/maxsim.md @@ -30,7 +30,7 @@ packaging, `slashkit link`) builds the image. There is no Chisel, Golden Gate, Yosys or sv2v anywhere in it, and no FireSim checkout. ```bash -export MAXSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift VX_lsu_scheduler__lift VX_fpu_std" +export MAXSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift+VX_lsu_scheduler__lift VX_fpu_std" export MAXSIM_LIFT="VX_socket:drop=INSTANCE_ID;port=SOCKET_ID VX_issue_slice:drop=ISSUE_ID VX_alu_int:port=BLOCK_IDX VX_alu_muldiv VX_lsu_slice:port=CORE_ID VX_lsu_scheduler" CONFIGS="-DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_ISSUE_WIDTH=2" CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=maxsim --app=sgemm # Verilator @@ -46,7 +46,17 @@ interface-port units a plain-port boundary first, so they can be classes RTL is not changed). `port=P` turns an instance-ID parameter into an input port, and every parameter derived from it below, so that all instances of the class elaborate to one module: the socket's `SOCKET_ID` carries every core's -`CORE_ID` with it. The tools are found through `CIRCT_BIN`. +`CORE_ID` with it. The tools are found through `CIRCT_BIN` (by default +`hw/syn/common.mk`'s CIRCT). + +A class `A+B` fuses two modules of the same parent (`maxsim-fuse-models`): +each `A` is paired with the one `B` it exchanges bits with, and each pair is +simulated as one model, so a path from one through the other and back takes +a thread one host cycle instead of a round trip through their channels. The +LSU slice and its scheduler are such a pair (the slice's request, the +scheduler's `ready` on it, the slice's `ready` on that): fused, tinygpu runs +at 3.00 host cycles per target cycle instead of 4.00 (V80), with the same +threads and copies and fewer LUTs. The flow reads Vortex as source, so it builds the configuration whose units are source: the STD FPU (`-DVX_CFG_FPU_TYPE_STD`) and the RTU without vendor diff --git a/hw/syn/maxsim/maxsim.py b/hw/syn/maxsim/maxsim.py index 4389bd5cdc..50ae779bca 100644 --- a/hw/syn/maxsim/maxsim.py +++ b/hw/syn/maxsim/maxsim.py @@ -11,6 +11,9 @@ models (maxsim-extract-models); the target's netlist is then composed into the simulator (maxsim-compose): one threaded module per class, the rest of the target as a model of one thread, and a fabric of token channels between them. +A class `A+B` is two modules of the same parent whose instances are paired and +simulated as one model (maxsim-fuse-models): what they exchange combinationally +both ways then takes a thread one host cycle, not a round trip through channels. Writes /maxsim_sim.sv, the module `maxsim_sim` with a channel per target port (`_valid`, `_ready`, `_bits`) and a `profile_finish` bit per class; @@ -88,18 +91,23 @@ def main(): '--allow-hierarchical-const', '--ignore-uninstantiated-modules', '--detect-memories', '--preserve-unpacked-ports', *defines, *incdirs, *files, '-o', design]) + groups = [c.split('+') for c in a.classes if '+' in c] + if any(len(g) != 2 for g in groups): + sys.exit('a fused class is two modules, A+B') + modules = [m for c in a.classes for m in c.split('+')] + # Flat but for the classes. flat = out('flat.mlir') with open(flat + '.in', 'w') as f: - f.write(publish(open(design).read(), [a.top] + a.classes)) + f.write(publish(open(design).read(), [a.top] + modules)) run([tool('circt-opt'), flat + '.in', '--symbol-dce', '--hw-flatten-modules', '--symbol-dce', '--canonicalize', '--cse', '-o', flat]) os.remove(flat + '.in') text = open(flat).read() # Each module with class instances, innermost first, becomes a netlist. - classes = set(a.classes) - children = {m: instantiated(text, m, classes) for m in [a.top] + a.classes} + classes = set(modules) + children = {m: instantiated(text, m, classes) for m in [a.top] + modules} order, seen = [], set() def visit(m): @@ -114,8 +122,15 @@ def visit(m): unused = classes - seen if unused: sys.exit(f'classes not in {a.top}: {" ".join(sorted(unused))}') - passes = [f'maxsim-extract-models{{module={m} models={",".join(sorted(children[m]))} netlist}}' - for m in order] + passes = [] + for m in order: + passes.append(f'maxsim-extract-models{{module={m} models={",".join(sorted(children[m]))} netlist}}') + fused = [g for g in groups if set(g) <= children[m]] + if fused: + passes.append(f'maxsim-fuse-models{{module={m} groups={",".join("+".join(g) for g in fused)}}}') + for g in groups: + if not any(set(g) <= children[m] for m in order): + sys.exit(f'{"+".join(g)}: not instantiated by the same module') pipeline = passes + OPTIMIZE + [f'maxsim-compose{{module={a.top} sim=maxsim_sim}}', 'canonicalize', 'cse'] sim = out('maxsim_sim.mlir') From 4c655f0264b720f318d8916fbc0ac67a16f0fca5 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 12 Oct 2026 10:03:14 -0700 Subject: [PATCH 188/336] maxsim runtime: poll backoff, exact-range DMA, memory-error report - Poll backoff: the card runs the target on its own, so after ~1 ms of busy polls the status poll sleeps, doubling up to 100 us. The in-process kernel advances only when it is accessed and keeps polling back to back. - DMA: QDMA needs only the staging-buffer offset page aligned, not the device address or the length, so a transfer is now exactly its bytes; a partial-page write no longer reads its first and last pages first. - MAXSIM_STATUS_MEM_ERR (a burst, which the memory model does not take) is reported and ends the wait instead of polling a stopped target. V80, tinygpu image, sgemm -n128: 2,785,811 -> 14,760 register reads and 2.10 -> 0.11 s of host CPU (63% -> 2%), kernel time unchanged (2,057 vs 2,059 ms); demo and sgemm -n16 cycle-identical. Writes and reads at unaligned offsets and lengths (1 B to 3 pages + 5 B, across page edges) match a host copy of the buffer exactly. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 867a01123247aba2eb2a3fef6cdb29ab7d710c12) --- sw/runtime/maxsim/vortex.cpp | 83 +++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/sw/runtime/maxsim/vortex.cpp b/sw/runtime/maxsim/vortex.cpp index 2f3236b14a..05b71fcc56 100644 --- a/sw/runtime/maxsim/vortex.cpp +++ b/sw/runtime/maxsim/vortex.cpp @@ -71,6 +71,8 @@ constexpr uint64_t kMemBase = 0x4000000000ull; class transport_t { public: + // The kernel advances while it is accessed. + static constexpr bool kRunsOnItsOwn = false; int init() { return sim_.init(); } int reg_write(uint32_t off, uint32_t v) { return sim_.reg_write(off, v); } int reg_read(uint32_t off, uint32_t* v) { return sim_.reg_read(off, v); } @@ -100,6 +102,8 @@ class transport_t { class transport_t { public: + // The kernel runs on the card whether or not the host looks. + static constexpr bool kRunsOnItsOwn = true; int init() { const char* vbin = getenv("VRT_VBIN_PATH"); if (vbin == nullptr || vbin[0] == '\0') { @@ -176,24 +180,16 @@ class transport_t { return 0; } - // QDMA moves whole 4 KB pages; a partial one is read, merged and written. + // QDMA moves any byte range: only the offset into the staging buffer must + // be page aligned, not the device address or the length. A write is the + // bytes it writes, with no read of the pages around them. int mem_write(uint64_t addr, const void* src, uint64_t n) { std::lock_guard lock(mutex_); auto* bytes = static_cast(src); while (n != 0) { - const uint64_t page = addr & ~(kPage - 1); - const uint64_t off = addr - page; - const uint64_t span = std::min(kDmaChunk - off, n); - const uint64_t len = (off + span + kPage - 1) & ~(kPage - 1); - if (off != 0 || (span % kPage) != 0) { - // Only the first and last pages can be partial. - if (dma(page, staging_, kPage, false) != 0 || - (len > kPage && - dma(page + len - kPage, staging_ + len - kPage, kPage, false) != 0)) - return -1; - } - std::memcpy(staging_ + off, bytes, span); - if (dma(page, staging_, len, true) != 0) + const uint64_t span = std::min(kDmaChunk, n); + std::memcpy(staging_, bytes, span); + if (dma(addr, staging_, span, true) != 0) return -1; addr += span; bytes += span; @@ -206,13 +202,10 @@ class transport_t { std::lock_guard lock(mutex_); auto* bytes = static_cast(dst); while (n != 0) { - const uint64_t page = addr & ~(kPage - 1); - const uint64_t off = addr - page; - const uint64_t span = std::min(kDmaChunk - off, n); - const uint64_t len = (off + span + kPage - 1) & ~(kPage - 1); - if (dma(page, staging_, len, false) != 0) + const uint64_t span = std::min(kDmaChunk, n); + if (dma(addr, staging_, span, false) != 0) return -1; - std::memcpy(bytes, staging_ + off, span); + std::memcpy(bytes, staging_, span); addr += span; bytes += span; n -= span; @@ -221,12 +214,11 @@ class transport_t { } private: - static constexpr uint64_t kPage = 4096; static constexpr uint64_t kDmaChunk = 64ull << 20; - static constexpr uint64_t kStaging = kDmaChunk + kPage; + static constexpr uint64_t kStaging = kDmaChunk; // `len` bytes at target address `addr`, from or to `host` in the staging - // buffer, both page aligned. + // buffer, page aligned. int dma(uint64_t addr, uint8_t* host, uint64_t len, bool to_device) { struct slash_qdma_transfer xfer = {}; xfer.size = sizeof(xfer); @@ -465,12 +457,42 @@ class vx_device { int wait_idle() { uint32_t s = 0; do { - if (status(&s) != 0) + if (status(&s) != 0 || mem_error(s)) return -1; } while (s & MAXSIM_STATUS_BUSY); return 0; } + // The memory endpoint takes single-beat transactions; a target that makes a + // burst stops the simulation there, busy for good. + bool mem_error(uint32_t s) { + if (!(s & MAXSIM_STATUS_MEM_ERR)) + return false; + if (!mem_error_reported_) { + fprintf(stderr, "[VXDRV] the target made a memory burst, which the memory " + "model does not take; the simulation is stopped\n"); + mem_error_reported_ = true; + } + return true; + } + + // The target runs on its own: once it has been busy for a while, polling it + // back to back only spins a core, so the poll sleeps, longer the longer the + // target stays busy (up to kMaxPollSleepUs). + void poll_backoff(bool busy) { + if (!transport_t::kRunsOnItsOwn) + return; + if (!busy) { + busy_polls_ = 0; + return; + } + if (++busy_polls_ <= kSpinPolls) + return; + const uint64_t us = std::min(1ull << std::min(busy_polls_ - kSpinPolls, 7), + kMaxPollSleepUs); + std::this_thread::sleep_for(std::chrono::microseconds(us)); + } + void* host_region_ptr(uint64_t addr) { std::lock_guard g(host_mu_); if (host_regions_.empty()) @@ -517,9 +539,11 @@ class vx_device { h.vortex_start = [this]() { command(MAXSIM_CMD_START, 0, 0); }; h.vortex_busy = [this]() -> bool { uint32_t s = 0; - if (status(&s) != 0) + if (status(&s) != 0 || mem_error(s)) return false; - return (s & MAXSIM_STATUS_BUSY) != 0; + const bool busy = (s & MAXSIM_STATUS_BUSY) != 0; + poll_backoff(busy); + return busy; }; #ifdef VX_CFG_VM_ENABLE h.mmu_fault_report = true; @@ -527,7 +551,14 @@ class vx_device { return h; } + // Polls back to back before the first sleep: about a millisecond at a PCIe + // round trip each. + static constexpr uint64_t kSpinPolls = 1024; + static constexpr uint64_t kMaxPollSleepUs = 100; + counted_transport_t transport_; + uint64_t busy_polls_ = 0; + bool mem_error_reported_ = false; bool open_ = false; vortex::CommandProcessor cp_; std::mutex host_mu_; From 3833fc711f035d977987ab20140ad8d37e4131f8 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 12 Oct 2026 21:59:09 -0700 Subject: [PATCH 189/336] maxsim: build the full GPU's extensions, and its RTU on DSPs The flow now reads the extensions' packages and include paths from hw/syn/extensions.mk, as every synthesis flow does, so a configuration with TCU, DXA, raster, texture, OM or RTU elaborates. The RTU no longer forces VX_CFG_RTU_USE_DSP=0. It kept the RTU's F32 FMAs and dividers off Xilinx's floating-point cores, whose netlists the flow can neither read nor thread, but it also took away their DSP multiplies. Instead, the staged copies of VX_fma_unit, VX_fdiv_unit and VX_fsqrt_unit no longer select their vendor core (VENDOR_OK = 0), so RTU_USE_DSP keeps Vortex's FPGA default and the RTU's soft FMAs multiply on DSPs. The build stops if a unit ever selects its vendor core some other way. -DVIVADO stays, so the rest of Vortex elaborates as the native Vivado flows see it. The RTU divider keeps the soft divider's latency. Validated: 4-core full GPU (L2, A, TCU, DXA, TEX, RASTER, OM, RTU) demo in Verilator emulation PASSED, instrs=39104 cycles=72519 (unchanged). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 56b5feeaf8b9753aa32f944c768dec8c066f6c26) --- hw/syn/maxsim/Makefile | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/hw/syn/maxsim/Makefile b/hw/syn/maxsim/Makefile index 15ebe55918..d38baec613 100644 --- a/hw/syn/maxsim/Makefile +++ b/hw/syn/maxsim/Makefile @@ -64,16 +64,21 @@ RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG # One memory port, as the memory endpoint models; 4 AXI id bits, what Vortex # drives with its tag buffer engaged. The FPU and RTU use Vortex's own -# datapaths (the STD FPU, the RTU without vendor cores): a vendor netlist is not -# source the flow can read, nor state it can thread. +# datapaths (the STD FPU, the RTU's soft cores with their DSP multiplies): a +# vendor netlist is not source the flow can read, nor state it can thread. The +# staged sources therefore never select the floating-point cores (see +# VENDOR_FP_UNITS), and the RTU divider has the soft one's latency. PLATFORM_DEFINES := -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DPLATFORM_MEMORY_ID_WIDTH=4 \ - -DVX_CFG_RTU_USE_DSP=0 -DVX_CFG_RTU_FDIV_LATENCY=17 \ - -DVX_CFG_FPU_TYPE_STD + -DVX_CFG_RTU_FDIV_LATENCY=17 -DVX_CFG_FPU_TYPE_STD ifdef PERF PLATFORM_DEFINES += -DPERF_ENABLE endif -TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(shell python3 $(VORTEX_HOME)/ci/gen_config.py \ - --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN) $(SYNTH_DEFINES) $(PLATFORM_DEFINES)')) +XCONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py \ + --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN) $(SYNTH_DEFINES) $(PLATFORM_DEFINES)') +TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) +# The extensions' packages and include paths, as every synthesis flow finds +# them. +include $(VORTEX_HOME)/hw/syn/extensions.mk GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) \ $(SYNTH_DEFINES) $(RTL_INCLUDE) @@ -111,12 +116,21 @@ all: $(VBIN) # The flow threads memories itself, so an asynchronous-read RAM stays one # (VX_async_ram_patch.sv) instead of becoming placeholders for a Vivado hook. +# The units that would wrap a Xilinx floating-point core take their soft path, +# the one they take wherever no vendor core exists. +VENDOR_FP_UNITS := VX_fma_unit.sv VX_fdiv_unit.sv VX_fsqrt_unit.sv sources: $(SRCS_LIST) $(SRCS_LIST): $(RTL_SRCS) $(SRC_DIR)/VX_async_ram_patch.sv rm -rf $(SRCS_DIR) cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) -T$(TARGET_TOP) -C$(SRCS_NAME) \ - -O$(SRCS_NAME).f + -O$(SRCS_NAME).f $(RTL_PKGS) cp $(SRC_DIR)/VX_async_ram_patch.sv $(SRCS_DIR)/VX_async_ram_patch.sv + @for f in $(VENDOR_FP_UNITS); do \ + [ -f $(SRCS_DIR)/$$f ] || continue; \ + grep -q 'localparam VENDOR_OK = 1;' $(SRCS_DIR)/$$f || \ + { echo "ERROR: $$f no longer selects its vendor core by VENDOR_OK"; exit 1; }; \ + sed -i 's/localparam VENDOR_OK = 1;/localparam VENDOR_OK = 0;/' $(SRCS_DIR)/$$f; \ + done @for f in $(SHELL_MODULES); do \ rm -f $(SRCS_DIR)/$$f; \ grep -v "/$$f$$" $@ > $@.tmp && mv $@.tmp $@; \ From b81c74cc91a474315af2a9b63757c3a5d0c38dd8 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 12 Oct 2026 21:59:09 -0700 Subject: [PATCH 190/336] maxsim: read block RAMs synchronously, as the RTL asks The flow's VX_async_ram_patch, which VX_dp_ram and VX_sp_ram instantiate only for a RAM that must be block RAM, now says so with the synthesis attribute (* ram_style = "block" *). Once the target is flattened (but for its classes), CIRCT's seq-retime-firmem-reads makes each such read synchronous: it reads at the address the address registers are about to take, so the RAM can be a block RAM. Unhinted RAMs are left as they are. The pass writes gen/memories.rpt, one line per asynchronous-read RAM with its size, hint and decision. On the 4-core full GPU, one RAM asks for block RAM, the IPDOM stack store (112x39, four threads), and it is converted. The other 114 asynchronous RAMs are below Vortex's block RAM threshold. Validated: tinygpu token co-simulation against the unconverted reference, 2000 cycles, 0 errors (host/target 3.00); full GPU demo in Verilator emulation PASSED, instrs=39104 cycles=72519 (unchanged). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 285672bf9b1f38ded3c22995fc2ee9162a59717a) --- hw/syn/maxsim/VX_async_ram_patch.sv | 9 ++++++--- hw/syn/maxsim/maxsim.py | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hw/syn/maxsim/VX_async_ram_patch.sv b/hw/syn/maxsim/VX_async_ram_patch.sv index 43ca5617a4..9b702be3a7 100644 --- a/hw/syn/maxsim/VX_async_ram_patch.sv +++ b/hw/syn/maxsim/VX_async_ram_patch.sv @@ -19,8 +19,8 @@ // rewires; with or without RADDR_REG it reads the word at raddr. The maxsim // flow threads its memories itself and cannot run that hook, so it replaces the // staged copy of the module with this one: same parameters and ports, same -// contents and reads. Plain SystemVerilog, as the staged sources are already -// preprocessed. +// contents and reads, and the block RAM request as a synthesis attribute. +// Plain SystemVerilog, as the staged sources are already preprocessed. module VX_async_ram_patch #( parameter DATAW = 1, parameter SIZE = 1, @@ -47,7 +47,10 @@ module VX_async_ram_patch #( ); localparam WSELW = DATAW / WRENW; - reg [DATAW-1:0] ram [0:SIZE-1]; + // VX_dp_ram and VX_sp_ram instantiate this module only for a RAM that + // must be block RAM; the compiler makes its read synchronous + // (seq-retime-firmem-reads), as the attribute asks. + (* ram_style = "block" *) reg [DATAW-1:0] ram [0:SIZE-1]; if (INIT_ENABLE != 0) begin : g_init if (INIT_FILE != "") begin : g_file diff --git a/hw/syn/maxsim/maxsim.py b/hw/syn/maxsim/maxsim.py index 50ae779bca..e6a28e8a27 100644 --- a/hw/syn/maxsim/maxsim.py +++ b/hw/syn/maxsim/maxsim.py @@ -100,7 +100,10 @@ def main(): flat = out('flat.mlir') with open(flat + '.in', 'w') as f: f.write(publish(open(design).read(), [a.top] + modules)) + # A RAM that asks for block RAM reads synchronously once its wrapper is + # flattened into the module that registers its address. run([tool('circt-opt'), flat + '.in', '--symbol-dce', '--hw-flatten-modules', '--symbol-dce', + '--canonicalize', '--cse', f'--seq-retime-firmem-reads=report={out("memories.rpt")}', '--canonicalize', '--cse', '-o', flat]) os.remove(flat + '.in') text = open(flat).read() From b6263ad26d7d647c32fe3b2d1eed631e10b13725 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 01:03:45 -0700 Subject: [PATCH 191/336] Block RAM by CIRCT, not a netlist patch; all synthesis flows on CIRCT A RAM that must be block RAM (FORCE_BRAM) but reads asynchronously (OUT_REG=0, read-first) is now a plain array tagged with the standard synthesis attribute, `(* ram_style = "block" *)` (USE_BLOCK_BRAM), read asynchronously. CIRCT's seq-retime-firmem-reads makes the read synchronous, reading at the address the address registers are about to take, after inlining the modules between the RAM and those registers. A block RAM request it cannot honor stops the build with the reason. Every synthesis flow now reads one netlist CIRCT elaborates: hw/scripts/circt_elaborate.sh (was sv_to_v.sh) writes SystemVerilog for Vivado (-f sv) or Verilog-2005 for Yosys (-f v2005), expands conditional array writes into per-element selects, merges duplicate logic, and writes .memories.rpt, one line per asynchronous-read RAM. gen_sources.sh -E runs it on the staged tree and makes the filelist name the netlist (xrt, aved, dut, sandbox); FireSim and maxsim run it in their own recipes. The Xilinx floating-point cores stay instances, declared to the elaboration in hw/scripts/xilinx_ip_stubs.sv. Removed: ASYNC_BRAM_PATCH, VX_async_ram_patch (both copies), VX_placeholder, xilinx_async_bram_patch.tcl and the hooks that sourced it (the sandbox and FireSim pre-opt hooks entirely), and RADDR_RESET, which only the patch read. RADDR_REG stays: the ASIC branch uses it. xilinx_slr_pblocks.tcl accepts the netlist's generate-block names, g_x_0_inst, as well as g_x[0].inst. Validated: - xilinx/dut core, 16 warps, 8 threads, V80 part, 300 MHz, implemented both ways: 90,481 logic LUTs against 92,608 today, the same BRAM (65+9), LUTRAM (2,998) and DSPs (40), timing met both ways. The FF count differs only by registers Vivado retimed (`_bret`): the RTL registers are the same. The IPDOM stack store (112x39) is converted. - Yosys: the core (16 warps, 16 threads) exports to Verilog-2005. - maxsim: tinygpu token co-simulation, 0 errors; 4-core full GPU demo in Verilator emulation PASSED, instrs=39104 cycles=72519 (unchanged). Needs the CIRCT of maxsim/phase1 (circt-maxsim) in the toolchain; the synthesis gates' baselines move with the netlist. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c87d95144e3eaf310d589efefe0cda1baffa796e) Integration (phase 2): the Xilinx flows' source rule keeps phase 2's dependency on the AFU memory-port stamp; the library catalog keeps phase 2's VX_rom and drops the two removed modules. --- CHANGELOG.md | 3 + ci/synth_gate.py | 4 +- docs/assets/img/firesim_build_flow.svg | 4 +- docs/designs/firesim_integration.md | 8 +- docs/designs/maxsim.md | 10 +- docs/hardware_library.md | 218 +++++----- docs/synthesis_analysis.md | 2 +- hw/rtl/VX_platform.vh | 3 - hw/rtl/libs/VX_async_ram_patch.sv | 277 ------------ hw/rtl/libs/VX_dp_ram.sv | 37 +- hw/rtl/libs/VX_fifo_queue.sv | 3 +- hw/rtl/libs/VX_placeholder.sv | 26 -- hw/rtl/libs/VX_sp_ram.sv | 37 +- hw/scripts/circt_elaborate.sh | 157 +++++++ hw/scripts/gen_sources.sh | 33 +- hw/scripts/sv_to_v.sh | 92 ---- hw/scripts/xilinx_async_bram_patch.tcl | 563 ------------------------- hw/scripts/xilinx_ip_stubs.sv | 88 ++++ hw/scripts/xilinx_slr_pblocks.tcl | 17 +- hw/syn/common.mk | 3 +- hw/syn/firesim/Makefile | 30 +- hw/syn/firesim/pre_opt_hook.tcl | 11 - hw/syn/maxsim/Makefile | 5 +- hw/syn/maxsim/VX_async_ram_patch.sv | 79 ---- hw/syn/xilinx/aved/Makefile | 7 +- hw/syn/xilinx/aved/pre_opt_hook.tcl | 11 - hw/syn/xilinx/aved/pre_synth_hook.tcl | 5 - hw/syn/xilinx/common.mk | 6 +- hw/syn/xilinx/dut/common.mk | 3 +- hw/syn/xilinx/dut/pre_opt_hook.tcl | 1 - hw/syn/xilinx/sandbox/Makefile | 8 +- hw/syn/xilinx/sandbox/pre_opt_hook.tcl | 2 - hw/syn/xilinx/sandbox/project.tcl.in | 2 +- hw/syn/xilinx/xrt/pre_opt_hook.tcl | 1 - hw/syn/yosys/Makefile | 2 +- 35 files changed, 442 insertions(+), 1316 deletions(-) delete mode 100644 hw/rtl/libs/VX_async_ram_patch.sv delete mode 100644 hw/rtl/libs/VX_placeholder.sv create mode 100755 hw/scripts/circt_elaborate.sh delete mode 100755 hw/scripts/sv_to_v.sh delete mode 100644 hw/scripts/xilinx_async_bram_patch.tcl create mode 100644 hw/scripts/xilinx_ip_stubs.sv delete mode 100644 hw/syn/firesim/pre_opt_hook.tcl delete mode 100644 hw/syn/maxsim/VX_async_ram_patch.sv delete mode 100644 hw/syn/xilinx/sandbox/pre_opt_hook.tcl diff --git a/CHANGELOG.md b/CHANGELOG.md index c81cc47c33..93352c63f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,9 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, value exactly. - **FPGA run instructions.** The docs exported `TARGET=hw`, which `blackbox.sh` ignores, so runs fell back to xrtsim; they now pass `--target=hw`. +- **Block RAM for asynchronous-read RAMs is a CIRCT pass, not a Vivado netlist patch; every synthesis flow reads one CIRCT-elaborated netlist.** A RAM that must be block RAM but reads asynchronously is a plain array tagged `(* ram_style = "block" *)`; CIRCT's `seq-retime-firmem-reads` makes its read synchronous, reading at the address its registers are about to take, and stops the build with the reason if it cannot. `hw/scripts/circt_elaborate.sh` (`gen_sources.sh -E`) elaborates the design for Vivado (xrt, aved, dut, sandbox), FireSim, maxsim and Yosys alike, and writes a report of every asynchronous-read RAM. `ASYNC_BRAM_PATCH`, `VX_async_ram_patch`, `VX_placeholder`, `xilinx_async_bram_patch.tcl` and its pre-opt hooks are gone; Xilinx floating-point cores are declared to the elaboration in `hw/scripts/xilinx_ip_stubs.sv`. + *Why:* the patch rewired Vivado's synthesized netlist in Tcl, ran in Vivado flows only, failed late on large builds (`DRC INBB-3`), and produced a netlist nothing simulated. On the core at 16 warps and 8 threads (V80, 300 MHz met both ways) the CIRCT netlist implements in 90,481 logic LUTs against 92,608, with the same BRAM, LUTRAM and DSPs. + *Limits:* generate-block instances are named `g_x_0_inst`, not `g_x[0].inst`, in the netlist (`xilinx_slr_pblocks.tcl` accepts both); the modules between a converted RAM and its address registers are inlined; the synthesis baselines move with the netlist and are regenerated by hand. - **CIRCT is the SystemVerilog frontend of every synthesis flow; sv2v and Verilator's preprocessor are gone.** `gen_sources.sh -P` preprocesses with `circt-verilog -E` (identical code to `verilator -E` over the whole RTL tree; only comments differ), and the Yosys flow elaborates with CIRCT into one Verilog-2005 file ([hw/scripts/sv_to_v.sh](hw/scripts/sv_to_v.sh)) instead of sv2v: packed arrays and structs become vectors, and the SRAM macros (`BB_MODULES`) stay instances with their parameter values, read in SystemVerilog mode beside their stubs. CIRCT installs under `$TOOLDIR/circt` (`toolchain_install.sh --circt`; `hw/syn/common.mk` exports `CIRCT_VERILOG`, `CIRCT_OPT`, `FIRTOOL`), and the `asic_gate` fingerprints its baselines on the CIRCT version. Verilator remains the simulator (and `--xml-only` scope metadata). On the cache DUT (ASAP7), SRAM area is identical and logic area is 5.6% higher than through sv2v: CIRCT folds synchronous resets into the next-value logic and lowers array reads to mux trees, both open items. ## [3.0] — 2026-10-07 diff --git a/ci/synth_gate.py b/ci/synth_gate.py index 2750d27690..92dabb07d3 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -334,14 +334,14 @@ class Yosys(Tool): job_label = "make" # CIRCT is the parser: a bad define, a missing include or a hierarchy error - # dies in sv_to_v.sh, before Yosys ever reads a line. run_synth.sh stamps + # dies in circt_elaborate.sh, before Yosys ever reads a line. run_synth.sh stamps # `gen-ys` immediately after source generation and conversion have both # succeeded, so that stamp is this flow's "past the typo risk" marker. elab_done = "TIME gen-ys" elab_label = "source conversion" phases = ( ("gen_sources.sh", "sources"), - ("sv_to_v.sh", "verilog"), + ("circt_elaborate.sh", "verilog"), (elab_done, "synthesis"), ("TIME yosys", "sram"), ("run_sta.tcl", "timing"), diff --git a/docs/assets/img/firesim_build_flow.svg b/docs/assets/img/firesim_build_flow.svg index 983187e59d..c938408f81 100644 --- a/docs/assets/img/firesim_build_flow.svg +++ b/docs/assets/img/firesim_build_flow.svg @@ -52,8 +52,8 @@ Platform build — the card's own flow vitis: v++ link · aved: slashkit. The kernel is packaged from three inputs: - VORTEX_VIVADO_SRCS = the flattened tree VORTEX_XILINX_IP_DIR = the vendor cores - FireSim-generated.sv = the harness VORTEX_PRE_OPT_TCL = async-BRAM patch hook + VORTEX_VIVADO_SRCS = the CIRCT netlist VORTEX_XILINX_IP_DIR = the vendor cores + FireSim-generated.sv = the harness Vivado synthesis + place-and-route at FREQUENCY (default 60 MHz), STRATEGY=TIMING; on aved, FireSim's clock gates become clock enables (no BUFGCE in the partition). diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index 703b3e7b74..481e0bfa5c 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -193,15 +193,11 @@ changed, since FireSim's make judges elaboration by these timestamps), `replace-rtl` (Golden Gate), and the platform's own build (`v++` on vitis, slashkit on aved), handed: -- `VORTEX_VIVADO_SRCS` — the flattened tree, packaged into the kernel; +- `VORTEX_VIVADO_SRCS` — the netlist CIRCT elaborates from the flattened + tree, packaged into the kernel; - `VORTEX_XILINX_IP_DIR` — the vendor cores, copied in fresh directories (Tcl's `file copy` onto an existing directory nests instead of replacing, which silently reuses the previous run's cores); -- `VORTEX_PRE_OPT_TCL` — the async-BRAM patch hook. `VX_async_ram_patch` - leaves `VX_placeholder` black boxes wherever an asynchronous-read RAM must - live in block RAM; without the hook, `opt_design` aborts with `DRC INBB-3` - — but only once a configuration is large enough to push those RAMs out of - LUTRAM, so small builds pass and large ones do not; - `FREQUENCY` (default 60 MHz) and `STRATEGY` (default `TIMING`). --- diff --git a/docs/designs/maxsim.md b/docs/designs/maxsim.md index a478e26934..85d613d99e 100644 --- a/docs/designs/maxsim.md +++ b/docs/designs/maxsim.md @@ -59,10 +59,12 @@ at 3.00 host cycles per target cycle instead of 4.00 (V80), with the same threads and copies and fewer LUTs. The flow reads Vortex as source, so it builds the configuration whose units -are source: the STD FPU (`-DVX_CFG_FPU_TYPE_STD`) and the RTU without vendor -cores. Vivado's asynchronous-read RAM patch (placeholder cells a -post-synthesis hook rewires) is replaced in the flow's copy of the sources by -the RAM it stands for (`hw/syn/maxsim/VX_async_ram_patch.sv`). +are source: the STD FPU (`-DVX_CFG_FPU_TYPE_STD`), and the RTU's soft FMAs and +dividers, which the flow's copy of the floating-point units selects in place of +the Xilinx cores. A RAM that asks for block RAM but reads asynchronously reads +synchronously once the target is flattened (CIRCT `seq-retime-firmem-reads`, +see the hardware library's "Block RAM and asynchronous reads"); +`gen/memories.rpt` lists each such RAM and the decision. --- diff --git a/docs/hardware_library.md b/docs/hardware_library.md index 0ba4a8bdb5..471cff2cf4 100644 --- a/docs/hardware_library.md +++ b/docs/hardware_library.md @@ -27,7 +27,6 @@ All modules are parameterized for data width, depth, and implementation strategy 12. [Synchronization and Timing](#12-synchronization-and-timing) 13. [Allocators and Counters](#13-allocators-and-counters) 14. [Debug and Tracing](#14-debug-and-tracing) -15. [Utility and Placeholder](#15-utility-and-placeholder) --- @@ -1255,7 +1254,7 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn **Features:** - Single address shared between read and write - Three read-during-write modes including "N" (no-change) which holds previous read data -- `ASYNC_BRAM_PATCH` support for asynchronous read BRAM workarounds +- A forced-BRAM, read-first RAM read asynchronously is tagged block RAM (`ram_style = "block"`); the synthesis elaboration makes its read synchronous (see [Block RAM and asynchronous reads](#block-ram-and-asynchronous-reads)) - Same FPGA/ASIC implementation paths as VX_dp_ram --- @@ -1279,30 +1278,31 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn --- -### VX_async_ram_patch +### Block RAM and asynchronous reads -**File:** `VX_async_ram_patch.sv` -**Purpose:** Asynchronous RAM wrapper with patching support for synthesis tools. +Block RAMs and SRAM macros only read synchronously. A `VX_dp_ram` or +`VX_sp_ram` that must be block RAM (`FORCE_BRAM`) but reads asynchronously +(`OUT_REG=0`, read-first) is written as a plain array with the standard +synthesis attribute `(* ram_style = "block" *)` (`USE_BLOCK_BRAM`) and an +asynchronous read. Every synthesis flow elaborates the design with CIRCT +(`hw/scripts/circt_elaborate.sh`, `gen_sources.sh -E`), whose +`seq-retime-firmem-reads` pass then reads such a RAM one edge earlier, at the +address its address registers are about to take, so that it maps to a block +RAM. The modules between the RAM and its registers are inlined for it, and +only those. -| Parameter | Default | Description | -|-----------|---------|-------------| -| `DATAW` | 1 | Data width | -| `SIZE` | 1 | Number of entries | -| `WRENW` | 1 | Write enable granularity | -| `DUAL_PORT` | 0 | Enable dual-port operation | -| `FORCE_BRAM` | 0 | Force block RAM inference | -| `RADDR_REG` | 0 | Read address registered | -| `RADDR_RESET` | 0 | Read address has reset | -| `WRITE_FIRST` | 0 | Write-first semantics | -| `INIT_ENABLE` | 0 | Enable initialization | -| `INIT_FILE` | "" | Initialization file | -| `INIT_VALUE` | 0 | Default init value | +The attribute is a contract: -**Features:** -- Wraps synchronous RAM with asynchronous read path emulation -- Uses placeholder modules for tool-specific register/memory patching -- Supports dual-port and single-port configurations -- Handles vendor-specific synthesis quirks (Xilinx BRAM async read workaround) +- `ram_style = "block"`: the RAM is converted. If it cannot be (its address + depends on anything but registers and logic, or a register has an + asynchronous reset), the elaboration stops and names the reason; +- `ram_style = "distributed"`: the RAM stays as it is; +- no attribute: the RAM stays as it is (`circt_elaborate.sh -m auto` converts + those it can). + +The converted read is exact on every cycle after the first clock edge. Each +elaboration writes `.memories.rpt`, one line per asynchronous-read +RAM with its size, hint and decision. --- @@ -1798,107 +1798,81 @@ Infrastructure for runtime debug observation and serial trace extraction. --- -## 15. Utility and Placeholder - -### VX_placeholder - -**File:** `VX_placeholder.sv` -**Purpose:** Empty black-box module for hierarchical design placeholder. - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `I` | 1 | Input width | -| `O` | 1 | Output width | - -| Port | Direction | Width | Description | -|------|-----------|-------|-------------| -| `data_in` | input | I | Input (unconnected) | -| `data_out` | output | O | Output (unconnected) | - -**Features:** -- Marked with `BLACKBOX_CELL` synthesis directive -- Used as a structural placeholder for tool-specific cell replacement -- Zero logic -- purely a synthesis anchor point - ---- - ## Module Summary Table | # | Module | Category | Area | Latency | |---|--------|----------|------|---------| | 1 | VX_allocator | Allocator | Small | 0 cycles | -| 2 | VX_async_ram_patch | Memory | Varies | 0-1 cycles | -| 3 | VX_avs_adapter | Memory Adapter | Large | Multi-cycle | -| 4 | VX_axi_adapter | Memory Adapter | Large | Multi-cycle | -| 5 | VX_axi_write_ack | Memory Adapter | Small | 0-1 cycles | -| 6 | VX_bits_concat | Bit Manip | Zero | 0 cycles | -| 7 | VX_bits_insert | Bit Manip | Zero | 0 cycles | -| 8 | VX_bits_remove | Bit Manip | Zero | 0 cycles | -| 9 | VX_bypass_buffer | Buffer | Small | 0-1 cycles | -| 10 | VX_clockgate | Sync | Minimal | 0 cycles | -| 11 | VX_csa_32 | CSA | Small | 1 gate | -| 12 | VX_csa_42 | CSA | Medium | 2 gates | -| 13 | VX_csa_mod4 | CSA | Varies | O(log N) | -| 14 | VX_csa_tree | CSA | Varies | O(log N) | -| 15 | VX_cyclic_arbiter | Arbiter | Small | 0 cycles | -| 16 | VX_demux | Mux/Demux | Small | 0 cycles | -| 17 | VX_divider | Arithmetic | Large | Configurable | -| 18 | VX_dp_ram | Memory | Varies | 0-1 cycles | -| 19 | VX_edge_trigger | Sync | Minimal | 1 cycle | -| 20 | VX_elastic_adapter | Buffer | Minimal | 0-1 cycles | -| 21 | VX_elastic_buffer | Buffer | Varies | Configurable | -| 22 | VX_fifo_queue | Memory | Varies | 0-1 cycles | -| 23 | VX_find_first | Encoder | Small | O(log N) | -| 24 | VX_fold_mul | Arithmetic | Medium | Configurable | -| 25 | VX_generic_arbiter | Arbiter | Varies | 0 cycles | -| 26 | VX_gto_arbiter | Arbiter | Medium | 0 cycles | -| 27 | VX_index_buffer | Memory | Medium | 0-1 cycles | -| 28 | VX_index_queue | Memory | Medium | 0-1 cycles | -| 29 | VX_ks_adder | Arithmetic | Medium | O(log N) | -| 30 | VX_lzc | Encoder | Small | O(log N) | -| 31 | VX_matrix_arbiter | Arbiter | Medium | 0 cycles | -| 32 | VX_mem_bank_adapter | Memory Adapter | Large | Multi-cycle | -| 33 | VX_mem_coalescer | Memory Adapter | Large | Multi-cycle | -| 34 | VX_mem_data_adapter | Memory Adapter | Medium | Multi-cycle | -| 35 | VX_mem_scheduler | Scheduler | Large | Multi-cycle | -| 36 | VX_multiplier | Arithmetic | Varies | Configurable | -| 37 | VX_mux | Mux/Demux | Small | 0 cycles | -| 38 | VX_nz_iterator | Stream Util | Small | 0-1 cycles | -| 39 | VX_onehot_encoder | Encoder | Small | O(log N) | -| 40 | VX_onehot_mux | Mux/Demux | Small | 0 cycles | -| 41 | VX_onehot_shift | Bit Manip | Small | 0 cycles | -| 42 | VX_pe_serializer | Stream Util | Medium | Multi-cycle | -| 43 | VX_pending_size | Counter | Small | 0-1 cycles | -| 44 | VX_pipe_buffer | Buffer | Varies | N cycles | -| 45 | VX_pipe_register | Buffer | Varies | N cycles | -| 46 | VX_placeholder | Utility | Zero | 0 cycles | -| 47 | VX_popcount | Arithmetic | Small | O(log N) | -| 48 | VX_priority_arbiter | Arbiter | Small | 0 cycles | -| 49 | VX_priority_encoder | Encoder | Small | O(log N) | -| 50 | VX_reduce_tree | Arithmetic | Small | O(log N) | -| 51 | VX_reset_relay | Sync | Small | 1 cycle | -| 52 | VX_rom | Memory | Varies | 0-1 cycles | -| 53 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | -| 54 | VX_scan | Encoder | Medium | O(log N) | -| 55 | VX_scope_switch | Debug | Small | 0 cycles | -| 56 | VX_scope_tap | Debug | Large | Multi-cycle | -| 57 | VX_serial_div | Arithmetic | Small | Multi-cycle | -| 58 | VX_serial_mul | Arithmetic | Small | Multi-cycle | -| 59 | VX_shift_register | Buffer | Varies | N cycles | -| 60 | VX_skid_buffer | Buffer | Small | 0-1 cycles | -| 61 | VX_sp_ram | Memory | Varies | 0-1 cycles | -| 62 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | -| 63 | VX_stream_buffer | Buffer | Small | 0-1 cycles | -| 64 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | -| 65 | VX_stream_fork | Stream Util | Small | 0 cycles | -| 66 | VX_stream_join | Stream Util | Small | 0 cycles | -| 67 | VX_stream_omega | Interconnect | Large | O(log N) | -| 68 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | -| 69 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | -| 70 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | -| 71 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | -| 72 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | -| 73 | VX_ticket_lock | Allocator | Small | 0-1 cycles | -| 74 | VX_toggle_buffer | Buffer | Small | 1 cycle | -| 75 | VX_transpose | Bit Manip | Zero | 0 cycles | -| 76 | VX_wallace_mul | Arithmetic | Large | O(log N) | +| 2 | VX_avs_adapter | Memory Adapter | Large | Multi-cycle | +| 3 | VX_axi_adapter | Memory Adapter | Large | Multi-cycle | +| 4 | VX_axi_write_ack | Memory Adapter | Small | 0-1 cycles | +| 5 | VX_bits_concat | Bit Manip | Zero | 0 cycles | +| 6 | VX_bits_insert | Bit Manip | Zero | 0 cycles | +| 7 | VX_bits_remove | Bit Manip | Zero | 0 cycles | +| 8 | VX_bypass_buffer | Buffer | Small | 0-1 cycles | +| 9 | VX_clockgate | Sync | Minimal | 0 cycles | +| 10 | VX_csa_32 | CSA | Small | 1 gate | +| 11 | VX_csa_42 | CSA | Medium | 2 gates | +| 12 | VX_csa_mod4 | CSA | Varies | O(log N) | +| 13 | VX_csa_tree | CSA | Varies | O(log N) | +| 14 | VX_cyclic_arbiter | Arbiter | Small | 0 cycles | +| 15 | VX_demux | Mux/Demux | Small | 0 cycles | +| 16 | VX_divider | Arithmetic | Large | Configurable | +| 17 | VX_dp_ram | Memory | Varies | 0-1 cycles | +| 18 | VX_edge_trigger | Sync | Minimal | 1 cycle | +| 19 | VX_elastic_adapter | Buffer | Minimal | 0-1 cycles | +| 20 | VX_elastic_buffer | Buffer | Varies | Configurable | +| 21 | VX_fifo_queue | Memory | Varies | 0-1 cycles | +| 22 | VX_find_first | Encoder | Small | O(log N) | +| 23 | VX_fold_mul | Arithmetic | Medium | Configurable | +| 24 | VX_generic_arbiter | Arbiter | Varies | 0 cycles | +| 25 | VX_gto_arbiter | Arbiter | Medium | 0 cycles | +| 26 | VX_index_buffer | Memory | Medium | 0-1 cycles | +| 27 | VX_index_queue | Memory | Medium | 0-1 cycles | +| 28 | VX_ks_adder | Arithmetic | Medium | O(log N) | +| 29 | VX_lzc | Encoder | Small | O(log N) | +| 30 | VX_matrix_arbiter | Arbiter | Medium | 0 cycles | +| 31 | VX_mem_bank_adapter | Memory Adapter | Large | Multi-cycle | +| 32 | VX_mem_coalescer | Memory Adapter | Large | Multi-cycle | +| 33 | VX_mem_data_adapter | Memory Adapter | Medium | Multi-cycle | +| 34 | VX_mem_scheduler | Scheduler | Large | Multi-cycle | +| 35 | VX_multiplier | Arithmetic | Varies | Configurable | +| 36 | VX_mux | Mux/Demux | Small | 0 cycles | +| 37 | VX_nz_iterator | Stream Util | Small | 0-1 cycles | +| 38 | VX_onehot_encoder | Encoder | Small | O(log N) | +| 39 | VX_onehot_mux | Mux/Demux | Small | 0 cycles | +| 40 | VX_onehot_shift | Bit Manip | Small | 0 cycles | +| 41 | VX_pe_serializer | Stream Util | Medium | Multi-cycle | +| 42 | VX_pending_size | Counter | Small | 0-1 cycles | +| 43 | VX_pipe_buffer | Buffer | Varies | N cycles | +| 44 | VX_pipe_register | Buffer | Varies | N cycles | +| 45 | VX_popcount | Arithmetic | Small | O(log N) | +| 46 | VX_priority_arbiter | Arbiter | Small | 0 cycles | +| 47 | VX_priority_encoder | Encoder | Small | O(log N) | +| 48 | VX_reduce_tree | Arithmetic | Small | O(log N) | +| 49 | VX_reset_relay | Sync | Small | 1 cycle | +| 50 | VX_rom | Memory | Varies | 0-1 cycles | +| 51 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | +| 52 | VX_scan | Encoder | Medium | O(log N) | +| 53 | VX_scope_switch | Debug | Small | 0 cycles | +| 54 | VX_scope_tap | Debug | Large | Multi-cycle | +| 55 | VX_serial_div | Arithmetic | Small | Multi-cycle | +| 56 | VX_serial_mul | Arithmetic | Small | Multi-cycle | +| 57 | VX_shift_register | Buffer | Varies | N cycles | +| 58 | VX_skid_buffer | Buffer | Small | 0-1 cycles | +| 59 | VX_sp_ram | Memory | Varies | 0-1 cycles | +| 60 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | +| 61 | VX_stream_buffer | Buffer | Small | 0-1 cycles | +| 62 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | +| 63 | VX_stream_fork | Stream Util | Small | 0 cycles | +| 64 | VX_stream_join | Stream Util | Small | 0 cycles | +| 65 | VX_stream_omega | Interconnect | Large | O(log N) | +| 66 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | +| 67 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | +| 68 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | +| 69 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | +| 70 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | +| 71 | VX_ticket_lock | Allocator | Small | 0-1 cycles | +| 72 | VX_toggle_buffer | Buffer | Small | 1 cycle | +| 73 | VX_transpose | Bit Manip | Zero | 0 cycles | +| 74 | VX_wallace_mul | Arithmetic | Large | O(log N) | diff --git a/docs/synthesis_analysis.md b/docs/synthesis_analysis.md index d497e0750f..d6ee632a22 100644 --- a/docs/synthesis_analysis.md +++ b/docs/synthesis_analysis.md @@ -434,7 +434,7 @@ Output reports are written to `build/hw/syn/yosys/tfr_fedp_VX_tcu_fedp_tfr/repor Build directory: `_/` (e.g., `test_Vortex/`). -The flow elaborates the SystemVerilog sources with CIRCT ([hw/scripts/sv_to_v.sh](../hw/scripts/sv_to_v.sh)) into one Verilog-2005 file for Yosys: packed arrays and structs become plain vectors, and the SRAM macros (`BB_MODULES`) stay instances with their parameter values, bound to their stubs. +The flow elaborates the SystemVerilog sources with CIRCT ([hw/scripts/circt_elaborate.sh](../hw/scripts/circt_elaborate.sh) `-f v2005`) into one Verilog-2005 file for Yosys: packed arrays and structs become plain vectors, the SRAM macros (`BB_MODULES`) stay instances with their parameter values, bound to their stubs, and a RAM that asks for block RAM reads synchronously, so that it maps to an SRAM macro. #### Per-DUT synthesis diff --git a/hw/rtl/VX_platform.vh b/hw/rtl/VX_platform.vh index 85fae1d635..5b76c4fab3 100644 --- a/hw/rtl/VX_platform.vh +++ b/hw/rtl/VX_platform.vh @@ -192,9 +192,6 @@ `define PRESERVE_NET (* keep = "true" *) `define BLACKBOX_CELL (* black_box *) `define STRING -`ifndef SIMULATION - `define ASYNC_BRAM_PATCH -`endif `elsif ASIC `define MAX_FANOUT 8 `define LATENCY_IMUL 3 diff --git a/hw/rtl/libs/VX_async_ram_patch.sv b/hw/rtl/libs/VX_async_ram_patch.sv deleted file mode 100644 index dd4d2b42d5..0000000000 --- a/hw/rtl/libs/VX_async_ram_patch.sv +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_platform.vh" - -`define RAM_INITIALIZATION \ - if (INIT_ENABLE != 0) begin : g_init \ - if (INIT_FILE != "") begin : g_file \ - initial $readmemh(INIT_FILE, ram); \ - end else begin : g_value \ - initial begin \ - for (integer i = 0; i < SIZE; ++i) begin : g_i \ - ram[i] = INIT_VALUE; \ - end \ - end \ - end \ - end - -`define SYNC_RAM_WF_BLOCK(__d, __re, __we, __ra, __wa) \ - `RAM_ATTRIBUTES `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - reg [ADDRW-1:0] raddr_r; \ - always @(posedge clk) begin \ - if (__we) begin \ - ram[__wa] <= wdata; \ - end \ - if (__re) begin \ - raddr_r <= __ra; \ - end \ - end \ - assign __d = ram[raddr_r] - -`define SYNC_RAM_WF_WREN_BLOCK(__d, __re, __we, __ra, __wa) \ - `RAM_ATTRIBUTES `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - reg [ADDRW-1:0] raddr_r; \ - always @(posedge clk) begin \ - if (__we) begin \ - for (integer i = 0; i < WRENW; ++i) begin \ - if (wren[i]) begin \ - ram[__wa][i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; \ - end \ - end \ - end \ - if (__re) begin \ - raddr_r <= __ra; \ - end \ - end \ - assign __d = ram[raddr_r] - -`define SYNC_RAM_RF_BLOCK(__d, __re, __we, __ra, __wa) \ - `RAM_ATTRIBUTES reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - reg [DATAW-1:0] rdata_r; \ - always @(posedge clk) begin \ - if (__we) begin \ - ram[__wa] <= wdata; \ - end \ - if (__re) begin \ - rdata_r <= ram[__ra]; \ - end \ - end \ - assign __d = rdata_r - -`define SYNC_RAM_RF_WREN_BLOCK(__d, __re, __we, __ra, __wa) \ - `RAM_ATTRIBUTES reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - reg [DATAW-1:0] rdata_r; \ - always @(posedge clk) begin \ - if (__we) begin \ - for (integer i = 0; i < WRENW; ++i) begin \ - if (wren[i]) begin \ - ram[__wa][i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; \ - end \ - end \ - end \ - if (__re) begin \ - rdata_r <= ram[__ra]; \ - end \ - end \ - assign __d = rdata_r - -`define ASYNC_RAM_BLOCK(__d, __we, __ra, __wa) \ - `RAM_ATTRIBUTES reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - always @(posedge clk) begin \ - if (__we) begin \ - ram[__wa] <= wdata; \ - end \ - end \ - assign __d = ram[__ra] - -`define ASYNC_RAM_BLOCK_WREN(__d, __we, __ra, __wa) \ - `RAM_ATTRIBUTES reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - always @(posedge clk) begin \ - if (__we) begin \ - for (integer i = 0; i < WRENW; ++i) begin \ - if (wren[i]) begin \ - ram[__wa][i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; \ - end \ - end \ - end \ - end \ - assign __d = ram[__ra] - -`TRACING_OFF -module VX_async_ram_patch #( - parameter DATAW = 1, - parameter SIZE = 1, - parameter WRENW = 1, - parameter DUAL_PORT = 0, - parameter FORCE_BRAM = 0, - parameter RADDR_REG = 0, // read address registered hint - parameter RADDR_RESET = 0, // read address has reset - parameter WRITE_FIRST = 0, - parameter INIT_ENABLE = 0, - parameter INIT_FILE = "", - parameter [DATAW-1:0] INIT_VALUE = 0, - parameter ADDRW = `LOG2UP(SIZE) -) ( - input wire clk, - input wire reset, - input wire read, - input wire write, - input wire [WRENW-1:0] wren, - input wire [ADDRW-1:0] waddr, - input wire [DATAW-1:0] wdata, - input wire [ADDRW-1:0] raddr, - output wire [DATAW-1:0] rdata -); - localparam WSELW = DATAW / WRENW; - - `UNUSED_VAR (reset) - - (* keep = "true" *) wire [ADDRW-1:0] raddr_w, raddr_s; - (* keep = "true" *) wire read_s; - assign raddr_w = raddr; - - wire raddr_reset_w; - if (RADDR_RESET) begin : g_raddr_reset - (* keep = "true" *) wire raddr_reset; - assign raddr_reset = 0; - assign raddr_reset_w = raddr_reset; - end else begin : g_no_raddr_reset - assign raddr_reset_w = 0; - end - - VX_placeholder #( - .I (ADDRW + 1), - .O (ADDRW + 1) - ) placeholder1 ( - .in ({raddr_w, raddr_reset_w}), - .out ({raddr_s, read_s}) - ); - - wire [DATAW-1:0] rdata_s; - - if (1) begin : g_sync_ram - if (WRENW != 1) begin : g_wren - if (FORCE_BRAM) begin : g_bram - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `USE_BLOCK_BRAM - `SYNC_RAM_WF_WREN_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `USE_BLOCK_BRAM - `SYNC_RAM_RF_WREN_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end - end else begin : g_lutram - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES - `SYNC_RAM_WF_WREN_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES - `SYNC_RAM_RF_WREN_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end - end - end else begin : g_no_wren - if (FORCE_BRAM) begin : g_bram - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `USE_BLOCK_BRAM - `SYNC_RAM_WF_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `USE_BLOCK_BRAM - `SYNC_RAM_RF_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end - end else begin : g_lutram - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES - `SYNC_RAM_WF_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES - `SYNC_RAM_RF_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end - end - end - end - - if (RADDR_REG) begin : g_raddr_reg - assign rdata = rdata_s; - end else begin : g_async_ram - (* keep = "true" *) wire is_raddr_reg; - VX_placeholder #( - .O (1) - ) placeholder2 ( - .in (1'b0), - .out (is_raddr_reg) - ); - wire [DATAW-1:0] rdata_a; - if (DUAL_PORT) begin : g_dp - if (WRENW != 1) begin : g_wren - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `RW_RAM_CHECK - `ASYNC_RAM_BLOCK_WREN(rdata_a, write, raddr, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `NO_RW_RAM_CHECK - `ASYNC_RAM_BLOCK_WREN(rdata_a, write, raddr, waddr); - `undef RAM_ATTRIBUTES - end - end else begin : g_no_wren - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `RW_RAM_CHECK - `ASYNC_RAM_BLOCK(rdata_a, write, raddr, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `NO_RW_RAM_CHECK - `ASYNC_RAM_BLOCK(rdata_a, write, raddr, waddr); - `undef RAM_ATTRIBUTES - end - end - end else begin : g_sp - if (WRENW != 1) begin : g_wren - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `RW_RAM_CHECK - `ASYNC_RAM_BLOCK_WREN(rdata_a, write, waddr, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `NO_RW_RAM_CHECK - `ASYNC_RAM_BLOCK_WREN(rdata_a, write, waddr, waddr); - `undef RAM_ATTRIBUTES - end - end else begin : g_no_wren - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `RW_RAM_CHECK - `ASYNC_RAM_BLOCK(rdata_a, write, waddr, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `NO_RW_RAM_CHECK - `ASYNC_RAM_BLOCK(rdata_a, write, waddr, waddr); - `undef RAM_ATTRIBUTES - end - end - end - assign rdata = is_raddr_reg ? rdata_s : rdata_a; - end - -endmodule -`TRACING_ON diff --git a/hw/rtl/libs/VX_dp_ram.sv b/hw/rtl/libs/VX_dp_ram.sv index ab9cafde29..9552c33f85 100644 --- a/hw/rtl/libs/VX_dp_ram.sv +++ b/hw/rtl/libs/VX_dp_ram.sv @@ -76,7 +76,6 @@ module VX_dp_ram #( parameter LUTRAM = 0, parameter `STRING RDW_MODE = "W", // W: write-first, R: read-first parameter RADDR_REG = 0, // read address registered hint - parameter RADDR_RESET = 0, // read address has reset parameter RDW_ASSERT = 0, parameter RESET_RAM = 0, parameter INIT_ENABLE = 0, @@ -97,7 +96,6 @@ module VX_dp_ram #( localparam WSELW = DATAW / WRENW; `UNUSED_PARAM (LUTRAM) `UNUSED_PARAM (RADDR_REG) - `UNUSED_PARAM (RADDR_RESET) `STATIC_ASSERT(!(WRENW * WSELW != DATAW), ("invalid parameter")) `STATIC_ASSERT((RDW_MODE == "R" || RDW_MODE == "W"), ("invalid parameter")) @@ -105,9 +103,12 @@ module VX_dp_ram #( `ifdef SYNTHESIS localparam FORCE_BRAM = !LUTRAM && `FORCE_BRAM(SIZE, DATAW); - // Async reads have no native block-RAM form, so forcing BRAM pulls in the - // async-RAM patch. That only pays off for a read-first RAM; a write-first one - // adds a read/write collision bypass that is larger than plain distributed RAM. + // Async reads have no native block-RAM form: a forced-BRAM RAM read + // asynchronously is tagged block RAM, and the synthesis elaboration makes its + // read synchronous, reading at the address its registers are about to take + // (CIRCT seq-retime-firmem-reads). That only pays off for a read-first RAM; a + // write-first one adds a read/write collision bypass that is larger than + // plain distributed RAM. localparam FORCE_BRAM_ASYNC = FORCE_BRAM && (OUT_REG == 0) && (RDW_MODE == "R"); `ifdef ASIC if (FORCE_BRAM && (OUT_REG != 0 || RADDR_REG != 0)) begin : g_asic @@ -288,31 +289,6 @@ module VX_dp_ram #( end else begin : g_async `UNUSED_VAR (read) if (FORCE_BRAM_ASYNC) begin : g_bram - `ifdef ASYNC_BRAM_PATCH - VX_async_ram_patch #( - .DATAW (DATAW), - .SIZE (SIZE), - .WRENW (WRENW), - .DUAL_PORT (1), - .FORCE_BRAM (FORCE_BRAM), - .RADDR_REG (RADDR_REG), - .RADDR_RESET(RADDR_RESET), - .WRITE_FIRST(RDW_MODE == "W"), - .INIT_ENABLE(INIT_ENABLE), - .INIT_FILE (INIT_FILE), - .INIT_VALUE (INIT_VALUE) - ) async_ram_patch ( - .clk (clk), - .reset (reset), - .read (read), - .write (write), - .wren (wren), - .waddr (waddr), - .wdata (wdata), - .raddr (raddr), - .rdata (rdata) - ); - `else if (RDW_MODE == "W") begin : g_write_first if (WRENW != 1) begin : g_wren `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN @@ -346,7 +322,6 @@ module VX_dp_ram #( assign rdata = ram[raddr]; end end - `endif end else begin : g_auto if (RDW_MODE == "W") begin : g_write_first if (WRENW != 1) begin : g_wren diff --git a/hw/rtl/libs/VX_fifo_queue.sv b/hw/rtl/libs/VX_fifo_queue.sv index 624ad5786b..22adee2f56 100644 --- a/hw/rtl/libs/VX_fifo_queue.sv +++ b/hw/rtl/libs/VX_fifo_queue.sv @@ -106,8 +106,7 @@ module VX_fifo_queue #( .OUT_REG (OUT_REG), .LUTRAM (LUTRAM), .RDW_MODE ("W"), - .RADDR_REG (OUT_REG == 0), - .RADDR_RESET (OUT_REG == 0) + .RADDR_REG (OUT_REG == 0) ) dp_ram ( .clk (clk), .reset (reset), diff --git a/hw/rtl/libs/VX_placeholder.sv b/hw/rtl/libs/VX_placeholder.sv deleted file mode 100644 index 7642bf353a..0000000000 --- a/hw/rtl/libs/VX_placeholder.sv +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_platform.vh" - -`TRACING_OFF -`BLACKBOX_CELL module VX_placeholder #( - parameter I = 0, - parameter O = 0 -) ( - input wire [`UP(I)-1:0] in, - output wire [`UP(O)-1:0] out -); - -endmodule -`TRACING_ON diff --git a/hw/rtl/libs/VX_sp_ram.sv b/hw/rtl/libs/VX_sp_ram.sv index 0d3694a636..f7275814fd 100644 --- a/hw/rtl/libs/VX_sp_ram.sv +++ b/hw/rtl/libs/VX_sp_ram.sv @@ -76,7 +76,6 @@ module VX_sp_ram #( parameter LUTRAM = 0, parameter `STRING RDW_MODE = "W", // W: write-first, R: read-first, N: no-change parameter RADDR_REG = 0, // read address registered hint - parameter RADDR_RESET = 0, // read address has reset parameter RDW_ASSERT = 0, parameter RESET_RAM = 0, parameter INIT_ENABLE = 0, @@ -96,7 +95,6 @@ module VX_sp_ram #( localparam WSELW = DATAW / WRENW; `UNUSED_PARAM (LUTRAM) `UNUSED_PARAM (RADDR_REG) - `UNUSED_PARAM (RADDR_RESET) `STATIC_ASSERT(!(WRENW * WSELW != DATAW), ("invalid parameter")) `STATIC_ASSERT((RDW_MODE == "R" || RDW_MODE == "W" || RDW_MODE == "N"), ("invalid parameter")) @@ -104,9 +102,12 @@ module VX_sp_ram #( `ifdef SYNTHESIS localparam FORCE_BRAM = !LUTRAM && `FORCE_BRAM(SIZE, DATAW); - // Async reads have no native block-RAM form, so forcing BRAM pulls in the - // async-RAM patch. That only pays off for a read-first RAM; a write-first one - // adds a read/write collision bypass that is larger than plain distributed RAM. + // Async reads have no native block-RAM form: a forced-BRAM RAM read + // asynchronously is tagged block RAM, and the synthesis elaboration makes its + // read synchronous, reading at the address its registers are about to take + // (CIRCT seq-retime-firmem-reads). That only pays off for a read-first RAM; a + // write-first one adds a read/write collision bypass that is larger than + // plain distributed RAM. localparam FORCE_BRAM_ASYNC = FORCE_BRAM && (OUT_REG == 0) && (RDW_MODE == "R"); `ifdef ASIC if (FORCE_BRAM && (OUT_REG != 0 || RADDR_REG != 0)) begin : g_asic @@ -370,31 +371,6 @@ module VX_sp_ram #( end else begin : g_async `UNUSED_VAR (read) if (FORCE_BRAM_ASYNC) begin : g_bram - `ifdef ASYNC_BRAM_PATCH - VX_async_ram_patch #( - .DATAW (DATAW), - .SIZE (SIZE), - .WRENW (WRENW), - .DUAL_PORT (0), - .FORCE_BRAM (FORCE_BRAM), - .RADDR_REG (RADDR_REG), - .RADDR_RESET(RADDR_RESET), - .WRITE_FIRST(RDW_MODE == "W"), - .INIT_ENABLE(INIT_ENABLE), - .INIT_FILE (INIT_FILE), - .INIT_VALUE (INIT_VALUE) - ) async_ram_patch ( - .clk (clk), - .reset (reset), - .read (read), - .write (write), - .wren (wren), - .waddr (addr), - .wdata (wdata), - .raddr (addr), - .rdata (rdata) - ); - `else if (RDW_MODE == "W") begin : g_write_first if (WRENW != 1) begin : g_wren `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN @@ -428,7 +404,6 @@ module VX_sp_ram #( assign rdata = ram[addr]; end end - `endif end else begin : g_auto if (RDW_MODE == "W") begin : g_write_first if (WRENW != 1) begin : g_wren diff --git a/hw/scripts/circt_elaborate.sh b/hw/scripts/circt_elaborate.sh new file mode 100755 index 0000000000..c0af5f2636 --- /dev/null +++ b/hw/scripts/circt_elaborate.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Elaborates SystemVerilog sources with CIRCT and writes the design as one +# file, the netlist every synthesis flow reads: +# +# - memories are detected, and a RAM that asks for block RAM with its +# synthesis attribute (ram_style = "block", ...) reads synchronously, so +# that it maps to a block RAM or an SRAM macro (seq-retime-firmem-reads). +# A request that cannot be honored stops the build with the reason. -m +# decides for RAMs without a hint: off (the default) leaves them, auto +# converts those it can. A report lists every asynchronous-read RAM; +# - black-box modules (vendor IP, SRAM macros) stay instances of +# themselves, with their parameter values, for the tool to bind. -x names +# a file of their declarations (ports only), read for their interfaces; +# -b names modules declared in the sources themselves; +# - a conditional write of one element of an array at a dynamic index +# becomes a select per element, as the statement means; +# - -f sv (the default) writes SystemVerilog, for Vivado and Quartus; -f +# v2005 writes Verilog-2005 for tools with a Verilog frontend (yosys): no +# packed arrays or structs, no local variables. +# +# The sources are a filelist (-l, as gen_sources.sh writes it: +define+, +# +incdir+ and files in compile order), or include directories whose sources +# are all read (-I). CIRCT_VERILOG, CIRCT_OPT and FIRTOOL name the tools +# (hw/syn/common.mk). + +CIRCT_VERILOG="${CIRCT_VERILOG:-circt-verilog}" +CIRCT_OPT="${CIRCT_OPT:-circt-opt}" +FIRTOOL="${FIRTOOL:-firtool}" + +set -eo pipefail + +top="" +filelist="" +format=sv +mem_mode=off +report="" +includes=() +defines=() +blackboxes=() +stubs=() +output_file=out.sv + +usage() { echo "$0 usage:" && grep " .)\ #" $0; exit 0; } +[ $# -eq 0 ] && usage +while getopts "t:l:o:I:D:b:x:f:m:r:h" arg; do + case $arg in + t) # top module + top=${OPTARG} + ;; + l) # filelist (+define+, +incdir+, files) + filelist=${OPTARG} + ;; + o) # output file + output_file=${OPTARG} + ;; + I) # include directory, whose sources are read too + includes+=("${OPTARG}") + ;; + D) # macro definition + defines+=("-D${OPTARG}") + ;; + b) # black-box modules declared in the sources, comma-separated + IFS=',' read -r -a names <<< "${OPTARG}" + for name in "${names[@]}"; do + [ -n "$name" ] && blackboxes+=("--blackbox=$name") + done + ;; + x) # file of black-box declarations (vendor IP) + stubs+=("${OPTARG}") + ;; + f) # output format: sv or v2005 + format=${OPTARG} + ;; + m) # RAMs without a hint: off or auto + mem_mode=${OPTARG} + ;; + r) # memory report (default: next to the output) + report=${OPTARG} + ;; + h | *) + usage + exit 0 + ;; + esac +done + +if [ -z "$top" ]; then + echo "circt_elaborate.sh: error: no top module (-t)" >&2 + exit 1 +fi +case "$format" in + sv|v2005) ;; + *) echo "circt_elaborate.sh: error: format must be sv or v2005, not '$format'" >&2 + exit 1 ;; +esac +[ -z "$report" ] && report="${output_file%.*}.memories.rpt" + +args=() +files=() +if [ -n "$filelist" ]; then + while IFS= read -r line; do + case "$line" in + "") ;; + +define+*) args+=("-D${line#+define+}") ;; + +incdir+*) args+=("-I${line#+incdir+}") ;; + *) files+=("$line") ;; + esac + done < "$filelist" +fi +for dir in "${includes[@]}"; do + args+=("-I$dir") + while IFS= read -r file; do + files+=("$file") + done < <(find "$dir" -maxdepth 1 -type f \( -name '*.v' -o -name '*.sv' \) | sort) +done +for stub in "${stubs[@]}"; do + files+=("$stub") + while IFS= read -r name; do + blackboxes+=("--blackbox=$name") + done < <(sed -nE 's/^[[:space:]]*module[[:space:]]+([A-Za-z_][A-Za-z0-9_$]*).*/\1/p' "$stub") +done + +stem="${output_file%.*}" +"$CIRCT_VERILOG" --ir-hw --top="$top" --detect-memories --allow-use-before-declare \ + --allow-hierarchical-const --ignore-uninstantiated-modules "${blackboxes[@]}" \ + "${defines[@]}" "${args[@]}" "${files[@]}" -o "$stem.mlir" + +# Memories are detected on import; what remains of a conditional write at a +# dynamic index (`if (c) x[i] = v`) is expanded into per-element selects. +passes=(--hw-expand-array-writes "--seq-retime-firmem-reads=mode=$mem_mode report=$report") +if [ "$format" = v2005 ]; then + passes+=(--seq-lower-aggregate-regs --hw-aggregate-ports-to-int --hw-convert-bitcasts + --hw-aggregate-to-comb) + lowering=noAlwaysComb,disallowLocalVariables,disallowPackedArrays,disallowPackedStructAssignments,svVarDeclStyle=reg,locationInfoStyle=none,omitVersionComment +else + lowering=locationInfoStyle=none,omitVersionComment +fi +# A module body is a graph region, where CSE cannot replace a duplicate whose +# users come first; sorting the ops first lets it merge every duplicate. +"$CIRCT_OPT" "$stem.mlir" "${passes[@]}" --canonicalize --topological-sort --cse \ + -o "$stem.lowered.mlir" +"$FIRTOOL" --format=mlir "$stem.lowered.mlir" --disable-all-randomization \ + --lowering-options="$lowering" -o "$output_file" diff --git a/hw/scripts/gen_sources.sh b/hw/scripts/gen_sources.sh index a8514246cb..71d68a8e6d 100755 --- a/hw/scripts/gen_sources.sh +++ b/hw/scripts/gen_sources.sh @@ -33,6 +33,8 @@ define_header="" top_module="" copy_folder="" preprocessor=0 +elaborate=0 +stubs=() defines_str="" params_str="" @@ -48,7 +50,7 @@ add_option() { } # Parse command arguments -while getopts D:G:T:I:J:O:H:C:Ph flag +while getopts D:G:T:I:J:O:H:C:X:PEh flag do case "${flag}" in D) defines+=( ${OPTARG} ) @@ -72,7 +74,11 @@ do ;; P) preprocessor=1 ;; - h) echo "Usage: [-D] [-G=] [-T] [-I] [-J] [-O] [-C: copy to] [-H] [-P: macro preprocessing] [-h help] [files...]" + E) elaborate=1 + ;; + X) stubs+=( "${OPTARG}" ) + ;; + h) echo "Usage: [-D] [-G=] [-T] [-I] [-J] [-O] [-C: copy to] [-H] [-P: macro preprocessing] [-E: elaborate into one netlist] [-X] [-h help] [files...]" exit 0 ;; \?) echo "Invalid option: -$OPTARG" 1>&2 @@ -268,4 +274,25 @@ if [ "$output_file" != "" ]; then done fi } > "$output_file" -fi \ No newline at end of file +fi + +# Optional elaboration (-E): CIRCT elaborates the listed sources into the one +# netlist synthesis reads (circt_elaborate.sh), and the filelist names it. The +# RTL filelist stays next to it as .rtl.f; the memory report as +# .memories.rpt. -X files declare the black boxes (vendor IP) the +# synthesis tool binds. +if [ "$elaborate" != 0 ]; then + if [ -z "$top_module" ] || [ -z "$output_file" ]; then + echo "gen_sources.sh: error: -E needs a top module (-T) and a filelist (-O)" >&2 + exit 1 + fi + stem="${output_file%.*}" + mv "$output_file" "$stem.rtl.f" + stub_args=() + for stub in "${stubs[@]}"; do + stub_args+=( -x "$(realpath "$stub")" ) + done + "$SCRIPT_DIR"/circt_elaborate.sh -t "$top_module" -l "$stem.rtl.f" "${stub_args[@]}" \ + -r "$stem.memories.rpt" -o "$stem.elab.sv" + realpath "$stem.elab.sv" > "$output_file" +fi diff --git a/hw/scripts/sv_to_v.sh b/hw/scripts/sv_to_v.sh deleted file mode 100755 index bd1cbab9b5..0000000000 --- a/hw/scripts/sv_to_v.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -# Copyright © 2019-2023 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Elaborates SystemVerilog sources with CIRCT and writes the design as one -# Verilog-2005 file, for synthesis tools with a Verilog frontend (yosys): -# no packed arrays or structs (ports, registers and logic become plain -# vectors), no local variables, and black-box modules left as instances of -# themselves with their parameter values, for the tool to bind. -# CIRCT_VERILOG, CIRCT_OPT and FIRTOOL name the tools (hw/syn/common.mk). - -CIRCT_VERILOG="${CIRCT_VERILOG:-circt-verilog}" -CIRCT_OPT="${CIRCT_OPT:-circt-opt}" -FIRTOOL="${FIRTOOL:-firtool}" - -set -eo pipefail - -top="" -source="" -includes=() -defines=() -blackboxes=() -output_file=out.v - -usage() { echo "$0 usage:" && grep " .)\ #" $0; exit 0; } -[ $# -eq 0 ] && usage -while getopts "t:s:o:I:D:b:h" arg; do - case $arg in - t) # top module - top=${OPTARG} - ;; - s) # source - source=${OPTARG} - ;; - o) # output-file - output_file=${OPTARG} - ;; - I) # include directory, whose sources are read too - includes+=("${OPTARG}") - ;; - D) # macro definition - defines+=("-D${OPTARG}") - ;; - b) # black-box modules, comma-separated - IFS=',' read -r -a names <<< "${OPTARG}" - for name in "${names[@]}"; do - [ -n "$name" ] && blackboxes+=("--blackbox=$name") - done - ;; - h | *) - usage - exit 0 - ;; - esac -done - -if [ -z "$top" ]; then - echo "sv_to_v.sh: error: no top module (-t)" >&2 - exit 1 -fi - -inc_args=() -files=() -[ -n "$source" ] && files+=("$source") -for dir in "${includes[@]}"; do - inc_args+=("-I$dir") - while IFS= read -r file; do - files+=("$file") - done < <(find "$dir" -maxdepth 1 -type f \( -name '*.v' -o -name '*.sv' \) | sort) -done - -ir="${output_file%.*}.mlir" -lowered="${output_file%.*}.lowered.mlir" -"$CIRCT_VERILOG" --ir-hw --top="$top" --detect-memories --allow-use-before-declare \ - --ignore-uninstantiated-modules "${blackboxes[@]}" "${defines[@]}" "${inc_args[@]}" \ - "${files[@]}" -o "$ir" -"$CIRCT_OPT" "$ir" --seq-lower-aggregate-regs --hw-aggregate-ports-to-int \ - --hw-convert-bitcasts --hw-aggregate-to-comb --canonicalize --cse -o "$lowered" -"$FIRTOOL" --format=mlir "$lowered" --disable-all-randomization \ - --lowering-options=noAlwaysComb,disallowLocalVariables,disallowPackedArrays,disallowPackedStructAssignments,svVarDeclStyle=reg,locationInfoStyle=none,omitVersionComment \ - -o "$output_file" diff --git a/hw/scripts/xilinx_async_bram_patch.tcl b/hw/scripts/xilinx_async_bram_patch.tcl deleted file mode 100644 index f145035899..0000000000 --- a/hw/scripts/xilinx_async_bram_patch.tcl +++ /dev/null @@ -1,563 +0,0 @@ -# Copyright © 2019-2023 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -namespace eval vortex { - -variable debug 0 -variable start_time [clock seconds] -variable cpu_start_time [clock clicks -milliseconds] - -# --- Standardized Logging Wrappers --- - -proc print_msg {level msg} { - variable debug - set prefix "\[AsyncBRAM Patch\]" - switch $level { - "ERROR" { puts "ERROR: $prefix $msg" } - "CRITICAL" { puts "CRITICAL WARNING: $prefix $msg" } - "WARNING" { puts "WARNING: $prefix $msg" } - "INFO" { puts "INFO: $prefix $msg" } - "DEBUG" { if {$debug} { puts "DEBUG: $prefix $msg" } } - } -} - -proc log_error {msg {exit_code -1}} { - print_msg "ERROR" $msg - if {$exit_code != 0} { exit $exit_code } -} - -proc log_critical {msg} { - print_msg "CRITICAL" $msg -} - -proc log_warning {msg} { - print_msg "WARNING" $msg -} - -proc log_info {msg} { - print_msg "INFO" $msg -} - -proc log_debug {msg} { - print_msg "DEBUG" $msg -} - -# --- Time Formatting Utils --- - -proc format_time_val {total_seconds} { - set s [expr {int($total_seconds)}] - set ms [expr {int(($total_seconds - $s) * 100)}] - set h [expr {$s / 3600}] - set m [expr {($s % 3600) / 60}] - set s [expr {$s % 60}] - - if {$ms > 0} { - return [format "%02d:%02d:%02d.%02d" $h $m $s $ms] - } else { - return [format "%02d:%02d:%02d" $h $m $s] - } -} - -proc log_time {} { - variable start_time - variable cpu_start_time - - set now [clock seconds] - set cpu_now [clock clicks -milliseconds] - - set elapsed_s [expr {$now - $start_time}] - set cpu_s [expr {($cpu_now - $cpu_start_time) / 1000.0}] - - set cpu_fmt [format_time_val $cpu_s] - set elapsed_fmt [format_time_val $elapsed_s] - - puts "INFO: \[AsyncBRAM Patch\] Time (s): cpu = $cpu_fmt ; elapsed = $elapsed_fmt ." -} - -# --- String & Regex Utilities --- - -proc str_replace {str match repl} { - set result "" - regsub $match $str $repl result - return $result -} - -proc regex_escape {str} { - return [string map { - \\ \\\\ - ^ \\^ - . \\. - \[ \\\[ - \] \\\] - \$ \\\$ - \( \\\( - \) \\\) - | \\| - * \\* - + \\+ - ? \\? - \{ \\\{ - \} \\\} - } $str] -} - -proc unique_cell_name {name} { - if {[get_cells -quiet $name] == {}} { return $name } - set index 0 - while {[get_cells -quiet ${name}_${index}] != {}} { incr index } - return ${name}_${index} -} - -proc unique_net_name {name} { - if {[get_nets -quiet $name] == {}} { return $name } - set index 0 - while {[get_nets -quiet ${name}_${index}] != {}} { incr index } - return ${name}_${index} -} - -# --- Net Finding Utilities --- - -# Fast retrieval of the nets one level inside $cell (i.e. PARENT_CELL == $cell). -# Scoping with current_instance is O(nets-in-cell); the equivalent -# 'get_nets -hierarchical -filter {PARENT_CELL == $cell}' rescans the entire -# design's net database on every call (O(total_nets)), which dominated runtime -# on large multi-core designs. The NAME property is still the full hierarchical -# name, so all downstream regexp/name matching is unchanged. -proc cell_nets {cell} { - current_instance -quiet $cell - set nets [get_nets -quiet *] - current_instance -quiet - return $nets -} - -proc find_cell_nets {cell name_match {required 1}} { - set matching_nets {} - foreach net [cell_nets $cell] { - set name [get_property NAME $net] - if {[regexp $name_match $name]} { - lappend matching_nets $net - } - } - - if {[llength $matching_nets] == 0 && $required} { - log_error "No matching net found for '$cell' matching '$name_match'." - } - return $matching_nets -} - -proc find_cell_net {cell name_match {required 1}} { - set nets [find_cell_nets $cell $name_match $required] - - if {[llength $nets] == 0} { - return "" - } elseif {[llength $nets] > 1} { - log_error "Multiple matching nets found for '$cell' matching '$name_match'." - } - return [lindex $nets 0] -} - -proc get_cell_net {cell name} { - set net {} - foreach n [cell_nets $cell] { - if {[get_property NAME $n] == $name} { set net $n; break } - } - if {[llength $net] == 0} { - log_error "No matching net found for '$cell' matching '$name'." - } - return $net -} - -proc get_cell_pin {cell name} { - set pin [get_pins -of_objects $cell -filter "NAME == $name"] - if {[llength $pin] == 0} { - log_error "No matching pin found for '$cell' matching '$name'." - } - return $pin -} - -# --- Modification Utilities --- - -proc remove_cell_from_netlist {cell} { - foreach pin [get_pins -quiet -of_objects $cell] { - foreach net [get_nets -quiet -of_objects $pin] { - disconnect_net -net $net -objects $pin - log_debug "Disconnected net '$net' from pin '$pin'." - } - } - remove_cell $cell - log_debug "Cell '$cell' was removed successfully." -} - -proc find_net_driver {target_net {required 1}} { - set driverPins [get_pins -quiet -leaf -of_objects $target_net -filter {DIRECTION == "OUT"}] - - if {[llength $driverPins] == 0} { - set driverPorts [get_ports -quiet -of_objects $target_net -filter {DIRECTION == "IN"}] - - if {[llength $driverPorts] == 0} { - if {$required} { log_error "No driver found for '$target_net'." } - return "" - } elseif {[llength $driverPorts] > 1} { - log_warning "Multiple driver ports found for '$target_net'. Using first." - return [lindex $driverPorts 0] - } - return $driverPorts - } elseif {[llength $driverPins] > 1} { - log_warning "Multiple driver pins found for '$target_net'. Using first." - return [lindex $driverPins 0] - } - return $driverPins -} - -proc find_pin_driver {target_pin {required 1}} { - set net [get_nets -quiet -of_objects $target_pin] - - if {[llength $net] == 0} { - if {$required} { log_error "No net connected to pin '$target_pin'." } - return "" - } elseif {[llength $net] > 1} { - log_error "Multiple nets connected to pin '$target_pin'." - } - return [find_net_driver $net] -} - -# --- Core Logic --- - -proc create_register_next {parent reg_cell raddr_reset} { - set hier_sep [get_hierarchy_separator] - - set reg_d_pin [get_pins "${reg_cell}${hier_sep}D"] - if {[llength $reg_d_pin] != 1} { - log_error "Found [llength $reg_d_pin] D pins on register cell '$reg_cell'. Expected 1." - } - - log_debug "reg_d_pin: '$reg_d_pin'" - - set reg_d_src_pin [find_pin_driver $reg_d_pin] - log_debug "reg_d_src_pin: '$reg_d_src_pin'" - - # [Optimization] 1. Check if reset is requested by the patch instance - if {$raddr_reset == ""} { - log_debug "No reset requested by patch. Skipping LUT." - return $reg_d_src_pin - } - - set register_type [get_property REF_NAME $reg_cell] - set reg_r_pin "" - set lut_init "" - - if {$register_type == "FDRE"} { - set reg_r_pin [get_pins -quiet "${reg_cell}${hier_sep}R"] - set lut_init "4'b0010" ;# Reset dominates - } elseif {$register_type == "FDSE"} { - set reg_r_pin [get_pins -quiet "${reg_cell}${hier_sep}S"] - set lut_init "4'b1110" ;# Set dominates - } else { - log_error "Unsupported register type: '$register_type'." - } - - if {[llength $reg_r_pin] == 0} { - log_error "No Reset/Set pin found on $register_type cell '$reg_cell'." - } - - set reg_r_src_pin [find_pin_driver $reg_r_pin 0] - - # [Optimization] 2. Check if the register's reset pin is unconnected or grounded - if {$reg_r_src_pin == ""} { - log_debug "Reset pin unconnected. Optimization: Skipping LUT." - return $reg_d_src_pin - } else { - set driver_cell [get_cells -quiet -of_objects $reg_r_src_pin] - if {[llength $driver_cell] > 0} { - set driver_ref [get_property REF_NAME $driver_cell] - if {$driver_ref == "GND"} { - log_debug "Reset driven by GND. Optimization: Skipping LUT." - return $reg_d_src_pin - } - } - } - - log_debug "reg_r_src_pin: '$reg_r_src_pin'" - - set reg_d_src_net [get_nets -of_objects $reg_d_src_pin] - set reg_r_src_net [get_nets -of_objects $reg_r_src_pin] - - # Create LUT2 MUX - set lut_name [unique_cell_name "${parent}${hier_sep}raddr_next"] - set lut_cell [create_cell -reference LUT2 $lut_name] - set_property INIT $lut_init $lut_cell - - log_debug "Created LUT cell: '$lut_cell' (Init: $lut_init)" - - set lut_i0_pin [get_pins "${lut_cell}${hier_sep}I0"] - set lut_i1_pin [get_pins "${lut_cell}${hier_sep}I1"] - set lut_o_pin [get_pins "${lut_cell}${hier_sep}O"] - - connect_net -net $reg_d_src_net -objects $lut_i0_pin -hierarchical - connect_net -net $reg_r_src_net -objects $lut_i1_pin -hierarchical - - return $lut_o_pin -} - -proc getOrCreateVCCPin {parent} { - set hier_sep [get_hierarchy_separator] - set cell_name "${parent}${hier_sep}VCC" - - set vcc_cell [get_cells -quiet $cell_name] - if {[llength $vcc_cell] == 0} { - set vcc_cell [create_cell -reference VCC $cell_name] - log_debug "Created VCC cell: '$vcc_cell'" - } - - set vcc_pin [get_pins "${vcc_cell}${hier_sep}P"] - if {[llength $vcc_pin] == 0} { - log_error "No VCC pin found on VCC cell '$vcc_cell'." - } - return $vcc_pin -} - -proc getOrCreateGNDPin {parent} { - set hier_sep [get_hierarchy_separator] - set cell_name "${parent}${hier_sep}GND" - - set gnd_cell [get_cells -quiet $cell_name] - if {[llength $gnd_cell] == 0} { - set gnd_cell [create_cell -reference GND $cell_name] - log_debug "Created GND cell: '$gnd_cell'" - } - - set gnd_pin [get_pins "${gnd_cell}${hier_sep}G"] - if {[llength $gnd_pin] == 0} { - log_error "No GND pin found on GND cell '$gnd_cell'." - } - return $gnd_pin -} - -proc find_net_sinks {source_net {required 1}} { - set sink_pins {} - - foreach pin [get_pins -quiet -of_objects $source_net] { - set direction [get_property DIRECTION $pin] - # Input pins of nested cells - if {$direction == "IN"} { - lappend sink_pins $pin - } - # Output pins of the parent cell - set pin_cell [get_cells -of_objects $pin] - set is_primitive [get_property IS_PRIMITIVE $pin_cell] - if {$direction == "OUT" && !$is_primitive} { - lappend sink_pins $pin - } - } - - foreach port [get_ports -quiet -of_objects $source_net -filter {DIRECTION == "OUT"}] { - lappend sink_pins $port - } - - if {[llength $sink_pins] == 0 && $required} { - log_error "No sink found for '$source_net'." - } - return $sink_pins -} - -proc find_matching_nets {cell nets match repl} { - set matching_nets {} - foreach net $nets { - set net_name [str_replace $net $match $repl] - set matching_net [get_cell_net $cell $net_name] - if {$matching_net != ""} { - lappend matching_nets $matching_net - } - } - - if {[llength $matching_nets] != [llength $nets]} { - log_error "Mismatch in number of matching nets. Found [llength $matching_nets], expected [llength $nets]." - } - return $matching_nets -} - -proc replace_net_source {net source_pin} { - foreach pin [find_net_sinks $net 0] { - # 1. Disconnect - disconnect_net -net $net -objects $pin - log_debug "Disconnected net '$net' from pin '$pin'." - - # 2. Find/Create Source Net - set source_net [get_nets -quiet -of_objects $source_pin] - - if {[llength $source_net] == 0} { - set source_cell [get_cells -of_objects $source_pin] - set net_name [unique_net_name "${source_cell}_tmp_net"] - set source_net [create_net $net_name] - connect_net -net $source_net -objects $source_pin -hierarchical - log_debug "Created new source_net '$source_net' and connected to '$source_pin'." - } elseif {[llength $source_net] > 1} { - log_error "Multiple nets connected to pin '$source_pin'." - } - - # 3. Connect Sink - set external_net [get_nets -quiet -of_objects $pin] - - if {[llength $external_net] == 0} { - connect_net -net $source_net -objects $pin -hierarchical - log_debug "Connected net '$source_net' to pin '$pin'." - } elseif {[llength $external_net] == 1} { - foreach external_pin [get_pins -of_objects $external_net] { - disconnect_net -net $external_net -objects $pin - replace_net_source $external_net $source_pin - } - } else { - log_error "Multiple nets connected to pin '$pin'." - } - } -} - -# --- Main Patching Routine --- - -proc resolve_async_bram {inst} { - log_info "Resolving asynchronous BRAM patch: '$inst'." - - set hier_sep [get_hierarchy_separator] - - # 1. Find Nets - set raddr_w_nets [find_cell_nets $inst "raddr_w(\\\[\\d+\\\])?$"] - set raddr_s_nets [find_matching_nets $inst $raddr_w_nets "raddr_w(\\\[\\d+\\\])?$" "raddr_s\\1"] - - set read_s_net [find_cell_net $inst "read_s$"] - log_debug "read_s_net: '$read_s_net'" - - set is_raddr_reg_net [find_cell_net $inst "g_async_ram.is_raddr_reg$" 0] - set raddr_reset_net [find_cell_net $inst "raddr_reset$" 0] - log_debug "raddr_reset: '$raddr_reset_net'" - - set reg_next_pins {} - set reg_ce_src_pin "" - - # 2. Analyze Drivers & Create Logic - foreach raddr_w_net $raddr_w_nets { - log_debug "Processing raddr_w net: '$raddr_w_net'" - - set raddr_src_pin [find_net_driver $raddr_w_net] - - if {[get_ports -quiet $raddr_src_pin] ne ""} { - log_warning "Net '$raddr_w_net' driven by port. Skipping patch for this instance." - break - } - - set raddr_src_cell [get_cells -of_objects $raddr_src_pin] - set driver_type [get_property REF_NAME $raddr_src_cell] - - if {$driver_type != "FDRE" && $driver_type != "FDSE"} { - log_warning "Net '$raddr_w_net' driven by '$driver_type' (not FDRE/FDSE). Skipping." - break - } - - # Create Next Logic (with optimization) - set reg_next_pin [create_register_next $inst $raddr_src_cell $raddr_reset_net] - if {$reg_next_pin == ""} { - log_error "Failed to create register next value for '$raddr_src_cell'." - } - - lappend reg_next_pins $reg_next_pin - - # Capture CE Pin (once) - if {$reg_ce_src_pin == ""} { - set reg_ce_pin [get_pins "${raddr_src_cell}${hier_sep}CE"] - if {[llength $reg_ce_pin] != 1} { - log_error "Expected 1 CE pin on '$raddr_src_cell', found [llength $reg_ce_pin]." - } - set reg_ce_src_pin [find_pin_driver $reg_ce_pin] - log_debug "reg_ce_src_pin: '$reg_ce_src_pin'" - } - } - - set addr_width [llength $raddr_w_nets] - - # 3. Apply Patch or Fallback - if {[llength $reg_next_pins] == $addr_width} { - log_debug "Fully registered read address detected. Patching..." - - # Connect Address - for {set i 0} {$i < $addr_width} {incr i} { - replace_net_source [lindex $raddr_s_nets $i] [lindex $reg_next_pins $i] - } - - # Connect CE - replace_net_source $read_s_net $reg_ce_src_pin - - # Connect Const 1 (Valid Patch) - if {$is_raddr_reg_net != ""} { - set vcc_pin [getOrCreateVCCPin $inst] - replace_net_source $is_raddr_reg_net $vcc_pin - } - } else { - log_critical "Read address not fully registered. Falling back to Asynchronous (Grounding)." - - set gnd_pin [getOrCreateGNDPin $inst] - - # Ground Address & Control - foreach net $raddr_s_nets { replace_net_source $net $gnd_pin } - replace_net_source $read_s_net $gnd_pin - - # Connect Const 0 (Invalid Patch) - if {$is_raddr_reg_net != ""} { - replace_net_source $is_raddr_reg_net $gnd_pin - } - } - - # 4. Cleanup (Optimized for Speed) - remove_cell_from_netlist [get_cells -quiet "${inst}${hier_sep}placeholder1"] - if {$is_raddr_reg_net != ""} { - remove_cell_from_netlist [get_cells -quiet "${inst}${hier_sep}*placeholder2*"] - } -} - -proc resolve_async_brams {} { - variable debug - set bram_patch_cells [get_cells -hierarchical -filter {REF_NAME =~ "*VX_async_ram_patch*"}] - if {[llength $bram_patch_cells] != 0} { - foreach cell $bram_patch_cells { - resolve_async_bram $cell - } - } else { - log_info "No async BRAM patch cells found in the design." - } -} - -proc dump_cell_hierarchy {parent_cell indent} { - set children [get_cells -quiet -hierarchical -filter "PARENT_CELL == $parent_cell"] - foreach child $children { - set type [get_property REF_NAME $child] - log_info "${indent}child cell: '$child', type: '$type'" - dump_cell_hierarchy $child "${indent} " - } -} - -proc dump_async_bram_cells {} { - set bram_patch_cells [get_cells -hierarchical -filter {REF_NAME =~ "*VX_async_ram_patch*"}] - if {[llength $bram_patch_cells] != 0} { - foreach cell $bram_patch_cells { - log_info "Found async BRAM patch cell: '$cell'." - dump_cell_hierarchy $cell " " - } - } else { - log_info "No async BRAM patch cells found in the design." - } -} - -} - -# Run -vortex::resolve_async_brams -# vortex::dump_async_bram_cells -vortex::log_time \ No newline at end of file diff --git a/hw/scripts/xilinx_ip_stubs.sv b/hw/scripts/xilinx_ip_stubs.sv new file mode 100644 index 0000000000..f1ecb14186 --- /dev/null +++ b/hw/scripts/xilinx_ip_stubs.sv @@ -0,0 +1,88 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Black-box declarations of the Xilinx floating-point cores that +// xilinx_ip_gen.tcl generates: their ports as configured there, no contents. +// The CIRCT elaboration reads them to know each core's interface and keeps +// every instance as an instance; Vivado binds it to the generated core. +// Synthesis never reads this file. Keep it in step with xilinx_ip_gen.tcl. + +(* black_box *) +module xil_fdiv ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [3:0] m_axis_result_tuser // underflow, overflow, invalid, divide by zero +); +endmodule + +(* black_box *) +module xil_fsqrt ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [0:0] m_axis_result_tuser // invalid +); +endmodule + +(* black_box *) +module xil_fma ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + input wire s_axis_c_tvalid, + input wire [31:0] s_axis_c_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [2:0] m_axis_result_tuser // underflow, overflow, invalid +); +endmodule + +(* black_box *) +module xil_fmul ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata +); +endmodule + +(* black_box *) +module xil_fadd ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + input wire s_axis_operation_tvalid, + input wire [7:0] s_axis_operation_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata +); +endmodule diff --git a/hw/scripts/xilinx_slr_pblocks.tcl b/hw/scripts/xilinx_slr_pblocks.tcl index 3001d908a5..3bd5540103 100644 --- a/hw/scripts/xilinx_slr_pblocks.tcl +++ b/hw/scripts/xilinx_slr_pblocks.tcl @@ -186,7 +186,10 @@ proc slr_pblocks_run {} { # Per-cluster reconciliation: everything sizeable inside a cluster must be # captured by an atom or a cluster cache. If it is not, the atom model is # stale (RTL changed) and we must NOT floorplan on a partial view. - set clusters [cells_re {.*/g_clusters\[[0-9]+\]\.cluster}] + # + # A generate-loop instance is g_x[i].inst read from the RTL, and g_x_i_inst + # read from the CIRCT netlist, which names generate scopes with `_`. + set clusters [cells_re {.*/g_clusters(\[[0-9]+\]\.|_[0-9]+_)cluster}] if {[llength $clusters] == 0} { # Non-clustered top (single cluster inlined): treat vortex as one region. set clusters [cells_re {.*/(Vortex|vortex)$}] @@ -199,9 +202,9 @@ proc slr_pblocks_run {} { set acc 0 # sockets (cores + private L1s), whole. - foreach s [cells_re "[slr_re_escape $clp]/g_sockets\\\[\[0-9\]+\\\]\\.socket"] { + foreach s [cells_re "[slr_re_escape $clp]/g_sockets(\\\[\[0-9\]+\\\]\\.|_\[0-9\]+_)socket"] { set g {} - foreach c [cells_re "[slr_re_escape [get_property NAME $s]]/g_cores\\\[\[0-9\]+\\\]\\.core"] { lappend g $c } + foreach c [cells_re "[slr_re_escape [get_property NAME $s]]/g_cores(\\\[\[0-9\]+\\\]\\.|_\[0-9\]+_)core"] { lappend g $c } foreach c [cells_re "[slr_re_escape [get_property NAME $s]]/(icache|dcache)"] { lappend g $c } if {[llength $g] == 0} { set g [list $s] } set f {0 0 0 0} @@ -212,10 +215,10 @@ proc slr_pblocks_run {} { # extension core BUNDLED with its private cache. foreach {corere cachere} { - {.*\.tex_core} {.*/tcache} - {.*\.raster_core} {.*/rcache} - {.*\.om_core} {.*/ocache} - {.*\.rtu_core} {.*/rtcache} + {.*[._]tex_core} {.*/tcache} + {.*[._]raster_core} {.*/rcache} + {.*[._]om_core} {.*/ocache} + {.*[._]rtu_core} {.*/rtcache} {.*/dxa_core} {} } { foreach core [get_cells -quiet -hierarchical -regexp $corere] { diff --git a/hw/syn/common.mk b/hw/syn/common.mk index 84604150da..7ade2a9809 100644 --- a/hw/syn/common.mk +++ b/hw/syn/common.mk @@ -23,7 +23,8 @@ endif # Tool install prefixes (overridable; default under $(TOOLDIR)). CIRCT is the # SystemVerilog frontend of every flow: its preprocessor (gen_sources.sh -P) -# and, for yosys, its Verilog-2005 export (sv_to_v.sh). +# and its elaboration into the one netlist a synthesis tool reads +# (circt_elaborate.sh; gen_sources.sh -E). CIRCT_PATH ?= $(TOOLDIR)/circt YOSYS_PATH ?= $(TOOLDIR)/yosys STA_PATH ?= $(TOOLDIR)/sta diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index b137e4e427..7ebfdac3c5 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -117,17 +117,15 @@ FIRESIM_TARGET_ARGS := PLATFORM=$(FIRESIM_PLATFORM) TARGET_PROJECT=midasexamples DESIGN=VortexTarget TARGET_CONFIG=$(VORTEX_TARGET_CONFIG) PLATFORM_CONFIG=$(PLATFORM_CONFIG) QUINTUPLET := $(FIRESIM_PLATFORM)-midasexamples-VortexTarget-$(VORTEX_TARGET_CONFIG)-$(PLATFORM_CONFIG) -# Flattened Vortex sources for the BlackBox. Golden Gate takes a file list, not -# include paths, so the tree is copied into one directory with the macros already -# inlined (-P) and emitted in elaboration order: packages, interfaces, modules. +# Vortex for the BlackBox. Golden Gate takes a file list, not include paths, so +# the tree is copied into one directory with the macros already inlined (-P) and +# emitted in elaboration order: packages, interfaces, modules. CIRCT then +# elaborates it into the one netlist the image reads, as every synthesis flow +# does (circt_elaborate.sh); the list names that netlist, and the RTL list stays +# next to it as $(VIVADO_SRCS_NAME).rtl.f. # # A simulation image (TARGET=sim or hw_emu) gets its own flattening, under -# -DSIMULATION. Under VIVADO, VX_platform.vh enables ASYNC_BRAM_PATCH unless -# SIMULATION is defined, and that patch leaves every async RAM's read address -# behind an empty VX_placeholder black box which only the opt_design hook -# rewires. A behavioural simulation never runs opt_design, so without the -# define those RAMs never latch an address and read back X -- the same reason -# the native xrt flow adds -DSIMULATION for hw_emu. +# -DSIMULATION, as the native xrt flow does for hw_emu. ifneq ($(filter $(TARGET),sim hw_emu),) IMAGE_KIND := sim IMAGE_SUFFIX := _sim @@ -379,7 +377,11 @@ $(VIVADO_SRCS_LIST): $(RTL_SRCS) echo " pass only XLEN and the TCU/FPU selectors; VX_config.vh computes the rest"; \ exit 1; \ fi - @echo "wrote $(VIVADO_SRCS_LIST) ($$(grep -cv '^+' $(VIVADO_SRCS_LIST)) files)" + mv $(VIVADO_SRCS_LIST) $(BUILD_DIR)/$(VIVADO_SRCS_NAME).rtl.f + $(SCRIPT_DIR)/circt_elaborate.sh -tVX_firesim_wrap -l$(BUILD_DIR)/$(VIVADO_SRCS_NAME).rtl.f \ + -x$(SCRIPT_DIR)/xilinx_ip_stubs.sv -o$(BUILD_DIR)/$(VIVADO_SRCS_NAME).elab.sv + realpath $(BUILD_DIR)/$(VIVADO_SRCS_NAME).elab.sv > $(VIVADO_SRCS_LIST) + @echo "wrote $(VIVADO_SRCS_LIST): $(VIVADO_SRCS_NAME).elab.sv, elaborated from $$(grep -cv '^+' $(BUILD_DIR)/$(VIVADO_SRCS_NAME).rtl.f) files" # The same flattening under the simulation defines. Verilator finds the modules # by library search over this directory, so only the tree has to be complete -- @@ -430,14 +432,11 @@ $(XILINX_IP_STAMP): $(SCRIPT_DIR)/xilinx_ip_gen.tcl @test -n "$$(find $(XILINX_IP_DIR) -name '*.xci')" || { echo "ERROR: no .xci generated under $(XILINX_IP_DIR)"; exit 1; } touch $@ -# The environment the FireSim Vitis kernel build reads: the sources and vendor -# cores package_kernel.tcl attaches, and the pre-opt hook the link runs to -# resolve the async-BRAM placeholders (it locates hw/scripts through TOOL_DIR). +# The environment the FireSim Vitis kernel build reads: the netlist and vendor +# cores package_kernel.tcl attaches. kernel-env: @echo 'export VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST)' @echo 'export VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR)' - @echo 'export VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl' - @echo 'export TOOL_DIR=$(SCRIPT_DIR)' # Simulator clock (MHz): the image is timed at it and the shell programs it. # V80: the Fmax measured on a FireSim image of Vortex (143.4 MHz). @@ -490,7 +489,6 @@ endif image: vivado-sources xilinx-ips stage $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ - VORTEX_PRE_OPT_TCL=$(SRC_DIR)/pre_opt_hook.tcl TOOL_DIR=$(SCRIPT_DIR) \ $(MAKE) -C $(PLATFORM_BUILD_DIR) $(PLATFORM_IMAGE_GOAL) \ DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) diff --git a/hw/syn/firesim/pre_opt_hook.tcl b/hw/syn/firesim/pre_opt_hook.tcl deleted file mode 100644 index f5b5046a15..0000000000 --- a/hw/syn/firesim/pre_opt_hook.tcl +++ /dev/null @@ -1,11 +0,0 @@ -# Runs on impl_1 immediately before opt_design, attached by the FireSim platform -# build through VORTEX_PRE_OPT_TCL. -# -# VX_async_ram_patch emits VX_placeholder black boxes wherever an -# asynchronous-read RAM has to be realized on a synchronous BRAM, and their -# contents are filled in here from the synthesized netlist. Without this, -# opt_design aborts with "DRC INBB-3 Black Box Instances" -- but only once a -# configuration is large enough to push those RAMs into BRAM. - -set tool_dir $::env(TOOL_DIR) -source ${tool_dir}/xilinx_async_bram_patch.tcl diff --git a/hw/syn/maxsim/Makefile b/hw/syn/maxsim/Makefile index d38baec613..cf252c772e 100644 --- a/hw/syn/maxsim/Makefile +++ b/hw/syn/maxsim/Makefile @@ -114,17 +114,14 @@ SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv) $(wildcard $(MA all: $(VBIN) -# The flow threads memories itself, so an asynchronous-read RAM stays one -# (VX_async_ram_patch.sv) instead of becoming placeholders for a Vivado hook. # The units that would wrap a Xilinx floating-point core take their soft path, # the one they take wherever no vendor core exists. VENDOR_FP_UNITS := VX_fma_unit.sv VX_fdiv_unit.sv VX_fsqrt_unit.sv sources: $(SRCS_LIST) -$(SRCS_LIST): $(RTL_SRCS) $(SRC_DIR)/VX_async_ram_patch.sv +$(SRCS_LIST): $(RTL_SRCS) rm -rf $(SRCS_DIR) cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) -T$(TARGET_TOP) -C$(SRCS_NAME) \ -O$(SRCS_NAME).f $(RTL_PKGS) - cp $(SRC_DIR)/VX_async_ram_patch.sv $(SRCS_DIR)/VX_async_ram_patch.sv @for f in $(VENDOR_FP_UNITS); do \ [ -f $(SRCS_DIR)/$$f ] || continue; \ grep -q 'localparam VENDOR_OK = 1;' $(SRCS_DIR)/$$f || \ diff --git a/hw/syn/maxsim/VX_async_ram_patch.sv b/hw/syn/maxsim/VX_async_ram_patch.sv deleted file mode 100644 index 9b702be3a7..0000000000 --- a/hw/syn/maxsim/VX_async_ram_patch.sv +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The maxsim flow's VX_async_ram_patch: the RAM it stands for, read -// asynchronously. -// -// On Vivado, hw/rtl/libs/VX_async_ram_patch realizes an asynchronous-read RAM -// on a synchronous BRAM through placeholder cells that a post-synthesis hook -// rewires; with or without RADDR_REG it reads the word at raddr. The maxsim -// flow threads its memories itself and cannot run that hook, so it replaces the -// staged copy of the module with this one: same parameters and ports, same -// contents and reads, and the block RAM request as a synthesis attribute. -// Plain SystemVerilog, as the staged sources are already preprocessed. -module VX_async_ram_patch #( - parameter DATAW = 1, - parameter SIZE = 1, - parameter WRENW = 1, - parameter DUAL_PORT = 0, - parameter FORCE_BRAM = 0, - parameter RADDR_REG = 0, - parameter RADDR_RESET = 0, - parameter WRITE_FIRST = 0, - parameter INIT_ENABLE = 0, - parameter INIT_FILE = "", - parameter [DATAW-1:0] INIT_VALUE = 0, - parameter ADDRW = (((SIZE) > 1) ? $clog2(SIZE) : 1) -) ( - input wire clk, - input wire reset, - input wire read, - input wire write, - input wire [WRENW-1:0] wren, - input wire [ADDRW-1:0] waddr, - input wire [DATAW-1:0] wdata, - input wire [ADDRW-1:0] raddr, - output wire [DATAW-1:0] rdata -); - localparam WSELW = DATAW / WRENW; - - // VX_dp_ram and VX_sp_ram instantiate this module only for a RAM that - // must be block RAM; the compiler makes its read synchronous - // (seq-retime-firmem-reads), as the attribute asks. - (* ram_style = "block" *) reg [DATAW-1:0] ram [0:SIZE-1]; - - if (INIT_ENABLE != 0) begin : g_init - if (INIT_FILE != "") begin : g_file - initial $readmemh(INIT_FILE, ram); - end else begin : g_value - initial begin - for (integer i = 0; i < SIZE; ++i) begin - ram[i] = INIT_VALUE; - end - end - end - end - - always @(posedge clk) begin - if (write) begin - for (integer i = 0; i < WRENW; ++i) begin - if (wren[i]) begin - ram[waddr][i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; - end - end - end - end - - assign rdata = ram[raddr]; - -endmodule diff --git a/hw/syn/xilinx/aved/Makefile b/hw/syn/xilinx/aved/Makefile index c2ce83a999..c7866c4809 100644 --- a/hw/syn/xilinx/aved/Makefile +++ b/hw/syn/xilinx/aved/Makefile @@ -106,10 +106,9 @@ $(IP_DIR)/component.xml: $(BUILD_DIR)/sources.txt cd $(BUILD_DIR); TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -source $(SRC_DIR)/package_ip.tcl -tclargs vortex_afu sources.txt . $(DEVICE) gen-bin: $(VBIN_CONTAINER) -# TOOL_DIR is consumed by pre_opt_hook.tcl to locate hw/scripts. The pre-synth -# hook is the linker's only opportunity to set per-step properties on impl_1, -# and it is what attaches the pre-opt hook that resolves the async-BRAM -# placeholders. +# The pre-synth hook is the linker's only opportunity to set per-step properties +# on impl_1, and it is what attaches the pre-opt hook (the hierarchical +# utilization report). $(VBIN_CONTAINER): $(IP_DIR)/component.xml $(BUILD_DIR)/config.cfg $(SCOPE_JSON) mkdir -p $(BIN_DIR) TOOL_DIR=$(SCRIPT_DIR) $(SLASHKIT) link -c $(BUILD_DIR)/config.cfg -p $(TARGET) -o $(VBIN_CONTAINER) \ diff --git a/hw/syn/xilinx/aved/pre_opt_hook.tcl b/hw/syn/xilinx/aved/pre_opt_hook.tcl index e1ec4d450f..b79091ad28 100644 --- a/hw/syn/xilinx/aved/pre_opt_hook.tcl +++ b/hw/syn/xilinx/aved/pre_opt_hook.tcl @@ -1,13 +1,5 @@ # Runs on impl_1 immediately before opt_design. # -# VX_async_ram_patch emits VX_placeholder black boxes wherever an -# asynchronous-read RAM has to be realized on a synchronous BRAM. Their -# contents are filled in here, from the synthesized netlist. Without this, -# opt_design aborts with "DRC INBB-3 Black Box Instances ... has undefined -# contents" -- but only once a configuration is large enough to push those -# RAMs out of distributed LUTRAM and into BRAM, so small builds pass and -# large ones do not. -# # Note: the xrt and dut flows source xilinx_slr_pblocks.tcl here, which SPREADS # a large design across SLRs to relieve congestion. That is deliberately still # omitted -- this AFU sits in a DFX reconfigurable partition and does not have @@ -18,7 +10,4 @@ # pre_synth_hook.tcl for the numbers. The SLR split those attempts were # chasing is fixed shell-side, in slash_base.tcl. -set tool_dir $::env(TOOL_DIR) -source ${tool_dir}/xilinx_async_bram_patch.tcl - report_utilization -file hier_utilization.rpt -hierarchical -hierarchical_percentages diff --git a/hw/syn/xilinx/aved/pre_synth_hook.tcl b/hw/syn/xilinx/aved/pre_synth_hook.tcl index ccea1e8766..fdacefb3e5 100644 --- a/hw/syn/xilinx/aved/pre_synth_hook.tcl +++ b/hw/syn/xilinx/aved/pre_synth_hook.tcl @@ -10,11 +10,6 @@ if {![file exists $pre_opt]} { error "pre_opt_hook.tcl not found next to pre_synth_hook.tcl: $pre_opt" } -# TOOL_DIR is read by the hook itself; fail here rather than mid-implementation. -if {![info exists ::env(TOOL_DIR)]} { - error "TOOL_DIR is not set -- the pre-opt hook needs it to locate hw/scripts" -} - set_property STEPS.OPT_DESIGN.TCL.PRE $pre_opt [get_runs impl_1] puts "INFO: attached pre-opt hook to impl_1: $pre_opt" diff --git a/hw/syn/xilinx/common.mk b/hw/syn/xilinx/common.mk index 66cf4806e2..7591821ef4 100644 --- a/hw/syn/xilinx/common.mk +++ b/hw/syn/xilinx/common.mk @@ -104,9 +104,13 @@ XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_D check-axil-blocks: $(if $(filter-out 0,$(PLATFORM_AXIL_BLOCK_BYTES)),@python3 $(SCRIPT_DIR)/check_axil_blocks.py $(RTL_DIR)/cp/VX_cp_axil_regfile.sv) +# The sources are elaborated by CIRCT into the one netlist Vivado reads +# (gen_sources.sh -E); the Xilinx floating-point cores stay instances, bound to +# the cores xilinx_ip_gen.tcl generates. gen-sources: $(BUILD_DIR)/sources.txt $(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt + mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt \ + -Tvortex_afu -E -X$(SCRIPT_DIR)/xilinx_ip_stubs.sv # The preprocessed sources bake in the kernel's memory port count (the flow's # AFU_MEM_PORTS): regenerate them whenever the configuration changes it. diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index 49c99f3867..8ab286ed21 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -60,7 +60,8 @@ all: $(PROJECT).xpr gen-sources: project_1/sources.txt project_1/sources.txt: mkdir -p project_1 - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -Cproject_1/src -Oproject_1/sources.txt $(RTL_PKGS) + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -Cproject_1/src -Oproject_1/sources.txt $(RTL_PKGS) \ + -E -X$(SCRIPT_DIR)/xilinx_ip_stubs.sv build: $(PROJECT).xpr $(PROJECT).xpr: project_1/sources.txt diff --git a/hw/syn/xilinx/dut/pre_opt_hook.tcl b/hw/syn/xilinx/dut/pre_opt_hook.tcl index 0bb8c7bbb7..b4405850c3 100644 --- a/hw/syn/xilinx/dut/pre_opt_hook.tcl +++ b/hw/syn/xilinx/dut/pre_opt_hook.tcl @@ -1,3 +1,2 @@ set tool_dir $::env(TOOL_DIR) -source ${tool_dir}/xilinx_async_bram_patch.tcl source ${tool_dir}/xilinx_slr_pblocks.tcl \ No newline at end of file diff --git a/hw/syn/xilinx/sandbox/Makefile b/hw/syn/xilinx/sandbox/Makefile index ab2cb72cd1..d2eda595eb 100644 --- a/hw/syn/xilinx/sandbox/Makefile +++ b/hw/syn/xilinx/sandbox/Makefile @@ -68,19 +68,17 @@ $(KERNEL).bin: kernel.bin.coe: $(KERNEL).bin $(SCRIPT_DIR)/bin2coe.py --out=$@ --binfile=8192:$(KERNEL).bin --depth=16384 --wordsize=64 --little_endian -pre_opt_hook.tcl: $(SRC_DIR)/pre_opt_hook.tcl - cp $< $@ - simulate.tcl: $(SRC_DIR)/simulate.tcl cp $< $@ gen-sources: project_1/sources.txt project_1/sources.txt: mkdir -p project_1 - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -P -Cproject_1/src -Oproject_1/sources.txt + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -P -Cproject_1/src -Oproject_1/sources.txt \ + -TVortex_top -E -X$(SCRIPT_DIR)/xilinx_ip_stubs.sv build: done.dcp -done.dcp: project_1/sources.txt kernel.bin.coe project.tcl pre_opt_hook.tcl +done.dcp: project_1/sources.txt kernel.bin.coe project.tcl MAX_JOBS=$(JOBS) OPT_LEVEL=$(OPT_LEVEL) TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -source project.tcl -tclargs $(DEVICE) project_1/sources.txt echo done > done.dcp diff --git a/hw/syn/xilinx/sandbox/pre_opt_hook.tcl b/hw/syn/xilinx/sandbox/pre_opt_hook.tcl deleted file mode 100644 index 92d1e94f97..0000000000 --- a/hw/syn/xilinx/sandbox/pre_opt_hook.tcl +++ /dev/null @@ -1,2 +0,0 @@ -set tool_dir $::env(TOOL_DIR) -source ${tool_dir}/xilinx_async_bram_patch.tcl \ No newline at end of file diff --git a/hw/syn/xilinx/sandbox/project.tcl.in b/hw/syn/xilinx/sandbox/project.tcl.in index 430c8c85a0..2bc478902f 100644 --- a/hw/syn/xilinx/sandbox/project.tcl.in +++ b/hw/syn/xilinx/sandbox/project.tcl.in @@ -422,7 +422,7 @@ proc run_setup {} { # register compilation hooks #set_property STEPS.SYNTH_DESIGN.TCL.PRE ${script_dir}/pre_synth_hook.tcl [get_runs synth_1] #set_property STEPS.SYNTH_DESIGN.TCL.POST ${script_dir}/post_synth_hook.tcl [get_runs synth_1] - set_property STEPS.OPT_DESIGN.TCL.PRE ${script_dir}/pre_opt_hook.tcl [get_runs impl_1] + #set_property STEPS.OPT_DESIGN.TCL.PRE ${script_dir}/pre_opt_hook.tcl [get_runs impl_1] #set_property STEPS.OPT_DESIGN.TCL.POST ${script_dir}/post_opt_hook.tcl [get_runs impl_1] #set_property STEPS.POWER_OPT_DESIGN.TCL.PRE ${script_dir}/pre_power_opt_hook.tcl [get_runs impl_1] #set_property STEPS.POWER_OPT_DESIGN.TCL.POST ${script_dir}/post_power_opt_hook.tcl [get_runs impl_1] diff --git a/hw/syn/xilinx/xrt/pre_opt_hook.tcl b/hw/syn/xilinx/xrt/pre_opt_hook.tcl index 7f1ad925f1..af9d25dffd 100644 --- a/hw/syn/xilinx/xrt/pre_opt_hook.tcl +++ b/hw/syn/xilinx/xrt/pre_opt_hook.tcl @@ -1,5 +1,4 @@ set tool_dir $::env(TOOL_DIR) -source ${tool_dir}/xilinx_async_bram_patch.tcl source ${tool_dir}/xilinx_slr_pblocks.tcl report_utilization -file hier_utilization.rpt -hierarchical -hierarchical_percentages diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index 0fd6a01175..231370f27b 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -209,7 +209,7 @@ $(BUILD_DIR)/src: | $(BUILD_DIR) # their stubs in the sources. verilog: $(BUILD_DIR)/project.v $(BUILD_DIR)/project.v: $(BUILD_DIR)/src - $(SCRIPT_DIR)/sv_to_v.sh -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -b'$(BB_MODULES)' \ + $(SCRIPT_DIR)/circt_elaborate.sh -fv2005 -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -b'$(BB_MODULES)' \ -o$(BUILD_DIR)/project.v # Remove severity/control system-task lines yosys can't synthesize # ($fatal/$error/$warning/$info/$stop are used by the cvfpu sources). From 7ecfadd9dd18e3b57d142ae1462ab5071bb2b64b Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 03:26:55 -0700 Subject: [PATCH 192/336] maxsim: expand array writes, sort before CSE and absorb input logic Each model now gets hw-expand-array-writes and a topological sort before CSE, and maxsim-extract-models moves logic of up to 64 ops that drives a model's input from other models into the model. s2w4i2 on the V80 part, out-of-context synthesis after opt_design: logic LUTs 97,398 -> 93,263 (-4.2%), total LUTs 163,458 -> 159,455 (-2.4%), LUTRAM +132, FFs +64, BRAM and DSP unchanged. Token co-simulation: s2w4i2 and tinygpu 0 errors at unchanged host cycles per target cycle; the 4-core full GPU (Verilator, avedsim) runs demo with 39,104 instructions in 72,519 cycles, as before. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7098d6b5b275d62de96f99b4473443529e61b2d5) --- docs/designs/maxsim.md | 14 ++++++++++---- hw/syn/maxsim/maxsim.py | 18 +++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/designs/maxsim.md b/docs/designs/maxsim.md index 85d613d99e..a3819a4a53 100644 --- a/docs/designs/maxsim.md +++ b/docs/designs/maxsim.md @@ -85,10 +85,16 @@ HW, Comb and Seq dialects: so the socket's netlist is cut out of the top in turn. Bits of a model output that are always the same constant stay in the parent, and an input driven only by the parent's inputs and constants, or exactly by another - model's output, is wired to those directly. -3. **Clean up the models**: `seq-reg-const-prop`, `comb-share-adders` - (`mux(c, a+b, a-b)` → one adder), `comb-low-bits-narrowing` (adders whose - operands have known-zero low bits), canonicalization. These are what a + model's output, is wired to those directly. Small logic (up to 64 ops) + that drives the same input of every instance of a model from other models' + outputs is moved into the model (`absorb-logic`), so that input comes from + those models rather than through the parent's model. +3. **Clean up the models**: `hw-expand-array-writes` (a conditional write of + one array element becomes a select per element), `seq-reg-const-prop`, + `comb-share-adders` (`mux(c, a+b, a-b)` → one adder), + `comb-low-bits-narrowing` (adders whose operands have known-zero low bits), + canonicalization, and CSE after a topological sort (in a module body, CSE + cannot merge a duplicate whose users come first). These are what a threaded model's area is sensitive to; ExportVerilog emits sign-extended multiplies as `$signed` so Vivado maps them to single DSPs. 4. **Compose** (`maxsim-compose`). The top netlist's nested netlists are diff --git a/hw/syn/maxsim/maxsim.py b/hw/syn/maxsim/maxsim.py index e6a28e8a27..6882111b7f 100644 --- a/hw/syn/maxsim/maxsim.py +++ b/hw/syn/maxsim/maxsim.py @@ -29,9 +29,17 @@ # Passes each model gets before it is threaded: register constants, shared # adders and narrowed low bits matter for area once every register of a model -# is replicated per thread. -OPTIMIZE = ['hw.module(seq-reg-const-prop)', 'hw.module(comb-share-adders)', 'canonicalize', - 'hw.module(comb-low-bits-narrowing)', 'canonicalize', 'cse'] +# is replicated per thread. A conditional write of one array element becomes a +# select per element, and sorting lets CSE merge duplicates whose users come +# first in a module body. +OPTIMIZE = ['hw.module(hw-expand-array-writes)', 'hw.module(seq-reg-const-prop)', + 'hw.module(comb-share-adders)', 'canonicalize', 'hw.module(comb-low-bits-narrowing)', + 'canonicalize', 'topological-sort', 'cse'] + +# Logic of up to this many ops that drives a model's input from other models +# is computed in the model, so the input comes from them directly rather than +# through the parent's model. +ABSORB_LOGIC = 64 def run(cmd, **kw): @@ -127,7 +135,7 @@ def visit(m): sys.exit(f'classes not in {a.top}: {" ".join(sorted(unused))}') passes = [] for m in order: - passes.append(f'maxsim-extract-models{{module={m} models={",".join(sorted(children[m]))} netlist}}') + passes.append(f'maxsim-extract-models{{module={m} models={",".join(sorted(children[m]))} netlist absorb-logic={ABSORB_LOGIC}}}') fused = [g for g in groups if set(g) <= children[m]] if fused: passes.append(f'maxsim-fuse-models{{module={m} groups={",".join("+".join(g) for g in fused)}}}') @@ -135,7 +143,7 @@ def visit(m): if not any(set(g) <= children[m] for m in order): sys.exit(f'{"+".join(g)}: not instantiated by the same module') pipeline = passes + OPTIMIZE + [f'maxsim-compose{{module={a.top} sim=maxsim_sim}}', - 'canonicalize', 'cse'] + 'canonicalize', 'topological-sort', 'cse'] sim = out('maxsim_sim.mlir') run([tool('circt-opt'), flat, f'--pass-pipeline=builtin.module({",".join(pipeline)})', '-o', sim]) sim_text = open(sim).read() From d4516572bba732ecaec21a24e453c5416d8c8b47 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 08:21:11 -0700 Subject: [PATCH 193/336] Flows: state RAM read-under-write behavior to Vivado circt_elaborate.sh and maxsim.py pass firtool's --add-vivado-ram-address-conflict-synthesis-bug-workaround: a RAM with a registered read gets rw_addr_collision = "yes" unless it states its own, and an asynchronous-read RAM without a style gets ram_style = "distributed". Without it Vivado leaves a read colliding with a write undefined in a write-first block RAM (Synth 8-6430 on the IPDOM store the read retiming converts). Other tools ignore the attributes. With CIRCT cc3053a6a (memory ports sharing a clock use one clock net) this fixes the single-bit errors of sgemm -n1024 on the full-GPU V80 image: Vivado had mapped the issue slices' register file and the L2 data store to block RAM in WRITE_FIRST mode with independent clocks. s2w4i2, V80 out-of-context synthesis: area unchanged (93,192 logic LUTs, 66,061 LUTRAM, 23,943 FFs, 66 BRAM). Token co-simulation 0 errors on s2w4i2 and tinygpu; the 4-core full GPU in Verilator runs demo with 39,104 instructions in 72,519 cycles, as before. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 81c97e12890de521710383e9ea16a173828fcd5a) --- hw/scripts/circt_elaborate.sh | 4 ++++ hw/syn/maxsim/maxsim.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hw/scripts/circt_elaborate.sh b/hw/scripts/circt_elaborate.sh index c0af5f2636..144e0b6cde 100755 --- a/hw/scripts/circt_elaborate.sh +++ b/hw/scripts/circt_elaborate.sh @@ -28,6 +28,9 @@ # -b names modules declared in the sources themselves; # - a conditional write of one element of an array at a dynamic index # becomes a select per element, as the statement means; +# - a RAM whose read port is registered states its read-under-write +# behavior to Vivado (rw_addr_collision), which otherwise leaves a read +# colliding with a write undefined in block RAM; other tools ignore it; # - -f sv (the default) writes SystemVerilog, for Vivado and Quartus; -f # v2005 writes Verilog-2005 for tools with a Verilog frontend (yosys): no # packed arrays or structs, no local variables. @@ -154,4 +157,5 @@ fi "$CIRCT_OPT" "$stem.mlir" "${passes[@]}" --canonicalize --topological-sort --cse \ -o "$stem.lowered.mlir" "$FIRTOOL" --format=mlir "$stem.lowered.mlir" --disable-all-randomization \ + --add-vivado-ram-address-conflict-synthesis-bug-workaround \ --lowering-options="$lowering" -o "$output_file" diff --git a/hw/syn/maxsim/maxsim.py b/hw/syn/maxsim/maxsim.py index 6882111b7f..195a2ab643 100644 --- a/hw/syn/maxsim/maxsim.py +++ b/hw/syn/maxsim/maxsim.py @@ -147,7 +147,10 @@ def visit(m): sim = out('maxsim_sim.mlir') run([tool('circt-opt'), flat, f'--pass-pipeline=builtin.module({",".join(pipeline)})', '-o', sim]) sim_text = open(sim).read() - run([tool('firtool'), '--format=mlir', sim, '-o', out('maxsim_sim.sv')]) + # A RAM with a registered read states its read-under-write behavior to + # Vivado, which otherwise leaves a read colliding with a write undefined. + run([tool('firtool'), '--format=mlir', sim, + '--add-vivado-ram-address-conflict-synthesis-bug-workaround', '-o', out('maxsim_sim.sv')]) threads = re.search(r'maxsim\.threads = \[(.*?)\]', sim_text).group(1) info = { From b8a14e6ceea9e1a0b7ee6c29721d4957cde1836f Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 08:33:36 -0700 Subject: [PATCH 194/336] RAMs: a block RAM made from an asynchronous read checks collisions A forced-block-RAM RAM read asynchronously (FORCE_BRAM_ASYNC) said rw_addr_collision = "no". The elaboration makes its read synchronous at the address its registers are about to take, which returns a write of the same edge, as the asynchronous read did: write-first. With "no", Vivado built the IPDOM stack store as a NO_CHANGE block RAM and warned (Synth 8-6430) that a read colliding with a write is undefined. The branch now uses RW_RAM_CHECK like every other write-first block RAM. ASIC is unaffected (both macros are empty there). xilinx/dut core, 16 warps, 8 threads, V80 part, synthesis: every block RAM is READ_FIRST with CLOCK_DOMAINS = COMMON, no 8-6430 warning; LUTs 62,725 -> 62,429, block RAM unchanged (69.5). hardware_library.md says how the elaborated netlist states read-under-write behavior, and to check a netlist on the primitives' WRITE_MODE and CLOCK_DOMAINS rather than the synthesis report. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 25ae11e351f4e0c6ada2a76e5603e7f68f74eaf5) --- docs/hardware_library.md | 9 +++++++++ hw/rtl/libs/VX_dp_ram.sv | 4 ++-- hw/rtl/libs/VX_sp_ram.sv | 4 ++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/hardware_library.md b/docs/hardware_library.md index 471cff2cf4..b9ccc4182f 100644 --- a/docs/hardware_library.md +++ b/docs/hardware_library.md @@ -1304,6 +1304,15 @@ The converted read is exact on every cycle after the first clock edge. Each elaboration writes `.memories.rpt`, one line per asynchronous-read RAM with its size, hint and decision. +A converted RAM returns the data written in the same cycle, as the +asynchronous read did (write-first). The elaborated netlist states this to +Vivado with `rw_addr_collision = "yes"`, and the ports of every RAM share one +clock net, so Vivado keeps each RAM's read-under-write behavior. A block RAM +whose ports Vivado takes as independently clocked returns undefined bits when +a read meets a write to the same address; simulation never shows it. To +check a netlist, read the block RAM primitives' `WRITE_MODE_A/B` and +`CLOCK_DOMAINS`, not the synthesis report's mapping table. + --- ### VX_fifo_queue diff --git a/hw/rtl/libs/VX_dp_ram.sv b/hw/rtl/libs/VX_dp_ram.sv index 9552c33f85..8cd451e282 100644 --- a/hw/rtl/libs/VX_dp_ram.sv +++ b/hw/rtl/libs/VX_dp_ram.sv @@ -307,14 +307,14 @@ module VX_dp_ram #( end end else begin : g_read_first if (WRENW != 1) begin : g_wren - `NO_RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN + `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN `RAM_INITIALIZATION always @(posedge clk) begin `RAM_WRITE_WREN end assign rdata = ram[raddr]; end else begin : g_no_wren - `NO_RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; `RAM_INITIALIZATION always @(posedge clk) begin `RAM_WRITE_ALL diff --git a/hw/rtl/libs/VX_sp_ram.sv b/hw/rtl/libs/VX_sp_ram.sv index f7275814fd..319bcffb87 100644 --- a/hw/rtl/libs/VX_sp_ram.sv +++ b/hw/rtl/libs/VX_sp_ram.sv @@ -389,14 +389,14 @@ module VX_sp_ram #( end end else begin : g_read_first if (WRENW != 1) begin : g_wren - `NO_RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN + `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN `RAM_INITIALIZATION always @(posedge clk) begin `RAM_WRITE_WREN end assign rdata = ram[addr]; end else begin : g_no_wren - `NO_RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; `RAM_INITIALIZATION always @(posedge clk) begin `RAM_WRITE_ALL From 5f439d5a7a84be4b0a74a304a53e2fb4ca22a493 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 13:40:54 -0700 Subject: [PATCH 195/336] Rename maxsim to foldsim; foldsim is its own project on CIRCT maxsim becomes foldsim, and its compiler moves out of the CIRCT fork into its own project that builds on CIRCT, as POCL builds on LLVM: the flow calls the installed foldsim driver (hw/syn/common.mk's FOLDSIM), and the toolchain installs and packages circt and foldsim as components with their own producer scripts (ci/circt_install.sh.in, ci/foldsim_install.sh.in) and --circt/--foldsim flags. foldsim now owns its target, foldsim_target (hw/rtl/afu/foldsim/target): Vortex_axi with its ports flattened, the boundary foldsim channelizes. It no longer elaborates the firesim backend's wrapper, and the flow, sources and docs describe the transformations in their own terms. Validated in-process (avedsim, Verilator) with --driver=foldsim: tinygpu demo PASSED (1222 instrs, 24402 cycles) and the 4-core full GPU demo PASSED (115459 target cycles), both unchanged from before the rename. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 68e2f0fa573032476e8ded50a53e1f6add155bf2) --- AGENTS.md | 2 +- CHANGELOG.md | 6 +- README.md | 3 +- ci/blackbox.sh | 4 +- ci/circt_install.sh.in | 82 ++++++++ ci/foldsim_install.sh.in | 77 +++++++ ci/toolchain_install.sh.in | 15 +- ci/toolchain_prebuilt.sh.in | 14 +- docs/assets/img/firesim_build_flow.svg | 2 +- docs/assets/img/firesim_fpga_architecture.svg | 2 +- ...hitecture.svg => foldsim_architecture.svg} | 22 +- docs/building_toolchain.md | 100 ++++++++- docs/designs/firesim_integration.md | 10 +- docs/designs/{maxsim.md => foldsim.md} | 107 +++++----- docs/index.md | 3 +- docs/simulation.md | 34 +-- .../maxsim_afu.sv => foldsim/foldsim_afu.sv} | 198 +++++++++--------- .../foldsim_ctrl.sv} | 112 +++++----- .../maxsim_mem.sv => foldsim/foldsim_mem.sv} | 56 ++--- hw/rtl/afu/foldsim/foldsim_regs.vh | 57 +++++ .../foldsim_token.sv} | 6 +- hw/rtl/afu/foldsim/target/foldsim_target.sv | 151 +++++++++++++ hw/rtl/afu/maxsim/maxsim_regs.vh | 57 ----- hw/syn/common.mk | 5 +- hw/syn/{maxsim => foldsim}/Makefile | 69 +++--- hw/syn/{maxsim => foldsim}/lift_boundaries.py | 2 +- hw/syn/maxsim/maxsim.py | 174 --------------- hw/syn/xilinx/aved/package_ip.tcl | 2 +- sim/foldsim/Makefile | 56 +++++ .../foldsim_emul.cpp} | 54 ++--- .../maxsim_emul.h => foldsim/foldsim_emul.h} | 10 +- sim/maxsim/Makefile | 56 ----- sw/common/foldsim_regs.h | 55 +++++ sw/common/maxsim_regs.h | 55 ----- sw/runtime/{maxsim => foldsim}/Makefile | 22 +- sw/runtime/{maxsim => foldsim}/vortex.cpp | 105 +++++----- tests/regression/common.mk | 10 +- 37 files changed, 1031 insertions(+), 764 deletions(-) create mode 100755 ci/circt_install.sh.in create mode 100755 ci/foldsim_install.sh.in rename docs/assets/img/{maxsim_architecture.svg => foldsim_architecture.svg} (88%) rename docs/designs/{maxsim.md => foldsim.md} (71%) rename hw/rtl/afu/{maxsim/maxsim_afu.sv => foldsim/foldsim_afu.sv} (65%) rename hw/rtl/afu/{maxsim/maxsim_ctrl.sv => foldsim/foldsim_ctrl.sv} (70%) rename hw/rtl/afu/{maxsim/maxsim_mem.sv => foldsim/foldsim_mem.sv} (84%) create mode 100644 hw/rtl/afu/foldsim/foldsim_regs.vh rename hw/rtl/afu/{maxsim/maxsim_token.sv => foldsim/foldsim_token.sv} (94%) create mode 100644 hw/rtl/afu/foldsim/target/foldsim_target.sv delete mode 100644 hw/rtl/afu/maxsim/maxsim_regs.vh rename hw/syn/{maxsim => foldsim}/Makefile (75%) rename hw/syn/{maxsim => foldsim}/lift_boundaries.py (99%) delete mode 100644 hw/syn/maxsim/maxsim.py create mode 100644 sim/foldsim/Makefile rename sim/{maxsim/maxsim_emul.cpp => foldsim/foldsim_emul.cpp} (84%) rename sim/{maxsim/maxsim_emul.h => foldsim/foldsim_emul.h} (85%) delete mode 100644 sim/maxsim/Makefile create mode 100644 sw/common/foldsim_regs.h delete mode 100644 sw/common/maxsim_regs.h rename sw/runtime/{maxsim => foldsim}/Makefile (71%) rename sw/runtime/{maxsim => foldsim}/vortex.cpp (83%) diff --git a/AGENTS.md b/AGENTS.md index 59757bf89d..6d9357efa2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ This is the canonical entry point for **both human contributors and AI coding ag ### Setup & build - [docs/install_vortex.md](docs/install_vortex.md) — initial install + build setup -- [docs/building_toolchain.md](docs/building_toolchain.md) — building Verilator, RISC-V GNU, LLVM, compiler-rt, musl, POCL from source +- [docs/building_toolchain.md](docs/building_toolchain.md) — building Verilator, RISC-V GNU, LLVM, compiler-rt, musl, POCL, CIRCT, foldsim from source - [docs/environment_setup.md](docs/environment_setup.md) — environment variables and toolchain layout - [docs/fpga_setup.md](docs/fpga_setup.md) — FPGA target setup diff --git a/CHANGELOG.md b/CHANGELOG.md index 93352c63f5..8d36694868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,9 +34,9 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, of at least half the LLC write around it. A launch behind DMA writes waits for an L1-only flush token sent on the slot. The runtime sees the new `DMA_LLC_CLIENT` CP capability. *Why:* rtlsim at 2 clusters + L2 + L3: dogfood -30%, multikernel -42.7%; dogfood -64% with an 8 MB L3. -- **maxsim: threaded FPGA simulation of Vortex on the Alveo V80, compiled by CIRCT.** Every class of identical units (sockets, issue slices, ALUs, FPUs, load-store units) is time-multiplexed through one physical copy, cycle-exact against the RTL, with no Chisel, Golden Gate, Yosys or sv2v: `circt-verilog` imports Vortex, CIRCT's `maxsim-extract-models` / `maxsim-compose` cut it into threaded models joined by one token fabric ([hw/syn/maxsim/maxsim.py](hw/syn/maxsim/maxsim.py)), and the V80's native flow packages it with a latency-bandwidth memory endpoint and a command-queue control block ([hw/rtl/afu/maxsim/](hw/rtl/afu/maxsim/)). `--driver=maxsim` runs it on the card (`TARGET=hw`, VRT + QDMA) or in-process in Verilator (`avedsim`, the default). Vortex's RTL is unchanged: units get plain-port boundaries and instance IDs as ports from a source-to-source pass on the flow's copy ([hw/syn/maxsim/lift_boundaries.py](hw/syn/maxsim/lift_boundaries.py)). See [docs/designs/maxsim.md](docs/designs/maxsim.md). +- **foldsim: threaded FPGA simulation of Vortex on the Alveo V80, built on CIRCT.** Every class of identical units (sockets, issue slices, ALUs, FPUs, load-store units) is time-multiplexed through one physical copy, cycle-exact against the RTL, with no Yosys or sv2v: `circt-verilog` imports Vortex, foldsim's `foldsim-extract-models` / `foldsim-compose` cut it into threaded models joined by one token fabric (foldsim is its own project on CIRCT, installed by `ci/toolchain_install.sh --foldsim`; [hw/syn/foldsim/](hw/syn/foldsim/)), and the V80's native flow packages it with a latency-bandwidth memory endpoint and a command-queue control block ([hw/rtl/afu/foldsim/](hw/rtl/afu/foldsim/)). `--driver=foldsim` runs it on the card (`TARGET=hw`, VRT + QDMA) or in-process in Verilator (`avedsim`, the default). Vortex's RTL is unchanged: units get plain-port boundaries and instance IDs as ports from a source-to-source pass on the flow's copy ([hw/syn/foldsim/lift_boundaries.py](hw/syn/foldsim/lift_boundaries.py)). See [docs/designs/foldsim.md](docs/designs/foldsim.md). *Why:* the simulated GPU is no longer bounded by the card's size, and the shell around it is ~8k LUTs instead of FireSim's ~93k — s2w4i2 (2 cores, NW=NT=4, IW=2) routes in 138k LUTs vs 219k and runs sgemm n=128 in 196 ms on V80: 50× SimX on the same host, 1.7× the FireSim image, cycle counts identical to Verilator. - *Limits:* V80 only; one memory port, single-beat transactions; classes are named (`MAXSIM_CLASSES`), not found; kernel register offsets 0x40–0xBF are unusable on the V80 shell (its PCIe MSI-X table answers them). + *Limits:* V80 only; one memory port, single-beat transactions; classes are named (`FOLDSIM_CLASSES`), not found; kernel register offsets 0x40–0xBF are unusable on the V80 shell (its PCIe MSI-X table answers them). ### Changed @@ -90,7 +90,7 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, value exactly. - **FPGA run instructions.** The docs exported `TARGET=hw`, which `blackbox.sh` ignores, so runs fell back to xrtsim; they now pass `--target=hw`. -- **Block RAM for asynchronous-read RAMs is a CIRCT pass, not a Vivado netlist patch; every synthesis flow reads one CIRCT-elaborated netlist.** A RAM that must be block RAM but reads asynchronously is a plain array tagged `(* ram_style = "block" *)`; CIRCT's `seq-retime-firmem-reads` makes its read synchronous, reading at the address its registers are about to take, and stops the build with the reason if it cannot. `hw/scripts/circt_elaborate.sh` (`gen_sources.sh -E`) elaborates the design for Vivado (xrt, aved, dut, sandbox), FireSim, maxsim and Yosys alike, and writes a report of every asynchronous-read RAM. `ASYNC_BRAM_PATCH`, `VX_async_ram_patch`, `VX_placeholder`, `xilinx_async_bram_patch.tcl` and its pre-opt hooks are gone; Xilinx floating-point cores are declared to the elaboration in `hw/scripts/xilinx_ip_stubs.sv`. +- **Block RAM for asynchronous-read RAMs is a CIRCT pass, not a Vivado netlist patch; every synthesis flow reads one CIRCT-elaborated netlist.** A RAM that must be block RAM but reads asynchronously is a plain array tagged `(* ram_style = "block" *)`; CIRCT's `seq-retime-firmem-reads` makes its read synchronous, reading at the address its registers are about to take, and stops the build with the reason if it cannot. `hw/scripts/circt_elaborate.sh` (`gen_sources.sh -E`) elaborates the design for Vivado (xrt, aved, dut, sandbox), FireSim, foldsim and Yosys alike, and writes a report of every asynchronous-read RAM. `ASYNC_BRAM_PATCH`, `VX_async_ram_patch`, `VX_placeholder`, `xilinx_async_bram_patch.tcl` and its pre-opt hooks are gone; Xilinx floating-point cores are declared to the elaboration in `hw/scripts/xilinx_ip_stubs.sv`. *Why:* the patch rewired Vivado's synthesized netlist in Tcl, ran in Vivado flows only, failed late on large builds (`DRC INBB-3`), and produced a netlist nothing simulated. On the core at 16 warps and 8 threads (V80, 300 MHz met both ways) the CIRCT netlist implements in 90,481 logic LUTs against 92,608, with the same BRAM, LUTRAM and DSPs. *Limits:* generate-block instances are named `g_x_0_inst`, not `g_x[0].inst`, in the netlist (`xilinx_slr_pblocks.tcl` accepts both); the modules between a converted RAM and its address registers are inlined; the synthesis baselines move with the netlist and are regenerated by hand. - **CIRCT is the SystemVerilog frontend of every synthesis flow; sv2v and Verilator's preprocessor are gone.** `gen_sources.sh -P` preprocesses with `circt-verilog -E` (identical code to `verilator -E` over the whole RTL tree; only comments differ), and the Yosys flow elaborates with CIRCT into one Verilog-2005 file ([hw/scripts/sv_to_v.sh](hw/scripts/sv_to_v.sh)) instead of sv2v: packed arrays and structs become vectors, and the SRAM macros (`BB_MODULES`) stay instances with their parameter values, read in SystemVerilog mode beside their stubs. CIRCT installs under `$TOOLDIR/circt` (`toolchain_install.sh --circt`; `hw/syn/common.mk` exports `CIRCT_VERILOG`, `CIRCT_OPT`, `FIRTOOL`), and the `asic_gate` fingerprints its baselines on the CIRCT version. Verilator remains the simulator (and `--xml-only` scope metadata). On the cache DUT (ASAP7), SRAM area is identical and logic area is 5.6% higher than through sv2v: CIRCT folds synchronous resets into the next-value logic and lowers array reads to mux trees, both open items. diff --git a/README.md b/README.md index ddd3f6baa0..8faa639971 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,8 @@ The following dependencies will be fetched prebuilt by `toolchain_install.sh`. - [SoftFloat](https://github.com/ucb-bar/berkeley-softfloat-3.git) - [Ramulator](https://github.com/CMU-SAFARI/ramulator.git) - [Yosys](https://github.com/YosysHQ/yosys) -- [CIRCT](https://github.com/llvm/circt) +- [CIRCT](https://github.com/llvm/circt) (Vortex's fork) +- foldsim: FAME-5 FPGA simulators of Vortex, built on CIRCT - [Mesa](https://www.mesa3d.org/) - [chipStar](https://github.com/CHIP-Star/chipStar) ### Install Vortex codebase diff --git a/ci/blackbox.sh b/ci/blackbox.sh index d0e9fa32c6..319abe6b85 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -26,7 +26,7 @@ show_help() { show_usage echo " where" - echo "--driver: gpu, simx, rtlsim, opae, xrt, aved, firesim, maxsim" + echo "--driver: gpu, simx, rtlsim, opae, xrt, aved, firesim, foldsim" echo "--target: xrt: xrtsim (default), hw_emu, hw -- aved: avedsim (default), sim, hw" echo " firesim: the targets of the card's native driver; xrtsim (default) or" echo " avedsim need no image, the others run FPGA_BIN_DIR's (hw/syn/firesim)" @@ -98,7 +98,7 @@ parse_args() { set_driver_path() { case $DRIVER in gpu) DRIVER_PATH="" ;; - simx|rtlsim|opae|xrt|aved|firesim|maxsim) DRIVER_PATH="$ROOT_DIR/sw/runtime/$DRIVER" ;; + simx|rtlsim|opae|xrt|aved|firesim|foldsim) DRIVER_PATH="$ROOT_DIR/sw/runtime/$DRIVER" ;; *) echo "Invalid driver: $DRIVER"; exit 1 ;; esac } diff --git a/ci/circt_install.sh.in b/ci/circt_install.sh.in new file mode 100755 index 0000000000..b7928947dc --- /dev/null +++ b/ci/circt_install.sh.in @@ -0,0 +1,82 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# CIRCT install: the SystemVerilog frontend of every synthesis flow +# (circt-verilog, circt-opt, firtool) and the compiler foldsim builds on. +# +# Builds Vortex's CIRCT fork (LLVM, MLIR and CIRCT in one build) and installs +# it into $TOOLDIR/circt: tools, libraries, headers and CMake packages, which +# ci/foldsim_install.sh builds against. This script is the *producer* of the +# circt prebuilt: build here (once per OSVERSION, in that OS's container), +# then `ci/toolchain_prebuilt.sh --circt` packages the three tools and +# `ci/toolchain_install.sh --circt` fetches them. Most users install the +# prebuilt and never run this script. +# +# Release, assertions off, libstdc++ linked statically, no zlib/zstd: the +# tools run on any machine of the OSVERSION. Needs git, cmake >= 3.20, ninja, +# a C++17 compiler and python3; installs nothing system-wide. +# +# Idempotent: re-running rebuilds only what changed. + +# exit when any command fails +set -e + +TOOLDIR=${TOOLDIR:=@TOOLDIR@} + +CIRCT_REPO=${CIRCT_REPO:=https://github.com/vortexgpgpu/circt.git} +CIRCT_REV=${CIRCT_REV:=vortex_3.x} + +CIRCT_SRC=${CIRCT_SRC:=$TOOLDIR/circt-src} +CIRCT_PREFIX=${CIRCT_PREFIX:=$TOOLDIR/circt} +CIRCT_BUILD=$CIRCT_SRC/build +JOBS=${JOBS:=$(nproc)} + +# ---------------------------------------------------------------------- +# 1. Fetch: LLVM is a submodule pinned by the CIRCT revision. +# ---------------------------------------------------------------------- +if [ ! -d "$CIRCT_SRC/.git" ]; then + git clone --branch "$CIRCT_REV" "$CIRCT_REPO" "$CIRCT_SRC" +else + git -C "$CIRCT_SRC" fetch origin + git -C "$CIRCT_SRC" checkout "$CIRCT_REV" +fi +git -C "$CIRCT_SRC" submodule update --init --depth 1 llvm + +# ---------------------------------------------------------------------- +# 2. Configure and build +# ---------------------------------------------------------------------- +cmake -G Ninja -S "$CIRCT_SRC/llvm/llvm" -B "$CIRCT_BUILD" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$CIRCT_PREFIX" \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_EXTERNAL_PROJECTS=circt \ + -DLLVM_EXTERNAL_CIRCT_SOURCE_DIR="$CIRCT_SRC" \ + -DLLVM_TARGETS_TO_BUILD=host \ + -DLLVM_ENABLE_ASSERTIONS=OFF \ + -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ + -DLLVM_ENABLE_ZLIB=OFF \ + -DLLVM_ENABLE_ZSTD=OFF \ + -DLLVM_ENABLE_LIBXML2=OFF \ + -DLLVM_INSTALL_UTILS=ON \ + -DCIRCT_SLANG_FRONTEND_ENABLED=ON \ + -DCIRCT_SLANG_BUILD_FROM_SOURCE=ON +ninja -C "$CIRCT_BUILD" -j "$JOBS" circt-verilog circt-opt firtool +ninja -C "$CIRCT_BUILD" -j "$JOBS" install + +# ---------------------------------------------------------------------- +# 3. Check +# ---------------------------------------------------------------------- +"$CIRCT_PREFIX/bin/circt-verilog" --version | head -1 +echo "circt installed in $CIRCT_PREFIX" diff --git a/ci/foldsim_install.sh.in b/ci/foldsim_install.sh.in new file mode 100755 index 0000000000..1a21d1751a --- /dev/null +++ b/ci/foldsim_install.sh.in @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# foldsim install: the FAME-5 simulator compiler of hw/syn/foldsim (the +# `foldsim` driver and foldsim-opt). +# +# foldsim is its own project and builds on CIRCT, as POCL builds on LLVM: it +# needs the CIRCT installation ci/circt_install.sh leaves in $TOOLDIR/circt +# (libraries, headers and CMake packages, not only the prebuilt's tools). This +# script is the *producer* of the foldsim prebuilt: build here (once per +# OSVERSION, in that OS's container, after ci/circt_install.sh), then +# `ci/toolchain_prebuilt.sh --foldsim` packages $TOOLDIR/foldsim and +# `ci/toolchain_install.sh --foldsim` fetches it. Most users install the +# prebuilt and never run this script. +# +# Release, assertions off, libstdc++ linked statically. Needs git, cmake >= +# 3.20, ninja and a C++17 compiler; installs nothing system-wide. +# +# Idempotent: re-running rebuilds only what changed. + +# exit when any command fails +set -e + +TOOLDIR=${TOOLDIR:=@TOOLDIR@} + +FOLDSIM_REPO=${FOLDSIM_REPO:=https://github.com/vortexgpgpu/foldsim.git} +FOLDSIM_REV=${FOLDSIM_REV:=main} + +CIRCT_PREFIX=${CIRCT_PREFIX:=$TOOLDIR/circt} +FOLDSIM_SRC=${FOLDSIM_SRC:=$TOOLDIR/foldsim-src} +FOLDSIM_PREFIX=${FOLDSIM_PREFIX:=$TOOLDIR/foldsim} +FOLDSIM_BUILD=$FOLDSIM_SRC/build + +if [ ! -f "$CIRCT_PREFIX/lib/cmake/circt/CIRCTConfig.cmake" ]; then + echo "error: no CIRCT installation in $CIRCT_PREFIX (CIRCTConfig.cmake)." + echo " run ci/circt_install.sh first, or set CIRCT_PREFIX." + exit 1 +fi + +# ---------------------------------------------------------------------- +# 1. Fetch +# ---------------------------------------------------------------------- +if [ ! -d "$FOLDSIM_SRC/.git" ]; then + git clone --branch "$FOLDSIM_REV" "$FOLDSIM_REPO" "$FOLDSIM_SRC" +else + git -C "$FOLDSIM_SRC" fetch origin + git -C "$FOLDSIM_SRC" checkout "$FOLDSIM_REV" +fi + +# ---------------------------------------------------------------------- +# 2. Configure, build and test +# ---------------------------------------------------------------------- +cmake -G Ninja -S "$FOLDSIM_SRC" -B "$FOLDSIM_BUILD" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$FOLDSIM_PREFIX" \ + -DCIRCT_DIR="$CIRCT_PREFIX/lib/cmake/circt" \ + -DMLIR_DIR="$CIRCT_PREFIX/lib/cmake/mlir" \ + -DLLVM_DIR="$CIRCT_PREFIX/lib/cmake/llvm" \ + -DLLVM_EXTERNAL_LIT="$CIRCT_PREFIX/bin/llvm-lit" \ + -DLLVM_STATIC_LINK_CXX_STDLIB=ON +ninja -C "$FOLDSIM_BUILD" +ninja -C "$FOLDSIM_BUILD" check-foldsim +cmake --install "$FOLDSIM_BUILD" + +echo "foldsim installed in $FOLDSIM_PREFIX" diff --git a/ci/toolchain_install.sh.in b/ci/toolchain_install.sh.in index a9c58dbbc8..397132025e 100755 --- a/ci/toolchain_install.sh.in +++ b/ci/toolchain_install.sh.in @@ -177,6 +177,16 @@ circt() mkdir -p $TOOLDIR && rm -rf $TOOLDIR/circt && mv circt $TOOLDIR rm -rf circt circt.tar.bz2* } +# foldsim: the FAME-5 simulator compiler of hw/syn/foldsim (the foldsim +# driver and foldsim-opt), built on CIRCT. +foldsim() +{ + rm -f foldsim.tar.bz2 + wget $REPOSITORY/foldsim/$OSVERSION/foldsim.tar.bz2 + tar -xvf foldsim.tar.bz2 + mkdir -p $TOOLDIR && rm -rf $TOOLDIR/foldsim && mv foldsim $TOOLDIR + rm -rf foldsim foldsim.tar.bz2 +} yosys() { @@ -240,6 +250,7 @@ install_base() riscv32 riscv64 circt + foldsim yosys sta mesa @@ -251,7 +262,7 @@ install_base() show_usage() { echo "Install Pre-built Vortex Toolchain" - echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" + echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--foldsim] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" echo " (no arguments: install all components except gem5)" echo " gem5 is opt-in: it is a large source build." } @@ -289,6 +300,8 @@ while [ "$1" != "" ]; do ;; --circt ) circt ;; + --foldsim ) foldsim + ;; --yosys ) yosys ;; --sta ) sta diff --git a/ci/toolchain_prebuilt.sh.in b/ci/toolchain_prebuilt.sh.in index 3e7c6bb65c..354b4caa1c 100755 --- a/ci/toolchain_prebuilt.sh.in +++ b/ci/toolchain_prebuilt.sh.in @@ -134,6 +134,15 @@ circt() rm circt.tar.bz2 } +# foldsim's driver and foldsim-opt (ci/foldsim_install.sh), one tarball. +foldsim() +{ + echo "prebuilt foldsim..." + tar -C $TOOLDIR -cvjf foldsim.tar.bz2 foldsim/bin/foldsim foldsim/bin/foldsim-opt + mkdir -p ./foldsim/$OSVERSION + mv foldsim.tar.bz2 ./foldsim/$OSVERSION +} + yosys() { echo "prebuilt yosys..." @@ -174,7 +183,7 @@ gem5() show_usage() { echo "Setup Pre-built Vortex Toolchain" - echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" + echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--foldsim] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" } while [ "$1" != "" ]; do @@ -203,6 +212,8 @@ while [ "$1" != "" ]; do ;; --circt ) circt ;; + --foldsim ) foldsim + ;; --yosys ) yosys ;; --sta ) sta @@ -220,6 +231,7 @@ while [ "$1" != "" ]; do libc32 libc64 circt + foldsim yosys sta mesa diff --git a/docs/assets/img/firesim_build_flow.svg b/docs/assets/img/firesim_build_flow.svg index c938408f81..3654f74b50 100644 --- a/docs/assets/img/firesim_build_flow.svg +++ b/docs/assets/img/firesim_build_flow.svg @@ -24,7 +24,7 @@ Build flow — how Vortex RTL becomes a FireSim image Vortex stays Verilog: Golden Gate never sees inside it. Runs from hw/syn/firesim/Makefile with the native flows' tokens. - Threaded images (FAME-5) are the maxsim flow, compiled by CIRCT: see maxsim.md. + Threaded images (FAME-5) are the foldsim flow, compiled by CIRCT: see foldsim.md. diff --git a/docs/assets/img/firesim_fpga_architecture.svg b/docs/assets/img/firesim_fpga_architecture.svg index 82dce0c875..7368968d90 100644 --- a/docs/assets/img/firesim_fpga_architecture.svg +++ b/docs/assets/img/firesim_fpga_architecture.svg @@ -140,7 +140,7 @@ g_sockets — N × VX_socket_top Each socket: 1 core + I$/D$/LMEM, socket_id as a port. One physical copy per socket. Threading the sockets - through one copy (FAME-5) is the maxsim flow: maxsim.md. + through one copy (FAME-5) is the foldsim flow: foldsim.md. diff --git a/docs/assets/img/maxsim_architecture.svg b/docs/assets/img/foldsim_architecture.svg similarity index 88% rename from docs/assets/img/maxsim_architecture.svg rename to docs/assets/img/foldsim_architecture.svg index 7ba7d389bf..c315a0d9ae 100644 --- a/docs/assets/img/maxsim_architecture.svg +++ b/docs/assets/img/foldsim_architecture.svg @@ -19,13 +19,13 @@ - Compile: one IR from SystemVerilog to the kernel (hw/syn/maxsim/maxsim.py) + Compile: one IR from SystemVerilog to the kernel (the foldsim driver, on CIRCT) circt-verilog Vortex, flat but for the classes - maxsim-extract-models + foldsim-extract-models each level a netlist of its own logic and the class instances @@ -33,24 +33,24 @@ share-adders, low-bits narrowing, const-prop - maxsim-compose + foldsim-compose classes threaded (FAME-5), channels in one token fabric firtool, Vivado - maxsim_sim.sv, packaged + foldsim_sim.sv, packaged and linked as on the V80 - Run: the kernel (hw/rtl/afu/maxsim) and its host + Run: the kernel (hw/rtl/afu/foldsim) and its host - maxsim_afu + foldsim_afu - maxsim_sim (generated) + foldsim_sim (generated) every channel: two slots per producer thread, a count per reader rest of Vortex @@ -69,13 +69,13 @@ state in LUTRAM per thread, written with the clock enable of that finish - maxsim_ctrl + foldsim_ctrl AXI4-Lite registers command queue: DCR, start, reset; busy, cycle counters - maxsim_mem + foldsim_mem latency-bandwidth pipe in target cycles; requests to HBM at once, @@ -84,10 +84,10 @@ Endpoints offer and take tokens with valid and ready from their own state, so they connect to the simulator's channels directly. The target runs free: one token per channel per target cycle. - Shown for one memory port and Vortex's DCR, start and busy (the target VX_firesim_wrap). + Shown for one memory port and Vortex's DCR, start and busy (the target foldsim_target). - host: --driver=maxsim + host: --driver=foldsim the runtime's command processor model, in software; VRT on the V80, or the kernel diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index c5177b458b..926c2477ea 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -25,6 +25,8 @@ The components covered here are: 11. [SST](#11-sst) — Structural Simulation Toolkit (sst-core + sst-elements + OpenMPI); consumed by the SST-driven regression configurations 12. [FireSim](#12-firesim) — FPGA-accelerated host-decoupled simulation on the Alveo U55C; needed only to elaborate a target or build a bitstream (optional) 13. [SLASH / Alveo V80](#13-slash--alveo-v80-the-aved-backend) — AMD Alveo V80 platform stack for the `aved` backend (optional) +14. [CIRCT](#14-circt) — the SystemVerilog frontend of every synthesis flow (`circt-verilog`, `circt-opt`, `firtool`) +15. [foldsim](#15-foldsim) — FAME-5 simulator compiler of `hw/syn/foldsim`, built on CIRCT (optional) --- @@ -1098,6 +1100,96 @@ for the backend's architecture and [`xilinx_slash_setup.md`](xilinx_slash_setup.md) for board bring-up. +--- + +## 14. CIRCT + +**Purpose**: the SystemVerilog frontend of every synthesis flow. Each flow +reads the netlist CIRCT elaborates (`hw/scripts/circt_elaborate.sh`, +`gen_sources.sh -E`; see the hardware library's "Block RAM and asynchronous +reads"), and foldsim builds on it. + +> **Baseline:** Vortex's CIRCT fork, branch `vortex_3.x` of +> `vortexgpgpu/circt`: upstream CIRCT with the import, lowering and +> memory fixes the Vortex flows need. It pins its own LLVM (a submodule), +> separate from `llvm-vortex`: CIRCT tracks LLVM main. + +### Build + +`ci/circt_install.sh` builds LLVM, MLIR and CIRCT in one build and installs +the whole of it (tools, libraries, headers, CMake packages) into +`$TOOLDIR/circt`: + +```bash +./ci/circt_install.sh # CIRCT_REPO, CIRCT_REV, JOBS override the defaults +``` + +It configures with: + +```bash +cmake -G Ninja -S $CIRCT_SRC/llvm/llvm -B $CIRCT_SRC/build \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TOOLDIR/circt \ + -DLLVM_ENABLE_PROJECTS=mlir -DLLVM_EXTERNAL_PROJECTS=circt \ + -DLLVM_EXTERNAL_CIRCT_SOURCE_DIR=$CIRCT_SRC -DLLVM_TARGETS_TO_BUILD=host \ + -DLLVM_ENABLE_ASSERTIONS=OFF -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ + -DLLVM_ENABLE_ZLIB=OFF -DLLVM_ENABLE_ZSTD=OFF -DLLVM_ENABLE_LIBXML2=OFF \ + -DLLVM_INSTALL_UTILS=ON -DCIRCT_SLANG_FRONTEND_ENABLED=ON +``` + +### Notes + +- For a prebuilt, build once per `OSVERSION` in that OS's container + (`ubuntu/focal`, `centos/7`): assertions off and libstdc++ linked + statically, so the tools run on any machine of that OS. +- The prebuilt ships only `circt-verilog`, `circt-opt` and `firtool`. Building + foldsim needs the full installation this script leaves behind. +- To develop CIRCT itself, build it in its own checkout with assertions on + and point the flows at it: `CIRCT_PATH=` (the build directory has + the same `bin/` layout). + +--- + +## 15. foldsim + +**Purpose**: the compiler of the FAME-5 FPGA simulators (`hw/syn/foldsim`, +[designs/foldsim.md](designs/foldsim.md)): the `foldsim` driver and +`foldsim-opt`. foldsim is its own project and builds on CIRCT, as POCL builds +on LLVM. + +> **Baseline:** branch `main` of `vortexgpgpu/foldsim`. Its passes +> (`foldsim-extract-models`, `foldsim-fuse-models`, `foldsim-fame5`, +> `foldsim-compose`) were the `maxsim-*` passes of the CIRCT fork. + +### Build + +Build CIRCT first (§14), then: + +```bash +./ci/foldsim_install.sh # FOLDSIM_REPO, FOLDSIM_REV, CIRCT_PREFIX override the defaults +``` + +which configures foldsim against `$TOOLDIR/circt`, builds it, runs its lit +tests (`check-foldsim`) and installs `bin/foldsim` and `bin/foldsim-opt` into +`$TOOLDIR/foldsim`: + +```bash +cmake -G Ninja -S $FOLDSIM_SRC -B $FOLDSIM_SRC/build \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TOOLDIR/foldsim \ + -DCIRCT_DIR=$TOOLDIR/circt/lib/cmake/circt \ + -DMLIR_DIR=$TOOLDIR/circt/lib/cmake/mlir \ + -DLLVM_DIR=$TOOLDIR/circt/lib/cmake/llvm \ + -DLLVM_EXTERNAL_LIT=$TOOLDIR/circt/bin/llvm-lit \ + -DLLVM_STATIC_LINK_CXX_STDLIB=ON +``` + +### Notes + +- `foldsim-opt` links CIRCT statically; the driver finds it next to itself + and the CIRCT tools through `--circt-bin`. `hw/syn/common.mk` sets + `FOLDSIM_PATH ?= $(TOOLDIR)/foldsim` and `FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim`. +- A CIRCT change foldsim's passes rely on lands in CIRCT first; both + prebuilts are refreshed together. + --- ## Verifying an installed toolchain @@ -1119,6 +1211,8 @@ $TOOLDIR/ │ ├── bin/ etc/ lib/libOpenCL.so* share/pocl/ │ └── include/CL/*.h (REQUIRED — see §6; absence breaks tests/opencl) ├── chipstar/ (bin/hipcc, lib/libCHIP.so, include/hip/) +├── circt/bin/ (circt-verilog, circt-opt, firtool) +├── foldsim/bin/ (foldsim, foldsim-opt; optional) ├── sta/ (OpenSTA, optional) ├── mesa-vortex/ (lavapipe Vulkan ICD; share/vulkan/icd.d/, optional) ├── gem5/ (slim runtime — build/{X86,ARM}/gem5.opt + configs/) @@ -1140,6 +1234,8 @@ $LLVM_PREFIX/bin/llvm-spirv --version # if SPIRV-LLVM-Translator installed $TOOLDIR/riscv32-gnu-toolchain/bin/riscv32-unknown-elf-gcc --version $TOOLDIR/verilator/bin/verilator --version $TOOLDIR/chipstar/bin/hipcc --version # if chipStar installed +$TOOLDIR/circt/bin/circt-verilog --version +$TOOLDIR/foldsim/bin/foldsim-opt --version # if foldsim installed test -f $TOOLDIR/pocl/include/CL/opencl.h && echo "POCL host headers OK" # if gem5 installed — both ISAs are built by default @@ -1171,7 +1267,7 @@ Both accept the same per-component flags: ``` --pocl --chipstar --verilator --riscv32 --riscv64 --llvm --mesa ---libcrt32 --libcrt64 --libc32 --libc64 --circt --yosys --sta +--libcrt32 --libcrt64 --libc32 --libc64 --circt --foldsim --yosys --sta --gem5 (slim runtime — see §10) --all (every component above; SST is NOT included — see §11) ``` @@ -1188,7 +1284,7 @@ Both accept the same per-component flags: Run this **after** building the components above (or after rebuilding just one — e.g. POCL). For each selected component it `tar`s the component directory out of `$TOOLDIR`, bzip2-compresses it, and — for -the large ones (`riscv32`, `riscv64`, `llvm`, `verilator`, `yosys`) — +the large ones (`riscv32`, `riscv64`, `llvm`, `verilator`, `circt`, `yosys`) — `split`s the archive into 50 MB parts to stay under GitHub's per-file limit. The bundles are written **relative to the current directory**, into `.//$OSVERSION/` (`libc*` / `libcrt*` go straight into diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index 481e0bfa5c..6c44095153 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -8,7 +8,7 @@ Alveo U55C (Vitis/XRT) and the Alveo V80 (AVED/VRT). |---|---| | [1](#1-system-architecture) | System architecture: host stack, FPGA payload, execution model | | [2](#2-synthesis-of-the-vortex-rtl) | Synthesis of the Vortex RTL | -| [3](#3-threaded-simulation) | Threaded simulation: maxsim | +| [3](#3-threaded-simulation) | Threaded simulation: foldsim | | [4](#4-the-simulation-timing-model) | The simulation timing model | | [5](#5-the-memory-system) | The memory system and its configuration; shell configuration | | [6](#6-observability) | Observability: wave traces, RTL messages and assertions, counters | @@ -206,8 +206,8 @@ slashkit on aved), handed: This backend simulates Vortex unthreaded: one physical copy of every module. Threading a GPU's identical units through one physical copy each (FAME-5) is -the maxsim flow, which compiles Vortex with CIRCT rather than Golden Gate and -has its own kernel, driver and image: see [maxsim.md](maxsim.md). +the foldsim flow, which compiles Vortex with CIRCT rather than Golden Gate and +has its own kernel, driver and image: see [foldsim.md](foldsim.md). --- @@ -521,8 +521,8 @@ faster than rtlsim on `sgemm -n32`). Instruction counts match rtlsim exactly; cycle counts differ −1.3%…+7.1% — the FASED timing model, not the core: FPGA and hw_emu agree exactly. -Threaded images and their results are the maxsim flow's: -[maxsim.md](maxsim.md). +Threaded images and their results are the foldsim flow's: +[foldsim.md](foldsim.md). --- diff --git a/docs/designs/maxsim.md b/docs/designs/foldsim.md similarity index 71% rename from docs/designs/maxsim.md rename to docs/designs/foldsim.md index a3819a4a53..56f2e53ca5 100644 --- a/docs/designs/maxsim.md +++ b/docs/designs/foldsim.md @@ -1,6 +1,6 @@ -# maxsim — threaded FPGA simulation of Vortex, compiled by CIRCT +# foldsim — threaded FPGA simulation of Vortex, on CIRCT -maxsim runs Vortex on the Alveo V80 as a host-decoupled, cycle-exact +foldsim runs Vortex on the Alveo V80 as a host-decoupled, cycle-exact simulator in which every class of identical units — sockets, issue slices, ALUs, FPUs, load-store units — is time-multiplexed through **one physical copy** (FAME-5 model multithreading). A GPU many times the card's size fits @@ -21,35 +21,37 @@ on it, and runs orders of magnitude faster than Verilator. ## 1. Overview -![maxsim architecture](../assets/img/maxsim_architecture.svg) +![foldsim architecture](../assets/img/foldsim_architecture.svg) The whole flow is one IR from SystemVerilog to Verilog: `circt-verilog` -imports Vortex, a few CIRCT passes cut it into models and compose the +imports Vortex, foldsim's passes cut it into models and compose the simulator, `firtool` writes it out, and the card's native flow (IP-XACT -packaging, `slashkit link`) builds the image. There is no Chisel, Golden -Gate, Yosys or sv2v anywhere in it, and no FireSim checkout. +packaging, `slashkit link`) builds the image. foldsim is a project of its own +that builds on CIRCT, as POCL builds on LLVM: the `foldsim` driver and +`foldsim-opt` come from the toolchain (`$(TOOLDIR)/foldsim`), CIRCT from +`$(TOOLDIR)/circt`. Nothing else is in the flow: no Yosys, no sv2v. ```bash -export MAXSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift+VX_lsu_scheduler__lift VX_fpu_std" -export MAXSIM_LIFT="VX_socket:drop=INSTANCE_ID;port=SOCKET_ID VX_issue_slice:drop=ISSUE_ID VX_alu_int:port=BLOCK_IDX VX_alu_muldiv VX_lsu_slice:port=CORE_ID VX_lsu_scheduler" +export FOLDSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift+VX_lsu_scheduler__lift VX_fpu_std" +export FOLDSIM_LIFT="VX_socket:drop=INSTANCE_ID;port=SOCKET_ID VX_issue_slice:drop=ISSUE_ID VX_alu_int:port=BLOCK_IDX VX_alu_muldiv VX_lsu_slice:port=CORE_ID VX_lsu_scheduler" CONFIGS="-DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_ISSUE_WIDTH=2" -CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=maxsim --app=sgemm # Verilator -make -C hw/syn/maxsim CONFIGS="$CONFIGS" # the image -FPGA_BIN_DIR=$PWD/hw/syn/maxsim/build32_maxsim_hw/bin CONFIGS="$CONFIGS" \ - ./ci/blackbox.sh --driver=maxsim --target=hw --app=sgemm # the V80 +CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=foldsim --app=sgemm # Verilator +make -C hw/syn/foldsim CONFIGS="$CONFIGS" # the image +FPGA_BIN_DIR=$PWD/hw/syn/foldsim/build32_foldsim_hw/bin CONFIGS="$CONFIGS" \ + ./ci/blackbox.sh --driver=foldsim --target=hw --app=sgemm # the V80 ``` -`MAXSIM_CLASSES` names the modules threaded as classes; a class may contain -others (the socket contains the issue slices). `MAXSIM_LIFT` gives +`FOLDSIM_CLASSES` names the modules threaded as classes; a class may contain +others (the socket contains the issue slices). `FOLDSIM_LIFT` gives interface-port units a plain-port boundary first, so they can be classes -(`hw/syn/maxsim/lift_boundaries.py`, a source-to-source pass on pyslang; the +(`hw/syn/foldsim/lift_boundaries.py`, a source-to-source pass on pyslang; the RTL is not changed). `port=P` turns an instance-ID parameter into an input port, and every parameter derived from it below, so that all instances of the class elaborate to one module: the socket's `SOCKET_ID` carries every core's -`CORE_ID` with it. The tools are found through `CIRCT_BIN` (by default -`hw/syn/common.mk`'s CIRCT). +`CORE_ID` with it. The tools are found through `FOLDSIM` (the driver) and +`CIRCT_BIN`, by default `hw/syn/common.mk`'s `FOLDSIM_PATH` and `CIRCT_PATH`. -A class `A+B` fuses two modules of the same parent (`maxsim-fuse-models`): +A class `A+B` fuses two modules of the same parent (`foldsim-fuse-models`): each `A` is paired with the one `B` it exchanges bits with, and each pair is simulated as one model, so a path from one through the other and back takes a thread one host cycle instead of a round trip through their channels. The @@ -70,14 +72,14 @@ see the hardware library's "Block RAM and asynchronous reads"); ## 2. The compiler -`hw/syn/maxsim/maxsim.py` drives four steps, all in one MLIR module of the -HW, Comb and Seq dialects: +The `foldsim` driver drives four steps, all in one MLIR module of the HW, +Comb and Seq dialects; steps 2 to 4 run in `foldsim-opt`: 1. **Import and flatten.** `circt-verilog` elaborates the target - (`VX_firesim_wrap`: Vortex with its AXI4 port, DCR, start and busy as plain + (`foldsim_target`: Vortex with its AXI4 port, DCR, start and busy as plain ports) with `--detect-memories`; everything but the class modules is inlined. -2. **Cut netlists, innermost first** (`maxsim-extract-models{netlist}`). A +2. **Cut netlists, innermost first** (`foldsim-extract-models{netlist}`). A module with class instances becomes `_model` — its own logic, with ports for what it exchanges with the instances — and `` becomes a netlist of the same ports, instantiating `_model` beside the class @@ -97,22 +99,22 @@ HW, Comb and Seq dialects: cannot merge a duplicate whose users come first). These are what a threaded model's area is sensitive to; ExportVerilog emits sign-extended multiplies as `$signed` so Vivado maps them to single DSPs. -4. **Compose** (`maxsim-compose`). The top netlist's nested netlists are +4. **Compose** (`foldsim-compose`). The top netlist's nested netlists are inlined, every instance of a model becomes one thread of that model's single threaded copy (§3), and every wire between them a channel of the - token fabric. The result is `maxsim_sim`: `hostClock`, `hostReset`, a + token fabric. The result is `foldsim_sim`: `hostClock`, `hostReset`, a channel per target port (`_valid`, `_ready`, `_bits`) and a `profile_finish` bit per class. -`maxsim.py` also writes `maxsim_sim.json` (the ports, and each class's thread -count) and `maxsim_sim.vh` (the class count, for the kernel). +The driver also writes `foldsim_sim.json` (the ports, and each class's thread +count) and `foldsim_sim.vh` (the class count, for the kernel). --- ## 3. The simulator **Threads.** A model of N instances is one module that simulates them in turn -(`maxsim-fame5`'s threader): every register becomes an N-entry memory read at +(`foldsim-fame5`'s threader): every register becomes an N-entry memory read at the current thread, every memory N times deeper. A thread finishes its target cycle when each input channel has its token and each output's token was taken; its state is then written, on the host clock with that finish as @@ -127,9 +129,9 @@ rather than keeping its own counter. Each producer thread has two token slots and a count of tokens written; each reader of it a count of tokens read. A producer is ready while every reader of the thread has read all but one of its tokens; a consumer's token is valid -when each producer it reads has one for it. This is the two-entry, no -flow-through channel of FireSim's LI-BDN simulators, with fan-out, so the -target's behaviour does not depend on the host's schedule. +when each producer it reads has one for it. This is a two-entry channel +with no flow-through, and with fan-out, so the target's behaviour does not +depend on the host's schedule. **Storage.** A producer read only by threaded consumers that read the same bits of it for every thread keeps its slots in a memory addressed by the @@ -141,20 +143,20 @@ directly, since the shell's valid and ready depend on its own state alone. ## 4. The kernel -`hw/rtl/afu/maxsim/maxsim_afu.sv` has the native V80 kernel's ports but for +`hw/rtl/afu/foldsim/foldsim_afu.sv` has the native V80 kernel's ports but for the host-memory master: `s_axi_control` and `m_axi_mem_0` (linked to the -`MEM` HBM aperture). It instantiates `maxsim_sim` and two endpoints, both +`MEM` HBM aperture). It instantiates `foldsim_sim` and two endpoints, both models of the target's surroundings that take one token per channel per -target cycle (`maxsim_token.sv`): +target cycle (`foldsim_token.sv`): -- **`maxsim_mem`** is the target's AXI4 port, timed as a latency-bandwidth +- **`foldsim_mem`** is the target's AXI4 port, timed as a latency-bandwidth pipe — cycle for cycle `sim/common/axi_latency_pipe.h`: a fixed latency per transaction, bounded reads and writes in flight, one beat per cycle, responses in request order. A request goes to HBM (`base` + target address) the cycle the target makes it; a target cycle that should present a response memory has not returned yet does not finish until it has, and a write is acknowledged to the target only after HBM acknowledged it. -- **`maxsim_ctrl`** is the AXI4-Lite register block and the endpoint of the +- **`foldsim_ctrl`** is the AXI4-Lite register block and the endpoint of the target's reset, DCR, start and busy. The target runs free: the endpoint gives it a token every target cycle. Host commands are queued and each is applied for one target cycle (a reset for its count). Kernel reset holds the @@ -164,16 +166,16 @@ target cycle (`maxsim_token.sv`): ## 5. The runtime -`sw/runtime/maxsim/vortex.cpp` (`--driver=maxsim`) is a transport under the -runtime's software command-processor model, as the FireSim backend is: the +`sw/runtime/foldsim/vortex.cpp` (`--driver=foldsim`) is a transport under the +runtime's software command-processor model: the model's DCR writes and reads, start and busy become commands and status reads of the control block, and its device-memory accesses DMA at the target's addresses. `TARGET=hw` uses VRT — the kernel's registers through `vrt::Kernel`, device memory through a QDMA queue — and `TARGET=avedsim` -the same kernel in Verilator (`sim/maxsim`), with a sparse device memory and +the same kernel in Verilator (`sim/foldsim`), with a sparse device memory and each register access costing the kernel cycles a PCIe access would. -The register map is `sw/common/maxsim_regs.h` (`hw/rtl/afu/maxsim/maxsim_regs.vh` +The register map is `sw/common/foldsim_regs.h` (`hw/rtl/afu/foldsim/foldsim_regs.vh` for the RTL): | offset | register | @@ -201,7 +203,7 @@ the target idle has every effect of what it asked for. ## 6. Validation -- **Token co-simulation** (`maxsim_sim` against the RTL it was compiled from, +- **Token co-simulation** (`foldsim_sim` against the RTL it was compiled from, in Verilator): random target inputs, every channel stalled at random, every output token compared. s2w4i2: 3,000 target cycles, 51,000 tokens, 0 differences. @@ -209,7 +211,7 @@ the target idle has every effect of what it asked for. requests and device-memory latency, every target cycle's outputs and every read's data compared; five latency and limit settings, 0 differences. - **Applications** in the in-process simulator and on the V80 (§7). -- **CIRCT lit tests** for every pass (`test/Transforms/maxsim-*.mlir`). +- **CIRCT lit tests** for every pass (`test/Transforms/foldsim-*.mlir`). --- @@ -221,22 +223,21 @@ classes (4 threads each). | | where | demo cycles | sgemm cycles | sgemm n=128 cycles | host / target cycle | |---|---|---:|---:|---:|---:| -| maxsim, threads in turn, in-process | Verilator | 8,702 | 415,454 | | 12.06 | -| maxsim, ready scheduling, in-process | Verilator | 8,702 | 415,454 | | 8.00 | -| maxsim, threads in turn | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 or 12.00 | -| maxsim, ready scheduling | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 | +| foldsim, threads in turn, in-process | Verilator | 8,702 | 415,454 | | 12.06 | +| foldsim, ready scheduling, in-process | Verilator | 8,702 | 415,454 | | 8.00 | +| foldsim, threads in turn | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 or 12.00 | +| foldsim, ready scheduling | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 | | rtlsim, AXI4 with `AxiLatencyPipe` | Verilator | 8,710 | | | | On the V80 the kernel closes at 149.8 MHz (run at 143 MHz) in 138k routed -LUTs, against 219k for the FireSim image of the same configuration. +LUTs, against 219k for the firesim backend's image of the same configuration. sgemm n=128 takes 196 ms with ready scheduling: 50× SimX on the same host, -1.7× the FireSim image. +1.7× the firesim backend's image. -Instruction counts equal the FireSim images' and rtlsim's. Cycle counts are -1.5–6 % below the FireSim images', which modelled memory with FASED: its -latency-bandwidth pipe has the same settings, but its channel mechanics add -cycles `AxiLatencyPipe` does not; maxsim agrees with `AxiLatencyPipe` -(rtlsim) instead. +Instruction counts equal the firesim backend's and rtlsim's. Cycle counts are +1.5–6 % below the firesim backend's, whose memory model has the same +latency-bandwidth settings but adds channel cycles `AxiLatencyPipe` does not; +foldsim agrees with `AxiLatencyPipe` (rtlsim) instead. --- @@ -251,7 +252,7 @@ cycles `AxiLatencyPipe` does not; maxsim agrees with `AxiLatencyPipe` 3. **The memory endpoint takes single-beat transactions**, as Vortex issues them; a burst sets the status register's memory error. 4. **One memory port**: the target is built with one platform memory bank. -5. **Classes are named, not found**: `MAXSIM_CLASSES` lists them. +5. **Classes are named, not found**: `FOLDSIM_CLASSES` lists them. 6. **No kernel register in `0x40`–`0xBF`.** The V80 shell configures the PCIe block's MSI-X table internally, at offset `0x40` of the BAR whose offset 0 is the first kernel; reads there return the table's entries and diff --git a/docs/index.md b/docs/index.md index 103fbc2c2b..1bcb52a899 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ - [Environment Setup](environment_setup.md): system prerequisites and toolchain setup - [Installing Vortex](install_vortex.md): building and installing the Vortex stack -- [Building the Toolchain from Source](building_toolchain.md): maintainer-facing build recipes for Verilator, RISC-V GNU, LLVM (with X86 + lld + SPIR-V), compiler-rt, musl, POCL, Mesa, chipStar, and gem5 +- [Building the Toolchain from Source](building_toolchain.md): maintainer-facing build recipes for Verilator, RISC-V GNU, LLVM (with X86 + lld + SPIR-V), compiler-rt, musl, POCL, Mesa, chipStar, gem5, CIRCT, and foldsim - [FPGA Setup](fpga_setup.md): running Vortex on Altera and Xilinx FPGA cards - [SLASH / Alveo V80 Setup](xilinx_slash_setup.md): building the SLASH platform stack, the prerequisite for the `aved` backend @@ -75,4 +75,5 @@ Detailed architectural specifications for Vortex subsystems live under - [SimX Simulator Architecture](designs/simx_simulator_architecture.md) - [gem5 Integration](designs/vortex_gem5_integration.md) - [FireSim Integration](designs/firesim_integration.md) + - [foldsim: threaded FPGA simulation on CIRCT](designs/foldsim.md) - [Continuous Integration Design](designs/continuous_integration.md) diff --git a/docs/simulation.md b/docs/simulation.md index cde48cf8d6..bb2c5aa5ac 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -83,9 +83,9 @@ One caveat worth knowing when reading its output: FireSim's kernel took, and is quantized by the driver's polling schedule. Use Vortex's own `PERF: cycles=` line, which is exact. -### Threaded FPGA Simulation (maxsim) +### Threaded FPGA Simulation (foldsim) -maxsim compiles Vortex into a simulator for the V80 with CIRCT: each class of +foldsim compiles Vortex into a simulator for the V80 with CIRCT: each class of identical units (sockets, issue slices, ALUs, ...) is threaded through one physical copy, so a GPU far larger than the card fits on it, still cycle-exact against the RTL. Its kernel carries a small control block and a @@ -95,31 +95,33 @@ it like the native V80 driver does (AXI4-Lite registers, QDMA for memory). | target | what runs | image built by | |---|---|---| | `avedsim` (default) | the same kernel in Verilator, in-process | nothing to build | -| `hw` | the image on the V80 | `make -C hw/syn/maxsim` | +| `hw` | the image on the V80 | `make -C hw/syn/foldsim` | ```bash # the classes to thread, and the units to give plain-port boundaries first -export MAXSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift VX_lsu_scheduler__lift VX_fpu_std" -export MAXSIM_LIFT="VX_socket:drop=INSTANCE_ID;port=SOCKET_ID VX_issue_slice:drop=ISSUE_ID VX_alu_int:port=BLOCK_IDX VX_alu_muldiv VX_lsu_slice:port=CORE_ID VX_lsu_scheduler" +export FOLDSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift VX_lsu_scheduler__lift VX_fpu_std" +export FOLDSIM_LIFT="VX_socket:drop=INSTANCE_ID;port=SOCKET_ID VX_issue_slice:drop=ISSUE_ID VX_alu_int:port=BLOCK_IDX VX_alu_muldiv VX_lsu_slice:port=CORE_ID VX_lsu_scheduler" CONFIGS="-DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_ISSUE_WIDTH=2" # in Verilator, no image needed -CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=maxsim --app=sgemm +CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=foldsim --app=sgemm # build the image, then run on the V80 -make -C hw/syn/maxsim CONFIGS="$CONFIGS" -FPGA_BIN_DIR=$PWD/hw/syn/maxsim/build32_maxsim_hw/bin \ - CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=maxsim --target=hw --app=sgemm +make -C hw/syn/foldsim CONFIGS="$CONFIGS" +FPGA_BIN_DIR=$PWD/hw/syn/foldsim/build32_foldsim_hw/bin \ + CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=foldsim --target=hw --app=sgemm ``` -The flow needs `circt-verilog`, `circt-opt` and `firtool` from a CIRCT build -with the maxsim passes (`CIRCT_BIN=`, or `PATH`), and pyslang for the -boundary lifting (`LIFT_PYTHON`). The memory model's settings are runtime -knobs: `MAXSIM_MEM_READ_LATENCY`, `MAXSIM_MEM_WRITE_LATENCY` (target cycles, -30 by default) and `MAXSIM_MEM_MAX_READS`, `MAXSIM_MEM_MAX_WRITES` (16). At +The flow needs CIRCT (`circt-verilog`, `circt-opt`, `firtool`) and foldsim +(the `foldsim` driver and `foldsim-opt`) from the toolchain +(`./ci/toolchain_install.sh --circt --foldsim`; `CIRCT_BIN=` and +`FOLDSIM=` override them), and pyslang for the boundary lifting +(`LIFT_PYTHON`). The memory model's settings are runtime +knobs: `FOLDSIM_MEM_READ_LATENCY`, `FOLDSIM_MEM_WRITE_LATENCY` (target cycles, +30 by default) and `FOLDSIM_MEM_MAX_READS`, `FOLDSIM_MEM_MAX_WRITES` (16). At exit the runtime reports the target and kernel-clock cycles the simulator ran and how busy each threaded class was. The design is -[designs/maxsim.md](designs/maxsim.md). +[designs/foldsim.md](designs/foldsim.md). ### How to Test (using `blackbox.sh`) @@ -131,7 +133,7 @@ Running tests under specific drivers (rtlsim,simx,fpga) is done using the script - *Threads* - used to specify the number of threads (smallest unit of computation) within a configuration. - *L2cache* - used to enable the shared l2cache among the Vortex cores. - *L3cache* - used to enable the shared l3cache among the Vortex clusters. -- *Driver* - used to specify which driver to run the Vortex simulation (either rtlsim, opae, xrt, simx, firesim, maxsim). +- *Driver* - used to specify which driver to run the Vortex simulation (either rtlsim, opae, xrt, simx, firesim, foldsim). - *Debug* - used to enable debug mode for the Vortex simulation. - *Perf* - used to enable the detailed performance counters within the Vortex simulation. - *App* - used to specify which test/benchmark to run in the Vortex simulation. The main choices are vecadd, sgemm, basic, demo, and dogfood. Other tests/benchmarks are located in the `/benchmarks/opencl` folder though not all of them work wit the current version of Vortex. diff --git a/hw/rtl/afu/maxsim/maxsim_afu.sv b/hw/rtl/afu/foldsim/foldsim_afu.sv similarity index 65% rename from hw/rtl/afu/maxsim/maxsim_afu.sv rename to hw/rtl/afu/foldsim/foldsim_afu.sv index bf67ce4ac9..76dbd79c96 100644 --- a/hw/rtl/afu/maxsim/maxsim_afu.sv +++ b/hw/rtl/afu/foldsim/foldsim_afu.sv @@ -11,13 +11,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -`include "maxsim_sim.vh" +`include "foldsim_sim.vh" -// The maxsim kernel: the simulator CIRCT composes of Vortex (maxsim_sim, of -// the target VX_firesim_wrap), the memory endpoint on its AXI4 port and the +// The foldsim kernel: the simulator CIRCT composes of Vortex (foldsim_sim, of +// the target foldsim_target), the memory endpoint on its AXI4 port and the // control endpoint on its DCR, start and busy signals. Its ports are the // native V80 kernel's, but for the host-memory master it does not need. -module maxsim_afu #( +module foldsim_afu #( parameter C_S_AXI_CONTROL_ADDR_WIDTH = 16, parameter C_S_AXI_CONTROL_DATA_WIDTH = 32, parameter C_M_AXI_MEM_ID_WIDTH = 1, @@ -84,92 +84,92 @@ module maxsim_afu #( reset <= ~ap_rst_n; // ---- The simulator's channels, named after the target's ports ---- -`define MAXSIM_CH(name, w) \ +`define FOLDSIM_CH(name, w) \ wire name``_valid, name``_ready; \ wire [w-1:0] name``_bits; - `MAXSIM_CH(reset, 1) - `MAXSIM_CH(m_axi_mem_0_awready, 1) - `MAXSIM_CH(m_axi_mem_0_wready, 1) - `MAXSIM_CH(m_axi_mem_0_arready, 1) - `MAXSIM_CH(m_axi_mem_0_rvalid, 1) - `MAXSIM_CH(m_axi_mem_0_rdata, DATA_W) - `MAXSIM_CH(m_axi_mem_0_rlast, 1) - `MAXSIM_CH(m_axi_mem_0_rid, TARGET_ID_W) - `MAXSIM_CH(m_axi_mem_0_rresp, 2) - `MAXSIM_CH(m_axi_mem_0_bvalid, 1) - `MAXSIM_CH(m_axi_mem_0_bresp, 2) - `MAXSIM_CH(m_axi_mem_0_bid, TARGET_ID_W) - `MAXSIM_CH(dcr_req_valid, 1) - `MAXSIM_CH(dcr_req_rw, 1) - `MAXSIM_CH(dcr_req_addr, DCR_ADDR_W) - `MAXSIM_CH(dcr_req_data, DCR_DATA_W) - `MAXSIM_CH(start, 1) - `MAXSIM_CH(m_axi_mem_0_awvalid, 1) - `MAXSIM_CH(m_axi_mem_0_awaddr, TARGET_ADDR_W) - `MAXSIM_CH(m_axi_mem_0_awid, TARGET_ID_W) - `MAXSIM_CH(m_axi_mem_0_awlen, 8) - `MAXSIM_CH(m_axi_mem_0_wvalid, 1) - `MAXSIM_CH(m_axi_mem_0_wdata, DATA_W) - `MAXSIM_CH(m_axi_mem_0_wstrb, DATA_W/8) - `MAXSIM_CH(m_axi_mem_0_wlast, 1) - `MAXSIM_CH(m_axi_mem_0_arvalid, 1) - `MAXSIM_CH(m_axi_mem_0_araddr, TARGET_ADDR_W) - `MAXSIM_CH(m_axi_mem_0_arid, TARGET_ID_W) - `MAXSIM_CH(m_axi_mem_0_arlen, 8) - `MAXSIM_CH(m_axi_mem_0_rready, 1) - `MAXSIM_CH(m_axi_mem_0_bready, 1) - `MAXSIM_CH(dcr_rsp_valid, 1) - `MAXSIM_CH(dcr_rsp_data, DCR_DATA_W) - `MAXSIM_CH(busy, 1) -`undef MAXSIM_CH + `FOLDSIM_CH(reset, 1) + `FOLDSIM_CH(m_axi_mem_0_awready, 1) + `FOLDSIM_CH(m_axi_mem_0_wready, 1) + `FOLDSIM_CH(m_axi_mem_0_arready, 1) + `FOLDSIM_CH(m_axi_mem_0_rvalid, 1) + `FOLDSIM_CH(m_axi_mem_0_rdata, DATA_W) + `FOLDSIM_CH(m_axi_mem_0_rlast, 1) + `FOLDSIM_CH(m_axi_mem_0_rid, TARGET_ID_W) + `FOLDSIM_CH(m_axi_mem_0_rresp, 2) + `FOLDSIM_CH(m_axi_mem_0_bvalid, 1) + `FOLDSIM_CH(m_axi_mem_0_bresp, 2) + `FOLDSIM_CH(m_axi_mem_0_bid, TARGET_ID_W) + `FOLDSIM_CH(dcr_req_valid, 1) + `FOLDSIM_CH(dcr_req_rw, 1) + `FOLDSIM_CH(dcr_req_addr, DCR_ADDR_W) + `FOLDSIM_CH(dcr_req_data, DCR_DATA_W) + `FOLDSIM_CH(start, 1) + `FOLDSIM_CH(m_axi_mem_0_awvalid, 1) + `FOLDSIM_CH(m_axi_mem_0_awaddr, TARGET_ADDR_W) + `FOLDSIM_CH(m_axi_mem_0_awid, TARGET_ID_W) + `FOLDSIM_CH(m_axi_mem_0_awlen, 8) + `FOLDSIM_CH(m_axi_mem_0_wvalid, 1) + `FOLDSIM_CH(m_axi_mem_0_wdata, DATA_W) + `FOLDSIM_CH(m_axi_mem_0_wstrb, DATA_W/8) + `FOLDSIM_CH(m_axi_mem_0_wlast, 1) + `FOLDSIM_CH(m_axi_mem_0_arvalid, 1) + `FOLDSIM_CH(m_axi_mem_0_araddr, TARGET_ADDR_W) + `FOLDSIM_CH(m_axi_mem_0_arid, TARGET_ID_W) + `FOLDSIM_CH(m_axi_mem_0_arlen, 8) + `FOLDSIM_CH(m_axi_mem_0_rready, 1) + `FOLDSIM_CH(m_axi_mem_0_bready, 1) + `FOLDSIM_CH(dcr_rsp_valid, 1) + `FOLDSIM_CH(dcr_rsp_data, DCR_DATA_W) + `FOLDSIM_CH(busy, 1) +`undef FOLDSIM_CH -`define MAXSIM_IN(name) \ +`define FOLDSIM_IN(name) \ .name``_valid (name``_valid), .name``_ready (name``_ready), .name``_bits (name``_bits) - wire [`MAXSIM_NUM_CLASSES-1:0] profile_finish /*verilator public_flat_rd*/; + wire [`FOLDSIM_NUM_CLASSES-1:0] profile_finish /*verilator public_flat_rd*/; - maxsim_sim sim ( + foldsim_sim sim ( .hostClock (ap_clk), .hostReset (reset), .profile_finish (profile_finish), - `MAXSIM_IN(reset), - `MAXSIM_IN(m_axi_mem_0_awready), - `MAXSIM_IN(m_axi_mem_0_wready), - `MAXSIM_IN(m_axi_mem_0_arready), - `MAXSIM_IN(m_axi_mem_0_rvalid), - `MAXSIM_IN(m_axi_mem_0_rdata), - `MAXSIM_IN(m_axi_mem_0_rlast), - `MAXSIM_IN(m_axi_mem_0_rid), - `MAXSIM_IN(m_axi_mem_0_rresp), - `MAXSIM_IN(m_axi_mem_0_bvalid), - `MAXSIM_IN(m_axi_mem_0_bresp), - `MAXSIM_IN(m_axi_mem_0_bid), - `MAXSIM_IN(dcr_req_valid), - `MAXSIM_IN(dcr_req_rw), - `MAXSIM_IN(dcr_req_addr), - `MAXSIM_IN(dcr_req_data), - `MAXSIM_IN(start), - `MAXSIM_IN(m_axi_mem_0_awvalid), - `MAXSIM_IN(m_axi_mem_0_awaddr), - `MAXSIM_IN(m_axi_mem_0_awid), - `MAXSIM_IN(m_axi_mem_0_awlen), - `MAXSIM_IN(m_axi_mem_0_wvalid), - `MAXSIM_IN(m_axi_mem_0_wdata), - `MAXSIM_IN(m_axi_mem_0_wstrb), - `MAXSIM_IN(m_axi_mem_0_wlast), - `MAXSIM_IN(m_axi_mem_0_arvalid), - `MAXSIM_IN(m_axi_mem_0_araddr), - `MAXSIM_IN(m_axi_mem_0_arid), - `MAXSIM_IN(m_axi_mem_0_arlen), - `MAXSIM_IN(m_axi_mem_0_rready), - `MAXSIM_IN(m_axi_mem_0_bready), - `MAXSIM_IN(dcr_rsp_valid), - `MAXSIM_IN(dcr_rsp_data), - `MAXSIM_IN(busy) + `FOLDSIM_IN(reset), + `FOLDSIM_IN(m_axi_mem_0_awready), + `FOLDSIM_IN(m_axi_mem_0_wready), + `FOLDSIM_IN(m_axi_mem_0_arready), + `FOLDSIM_IN(m_axi_mem_0_rvalid), + `FOLDSIM_IN(m_axi_mem_0_rdata), + `FOLDSIM_IN(m_axi_mem_0_rlast), + `FOLDSIM_IN(m_axi_mem_0_rid), + `FOLDSIM_IN(m_axi_mem_0_rresp), + `FOLDSIM_IN(m_axi_mem_0_bvalid), + `FOLDSIM_IN(m_axi_mem_0_bresp), + `FOLDSIM_IN(m_axi_mem_0_bid), + `FOLDSIM_IN(dcr_req_valid), + `FOLDSIM_IN(dcr_req_rw), + `FOLDSIM_IN(dcr_req_addr), + `FOLDSIM_IN(dcr_req_data), + `FOLDSIM_IN(start), + `FOLDSIM_IN(m_axi_mem_0_awvalid), + `FOLDSIM_IN(m_axi_mem_0_awaddr), + `FOLDSIM_IN(m_axi_mem_0_awid), + `FOLDSIM_IN(m_axi_mem_0_awlen), + `FOLDSIM_IN(m_axi_mem_0_wvalid), + `FOLDSIM_IN(m_axi_mem_0_wdata), + `FOLDSIM_IN(m_axi_mem_0_wstrb), + `FOLDSIM_IN(m_axi_mem_0_wlast), + `FOLDSIM_IN(m_axi_mem_0_arvalid), + `FOLDSIM_IN(m_axi_mem_0_araddr), + `FOLDSIM_IN(m_axi_mem_0_arid), + `FOLDSIM_IN(m_axi_mem_0_arlen), + `FOLDSIM_IN(m_axi_mem_0_rready), + `FOLDSIM_IN(m_axi_mem_0_bready), + `FOLDSIM_IN(dcr_rsp_valid), + `FOLDSIM_IN(dcr_rsp_data), + `FOLDSIM_IN(busy) ); -`define MAXSIM_MEM(name) \ +`define FOLDSIM_MEM(name) \ .name``_valid (m_axi_mem_0_``name``_valid), .name``_ready (m_axi_mem_0_``name``_ready), \ .name``_bits (m_axi_mem_0_``name``_bits) @@ -178,7 +178,7 @@ module maxsim_afu #( wire [63:0] mem_base; wire mem_idle, mem_error, mem_stall; - maxsim_mem #( + foldsim_mem #( .ID_W (TARGET_ID_W), .ADDR_W (TARGET_ADDR_W), .DATA_W (DATA_W) @@ -193,13 +193,13 @@ module maxsim_afu #( .idle (mem_idle), .error (mem_error), .stall (mem_stall), - `MAXSIM_MEM(awvalid), `MAXSIM_MEM(awaddr), `MAXSIM_MEM(awid), `MAXSIM_MEM(awlen), - `MAXSIM_MEM(wvalid), `MAXSIM_MEM(wdata), `MAXSIM_MEM(wstrb), `MAXSIM_MEM(wlast), - `MAXSIM_MEM(arvalid), `MAXSIM_MEM(araddr), `MAXSIM_MEM(arid), `MAXSIM_MEM(arlen), - `MAXSIM_MEM(rready), `MAXSIM_MEM(bready), - `MAXSIM_MEM(awready), `MAXSIM_MEM(wready), `MAXSIM_MEM(arready), - `MAXSIM_MEM(rvalid), `MAXSIM_MEM(rdata), `MAXSIM_MEM(rlast), `MAXSIM_MEM(rid), `MAXSIM_MEM(rresp), - `MAXSIM_MEM(bvalid), `MAXSIM_MEM(bresp), `MAXSIM_MEM(bid), + `FOLDSIM_MEM(awvalid), `FOLDSIM_MEM(awaddr), `FOLDSIM_MEM(awid), `FOLDSIM_MEM(awlen), + `FOLDSIM_MEM(wvalid), `FOLDSIM_MEM(wdata), `FOLDSIM_MEM(wstrb), `FOLDSIM_MEM(wlast), + `FOLDSIM_MEM(arvalid), `FOLDSIM_MEM(araddr), `FOLDSIM_MEM(arid), `FOLDSIM_MEM(arlen), + `FOLDSIM_MEM(rready), `FOLDSIM_MEM(bready), + `FOLDSIM_MEM(awready), `FOLDSIM_MEM(wready), `FOLDSIM_MEM(arready), + `FOLDSIM_MEM(rvalid), `FOLDSIM_MEM(rdata), `FOLDSIM_MEM(rlast), `FOLDSIM_MEM(rid), `FOLDSIM_MEM(rresp), + `FOLDSIM_MEM(bvalid), `FOLDSIM_MEM(bresp), `FOLDSIM_MEM(bid), .m_awvalid (m_axi_mem_0_awvalid), .m_awready (m_axi_mem_0_awready), .m_awaddr (m_axi_mem_0_awaddr), @@ -222,10 +222,10 @@ module maxsim_afu #( assign m_axi_mem_0_awid = '0; assign m_axi_mem_0_arid = '0; - maxsim_ctrl #( + foldsim_ctrl #( .DCR_ADDR_W (DCR_ADDR_W), .DCR_DATA_W (DCR_DATA_W), - .NUM_CLASSES (`MAXSIM_NUM_CLASSES) + .NUM_CLASSES (`FOLDSIM_NUM_CLASSES) ) ctrl ( .clk (ap_clk), .reset (reset), @@ -252,20 +252,20 @@ module maxsim_afu #( .mem_error (mem_error), .mem_stall (mem_stall), .profile_finish (profile_finish), - `MAXSIM_IN(reset), - `MAXSIM_IN(dcr_req_valid), - `MAXSIM_IN(dcr_req_rw), - `MAXSIM_IN(dcr_req_addr), - `MAXSIM_IN(dcr_req_data), - `MAXSIM_IN(start), - `MAXSIM_IN(dcr_rsp_valid), - `MAXSIM_IN(dcr_rsp_data), - `MAXSIM_IN(busy) + `FOLDSIM_IN(reset), + `FOLDSIM_IN(dcr_req_valid), + `FOLDSIM_IN(dcr_req_rw), + `FOLDSIM_IN(dcr_req_addr), + `FOLDSIM_IN(dcr_req_data), + `FOLDSIM_IN(start), + `FOLDSIM_IN(dcr_rsp_valid), + `FOLDSIM_IN(dcr_rsp_data), + `FOLDSIM_IN(busy) ); assign s_axi_control_rresp = 2'b00; assign s_axi_control_bresp = 2'b00; -`undef MAXSIM_IN -`undef MAXSIM_MEM +`undef FOLDSIM_IN +`undef FOLDSIM_MEM endmodule diff --git a/hw/rtl/afu/maxsim/maxsim_ctrl.sv b/hw/rtl/afu/foldsim/foldsim_ctrl.sv similarity index 70% rename from hw/rtl/afu/maxsim/maxsim_ctrl.sv rename to hw/rtl/afu/foldsim/foldsim_ctrl.sv index 491766e212..91872f58d9 100644 --- a/hw/rtl/afu/maxsim/maxsim_ctrl.sv +++ b/hw/rtl/afu/foldsim/foldsim_ctrl.sv @@ -11,9 +11,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -`include "maxsim_regs.vh" +`include "foldsim_regs.vh" -// The host's view of a maxsim simulator: an AXI4-Lite register block, and the +// The host's view of a foldsim simulator: an AXI4-Lite register block, and the // endpoint that drives the target's control inputs from it. // // The target runs free: the endpoint gives it a token every cycle. The host @@ -23,8 +23,8 @@ // kept for the host; `busy` covers everything queued or in flight, so a host // that sees the target idle has every effect of what it asked for. // -// See maxsim_regs.vh for the register map. -module maxsim_ctrl #( +// See foldsim_regs.vh for the register map. +module foldsim_ctrl #( parameter DCR_ADDR_W = 12, parameter DCR_DATA_W = 32, parameter QUEUE = 16, @@ -107,25 +107,25 @@ module maxsim_ctrl #( wire q_empty = (q_count == 0); // ---- The target's control inputs this cycle ---- - wire in_reset = (reset_count != 0) | (cur_op == `MAXSIM_CMD_RESET); - wire dcr_valid = ~in_reset & ((cur_op == `MAXSIM_CMD_DCR_WRITE) | (cur_op == `MAXSIM_CMD_DCR_READ)); - wire dcr_rw = (cur_op == `MAXSIM_CMD_DCR_WRITE); - wire do_start = ~in_reset & (cur_op == `MAXSIM_CMD_START); + wire in_reset = (reset_count != 0) | (cur_op == `FOLDSIM_CMD_RESET); + wire dcr_valid = ~in_reset & ((cur_op == `FOLDSIM_CMD_DCR_WRITE) | (cur_op == `FOLDSIM_CMD_DCR_READ)); + wire dcr_rw = (cur_op == `FOLDSIM_CMD_DCR_WRITE); + wire do_start = ~in_reset & (cur_op == `FOLDSIM_CMD_START); // Reset cycles left after this one; a reset command lasts its count of // cycles, at least one. wire [31:0] reset_next = (reset_count != 0) ? reset_count - 32'd1 - : ((cur_op == `MAXSIM_CMD_RESET) && (cur_data > 1)) ? 32'(cur_data) - 32'd1 + : ((cur_op == `FOLDSIM_CMD_RESET) && (cur_data > 1)) ? 32'(cur_data) - 32'd1 : 32'd0; wire finish; wire [5:0] done; wire [2:0] have; - maxsim_token_out o_reset (clk, reset, reset_valid, reset_ready, run, finish, done[0]); - maxsim_token_out o_dvalid (clk, reset, dcr_req_valid_valid, dcr_req_valid_ready, run, finish, done[1]); - maxsim_token_out o_drw (clk, reset, dcr_req_rw_valid, dcr_req_rw_ready, run, finish, done[2]); - maxsim_token_out o_daddr (clk, reset, dcr_req_addr_valid, dcr_req_addr_ready, run, finish, done[3]); - maxsim_token_out o_ddata (clk, reset, dcr_req_data_valid, dcr_req_data_ready, run, finish, done[4]); - maxsim_token_out o_start (clk, reset, start_valid, start_ready, run, finish, done[5]); + foldsim_token_out o_reset (clk, reset, reset_valid, reset_ready, run, finish, done[0]); + foldsim_token_out o_dvalid (clk, reset, dcr_req_valid_valid, dcr_req_valid_ready, run, finish, done[1]); + foldsim_token_out o_drw (clk, reset, dcr_req_rw_valid, dcr_req_rw_ready, run, finish, done[2]); + foldsim_token_out o_daddr (clk, reset, dcr_req_addr_valid, dcr_req_addr_ready, run, finish, done[3]); + foldsim_token_out o_ddata (clk, reset, dcr_req_data_valid, dcr_req_data_ready, run, finish, done[4]); + foldsim_token_out o_start (clk, reset, start_valid, start_ready, run, finish, done[5]); assign reset_bits = in_reset; assign dcr_req_valid_bits = dcr_valid; assign dcr_req_rw_bits = dcr_rw; @@ -135,9 +135,9 @@ module maxsim_ctrl #( wire rsp_v, busy; wire [DCR_DATA_W-1:0] rsp_d; - maxsim_token_in #(1) t_rspv (clk, reset, dcr_rsp_valid_valid, dcr_rsp_valid_ready, dcr_rsp_valid_bits, finish, have[0], rsp_v); - maxsim_token_in #(DCR_DATA_W) t_rspd (clk, reset, dcr_rsp_data_valid, dcr_rsp_data_ready, dcr_rsp_data_bits, finish, have[1], rsp_d); - maxsim_token_in #(1) t_busy (clk, reset, busy_valid, busy_ready, busy_bits, finish, have[2], busy); + foldsim_token_in #(1) t_rspv (clk, reset, dcr_rsp_valid_valid, dcr_rsp_valid_ready, dcr_rsp_valid_bits, finish, have[0], rsp_v); + foldsim_token_in #(DCR_DATA_W) t_rspd (clk, reset, dcr_rsp_data_valid, dcr_rsp_data_ready, dcr_rsp_data_bits, finish, have[1], rsp_d); + foldsim_token_in #(1) t_busy (clk, reset, busy_valid, busy_ready, busy_bits, finish, have[2], busy); assign finish = run & (&have) & (&done); @@ -151,10 +151,10 @@ module maxsim_ctrl #( wire wr = aw_got & w_got & ~s_bvalid; wire [11:0] wa = {aw_addr[11:2], 2'b00}; - wire push = wr & (wa == `MAXSIM_REG_CMD) & (q_count != QC'(QUEUE)); + wire push = wr & (wa == `FOLDSIM_REG_CMD) & (q_count != QC'(QUEUE)); wire pop = finish & ~q_empty & (reset_next == 0); - wire status_busy = ~q_empty | (cur_op != `MAXSIM_CMD_NONE) | start_pending | target_busy + wire status_busy = ~q_empty | (cur_op != `FOLDSIM_CMD_NONE) | start_pending | target_busy | ~mem_idle | (reset_count != 0); always @(posedge clk) begin @@ -164,7 +164,7 @@ module maxsim_ctrl #( q_count <= 0; run <= 1'b1; reset_count <= RESET_CYCLES; - cur_op <= `MAXSIM_CMD_NONE; + cur_op <= `FOLDSIM_CMD_NONE; start_pending <= 1'b0; target_busy <= 1'b0; rsp_valid <= 1'b0; @@ -195,7 +195,7 @@ module maxsim_ctrl #( if (finish) begin target_cycles <= target_cycles + 64'd1; reset_count <= reset_next; - cur_op <= pop ? q_op[q_head] : `MAXSIM_CMD_NONE; + cur_op <= pop ? q_op[q_head] : `FOLDSIM_CMD_NONE; cur_addr <= q_addr[q_head]; cur_data <= q_data[q_head]; if (rsp_v) begin @@ -222,7 +222,7 @@ module maxsim_ctrl #( q_data[q_tail] <= cmd_data; q_tail <= q_tail + 1'b1; // A read's response is the next one. - if (w_data[2:0] == `MAXSIM_CMD_DCR_READ) + if (w_data[2:0] == `FOLDSIM_CMD_DCR_READ) rsp_valid <= 1'b0; end q_count <= q_count + QC'(push) - QC'(pop); @@ -241,15 +241,15 @@ module maxsim_ctrl #( w_got <= 1'b0; s_bvalid <= 1'b1; case (wa) - `MAXSIM_REG_CTRL: run <= w_data[0]; - `MAXSIM_REG_CMD_ADDR: cmd_addr <= DCR_ADDR_W'(w_data); - `MAXSIM_REG_CMD_DATA: cmd_data <= DCR_DATA_W'(w_data); - `MAXSIM_REG_MEM_BASE_LO: mem_base[31:0] <= w_data; - `MAXSIM_REG_MEM_BASE_HI: mem_base[63:32] <= w_data; - `MAXSIM_REG_MEM_RLAT: mem_read_latency <= (w_data == 0) ? 32'd1 : w_data; - `MAXSIM_REG_MEM_WLAT: mem_write_latency <= (w_data == 0) ? 32'd1 : w_data; - `MAXSIM_REG_MEM_MAXR: mem_max_reads <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); - `MAXSIM_REG_MEM_MAXW: mem_max_writes <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); + `FOLDSIM_REG_CTRL: run <= w_data[0]; + `FOLDSIM_REG_CMD_ADDR: cmd_addr <= DCR_ADDR_W'(w_data); + `FOLDSIM_REG_CMD_DATA: cmd_data <= DCR_DATA_W'(w_data); + `FOLDSIM_REG_MEM_BASE_LO: mem_base[31:0] <= w_data; + `FOLDSIM_REG_MEM_BASE_HI: mem_base[63:32] <= w_data; + `FOLDSIM_REG_MEM_RLAT: mem_read_latency <= (w_data == 0) ? 32'd1 : w_data; + `FOLDSIM_REG_MEM_WLAT: mem_write_latency <= (w_data == 0) ? 32'd1 : w_data; + `FOLDSIM_REG_MEM_MAXR: mem_max_reads <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); + `FOLDSIM_REG_MEM_MAXW: mem_max_writes <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); default:; endcase end @@ -264,33 +264,33 @@ module maxsim_ctrl #( ar_got <= 1'b0; s_rvalid <= 1'b1; case ({ar_addr[11:2], 2'b00}) - `MAXSIM_REG_ID: s_rdata <= `MAXSIM_ID; - `MAXSIM_REG_VERSION: s_rdata <= `MAXSIM_VERSION; - `MAXSIM_REG_CTRL: s_rdata <= {31'b0, run}; - `MAXSIM_REG_STATUS: s_rdata <= {27'b0, mem_error, rsp_valid, + `FOLDSIM_REG_ID: s_rdata <= `FOLDSIM_ID; + `FOLDSIM_REG_VERSION: s_rdata <= `FOLDSIM_VERSION; + `FOLDSIM_REG_CTRL: s_rdata <= {31'b0, run}; + `FOLDSIM_REG_STATUS: s_rdata <= {27'b0, mem_error, rsp_valid, (q_count == QC'(QUEUE)), status_busy}; - `MAXSIM_REG_CMD_ADDR: s_rdata <= 32'(cmd_addr); - `MAXSIM_REG_CMD_DATA: s_rdata <= 32'(cmd_data); - `MAXSIM_REG_DCR_RSP: s_rdata <= 32'(rsp_data); - `MAXSIM_REG_TCYCLES_LO: s_rdata <= target_cycles[31:0]; - `MAXSIM_REG_TCYCLES_HI: s_rdata <= target_cycles[63:32]; - `MAXSIM_REG_HCYCLES_LO: s_rdata <= host_cycles[31:0]; - `MAXSIM_REG_HCYCLES_HI: s_rdata <= host_cycles[63:32]; - `MAXSIM_REG_MEM_BASE_LO: s_rdata <= mem_base[31:0]; - `MAXSIM_REG_MEM_BASE_HI: s_rdata <= mem_base[63:32]; - `MAXSIM_REG_MEM_RLAT: s_rdata <= mem_read_latency; - `MAXSIM_REG_MEM_WLAT: s_rdata <= mem_write_latency; - `MAXSIM_REG_MEM_MAXR: s_rdata <= 32'(mem_max_reads); - `MAXSIM_REG_MEM_MAXW: s_rdata <= 32'(mem_max_writes); - `MAXSIM_REG_MSTALL_LO: s_rdata <= mem_stalls[31:0]; - `MAXSIM_REG_MSTALL_HI: s_rdata <= mem_stalls[63:32]; - `MAXSIM_REG_NUM_CLASSES: s_rdata <= 32'(NUM_CLASSES); + `FOLDSIM_REG_CMD_ADDR: s_rdata <= 32'(cmd_addr); + `FOLDSIM_REG_CMD_DATA: s_rdata <= 32'(cmd_data); + `FOLDSIM_REG_DCR_RSP: s_rdata <= 32'(rsp_data); + `FOLDSIM_REG_TCYCLES_LO: s_rdata <= target_cycles[31:0]; + `FOLDSIM_REG_TCYCLES_HI: s_rdata <= target_cycles[63:32]; + `FOLDSIM_REG_HCYCLES_LO: s_rdata <= host_cycles[31:0]; + `FOLDSIM_REG_HCYCLES_HI: s_rdata <= host_cycles[63:32]; + `FOLDSIM_REG_MEM_BASE_LO: s_rdata <= mem_base[31:0]; + `FOLDSIM_REG_MEM_BASE_HI: s_rdata <= mem_base[63:32]; + `FOLDSIM_REG_MEM_RLAT: s_rdata <= mem_read_latency; + `FOLDSIM_REG_MEM_WLAT: s_rdata <= mem_write_latency; + `FOLDSIM_REG_MEM_MAXR: s_rdata <= 32'(mem_max_reads); + `FOLDSIM_REG_MEM_MAXW: s_rdata <= 32'(mem_max_writes); + `FOLDSIM_REG_MSTALL_LO: s_rdata <= mem_stalls[31:0]; + `FOLDSIM_REG_MSTALL_HI: s_rdata <= mem_stalls[63:32]; + `FOLDSIM_REG_NUM_CLASSES: s_rdata <= 32'(NUM_CLASSES); default: s_rdata <= 32'd0; endcase - if (ar_addr >= `MAXSIM_REG_FINISH_BASE - && ar_addr < `MAXSIM_REG_FINISH_BASE + 12'(8 * NUM_CLASSES)) begin - s_rdata <= ar_addr[2] ? 32'(finishes[(ar_addr - `MAXSIM_REG_FINISH_BASE) >> 3] >> 32) - : finishes[(ar_addr - `MAXSIM_REG_FINISH_BASE) >> 3][31:0]; + if (ar_addr >= `FOLDSIM_REG_FINISH_BASE + && ar_addr < `FOLDSIM_REG_FINISH_BASE + 12'(8 * NUM_CLASSES)) begin + s_rdata <= ar_addr[2] ? 32'(finishes[(ar_addr - `FOLDSIM_REG_FINISH_BASE) >> 3] >> 32) + : finishes[(ar_addr - `FOLDSIM_REG_FINISH_BASE) >> 3][31:0]; end end if (s_rvalid & s_rready) diff --git a/hw/rtl/afu/maxsim/maxsim_mem.sv b/hw/rtl/afu/foldsim/foldsim_mem.sv similarity index 84% rename from hw/rtl/afu/maxsim/maxsim_mem.sv rename to hw/rtl/afu/foldsim/foldsim_mem.sv index 849100dce7..3c46d64cbb 100644 --- a/hw/rtl/afu/maxsim/maxsim_mem.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem.sv @@ -11,10 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -// The target's AXI4 memory port, as a maxsim endpoint. +// The target's AXI4 memory port, as a foldsim endpoint. // // Timing is a latency-bandwidth pipe, cycle for cycle the model of -// sim/common/axi_latency_pipe.h (FireSim FASED's "Latency Bandwidth Pipe"): a +// sim/common/axi_latency_pipe.h: a // fixed latency per transaction, at most `max_reads` reads and `max_writes` // writes in flight, one beat per cycle on each response channel, responses in // the order their requests were accepted. Every output is a function of the @@ -29,7 +29,7 @@ // // Single-beat transactions only, as Vortex issues them: a longer burst sets // `error`. -module maxsim_mem #( +module foldsim_mem #( parameter ID_W = 4, parameter ADDR_W = 64, parameter DATA_W = 512, @@ -112,20 +112,20 @@ module maxsim_mem #( wire [DATA_W/8-1:0] wstrb; wire [13:0] have; - maxsim_token_in #(1) t_awvalid (clk, reset, awvalid_valid, awvalid_ready, awvalid_bits, finish, have[0], awvalid); - maxsim_token_in #(ADDR_W) t_awaddr (clk, reset, awaddr_valid, awaddr_ready, awaddr_bits, finish, have[1], awaddr); - maxsim_token_in #(ID_W) t_awid (clk, reset, awid_valid, awid_ready, awid_bits, finish, have[2], awid); - maxsim_token_in #(8) t_awlen (clk, reset, awlen_valid, awlen_ready, awlen_bits, finish, have[3], awlen); - maxsim_token_in #(1) t_wvalid (clk, reset, wvalid_valid, wvalid_ready, wvalid_bits, finish, have[4], wvalid); - maxsim_token_in #(DATA_W) t_wdata (clk, reset, wdata_valid, wdata_ready, wdata_bits, finish, have[5], wdata); - maxsim_token_in #(DATA_W/8) t_wstrb (clk, reset, wstrb_valid, wstrb_ready, wstrb_bits, finish, have[6], wstrb); - maxsim_token_in #(1) t_wlast (clk, reset, wlast_valid, wlast_ready, wlast_bits, finish, have[7], wlast); - maxsim_token_in #(1) t_arvalid (clk, reset, arvalid_valid, arvalid_ready, arvalid_bits, finish, have[8], arvalid); - maxsim_token_in #(ADDR_W) t_araddr (clk, reset, araddr_valid, araddr_ready, araddr_bits, finish, have[9], araddr); - maxsim_token_in #(ID_W) t_arid (clk, reset, arid_valid, arid_ready, arid_bits, finish, have[10], arid); - maxsim_token_in #(8) t_arlen (clk, reset, arlen_valid, arlen_ready, arlen_bits, finish, have[11], arlen); - maxsim_token_in #(1) t_rready (clk, reset, rready_valid, rready_ready, rready_bits, finish, have[12], rready); - maxsim_token_in #(1) t_bready (clk, reset, bready_valid, bready_ready, bready_bits, finish, have[13], bready); + foldsim_token_in #(1) t_awvalid (clk, reset, awvalid_valid, awvalid_ready, awvalid_bits, finish, have[0], awvalid); + foldsim_token_in #(ADDR_W) t_awaddr (clk, reset, awaddr_valid, awaddr_ready, awaddr_bits, finish, have[1], awaddr); + foldsim_token_in #(ID_W) t_awid (clk, reset, awid_valid, awid_ready, awid_bits, finish, have[2], awid); + foldsim_token_in #(8) t_awlen (clk, reset, awlen_valid, awlen_ready, awlen_bits, finish, have[3], awlen); + foldsim_token_in #(1) t_wvalid (clk, reset, wvalid_valid, wvalid_ready, wvalid_bits, finish, have[4], wvalid); + foldsim_token_in #(DATA_W) t_wdata (clk, reset, wdata_valid, wdata_ready, wdata_bits, finish, have[5], wdata); + foldsim_token_in #(DATA_W/8) t_wstrb (clk, reset, wstrb_valid, wstrb_ready, wstrb_bits, finish, have[6], wstrb); + foldsim_token_in #(1) t_wlast (clk, reset, wlast_valid, wlast_ready, wlast_bits, finish, have[7], wlast); + foldsim_token_in #(1) t_arvalid (clk, reset, arvalid_valid, arvalid_ready, arvalid_bits, finish, have[8], arvalid); + foldsim_token_in #(ADDR_W) t_araddr (clk, reset, araddr_valid, araddr_ready, araddr_bits, finish, have[9], araddr); + foldsim_token_in #(ID_W) t_arid (clk, reset, arid_valid, arid_ready, arid_bits, finish, have[10], arid); + foldsim_token_in #(8) t_arlen (clk, reset, arlen_valid, arlen_ready, arlen_bits, finish, have[11], arlen); + foldsim_token_in #(1) t_rready (clk, reset, rready_valid, rready_ready, rready_bits, finish, have[12], rready); + foldsim_token_in #(1) t_bready (clk, reset, bready_valid, bready_ready, bready_bits, finish, have[13], bready); // ---- The pipe's state ---- reg [31:0] cycle; @@ -165,17 +165,17 @@ module maxsim_mem #( wire ok = (~cur_r_valid | (rd_count != 0)) & (~cur_w_valid | (b_acked != b_given)); wire [10:0] done; - maxsim_token_out o_awready (clk, reset, awready_valid, awready_ready, ok, finish, done[0]); - maxsim_token_out o_wready (clk, reset, wready_valid, wready_ready, ok, finish, done[1]); - maxsim_token_out o_arready (clk, reset, arready_valid, arready_ready, ok, finish, done[2]); - maxsim_token_out o_rvalid (clk, reset, rvalid_valid, rvalid_ready, ok, finish, done[3]); - maxsim_token_out o_rdata (clk, reset, rdata_valid, rdata_ready, ok, finish, done[4]); - maxsim_token_out o_rlast (clk, reset, rlast_valid, rlast_ready, ok, finish, done[5]); - maxsim_token_out o_rid (clk, reset, rid_valid, rid_ready, ok, finish, done[6]); - maxsim_token_out o_rresp (clk, reset, rresp_valid, rresp_ready, ok, finish, done[7]); - maxsim_token_out o_bvalid (clk, reset, bvalid_valid, bvalid_ready, ok, finish, done[8]); - maxsim_token_out o_bresp (clk, reset, bresp_valid, bresp_ready, ok, finish, done[9]); - maxsim_token_out o_bid (clk, reset, bid_valid, bid_ready, ok, finish, done[10]); + foldsim_token_out o_awready (clk, reset, awready_valid, awready_ready, ok, finish, done[0]); + foldsim_token_out o_wready (clk, reset, wready_valid, wready_ready, ok, finish, done[1]); + foldsim_token_out o_arready (clk, reset, arready_valid, arready_ready, ok, finish, done[2]); + foldsim_token_out o_rvalid (clk, reset, rvalid_valid, rvalid_ready, ok, finish, done[3]); + foldsim_token_out o_rdata (clk, reset, rdata_valid, rdata_ready, ok, finish, done[4]); + foldsim_token_out o_rlast (clk, reset, rlast_valid, rlast_ready, ok, finish, done[5]); + foldsim_token_out o_rid (clk, reset, rid_valid, rid_ready, ok, finish, done[6]); + foldsim_token_out o_rresp (clk, reset, rresp_valid, rresp_ready, ok, finish, done[7]); + foldsim_token_out o_bvalid (clk, reset, bvalid_valid, bvalid_ready, ok, finish, done[8]); + foldsim_token_out o_bresp (clk, reset, bresp_valid, bresp_ready, ok, finish, done[9]); + foldsim_token_out o_bid (clk, reset, bid_valid, bid_ready, ok, finish, done[10]); assign awready_bits = aw_ready; assign wready_bits = w_ready; diff --git a/hw/rtl/afu/foldsim/foldsim_regs.vh b/hw/rtl/afu/foldsim/foldsim_regs.vh new file mode 100644 index 0000000000..882bbfcaaa --- /dev/null +++ b/hw/rtl/afu/foldsim/foldsim_regs.vh @@ -0,0 +1,57 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The foldsim simulator's register map, as sw/common/foldsim_regs.h gives it to +// the runtime. Byte offsets on s_axi_control; unlisted offsets read zero. +// +// 0x040-0x0BF is left empty: on the V80 the PCIe block's MSI-X table sits at +// that offset of the BAR the first kernel is mapped at, and answers those +// accesses itself. + +`ifndef FOLDSIM_REGS_VH +`define FOLDSIM_REGS_VH + +`define FOLDSIM_ID 32'h4D58534D +`define FOLDSIM_VERSION 32'd2 + +`define FOLDSIM_REG_ID 12'h000 +`define FOLDSIM_REG_VERSION 12'h004 +`define FOLDSIM_REG_CTRL 12'h008 +`define FOLDSIM_REG_STATUS 12'h00C +`define FOLDSIM_REG_CMD_ADDR 12'h010 +`define FOLDSIM_REG_CMD_DATA 12'h014 +`define FOLDSIM_REG_CMD 12'h018 +`define FOLDSIM_REG_DCR_RSP 12'h01C +`define FOLDSIM_REG_TCYCLES_LO 12'h020 +`define FOLDSIM_REG_TCYCLES_HI 12'h024 +`define FOLDSIM_REG_HCYCLES_LO 12'h028 +`define FOLDSIM_REG_HCYCLES_HI 12'h02C +`define FOLDSIM_REG_MSTALL_LO 12'h030 +`define FOLDSIM_REG_MSTALL_HI 12'h034 +`define FOLDSIM_REG_NUM_CLASSES 12'h038 +`define FOLDSIM_REG_MEM_BASE_LO 12'h0C0 +`define FOLDSIM_REG_MEM_BASE_HI 12'h0C4 +`define FOLDSIM_REG_MEM_RLAT 12'h0C8 +`define FOLDSIM_REG_MEM_WLAT 12'h0CC +`define FOLDSIM_REG_MEM_MAXR 12'h0D0 +`define FOLDSIM_REG_MEM_MAXW 12'h0D4 +// Per class c: the host cycles a thread of it finished its target cycle. +`define FOLDSIM_REG_FINISH_BASE 12'h100 + +`define FOLDSIM_CMD_NONE 3'd0 +`define FOLDSIM_CMD_DCR_WRITE 3'd1 +`define FOLDSIM_CMD_DCR_READ 3'd2 +`define FOLDSIM_CMD_START 3'd3 +`define FOLDSIM_CMD_RESET 3'd4 + +`endif // FOLDSIM_REGS_VH diff --git a/hw/rtl/afu/maxsim/maxsim_token.sv b/hw/rtl/afu/foldsim/foldsim_token.sv similarity index 94% rename from hw/rtl/afu/maxsim/maxsim_token.sv rename to hw/rtl/afu/foldsim/foldsim_token.sv index 531f9fb63d..9620a31265 100644 --- a/hw/rtl/afu/maxsim/maxsim_token.sv +++ b/hw/rtl/afu/foldsim/foldsim_token.sv @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// The token ends of a maxsim endpoint: a model of the target's surroundings +// The token ends of a foldsim endpoint: a model of the target's surroundings // that takes one token on each input channel and gives one on each output // channel per target cycle, finishing the cycle once it has them all. // @@ -20,7 +20,7 @@ // combinationally. // An input channel: the token is kept until the endpoint finishes its cycle. -module maxsim_token_in #( +module foldsim_token_in #( parameter W = 1 ) ( input wire clk, @@ -52,7 +52,7 @@ module maxsim_token_in #( endmodule // An output channel: the token is offered once `ok` until it is taken. -module maxsim_token_out ( +module foldsim_token_out ( input wire clk, input wire reset, output wire valid, diff --git a/hw/rtl/afu/foldsim/target/foldsim_target.sv b/hw/rtl/afu/foldsim/target/foldsim_target.sv new file mode 100644 index 0000000000..aa82553683 --- /dev/null +++ b/hw/rtl/afu/foldsim/target/foldsim_target.sv @@ -0,0 +1,151 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "vortex_afu.vh" + +// The foldsim target: Vortex_axi with its port surface flattened to packed, +// scalar ports, the boundary foldsim channelizes. +// +// Vortex_axi declares its AXI channels as SystemVerilog unpacked arrays indexed +// by bank; each bank is broken out into discrete m_axi_mem__* signals. The +// port shape and the array conversion are the same ones the XRT AFU uses, taken +// from its macros so the two cannot drift apart. +// +// The control surface (DCR request/response, start, busy) is already scalar and +// passes straight through, so the control endpoint drives it directly rather +// than through an AXI4-Lite control block. +module foldsim_target import VX_gpu_pkg::*; #( + parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, + parameter C_M_AXI_MEM_DATA_WIDTH = `VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8, + parameter C_M_AXI_MEM_ADDR_WIDTH = 64, + parameter C_M_AXI_MEM_NUM_BANKS = 1 +) ( + input wire clk, + input wire reset, + + `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), + + // DCR write request + input wire dcr_req_valid, + input wire dcr_req_rw, + input wire [VX_DCR_ADDR_WIDTH-1:0] dcr_req_addr, + input wire [VX_DCR_DATA_WIDTH-1:0] dcr_req_data, + + // DCR read response + output wire dcr_rsp_valid, + output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + + // ctrl/status + input wire start, + output wire busy +); + localparam VX_MEM_ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH; + + wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [VX_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS]; + + wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS]; + + wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS]; + + wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [VX_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS]; + + wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS]; + + `AXI_MEM_TO_ARRAY (0); + + Vortex_axi #( + .AXI_DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), + .AXI_ADDR_WIDTH (VX_MEM_ADDR_WIDTH), + .AXI_TID_WIDTH (C_M_AXI_MEM_ID_WIDTH), + .AXI_NUM_BANKS (C_M_AXI_MEM_NUM_BANKS) + ) vortex_axi ( + .clk (clk), + .reset (reset), + + .m_axi_awvalid (m_axi_mem_awvalid_a), + .m_axi_awready (m_axi_mem_awready_a), + .m_axi_awaddr (m_axi_mem_awaddr_a), + .m_axi_awid (m_axi_mem_awid_a), + .m_axi_awlen (m_axi_mem_awlen_a), + .m_axi_awsize (), + .m_axi_awburst (), + .m_axi_awlock (), + .m_axi_awcache (), + .m_axi_awprot (), + .m_axi_awqos (), + .m_axi_awregion (), + + .m_axi_wvalid (m_axi_mem_wvalid_a), + .m_axi_wready (m_axi_mem_wready_a), + .m_axi_wdata (m_axi_mem_wdata_a), + .m_axi_wstrb (m_axi_mem_wstrb_a), + .m_axi_wlast (m_axi_mem_wlast_a), + + .m_axi_bvalid (m_axi_mem_bvalid_a), + .m_axi_bready (m_axi_mem_bready_a), + .m_axi_bid (m_axi_mem_bid_a), + .m_axi_bresp (m_axi_mem_bresp_a), + + .m_axi_arvalid (m_axi_mem_arvalid_a), + .m_axi_arready (m_axi_mem_arready_a), + .m_axi_araddr (m_axi_mem_araddr_a), + .m_axi_arid (m_axi_mem_arid_a), + .m_axi_arlen (m_axi_mem_arlen_a), + .m_axi_arsize (), + .m_axi_arburst (), + .m_axi_arlock (), + .m_axi_arcache (), + .m_axi_arprot (), + .m_axi_arqos (), + .m_axi_arregion (), + + .m_axi_rvalid (m_axi_mem_rvalid_a), + .m_axi_rready (m_axi_mem_rready_a), + .m_axi_rdata (m_axi_mem_rdata_a), + .m_axi_rlast (m_axi_mem_rlast_a), + .m_axi_rid (m_axi_mem_rid_a), + .m_axi_rresp (m_axi_mem_rresp_a), + + .dcr_req_valid (dcr_req_valid), + .dcr_req_rw (dcr_req_rw), + .dcr_req_addr (dcr_req_addr), + .dcr_req_data (dcr_req_data), + + .dcr_rsp_valid (dcr_rsp_valid), + .dcr_rsp_data (dcr_rsp_data), + + .start (start), + .busy (busy) + ); + +endmodule diff --git a/hw/rtl/afu/maxsim/maxsim_regs.vh b/hw/rtl/afu/maxsim/maxsim_regs.vh deleted file mode 100644 index 6e9def4e42..0000000000 --- a/hw/rtl/afu/maxsim/maxsim_regs.vh +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The maxsim simulator's register map, as sw/common/maxsim_regs.h gives it to -// the runtime. Byte offsets on s_axi_control; unlisted offsets read zero. -// -// 0x040-0x0BF is left empty: on the V80 the PCIe block's MSI-X table sits at -// that offset of the BAR the first kernel is mapped at, and answers those -// accesses itself. - -`ifndef MAXSIM_REGS_VH -`define MAXSIM_REGS_VH - -`define MAXSIM_ID 32'h4D58534D -`define MAXSIM_VERSION 32'd2 - -`define MAXSIM_REG_ID 12'h000 -`define MAXSIM_REG_VERSION 12'h004 -`define MAXSIM_REG_CTRL 12'h008 -`define MAXSIM_REG_STATUS 12'h00C -`define MAXSIM_REG_CMD_ADDR 12'h010 -`define MAXSIM_REG_CMD_DATA 12'h014 -`define MAXSIM_REG_CMD 12'h018 -`define MAXSIM_REG_DCR_RSP 12'h01C -`define MAXSIM_REG_TCYCLES_LO 12'h020 -`define MAXSIM_REG_TCYCLES_HI 12'h024 -`define MAXSIM_REG_HCYCLES_LO 12'h028 -`define MAXSIM_REG_HCYCLES_HI 12'h02C -`define MAXSIM_REG_MSTALL_LO 12'h030 -`define MAXSIM_REG_MSTALL_HI 12'h034 -`define MAXSIM_REG_NUM_CLASSES 12'h038 -`define MAXSIM_REG_MEM_BASE_LO 12'h0C0 -`define MAXSIM_REG_MEM_BASE_HI 12'h0C4 -`define MAXSIM_REG_MEM_RLAT 12'h0C8 -`define MAXSIM_REG_MEM_WLAT 12'h0CC -`define MAXSIM_REG_MEM_MAXR 12'h0D0 -`define MAXSIM_REG_MEM_MAXW 12'h0D4 -// Per class c: the host cycles a thread of it finished its target cycle. -`define MAXSIM_REG_FINISH_BASE 12'h100 - -`define MAXSIM_CMD_NONE 3'd0 -`define MAXSIM_CMD_DCR_WRITE 3'd1 -`define MAXSIM_CMD_DCR_READ 3'd2 -`define MAXSIM_CMD_START 3'd3 -`define MAXSIM_CMD_RESET 3'd4 - -`endif // MAXSIM_REGS_VH diff --git a/hw/syn/common.mk b/hw/syn/common.mk index 7ade2a9809..b02c8519df 100644 --- a/hw/syn/common.mk +++ b/hw/syn/common.mk @@ -26,6 +26,8 @@ endif # and its elaboration into the one netlist a synthesis tool reads # (circt_elaborate.sh; gen_sources.sh -E). CIRCT_PATH ?= $(TOOLDIR)/circt +# foldsim builds FAME-5 simulators on CIRCT (hw/syn/foldsim). +FOLDSIM_PATH ?= $(TOOLDIR)/foldsim YOSYS_PATH ?= $(TOOLDIR)/yosys STA_PATH ?= $(TOOLDIR)/sta VERILATOR_PATH ?= $(TOOLDIR)/verilator @@ -35,8 +37,9 @@ VERILATOR_PATH ?= $(TOOLDIR)/verilator CIRCT_VERILOG ?= $(CIRCT_PATH)/bin/circt-verilog CIRCT_OPT ?= $(CIRCT_PATH)/bin/circt-opt FIRTOOL ?= $(CIRCT_PATH)/bin/firtool +FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim YOSYS ?= $(YOSYS_PATH)/bin/yosys STA ?= $(STA_PATH)/bin/sta VERILATOR ?= $(VERILATOR_PATH)/bin/verilator -export CIRCT_PATH YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL YOSYS STA VERILATOR +export CIRCT_PATH FOLDSIM_PATH YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL FOLDSIM YOSYS STA VERILATOR diff --git a/hw/syn/maxsim/Makefile b/hw/syn/foldsim/Makefile similarity index 75% rename from hw/syn/maxsim/Makefile rename to hw/syn/foldsim/Makefile index cf252c772e..fd9a3b35d7 100644 --- a/hw/syn/maxsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -1,25 +1,25 @@ -# maxsim image build for the V80, driven as the native aved flow +# foldsim image build for the V80, driven as the native aved flow # (hw/syn/xilinx/aved): # # make [DEVICE=] [PREFIX=...] [CONFIGS="-DVX_CFG_..."] [PERF=1] -# [MAXSIM_CLASSES="..."] [MAXSIM_LIFT="..."] [CIRCT_BIN=] +# [FOLDSIM_CLASSES="..."] [FOLDSIM_LIFT="..."] [CIRCT_BIN=] [FOLDSIM=] # -# CIRCT compiles Vortex into a simulator (maxsim.py): each class threaded +# foldsim compiles Vortex into a simulator, on CIRCT: each class threaded # through one physical copy (FAME-5), the rest of the target a model of its # own, and every channel between them in one token fabric. The kernel -# (hw/rtl/afu/maxsim) adds the memory and control endpoints around it. The -# image lands in $(BUILD_DIR)/bin/maxsim.vbin; run it with -# FPGA_BIN_DIR= ./ci/blackbox.sh --driver=maxsim --target=hw ... +# (hw/rtl/afu/foldsim) adds the memory and control endpoints around it. The +# image lands in $(BUILD_DIR)/bin/foldsim.vbin; run it with +# FPGA_BIN_DIR= ./ci/blackbox.sh --driver=foldsim --target=hw ... # The runtime's avedsim target simulates the same generated design in-process. ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk include $(VORTEX_HOME)/hw/syn/common.mk -SRC_DIR := $(VORTEX_HOME)/hw/syn/maxsim +SRC_DIR := $(VORTEX_HOME)/hw/syn/foldsim RTL_DIR := $(VORTEX_HOME)/hw/rtl SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts -MAXSIM_RTL_DIR := $(RTL_DIR)/afu/maxsim +FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim VIVADO := $(XILINX_VIVADO)/bin/vivado SLASHKIT ?= slashkit # Parallel link runs; each Vivado of a large simulator needs tens of GB. @@ -34,33 +34,34 @@ ifneq ($(TARGET), hw) $(error TARGET must be hw: the runtime's avedsim target simulates the generated design itself) endif -BUILD_DIR := $(abspath $(PREFIX)_maxsim_$(TARGET)) +BUILD_DIR := $(abspath $(PREFIX)_foldsim_$(TARGET)) # The runtime's in-process simulator generates the design in a directory of # its own. -ifneq ($(MAXSIM_WORK_DIR),) -BUILD_DIR := $(abspath $(MAXSIM_WORK_DIR)) +ifneq ($(FOLDSIM_WORK_DIR),) +BUILD_DIR := $(abspath $(FOLDSIM_WORK_DIR)) endif BIN_DIR := $(BUILD_DIR)/bin GEN_DIR := $(BUILD_DIR)/gen # The classes threaded by FAME-5, and the units given plain-port boundaries # first so they can be classes (lift_boundaries.py; [:drop=P,...][;port=P,...]). -MAXSIM_CLASSES ?= VX_socket__lift -MAXSIM_LIFT ?= VX_socket:drop=INSTANCE_ID;port=SOCKET_ID -# Where circt-verilog, circt-opt and firtool are (hw/syn/common.mk's CIRCT). +FOLDSIM_CLASSES ?= VX_socket__lift +FOLDSIM_LIFT ?= VX_socket:drop=INSTANCE_ID;port=SOCKET_ID +# Where circt-verilog, circt-opt and firtool are (hw/syn/common.mk's CIRCT); +# the foldsim driver is hw/syn/common.mk's FOLDSIM. CIRCT_BIN ?= $(CIRCT_PATH)/bin LIFT_PYTHON ?= python3 # The kernel clock (MHz): the image is timed at it and the shell programs it. -MAXSIM_FREQ ?= 143 +FOLDSIM_FREQ ?= 143 # The target: Vortex with its AXI4 port, DCR, start and busy as plain ports. -TARGET_TOP := VX_firesim_wrap +TARGET_TOP := foldsim_target # The same tree the native synthesis flows elaborate. CONFIGS ?= RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs \ -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm \ - -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(RTL_DIR)/afu/firesim + -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(FOLDSIM_RTL_DIR)/target SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG # One memory port, as the memory endpoint models; 4 AXI id bits, what Vortex # drives with its tag buffer engaged. The FPU and RTU use Vortex's own @@ -90,25 +91,25 @@ endef CONFIG_STAMP := $(BUILD_DIR)/cflags.stamp CLASSES_STAMP := $(BUILD_DIR)/classes.stamp $(call stamp,$(CONFIG_STAMP),$(GEN_CFLAGS)) -$(call stamp,$(CLASSES_STAMP),$(MAXSIM_CLASSES) | $(MAXSIM_LIFT)) +$(call stamp,$(CLASSES_STAMP),$(FOLDSIM_CLASSES) | $(FOLDSIM_LIFT)) SRCS_NAME := vortex_srcs SRCS_DIR := $(BUILD_DIR)/$(SRCS_NAME) SRCS_LIST := $(BUILD_DIR)/$(SRCS_NAME).f LIFT_LIST := $(BUILD_DIR)/lift/srcs.f -TARGET_SRCS := $(if $(strip $(MAXSIM_LIFT)),$(LIFT_LIST),$(SRCS_LIST)) -GEN_SV := $(GEN_DIR)/maxsim_sim.sv +TARGET_SRCS := $(if $(strip $(FOLDSIM_LIFT)),$(LIFT_LIST),$(SRCS_LIST)) +GEN_SV := $(GEN_DIR)/foldsim_sim.sv KERNEL_SRCS := $(BUILD_DIR)/kernel_srcs.f -IP_DIR := $(BUILD_DIR)/maxsim_afu_ip +IP_DIR := $(BUILD_DIR)/foldsim_afu_ip CFG := $(BUILD_DIR)/config.cfg -VBIN := $(BIN_DIR)/maxsim.vbin +VBIN := $(BIN_DIR)/foldsim.vbin RTL_SRCS := $(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh') \ $(wildcard $(ROOT_DIR)/hw/VX_config.vh) $(wildcard $(ROOT_DIR)/hw/VX_types.vh) \ $(firstword $(MAKEFILE_LIST)) $(CONFIG_STAMP) -# The AFU shells of the native flows are second tops, and the maxsim kernel is +# The AFU shells of the native flows are second tops, and the foldsim kernel is # compiled apart from the target. -SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv) $(wildcard $(MAXSIM_RTL_DIR)/*.sv)) +SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv) $(wildcard $(FOLDSIM_RTL_DIR)/*.sv)) .PHONY: all sources generate package-ip clean FORCE @@ -139,28 +140,28 @@ $(SRCS_LIST): $(RTL_SRCS) $(LIFT_LIST): $(SRCS_LIST) $(SRC_DIR)/lift_boundaries.py $(CLASSES_STAMP) rm -rf $(BUILD_DIR)/lift $(LIFT_PYTHON) $(SRC_DIR)/lift_boundaries.py --srcs $(SRCS_LIST) --top $(TARGET_TOP) \ - --out $(BUILD_DIR)/lift $(foreach u,$(MAXSIM_LIFT),--unit '$(u)') + --out $(BUILD_DIR)/lift $(foreach u,$(FOLDSIM_LIFT),--unit '$(u)') generate: $(GEN_SV) -$(GEN_SV): $(TARGET_SRCS) $(SRC_DIR)/maxsim.py $(CLASSES_STAMP) - python3 $(SRC_DIR)/maxsim.py --srcs $(TARGET_SRCS) --top $(TARGET_TOP) --out $(GEN_DIR) \ - $(if $(CIRCT_BIN),--circt-bin $(CIRCT_BIN)) $(MAXSIM_CLASSES) +$(GEN_SV): $(TARGET_SRCS) $(FOLDSIM) $(CLASSES_STAMP) + $(FOLDSIM) --srcs $(TARGET_SRCS) --top $(TARGET_TOP) --out $(GEN_DIR) \ + $(if $(CIRCT_BIN),--circt-bin $(CIRCT_BIN)) $(FOLDSIM_CLASSES) $(KERNEL_SRCS): $(GEN_SV) - @{ echo "+incdir+$(MAXSIM_RTL_DIR)"; echo "+incdir+$(GEN_DIR)"; \ + @{ echo "+incdir+$(FOLDSIM_RTL_DIR)"; echo "+incdir+$(GEN_DIR)"; \ echo "$(GEN_SV)"; \ - for f in maxsim_token.sv maxsim_mem.sv maxsim_ctrl.sv maxsim_afu.sv; do echo "$(MAXSIM_RTL_DIR)/$$f"; done; \ + for f in foldsim_token.sv foldsim_mem.sv foldsim_ctrl.sv foldsim_afu.sv; do echo "$(FOLDSIM_RTL_DIR)/$$f"; done; \ } > $@ package-ip: $(IP_DIR)/component.xml -$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(wildcard $(MAXSIM_RTL_DIR)/*) +$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(wildcard $(FOLDSIM_RTL_DIR)/*) cd $(BUILD_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -nojournal -nolog \ -source $(VORTEX_HOME)/hw/syn/xilinx/aved/package_ip.tcl \ - -tclargs maxsim_afu $(KERNEL_SRCS) . $(DEVICE) no-vendor-ip + -tclargs foldsim_afu $(KERNEL_SRCS) . $(DEVICE) no-vendor-ip $(CFG): FORCE - @python3 $(SCRIPT_DIR)/gen_platform.py --flow=aved --device=$(DEVICE) --emit=cfg --kernel=maxsim_afu \ - --no-host --kernel-mhz=$(MAXSIM_FREQ) -o $@.tmp + @python3 $(SCRIPT_DIR)/gen_platform.py --flow=aved --device=$(DEVICE) --emit=cfg --kernel=foldsim_afu \ + --no-host --kernel-mhz=$(FOLDSIM_FREQ) -o $@.tmp @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ FORCE: diff --git a/hw/syn/maxsim/lift_boundaries.py b/hw/syn/foldsim/lift_boundaries.py similarity index 99% rename from hw/syn/maxsim/lift_boundaries.py rename to hw/syn/foldsim/lift_boundaries.py index 43b8950879..282551bfe4 100644 --- a/hw/syn/maxsim/lift_boundaries.py +++ b/hw/syn/foldsim/lift_boundaries.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Interface boundary lifting: make interface-port units maxsim classes. +"""Interface boundary lifting: make interface-port units foldsim classes. A class is threaded and wired into the token fabric port by port, so it must be a module whose ports are plain signals; units such as VX_issue_slice or diff --git a/hw/syn/maxsim/maxsim.py b/hw/syn/maxsim/maxsim.py deleted file mode 100644 index 195a2ab643..0000000000 --- a/hw/syn/maxsim/maxsim.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -"""Compile a target to a maxsim simulator with CIRCT. - -usage: maxsim.py --srcs --top --out [--circt-bin ] - [ ...] - -The target is imported with circt-verilog and flattened, but for the classes: -the modules simulated as models of their own, each class's instances threaded -through one copy (FAME-5). A class may contain others. Each module with class -instances, innermost first, is cut into a netlist of its own logic and the -models (maxsim-extract-models); the target's netlist is then composed into the -simulator (maxsim-compose): one threaded module per class, the rest of the -target as a model of one thread, and a fabric of token channels between them. -A class `A+B` is two modules of the same parent whose instances are paired and -simulated as one model (maxsim-fuse-models): what they exchange combinationally -both ways then takes a thread one host cycle, not a round trip through channels. - -Writes /maxsim_sim.sv, the module `maxsim_sim` with a channel per target -port (`_valid`, `_ready`, `_bits`) and a `profile_finish` bit per class; -/maxsim_sim.json, its port list and the classes' thread counts, in the -order of those bits; and /maxsim_sim.vh, their number for the kernel. -""" -import argparse -import json -import os -import re -import subprocess -import sys - -# Passes each model gets before it is threaded: register constants, shared -# adders and narrowed low bits matter for area once every register of a model -# is replicated per thread. A conditional write of one array element becomes a -# select per element, and sorting lets CSE merge duplicates whose users come -# first in a module body. -OPTIMIZE = ['hw.module(hw-expand-array-writes)', 'hw.module(seq-reg-const-prop)', - 'hw.module(comb-share-adders)', 'canonicalize', 'hw.module(comb-low-bits-narrowing)', - 'canonicalize', 'topological-sort', 'cse'] - -# Logic of up to this many ops that drives a model's input from other models -# is computed in the model, so the input comes from them directly rather than -# through the parent's model. -ABSORB_LOGIC = 64 - - -def run(cmd, **kw): - print('+', ' '.join(cmd), file=sys.stderr) - return subprocess.run(cmd, check=True, text=True, **kw) - - -def publish(src, public): - """The design with only the `public` modules public.""" - s = re.sub(r'hw\.module @', 'hw.module private @', src) - for m in public: - s, n = re.subn(r'hw\.module private @' + re.escape(m) + r'\(', f'hw.module @{m}(', s, count=1) - if not n: - sys.exit(f'no module {m} in the design') - return s - - -def instantiated(text, module, names): - """The modules of `names` the module `module` instantiates.""" - m = re.search(r'hw\.module (?:private )?@' + re.escape(module) + r'\(.*?\n \}\n', text, re.S) - return {n for n in re.findall(r'hw\.instance "[^"]*" @(\w+)\(', m.group(0)) if n in names} - - -def signature(text, module): - """A module's ports: [(direction, name, width)].""" - m = re.search(r'hw\.module (?:private )?@' + re.escape(module) + r'\((.*?)\) (?:attributes .*?)?\{', - text, re.S) - ports = [] - for d, n, t in re.findall(r'(in|out) %?(\w+) : ([^,)]+)', m.group(1)): - w = re.fullmatch(r'i(\d+)', t.strip()) - ports.append((d, n, int(w.group(1)) if w else 0)) - return ports - - -def main(): - ap = argparse.ArgumentParser(description=__doc__.split('\n')[0]) - ap.add_argument('--srcs', required=True, help='source list: +define+, +incdir+ and files') - ap.add_argument('--top', required=True) - ap.add_argument('--out', required=True) - ap.add_argument('--circt-bin', default='') - ap.add_argument('classes', nargs='+') - a = ap.parse_args() - tool = lambda t: os.path.join(a.circt_bin, t) if a.circt_bin else t - os.makedirs(a.out, exist_ok=True) - out = lambda f: os.path.join(a.out, f) - - defines, incdirs, files = [], [], [] - for line in open(a.srcs).read().split('\n'): - if line.startswith('+define+'): - defines.append('-D' + line[len('+define+'):]) - elif line.startswith('+incdir+'): - incdirs.append('-I' + line[len('+incdir+'):]) - elif line.strip(): - files.append(line.strip()) - design = out('design.mlir') - run([tool('circt-verilog'), '--ir-hw', f'--top={a.top}', '--allow-use-before-declare', - '--allow-hierarchical-const', '--ignore-uninstantiated-modules', '--detect-memories', - '--preserve-unpacked-ports', *defines, *incdirs, *files, '-o', design]) - - groups = [c.split('+') for c in a.classes if '+' in c] - if any(len(g) != 2 for g in groups): - sys.exit('a fused class is two modules, A+B') - modules = [m for c in a.classes for m in c.split('+')] - - # Flat but for the classes. - flat = out('flat.mlir') - with open(flat + '.in', 'w') as f: - f.write(publish(open(design).read(), [a.top] + modules)) - # A RAM that asks for block RAM reads synchronously once its wrapper is - # flattened into the module that registers its address. - run([tool('circt-opt'), flat + '.in', '--symbol-dce', '--hw-flatten-modules', '--symbol-dce', - '--canonicalize', '--cse', f'--seq-retime-firmem-reads=report={out("memories.rpt")}', - '--canonicalize', '--cse', '-o', flat]) - os.remove(flat + '.in') - text = open(flat).read() - - # Each module with class instances, innermost first, becomes a netlist. - classes = set(modules) - children = {m: instantiated(text, m, classes) for m in [a.top] + modules} - order, seen = [], set() - - def visit(m): - if m in seen: - return - seen.add(m) - for c in sorted(children[m]): - visit(c) - if children[m]: - order.append(m) - visit(a.top) - unused = classes - seen - if unused: - sys.exit(f'classes not in {a.top}: {" ".join(sorted(unused))}') - passes = [] - for m in order: - passes.append(f'maxsim-extract-models{{module={m} models={",".join(sorted(children[m]))} netlist absorb-logic={ABSORB_LOGIC}}}') - fused = [g for g in groups if set(g) <= children[m]] - if fused: - passes.append(f'maxsim-fuse-models{{module={m} groups={",".join("+".join(g) for g in fused)}}}') - for g in groups: - if not any(set(g) <= children[m] for m in order): - sys.exit(f'{"+".join(g)}: not instantiated by the same module') - pipeline = passes + OPTIMIZE + [f'maxsim-compose{{module={a.top} sim=maxsim_sim}}', - 'canonicalize', 'topological-sort', 'cse'] - sim = out('maxsim_sim.mlir') - run([tool('circt-opt'), flat, f'--pass-pipeline=builtin.module({",".join(pipeline)})', '-o', sim]) - sim_text = open(sim).read() - # A RAM with a registered read states its read-under-write behavior to - # Vivado, which otherwise leaves a read colliding with a write undefined. - run([tool('firtool'), '--format=mlir', sim, - '--add-vivado-ram-address-conflict-synthesis-bug-workaround', '-o', out('maxsim_sim.sv')]) - - threads = re.search(r'maxsim\.threads = \[(.*?)\]', sim_text).group(1) - info = { - 'top': a.top, - 'threads': dict(t.strip('"').split('=') for t in re.findall(r'"[^"]*"', threads)), - 'ports': [{'dir': d, 'name': n, 'width': w} for d, n, w in signature(sim_text, 'maxsim_sim')], - } - for name, n in info['threads'].items(): - info['threads'][name] = int(n) - with open(out('maxsim_sim.json'), 'w') as f: - json.dump(info, f, indent=1) - # The kernel's view of it: how many classes report their finishes, bit c - # of `profile_finish` for the c-th class of the JSON's `threads`. - with open(out('maxsim_sim.vh'), 'w') as f: - f.write('// Generated by maxsim.py. Do not edit.\n' - f'`define MAXSIM_NUM_CLASSES {len(info["threads"])}\n') - print(f'maxsim: {", ".join(f"{k} x{v}" for k, v in info["threads"].items())}', file=sys.stderr) - - -if __name__ == '__main__': - main() diff --git a/hw/syn/xilinx/aved/package_ip.tcl b/hw/syn/xilinx/aved/package_ip.tcl index b1b9bb0ed6..be4b3ec5ca 100644 --- a/hw/syn/xilinx/aved/package_ip.tcl +++ b/hw/syn/xilinx/aved/package_ip.tcl @@ -24,7 +24,7 @@ set krnl_name [lindex $::argv 0] set vcs_file [lindex $::argv 1] set build_dir [lindex $::argv 2] set part [lindex $::argv 3] -# A kernel whose RTL instantiates no vendor floating-point core (a maxsim +# A kernel whose RTL instantiates no vendor floating-point core (a foldsim # simulator, whose FPU is soft logic) is packaged without them. set vendor_ip [expr {$::argc != 5 || [lindex $::argv 4] ne "no-vendor-ip"}] diff --git a/sim/foldsim/Makefile b/sim/foldsim/Makefile new file mode 100644 index 0000000000..fc8b32a051 --- /dev/null +++ b/sim/foldsim/Makefile @@ -0,0 +1,56 @@ +include ../common.mk + +# The foldsim kernel in Verilator, as a library the runtime's avedsim target +# links: the same simulator hw/syn/foldsim generates for the image, so a run +# here is the image's run, cycle for cycle, but for device memory's timing. + +DESTDIR ?= $(CURDIR) + +SRC_DIR := $(VORTEX_HOME)/sim/foldsim +FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim +WORK_DIR := $(DESTDIR)/foldsim_emul +GEN_SV := $(WORK_DIR)/gen/foldsim_sim.sv + +CONFIG_FILE := $(DESTDIR)/foldsim_emul_config.stamp + +CXXFLAGS += -std=c++17 -fPIC -O2 -DNDEBUG -I$(SRC_DIR) +LDFLAGS += -shared -pthread + +VL_FLAGS += -O3 -Wno-fatal -Wno-lint -Wno-style -Wno-MULTIDRIVEN --x-initial 0 --x-assign 0 +VL_FLAGS += -I$(FOLDSIM_RTL_DIR) -I$(WORK_DIR)/gen +ifdef VCD +VL_FLAGS += --trace --trace-structs +endif +THREADS ?= $(shell python3 -c 'import multiprocessing as mp; print(min(32, mp.cpu_count()))') +VL_FLAGS += -j $(THREADS) + +TOP := foldsim_afu +RTL_SRCS := $(GEN_SV) $(FOLDSIM_RTL_DIR)/foldsim_token.sv $(FOLDSIM_RTL_DIR)/foldsim_mem.sv \ + $(FOLDSIM_RTL_DIR)/foldsim_ctrl.sv $(FOLDSIM_RTL_DIR)/foldsim_afu.sv +SRCS := $(SRC_DIR)/foldsim_emul.cpp + +PROJECT := libfoldsim_emul.so + +.PHONY: all lib force clean + +all: lib +lib: $(DESTDIR)/$(PROJECT) + +# The simulator, generated for these CONFIGS by the image's own flow. +$(GEN_SV): force + $(MAKE) -C $(ROOT_DIR)/hw/syn/foldsim generate TARGET=hw FOLDSIM_WORK_DIR=$(WORK_DIR) CONFIGS='$(CONFIGS)' \ + $(if $(PERF),PERF=$(PERF)) $(if $(FOLDSIM_CLASSES),FOLDSIM_CLASSES='$(FOLDSIM_CLASSES)') \ + $(if $(FOLDSIM_LIFT),FOLDSIM_LIFT='$(FOLDSIM_LIFT)') $(if $(CIRCT_BIN),CIRCT_BIN=$(CIRCT_BIN)) \ + $(if $(LIFT_PYTHON),LIFT_PYTHON=$(LIFT_PYTHON)) $(if $(FOLDSIM),FOLDSIM=$(FOLDSIM)) + +$(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(wildcard $(FOLDSIM_RTL_DIR)/*) $(CONFIG_FILE) + $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(RTL_SRCS) --top-module $(TOP) \ + $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + +$(CONFIG_FILE): force + @mkdir -p $(@D) + @printf '%s\n' "$(VL_FLAGS)" "$(CXXFLAGS)" "$(CONFIGS)" > $@.tmp + @if ! cmp -s $@.tmp $@; then mv $@.tmp $@; else rm $@.tmp; fi + +clean: + rm -rf $(DESTDIR)/$(PROJECT).obj_dir $(DESTDIR)/$(PROJECT) $(WORK_DIR) diff --git a/sim/maxsim/maxsim_emul.cpp b/sim/foldsim/foldsim_emul.cpp similarity index 84% rename from sim/maxsim/maxsim_emul.cpp rename to sim/foldsim/foldsim_emul.cpp index 8280fa2182..45e0b509f0 100644 --- a/sim/maxsim/maxsim_emul.cpp +++ b/sim/foldsim/foldsim_emul.cpp @@ -11,10 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "maxsim_emul.h" +#include "foldsim_emul.h" -#include "Vmaxsim_afu.h" -#include "Vmaxsim_afu___024root.h" +#include "Vfoldsim_afu.h" +#include "Vfoldsim_afu___024root.h" #include "verilated.h" #ifdef VM_TRACE #include "verilated_vcd_c.h" @@ -69,11 +69,11 @@ class memory_t { } // namespace -class maxsim_emul::Impl { +class foldsim_emul::Impl { public: Impl() - : mem_latency_(env_or("MAXSIM_EMUL_MEM_LATENCY", 100)), - mmio_cycles_(env_or("MAXSIM_EMUL_MMIO_CYCLES", 250)) {} + : mem_latency_(env_or("FOLDSIM_EMUL_MEM_LATENCY", 100)), + mmio_cycles_(env_or("FOLDSIM_EMUL_MMIO_CYCLES", 250)) {} ~Impl() { #ifdef VM_TRACE @@ -91,28 +91,28 @@ class maxsim_emul::Impl { } int init() { - // MAXSIM_EMUL_TRACE=: which classes finish a thread, each kernel + // FOLDSIM_EMUL_TRACE=: which classes finish a thread, each kernel // cycle, one byte per cycle of up to 8 classes. - if (const char* path = getenv("MAXSIM_EMUL_TRACE")) + if (const char* path = getenv("FOLDSIM_EMUL_TRACE")) trace_ = fopen(path, "wb"); - // MAXSIM_EMUL_AXI_TRACE=: the kernel's memory port, one line per + // FOLDSIM_EMUL_AXI_TRACE=: the kernel's memory port, one line per // request and response. - if (const char* path = getenv("MAXSIM_EMUL_AXI_TRACE")) + if (const char* path = getenv("FOLDSIM_EMUL_AXI_TRACE")) axi_trace_ = fopen(path, "w"); - // MAXSIM_EMUL_REG_TRACE=: every register access of the host. - if (const char* path = getenv("MAXSIM_EMUL_REG_TRACE")) + // FOLDSIM_EMUL_REG_TRACE=: every register access of the host. + if (const char* path = getenv("FOLDSIM_EMUL_REG_TRACE")) reg_trace_ = fopen(path, "w"); context_ = std::make_unique(); - top_ = std::make_unique(context_.get()); + top_ = std::make_unique(context_.get()); #ifdef VM_TRACE - // MAXSIM_EMUL_VCD=: a waveform of the kernel, in a build with --trace, - // from kernel cycle MAXSIM_EMUL_VCD_FROM on, MAXSIM_EMUL_VCD_DEPTH levels + // FOLDSIM_EMUL_VCD=: a waveform of the kernel, in a build with --trace, + // from kernel cycle FOLDSIM_EMUL_VCD_FROM on, FOLDSIM_EMUL_VCD_DEPTH levels // of hierarchy deep (2: the kernel's channels). - vcd_from_ = env_or("MAXSIM_EMUL_VCD_FROM", 0); - if (const char* path = getenv("MAXSIM_EMUL_VCD")) { + vcd_from_ = env_or("FOLDSIM_EMUL_VCD_FROM", 0); + if (const char* path = getenv("FOLDSIM_EMUL_VCD")) { context_->traceEverOn(true); vcd_ = std::make_unique(); - top_->trace(vcd_.get(), int(env_or("MAXSIM_EMUL_VCD_DEPTH", 99))); + top_->trace(vcd_.get(), int(env_or("FOLDSIM_EMUL_VCD_DEPTH", 99))); vcd_->open(path); } #endif @@ -255,7 +255,7 @@ class maxsim_emul::Impl { bool b_fire = top_->m_axi_mem_0_bvalid && top_->m_axi_mem_0_bready; if (trace_ != nullptr) - fputc(top_->rootp->maxsim_afu__DOT__profile_finish & 0xff, trace_); + fputc(top_->rootp->foldsim_afu__DOT__profile_finish & 0xff, trace_); top_->ap_clk = 1; top_->eval(); #ifdef VM_TRACE @@ -303,7 +303,7 @@ class maxsim_emul::Impl { } std::unique_ptr context_; - std::unique_ptr top_; + std::unique_ptr top_; #ifdef VM_TRACE std::unique_ptr vcd_; uint64_t vcd_from_ = 0; @@ -321,24 +321,24 @@ class maxsim_emul::Impl { std::deque writes_; }; -maxsim_emul::maxsim_emul() : impl_(new Impl()) {} +foldsim_emul::foldsim_emul() : impl_(new Impl()) {} -maxsim_emul::~maxsim_emul() { delete impl_; } +foldsim_emul::~foldsim_emul() { delete impl_; } -int maxsim_emul::init() { return impl_->init(); } +int foldsim_emul::init() { return impl_->init(); } -int maxsim_emul::reg_write(uint32_t offset, uint32_t value) { +int foldsim_emul::reg_write(uint32_t offset, uint32_t value) { return impl_->reg_write(offset, value); } -int maxsim_emul::reg_read(uint32_t offset, uint32_t* value) { +int foldsim_emul::reg_read(uint32_t offset, uint32_t* value) { return impl_->reg_read(offset, value); } -int maxsim_emul::mem_write(uint64_t addr, const void* data, uint64_t size) { +int foldsim_emul::mem_write(uint64_t addr, const void* data, uint64_t size) { return impl_->mem_write(addr, data, size); } -int maxsim_emul::mem_read(uint64_t addr, void* data, uint64_t size) { +int foldsim_emul::mem_read(uint64_t addr, void* data, uint64_t size) { return impl_->mem_read(addr, data, size); } diff --git a/sim/maxsim/maxsim_emul.h b/sim/foldsim/foldsim_emul.h similarity index 85% rename from sim/maxsim/maxsim_emul.h rename to sim/foldsim/foldsim_emul.h index f5b0adc522..38ecbc988f 100644 --- a/sim/maxsim/maxsim_emul.h +++ b/sim/foldsim/foldsim_emul.h @@ -17,16 +17,16 @@ namespace vortex { -// The maxsim kernel (hw/rtl/afu/maxsim) simulated in-process, as the +// The foldsim kernel (hw/rtl/afu/foldsim) simulated in-process, as the // runtime's avedsim target reaches it: its control registers, and the device // memory behind its AXI4 port. The kernel clock runs only while the host // touches a register, each access costing the kernel cycles a register access -// over PCIe would (MAXSIM_EMUL_MMIO_CYCLES, 250 by default), so a host that +// over PCIe would (FOLDSIM_EMUL_MMIO_CYCLES, 250 by default), so a host that // polls a register is what lets the target run. -class maxsim_emul { +class foldsim_emul { public: - maxsim_emul(); - ~maxsim_emul(); + foldsim_emul(); + ~foldsim_emul(); int init(); diff --git a/sim/maxsim/Makefile b/sim/maxsim/Makefile deleted file mode 100644 index 2f66c89b00..0000000000 --- a/sim/maxsim/Makefile +++ /dev/null @@ -1,56 +0,0 @@ -include ../common.mk - -# The maxsim kernel in Verilator, as a library the runtime's avedsim target -# links: the same simulator hw/syn/maxsim generates for the image, so a run -# here is the image's run, cycle for cycle, but for device memory's timing. - -DESTDIR ?= $(CURDIR) - -SRC_DIR := $(VORTEX_HOME)/sim/maxsim -MAXSIM_RTL_DIR := $(RTL_DIR)/afu/maxsim -WORK_DIR := $(DESTDIR)/maxsim_emul -GEN_SV := $(WORK_DIR)/gen/maxsim_sim.sv - -CONFIG_FILE := $(DESTDIR)/maxsim_emul_config.stamp - -CXXFLAGS += -std=c++17 -fPIC -O2 -DNDEBUG -I$(SRC_DIR) -LDFLAGS += -shared -pthread - -VL_FLAGS += -O3 -Wno-fatal -Wno-lint -Wno-style -Wno-MULTIDRIVEN --x-initial 0 --x-assign 0 -VL_FLAGS += -I$(MAXSIM_RTL_DIR) -I$(WORK_DIR)/gen -ifdef VCD -VL_FLAGS += --trace --trace-structs -endif -THREADS ?= $(shell python3 -c 'import multiprocessing as mp; print(min(32, mp.cpu_count()))') -VL_FLAGS += -j $(THREADS) - -TOP := maxsim_afu -RTL_SRCS := $(GEN_SV) $(MAXSIM_RTL_DIR)/maxsim_token.sv $(MAXSIM_RTL_DIR)/maxsim_mem.sv \ - $(MAXSIM_RTL_DIR)/maxsim_ctrl.sv $(MAXSIM_RTL_DIR)/maxsim_afu.sv -SRCS := $(SRC_DIR)/maxsim_emul.cpp - -PROJECT := libmaxsim_emul.so - -.PHONY: all lib force clean - -all: lib -lib: $(DESTDIR)/$(PROJECT) - -# The simulator, generated for these CONFIGS by the image's own flow. -$(GEN_SV): force - $(MAKE) -C $(ROOT_DIR)/hw/syn/maxsim generate TARGET=hw MAXSIM_WORK_DIR=$(WORK_DIR) CONFIGS='$(CONFIGS)' \ - $(if $(PERF),PERF=$(PERF)) $(if $(MAXSIM_CLASSES),MAXSIM_CLASSES='$(MAXSIM_CLASSES)') \ - $(if $(MAXSIM_LIFT),MAXSIM_LIFT='$(MAXSIM_LIFT)') $(if $(CIRCT_BIN),CIRCT_BIN=$(CIRCT_BIN)) \ - $(if $(LIFT_PYTHON),LIFT_PYTHON=$(LIFT_PYTHON)) - -$(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(wildcard $(MAXSIM_RTL_DIR)/*) $(CONFIG_FILE) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(RTL_SRCS) --top-module $(TOP) \ - $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ - -$(CONFIG_FILE): force - @mkdir -p $(@D) - @printf '%s\n' "$(VL_FLAGS)" "$(CXXFLAGS)" "$(CONFIGS)" > $@.tmp - @if ! cmp -s $@.tmp $@; then mv $@.tmp $@; else rm $@.tmp; fi - -clean: - rm -rf $(DESTDIR)/$(PROJECT).obj_dir $(DESTDIR)/$(PROJECT) $(WORK_DIR) diff --git a/sw/common/foldsim_regs.h b/sw/common/foldsim_regs.h new file mode 100644 index 0000000000..695aa9c363 --- /dev/null +++ b/sw/common/foldsim_regs.h @@ -0,0 +1,55 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The foldsim simulator's register map (hw/rtl/afu/foldsim/foldsim_regs.vh). +// Byte offsets on the kernel's AXI4-Lite control port. 0x040-0x0BF is left +// empty: on the V80 the PCIe block's MSI-X table sits at that offset of the BAR +// the first kernel is mapped at, and answers those accesses itself. + +#pragma once + +#define FOLDSIM_ID 0x4D58534Du +#define FOLDSIM_VERSION 2u + +#define FOLDSIM_REG_ID 0x000u +#define FOLDSIM_REG_VERSION 0x004u +#define FOLDSIM_REG_CTRL 0x008u // bit 0: the target runs +#define FOLDSIM_REG_STATUS 0x00Cu +#define FOLDSIM_REG_CMD_ADDR 0x010u // DCR address of the next command +#define FOLDSIM_REG_CMD_DATA 0x014u // DCR data, or reset cycles +#define FOLDSIM_REG_CMD 0x018u // write: queue a FOLDSIM_CMD_* +#define FOLDSIM_REG_DCR_RSP 0x01Cu // the last DCR read's response +#define FOLDSIM_REG_TCYCLES_LO 0x020u // target cycles since the kernel's reset +#define FOLDSIM_REG_TCYCLES_HI 0x024u +#define FOLDSIM_REG_HCYCLES_LO 0x028u // host (kernel clock) cycles +#define FOLDSIM_REG_HCYCLES_HI 0x02Cu +#define FOLDSIM_REG_MSTALL_LO 0x030u // host cycles waiting for device memory +#define FOLDSIM_REG_MSTALL_HI 0x034u +#define FOLDSIM_REG_NUM_CLASSES 0x038u // classes of threaded models +#define FOLDSIM_REG_MEM_BASE_LO 0x0C0u // device address of target address 0 +#define FOLDSIM_REG_MEM_BASE_HI 0x0C4u +#define FOLDSIM_REG_MEM_RLAT 0x0C8u // read latency, target cycles +#define FOLDSIM_REG_MEM_WLAT 0x0CCu // write latency, target cycles +#define FOLDSIM_REG_MEM_MAXR 0x0D0u // most reads in flight +#define FOLDSIM_REG_MEM_MAXW 0x0D4u // most writes in flight +#define FOLDSIM_REG_FINISH(c) (0x100u + 8u * (c)) // threads of class c finished, lo/hi + +#define FOLDSIM_STATUS_BUSY 0x1u // queued, running, or memory in flight +#define FOLDSIM_STATUS_FULL 0x2u // the command queue is full +#define FOLDSIM_STATUS_RSP 0x4u // a DCR read has its response +#define FOLDSIM_STATUS_MEM_ERR 0x8u // the target made a burst + +#define FOLDSIM_CMD_DCR_WRITE 1u +#define FOLDSIM_CMD_DCR_READ 2u +#define FOLDSIM_CMD_START 3u +#define FOLDSIM_CMD_RESET 4u diff --git a/sw/common/maxsim_regs.h b/sw/common/maxsim_regs.h deleted file mode 100644 index 55532d0e65..0000000000 --- a/sw/common/maxsim_regs.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The maxsim simulator's register map (hw/rtl/afu/maxsim/maxsim_regs.vh). -// Byte offsets on the kernel's AXI4-Lite control port. 0x040-0x0BF is left -// empty: on the V80 the PCIe block's MSI-X table sits at that offset of the BAR -// the first kernel is mapped at, and answers those accesses itself. - -#pragma once - -#define MAXSIM_ID 0x4D58534Du -#define MAXSIM_VERSION 2u - -#define MAXSIM_REG_ID 0x000u -#define MAXSIM_REG_VERSION 0x004u -#define MAXSIM_REG_CTRL 0x008u // bit 0: the target runs -#define MAXSIM_REG_STATUS 0x00Cu -#define MAXSIM_REG_CMD_ADDR 0x010u // DCR address of the next command -#define MAXSIM_REG_CMD_DATA 0x014u // DCR data, or reset cycles -#define MAXSIM_REG_CMD 0x018u // write: queue a MAXSIM_CMD_* -#define MAXSIM_REG_DCR_RSP 0x01Cu // the last DCR read's response -#define MAXSIM_REG_TCYCLES_LO 0x020u // target cycles since the kernel's reset -#define MAXSIM_REG_TCYCLES_HI 0x024u -#define MAXSIM_REG_HCYCLES_LO 0x028u // host (kernel clock) cycles -#define MAXSIM_REG_HCYCLES_HI 0x02Cu -#define MAXSIM_REG_MSTALL_LO 0x030u // host cycles waiting for device memory -#define MAXSIM_REG_MSTALL_HI 0x034u -#define MAXSIM_REG_NUM_CLASSES 0x038u // classes of threaded models -#define MAXSIM_REG_MEM_BASE_LO 0x0C0u // device address of target address 0 -#define MAXSIM_REG_MEM_BASE_HI 0x0C4u -#define MAXSIM_REG_MEM_RLAT 0x0C8u // read latency, target cycles -#define MAXSIM_REG_MEM_WLAT 0x0CCu // write latency, target cycles -#define MAXSIM_REG_MEM_MAXR 0x0D0u // most reads in flight -#define MAXSIM_REG_MEM_MAXW 0x0D4u // most writes in flight -#define MAXSIM_REG_FINISH(c) (0x100u + 8u * (c)) // threads of class c finished, lo/hi - -#define MAXSIM_STATUS_BUSY 0x1u // queued, running, or memory in flight -#define MAXSIM_STATUS_FULL 0x2u // the command queue is full -#define MAXSIM_STATUS_RSP 0x4u // a DCR read has its response -#define MAXSIM_STATUS_MEM_ERR 0x8u // the target made a burst - -#define MAXSIM_CMD_DCR_WRITE 1u -#define MAXSIM_CMD_DCR_READ 2u -#define MAXSIM_CMD_START 3u -#define MAXSIM_CMD_RESET 4u diff --git a/sw/runtime/maxsim/Makefile b/sw/runtime/foldsim/Makefile similarity index 71% rename from sw/runtime/maxsim/Makefile rename to sw/runtime/foldsim/Makefile index 081ad37e50..64e7473fb9 100644 --- a/sw/runtime/maxsim/Makefile +++ b/sw/runtime/foldsim/Makefile @@ -1,15 +1,15 @@ include ../common.mk -# TARGET=hw runs a maxsim image (hw/syn/maxsim) on the V80 through VRT; -# TARGET=avedsim simulates the same kernel in-process (sim/maxsim). +# TARGET=hw runs a foldsim image (hw/syn/foldsim) on the V80 through VRT; +# TARGET=avedsim simulates the same kernel in-process (sim/foldsim). ifeq ($(strip $(TARGET)),) override TARGET := avedsim endif DESTDIR ?= $(CURDIR)/.. -CONFIG_FILE := $(DESTDIR)/maxsim_rt_config.stamp -SRC_DIR := $(VORTEX_HOME)/sw/runtime/maxsim +CONFIG_FILE := $(DESTDIR)/foldsim_rt_config.stamp +SRC_DIR := $(VORTEX_HOME)/sw/runtime/foldsim CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror CXXFLAGS += -I$(INC_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) -I$(RT_COMMON_DIR) @@ -22,9 +22,9 @@ LDFLAGS += -shared -pthread -Wl,-rpath,'$$ORIGIN' SRCS := $(SRC_DIR)/vortex.cpp $(RT_COMMON_DIR)/utils.cpp $(SIM_COMMON_DIR)/cmd_processor.cpp ifeq ($(TARGET), avedsim) -CXXFLAGS += -DMAXSIM_EMUL -I$(SIM_DIR)/maxsim -LDFLAGS += -L$(DESTDIR) -lmaxsim_emul -EMUL := $(DESTDIR)/libmaxsim_emul.so +CXXFLAGS += -DFOLDSIM_EMUL -I$(SIM_DIR)/foldsim +LDFLAGS += -L$(DESTDIR) -lfoldsim_emul +EMUL := $(DESTDIR)/libfoldsim_emul.so else ifeq ($(TARGET), hw) # VRT's public headers include and from # versioned subdirectories. @@ -40,7 +40,7 @@ else CXXFLAGS += -O2 -DNDEBUG endif -PROJECT := libvortex-maxsim.so +PROJECT := libvortex-foldsim.so .PHONY: all force driver clean-driver clean-runtime clean @@ -48,8 +48,8 @@ all: $(DESTDIR)/$(PROJECT) driver: $(EMUL) -$(DESTDIR)/libmaxsim_emul.so: force - DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/maxsim lib CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) +$(DESTDIR)/libfoldsim_emul.so: force + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/foldsim lib CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) $(DESTDIR)/$(PROJECT): $(SRCS) $(EMUL) $(CONFIG_FILE) $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ @@ -61,7 +61,7 @@ $(CONFIG_FILE): force if ! cmp -s $$TMP $@ 2>/dev/null; then mv -f $$TMP $@ ; else rm -f $$TMP ; fi clean-driver: - DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/maxsim clean + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/foldsim clean clean-runtime: rm -f $(DESTDIR)/$(PROJECT) diff --git a/sw/runtime/maxsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp similarity index 83% rename from sw/runtime/maxsim/vortex.cpp rename to sw/runtime/foldsim/vortex.cpp index 05b71fcc56..ece4dbf089 100644 --- a/sw/runtime/maxsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -12,23 +12,23 @@ // limitations under the License. // ============================================================================ -// maxsim backend — a pure transport HAL (see callbacks.h), like the firesim -// backend: the target carries no Command Processor, so the regfile surface -// comes from the functional CommandProcessor model, and its hooks reach the -// maxsim simulator. Device memory is the simulator's, in device memory at -// MAXSIM_REG_MEM_BASE; the target's DCR, start and busy are its command queue -// and status (sw/common/maxsim_regs.h). +// foldsim backend — a pure transport HAL (see callbacks.h): the target +// carries no Command Processor, so the regfile surface comes from the +// functional CommandProcessor model, and its hooks reach the foldsim +// simulator. Device memory is the simulator's, in device memory at +// FOLDSIM_REG_MEM_BASE; the target's DCR, start and busy are its command queue +// and status (sw/common/foldsim_regs.h). // // TARGET=hw runs the image on a V80 through VRT: registers over AXI4-Lite, // device memory over a QDMA queue. TARGET=avedsim simulates the same kernel -// in-process (sim/maxsim). +// in-process (sim/foldsim). // ============================================================================ #include #include #include -#include +#include #include #include @@ -42,8 +42,8 @@ #include #include -#ifdef MAXSIM_EMUL -#include +#ifdef FOLDSIM_EMUL +#include #else #include #include @@ -67,7 +67,7 @@ namespace { // v80.toml). The target's 4 GB lie in HBM0..HBM7, which nothing else uses. constexpr uint64_t kMemBase = 0x4000000000ull; -#ifdef MAXSIM_EMUL +#ifdef FOLDSIM_EMUL class transport_t { public: @@ -84,12 +84,12 @@ class transport_t { } private: - maxsim_emul sim_; + foldsim_emul sim_; }; #else -#define KERNEL_NAME "maxsim_afu_0" +#define KERNEL_NAME "foldsim_afu_0" // VRT throws; nothing may cross the callbacks' C boundary. #define VRT_TRY() try { @@ -107,7 +107,7 @@ class transport_t { int init() { const char* vbin = getenv("VRT_VBIN_PATH"); if (vbin == nullptr || vbin[0] == '\0') { - fprintf(stderr, "[VXDRV] VRT_VBIN_PATH must name the maxsim .vbin\n"); + fprintf(stderr, "[VXDRV] VRT_VBIN_PATH must name the foldsim .vbin\n"); return -1; } const char* bdf_env = getenv("VRT_DEVICE_BDF"); @@ -130,7 +130,7 @@ class transport_t { (unsigned long)clk_hz, (unsigned long)timed_hz); return -1; } - printf("[VXDRV] maxsim kernel clock %lu Hz\n", (unsigned long)clk_hz); + printf("[VXDRV] foldsim kernel clock %lu Hz\n", (unsigned long)clk_hz); kernel_ = std::make_unique(*device_, KERNEL_NAME); // A QDMA queue for device memory: the buffer's own memory is the // smallest allocation, outside the target's 4 GB, and only its queue is @@ -319,32 +319,33 @@ class vx_device { if (transport_.init() != 0) return -1; uint32_t id = 0, version = 0; - if (transport_.reg_read(MAXSIM_REG_ID, &id) != 0 || id != MAXSIM_ID) { - fprintf(stderr, "[VXDRV] the kernel is not a maxsim simulator (id 0x%08x)\n", id); + if (transport_.reg_read(FOLDSIM_REG_ID, &id) != 0 || id != FOLDSIM_ID) { + fprintf(stderr, "[VXDRV] the kernel is not a foldsim simulator (id 0x%08x)\n", id); return -1; } - if (transport_.reg_read(MAXSIM_REG_VERSION, &version) != 0 || version != MAXSIM_VERSION) { + if (transport_.reg_read(FOLDSIM_REG_VERSION, &version) != 0 || version != FOLDSIM_VERSION) { fprintf(stderr, "[VXDRV] the kernel's register map is version %u, this runtime's %u\n", - version, MAXSIM_VERSION); + version, FOLDSIM_VERSION); return -1; } - // The memory model: a latency-bandwidth pipe, FASED's defaults unless set. - const uint32_t rlat = env_or("MAXSIM_MEM_READ_LATENCY", 30); - const uint32_t wlat = env_or("MAXSIM_MEM_WRITE_LATENCY", 30); - const uint32_t maxr = env_or("MAXSIM_MEM_MAX_READS", 16); - const uint32_t maxw = env_or("MAXSIM_MEM_MAX_WRITES", 16); - if (reg_write(MAXSIM_REG_MEM_BASE_LO, uint32_t(kMemBase)) != 0 || - reg_write(MAXSIM_REG_MEM_BASE_HI, uint32_t(kMemBase >> 32)) != 0 || - reg_write(MAXSIM_REG_MEM_RLAT, rlat) != 0 || - reg_write(MAXSIM_REG_MEM_WLAT, wlat) != 0 || - reg_write(MAXSIM_REG_MEM_MAXR, maxr) != 0 || - reg_write(MAXSIM_REG_MEM_MAXW, maxw) != 0 || - reg_write(MAXSIM_REG_CTRL, 1) != 0) + // The memory model: a latency-bandwidth pipe, 30-cycle latency and 16 + // transactions in flight each way unless set. + const uint32_t rlat = env_or("FOLDSIM_MEM_READ_LATENCY", 30); + const uint32_t wlat = env_or("FOLDSIM_MEM_WRITE_LATENCY", 30); + const uint32_t maxr = env_or("FOLDSIM_MEM_MAX_READS", 16); + const uint32_t maxw = env_or("FOLDSIM_MEM_MAX_WRITES", 16); + if (reg_write(FOLDSIM_REG_MEM_BASE_LO, uint32_t(kMemBase)) != 0 || + reg_write(FOLDSIM_REG_MEM_BASE_HI, uint32_t(kMemBase >> 32)) != 0 || + reg_write(FOLDSIM_REG_MEM_RLAT, rlat) != 0 || + reg_write(FOLDSIM_REG_MEM_WLAT, wlat) != 0 || + reg_write(FOLDSIM_REG_MEM_MAXR, maxr) != 0 || + reg_write(FOLDSIM_REG_MEM_MAXW, maxw) != 0 || + reg_write(FOLDSIM_REG_CTRL, 1) != 0) return -1; // A target left running by an earlier process is reset, as a card's // native driver resets its device on open: Vortex counts VX_CFG_RESET_DELAY // cycles of reset before it takes a request. - if (command(MAXSIM_CMD_RESET, 0, 64) != 0 || wait_idle() != 0) + if (command(FOLDSIM_CMD_RESET, 0, 64) != 0 || wait_idle() != 0) return -1; open_ = true; return 0; @@ -412,34 +413,34 @@ class vx_device { // How fast the simulator ran since the kernel's reset: target cycles, the // kernel clock's cycles, and those a target cycle waited for device memory. void report() { - const uint64_t target = read64(MAXSIM_REG_TCYCLES_LO); - const uint64_t host = read64(MAXSIM_REG_HCYCLES_LO); - const uint64_t stall = read64(MAXSIM_REG_MSTALL_LO); + const uint64_t target = read64(FOLDSIM_REG_TCYCLES_LO); + const uint64_t host = read64(FOLDSIM_REG_HCYCLES_LO); + const uint64_t stall = read64(FOLDSIM_REG_MSTALL_LO); fprintf(stderr, - "[maxsim] target cycles=%lu host cycles=%lu host/target=%.2f memory waits=%lu\n", + "[foldsim] target cycles=%lu host cycles=%lu host/target=%.2f memory waits=%lu\n", (unsigned long)target, (unsigned long)host, target ? double(host) / double(target) : 0.0, (unsigned long)stall); // Each class finishes each of its threads once a target cycle, so its // finishes per host cycle is how busy its one physical copy is. uint32_t classes = 0; - transport_.reg_read(MAXSIM_REG_NUM_CLASSES, &classes); + transport_.reg_read(FOLDSIM_REG_NUM_CLASSES, &classes); std::string line; for (uint32_t c = 0; c < classes && host != 0; ++c) { - const uint64_t finishes = read64(MAXSIM_REG_FINISH(c)); + const uint64_t finishes = read64(FOLDSIM_REG_FINISH(c)); char buf[64]; snprintf(buf, sizeof(buf), " %u:%.1f%%", c, 100.0 * double(finishes) / double(host)); line += buf; } if (!line.empty()) - fprintf(stderr, "[maxsim] class busy:%s\n", line.c_str()); + fprintf(stderr, "[foldsim] class busy:%s\n", line.c_str()); const auto& st = transport_.stats; fprintf(stderr, - "[maxsim] host: %lu register reads, %lu writes in %.3f s; %lu bytes of device memory in %.3f s\n", + "[foldsim] host: %lu register reads, %lu writes in %.3f s; %lu bytes of device memory in %.3f s\n", (unsigned long)st.reads, (unsigned long)st.writes, st.reg_s, (unsigned long)st.dma_bytes, st.dma_s); } - int status(uint32_t* value) { return transport_.reg_read(MAXSIM_REG_STATUS, value); } + int status(uint32_t* value) { return transport_.reg_read(FOLDSIM_REG_STATUS, value); } // Queues a command once the queue has room. int command(uint32_t op, uint32_t addr, uint32_t data) { @@ -447,9 +448,9 @@ class vx_device { do { if (status(&s) != 0) return -1; - } while (s & MAXSIM_STATUS_FULL); - if (reg_write(MAXSIM_REG_CMD_ADDR, addr) != 0 || reg_write(MAXSIM_REG_CMD_DATA, data) != 0 || - reg_write(MAXSIM_REG_CMD, op) != 0) + } while (s & FOLDSIM_STATUS_FULL); + if (reg_write(FOLDSIM_REG_CMD_ADDR, addr) != 0 || reg_write(FOLDSIM_REG_CMD_DATA, data) != 0 || + reg_write(FOLDSIM_REG_CMD, op) != 0) return -1; return 0; } @@ -459,14 +460,14 @@ class vx_device { do { if (status(&s) != 0 || mem_error(s)) return -1; - } while (s & MAXSIM_STATUS_BUSY); + } while (s & FOLDSIM_STATUS_BUSY); return 0; } // The memory endpoint takes single-beat transactions; a target that makes a // burst stops the simulation there, busy for good. bool mem_error(uint32_t s) { - if (!(s & MAXSIM_STATUS_MEM_ERR)) + if (!(s & FOLDSIM_STATUS_MEM_ERR)) return false; if (!mem_error_reported_) { fprintf(stderr, "[VXDRV] the target made a memory burst, which the memory " @@ -523,25 +524,25 @@ class vx_device { transport_.mem_write(addr, src, bytes); }; h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { - command(MAXSIM_CMD_DCR_WRITE, addr, value); + command(FOLDSIM_CMD_DCR_WRITE, addr, value); }; h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { - if (command(MAXSIM_CMD_DCR_READ, addr, tag) != 0) + if (command(FOLDSIM_CMD_DCR_READ, addr, tag) != 0) return 0; uint32_t s = 0, value = 0; do { if (status(&s) != 0) return 0; - } while (!(s & MAXSIM_STATUS_RSP)); - transport_.reg_read(MAXSIM_REG_DCR_RSP, &value); + } while (!(s & FOLDSIM_STATUS_RSP)); + transport_.reg_read(FOLDSIM_REG_DCR_RSP, &value); return value; }; - h.vortex_start = [this]() { command(MAXSIM_CMD_START, 0, 0); }; + h.vortex_start = [this]() { command(FOLDSIM_CMD_START, 0, 0); }; h.vortex_busy = [this]() -> bool { uint32_t s = 0; if (status(&s) != 0 || mem_error(s)) return false; - const bool busy = (s & MAXSIM_STATUS_BUSY) != 0; + const bool busy = (s & FOLDSIM_STATUS_BUSY) != 0; poll_backoff(busy); return busy; }; diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 273140d244..dcb15840dc 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -269,15 +269,15 @@ else XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) endif -# maxsim: TARGET=hw runs FPGA_BIN_DIR's maxsim.vbin (hw/syn/maxsim) on the V80; +# foldsim: TARGET=hw runs FPGA_BIN_DIR's foldsim.vbin (hw/syn/foldsim) on the V80; # TARGET=avedsim, the default, simulates the same kernel in-process. -run-maxsim: $(PROJECT) kernel.vxbin - $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/maxsim DESTDIR=$(VORTEX_RT_LIB) TARGET=$(if $(filter hw,$(TARGET)),hw,avedsim) +run-foldsim: $(PROJECT) kernel.vxbin + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/foldsim DESTDIR=$(VORTEX_RT_LIB) TARGET=$(if $(filter hw,$(TARGET)),hw,avedsim) ifeq ($(TARGET), hw) $(VORTEX_HOME)/ci/v80_preflight.sh - VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/maxsim.vbin LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=maxsim ./$(PROJECT) $(OPTS) + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/foldsim.vbin LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) else - LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=maxsim ./$(PROJECT) $(OPTS) + LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) endif run-opae: $(PROJECT) kernel.vxbin From d0d402c9dc65055e3bf4c6507b1fd7a5ff3a2f19 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:21:37 -0700 Subject: [PATCH 196/336] foldsim: kernel ID "FOLD"; install foldsim from its vortex_3.x branch The kernel's ID register reads "FOLD" (0x464F4C44), the runtime checks it, and ci/foldsim_install.sh takes vortexgpgpu/foldsim at vortex_3.x by default, the baseline building_toolchain.md names. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e077da5fbf614604b068a4d6a672abf1fdd7fe25) --- ci/foldsim_install.sh.in | 2 +- docs/building_toolchain.md | 4 +--- docs/designs/foldsim.md | 2 +- hw/rtl/afu/foldsim/foldsim_regs.vh | 2 +- sw/common/foldsim_regs.h | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ci/foldsim_install.sh.in b/ci/foldsim_install.sh.in index 1a21d1751a..5681ac8c01 100755 --- a/ci/foldsim_install.sh.in +++ b/ci/foldsim_install.sh.in @@ -36,7 +36,7 @@ set -e TOOLDIR=${TOOLDIR:=@TOOLDIR@} FOLDSIM_REPO=${FOLDSIM_REPO:=https://github.com/vortexgpgpu/foldsim.git} -FOLDSIM_REV=${FOLDSIM_REV:=main} +FOLDSIM_REV=${FOLDSIM_REV:=vortex_3.x} CIRCT_PREFIX=${CIRCT_PREFIX:=$TOOLDIR/circt} FOLDSIM_SRC=${FOLDSIM_SRC:=$TOOLDIR/foldsim-src} diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index 926c2477ea..5a9b112758 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -1156,9 +1156,7 @@ cmake -G Ninja -S $CIRCT_SRC/llvm/llvm -B $CIRCT_SRC/build \ `foldsim-opt`. foldsim is its own project and builds on CIRCT, as POCL builds on LLVM. -> **Baseline:** branch `main` of `vortexgpgpu/foldsim`. Its passes -> (`foldsim-extract-models`, `foldsim-fuse-models`, `foldsim-fame5`, -> `foldsim-compose`) were the `maxsim-*` passes of the CIRCT fork. +> **Baseline:** branch `vortex_3.x` of `vortexgpgpu/foldsim`. ### Build diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 56f2e53ca5..cb48bb2210 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -180,7 +180,7 @@ for the RTL): | offset | register | |---|---| -| `0x00`, `0x04` | ID `"MXSM"`, version | +| `0x00`, `0x04` | ID `"FOLD"`, version | | `0x08` | control: bit 0 the target runs | | `0x0C` | status: busy (anything queued, running, or in flight), queue full, DCR response ready, memory error | | `0x10`, `0x14`, `0x18` | command address, data, and the command: DCR write, DCR read, start, reset | diff --git a/hw/rtl/afu/foldsim/foldsim_regs.vh b/hw/rtl/afu/foldsim/foldsim_regs.vh index 882bbfcaaa..916c03bd43 100644 --- a/hw/rtl/afu/foldsim/foldsim_regs.vh +++ b/hw/rtl/afu/foldsim/foldsim_regs.vh @@ -21,7 +21,7 @@ `ifndef FOLDSIM_REGS_VH `define FOLDSIM_REGS_VH -`define FOLDSIM_ID 32'h4D58534D +`define FOLDSIM_ID 32'h464F4C44 `define FOLDSIM_VERSION 32'd2 `define FOLDSIM_REG_ID 12'h000 diff --git a/sw/common/foldsim_regs.h b/sw/common/foldsim_regs.h index 695aa9c363..9d7870785a 100644 --- a/sw/common/foldsim_regs.h +++ b/sw/common/foldsim_regs.h @@ -18,7 +18,7 @@ #pragma once -#define FOLDSIM_ID 0x4D58534Du +#define FOLDSIM_ID 0x464F4C44u #define FOLDSIM_VERSION 2u #define FOLDSIM_REG_ID 0x000u From ae177549bcdaf3f308a769b19dea53969b6c0272 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:38:29 -0700 Subject: [PATCH 197/336] toolchain_install: foldsim is opt-in; fetch the circt parts explicitly foldsim serves only the foldsim flow, so the default install (no flags) no longer fetches it; --foldsim and --all still do, as the docs say of an optional component. circt stays in the default install: every synthesis flow elaborates with circt-verilog. circt() fetched parts until any wget failed, so a network error read as the end of the archive, and parts left by an interrupted run were concatenated into the next one. It now removes old parts first and fetches an explicit count, as verilator() does; the count moves with the published parts. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a2a48952bde75c386cef01c5036508d8df195e9d) --- ci/toolchain_install.sh.in | 25 ++++++++++++++++--------- docs/building_toolchain.md | 8 +++++++- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/ci/toolchain_install.sh.in b/ci/toolchain_install.sh.in index 397132025e..43f3f7308d 100755 --- a/ci/toolchain_install.sh.in +++ b/ci/toolchain_install.sh.in @@ -167,10 +167,17 @@ verilator() # circt-opt, firtool), packaged in parts as verilator is. circt() { - parts=$(eval echo {a..z}) + # The count of what ci/toolchain_prebuilt.sh's `split -b 50M` produced for + # this release, as for verilator: re-check it whenever the circt prebuilt + # is refreshed. + case $OSVERSION in + *) parts=$(eval echo {a..b}) ;; + esac + echo $parts + rm -f circt.tar.bz2.parta* for x in $parts do - wget $REPOSITORY/circt/$OSVERSION/circt.tar.bz2.parta$x || break + wget $REPOSITORY/circt/$OSVERSION/circt.tar.bz2.parta$x done cat circt.tar.bz2.parta* > circt.tar.bz2 tar -xvf circt.tar.bz2 @@ -235,8 +242,8 @@ gem5() rm -rf gem5 gem5.tar.bz2 } -# Every component except gem5 — the default when the script is run with no -# arguments, and the shared body of --all. +# Every component but the optional ones, foldsim and gem5 — the default when +# the script is run with no arguments, and the shared body of --all. install_base() { pocl @@ -250,7 +257,6 @@ install_base() riscv32 riscv64 circt - foldsim yosys sta mesa @@ -263,12 +269,12 @@ show_usage() { echo "Install Pre-built Vortex Toolchain" echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--foldsim] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" - echo " (no arguments: install all components except gem5)" - echo " gem5 is opt-in: it is a large source build." + echo " (no arguments: install all components except foldsim and gem5)" + echo " foldsim and gem5 are opt-in: foldsim serves only the foldsim flow, and" + echo " gem5 is a large source build. --all installs them too." } -# Default with no arguments: install every component except gem5, which is -# opt-in because it is large. +# Default with no arguments: install every component except the opt-in ones. if [ "$#" -eq 0 ]; then install_base exit 0 @@ -309,6 +315,7 @@ while [ "$1" != "" ]; do --gem5 ) gem5 ;; --all ) install_base + foldsim gem5 ;; -h | --help ) show_usage diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index 5a9b112758..2947fd9ac5 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -1187,6 +1187,9 @@ cmake -G Ninja -S $FOLDSIM_SRC -B $FOLDSIM_SRC/build \ `FOLDSIM_PATH ?= $(TOOLDIR)/foldsim` and `FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim`. - A CIRCT change foldsim's passes rely on lands in CIRCT first; both prebuilts are refreshed together. +- The prebuilt is opt-in: `./ci/toolchain_install.sh --foldsim` (or `--all`) + installs it; the default install leaves it out, since only the foldsim + flow needs it. --- @@ -1270,6 +1273,9 @@ Both accept the same per-component flags: --all (every component above; SST is NOT included — see §11) ``` +With no flag, `ci/toolchain_install.sh` installs every component but the +opt-in `foldsim` and `gem5`; `--all` adds them. + > **Note**: `--all` covers every component in this list except SST. > SST is intentionally not packaged as a prebuilt (autoconf bakes > absolute paths into `sstsimulator.conf` and the `sst` wrapper @@ -1313,7 +1319,7 @@ split parts) from the `vortex-toolchain-prebuilt` repo at the directory: ```bash -# install the full toolchain: +# install the toolchain (all but the opt-in foldsim and gem5): ./ci/toolchain_install.sh # or refresh a single component (e.g. after a new POCL release): From d1b5792acff2b516e1963fda02e2a0f5a308cc5c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:38:29 -0700 Subject: [PATCH 198/336] circt/foldsim install: check out the requested revision, detached A re-run fetched origin and then checked out the local branch, which stays where the first clone left it, so the build never moved to the branch's new tip. The source is now checked out detached at origin/ when REV is a branch, or at REV itself when it names a tag or a commit, with tags fetched (and moved tags updated) first. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 888ace0be7d47e2d3d6c68aac8548b5a7c3410bb) --- ci/circt_install.sh.in | 12 +++++++++--- ci/foldsim_install.sh.in | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ci/circt_install.sh.in b/ci/circt_install.sh.in index b7928947dc..db58fd1e96 100755 --- a/ci/circt_install.sh.in +++ b/ci/circt_install.sh.in @@ -46,11 +46,17 @@ JOBS=${JOBS:=$(nproc)} # ---------------------------------------------------------------------- # 1. Fetch: LLVM is a submodule pinned by the CIRCT revision. # ---------------------------------------------------------------------- +# CIRCT_REV is a branch, taken at its tip on origin, or a tag or a commit: the +# checkout is detached either way, so a re-run never stays on a stale local +# branch. if [ ! -d "$CIRCT_SRC/.git" ]; then - git clone --branch "$CIRCT_REV" "$CIRCT_REPO" "$CIRCT_SRC" + git clone "$CIRCT_REPO" "$CIRCT_SRC" +fi +git -C "$CIRCT_SRC" fetch --force --tags origin +if git -C "$CIRCT_SRC" rev-parse -q --verify "refs/remotes/origin/$CIRCT_REV^{commit}" >/dev/null; then + git -C "$CIRCT_SRC" checkout --detach "origin/$CIRCT_REV" else - git -C "$CIRCT_SRC" fetch origin - git -C "$CIRCT_SRC" checkout "$CIRCT_REV" + git -C "$CIRCT_SRC" checkout --detach "$CIRCT_REV^{commit}" fi git -C "$CIRCT_SRC" submodule update --init --depth 1 llvm diff --git a/ci/foldsim_install.sh.in b/ci/foldsim_install.sh.in index 5681ac8c01..35ffc04ebd 100755 --- a/ci/foldsim_install.sh.in +++ b/ci/foldsim_install.sh.in @@ -52,11 +52,17 @@ fi # ---------------------------------------------------------------------- # 1. Fetch # ---------------------------------------------------------------------- +# FOLDSIM_REV is a branch, taken at its tip on origin, or a tag or a commit: the +# checkout is detached either way, so a re-run never stays on a stale local +# branch. if [ ! -d "$FOLDSIM_SRC/.git" ]; then - git clone --branch "$FOLDSIM_REV" "$FOLDSIM_REPO" "$FOLDSIM_SRC" + git clone "$FOLDSIM_REPO" "$FOLDSIM_SRC" +fi +git -C "$FOLDSIM_SRC" fetch --force --tags origin +if git -C "$FOLDSIM_SRC" rev-parse -q --verify "refs/remotes/origin/$FOLDSIM_REV^{commit}" >/dev/null; then + git -C "$FOLDSIM_SRC" checkout --detach "origin/$FOLDSIM_REV" else - git -C "$FOLDSIM_SRC" fetch origin - git -C "$FOLDSIM_SRC" checkout "$FOLDSIM_REV" + git -C "$FOLDSIM_SRC" checkout --detach "$FOLDSIM_REV^{commit}" fi # ---------------------------------------------------------------------- From 730751fc2afe2a86e1ee924f3f7922826f2b5fa7 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:38:39 -0700 Subject: [PATCH 199/336] foldsim: a target reset resets the memory endpoint; bursts stay off HBM FOLDSIM_CMD_RESET drove only the target's reset token. The memory endpoint kept its pipe and its sticky error until the kernel's own reset, so after a reset the fresh target was handed responses to requests its previous run made, and a memory error could never be cleared. The control endpoint now gives the memory endpoint the target's reset as a token of the same target cycle. A target cycle in reset takes no request and finishes only once HBM has answered everything it has from before -- reads returned, writes acknowledged, nothing left offered -- then drops the pipe (requests not answered, data not delivered, the lone half of a write) and the error. HBM-side AXI stays in protocol, and no response from before a reset reaches the target after it. A burst set the error but still went out: its non-last W beats were pushed to HBM without an AW and were not limited by w_ready, so the 16-entry W queue could overflow and the HBM port went out of protocol. Now a burst only sets the error and is dropped; from the next cycle the endpoint is ready for nothing. Writes go to HBM only once both halves are in, so a lone half can be dropped without withdrawing a valid, and `idle` covers transactions HBM still owes. The max-in-flight compares no longer truncate the 8-bit limits to the counters' width. hw/unittest/foldsim_mem drives the endpoint with an AXI4 master as the target against an HBM model that checks the device-side protocol: responses, ids and latencies, resets at random points with HBM busy, and each kind of burst. The previous endpoint fails it. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 90118165dc39f85901c1708f0bfb667c23f42d19) --- docs/designs/foldsim.md | 31 +- hw/rtl/afu/foldsim/foldsim_afu.sv | 7 + hw/rtl/afu/foldsim/foldsim_ctrl.sv | 10 +- hw/rtl/afu/foldsim/foldsim_mem.sv | 239 ++++++----- hw/unittest/Makefile | 3 + hw/unittest/foldsim_mem/Makefile | 24 ++ hw/unittest/foldsim_mem/main.cpp | 631 +++++++++++++++++++++++++++++ 7 files changed, 845 insertions(+), 100 deletions(-) create mode 100644 hw/unittest/foldsim_mem/Makefile create mode 100644 hw/unittest/foldsim_mem/main.cpp diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index cb48bb2210..ce14fd90ab 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -152,16 +152,31 @@ target cycle (`foldsim_token.sv`): - **`foldsim_mem`** is the target's AXI4 port, timed as a latency-bandwidth pipe — cycle for cycle `sim/common/axi_latency_pipe.h`: a fixed latency per transaction, bounded reads and writes in flight, one beat per cycle, - responses in request order. A request goes to HBM (`base` + target address) - the cycle the target makes it; a target cycle that should present a - response memory has not returned yet does not finish until it has, and a - write is acknowledged to the target only after HBM acknowledged it. + responses in request order. A read goes to HBM (`base` + target address) + the cycle the target makes it, a write the cycle it has both its AW and its + W, so HBM only ever sees whole single-beat writes; a target cycle that + should present a response memory has not returned yet does not finish until + it has, and a write is acknowledged to the target only after HBM + acknowledged it. A burst sets the memory error and goes nowhere; from the + next cycle on the endpoint is ready for nothing, and HBM sees none of it. - **`foldsim_ctrl`** is the AXI4-Lite register block and the endpoint of the target's reset, DCR, start and busy. The target runs free: the endpoint gives it a token every target cycle. Host commands are queued and each is applied for one target cycle (a reset for its count). Kernel reset holds the target in reset for 64 cycles. +**Reset.** The control endpoint also gives the memory endpoint a token each +target cycle: the target's reset in that cycle. A reset command thereby +resets the target's whole memory port, not only the target. A target cycle in +reset takes no request, and finishes only once HBM has answered every +transaction it has from before the reset — reads returned, writes +acknowledged, nothing left offered to it — so the AXI4 port to HBM stays in +protocol. The cycle then drops the pipe: requests the target made and was not +answered, data not delivered to it, the lone half of a write, and the memory +error. No response from before a reset reaches the target after it; writes +HBM already took stay written, as they would behind a real memory +controller. + --- ## 5. The runtime @@ -210,6 +225,11 @@ the target idle has every effect of what it asked for. - **The memory endpoint against `AxiLatencyPipe`**, in Verilator: random requests and device-memory latency, every target cycle's outputs and every read's data compared; five latency and limit settings, 0 differences. +- **The memory endpoint's reset and bursts** (`hw/unittest/foldsim_mem`, in + Verilator): an AXI4 master as the target, HBM with random ready and + latency checking the device-side protocol, resets at random points with + HBM busy, and each kind of burst; every response and id checked, none from + before a reset delivered after it. - **Applications** in the in-process simulator and on the V80 (§7). - **CIRCT lit tests** for every pass (`test/Transforms/foldsim-*.mlir`). @@ -250,7 +270,8 @@ foldsim agrees with `AxiLatencyPipe` (rtlsim) instead. is enabled from the first finish after reset, so token k of every channel is target cycle k. 3. **The memory endpoint takes single-beat transactions**, as Vortex issues - them; a burst sets the status register's memory error. + them; a burst sets the status register's memory error, stops the + endpoint, and only a reset clears it. 4. **One memory port**: the target is built with one platform memory bank. 5. **Classes are named, not found**: `FOLDSIM_CLASSES` lists them. 6. **No kernel register in `0x40`–`0xBF`.** The V80 shell configures the diff --git a/hw/rtl/afu/foldsim/foldsim_afu.sv b/hw/rtl/afu/foldsim/foldsim_afu.sv index 76dbd79c96..cfe9ff1a95 100644 --- a/hw/rtl/afu/foldsim/foldsim_afu.sv +++ b/hw/rtl/afu/foldsim/foldsim_afu.sv @@ -177,6 +177,7 @@ module foldsim_afu #( wire [7:0] mem_max_reads, mem_max_writes; wire [63:0] mem_base; wire mem_idle, mem_error, mem_stall; + wire mem_reset_valid, mem_reset_ready, mem_reset_bits; foldsim_mem #( .ID_W (TARGET_ID_W), @@ -193,6 +194,9 @@ module foldsim_afu #( .idle (mem_idle), .error (mem_error), .stall (mem_stall), + .treset_valid (mem_reset_valid), + .treset_ready (mem_reset_ready), + .treset_bits (mem_reset_bits), `FOLDSIM_MEM(awvalid), `FOLDSIM_MEM(awaddr), `FOLDSIM_MEM(awid), `FOLDSIM_MEM(awlen), `FOLDSIM_MEM(wvalid), `FOLDSIM_MEM(wdata), `FOLDSIM_MEM(wstrb), `FOLDSIM_MEM(wlast), `FOLDSIM_MEM(arvalid), `FOLDSIM_MEM(araddr), `FOLDSIM_MEM(arid), `FOLDSIM_MEM(arlen), @@ -252,6 +256,9 @@ module foldsim_afu #( .mem_error (mem_error), .mem_stall (mem_stall), .profile_finish (profile_finish), + .mem_reset_valid (mem_reset_valid), + .mem_reset_ready (mem_reset_ready), + .mem_reset_bits (mem_reset_bits), `FOLDSIM_IN(reset), `FOLDSIM_IN(dcr_req_valid), `FOLDSIM_IN(dcr_req_rw), diff --git a/hw/rtl/afu/foldsim/foldsim_ctrl.sv b/hw/rtl/afu/foldsim/foldsim_ctrl.sv index 91872f58d9..aae283ec6d 100644 --- a/hw/rtl/afu/foldsim/foldsim_ctrl.sv +++ b/hw/rtl/afu/foldsim/foldsim_ctrl.sv @@ -19,7 +19,9 @@ // The target runs free: the endpoint gives it a token every cycle. The host // queues commands -- a DCR write or read, a start, a reset -- and each is // applied to the target for one cycle (a reset for its count of cycles), in -// order, from the cycle after the queue reaches it. A DCR read's response is +// order, from the cycle after the queue reaches it. The memory endpoint gets +// the target's reset for each target cycle too, and resets its side of the +// target's memory port with it (foldsim_mem). A DCR read's response is // kept for the host; `busy` covers everything queued or in flight, so a host // that sees the target idle has every effect of what it asked for. // @@ -61,6 +63,8 @@ module foldsim_ctrl #( input wire mem_stall, // Which classes of the simulator's models finish a thread this cycle. input wire [NUM_CLASSES-1:0] profile_finish, + // The target's reset, a token per target cycle for the memory endpoint. + output wire mem_reset_valid, input wire mem_reset_ready, output wire mem_reset_bits, // Target inputs, as channels of the simulator: the endpoint's outputs. output wire reset_valid, input wire reset_ready, output wire reset_bits, @@ -118,7 +122,7 @@ module foldsim_ctrl #( : 32'd0; wire finish; - wire [5:0] done; + wire [6:0] done; wire [2:0] have; foldsim_token_out o_reset (clk, reset, reset_valid, reset_ready, run, finish, done[0]); foldsim_token_out o_dvalid (clk, reset, dcr_req_valid_valid, dcr_req_valid_ready, run, finish, done[1]); @@ -126,12 +130,14 @@ module foldsim_ctrl #( foldsim_token_out o_daddr (clk, reset, dcr_req_addr_valid, dcr_req_addr_ready, run, finish, done[3]); foldsim_token_out o_ddata (clk, reset, dcr_req_data_valid, dcr_req_data_ready, run, finish, done[4]); foldsim_token_out o_start (clk, reset, start_valid, start_ready, run, finish, done[5]); + foldsim_token_out o_mreset (clk, reset, mem_reset_valid, mem_reset_ready, run, finish, done[6]); assign reset_bits = in_reset; assign dcr_req_valid_bits = dcr_valid; assign dcr_req_rw_bits = dcr_rw; assign dcr_req_addr_bits = cur_addr; assign dcr_req_data_bits = cur_data; assign start_bits = do_start; + assign mem_reset_bits = in_reset; wire rsp_v, busy; wire [DCR_DATA_W-1:0] rsp_d; diff --git a/hw/rtl/afu/foldsim/foldsim_mem.sv b/hw/rtl/afu/foldsim/foldsim_mem.sv index 3c46d64cbb..9e06527bd9 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem.sv @@ -20,15 +20,22 @@ // the order their requests were accepted. Every output is a function of the // pipe's state, so all of them are offered at the start of a target cycle. // -// Data lives in device memory at `base` plus the target's address. A request -// goes to memory the cycle the target makes it; the response is offered to +// Data lives in device memory at `base` plus the target's address. A read +// goes to memory the cycle the target makes it, a write the cycle it has both +// its AW and its W; the response is offered to // the target when the pipe releases it, and a target cycle that would offer // one memory has not returned yet simply does not finish until it has. A // write is acknowledged to the target only once memory has acknowledged it, // so any later request of the target sees it. // -// Single-beat transactions only, as Vortex issues them: a longer burst sets -// `error`. +// Single-beat transactions only, as Vortex issues them: a burst sets `error`, +// goes nowhere, and from the next cycle on the endpoint is ready for nothing. +// +// The target's reset is a token of the control endpoint's. A target cycle in +// reset takes no request; it finishes once every transaction memory has from +// before it has completed, and drops the pipe -- requests not answered, data +// not delivered, a write's lone half -- and `error`. A response from before a +// reset never reaches the target after it. module foldsim_mem #( parameter ID_W = 4, parameter ADDR_W = 64, @@ -43,12 +50,15 @@ module foldsim_mem #( input wire [7:0] max_reads, input wire [7:0] max_writes, input wire [63:0] base, - // No transaction in flight at the target. + // No transaction in flight, at the target or in memory. output wire idle, output reg error, // A host cycle a target cycle waits for memory. output wire stall, + // The target's reset, from the control endpoint. + input wire treset_valid, output wire treset_ready, input wire treset_bits, + // Target outputs, as channels of the simulator: the endpoint's inputs. input wire awvalid_valid, output wire awvalid_ready, input wire awvalid_bits, input wire awaddr_valid, output wire awaddr_ready, input wire [ADDR_W-1:0] awaddr_bits, @@ -104,13 +114,13 @@ module foldsim_mem #( wire finish; // ---- The target's request side, one token per channel per cycle ---- - wire awvalid, wvalid, wlast, arvalid, rready, bready; + wire awvalid, wvalid, wlast, arvalid, rready, bready, treset; wire [ADDR_W-1:0] awaddr, araddr; wire [ID_W-1:0] awid, arid; wire [7:0] awlen, arlen; wire [DATA_W-1:0] wdata; wire [DATA_W/8-1:0] wstrb; - wire [13:0] have; + wire [14:0] have; foldsim_token_in #(1) t_awvalid (clk, reset, awvalid_valid, awvalid_ready, awvalid_bits, finish, have[0], awvalid); foldsim_token_in #(ADDR_W) t_awaddr (clk, reset, awaddr_valid, awaddr_ready, awaddr_bits, finish, have[1], awaddr); @@ -126,6 +136,7 @@ module foldsim_mem #( foldsim_token_in #(8) t_arlen (clk, reset, arlen_valid, arlen_ready, arlen_bits, finish, have[11], arlen); foldsim_token_in #(1) t_rready (clk, reset, rready_valid, rready_ready, rready_bits, finish, have[12], rready); foldsim_token_in #(1) t_bready (clk, reset, bready_valid, bready_ready, bready_bits, finish, have[13], bready); + foldsim_token_in #(1) t_treset (clk, reset, treset_valid, treset_ready, treset_bits, finish, have[14], treset); // ---- The pipe's state ---- reg [31:0] cycle; @@ -156,11 +167,14 @@ module foldsim_mem #( reg [PW-1:0] rd_head, rd_tail; reg [CW-1:0] rd_count; reg [7:0] b_acked, b_given; + // Reads memory has not answered, and writes it has not acknowledged. + reg [CW-1:0] rd_out, wr_out; + wire drained = (rd_out == 0) & (wr_out == 0); // ---- The pipe's outputs: functions of its state ---- - wire ar_ready = (pending_reads < CW'(max_reads)); - wire aw_ready = (pending_aw < CW'(max_writes)); - wire w_ready = (pending_w < CW'(max_writes)); + wire ar_ready = ~error & (32'(pending_reads) < 32'(max_reads)); + wire aw_ready = ~error & (32'(pending_aw) < 32'(max_writes)); + wire w_ready = ~error & (32'(pending_w) < 32'(max_writes)); // R needs its data back from memory, and B its write acknowledged. wire ok = (~cur_r_valid | (rd_count != 0)) & (~cur_w_valid | (b_acked != b_given)); wire [10:0] done; @@ -189,17 +203,22 @@ module foldsim_mem #( assign bresp_bits = 2'b00; assign bid_bits = cur_w_id; - assign finish = (&have) & (&done); - assign stall = ~ok; - assign idle = (pending_reads == 0) & (pending_aw == 0) & (pending_w == 0); + // A reset cycle waits for memory to answer everything it has. + wire drain_wait = have[14] & treset & ~drained; + assign finish = (&have) & (&done) & ~drain_wait; + assign stall = ~ok | drain_wait; + assign idle = (pending_reads == 0) & (pending_aw == 0) & (pending_w == 0) & drained; // ---- One target cycle of the pipe, on the request side's tokens ---- - wire ar_fire = arvalid & ar_ready; - wire aw_fire = awvalid & aw_ready; - wire w_fire = wvalid & w_ready; - wire wlast_fire = w_fire & wlast; - wire r_fire = cur_r_valid & rready; - wire b_fire = cur_w_valid & bready; + wire ar_hs = arvalid & ar_ready; + wire aw_hs = awvalid & aw_ready; + wire w_hs = wvalid & w_ready; + wire burst = (ar_hs & (arlen != 0)) | (aw_hs & (awlen != 0)) | (w_hs & ~wlast); + wire ar_fire = ar_hs & (arlen == 0); + wire aw_fire = aw_hs & (awlen == 0); + wire w_fire = w_hs & wlast; + wire r_fire = cur_r_valid & rready; + wire b_fire = cur_w_valid & bready; // Reads: the latency pipe, into the stage driving R; a read released the // cycle it arrives skips the pipe. @@ -213,11 +232,11 @@ module foldsim_mem #( wire rp_push = ar_fire & ~(r_take & rp_empty); wire rp_pop = r_take & ~rp_empty; - // Writes: an AW and its last W beat, in either order, form one write, its - // latency counted from the cycle both are in. + // Writes: an AW and its W, in either order, form one write, its latency + // counted from the cycle both are in. wire new_write = ((pending_w > pending_aw) & aw_fire) - | ((pending_w < pending_aw) & wlast_fire) - | (aw_fire & wlast_fire); + | ((pending_w < pending_aw) & w_fire) + | (aw_fire & w_fire); wire awq_empty = (awq_count == 0); wire [ID_W-1:0] w_form_id = awq_empty ? awid : awq_id[awq_head]; wire awq_pop = new_write & ~awq_empty; @@ -256,138 +275,172 @@ module foldsim_mem #( error <= 1'b0; end else if (finish) begin cycle <= cycle + 32'd1; - if (rp_push) begin - rp_release[rp_tail] <= ar_release; - rp_id[rp_tail] <= arid; - rp_tail <= rp_tail + 1'b1; - end - if (rp_pop) - rp_head <= rp_head + 1'b1; - rp_count <= rp_count + CW'(rp_push) - CW'(rp_pop); - if (r_take) begin - cur_r_valid <= 1'b1; - cur_r_id <= r_head_id; - end else if (r_fire) begin - cur_r_valid <= 1'b0; - end - - if (awq_push) begin - awq_id[awq_tail] <= awid; - awq_tail <= awq_tail + 1'b1; + if (treset) begin + pending_reads <= 0; + pending_aw <= 0; + pending_w <= 0; + rp_head <= 0; + rp_tail <= 0; + rp_count <= 0; + cur_r_valid <= 1'b0; + awq_head <= 0; + awq_tail <= 0; + awq_count <= 0; + wp_head <= 0; + wp_tail <= 0; + wp_count <= 0; + cur_w_valid <= 1'b0; + error <= 1'b0; + end else begin + if (rp_push) begin + rp_release[rp_tail] <= ar_release; + rp_id[rp_tail] <= arid; + rp_tail <= rp_tail + 1'b1; + end + if (rp_pop) + rp_head <= rp_head + 1'b1; + rp_count <= rp_count + CW'(rp_push) - CW'(rp_pop); + if (r_take) begin + cur_r_valid <= 1'b1; + cur_r_id <= r_head_id; + end else if (r_fire) begin + cur_r_valid <= 1'b0; + end + + if (awq_push) begin + awq_id[awq_tail] <= awid; + awq_tail <= awq_tail + 1'b1; + end + if (awq_pop) + awq_head <= awq_head + 1'b1; + awq_count <= awq_count + CW'(awq_push) - CW'(awq_pop); + if (wp_push) begin + wp_release[wp_tail] <= w_release; + wp_id[wp_tail] <= w_form_id; + wp_tail <= wp_tail + 1'b1; + end + if (wp_pop) + wp_head <= wp_head + 1'b1; + wp_count <= wp_count + CW'(wp_push) - CW'(wp_pop); + if (w_take) begin + cur_w_valid <= 1'b1; + cur_w_id <= w_head_id; + end else if (b_fire) begin + cur_w_valid <= 1'b0; + end + + pending_reads <= count(pending_reads, ar_fire, r_fire); + pending_aw <= count(pending_aw, aw_fire, b_fire); + pending_w <= count(pending_w, w_fire, b_fire); + if (burst) + error <= 1'b1; end - if (awq_pop) - awq_head <= awq_head + 1'b1; - awq_count <= awq_count + CW'(awq_push) - CW'(awq_pop); - if (wp_push) begin - wp_release[wp_tail] <= w_release; - wp_id[wp_tail] <= w_form_id; - wp_tail <= wp_tail + 1'b1; - end - if (wp_pop) - wp_head <= wp_head + 1'b1; - wp_count <= wp_count + CW'(wp_push) - CW'(wp_pop); - if (w_take) begin - cur_w_valid <= 1'b1; - cur_w_id <= w_head_id; - end else if (b_fire) begin - cur_w_valid <= 1'b0; - end - - pending_reads <= count(pending_reads, ar_fire, r_fire); - pending_aw <= count(pending_aw, aw_fire, b_fire); - pending_w <= count(pending_w, wlast_fire, b_fire); - if ((ar_fire && arlen != 0) || (aw_fire && awlen != 0)) - error <= 1'b1; end end // ---- Memory: requests as the target makes them, in order ---- + // A write goes to memory once both its halves are in: only whole writes + // are offered, so a lone half can be dropped without breaking AXI. reg [63:0] arq_addr [DEPTH]; reg [PW-1:0] arq_head, arq_tail; reg [CW-1:0] arq_count; reg [63:0] awm_addr [DEPTH]; reg [PW-1:0] awm_head, awm_tail; - reg [CW-1:0] awm_count; reg [DATA_W-1:0] wm_data [DEPTH]; reg [DATA_W/8-1:0] wm_strb [DEPTH]; reg [PW-1:0] wm_head, wm_tail; - reg [CW-1:0] wm_count; + // Whole writes whose AW, and whose W, memory has not taken yet. + reg [CW-1:0] aw_formed, w_formed; - wire arq_push = finish & ar_fire; + wire take = finish & ~treset; + wire flush = finish & treset; + wire arq_push = take & ar_fire; wire arq_pop = m_arvalid & m_arready; - wire awm_push = finish & aw_fire; + wire awm_push = take & aw_fire; wire awm_pop = m_awvalid & m_awready; - wire wm_push = finish & w_fire; + wire wm_push = take & w_fire; wire wm_pop = m_wvalid & m_wready; + wire form = take & new_write; wire rd_push = m_rvalid & m_rready; - wire rd_pop = finish & r_fire; + wire rd_pop = take & r_fire; + wire b_push = m_bvalid & m_bready; assign m_arvalid = (arq_count != 0); assign m_araddr = arq_addr[arq_head]; assign m_arlen = 8'd0; assign m_rready = 1'b1; - assign m_awvalid = (awm_count != 0); + assign m_awvalid = (aw_formed != 0); assign m_awaddr = awm_addr[awm_head]; assign m_awlen = 8'd0; - assign m_wvalid = (wm_count != 0); + assign m_wvalid = (w_formed != 0); assign m_wdata = wm_data[wm_head]; assign m_wstrb = wm_strb[wm_head]; assign m_wlast = 1'b1; assign m_bready = 1'b1; always @(posedge clk) begin - if (reset) begin + if (arq_push) + arq_addr[arq_tail] <= base + 64'(araddr); + if (awm_push) + awm_addr[awm_tail] <= base + 64'(awaddr); + if (wm_push) begin + wm_data[wm_tail] <= wdata; + wm_strb[wm_tail] <= wstrb; + end + if (rd_push) + rd_data[rd_tail] <= m_rdata; + end + + // A flush comes once memory has answered everything: nothing is offered + // to it or expected from it, and what is left are lone halves of writes + // and data the target will no longer take. + always @(posedge clk) begin + if (reset || flush) begin arq_head <= 0; arq_tail <= 0; arq_count <= 0; awm_head <= 0; awm_tail <= 0; - awm_count <= 0; wm_head <= 0; wm_tail <= 0; - wm_count <= 0; + aw_formed <= 0; + w_formed <= 0; rd_head <= 0; rd_tail <= 0; rd_count <= 0; + rd_out <= 0; + wr_out <= 0; b_acked <= 0; b_given <= 0; end else begin - if (arq_push) begin - arq_addr[arq_tail] <= base + 64'(araddr); + if (arq_push) arq_tail <= arq_tail + 1'b1; - end if (arq_pop) arq_head <= arq_head + 1'b1; arq_count <= arq_count + CW'(arq_push) - CW'(arq_pop); - if (awm_push) begin - awm_addr[awm_tail] <= base + 64'(awaddr); + if (awm_push) awm_tail <= awm_tail + 1'b1; - end if (awm_pop) awm_head <= awm_head + 1'b1; - awm_count <= awm_count + CW'(awm_push) - CW'(awm_pop); - - if (wm_push) begin - wm_data[wm_tail] <= wdata; - wm_strb[wm_tail] <= wstrb; + if (wm_push) wm_tail <= wm_tail + 1'b1; - end if (wm_pop) wm_head <= wm_head + 1'b1; - wm_count <= wm_count + CW'(wm_push) - CW'(wm_pop); + aw_formed <= aw_formed + CW'(form) - CW'(awm_pop); + w_formed <= w_formed + CW'(form) - CW'(wm_pop); - if (rd_push) begin - rd_data[rd_tail] <= m_rdata; + if (rd_push) rd_tail <= rd_tail + 1'b1; - end if (rd_pop) rd_head <= rd_head + 1'b1; rd_count <= rd_count + CW'(rd_push) - CW'(rd_pop); - if (m_bvalid) + rd_out <= rd_out + CW'(arq_push) - CW'(rd_push); + wr_out <= wr_out + CW'(form) - CW'(b_push); + if (b_push) b_acked <= b_acked + 8'd1; - if (finish & b_fire) + if (take & b_fire) b_given <= b_given + 8'd1; end end diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index c6c1f5f4e1..c1531a435f 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -20,6 +20,7 @@ all: $(MAKE) -C afu_axil_demux $(MAKE) -C afu_reset_seq $(MAKE) -C mm_axi_arb + $(MAKE) -C foldsim_mem $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma $(MAKE) -C cp_llc_port @@ -55,6 +56,7 @@ run: $(MAKE) -C afu_axil_demux run $(MAKE) -C afu_reset_seq run $(MAKE) -C mm_axi_arb run + $(MAKE) -C foldsim_mem run $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run $(MAKE) -C cp_llc_port run @@ -207,6 +209,7 @@ clean: $(MAKE) -C stream_arb clean $(MAKE) -C mm_axi_arb clean $(MAKE) -C cp_core clean + $(MAKE) -C foldsim_mem clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean $(MAKE) -C fsqrt_unit clean diff --git a/hw/unittest/foldsim_mem/Makefile b/hw/unittest/foldsim_mem/Makefile new file mode 100644 index 0000000000..74e830f037 --- /dev/null +++ b/hw/unittest/foldsim_mem/Makefile @@ -0,0 +1,24 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := foldsim_mem + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_INCLUDE := -I$(FOLDSIM_RTL_DIR) + +# The token modules are not named after their file. +VL_FLAGS += $(FOLDSIM_RTL_DIR)/foldsim_token.sv + +TOP := foldsim_mem + +include ../common.mk diff --git a/hw/unittest/foldsim_mem/main.cpp b/hw/unittest/foldsim_mem/main.cpp new file mode 100644 index 0000000000..0afda874b5 --- /dev/null +++ b/hw/unittest/foldsim_mem/main.cpp @@ -0,0 +1,631 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// foldsim's memory endpoint (hw/rtl/afu/foldsim/foldsim_mem.sv), driven as the +// simulator and the control endpoint drive it -- one token per channel per +// target cycle -- by a model of an AXI4 master, against a model of HBM that +// checks the AXI4 protocol on the device side. The properties under test: +// * read data, write acknowledgements and ids are right, and no response +// comes before its latency; +// * a reset with HBM transactions in flight finishes only once HBM has +// answered them all, takes no request, and delivers nothing from before +// it to the target after it; +// * a burst sets `error` and reaches HBM in no part, and a reset clears it. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Vfoldsim_mem.h" +#include "verilated.h" + +namespace { + +constexpr int kWords = 16; // DATA_W = 512 +constexpr uint32_t kReadLatency = 10; +constexpr uint32_t kWriteLatency = 12; +constexpr uint64_t kBase = 0x10000000ull; +constexpr uint64_t kLines = 64; // target addresses in use +constexpr uint64_t kMagic = 0x7f000ull; // offered only in reset cycles +constexpr uint64_t kBurstAddr = 0x7e000ull; // offered only as a burst + +using Line = std::array; + +Vfoldsim_mem* dut = nullptr; +std::mt19937 rng(7); +uint64_t host_cycle = 0; +uint64_t tcycle = 0; +int failures = 0; + +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + if (failures++ < 20) { \ + printf("FAIL host %lu target %lu: ", (unsigned long)host_cycle, \ + (unsigned long)tcycle); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + } \ + } \ + } while (0) + +bool chance(double p) { return std::uniform_real_distribution<>(0, 1)(rng) < p; } +int rand_int(int lo, int hi) { return std::uniform_int_distribution<>(lo, hi)(rng); } + +Line make_line(uint64_t addr, uint32_t version) { + Line l; + for (int i = 0; i < kWords; ++i) + l[i] = uint32_t(addr * 2654435761u) ^ (version << 12) ^ (uint32_t(i) * 0x9e3779b9u); + return l; +} + +// ---- HBM: an AXI4 slave with random ready and latency ---- +struct Hbm { + std::map mem; + double ready_p = 0.7; + int lat_lo = 1, lat_hi = 40; + struct Pending { uint64_t at; Line data; }; + std::deque r; // reads accepted, not returned + std::deque b; // writes done, not acknowledged + std::deque aw; // AWs waiting for their W + std::deque> w; // Ws waiting for their AW + std::set ar_seen; + uint64_t n_ar = 0, n_aw = 0, n_w = 0; + bool arready = false, awready = false, wready = false; + bool ar_held = false, aw_held = false, w_held = false; + uint64_t ar_a = 0, aw_a = 0; + Line w_d{}; + + size_t outstanding() const { return r.size() + b.size() + aw.size() + w.size(); } + + uint64_t when(uint64_t last) { + return std::max(last, host_cycle + uint64_t(rand_int(lat_lo, lat_hi))); + } + + void drive() { + arready = chance(ready_p); + awready = chance(ready_p); + wready = chance(ready_p); + dut->m_arready = arready; + dut->m_awready = awready; + dut->m_wready = wready; + dut->m_rvalid = !r.empty() && r.front().at <= host_cycle; + for (int i = 0; i < kWords; ++i) + dut->m_rdata[i] = r.empty() ? 0 : r.front().data[i]; + dut->m_bvalid = !b.empty() && b.front() <= host_cycle; + } + + Line wdata() const { + Line l; + for (int i = 0; i < kWords; ++i) + l[i] = dut->m_wdata[i]; + return l; + } + + void sample() { + if (ar_held) + CHECK(dut->m_arvalid && dut->m_araddr == ar_a, "HBM: AR withdrawn or changed"); + if (aw_held) + CHECK(dut->m_awvalid && dut->m_awaddr == aw_a, "HBM: AW withdrawn or changed"); + if (w_held) + CHECK(dut->m_wvalid && wdata() == w_d, "HBM: W withdrawn or changed"); + if (dut->m_arvalid) + CHECK(dut->m_arlen == 0, "HBM: AR burst"); + if (dut->m_awvalid) + CHECK(dut->m_awlen == 0, "HBM: AW burst"); + if (dut->m_wvalid) + CHECK(dut->m_wlast, "HBM: W without WLAST"); + CHECK(dut->m_rready && dut->m_bready, "HBM: R or B not ready"); + + const bool ar_hs = dut->m_arvalid && arready; + const bool aw_hs = dut->m_awvalid && awready; + const bool w_hs = dut->m_wvalid && wready; + ar_held = dut->m_arvalid && !arready; + aw_held = dut->m_awvalid && !awready; + w_held = dut->m_wvalid && !wready; + ar_a = dut->m_araddr; + aw_a = dut->m_awaddr; + w_d = wdata(); + + if (dut->m_rvalid) + r.pop_front(); + if (dut->m_bvalid) + b.pop_front(); + if (ar_hs) { + ++n_ar; + const uint64_t a = dut->m_araddr; + ar_seen.insert(a); + auto it = mem.find(a); + r.push_back({when(r.empty() ? 0 : r.back().at), it != mem.end() ? it->second : Line{}}); + } + if (aw_hs) { + ++n_aw; + aw.push_back(dut->m_awaddr); + } + if (w_hs) { + ++n_w; + w.push_back({wdata(), dut->m_wstrb}); + } + while (!aw.empty() && !w.empty()) { + Line& m = mem[aw.front()]; + const Line& d = w.front().first; + const uint64_t strb = w.front().second; + for (int i = 0; i < kWords; ++i) + for (int k = 0; k < 4; ++k) + if ((strb >> (4 * i + k)) & 1) + m[i] = (m[i] & ~(0xffu << (8 * k))) | (d[i] & (0xffu << (8 * k))); + aw.pop_front(); + w.pop_front(); + b.push_back(when(b.empty() ? 0 : b.back())); + } + } +}; + +Hbm hbm; + +// ---- The target: an AXI4 master issuing single-beat reads and writes ---- +struct Op { uint64_t addr; int id; Line data; }; + +struct Master { + bool generate = false; + bool reset = false; // this target cycle is a reset cycle + bool offer_magic = false; + // A burst to offer: 0 none, 1 AR, 2 AW, 3 a W without WLAST. + int burst = 0; + + // What each channel offers this cycle, held until it fires. + bool ar_v = false, aw_v = false, w_v = false; + Op ar{}, aw{}, w{}; + int ar_len = 0, aw_len = 0; + bool w_last = true; + + std::deque ar_q, aw_q, w_q; // queued, not yet offered + struct ReadExp { Op op; bool known; uint64_t fired; }; + std::deque reads; // fired, awaiting R + std::deque aw_only, w_only; // fired halves not yet formed + struct WriteExp { Op op; uint64_t formed; }; + std::deque writes; // formed, awaiting B + + std::map known; // data of every acknowledged write + std::set unknown; // writes a reset left undecided + std::map busy; // reads or writes in flight per address + uint32_t version = 1; + int next_id = 0; + uint64_t n_r = 0, n_b = 0, stale = 0; + + bool idle() const { + return ar_q.empty() && aw_q.empty() && w_q.empty() && reads.empty() && aw_only.empty() + && w_only.empty() && writes.empty() && !ar_v && !aw_v && !w_v; + } + + void make_requests() { + if (!generate || reset) + return; + if (ar_q.size() + reads.size() < 24 && chance(0.3)) { + const uint64_t a = uint64_t(rand_int(0, kLines - 1)) * 64; + if (busy[a] == 0 || (writes_to(a) == 0)) { + ++busy[a]; + ar_q.push_back({a, next_id++ & 15, {}}); + } + } + if (aw_q.size() + writes.size() < 24 && chance(0.25)) { + const uint64_t a = uint64_t(rand_int(0, kLines - 1)) * 64; + if (busy[a] == 0) { + ++busy[a]; + Op op{a, next_id++ & 15, make_line(a, version++)}; + aw_q.push_back(op); + w_q.push_back(op); + pending_writes.insert(a); + } + } + } + + std::set pending_writes; + int writes_to(uint64_t a) const { return pending_writes.count(a) ? 1 : 0; } + + // The channels' tokens for this cycle. + void offer() { + if (reset) { + ar_v = offer_magic; + ar = {kMagic, 0, {}}; + ar_len = 0; + aw_v = w_v = false; + return; + } + if (burst == 1 && !ar_v) { + ar_v = true; + ar = {kBurstAddr, 1, {}}; + ar_len = 3; + } else if (burst == 2 && !aw_v) { + aw_v = true; + aw = {kBurstAddr, 2, {}}; + aw_len = 1; + } else if (burst == 3 && !w_v) { + w_v = true; + w = {kBurstAddr, 3, make_line(kBurstAddr, 0)}; + w_last = false; + } + if (!ar_v && !ar_q.empty() && chance(0.8)) { + ar_v = true; + ar = ar_q.front(); + ar_len = 0; + ar_q.pop_front(); + } + if (!aw_v && !aw_q.empty() && chance(0.6)) { + aw_v = true; + aw = aw_q.front(); + aw_len = 0; + aw_q.pop_front(); + } + if (!w_v && !w_q.empty() && chance(0.6)) { + w_v = true; + w = w_q.front(); + w_last = true; + w_q.pop_front(); + } + } + + void read_done(uint64_t a) { + if (--busy[a] == 0) + busy.erase(a); + } + + // One target cycle, given the endpoint's tokens for it. + void cycle(bool arready, bool awready, bool wready, bool rvalid, const Line& rdata, int rid, + bool bvalid, int bid) { + if (reset) { + // The target is in reset: it ignores its inputs and loses its state. + for (auto& r : reads) + read_done(r.op.addr); + for (auto* q : {&aw_only, &w_only}) + for (auto& op : *q) + unknown.insert(op.addr); + for (auto& wr : writes) + unknown.insert(wr.op.addr); + for (auto* q : {&ar_q}) + for (auto& op : *q) + read_done(op.addr); + for (auto& op : aw_q) + unknown.insert(op.addr); + for (auto& op : w_q) + unknown.insert(op.addr); + if (aw_v) unknown.insert(aw.addr); + if (w_v) unknown.insert(w.addr); + if (ar_v && ar.addr != kMagic) read_done(ar.addr); + reads.clear(); + aw_only.clear(); + w_only.clear(); + writes.clear(); + ar_q.clear(); + aw_q.clear(); + w_q.clear(); + ar_v = aw_v = w_v = false; + busy.clear(); + pending_writes.clear(); + return; + } + if (rvalid) { + ++n_r; + if (reads.empty()) { + ++stale; + CHECK(false, "R with no read outstanding"); + } else { + const ReadExp e = reads.front(); + reads.pop_front(); + CHECK(rid == e.op.id, "R id %d, expected %d", rid, e.op.id); + CHECK(tcycle >= e.fired + kReadLatency, "R at %lu for a read fired at %lu", + (unsigned long)tcycle, (unsigned long)e.fired); + if (e.known) + CHECK(rdata == e.op.data, "R data wrong at 0x%lx", (unsigned long)e.op.addr); + read_done(e.op.addr); + } + } + if (bvalid) { + ++n_b; + if (writes.empty()) { + ++stale; + CHECK(false, "B with no write outstanding"); + } else { + const WriteExp e = writes.front(); + writes.pop_front(); + CHECK(bid == e.op.id, "B id %d, expected %d", bid, e.op.id); + CHECK(tcycle >= e.formed + kWriteLatency, "B at %lu for a write formed at %lu", + (unsigned long)tcycle, (unsigned long)e.formed); + known[e.op.addr] = e.op.data; + unknown.erase(e.op.addr); + pending_writes.erase(e.op.addr); + read_done(e.op.addr); + } + } + if (ar_v && arready) { + ar_v = false; + if (ar_len == 0) { + const auto it = known.find(ar.addr); + const bool k = !unknown.count(ar.addr); + ReadExp e{ar, k, tcycle}; + e.op.data = (it != known.end()) ? it->second : Line{}; + reads.push_back(e); + } else { + burst = 0; + } + } + const bool aw_fire = aw_v && awready; + const bool w_fire = w_v && wready; + if (aw_fire) { + aw_v = false; + if (aw_len == 0) + aw_only.push_back(aw); + else + burst = 0; + } + if (w_fire) { + w_v = false; + if (w_last) + w_only.push_back(w); + else + burst = 0; + } + while (!aw_only.empty() && !w_only.empty()) { + writes.push_back({aw_only.front(), tcycle}); + aw_only.pop_front(); + w_only.pop_front(); + } + } +}; + +Master master; + +// ---- The token channels ---- +enum { I_AWVALID, I_AWADDR, I_AWID, I_AWLEN, I_WVALID, I_WDATA, I_WSTRB, I_WLAST, I_ARVALID, + I_ARADDR, I_ARID, I_ARLEN, I_RREADY, I_BREADY, I_TRESET, NUM_IN }; +enum { O_AWREADY, O_WREADY, O_ARREADY, O_RVALID, O_RDATA, O_RLAST, O_RID, O_RRESP, O_BVALID, + O_BRESP, O_BID, NUM_OUT }; + +bool taken[NUM_IN], got[NUM_OUT]; +bool o_awready, o_wready, o_arready, o_rvalid, o_bvalid; +int o_rid, o_bid; +Line o_rdata; +// Set when a reset cycle's tokens are all exchanged: the endpoint's next +// output token says it has finished that cycle. +bool after_reset = false; +uint64_t reset_cycles_done = 0, drained_at_reset = 0; + +void drive_tokens() { + const Master& m = master; +#define IN(ch, name, value) \ + dut->name##_valid = !taken[ch]; \ + dut->name##_bits = (value) + IN(I_AWVALID, awvalid, m.aw_v); + IN(I_AWADDR, awaddr, m.aw.addr); + IN(I_AWID, awid, m.aw.id); + IN(I_AWLEN, awlen, m.aw_len); + IN(I_WVALID, wvalid, m.w_v); + dut->wdata_valid = !taken[I_WDATA]; + for (int i = 0; i < kWords; ++i) + dut->wdata_bits[i] = m.w.data[i]; + IN(I_WSTRB, wstrb, ~0ull); + IN(I_WLAST, wlast, m.w_last); + IN(I_ARVALID, arvalid, m.ar_v); + IN(I_ARADDR, araddr, m.ar.addr); + IN(I_ARID, arid, m.ar.id); + IN(I_ARLEN, arlen, m.ar_len); + IN(I_RREADY, rready, 1); + IN(I_BREADY, bready, 1); + IN(I_TRESET, treset, m.reset); +#undef IN +#define OUT(ch, name) dut->name##_ready = !got[ch] + OUT(O_AWREADY, awready); OUT(O_WREADY, wready); OUT(O_ARREADY, arready); + OUT(O_RVALID, rvalid); OUT(O_RDATA, rdata); OUT(O_RLAST, rlast); OUT(O_RID, rid); + OUT(O_RRESP, rresp); OUT(O_BVALID, bvalid); OUT(O_BRESP, bresp); OUT(O_BID, bid); +#undef OUT +} + +void sample_tokens(bool in_take[NUM_IN], bool out_take[NUM_OUT]) { +#define IN(ch, name) in_take[ch] = dut->name##_valid && dut->name##_ready + IN(I_AWVALID, awvalid); IN(I_AWADDR, awaddr); IN(I_AWID, awid); IN(I_AWLEN, awlen); + IN(I_WVALID, wvalid); IN(I_WDATA, wdata); IN(I_WSTRB, wstrb); IN(I_WLAST, wlast); + IN(I_ARVALID, arvalid); IN(I_ARADDR, araddr); IN(I_ARID, arid); IN(I_ARLEN, arlen); + IN(I_RREADY, rready); IN(I_BREADY, bready); IN(I_TRESET, treset); +#undef IN +#define OUT(ch, name) out_take[ch] = dut->name##_valid && dut->name##_ready + OUT(O_AWREADY, awready); OUT(O_WREADY, wready); OUT(O_ARREADY, arready); + OUT(O_RVALID, rvalid); OUT(O_RDATA, rdata); OUT(O_RLAST, rlast); OUT(O_RID, rid); + OUT(O_RRESP, rresp); OUT(O_BVALID, bvalid); OUT(O_BRESP, bresp); OUT(O_BID, bid); +#undef OUT + if (out_take[O_AWREADY]) o_awready = dut->awready_bits; + if (out_take[O_WREADY]) o_wready = dut->wready_bits; + if (out_take[O_ARREADY]) o_arready = dut->arready_bits; + if (out_take[O_RVALID]) o_rvalid = dut->rvalid_bits; + if (out_take[O_RDATA]) + for (int i = 0; i < kWords; ++i) + o_rdata[i] = dut->rdata_bits[i]; + if (out_take[O_RID]) o_rid = dut->rid_bits; + if (out_take[O_BVALID]) o_bvalid = dut->bvalid_bits; + if (out_take[O_BID]) o_bid = dut->bid_bits; + if (out_take[O_RLAST]) CHECK(dut->rlast_bits, "RLAST low"); + if (out_take[O_RRESP]) CHECK(dut->rresp_bits == 0, "RRESP not OKAY"); + if (out_take[O_BRESP]) CHECK(dut->bresp_bits == 0, "BRESP not OKAY"); +} + +void host_step() { + drive_tokens(); + hbm.drive(); + dut->clk = 0; + dut->eval(); + bool in_take[NUM_IN], out_take[NUM_OUT]; + sample_tokens(in_take, out_take); + bool any_out = false; + for (int j = 0; j < NUM_OUT; ++j) + any_out |= out_take[j]; + if (after_reset && any_out) { + // The first token of the cycle after a reset cycle: that cycle has + // finished, so HBM has answered everything and nothing is offered to it. + after_reset = false; + ++reset_cycles_done; + CHECK(hbm.outstanding() == 0, "reset cycle finished with %zu HBM transactions open", + hbm.outstanding()); + CHECK(!dut->m_arvalid && !dut->m_awvalid && !dut->m_wvalid, + "reset cycle finished with requests offered to HBM"); + } + hbm.sample(); + dut->clk = 1; + dut->eval(); + ++host_cycle; + for (int i = 0; i < NUM_IN; ++i) + taken[i] |= in_take[i]; + for (int j = 0; j < NUM_OUT; ++j) + got[j] |= out_take[j]; +} + +// Runs target cycles until `n` have completed. +void run(uint64_t n) { + for (uint64_t k = 0; k < n; ++k) { + master.make_requests(); + master.offer(); + if (master.reset && hbm.outstanding() != 0) + ++drained_at_reset; + for (int i = 0; i < NUM_IN; ++i) + taken[i] = false; + for (int j = 0; j < NUM_OUT; ++j) + got[j] = false; + uint64_t spins = 0; + for (;;) { + bool all = true; + for (int i = 0; i < NUM_IN; ++i) + all &= taken[i]; + for (int j = 0; j < NUM_OUT; ++j) + all &= got[j]; + if (all) + break; + host_step(); + if (++spins > 200000) { + printf("FAIL: target cycle %lu never completes\n", (unsigned long)tcycle); + exit(1); + } + } + if (master.reset) + after_reset = true; + master.cycle(o_arready, o_awready, o_wready, o_rvalid, o_rdata, o_rid, o_bvalid, o_bid); + ++tcycle; + } +} + +void reset_cycles(int n, bool magic) { + master.reset = true; + master.offer_magic = magic; + run(n); + master.reset = false; + master.offer_magic = false; +} + +void drain() { + master.generate = false; + for (int i = 0; i < 100000 && !master.idle(); ++i) + run(1); + CHECK(master.idle(), "the target's transactions never complete"); + run(4); +} + +void burst_case(int kind, const char* what) { + drain(); + const uint64_t ar = hbm.n_ar, aw = hbm.n_aw, w = hbm.n_w; + master.burst = kind; + for (int i = 0; i < 64 && master.burst != 0; ++i) + run(1); + CHECK(master.burst == 0, "%s: never taken", what); + run(2); + CHECK(dut->error, "%s: no error", what); + CHECK(!o_arready && !o_awready && !o_wready, "%s: still ready after the error", what); + // Requests now wait; the endpoint stays stopped. + master.generate = true; + run(50); + master.generate = false; + CHECK(hbm.n_ar == ar && hbm.n_aw == aw && hbm.n_w == w, "%s: reached HBM", what); + CHECK(dut->error, "%s: error cleared without a reset", what); + reset_cycles(3, false); + CHECK(!dut->error, "%s: reset left the error", what); + master.generate = true; + run(800); +} + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + dut = new Vfoldsim_mem(); + dut->read_latency = kReadLatency; + dut->write_latency = kWriteLatency; + dut->max_reads = 16; + dut->max_writes = 16; + dut->base = kBase; + dut->reset = 1; + for (int i = 0; i < 8; ++i) { + hbm.drive(); + dut->clk = 0; + dut->eval(); + dut->clk = 1; + dut->eval(); + } + dut->reset = 0; + + reset_cycles(4, true); + master.generate = true; + run(3000); + + // Resets with HBM slow and busy, at random points. + for (int k = 0; k < 8; ++k) { + hbm.lat_lo = 100; + hbm.lat_hi = 400; + hbm.ready_p = 0.3; + run(rand_int(50, 300)); + hbm.lat_lo = 1; + hbm.lat_hi = 40; + hbm.ready_p = 0.7; + reset_cycles(rand_int(1, 4), true); + run(1500); + } + + burst_case(1, "AR burst"); + burst_case(2, "AW burst"); + burst_case(3, "W without WLAST"); + + drain(); + CHECK(hbm.n_aw == hbm.n_w, "HBM took %lu AWs and %lu Ws", (unsigned long)hbm.n_aw, + (unsigned long)hbm.n_w); + CHECK(hbm.outstanding() == 0, "HBM left with %zu transactions", hbm.outstanding()); + CHECK(!hbm.ar_seen.count(kBase + kMagic), "a request made in reset reached HBM"); + CHECK(!hbm.ar_seen.count(kBase + kBurstAddr), "a burst reached HBM"); + CHECK(dut->idle, "endpoint not idle at the end"); + CHECK(master.stale == 0, "%lu stale responses", (unsigned long)master.stale); + CHECK(drained_at_reset >= 4, "only %lu resets found HBM busy", (unsigned long)drained_at_reset); + + printf("target cycles %lu, host cycles %lu; %lu reads, %lu writes acknowledged; " + "%lu reset cycles, %lu of them with HBM busy\n", + (unsigned long)tcycle, (unsigned long)host_cycle, (unsigned long)master.n_r, + (unsigned long)master.n_b, (unsigned long)reset_cycles_done, + (unsigned long)drained_at_reset); + delete dut; + if (failures != 0) { + printf("FAILED: %d check(s)\n", failures); + return 1; + } + printf("PASSED!\n"); + return 0; +} From 87abc8975fcade39d8b7d2f8524243283745e862 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:39:48 -0700 Subject: [PATCH 200/336] foldsim runtime: latch device errors and bound every wait The command processor model's hooks return no status, and the backend dropped every failure: vortex_busy read a memory error or a failed status read as idle, so the launch retired as a success; DCR writes and starts ignored whether the command was queued; a DCR read polled for its response with no memory-error check and no limit. A device error -- a memory error, a failed register or memory access, a wait that timed out -- is now latched: it is reported once, the hooks stop touching the target, and every register access of the runtime core fails from then on, so the launch, DCR access or copy that hit it fails instead of completing. Waits for queue room, a DCR response or the idle target are bounded (60 s, or VORTEX_FOLDSIM_TIMEOUT_S), and a launch is bounded by VORTEX_FOLDSIM_TIMEOUT_S when it is set, after the firesim backend's VORTEX_FIRESIM_TIMEOUT_S. Open checks the memory error only after its reset, which now clears one an earlier process left. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 01cbe309a8b52beba7e6c43c2403904e7676fdc6) --- docs/designs/foldsim.md | 16 +++- docs/simulation.md | 6 +- sw/runtime/foldsim/vortex.cpp | 142 +++++++++++++++++++++++++--------- 3 files changed, 124 insertions(+), 40 deletions(-) diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index ce14fd90ab..b059bbf049 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -211,8 +211,20 @@ accesses itself (§8). The runtime refuses an image whose `VERSION` differs from its own map. `busy` covers queued commands, a start the target has not yet answered with -busy, and transactions the memory model has in flight, so a host that sees -the target idle has every effect of what it asked for. +busy, and transactions the memory model has in flight, at the target or in +HBM, so a host that sees the target idle has every effect of what it asked +for. + +The runtime resets the target on open, as a card's native driver resets its +device; that also clears a memory error an earlier process left. The command +processor model's hooks return no status, so the runtime latches a device +error instead — a memory error, a failed register or memory access, or a wait +that timed out — and from then on every register access the runtime core +makes fails, and the launch, DCR access or copy that hit it fails with it. A +launch longer than `VORTEX_FOLDSIM_TIMEOUT_S` seconds is such an error (no +limit by default), as is a command the target does not take, a DCR read it +does not answer, or a reset it does not finish within that time (60 s when +it is unset). --- diff --git a/docs/simulation.md b/docs/simulation.md index bb2c5aa5ac..59aa1e755e 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -118,8 +118,10 @@ The flow needs CIRCT (`circt-verilog`, `circt-opt`, `firtool`) and foldsim `FOLDSIM=` override them), and pyslang for the boundary lifting (`LIFT_PYTHON`). The memory model's settings are runtime knobs: `FOLDSIM_MEM_READ_LATENCY`, `FOLDSIM_MEM_WRITE_LATENCY` (target cycles, -30 by default) and `FOLDSIM_MEM_MAX_READS`, `FOLDSIM_MEM_MAX_WRITES` (16). At -exit the runtime reports the target and kernel-clock cycles the simulator ran +30 by default) and `FOLDSIM_MEM_MAX_READS`, `FOLDSIM_MEM_MAX_WRITES` (16). +`VORTEX_FOLDSIM_TIMEOUT_S=` fails a launch that runs longer; a command, +DCR read or reset the target leaves unanswered fails after it too, or after +60 s when it is unset. At exit the runtime reports the target and kernel-clock cycles the simulator ran and how busy each threaded class was. The design is [designs/foldsim.md](designs/foldsim.md). diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp index ece4dbf089..5dceb59571 100644 --- a/sw/runtime/foldsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -344,8 +344,12 @@ class vx_device { return -1; // A target left running by an earlier process is reset, as a card's // native driver resets its device on open: Vortex counts VX_CFG_RESET_DELAY - // cycles of reset before it takes a request. - if (command(FOLDSIM_CMD_RESET, 0, 64) != 0 || wait_idle() != 0) + // cycles of reset before it takes a request. The reset also clears a + // memory error that process left, so only the state after it counts. + uint32_t s = 0; + if (command(FOLDSIM_CMD_RESET, 0, 64) != 0 || wait_idle(false) != 0 || status(&s) != 0) + return -1; + if (mem_error(s)) return -1; open_ = true; return 0; @@ -353,18 +357,21 @@ class vx_device { // ----- CP register channel ----- // No hardware CP: the functional model answers, ticked around each access. + // The model's hooks return no status: a device error they hit fails every + // access from then on, so the command that hit it does not complete as a + // success. int cp_reg_write(uint32_t off, uint32_t value) { cp_.mmio_write(off, value); for (int i = 0; i < 256 && cp_.busy(); ++i) cp_.tick(); - return 0; + return failed_ ? -1 : 0; } int cp_reg_read(uint32_t off, uint32_t* value) { for (int i = 0; i < 256 && cp_.busy(); ++i) cp_.tick(); *value = cp_.mmio_read(off); - return 0; + return failed_ ? -1 : 0; } // ----- CP-visible host memory ----- @@ -440,28 +447,60 @@ class vx_device { (unsigned long)st.dma_bytes, st.dma_s); } - int status(uint32_t* value) { return transport_.reg_read(FOLDSIM_REG_STATUS, value); } + // Latches a device error: reported once, and every later access fails. + int fail(const char* what) { + if (!failed_) { + fprintf(stderr, "[VXDRV] %s\n", what); + failed_ = true; + } + return -1; + } - // Queues a command once the queue has room. - int command(uint32_t op, uint32_t addr, uint32_t data) { + int status(uint32_t* value) { + if (transport_.reg_read(FOLDSIM_REG_STATUS, value) != 0) + return fail("cannot read the simulator's status"); + return 0; + } + + // Polls the status until `done` holds of it. A memory error fails the wait + // (unless `error_fails` is false), and so does one longer than the control + // timeout: the target answers a command within a few cycles, so a wait + // that long is a target that stopped. + template + int poll(Done done, bool error_fails, const char* what) { + const auto start = std::chrono::steady_clock::now(); uint32_t s = 0; - do { - if (status(&s) != 0) + for (uint64_t n = 0;; ++n) { + if (failed_ || status(&s) != 0 || (error_fails && mem_error(s))) return -1; - } while (s & FOLDSIM_STATUS_FULL); + if (done(s)) + break; + // The clock is read only every so often: in-process, a status read is + // the cheap part of a poll. + if ((n & 0xff) == 0xff && elapsed_s(start) > control_timeout_s_) { + char msg[160]; + snprintf(msg, sizeof(msg), "the target has not %s in %.0f s (status 0x%x); " + "the simulation is stopped", what, control_timeout_s_, s); + return fail(msg); + } + } + return 0; + } + + // Queues a command once the queue has room. + int command(uint32_t op, uint32_t addr, uint32_t data) { + if (poll([](uint32_t s) { return !(s & FOLDSIM_STATUS_FULL); }, true, + "taken a command") != 0) + return -1; if (reg_write(FOLDSIM_REG_CMD_ADDR, addr) != 0 || reg_write(FOLDSIM_REG_CMD_DATA, data) != 0 || reg_write(FOLDSIM_REG_CMD, op) != 0) - return -1; + return fail("cannot queue a command"); return 0; } - int wait_idle() { - uint32_t s = 0; - do { - if (status(&s) != 0 || mem_error(s)) - return -1; - } while (s & FOLDSIM_STATUS_BUSY); - return 0; + int wait_idle(bool error_fails) { + return poll([](uint32_t s) { return !(s & FOLDSIM_STATUS_BUSY); }, error_fails, + "gone idle"); } // The memory endpoint takes single-beat transactions; a target that makes a @@ -469,14 +508,15 @@ class vx_device { bool mem_error(uint32_t s) { if (!(s & FOLDSIM_STATUS_MEM_ERR)) return false; - if (!mem_error_reported_) { - fprintf(stderr, "[VXDRV] the target made a memory burst, which the memory " - "model does not take; the simulation is stopped\n"); - mem_error_reported_ = true; - } + fail("the target made a memory burst, which the memory model does not take; " + "the simulation is stopped"); return true; } + static double elapsed_s(std::chrono::steady_clock::time_point since) { + return std::chrono::duration(std::chrono::steady_clock::now() - since).count(); + } + // The target runs on its own: once it has been busy for a while, polling it // back to back only spins a core, so the poll sleeps, longer the longer the // target stays busy (up to kMaxPollSleepUs). @@ -514,35 +554,53 @@ class vx_device { std::memcpy(dst, hp, bytes); return; } - transport_.mem_read(addr, dst, bytes); + if (failed_ || transport_.mem_read(addr, dst, bytes) != 0) { + fail("cannot read device memory"); + std::memset(dst, 0, bytes); + } }; h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { if (void* hp = host_region_ptr(addr)) { std::memcpy(hp, src, bytes); return; } - transport_.mem_write(addr, src, bytes); + if (failed_ || transport_.mem_write(addr, src, bytes) != 0) + fail("cannot write device memory"); }; h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { - command(FOLDSIM_CMD_DCR_WRITE, addr, value); + if (!failed_) + command(FOLDSIM_CMD_DCR_WRITE, addr, value); }; h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { - if (command(FOLDSIM_CMD_DCR_READ, addr, tag) != 0) + uint32_t value = 0; + if (failed_ || command(FOLDSIM_CMD_DCR_READ, addr, tag) != 0 || + poll([](uint32_t s) { return (s & FOLDSIM_STATUS_RSP) != 0; }, true, + "answered a DCR read") != 0) + return 0; + if (transport_.reg_read(FOLDSIM_REG_DCR_RSP, &value) != 0) { + fail("cannot read a DCR response"); return 0; - uint32_t s = 0, value = 0; - do { - if (status(&s) != 0) - return 0; - } while (!(s & FOLDSIM_STATUS_RSP)); - transport_.reg_read(FOLDSIM_REG_DCR_RSP, &value); + } return value; }; - h.vortex_start = [this]() { command(FOLDSIM_CMD_START, 0, 0); }; + h.vortex_start = [this]() { + if (!failed_ && command(FOLDSIM_CMD_START, 0, 0) == 0) + launch_start_ = std::chrono::steady_clock::now(); + }; + // A failed device reads idle, so the launch ends; the access after it + // reports the failure. h.vortex_busy = [this]() -> bool { uint32_t s = 0; - if (status(&s) != 0 || mem_error(s)) + if (failed_ || status(&s) != 0 || mem_error(s)) return false; const bool busy = (s & FOLDSIM_STATUS_BUSY) != 0; + if (busy && launch_timeout_s_ > 0 && elapsed_s(launch_start_) > launch_timeout_s_) { + char msg[128]; + snprintf(msg, sizeof(msg), "launch exceeded VORTEX_FOLDSIM_TIMEOUT_S=%.0f s; " + "the simulation is stopped", launch_timeout_s_); + fail(msg); + return false; + } poll_backoff(busy); return busy; }; @@ -557,9 +615,21 @@ class vx_device { static constexpr uint64_t kSpinPolls = 1024; static constexpr uint64_t kMaxPollSleepUs = 100; + // Wall-clock seconds a launch may run before the device is failed; 0, the + // default, never. A command, DCR read or reset the target leaves unanswered + // that long fails it too, after 60 s when it is unset. + static double timeout_env() { + const char* s = getenv("VORTEX_FOLDSIM_TIMEOUT_S"); + return (s != nullptr && s[0] != '\0') ? atof(s) : 0.0; + } + static constexpr double kControlTimeoutS = 60; + counted_transport_t transport_; uint64_t busy_polls_ = 0; - bool mem_error_reported_ = false; + const double launch_timeout_s_ = timeout_env(); + const double control_timeout_s_ = (launch_timeout_s_ > 0) ? launch_timeout_s_ : kControlTimeoutS; + std::chrono::steady_clock::time_point launch_start_; + bool failed_ = false; bool open_ = false; vortex::CommandProcessor cp_; std::mutex host_mu_; From cef2b4f3e483a0e9b819a7407daf5943d64ecc94 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:39:48 -0700 Subject: [PATCH 201/336] hw/syn/foldsim: no stale source list on failure; kernel RTL is not a target source gen_sources.sh wrote the source list before the VENDOR_OK patch, the shell-module removal and the VX_MEM_PORTS check, so a failing check left a list newer than its sources and the next make went on with it. The list is now built in a temporary and moved into place last. Every file under hw/rtl was a dependency of the list, so editing the kernel (foldsim_token/mem/ctrl/afu.sv, foldsim_regs.vh) regenerated the Vortex sources and reran the compiler. Those files are only read by the kernel's packaging and emulation, which depend on them themselves; the target's own wrapper under target/ stays a dependency. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 29ab07499d1cdc1a2c3cf596c34ba07cc0e6d3e2) --- hw/syn/foldsim/Makefile | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index fd9a3b35d7..f23a5f75de 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -104,7 +104,10 @@ IP_DIR := $(BUILD_DIR)/foldsim_afu_ip CFG := $(BUILD_DIR)/config.cfg VBIN := $(BIN_DIR)/foldsim.vbin -RTL_SRCS := $(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh') \ +# The kernel's own sources are not the target's: they are read only when the +# kernel is packaged or emulated. +KERNEL_RTL := $(wildcard $(FOLDSIM_RTL_DIR)/*.sv $(FOLDSIM_RTL_DIR)/*.vh) +RTL_SRCS := $(filter-out $(KERNEL_RTL),$(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh')) \ $(wildcard $(ROOT_DIR)/hw/VX_config.vh) $(wildcard $(ROOT_DIR)/hw/VX_types.vh) \ $(firstword $(MAKEFILE_LIST)) $(CONFIG_STAMP) # The AFU shells of the native flows are second tops, and the foldsim kernel is @@ -118,11 +121,13 @@ all: $(VBIN) # The units that would wrap a Xilinx floating-point core take their soft path, # the one they take wherever no vendor core exists. VENDOR_FP_UNITS := VX_fma_unit.sv VX_fdiv_unit.sv VX_fsqrt_unit.sv +# The list is written to a temporary and moved into place last, so a failing +# step leaves no list that looks current. sources: $(SRCS_LIST) $(SRCS_LIST): $(RTL_SRCS) - rm -rf $(SRCS_DIR) + rm -rf $(SRCS_DIR) $@ $@.tmp cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) -T$(TARGET_TOP) -C$(SRCS_NAME) \ - -O$(SRCS_NAME).f $(RTL_PKGS) + -O$(SRCS_NAME).f.tmp $(RTL_PKGS) @for f in $(VENDOR_FP_UNITS); do \ [ -f $(SRCS_DIR)/$$f ] || continue; \ grep -q 'localparam VENDOR_OK = 1;' $(SRCS_DIR)/$$f || \ @@ -131,11 +136,12 @@ $(SRCS_LIST): $(RTL_SRCS) done @for f in $(SHELL_MODULES); do \ rm -f $(SRCS_DIR)/$$f; \ - grep -v "/$$f$$" $@ > $@.tmp && mv $@.tmp $@; \ + grep -v "/$$f$$" $@.tmp > $@.tmp2 && mv $@.tmp2 $@.tmp; \ done @if grep -qE 'localparam +VX_MEM_PORTS *= *0 *;' $(SRCS_DIR)/VX_gpu_pkg.sv; then \ echo "ERROR: generated sources resolve VX_MEM_PORTS to 0 (no memory interface)"; exit 1; \ fi + mv $@.tmp $@ $(LIFT_LIST): $(SRCS_LIST) $(SRC_DIR)/lift_boundaries.py $(CLASSES_STAMP) rm -rf $(BUILD_DIR)/lift From d53cbc00d52874089d739024b74a6c87257ae7db Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:39:48 -0700 Subject: [PATCH 202/336] lift_boundaries: expand implicit and wildcard connections; refuse positional ones rewrite_site kept only connections with a name: `.*` and positional connections were dropped silently, and an implicit `.clk` became `.clk ()`, leaving the port unconnected. `.p` is now `.p (p)`, `.*` connects every port not named otherwise to the signal of its name, and a positional connection -- at a lifted site, or at a site a promoted parameter becomes a port of -- stops the lift with a message, since the rewrite reorders ports. srcs.f is written only after every edit applied, so a failed lift no longer leaves a list that looks finished. The lift of the tinygpu unit set is unchanged, byte for byte. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b8cab530bca99cc188641987e8678231acd89923) --- hw/syn/foldsim/lift_boundaries.py | 52 +++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/hw/syn/foldsim/lift_boundaries.py b/hw/syn/foldsim/lift_boundaries.py index 282551bfe4..ec8afdf965 100644 --- a/hw/syn/foldsim/lift_boundaries.py +++ b/hw/syn/foldsim/lift_boundaries.py @@ -43,6 +43,7 @@ from pyslang import ast, analysis SIG_KINDS = (ast.SymbolKind.Variable, ast.SymbolKind.Net) +SK = pyslang.syntax.SyntaxKind def read_srcs(path): @@ -251,17 +252,43 @@ def elem_expr(e, k, is_array): sys.exit(f"lift: cannot index connection expression '{e}'") +def site_connections(u, inst, shape): + """{port: expression text} of an instance's connections, by name. + + `.p` is `.p (p)`, and `.*` connects each port not named otherwise to the + signal of its name, as SystemVerilog defines them. Connections by + position are refused: the rewrite reorders the ports.""" + iname = str(inst.decl.name).strip() + conns, wildcard = {}, False + for c in inst.connections: + if isinstance(c, pyslang.parsing.Token): + continue # separator + k = c.kind + if k == SK.NamedPortConnection: + nm = str(c.name).strip() + if c.openParen.kind != pyslang.parsing.TokenKind.OpenParenthesis: + conns[nm] = nm + else: + conns[nm] = str(c.expr).strip() if c.expr is not None else "" + elif k == SK.WildcardPortConnection: + wildcard = True + elif k in (SK.OrderedPortConnection, SK.EmptyPortConnection): + sys.exit(f"lift: {u.name} instance {iname}: ports connected by position; " + f"connect them by name") + else: + sys.exit(f"lift: {u.name} instance {iname}: unsupported port connection {k}") + if wildcard: + for kind, *rest in shape: + if kind in ("plain", "iface") and rest[0] not in conns: + conns[rest[0]] = rest[0] + return conns + + def rewrite_site(u, site_syntax, shape, uid): """Replacement text for one instantiation statement.""" inst = site_syntax.instances[0] iname = str(inst.decl.name).strip() - conns = {} - for c in inst.connections: - if not hasattr(c, "name"): - continue # separator token - nm = str(c.name).strip() - ex = str(c.expr).strip() if c.expr is not None else "" - conns[nm] = ex + conns = site_connections(u, inst, shape) overrides = site_overrides(site_syntax) pre = f"__lift{uid}" decl, glue, pc = [], [], [] @@ -405,6 +432,12 @@ def convert_sites(self): else: ex = default inst_syn = site.instances[0] + if any(not isinstance(c, pyslang.parsing.Token) + and c.kind in (SK.OrderedPortConnection, SK.EmptyPortConnection) + for c in inst_syn.connections): + sys.exit(f"lift: {defname} instance {str(inst_syn.decl.name).strip()}: ports " + f"connected by position cannot take {param} as a port; " + f"connect them by name") op = inst_syn.openParen.location.offset + 1 self.edits[f].append((op, op, f"\n .{param} ({ex}),")) @@ -519,10 +552,13 @@ def visit(sym): else: shutil.copyfile(f, dst) out_files.append(dst) + if applied != len(edits): + sys.exit(f"lift: edits for unmatched files: {sorted(set(edits) - matched)}") for name, text in wrappers.items(): p = os.path.join(a.out, "src", f"{name}__lift.sv") open(p, "w").write(text) out_files.append(p) + # Written last: the list is the build's mark of a finished lift. with open(os.path.join(a.out, "srcs.f"), "w") as o: for x in defines: o.write(f"+define+{x}\n") @@ -532,8 +568,6 @@ def visit(sym): for x in out_files: o.write(x + "\n") print(f"lift: wrote {len(out_files)} files ({applied} of {len(edits)} rewritten) to {a.out}") - if applied != len(edits): - sys.exit(f"lift: edits for unmatched files: {sorted(set(edits) - matched)}") if __name__ == "__main__": From f27da4657feda4391036ec1e3535bda97b3e3288 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:39:48 -0700 Subject: [PATCH 203/336] tests: run-foldsim for the OpenCL and HIP suites Only tests/regression had run-foldsim; the OpenCL and HIP makefiles get the same target, with their run-firesim's environment: TARGET=hw runs FPGA_BIN_DIR's foldsim.vbin on the V80, anything else the in-process simulator. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 033c522df437b76917735ab44ccbe467f825cb53) --- tests/hip/common.mk | 14 +++++++++++++- tests/opencl/common.mk | 12 ++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/hip/common.mk b/tests/hip/common.mk index 07b530afe1..28101e6df1 100644 --- a/tests/hip/common.mk +++ b/tests/hip/common.mk @@ -194,9 +194,21 @@ else $(HIP_OCL_ENV) XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) endif +# foldsim: TARGET=hw runs FPGA_BIN_DIR's foldsim.vbin (hw/syn/foldsim) on the V80; +# any other TARGET, avedsim, simulates the same kernel in-process (convention +# established in tests/regression/common.mk). +run-foldsim: $(PROJECT) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/foldsim DESTDIR=$(VORTEX_RT_LIB) TARGET=$(if $(filter hw,$(TARGET)),hw,avedsim) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh + $(HIP_OCL_ENV) VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/foldsim.vbin LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) +else + $(HIP_OCL_ENV) LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) +endif + clean: rm -f $(PROJECT) *.o *.vxbin *.dump *.ll *.log *.spv common.h -.PHONY: all run-simx run-rtlsim run-opae run-xrt run-firesim clean +.PHONY: all run-simx run-rtlsim run-opae run-xrt run-firesim run-foldsim clean FORCE: diff --git a/tests/opencl/common.mk b/tests/opencl/common.mk index 39c3823cfd..facdae4d56 100644 --- a/tests/opencl/common.mk +++ b/tests/opencl/common.mk @@ -177,6 +177,18 @@ else XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) endif +# foldsim: TARGET=hw runs FPGA_BIN_DIR's foldsim.vbin (hw/syn/foldsim) on the V80; +# any other TARGET, avedsim, simulates the same kernel in-process (convention +# established in tests/regression/common.mk). +run-foldsim: $(PROJECT) $(KERNEL_SRCS) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/foldsim DESTDIR=$(VORTEX_RT_LIB) TARGET=$(if $(filter hw,$(TARGET)),hw,avedsim) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/foldsim.vbin LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) +else + LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) +endif + .depend: $(SRCS) $(CXX) $(CXXFLAGS) -MM $^ > .depend; From d93a8232054ee34327a28f17992a96d699e9d413 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:39:48 -0700 Subject: [PATCH 204/336] docs: foldsim's avedsim builds the kernel; CIRCT cmake flags; blackbox targets simulation.md said avedsim has nothing to build; the run compiles the simulator with foldsim and builds it with Verilator, for the app's CONFIGS. building_toolchain.md's CIRCT configure line now matches ci/circt_install.sh (-DCIRCT_SLANG_BUILD_FROM_SOURCE=ON), and blackbox.sh --help names foldsim's targets. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 55515a237eea6511380bd944dc281d175d5450a6) --- ci/blackbox.sh | 2 ++ docs/building_toolchain.md | 3 ++- docs/simulation.md | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/blackbox.sh b/ci/blackbox.sh index 319abe6b85..99a926e502 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -30,6 +30,8 @@ show_help() echo "--target: xrt: xrtsim (default), hw_emu, hw -- aved: avedsim (default), sim, hw" echo " firesim: the targets of the card's native driver; xrtsim (default) or" echo " avedsim need no image, the others run FPGA_BIN_DIR's (hw/syn/firesim)" + echo " foldsim: avedsim (default) simulates the kernel in Verilator, built" + echo " by the run; hw runs FPGA_BIN_DIR's foldsim.vbin (hw/syn/foldsim)" echo "--app: any subfolder test under regression, graphics, mpi, opencl, or hip" echo "--perf: 0=disable, 1=core, 3=icache, 4=dcache, 5=l2cache, 6=l3cache, 7=mem," echo " 11=tcu, 12=raster, 13=tex, 14=om, 15=rtu, 16=dxa" diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index 2947fd9ac5..9908b9e451 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -1133,7 +1133,8 @@ cmake -G Ninja -S $CIRCT_SRC/llvm/llvm -B $CIRCT_SRC/build \ -DLLVM_EXTERNAL_CIRCT_SOURCE_DIR=$CIRCT_SRC -DLLVM_TARGETS_TO_BUILD=host \ -DLLVM_ENABLE_ASSERTIONS=OFF -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ -DLLVM_ENABLE_ZLIB=OFF -DLLVM_ENABLE_ZSTD=OFF -DLLVM_ENABLE_LIBXML2=OFF \ - -DLLVM_INSTALL_UTILS=ON -DCIRCT_SLANG_FRONTEND_ENABLED=ON + -DLLVM_INSTALL_UTILS=ON -DCIRCT_SLANG_FRONTEND_ENABLED=ON \ + -DCIRCT_SLANG_BUILD_FROM_SOURCE=ON ``` ### Notes diff --git a/docs/simulation.md b/docs/simulation.md index 59aa1e755e..dea64ec0a2 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -92,9 +92,9 @@ against the RTL. Its kernel carries a small control block and a latency-bandwidth memory model; the target runs free, and the runtime talks to it like the native V80 driver does (AXI4-Lite registers, QDMA for memory). -| target | what runs | image built by | +| target | what runs | built by | |---|---|---| -| `avedsim` (default) | the same kernel in Verilator, in-process | nothing to build | +| `avedsim` (default) | the same kernel in Verilator, in-process | the run itself: the foldsim compiler, then a Verilator build of the kernel, for the app's `CONFIGS` | | `hw` | the image on the V80 | `make -C hw/syn/foldsim` | ```bash From fdcd33ebf5a9e1091ade0118339ce39195f98a83 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:45:53 -0700 Subject: [PATCH 205/336] foldsim runtime: a control wait fails on no progress, not on wall time A command, DCR read or reset now fails only once the target has advanced no cycle for the control timeout. In-process, a large target can take minutes of wall time to answer a DCR read that flushes its caches; a wall-clock bound failed such runs while the simulator was making progress. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit fc479c4eb45858ee60c41176083cec805f6c939b) --- docs/designs/foldsim.md | 4 ++-- docs/simulation.md | 4 ++-- sw/runtime/foldsim/vortex.cpp | 37 +++++++++++++++++++++++------------ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index b059bbf049..d8284ccd48 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -223,8 +223,8 @@ that timed out — and from then on every register access the runtime core makes fails, and the launch, DCR access or copy that hit it fails with it. A launch longer than `VORTEX_FOLDSIM_TIMEOUT_S` seconds is such an error (no limit by default), as is a command the target does not take, a DCR read it -does not answer, or a reset it does not finish within that time (60 s when -it is unset). +does not answer, or a reset it does not finish while its target cycles stop +advancing for that time (60 s when it is unset). --- diff --git a/docs/simulation.md b/docs/simulation.md index dea64ec0a2..c99f29b1dc 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -120,8 +120,8 @@ The flow needs CIRCT (`circt-verilog`, `circt-opt`, `firtool`) and foldsim knobs: `FOLDSIM_MEM_READ_LATENCY`, `FOLDSIM_MEM_WRITE_LATENCY` (target cycles, 30 by default) and `FOLDSIM_MEM_MAX_READS`, `FOLDSIM_MEM_MAX_WRITES` (16). `VORTEX_FOLDSIM_TIMEOUT_S=` fails a launch that runs longer; a command, -DCR read or reset the target leaves unanswered fails after it too, or after -60 s when it is unset. At exit the runtime reports the target and kernel-clock cycles the simulator ran +DCR read or reset the target leaves unanswered fails too once the target has +advanced no cycle for that long, or for 60 s when it is unset. At exit the runtime reports the target and kernel-clock cycles the simulator ran and how busy each threaded class was. The design is [designs/foldsim.md](designs/foldsim.md). diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp index 5dceb59571..c5249196ed 100644 --- a/sw/runtime/foldsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -463,25 +463,37 @@ class vx_device { } // Polls the status until `done` holds of it. A memory error fails the wait - // (unless `error_fails` is false), and so does one longer than the control - // timeout: the target answers a command within a few cycles, so a wait - // that long is a target that stopped. + // (unless `error_fails` is false), and so does a simulator whose target + // cycles stop advancing for the control timeout. Progress, not wall time, + // is the measure: in-process, a large target may legitimately take minutes + // of wall time to answer a DCR read that flushes its caches. template int poll(Done done, bool error_fails, const char* what) { - const auto start = std::chrono::steady_clock::now(); + auto last_progress = std::chrono::steady_clock::now(); + uint32_t last_tcycles = 0; + if (transport_.reg_read(FOLDSIM_REG_TCYCLES_LO, &last_tcycles) != 0) + return fail("cannot read the simulator's target cycles"); uint32_t s = 0; for (uint64_t n = 0;; ++n) { if (failed_ || status(&s) != 0 || (error_fails && mem_error(s))) return -1; if (done(s)) break; - // The clock is read only every so often: in-process, a status read is - // the cheap part of a poll. - if ((n & 0xff) == 0xff && elapsed_s(start) > control_timeout_s_) { - char msg[160]; - snprintf(msg, sizeof(msg), "the target has not %s in %.0f s (status 0x%x); " - "the simulation is stopped", what, control_timeout_s_, s); - return fail(msg); + // Checked only every so often: in-process, a status read is the cheap + // part of a poll. + if ((n & 0xff) == 0xff) { + uint32_t tcycles = 0; + if (transport_.reg_read(FOLDSIM_REG_TCYCLES_LO, &tcycles) != 0) + return fail("cannot read the simulator's target cycles"); + if (tcycles != last_tcycles) { + last_tcycles = tcycles; + last_progress = std::chrono::steady_clock::now(); + } else if (elapsed_s(last_progress) > control_timeout_s_) { + char msg[192]; + snprintf(msg, sizeof(msg), "the target has not %s, and has not advanced a cycle in " + "%.0f s (status 0x%x); the simulation is stopped", what, control_timeout_s_, s); + return fail(msg); + } } } return 0; @@ -617,7 +629,8 @@ class vx_device { // Wall-clock seconds a launch may run before the device is failed; 0, the // default, never. A command, DCR read or reset the target leaves unanswered - // that long fails it too, after 60 s when it is unset. + // while it advances no cycle for that long fails it too, after 60 s when it + // is unset. static double timeout_env() { const char* s = getenv("VORTEX_FOLDSIM_TIMEOUT_S"); return (s != nullptr && s[0] != '\0') ? atof(s) : 0.0; From 9b726904996f0b420347b0b3c106c9ae5c1d0aad Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:45:53 -0700 Subject: [PATCH 206/336] circt_elaborate: Verilog-2005 output carries no size casts -f v2005 is read by tools with a Verilog-2005 frontend, which reject SystemVerilog N'(...) casts; firtool's disallowSizeCasts keeps signed multiplies and bitcasts free of them. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 1bc09626b8e5ddfce257615688b5a6057fdcf8ff) --- hw/scripts/circt_elaborate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/scripts/circt_elaborate.sh b/hw/scripts/circt_elaborate.sh index 144e0b6cde..c6e07b1590 100755 --- a/hw/scripts/circt_elaborate.sh +++ b/hw/scripts/circt_elaborate.sh @@ -33,7 +33,7 @@ # colliding with a write undefined in block RAM; other tools ignore it; # - -f sv (the default) writes SystemVerilog, for Vivado and Quartus; -f # v2005 writes Verilog-2005 for tools with a Verilog frontend (yosys): no -# packed arrays or structs, no local variables. +# packed arrays or structs, no local variables, no size casts. # # The sources are a filelist (-l, as gen_sources.sh writes it: +define+, # +incdir+ and files in compile order), or include directories whose sources @@ -148,7 +148,7 @@ passes=(--hw-expand-array-writes "--seq-retime-firmem-reads=mode=$mem_mode repor if [ "$format" = v2005 ]; then passes+=(--seq-lower-aggregate-regs --hw-aggregate-ports-to-int --hw-convert-bitcasts --hw-aggregate-to-comb) - lowering=noAlwaysComb,disallowLocalVariables,disallowPackedArrays,disallowPackedStructAssignments,svVarDeclStyle=reg,locationInfoStyle=none,omitVersionComment + lowering=noAlwaysComb,disallowLocalVariables,disallowPackedArrays,disallowPackedStructAssignments,disallowSizeCasts,svVarDeclStyle=reg,locationInfoStyle=none,omitVersionComment else lowering=locationInfoStyle=none,omitVersionComment fi From d9a28ce73c016e269a3b20ed963715942fb32a5c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:45:53 -0700 Subject: [PATCH 207/336] firesim: no vendor FP core in an aved image On aved the simulator's clock gates become clock enables in synthesis, which cannot enter an out-of-context vendor core. VX_fpu_dsp's FMA, divider and square root select Xilinx cores on Vivado, so a V80 image failed the platform's check for logic-driven clocks. The aved build now takes the STD FPU and its units' soft paths in the flow's own copy of the sources, as the foldsim flow does; the metasimulator and the image run the same FPU. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 80f322aeb4a7a8b2c0c087212f77e0ccb1f14607) --- docs/designs/firesim_integration.md | 7 +++++-- hw/syn/firesim/Makefile | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index 6c44095153..e04d223bf8 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -93,8 +93,11 @@ AND marked `gated_clock` that synthesis's gated-clock conversion moves onto the clock-enable pin of every register, RAM and DSP it drives. `BUFGCE` semantics, no derived clock; a pre-opt check fails the build if any kernel clock is still driven by logic. (This is also why vendor OOC IP cannot sit -inside the kernel on aved: the conversion cannot enter it, so the RTU uses -the soft FMA/divider there — `VX_CFG_RTU_USE_DSP=0`.) +inside the kernel on aved: the conversion cannot enter it, so no unit takes +one there. The FPU is the STD one (`VX_CFG_FPU_TYPE_STD`), its FMA, divider +and square root take their soft path in the flow's copy of the sources, and +the RTU uses the soft FMA/divider — `VX_CFG_RTU_USE_DSP=0`. The metasimulator +and the image therefore run the same FPU, as the foldsim flow's do.) ### 1.3 Host-decoupled execution diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 7ebfdac3c5..c5f02886e0 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -242,12 +242,14 @@ PLATFORM_DEFINES += -DPLATFORM_MEMORY_ID_WIDTH=4 # synthesis (the V80's reconfigurable partition has no BUFGCE, and a fabric gate # is not safe; see FireSimFabricClockGate). That conversion cannot enter a # vendor core: each is synthesized out of context behind a kept hierarchy, so -# it would stay on a logic-driven clock and the platform build rejects it. The -# RTU is the only unit that reaches for them by default, so it uses the soft -# FMA and divider here -- the same RTL the metasimulator runs -- with the soft -# divider's latency in place of xil_fdiv's. +# it would stay on a logic-driven clock and the platform build rejects it. So +# no unit may take one here: the FPU is the STD one, as in the metasimulator, +# its units' copies in the flattened tree take their soft path (VENDOR_FP_UNITS +# below), and the RTU uses the soft FMA and divider, with the soft divider's +# latency in place of xil_fdiv's. ifeq ($(FIRESIM_PLATFORM), aved) -PLATFORM_DEFINES += -DVX_CFG_RTU_USE_DSP=0 -DVX_CFG_RTU_FDIV_LATENCY=17 +PLATFORM_DEFINES += -DVX_CFG_FPU_TYPE_STD -DVX_CFG_RTU_USE_DSP=0 -DVX_CFG_RTU_FDIV_LATENCY=17 +VENDOR_FP_UNITS := VX_fma_unit.sv VX_fdiv_unit.sv VX_fsqrt_unit.sv endif # Vortex's performance counters are the only view inside the core that does not @@ -360,6 +362,12 @@ $(VIVADO_SRCS_LIST): $(RTL_SRCS) grep -v "/$$f$$" $(VIVADO_SRCS_LIST) > $(VIVADO_SRCS_LIST).tmp && \ mv $(VIVADO_SRCS_LIST).tmp $(VIVADO_SRCS_LIST); \ done + @for f in $(VENDOR_FP_UNITS); do \ + [ -f $(VIVADO_SRCS_DIR)/$$f ] || continue; \ + grep -q 'localparam VENDOR_OK = 1;' $(VIVADO_SRCS_DIR)/$$f || \ + { echo "ERROR: $$f no longer selects its vendor core by VENDOR_OK"; exit 1; }; \ + sed -i 's/localparam VENDOR_OK = 1;/localparam VENDOR_OK = 0;/' $(VIVADO_SRCS_DIR)/$$f; \ + done @# gen_sources.sh lists packages in find(1) order, which is directory order, @# not dependency order. Synthesis reorders on its own; a simulator compiles @# the list as given. Under -DSIMULATION VX_trace_pkg imports VX_gpu_pkg, and From e65feeb8e3be3b916713272cb67a20b90a0f1e4e Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:45:53 -0700 Subject: [PATCH 208/336] hw/syn/foldsim: a rebuilt compiler tool regenerates the simulator The generated simulator depended on the foldsim driver script alone, so a rebuilt foldsim-opt or CIRCT tool left a stale simulator in place. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5238cd53a5fcf1f371aeb5aed7c04b7cba63d127) --- hw/syn/foldsim/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index f23a5f75de..c608f37151 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -149,7 +149,11 @@ $(LIFT_LIST): $(SRCS_LIST) $(SRC_DIR)/lift_boundaries.py $(CLASSES_STAMP) --out $(BUILD_DIR)/lift $(foreach u,$(FOLDSIM_LIFT),--unit '$(u)') generate: $(GEN_SV) -$(GEN_SV): $(TARGET_SRCS) $(FOLDSIM) $(CLASSES_STAMP) +# The compiler is the driver and the tools it runs: foldsim-opt beside it and +# CIRCT's, so a rebuilt tool regenerates the simulator. +FOLDSIM_TOOLS := $(FOLDSIM) $(wildcard $(dir $(FOLDSIM))foldsim-opt \ + $(addprefix $(CIRCT_BIN)/,circt-verilog circt-opt firtool)) +$(GEN_SV): $(TARGET_SRCS) $(FOLDSIM_TOOLS) $(CLASSES_STAMP) $(FOLDSIM) --srcs $(TARGET_SRCS) --top $(TARGET_TOP) --out $(GEN_DIR) \ $(if $(CIRCT_BIN),--circt-bin $(CIRCT_BIN)) $(FOLDSIM_CLASSES) From 82e512a39df02a096aab3b16094ec406123ae704 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 15:51:15 -0700 Subject: [PATCH 209/336] foldsim install: tests take lit from PATH; FireSim's toolchain notes cover the V80 An installed LLVM has no llvm-lit, so the producer could not run check-foldsim against an installed CIRCT; it now takes pip's lit (or LIT). building_toolchain.md's FireSim section described the U55C alone. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c808baa72ed4cb8f947be46a8dcd29e19d9bbe13) --- ci/foldsim_install.sh.in | 11 +++++++++-- docs/building_toolchain.md | 17 ++++++++++------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/ci/foldsim_install.sh.in b/ci/foldsim_install.sh.in index 35ffc04ebd..5ce95ff397 100755 --- a/ci/foldsim_install.sh.in +++ b/ci/foldsim_install.sh.in @@ -26,7 +26,7 @@ # prebuilt and never run this script. # # Release, assertions off, libstdc++ linked statically. Needs git, cmake >= -# 3.20, ninja and a C++17 compiler; installs nothing system-wide. +# 3.20, ninja, lit (pip) and a C++17 compiler; installs nothing system-wide. # # Idempotent: re-running rebuilds only what changed. @@ -68,13 +68,20 @@ fi # ---------------------------------------------------------------------- # 2. Configure, build and test # ---------------------------------------------------------------------- +# An installed LLVM has no llvm-lit; the tests take lit from PATH (pip's +# `lit`) unless LIT names one. +LIT=${LIT:-$(command -v lit || command -v llvm-lit || true)} +if [ -z "$LIT" ]; then + echo "error: no lit for foldsim's tests: pip install lit, or set LIT." + exit 1 +fi cmake -G Ninja -S "$FOLDSIM_SRC" -B "$FOLDSIM_BUILD" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX="$FOLDSIM_PREFIX" \ -DCIRCT_DIR="$CIRCT_PREFIX/lib/cmake/circt" \ -DMLIR_DIR="$CIRCT_PREFIX/lib/cmake/mlir" \ -DLLVM_DIR="$CIRCT_PREFIX/lib/cmake/llvm" \ - -DLLVM_EXTERNAL_LIT="$CIRCT_PREFIX/bin/llvm-lit" \ + -DLLVM_EXTERNAL_LIT="$LIT" \ -DLLVM_STATIC_LINK_CXX_STDLIB=ON ninja -C "$FOLDSIM_BUILD" ninja -C "$FOLDSIM_BUILD" check-foldsim diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index 9908b9e451..9c1225a61a 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -23,7 +23,7 @@ The components covered here are: 9. [Mesa for Vortex](#9-mesa-for-vortex-vulkan) — Vulkan software stack (lavapipe) the Vortex Vulkan driver builds on (optional) 10. [gem5](#10-gem5) — cycle-level CPU simulator hosting the Vortex SimObject (X86 + ARM); consumed by `ci/regression.sh --gem5` 11. [SST](#11-sst) — Structural Simulation Toolkit (sst-core + sst-elements + OpenMPI); consumed by the SST-driven regression configurations -12. [FireSim](#12-firesim) — FPGA-accelerated host-decoupled simulation on the Alveo U55C; needed only to elaborate a target or build a bitstream (optional) +12. [FireSim](#12-firesim) — FPGA-accelerated host-decoupled simulation on the Alveo U55C or V80; needed only to elaborate a target or build a bitstream (optional) 13. [SLASH / Alveo V80](#13-slash--alveo-v80-the-aved-backend) — AMD Alveo V80 platform stack for the `aved` backend (optional) 14. [CIRCT](#14-circt) — the SystemVerilog frontend of every synthesis flow (`circt-verilog`, `circt-opt`, `firtool`) 15. [foldsim](#15-foldsim) — FAME-5 simulator compiler of `hw/syn/foldsim`, built on CIRCT (optional) @@ -882,15 +882,17 @@ rm -rf "$SST_CORE_HOME/include" "$SST_ELEMENTS_HOME/include" \ ## 12. FireSim **Purpose**: FPGA-accelerated, host-decoupled simulation of Vortex on a -Xilinx Alveo U55C. Consumed by the `firesim` driver +Xilinx Alveo U55C (the `vitis` platform, XRT) or an AMD Alveo V80 (the `aved` +platform, SLASH/VRT). Consumed by the `firesim` driver (`./ci/blackbox.sh --driver=firesim`) and by the bitstream build under `hw/syn/firesim`. See [`designs/firesim_integration.md`](designs/firesim_integration.md) for the architecture. -**Only needed to build.** Running an existing `.xclbin` requires XRT and the -Vortex runtime and nothing from `$TOOLDIR/firesim` — the transport links -against XRT directly. Install this component if you intend to elaborate a +**Only needed to build.** Running an existing image — a `.xclbin` with XRT, +a `.vbin` with VRT — requires the card's runtime and the Vortex runtime and +nothing from `$TOOLDIR/firesim`: the transport links against XRT or VRT +directly. Install this component if you intend to elaborate a target or build a bitstream. ### Installing @@ -953,8 +955,9 @@ make -C hw/syn/firesim check-firesim activating conda at all. - **Vortex's changes to FireSim live on the branch, not as patches.** `git diff 1.21.0..$FIRESIM_REV` is the authoritative delta: Vitis platform - fixes, the host-memory base as a config field, driver build and teardown, - and cycle-counter accounting, all upstreamable. + fixes, the AVED platform for the V80 (shim, fabric clock gates, kernel + clock, partitioned synthesis of large models), the host-memory base as a + config field, driver build and teardown, and cycle-counter accounting. ## 13. SLASH / Alveo V80 (the `aved` backend) From fe777804c91e7594a78595360c35c4c61e2273a4 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 13 Oct 2026 17:29:02 -0700 Subject: [PATCH 210/336] docs: tinygpu on foldsim and the firesim backend, on the V80 foldsim.md gains the tinygpu comparison (area, timing, cycles, kernel time; same tree, configuration and card), its validation against rtlsim on AxiLatencyPipe, and why rtlsim's cached configurations read 8 cycles more: its harness launches while the L1 banks still sweep their sets after reset. firesim_integration.md gains its V80 results. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 41811b4764676b0b14a829c040b1a160257bf014) --- docs/designs/firesim_integration.md | 22 +++++++++++-- docs/designs/foldsim.md | 49 ++++++++++++++++++++++++++--- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index e04d223bf8..fec9b9d66f 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -524,8 +524,26 @@ faster than rtlsim on `sgemm -n32`). Instruction counts match rtlsim exactly; cycle counts differ −1.3%…+7.1% — the FASED timing model, not the core: FPGA and hw_emu agree exactly. -Threaded images and their results are the foldsim flow's: -[foldsim.md](foldsim.md). +### 9.2 V80 + +tinygpu: 1 cluster / 1 core / 2 warps / 2 threads, no L1 caches or local +memory, STD FPU, one memory bank, kernel clock 143 MHz (WNS +0.752 ns). +Kernel-only synthesis: 103,369 LUTs (51,943 as distributed RAM), 34,376 +FFs, 5 BRAM tiles, 8 URAM, 8 DSP. + +| app | instrs | cycles, metasim (`avedsim`) | cycles, V80 | kernel time, V80 | +|---|---:|---:|---:|---:| +| `demo` | 1,222 | 25,624 | 25,624 | | +| `sgemm -n16` | 15,240 | 317,163 | 317,163 | 5 ms | +| `sgemm -n64` | 673,800 | | 13,937,835 | 178 ms | + +The metasimulator and the image agree cycle for cycle. Instruction counts +match rtlsim's. Cycles are 5.0 % above rtlsim's on the same latency-bandwidth +settings (`AXI_BUS=1`; `demo`, `sgemm -n16`), and 5.1 % above the foldsim +image's on `sgemm -n64`, from the memory model's channel mechanics. + +Threaded images, and the same tinygpu on the foldsim flow, are in +[foldsim.md](foldsim.md) §7. --- diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index d8284ccd48..2549783982 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -242,8 +242,14 @@ advancing for that time (60 s when it is unset). latency checking the device-side protocol, resets at random points with HBM busy, and each kind of burst; every response and id checked, none from before a reset delivered after it. -- **Applications** in the in-process simulator and on the V80 (§7). -- **CIRCT lit tests** for every pass (`test/Transforms/foldsim-*.mlir`). +- **Applications** in the in-process simulator and on the V80, against rtlsim + behind the AXI4 adapter on `AxiLatencyPipe` (`AXI_BUS=1`) with the same + memory settings: retired instructions and cycles identical (§7). +- **foldsim's own tests** (`check-foldsim` in the foldsim repository): lit + tests of every pass and of the driver, and small threaded simulators run in + Verilator against their expected tokens — one and several threads, + registered memory reads, register initial values, eager outputs, a reset of + one cycle. --- @@ -259,7 +265,8 @@ classes (4 threads each). | foldsim, ready scheduling, in-process | Verilator | 8,702 | 415,454 | | 8.00 | | foldsim, threads in turn | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 or 12.00 | | foldsim, ready scheduling | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 | -| rtlsim, AXI4 with `AxiLatencyPipe` | Verilator | 8,710 | | | | +| rtlsim, AXI4 with `AxiLatencyPipe` | Verilator | 8,710 | 415,462 | | | +| rtlsim, same, launched after the caches' reset sweep | Verilator | 8,702 | 415,454 | | | On the V80 the kernel closes at 149.8 MHz (run at 143 MHz) in 138k routed LUTs, against 219k for the firesim backend's image of the same configuration. @@ -269,7 +276,41 @@ sgemm n=128 takes 196 ms with ready scheduling: 50× SimX on the same host, Instruction counts equal the firesim backend's and rtlsim's. Cycle counts are 1.5–6 % below the firesim backend's, whose memory model has the same latency-bandwidth settings but adds channel cycles `AxiLatencyPipe` does not; -foldsim agrees with `AxiLatencyPipe` (rtlsim) instead. +foldsim agrees with `AxiLatencyPipe` (rtlsim) instead: every memory +transaction in the same target cycle. rtlsim's harness starts the kernel 8 +cycles after reset, while each L1 bank still invalidates one set per cycle +(64 for the instruction cache), so its first fetch waits 8 cycles the +counters see; a host's launch comes thousands of cycles later. Given that +settle time, rtlsim's cycles are foldsim's exactly. + +### 7.1 tinygpu: foldsim and the firesim backend + +The same Vortex tree, configuration and V80 on both flows: 1 core, 2 warps, +2 threads, no L1 caches or local memory, STD FPU, one memory bank, kernel +clock 143 MHz. Every class has one instance, so foldsim folds nothing here: +this compares the two simulators' fidelity and overheads, not threading. + +| | foldsim | firesim backend | +|---|---:|---:| +| kernel LUTs (synthesis) | 34,393 | 103,369 | +| of them distributed RAM | 1,822 | 51,943 | +| FFs / BRAM tiles / URAM / DSP | 30,198 / 4 / 0 / 8 | 34,376 / 5 / 8 / 8 | +| WNS at 143 MHz | +0.282 ns | +0.752 ns | +| image build (side by side, same host) | 56 min | 80 min | +| `demo` cycles, V80 (= Verilator) | 24,402 | 25,624 | +| `sgemm -n16` cycles, V80 (= Verilator) | 301,923 | 317,163 | +| `sgemm -n64` cycles, V80 | 13,264,035 | 13,937,835 | +| `sgemm -n64` kernel time, V80 | 278 ms | 178 ms | +| host cycles per target cycle, `sgemm -n64` | 3.00 | 1.84 (FMR) | + +Instruction counts are identical on both flows, in Verilator and on the V80. +foldsim's cycles equal rtlsim's behind the AXI4 adapter on `AxiLatencyPipe` +with the same memory settings (`AXI_BUS=1`) exactly; the firesim backend's +are 5.0–5.1 % above, from its memory model's channel mechanics. With nothing +to fold, a foldsim target cycle costs three host cycles (the token exchange +of each one-thread model), so the firesim backend runs tinygpu faster; +foldsim's advantage is the GPU it fits, its area falling with every class it +folds (s2w4i2 above). --- From 283cc926b004f600bec1a332e6609124fe992c53 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 14 Oct 2026 01:04:08 -0700 Subject: [PATCH 211/336] tests/opencl: fix the defects GCC 13 rejects under -Werror The OpenCL samples failed to build with GCC 13 (-Werror=address, -Werror=maybe-uninitialized); GCC 11 on the CI runners does not diagnose them, but both are real bugs: - oclUtils.cpp (dotproduct, blackscholes, transpose): `device_string != 0` and `cDevString != 0` compare an array's address with NULL, which is always true. Test for a non-empty extension string instead, and start cDevString empty so the test holds if clGetDeviceInfo fails. - bfs/main.cc: the catch handler frees every cl_mem handle, but when _clInit or an early _clMalloc throws, the later handles are still uninitialized. Start them NULL; _clFree already skips NULL. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 844c11e9750cad883f088f01c9e7a7ae46c1a4eb) --- tests/opencl/bfs/main.cc | 5 +++-- tests/opencl/blackscholes/oclUtils.cpp | 6 +++--- tests/opencl/dotproduct/oclUtils.cpp | 6 +++--- tests/opencl/transpose/oclUtils.cpp | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/opencl/bfs/main.cc b/tests/opencl/bfs/main.cc index c676a40c53..490f6f4b24 100755 --- a/tests/opencl/bfs/main.cc +++ b/tests/opencl/bfs/main.cc @@ -75,8 +75,9 @@ void run_bfs_gpu(int no_of_nodes, Node *h_graph_nodes, int edge_list_size, // int number_elements = height*width; char h_over; - cl_mem d_graph_nodes, d_graph_edges, d_graph_mask, d_updating_graph_mask, - d_graph_visited, d_cost, d_over; + cl_mem d_graph_nodes = NULL, d_graph_edges = NULL, d_graph_mask = NULL, + d_updating_graph_mask = NULL, d_graph_visited = NULL, d_cost = NULL, + d_over = NULL; try { //--1 transfer data from host to device diff --git a/tests/opencl/blackscholes/oclUtils.cpp b/tests/opencl/blackscholes/oclUtils.cpp index 7a238b556d..8f82cf0d12 100644 --- a/tests/opencl/blackscholes/oclUtils.cpp +++ b/tests/opencl/blackscholes/oclUtils.cpp @@ -259,7 +259,7 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) // CL_DEVICE_EXTENSIONS: get device extensions, and if any then parse & log the string onto separate lines clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(device_string), &device_string, NULL); - if (device_string != 0) + if (device_string[0] != '\0') { shrLogEx(iLogMode, 0, "\n CL_DEVICE_EXTENSIONS:"); std::string stdDevString; @@ -341,13 +341,13 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) ////////////////////////////////////////////////////////////////////////////// int oclGetDevCap(cl_device_id device) { - char cDevString[1024]; + char cDevString[1024] = ""; bool bDevAttributeQuery = false; int iDevArch = -1; // Get device extensions, and if any then search for cl_nv_device_attribute_query clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(cDevString), &cDevString, NULL); - if (cDevString != 0) + if (cDevString[0] != '\0') { std::string stdDevString; stdDevString = std::string(cDevString); diff --git a/tests/opencl/dotproduct/oclUtils.cpp b/tests/opencl/dotproduct/oclUtils.cpp index 7a238b556d..8f82cf0d12 100644 --- a/tests/opencl/dotproduct/oclUtils.cpp +++ b/tests/opencl/dotproduct/oclUtils.cpp @@ -259,7 +259,7 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) // CL_DEVICE_EXTENSIONS: get device extensions, and if any then parse & log the string onto separate lines clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(device_string), &device_string, NULL); - if (device_string != 0) + if (device_string[0] != '\0') { shrLogEx(iLogMode, 0, "\n CL_DEVICE_EXTENSIONS:"); std::string stdDevString; @@ -341,13 +341,13 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) ////////////////////////////////////////////////////////////////////////////// int oclGetDevCap(cl_device_id device) { - char cDevString[1024]; + char cDevString[1024] = ""; bool bDevAttributeQuery = false; int iDevArch = -1; // Get device extensions, and if any then search for cl_nv_device_attribute_query clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(cDevString), &cDevString, NULL); - if (cDevString != 0) + if (cDevString[0] != '\0') { std::string stdDevString; stdDevString = std::string(cDevString); diff --git a/tests/opencl/transpose/oclUtils.cpp b/tests/opencl/transpose/oclUtils.cpp index 7a238b556d..8f82cf0d12 100644 --- a/tests/opencl/transpose/oclUtils.cpp +++ b/tests/opencl/transpose/oclUtils.cpp @@ -259,7 +259,7 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) // CL_DEVICE_EXTENSIONS: get device extensions, and if any then parse & log the string onto separate lines clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(device_string), &device_string, NULL); - if (device_string != 0) + if (device_string[0] != '\0') { shrLogEx(iLogMode, 0, "\n CL_DEVICE_EXTENSIONS:"); std::string stdDevString; @@ -341,13 +341,13 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) ////////////////////////////////////////////////////////////////////////////// int oclGetDevCap(cl_device_id device) { - char cDevString[1024]; + char cDevString[1024] = ""; bool bDevAttributeQuery = false; int iDevArch = -1; // Get device extensions, and if any then search for cl_nv_device_attribute_query clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(cDevString), &cDevString, NULL); - if (cDevString != 0) + if (cDevString[0] != '\0') { std::string stdDevString; stdDevString = std::string(cDevString); From f56857a8f4237a4418b6e67a549dfd50fd1968b5 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 14 Oct 2026 01:05:47 -0700 Subject: [PATCH 212/336] ci/circt_install.sh: require cmake >= 3.28, as slang does slang, CIRCT's SystemVerilog frontend, sets cmake_minimum_required(3.28); the script claimed 3.20, and on Ubuntu 22.04 (cmake 3.22) the build died in configure after the whole LLVM checkout. Check the version up front and let CMAKE name a newer cmake, as yosys_install.sh does; document both, and the C++20 compiler slang needs. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 59e9b9aa7f0d60c2635accdcd8ab0076eea6cb19) --- ci/circt_install.sh.in | 19 ++++++++++++++++--- docs/building_toolchain.md | 7 ++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ci/circt_install.sh.in b/ci/circt_install.sh.in index db58fd1e96..c8c94f6a2b 100755 --- a/ci/circt_install.sh.in +++ b/ci/circt_install.sh.in @@ -25,8 +25,9 @@ # prebuilt and never run this script. # # Release, assertions off, libstdc++ linked statically, no zlib/zstd: the -# tools run on any machine of the OSVERSION. Needs git, cmake >= 3.20, ninja, -# a C++17 compiler and python3; installs nothing system-wide. +# tools run on any machine of the OSVERSION. Needs git, cmake >= 3.28 (for +# slang, the SystemVerilog frontend), ninja, a C++20 compiler (GCC >= 11) and +# python3; installs nothing system-wide. # # Idempotent: re-running rebuilds only what changed. @@ -43,6 +44,18 @@ CIRCT_PREFIX=${CIRCT_PREFIX:=$TOOLDIR/circt} CIRCT_BUILD=$CIRCT_SRC/build JOBS=${JOBS:=$(nproc)} +# slang's CMakeLists sets cmake_minimum_required(3.28), which is NEWER than +# the cmake Ubuntu 22.04 ships (3.22); the failure would come only after the +# LLVM checkout, so check first. CMAKE may point at a newer cmake (e.g. pip's). +CMAKE=${CMAKE:=cmake} +cmake_ver=$($CMAKE --version | head -1 | awk '{print $3}') +cmake_maj=${cmake_ver%%.*}; cmake_rest=${cmake_ver#*.}; cmake_min=${cmake_rest%%.*} +if [ "$cmake_maj" -lt 3 ] || { [ "$cmake_maj" -eq 3 ] && [ "$cmake_min" -lt 28 ]; }; then + echo "error: CIRCT's slang frontend needs cmake >= 3.28, found $cmake_ver ($CMAKE)." + echo " set CMAKE=/path/to/newer/cmake (pip install cmake) and re-run." + exit 1 +fi + # ---------------------------------------------------------------------- # 1. Fetch: LLVM is a submodule pinned by the CIRCT revision. # ---------------------------------------------------------------------- @@ -63,7 +76,7 @@ git -C "$CIRCT_SRC" submodule update --init --depth 1 llvm # ---------------------------------------------------------------------- # 2. Configure and build # ---------------------------------------------------------------------- -cmake -G Ninja -S "$CIRCT_SRC/llvm/llvm" -B "$CIRCT_BUILD" \ +$CMAKE -G Ninja -S "$CIRCT_SRC/llvm/llvm" -B "$CIRCT_BUILD" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX="$CIRCT_PREFIX" \ -DLLVM_ENABLE_PROJECTS=mlir \ diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index 9c1225a61a..fb739cdc99 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -1124,9 +1124,14 @@ the whole of it (tools, libraries, headers, CMake packages) into `$TOOLDIR/circt`: ```bash -./ci/circt_install.sh # CIRCT_REPO, CIRCT_REV, JOBS override the defaults +./ci/circt_install.sh # CIRCT_REPO, CIRCT_REV, JOBS, CMAKE override the defaults ``` +It needs cmake >= 3.28 (slang, CIRCT's SystemVerilog frontend, requires it) +and a C++20 compiler (GCC >= 11). Ubuntu 22.04 ships cmake 3.22: install a +newer one (`pip install cmake`) and point `CMAKE` at it if it is not first on +`PATH`. + It configures with: ```bash From abc1d949ac717a3175509edb535e3f3431dcaaef Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 14 Oct 2026 03:05:45 -0700 Subject: [PATCH 213/336] ci/toolchain_install.sh: circt v3.0.1 prebuilt is one part The ubuntu/focal circt bundle built for v3.0.1 (circt-verilog, circt-opt, firtool) compresses to 51.2 MB, under split's 50 MiB part size, so the prebuilt has only circt.tar.bz2.partaa; fetching partab would 404 and fail the install. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a5143c9ae5f9f40a016bb54cc3a099f71eb53631) --- ci/toolchain_install.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/toolchain_install.sh.in b/ci/toolchain_install.sh.in index 43f3f7308d..f6694d8528 100755 --- a/ci/toolchain_install.sh.in +++ b/ci/toolchain_install.sh.in @@ -169,9 +169,9 @@ circt() { # The count of what ci/toolchain_prebuilt.sh's `split -b 50M` produced for # this release, as for verilator: re-check it whenever the circt prebuilt - # is refreshed. + # is refreshed. v3.0.1's bundle is 51.2 MB, one part. case $OSVERSION in - *) parts=$(eval echo {a..b}) ;; + *) parts=$(eval echo {a..a}) ;; esac echo $parts rm -f circt.tar.bz2.parta* From 30fc402b30d641370405d6db2486bbd45c59800f Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 14 Oct 2026 03:46:10 -0700 Subject: [PATCH 214/336] VERSION: pin FIRESIM_REV to the fork's vortex_3.x head 1ba01ca0 adds, on top of b36112f9: partitioned synthesis of chosen models on the aved platform under a memory budget, and build-setup's conda-lock environment pinned to firesim.yaml's python (a fresh setup failed on python 3.15). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b3ee2744c294940ba5bd47341fe1e97114145883) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8f00b14dfe..db7a93ab72 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=b36112f98f31e04d183e9aaed70fbfbe4c2fe191 +FIRESIM_REV=1ba01ca07ae78f7e2fdac50f70544602c80995d0 From bd3b7d5cdc8452ef479dac307adc2090738cd987 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 14 Oct 2026 03:54:00 -0700 Subject: [PATCH 215/336] docs: install FireSim from the prebuilt bundle Document the faster alternative to clone + build-setup: the v3.0.1 ubuntu/focal bundle in vortex-toolchain-prebuilt, i.e. the fork at FIRESIM_REV with its conda environment conda-packed (22 parts), unpacked and conda-unpacked in place. It ships a shallow .git so check-firesim can verify FIRESIM_REV, and an env.sh that finds its own directory. Verified by unpacking into a fresh directory and elaborating tinygpu for the V80 through ci/firesim_env.sh; sbt fetches its compiler jars on first use. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 793a626f52977e550cb8330d87db6a482aff77d0) --- docs/building_toolchain.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index fb739cdc99..f710ac3952 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -910,6 +910,29 @@ ln -s ~/dev/firesim $TOOLDIR/firesim The checkout must contain the fork commit pinned as `FIRESIM_REV` in `VERSION`; `make -C hw/syn/firesim stage` checks it. +**From the prebuilt (faster).** `vortex-toolchain-prebuilt` carries the same +tree for ubuntu/focal with its conda environment already resolved — the part +of a setup that takes time — conda-packed, plus a shallow `.git` of the +`FIRESIM_REV` commit so `check-firesim` passes and `git fetch` works: + +```bash +REPO=https://github.com/vortexgpgpu/vortex-toolchain-prebuilt/raw/v3.0.1/firesim/ubuntu/focal +for x in a b c d e f g h i j k l m n o p q r s t u v; do + wget $REPO/firesim.tar.bz2.parta$x +done +cat firesim.tar.bz2.parta* | tar -C ~/dev -xjf - && rm firesim.tar.bz2.parta* +cd ~/dev/firesim/.conda-env +tar -xf firesim-conda-env.tar && rm firesim-conda-env.tar +python3 bin/conda-unpack # rewrite the packed prefixes for this location +ln -s ~/dev/firesim $TOOLDIR/firesim +``` + +The v3.0.1 bundle is 22 parts (`partaa`..`partav`). `env.sh` locates its own +directory, so the tree works wherever it is unpacked; `conda` must still be on +`PATH` for `ci/firesim_env.sh`. The first elaboration downloads sbt's own +compiler jars into `~/.cache/coursier` and `~/.sbt`, so it needs network +access once. + `vortex_3.x` is based on upstream tag 1.21.0. Clone it wherever you keep sources; `$TOOLDIR/firesim` only has to resolve to the checkout. From 4cc9857a4af2e91b19377a2fe2e0492f1974c02f Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 14 Oct 2026 18:40:20 -0700 Subject: [PATCH 216/336] docs, ci: write the project's name as Foldsim Where the docs, figures and toolchain scripts name the project in prose (the design doc, the toolchain build guide, simulation and FireSim integration docs, README, AGENTS.md, CHANGELOG, three figures, and the comments and messages of the toolchain install, prebuilt, CIRCT and Foldsim install scripts), write Foldsim. Identifiers keep their case: foldsim_sim, foldsim-opt, --driver=foldsim, FOLDSIM_*, hw/syn/foldsim, paths, and everything in code. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4694ccbc645d6f6429f66e7eb7a6a8d5e8cdf7b5) --- AGENTS.md | 2 +- CHANGELOG.md | 4 +-- README.md | 2 +- ci/circt_install.sh.in | 2 +- ci/foldsim_install.sh.in | 10 +++--- ci/toolchain_install.sh.in | 8 ++--- ci/toolchain_prebuilt.sh.in | 2 +- docs/assets/img/firesim_build_flow.svg | 2 +- docs/assets/img/firesim_fpga_architecture.svg | 2 +- docs/assets/img/foldsim_architecture.svg | 2 +- docs/building_toolchain.md | 16 ++++----- docs/designs/firesim_integration.md | 10 +++--- docs/designs/foldsim.md | 36 +++++++++---------- docs/index.md | 4 +-- docs/simulation.md | 8 ++--- 15 files changed, 55 insertions(+), 55 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6d9357efa2..b3807145aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ This is the canonical entry point for **both human contributors and AI coding ag ### Setup & build - [docs/install_vortex.md](docs/install_vortex.md) — initial install + build setup -- [docs/building_toolchain.md](docs/building_toolchain.md) — building Verilator, RISC-V GNU, LLVM, compiler-rt, musl, POCL, CIRCT, foldsim from source +- [docs/building_toolchain.md](docs/building_toolchain.md) — building Verilator, RISC-V GNU, LLVM, compiler-rt, musl, POCL, CIRCT, Foldsim from source - [docs/environment_setup.md](docs/environment_setup.md) — environment variables and toolchain layout - [docs/fpga_setup.md](docs/fpga_setup.md) — FPGA target setup diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d36694868..1ee45652e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, of at least half the LLC write around it. A launch behind DMA writes waits for an L1-only flush token sent on the slot. The runtime sees the new `DMA_LLC_CLIENT` CP capability. *Why:* rtlsim at 2 clusters + L2 + L3: dogfood -30%, multikernel -42.7%; dogfood -64% with an 8 MB L3. -- **foldsim: threaded FPGA simulation of Vortex on the Alveo V80, built on CIRCT.** Every class of identical units (sockets, issue slices, ALUs, FPUs, load-store units) is time-multiplexed through one physical copy, cycle-exact against the RTL, with no Yosys or sv2v: `circt-verilog` imports Vortex, foldsim's `foldsim-extract-models` / `foldsim-compose` cut it into threaded models joined by one token fabric (foldsim is its own project on CIRCT, installed by `ci/toolchain_install.sh --foldsim`; [hw/syn/foldsim/](hw/syn/foldsim/)), and the V80's native flow packages it with a latency-bandwidth memory endpoint and a command-queue control block ([hw/rtl/afu/foldsim/](hw/rtl/afu/foldsim/)). `--driver=foldsim` runs it on the card (`TARGET=hw`, VRT + QDMA) or in-process in Verilator (`avedsim`, the default). Vortex's RTL is unchanged: units get plain-port boundaries and instance IDs as ports from a source-to-source pass on the flow's copy ([hw/syn/foldsim/lift_boundaries.py](hw/syn/foldsim/lift_boundaries.py)). See [docs/designs/foldsim.md](docs/designs/foldsim.md). +- **Foldsim: threaded FPGA simulation of Vortex on the Alveo V80, built on CIRCT.** Every class of identical units (sockets, issue slices, ALUs, FPUs, load-store units) is time-multiplexed through one physical copy, cycle-exact against the RTL, with no Yosys or sv2v: `circt-verilog` imports Vortex, Foldsim's `foldsim-extract-models` / `foldsim-compose` cut it into threaded models joined by one token fabric (Foldsim is its own project on CIRCT, installed by `ci/toolchain_install.sh --foldsim`; [hw/syn/foldsim/](hw/syn/foldsim/)), and the V80's native flow packages it with a latency-bandwidth memory endpoint and a command-queue control block ([hw/rtl/afu/foldsim/](hw/rtl/afu/foldsim/)). `--driver=foldsim` runs it on the card (`TARGET=hw`, VRT + QDMA) or in-process in Verilator (`avedsim`, the default). Vortex's RTL is unchanged: units get plain-port boundaries and instance IDs as ports from a source-to-source pass on the flow's copy ([hw/syn/foldsim/lift_boundaries.py](hw/syn/foldsim/lift_boundaries.py)). See [docs/designs/foldsim.md](docs/designs/foldsim.md). *Why:* the simulated GPU is no longer bounded by the card's size, and the shell around it is ~8k LUTs instead of FireSim's ~93k — s2w4i2 (2 cores, NW=NT=4, IW=2) routes in 138k LUTs vs 219k and runs sgemm n=128 in 196 ms on V80: 50× SimX on the same host, 1.7× the FireSim image, cycle counts identical to Verilator. *Limits:* V80 only; one memory port, single-beat transactions; classes are named (`FOLDSIM_CLASSES`), not found; kernel register offsets 0x40–0xBF are unusable on the V80 shell (its PCIe MSI-X table answers them). @@ -90,7 +90,7 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, value exactly. - **FPGA run instructions.** The docs exported `TARGET=hw`, which `blackbox.sh` ignores, so runs fell back to xrtsim; they now pass `--target=hw`. -- **Block RAM for asynchronous-read RAMs is a CIRCT pass, not a Vivado netlist patch; every synthesis flow reads one CIRCT-elaborated netlist.** A RAM that must be block RAM but reads asynchronously is a plain array tagged `(* ram_style = "block" *)`; CIRCT's `seq-retime-firmem-reads` makes its read synchronous, reading at the address its registers are about to take, and stops the build with the reason if it cannot. `hw/scripts/circt_elaborate.sh` (`gen_sources.sh -E`) elaborates the design for Vivado (xrt, aved, dut, sandbox), FireSim, foldsim and Yosys alike, and writes a report of every asynchronous-read RAM. `ASYNC_BRAM_PATCH`, `VX_async_ram_patch`, `VX_placeholder`, `xilinx_async_bram_patch.tcl` and its pre-opt hooks are gone; Xilinx floating-point cores are declared to the elaboration in `hw/scripts/xilinx_ip_stubs.sv`. +- **Block RAM for asynchronous-read RAMs is a CIRCT pass, not a Vivado netlist patch; every synthesis flow reads one CIRCT-elaborated netlist.** A RAM that must be block RAM but reads asynchronously is a plain array tagged `(* ram_style = "block" *)`; CIRCT's `seq-retime-firmem-reads` makes its read synchronous, reading at the address its registers are about to take, and stops the build with the reason if it cannot. `hw/scripts/circt_elaborate.sh` (`gen_sources.sh -E`) elaborates the design for Vivado (xrt, aved, dut, sandbox), FireSim, Foldsim and Yosys alike, and writes a report of every asynchronous-read RAM. `ASYNC_BRAM_PATCH`, `VX_async_ram_patch`, `VX_placeholder`, `xilinx_async_bram_patch.tcl` and its pre-opt hooks are gone; Xilinx floating-point cores are declared to the elaboration in `hw/scripts/xilinx_ip_stubs.sv`. *Why:* the patch rewired Vivado's synthesized netlist in Tcl, ran in Vivado flows only, failed late on large builds (`DRC INBB-3`), and produced a netlist nothing simulated. On the core at 16 warps and 8 threads (V80, 300 MHz met both ways) the CIRCT netlist implements in 90,481 logic LUTs against 92,608, with the same BRAM, LUTRAM and DSPs. *Limits:* generate-block instances are named `g_x_0_inst`, not `g_x[0].inst`, in the netlist (`xilinx_slr_pblocks.tcl` accepts both); the modules between a converted RAM and its address registers are inlined; the synthesis baselines move with the netlist and are regenerated by hand. - **CIRCT is the SystemVerilog frontend of every synthesis flow; sv2v and Verilator's preprocessor are gone.** `gen_sources.sh -P` preprocesses with `circt-verilog -E` (identical code to `verilator -E` over the whole RTL tree; only comments differ), and the Yosys flow elaborates with CIRCT into one Verilog-2005 file ([hw/scripts/sv_to_v.sh](hw/scripts/sv_to_v.sh)) instead of sv2v: packed arrays and structs become vectors, and the SRAM macros (`BB_MODULES`) stay instances with their parameter values, read in SystemVerilog mode beside their stubs. CIRCT installs under `$TOOLDIR/circt` (`toolchain_install.sh --circt`; `hw/syn/common.mk` exports `CIRCT_VERILOG`, `CIRCT_OPT`, `FIRTOOL`), and the `asic_gate` fingerprints its baselines on the CIRCT version. Verilator remains the simulator (and `--xml-only` scope metadata). On the cache DUT (ASAP7), SRAM area is identical and logic area is 5.6% higher than through sv2v: CIRCT folds synchronous resets into the next-value logic and lowers array reads to mux trees, both open items. diff --git a/README.md b/README.md index 8faa639971..8f29d9522a 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The following dependencies will be fetched prebuilt by `toolchain_install.sh`. - [Ramulator](https://github.com/CMU-SAFARI/ramulator.git) - [Yosys](https://github.com/YosysHQ/yosys) - [CIRCT](https://github.com/llvm/circt) (Vortex's fork) -- foldsim: FAME-5 FPGA simulators of Vortex, built on CIRCT +- Foldsim: FAME-5 FPGA simulators of Vortex, built on CIRCT - [Mesa](https://www.mesa3d.org/) - [chipStar](https://github.com/CHIP-Star/chipStar) ### Install Vortex codebase diff --git a/ci/circt_install.sh.in b/ci/circt_install.sh.in index c8c94f6a2b..cfa17f83c0 100755 --- a/ci/circt_install.sh.in +++ b/ci/circt_install.sh.in @@ -14,7 +14,7 @@ # limitations under the License. # # CIRCT install: the SystemVerilog frontend of every synthesis flow -# (circt-verilog, circt-opt, firtool) and the compiler foldsim builds on. +# (circt-verilog, circt-opt, firtool) and the compiler Foldsim builds on. # # Builds Vortex's CIRCT fork (LLVM, MLIR and CIRCT in one build) and installs # it into $TOOLDIR/circt: tools, libraries, headers and CMake packages, which diff --git a/ci/foldsim_install.sh.in b/ci/foldsim_install.sh.in index 5ce95ff397..65ff94e84c 100755 --- a/ci/foldsim_install.sh.in +++ b/ci/foldsim_install.sh.in @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# foldsim install: the FAME-5 simulator compiler of hw/syn/foldsim (the +# Foldsim install: the FAME-5 simulator compiler of hw/syn/foldsim (the # `foldsim` driver and foldsim-opt). # -# foldsim is its own project and builds on CIRCT, as POCL builds on LLVM: it +# Foldsim is its own project and builds on CIRCT, as POCL builds on LLVM: it # needs the CIRCT installation ci/circt_install.sh leaves in $TOOLDIR/circt # (libraries, headers and CMake packages, not only the prebuilt's tools). This -# script is the *producer* of the foldsim prebuilt: build here (once per +# script is the *producer* of the Foldsim prebuilt: build here (once per # OSVERSION, in that OS's container, after ci/circt_install.sh), then # `ci/toolchain_prebuilt.sh --foldsim` packages $TOOLDIR/foldsim and # `ci/toolchain_install.sh --foldsim` fetches it. Most users install the @@ -72,7 +72,7 @@ fi # `lit`) unless LIT names one. LIT=${LIT:-$(command -v lit || command -v llvm-lit || true)} if [ -z "$LIT" ]; then - echo "error: no lit for foldsim's tests: pip install lit, or set LIT." + echo "error: no lit for Foldsim's tests: pip install lit, or set LIT." exit 1 fi cmake -G Ninja -S "$FOLDSIM_SRC" -B "$FOLDSIM_BUILD" \ @@ -87,4 +87,4 @@ ninja -C "$FOLDSIM_BUILD" ninja -C "$FOLDSIM_BUILD" check-foldsim cmake --install "$FOLDSIM_BUILD" -echo "foldsim installed in $FOLDSIM_PREFIX" +echo "Foldsim installed in $FOLDSIM_PREFIX" diff --git a/ci/toolchain_install.sh.in b/ci/toolchain_install.sh.in index f6694d8528..e2df33253c 100755 --- a/ci/toolchain_install.sh.in +++ b/ci/toolchain_install.sh.in @@ -184,7 +184,7 @@ circt() mkdir -p $TOOLDIR && rm -rf $TOOLDIR/circt && mv circt $TOOLDIR rm -rf circt circt.tar.bz2* } -# foldsim: the FAME-5 simulator compiler of hw/syn/foldsim (the foldsim +# Foldsim: the FAME-5 simulator compiler of hw/syn/foldsim (the Foldsim # driver and foldsim-opt), built on CIRCT. foldsim() { @@ -242,7 +242,7 @@ gem5() rm -rf gem5 gem5.tar.bz2 } -# Every component but the optional ones, foldsim and gem5 — the default when +# Every component but the optional ones, Foldsim and gem5 — the default when # the script is run with no arguments, and the shared body of --all. install_base() { @@ -269,8 +269,8 @@ show_usage() { echo "Install Pre-built Vortex Toolchain" echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--foldsim] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" - echo " (no arguments: install all components except foldsim and gem5)" - echo " foldsim and gem5 are opt-in: foldsim serves only the foldsim flow, and" + echo " (no arguments: install all components except Foldsim and gem5)" + echo " Foldsim and gem5 are opt-in: Foldsim serves only the Foldsim flow, and" echo " gem5 is a large source build. --all installs them too." } diff --git a/ci/toolchain_prebuilt.sh.in b/ci/toolchain_prebuilt.sh.in index 354b4caa1c..da17faefe2 100755 --- a/ci/toolchain_prebuilt.sh.in +++ b/ci/toolchain_prebuilt.sh.in @@ -134,7 +134,7 @@ circt() rm circt.tar.bz2 } -# foldsim's driver and foldsim-opt (ci/foldsim_install.sh), one tarball. +# Foldsim's driver and foldsim-opt (ci/foldsim_install.sh), one tarball. foldsim() { echo "prebuilt foldsim..." diff --git a/docs/assets/img/firesim_build_flow.svg b/docs/assets/img/firesim_build_flow.svg index 3654f74b50..4c2b845074 100644 --- a/docs/assets/img/firesim_build_flow.svg +++ b/docs/assets/img/firesim_build_flow.svg @@ -24,7 +24,7 @@ Build flow — how Vortex RTL becomes a FireSim image Vortex stays Verilog: Golden Gate never sees inside it. Runs from hw/syn/firesim/Makefile with the native flows' tokens. - Threaded images (FAME-5) are the foldsim flow, compiled by CIRCT: see foldsim.md. + Threaded images (FAME-5) are the Foldsim flow, compiled by CIRCT: see foldsim.md. diff --git a/docs/assets/img/firesim_fpga_architecture.svg b/docs/assets/img/firesim_fpga_architecture.svg index 7368968d90..d6bb715603 100644 --- a/docs/assets/img/firesim_fpga_architecture.svg +++ b/docs/assets/img/firesim_fpga_architecture.svg @@ -140,7 +140,7 @@ g_sockets — N × VX_socket_top Each socket: 1 core + I$/D$/LMEM, socket_id as a port. One physical copy per socket. Threading the sockets - through one copy (FAME-5) is the foldsim flow: foldsim.md. + through one copy (FAME-5) is the Foldsim flow: foldsim.md. diff --git a/docs/assets/img/foldsim_architecture.svg b/docs/assets/img/foldsim_architecture.svg index c315a0d9ae..649c1349c2 100644 --- a/docs/assets/img/foldsim_architecture.svg +++ b/docs/assets/img/foldsim_architecture.svg @@ -19,7 +19,7 @@ - Compile: one IR from SystemVerilog to the kernel (the foldsim driver, on CIRCT) + Compile: one IR from SystemVerilog to the kernel (the Foldsim driver, on CIRCT) circt-verilog Vortex, flat but for diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index f710ac3952..63b5069b75 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -26,7 +26,7 @@ The components covered here are: 12. [FireSim](#12-firesim) — FPGA-accelerated host-decoupled simulation on the Alveo U55C or V80; needed only to elaborate a target or build a bitstream (optional) 13. [SLASH / Alveo V80](#13-slash--alveo-v80-the-aved-backend) — AMD Alveo V80 platform stack for the `aved` backend (optional) 14. [CIRCT](#14-circt) — the SystemVerilog frontend of every synthesis flow (`circt-verilog`, `circt-opt`, `firtool`) -15. [foldsim](#15-foldsim) — FAME-5 simulator compiler of `hw/syn/foldsim`, built on CIRCT (optional) +15. [Foldsim](#15-foldsim) — FAME-5 simulator compiler of `hw/syn/foldsim`, built on CIRCT (optional) --- @@ -1133,7 +1133,7 @@ for the backend's architecture and **Purpose**: the SystemVerilog frontend of every synthesis flow. Each flow reads the netlist CIRCT elaborates (`hw/scripts/circt_elaborate.sh`, `gen_sources.sh -E`; see the hardware library's "Block RAM and asynchronous -reads"), and foldsim builds on it. +reads"), and Foldsim builds on it. > **Baseline:** Vortex's CIRCT fork, branch `vortex_3.x` of > `vortexgpgpu/circt`: upstream CIRCT with the import, lowering and @@ -1174,18 +1174,18 @@ cmake -G Ninja -S $CIRCT_SRC/llvm/llvm -B $CIRCT_SRC/build \ (`ubuntu/focal`, `centos/7`): assertions off and libstdc++ linked statically, so the tools run on any machine of that OS. - The prebuilt ships only `circt-verilog`, `circt-opt` and `firtool`. Building - foldsim needs the full installation this script leaves behind. + Foldsim needs the full installation this script leaves behind. - To develop CIRCT itself, build it in its own checkout with assertions on and point the flows at it: `CIRCT_PATH=` (the build directory has the same `bin/` layout). --- -## 15. foldsim +## 15. Foldsim **Purpose**: the compiler of the FAME-5 FPGA simulators (`hw/syn/foldsim`, [designs/foldsim.md](designs/foldsim.md)): the `foldsim` driver and -`foldsim-opt`. foldsim is its own project and builds on CIRCT, as POCL builds +`foldsim-opt`. Foldsim is its own project and builds on CIRCT, as POCL builds on LLVM. > **Baseline:** branch `vortex_3.x` of `vortexgpgpu/foldsim`. @@ -1198,7 +1198,7 @@ Build CIRCT first (§14), then: ./ci/foldsim_install.sh # FOLDSIM_REPO, FOLDSIM_REV, CIRCT_PREFIX override the defaults ``` -which configures foldsim against `$TOOLDIR/circt`, builds it, runs its lit +which configures Foldsim against `$TOOLDIR/circt`, builds it, runs its lit tests (`check-foldsim`) and installs `bin/foldsim` and `bin/foldsim-opt` into `$TOOLDIR/foldsim`: @@ -1217,10 +1217,10 @@ cmake -G Ninja -S $FOLDSIM_SRC -B $FOLDSIM_SRC/build \ - `foldsim-opt` links CIRCT statically; the driver finds it next to itself and the CIRCT tools through `--circt-bin`. `hw/syn/common.mk` sets `FOLDSIM_PATH ?= $(TOOLDIR)/foldsim` and `FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim`. -- A CIRCT change foldsim's passes rely on lands in CIRCT first; both +- A CIRCT change Foldsim's passes rely on lands in CIRCT first; both prebuilts are refreshed together. - The prebuilt is opt-in: `./ci/toolchain_install.sh --foldsim` (or `--all`) - installs it; the default install leaves it out, since only the foldsim + installs it; the default install leaves it out, since only the Foldsim flow needs it. --- diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index fec9b9d66f..4935db1a97 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -8,7 +8,7 @@ Alveo U55C (Vitis/XRT) and the Alveo V80 (AVED/VRT). |---|---| | [1](#1-system-architecture) | System architecture: host stack, FPGA payload, execution model | | [2](#2-synthesis-of-the-vortex-rtl) | Synthesis of the Vortex RTL | -| [3](#3-threaded-simulation) | Threaded simulation: foldsim | +| [3](#3-threaded-simulation) | Threaded simulation: Foldsim | | [4](#4-the-simulation-timing-model) | The simulation timing model | | [5](#5-the-memory-system) | The memory system and its configuration; shell configuration | | [6](#6-observability) | Observability: wave traces, RTL messages and assertions, counters | @@ -97,7 +97,7 @@ inside the kernel on aved: the conversion cannot enter it, so no unit takes one there. The FPU is the STD one (`VX_CFG_FPU_TYPE_STD`), its FMA, divider and square root take their soft path in the flow's copy of the sources, and the RTU uses the soft FMA/divider — `VX_CFG_RTU_USE_DSP=0`. The metasimulator -and the image therefore run the same FPU, as the foldsim flow's do.) +and the image therefore run the same FPU, as the Foldsim flow's do.) ### 1.3 Host-decoupled execution @@ -209,7 +209,7 @@ slashkit on aved), handed: This backend simulates Vortex unthreaded: one physical copy of every module. Threading a GPU's identical units through one physical copy each (FAME-5) is -the foldsim flow, which compiles Vortex with CIRCT rather than Golden Gate and +the Foldsim flow, which compiles Vortex with CIRCT rather than Golden Gate and has its own kernel, driver and image: see [foldsim.md](foldsim.md). --- @@ -539,10 +539,10 @@ FFs, 5 BRAM tiles, 8 URAM, 8 DSP. The metasimulator and the image agree cycle for cycle. Instruction counts match rtlsim's. Cycles are 5.0 % above rtlsim's on the same latency-bandwidth -settings (`AXI_BUS=1`; `demo`, `sgemm -n16`), and 5.1 % above the foldsim +settings (`AXI_BUS=1`; `demo`, `sgemm -n16`), and 5.1 % above the Foldsim image's on `sgemm -n64`, from the memory model's channel mechanics. -Threaded images, and the same tinygpu on the foldsim flow, are in +Threaded images, and the same tinygpu on the Foldsim flow, are in [foldsim.md](foldsim.md) §7. --- diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 2549783982..d97ed4434d 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -1,6 +1,6 @@ -# foldsim — threaded FPGA simulation of Vortex, on CIRCT +# Foldsim — threaded FPGA simulation of Vortex, on CIRCT -foldsim runs Vortex on the Alveo V80 as a host-decoupled, cycle-exact +Foldsim runs Vortex on the Alveo V80 as a host-decoupled, cycle-exact simulator in which every class of identical units — sockets, issue slices, ALUs, FPUs, load-store units — is time-multiplexed through **one physical copy** (FAME-5 model multithreading). A GPU many times the card's size fits @@ -21,12 +21,12 @@ on it, and runs orders of magnitude faster than Verilator. ## 1. Overview -![foldsim architecture](../assets/img/foldsim_architecture.svg) +![Foldsim architecture](../assets/img/foldsim_architecture.svg) The whole flow is one IR from SystemVerilog to Verilog: `circt-verilog` -imports Vortex, foldsim's passes cut it into models and compose the +imports Vortex, Foldsim's passes cut it into models and compose the simulator, `firtool` writes it out, and the card's native flow (IP-XACT -packaging, `slashkit link`) builds the image. foldsim is a project of its own +packaging, `slashkit link`) builds the image. Foldsim is a project of its own that builds on CIRCT, as POCL builds on LLVM: the `foldsim` driver and `foldsim-opt` come from the toolchain (`$(TOOLDIR)/foldsim`), CIRCT from `$(TOOLDIR)/circt`. Nothing else is in the flow: no Yosys, no sv2v. @@ -245,7 +245,7 @@ advancing for that time (60 s when it is unset). - **Applications** in the in-process simulator and on the V80, against rtlsim behind the AXI4 adapter on `AxiLatencyPipe` (`AXI_BUS=1`) with the same memory settings: retired instructions and cycles identical (§7). -- **foldsim's own tests** (`check-foldsim` in the foldsim repository): lit +- **Foldsim's own tests** (`check-foldsim` in the Foldsim repository): lit tests of every pass and of the driver, and small threaded simulators run in Verilator against their expected tokens — one and several threads, registered memory reads, register initial values, eager outputs, a reset of @@ -261,10 +261,10 @@ classes (4 threads each). | | where | demo cycles | sgemm cycles | sgemm n=128 cycles | host / target cycle | |---|---|---:|---:|---:|---:| -| foldsim, threads in turn, in-process | Verilator | 8,702 | 415,454 | | 12.06 | -| foldsim, ready scheduling, in-process | Verilator | 8,702 | 415,454 | | 8.00 | -| foldsim, threads in turn | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 or 12.00 | -| foldsim, ready scheduling | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 | +| Foldsim, threads in turn, in-process | Verilator | 8,702 | 415,454 | | 12.06 | +| Foldsim, ready scheduling, in-process | Verilator | 8,702 | 415,454 | | 8.00 | +| Foldsim, threads in turn | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 or 12.00 | +| Foldsim, ready scheduling | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 | | rtlsim, AXI4 with `AxiLatencyPipe` | Verilator | 8,710 | 415,462 | | | | rtlsim, same, launched after the caches' reset sweep | Verilator | 8,702 | 415,454 | | | @@ -276,21 +276,21 @@ sgemm n=128 takes 196 ms with ready scheduling: 50× SimX on the same host, Instruction counts equal the firesim backend's and rtlsim's. Cycle counts are 1.5–6 % below the firesim backend's, whose memory model has the same latency-bandwidth settings but adds channel cycles `AxiLatencyPipe` does not; -foldsim agrees with `AxiLatencyPipe` (rtlsim) instead: every memory +Foldsim agrees with `AxiLatencyPipe` (rtlsim) instead: every memory transaction in the same target cycle. rtlsim's harness starts the kernel 8 cycles after reset, while each L1 bank still invalidates one set per cycle (64 for the instruction cache), so its first fetch waits 8 cycles the counters see; a host's launch comes thousands of cycles later. Given that -settle time, rtlsim's cycles are foldsim's exactly. +settle time, rtlsim's cycles are Foldsim's exactly. -### 7.1 tinygpu: foldsim and the firesim backend +### 7.1 tinygpu: Foldsim and the firesim backend The same Vortex tree, configuration and V80 on both flows: 1 core, 2 warps, 2 threads, no L1 caches or local memory, STD FPU, one memory bank, kernel -clock 143 MHz. Every class has one instance, so foldsim folds nothing here: +clock 143 MHz. Every class has one instance, so Foldsim folds nothing here: this compares the two simulators' fidelity and overheads, not threading. -| | foldsim | firesim backend | +| | Foldsim | firesim backend | |---|---:|---:| | kernel LUTs (synthesis) | 34,393 | 103,369 | | of them distributed RAM | 1,822 | 51,943 | @@ -304,12 +304,12 @@ this compares the two simulators' fidelity and overheads, not threading. | host cycles per target cycle, `sgemm -n64` | 3.00 | 1.84 (FMR) | Instruction counts are identical on both flows, in Verilator and on the V80. -foldsim's cycles equal rtlsim's behind the AXI4 adapter on `AxiLatencyPipe` +Foldsim's cycles equal rtlsim's behind the AXI4 adapter on `AxiLatencyPipe` with the same memory settings (`AXI_BUS=1`) exactly; the firesim backend's are 5.0–5.1 % above, from its memory model's channel mechanics. With nothing -to fold, a foldsim target cycle costs three host cycles (the token exchange +to fold, a Foldsim target cycle costs three host cycles (the token exchange of each one-thread model), so the firesim backend runs tinygpu faster; -foldsim's advantage is the GPU it fits, its area falling with every class it +Foldsim's advantage is the GPU it fits, its area falling with every class it folds (s2w4i2 above). --- diff --git a/docs/index.md b/docs/index.md index 1bcb52a899..9e4b858360 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ - [Environment Setup](environment_setup.md): system prerequisites and toolchain setup - [Installing Vortex](install_vortex.md): building and installing the Vortex stack -- [Building the Toolchain from Source](building_toolchain.md): maintainer-facing build recipes for Verilator, RISC-V GNU, LLVM (with X86 + lld + SPIR-V), compiler-rt, musl, POCL, Mesa, chipStar, gem5, CIRCT, and foldsim +- [Building the Toolchain from Source](building_toolchain.md): maintainer-facing build recipes for Verilator, RISC-V GNU, LLVM (with X86 + lld + SPIR-V), compiler-rt, musl, POCL, Mesa, chipStar, gem5, CIRCT, and Foldsim - [FPGA Setup](fpga_setup.md): running Vortex on Altera and Xilinx FPGA cards - [SLASH / Alveo V80 Setup](xilinx_slash_setup.md): building the SLASH platform stack, the prerequisite for the `aved` backend @@ -75,5 +75,5 @@ Detailed architectural specifications for Vortex subsystems live under - [SimX Simulator Architecture](designs/simx_simulator_architecture.md) - [gem5 Integration](designs/vortex_gem5_integration.md) - [FireSim Integration](designs/firesim_integration.md) - - [foldsim: threaded FPGA simulation on CIRCT](designs/foldsim.md) + - [Foldsim: threaded FPGA simulation on CIRCT](designs/foldsim.md) - [Continuous Integration Design](designs/continuous_integration.md) diff --git a/docs/simulation.md b/docs/simulation.md index c99f29b1dc..260d4fe3f3 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -83,9 +83,9 @@ One caveat worth knowing when reading its output: FireSim's kernel took, and is quantized by the driver's polling schedule. Use Vortex's own `PERF: cycles=` line, which is exact. -### Threaded FPGA Simulation (foldsim) +### Threaded FPGA Simulation (Foldsim) -foldsim compiles Vortex into a simulator for the V80 with CIRCT: each class of +Foldsim compiles Vortex into a simulator for the V80 with CIRCT: each class of identical units (sockets, issue slices, ALUs, ...) is threaded through one physical copy, so a GPU far larger than the card fits on it, still cycle-exact against the RTL. Its kernel carries a small control block and a @@ -94,7 +94,7 @@ it like the native V80 driver does (AXI4-Lite registers, QDMA for memory). | target | what runs | built by | |---|---|---| -| `avedsim` (default) | the same kernel in Verilator, in-process | the run itself: the foldsim compiler, then a Verilator build of the kernel, for the app's `CONFIGS` | +| `avedsim` (default) | the same kernel in Verilator, in-process | the run itself: the Foldsim compiler, then a Verilator build of the kernel, for the app's `CONFIGS` | | `hw` | the image on the V80 | `make -C hw/syn/foldsim` | ```bash @@ -112,7 +112,7 @@ FPGA_BIN_DIR=$PWD/hw/syn/foldsim/build32_foldsim_hw/bin \ CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=foldsim --target=hw --app=sgemm ``` -The flow needs CIRCT (`circt-verilog`, `circt-opt`, `firtool`) and foldsim +The flow needs CIRCT (`circt-verilog`, `circt-opt`, `firtool`) and Foldsim (the `foldsim` driver and `foldsim-opt`) from the toolchain (`./ci/toolchain_install.sh --circt --foldsim`; `CIRCT_BIN=` and `FOLDSIM=` override them), and pyslang for the boundary lifting From fd20a94d7e29682da7a4036988a014444ddc1991 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 14 Oct 2026 18:40:51 -0700 Subject: [PATCH 217/336] README: drop Foldsim from the toolchain dependencies Foldsim is an opt-in component of the Foldsim flow only (toolchain_install.sh --foldsim), not a dependency of a Vortex build. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e0274e1c31e6ace93f7824a5c11d3f0d29624fc1) --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8f29d9522a..f110ebad2b 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,6 @@ The following dependencies will be fetched prebuilt by `toolchain_install.sh`. - [Ramulator](https://github.com/CMU-SAFARI/ramulator.git) - [Yosys](https://github.com/YosysHQ/yosys) - [CIRCT](https://github.com/llvm/circt) (Vortex's fork) -- Foldsim: FAME-5 FPGA simulators of Vortex, built on CIRCT - [Mesa](https://www.mesa3d.org/) - [chipStar](https://github.com/CHIP-Star/chipStar) ### Install Vortex codebase From 1f90c062f479d20a1a11f35618ac529e6d2c07fe Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 03:12:06 -0700 Subject: [PATCH 218/336] foldsim: time the image at 250 MHz, its kernel synthesis against its clock FOLDSIM_FREQ defaults to 250. The kernel IP carries an out-of-context XDC with its clock at FOLDSIM_FREQ (package_ip.tcl's ooc-xdc= option), and a pre-synth hook turns on global retiming for its synthesis run and, as the native flow's hook does, post-route phys_opt_design. FOLDSIM_FLAGS passes options to the foldsim driver (e.g. --region-loads, --registered-state) and regenerates the simulator when they change. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8ef1e49c45fbd665a5e7895836e6f282714ae636) --- hw/syn/foldsim/Makefile | 23 +++++++++++++++++------ hw/syn/foldsim/pre_synth_hook.tcl | 13 +++++++++++++ hw/syn/xilinx/aved/package_ip.tcl | 22 ++++++++++++++++++---- sim/foldsim/Makefile | 1 + 4 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 hw/syn/foldsim/pre_synth_hook.tcl diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index c608f37151..2ca729a549 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -52,7 +52,9 @@ FOLDSIM_LIFT ?= VX_socket:drop=INSTANCE_ID;port=SOCKET_ID CIRCT_BIN ?= $(CIRCT_PATH)/bin LIFT_PYTHON ?= python3 # The kernel clock (MHz): the image is timed at it and the shell programs it. -FOLDSIM_FREQ ?= 143 +FOLDSIM_FREQ ?= 250 +# Options of the foldsim driver, e.g. --region-loads=2048 --registered-state. +FOLDSIM_FLAGS ?= # The target: Vortex with its AXI4 port, DCR, start and busy as plain ports. TARGET_TOP := foldsim_target @@ -91,7 +93,7 @@ endef CONFIG_STAMP := $(BUILD_DIR)/cflags.stamp CLASSES_STAMP := $(BUILD_DIR)/classes.stamp $(call stamp,$(CONFIG_STAMP),$(GEN_CFLAGS)) -$(call stamp,$(CLASSES_STAMP),$(FOLDSIM_CLASSES) | $(FOLDSIM_LIFT)) +$(call stamp,$(CLASSES_STAMP),$(FOLDSIM_CLASSES) | $(FOLDSIM_LIFT)$(if $(FOLDSIM_FLAGS), | $(FOLDSIM_FLAGS))) SRCS_NAME := vortex_srcs SRCS_DIR := $(BUILD_DIR)/$(SRCS_NAME) @@ -155,7 +157,7 @@ FOLDSIM_TOOLS := $(FOLDSIM) $(wildcard $(dir $(FOLDSIM))foldsim-opt \ $(addprefix $(CIRCT_BIN)/,circt-verilog circt-opt firtool)) $(GEN_SV): $(TARGET_SRCS) $(FOLDSIM_TOOLS) $(CLASSES_STAMP) $(FOLDSIM) --srcs $(TARGET_SRCS) --top $(TARGET_TOP) --out $(GEN_DIR) \ - $(if $(CIRCT_BIN),--circt-bin $(CIRCT_BIN)) $(FOLDSIM_CLASSES) + $(if $(CIRCT_BIN),--circt-bin $(CIRCT_BIN)) $(FOLDSIM_FLAGS) $(FOLDSIM_CLASSES) $(KERNEL_SRCS): $(GEN_SV) @{ echo "+incdir+$(FOLDSIM_RTL_DIR)"; echo "+incdir+$(GEN_DIR)"; \ @@ -163,11 +165,19 @@ $(KERNEL_SRCS): $(GEN_SV) for f in foldsim_token.sv foldsim_mem.sv foldsim_ctrl.sv foldsim_afu.sv; do echo "$(FOLDSIM_RTL_DIR)/$$f"; done; \ } > $@ +# The kernel's clock at FOLDSIM_FREQ, for its out-of-context synthesis. +OOC_XDC := $(BUILD_DIR)/foldsim_afu_ooc.xdc +$(OOC_XDC): FORCE + @mkdir -p $(@D) + @printf 'create_clock -name ap_clk -period %s [get_ports ap_clk]\n' \ + $$(python3 -c 'print(round(1000 / $(FOLDSIM_FREQ), 3))') > $@.tmp + @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ + package-ip: $(IP_DIR)/component.xml -$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(wildcard $(FOLDSIM_RTL_DIR)/*) +$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(OOC_XDC) $(wildcard $(FOLDSIM_RTL_DIR)/*) cd $(BUILD_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -nojournal -nolog \ -source $(VORTEX_HOME)/hw/syn/xilinx/aved/package_ip.tcl \ - -tclargs foldsim_afu $(KERNEL_SRCS) . $(DEVICE) no-vendor-ip + -tclargs foldsim_afu $(KERNEL_SRCS) . $(DEVICE) no-vendor-ip ooc-xdc=$(OOC_XDC) $(CFG): FORCE @python3 $(SCRIPT_DIR)/gen_platform.py --flow=aved --device=$(DEVICE) --emit=cfg --kernel=foldsim_afu \ @@ -179,7 +189,8 @@ FORCE: $(VBIN): $(IP_DIR)/component.xml $(CFG) mkdir -p $(BIN_DIR) TOOL_DIR=$(SCRIPT_DIR) $(SLASHKIT) link -c $(CFG) -p hw -o $@ -k $(IP_DIR)/component.xml \ - --vivado $(VIVADO) $(if $(SLASHKIT_JOBS),--jobs $(SLASHKIT_JOBS)) + --vivado $(VIVADO) $(if $(SLASHKIT_JOBS),--jobs $(SLASHKIT_JOBS)) \ + --pre-synth-tcls $(SRC_DIR)/pre_synth_hook.tcl @echo "image: $@" clean: diff --git a/hw/syn/foldsim/pre_synth_hook.tcl b/hw/syn/foldsim/pre_synth_hook.tcl new file mode 100644 index 0000000000..957a75ba28 --- /dev/null +++ b/hw/syn/foldsim/pre_synth_hook.tcl @@ -0,0 +1,13 @@ +# Sourced by slashkit via --pre-synth-tcls, after the linker has created the +# project and its runs but before it launches them. + +# The native flow's hook: the pre-opt utilization report and post-route +# phys_opt_design -directive AggressiveExplore. +source [file join [file dirname [file normalize [info script]]] .. xilinx aved pre_synth_hook.tcl] + +# The kernel's out-of-context synthesis sees its clock (the IP's OOC XDC) and +# retimes against it: a threaded model's registered state is retimable. +foreach run [get_runs -quiet -filter {IS_SYNTHESIS} *foldsim_afu*] { + set_property STEPS.SYNTH_DESIGN.ARGS.GLOBAL_RETIMING on $run + puts "INFO: global retiming on for $run" +} diff --git a/hw/syn/xilinx/aved/package_ip.tcl b/hw/syn/xilinx/aved/package_ip.tcl index be4b3ec5ca..5c1749a974 100644 --- a/hw/syn/xilinx/aved/package_ip.tcl +++ b/hw/syn/xilinx/aved/package_ip.tcl @@ -14,9 +14,9 @@ # Package the AFU as an IP-XACT component. The V80 linker consumes the # component.xml directly, so there is no intermediate object container. -if { $::argc != 4 && $::argc != 5 } { - puts "ERROR: Program \"$::argv0\" requires 4 or 5 arguments!\n" - puts "Usage: $::argv0 \[no-vendor-ip\]\n" +if { $::argc < 4 || $::argc > 6 } { + puts "ERROR: Program \"$::argv0\" requires 4 to 6 arguments!\n" + puts "Usage: $::argv0 \[no-vendor-ip\] \[ooc-xdc=\]\n" exit 1 } @@ -26,7 +26,16 @@ set build_dir [lindex $::argv 2] set part [lindex $::argv 3] # A kernel whose RTL instantiates no vendor floating-point core (a foldsim # simulator, whose FPU is soft logic) is packaged without them. -set vendor_ip [expr {$::argc != 5 || [lindex $::argv 4] ne "no-vendor-ip"}] +set options [lrange $::argv 4 end] +set vendor_ip [expr {[lsearch -exact $options no-vendor-ip] < 0}] +# The kernel's clock for its out-of-context synthesis, so that synthesis +# times and retimes it: without one the IP is synthesized unconstrained. +set ooc_xdc "" +foreach option $options { + if {[string match "ooc-xdc=*" $option]} { + set ooc_xdc [file normalize [string range $option 8 end]] + } +} set tool_dir $::env(TOOL_DIR) @@ -68,6 +77,11 @@ foreach line [split $data "\n"] { } } +if {$ooc_xdc ne ""} { + add_files -norecurse -fileset constrs_1 $ooc_xdc + set_property USED_IN {synthesis out_of_context} [get_files $ooc_xdc] +} + # Pull in the floating-point cores generated above so they are packaged # alongside the RTL. The set matches the XRT flow's package_kernel.tcl. if {$vendor_ip} { diff --git a/sim/foldsim/Makefile b/sim/foldsim/Makefile index fc8b32a051..e2cd0266c9 100644 --- a/sim/foldsim/Makefile +++ b/sim/foldsim/Makefile @@ -41,6 +41,7 @@ $(GEN_SV): force $(MAKE) -C $(ROOT_DIR)/hw/syn/foldsim generate TARGET=hw FOLDSIM_WORK_DIR=$(WORK_DIR) CONFIGS='$(CONFIGS)' \ $(if $(PERF),PERF=$(PERF)) $(if $(FOLDSIM_CLASSES),FOLDSIM_CLASSES='$(FOLDSIM_CLASSES)') \ $(if $(FOLDSIM_LIFT),FOLDSIM_LIFT='$(FOLDSIM_LIFT)') $(if $(CIRCT_BIN),CIRCT_BIN=$(CIRCT_BIN)) \ + $(if $(FOLDSIM_FLAGS),FOLDSIM_FLAGS='$(FOLDSIM_FLAGS)') \ $(if $(LIFT_PYTHON),LIFT_PYTHON=$(LIFT_PYTHON)) $(if $(FOLDSIM),FOLDSIM=$(FOLDSIM)) $(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(wildcard $(FOLDSIM_RTL_DIR)/*) $(CONFIG_FILE) From 2bb09de559e1f7199f7306aaf06e0618681d852d Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 03:20:55 -0700 Subject: [PATCH 219/336] foldsim: the control endpoint's token ends are staged foldsim_token_in and foldsim_token_out take STAGED: a token counts from the host cycle after it is taken, so have and done are registers and the endpoint's finish starts at its own state rather than at the simulator's valid and ready. foldsim_ctrl uses them; the memory endpoint keeps the unstaged ends. Target cycles and host cycles are unchanged on demo and sgemm (in-process simulator). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit cd4c38a84d3ace5f105887ff3b2c07fa09c4e6c6) --- hw/rtl/afu/foldsim/foldsim_ctrl.sv | 22 ++++++++++++---------- hw/rtl/afu/foldsim/foldsim_token.sv | 17 +++++++++++------ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/hw/rtl/afu/foldsim/foldsim_ctrl.sv b/hw/rtl/afu/foldsim/foldsim_ctrl.sv index aae283ec6d..11f18cabcc 100644 --- a/hw/rtl/afu/foldsim/foldsim_ctrl.sv +++ b/hw/rtl/afu/foldsim/foldsim_ctrl.sv @@ -124,13 +124,13 @@ module foldsim_ctrl #( wire finish; wire [6:0] done; wire [2:0] have; - foldsim_token_out o_reset (clk, reset, reset_valid, reset_ready, run, finish, done[0]); - foldsim_token_out o_dvalid (clk, reset, dcr_req_valid_valid, dcr_req_valid_ready, run, finish, done[1]); - foldsim_token_out o_drw (clk, reset, dcr_req_rw_valid, dcr_req_rw_ready, run, finish, done[2]); - foldsim_token_out o_daddr (clk, reset, dcr_req_addr_valid, dcr_req_addr_ready, run, finish, done[3]); - foldsim_token_out o_ddata (clk, reset, dcr_req_data_valid, dcr_req_data_ready, run, finish, done[4]); - foldsim_token_out o_start (clk, reset, start_valid, start_ready, run, finish, done[5]); - foldsim_token_out o_mreset (clk, reset, mem_reset_valid, mem_reset_ready, run, finish, done[6]); + foldsim_token_out #(1) o_reset (clk, reset, reset_valid, reset_ready, run, finish, done[0]); + foldsim_token_out #(1) o_dvalid (clk, reset, dcr_req_valid_valid, dcr_req_valid_ready, run, finish, done[1]); + foldsim_token_out #(1) o_drw (clk, reset, dcr_req_rw_valid, dcr_req_rw_ready, run, finish, done[2]); + foldsim_token_out #(1) o_daddr (clk, reset, dcr_req_addr_valid, dcr_req_addr_ready, run, finish, done[3]); + foldsim_token_out #(1) o_ddata (clk, reset, dcr_req_data_valid, dcr_req_data_ready, run, finish, done[4]); + foldsim_token_out #(1) o_start (clk, reset, start_valid, start_ready, run, finish, done[5]); + foldsim_token_out #(1) o_mreset (clk, reset, mem_reset_valid, mem_reset_ready, run, finish, done[6]); assign reset_bits = in_reset; assign dcr_req_valid_bits = dcr_valid; assign dcr_req_rw_bits = dcr_rw; @@ -141,10 +141,12 @@ module foldsim_ctrl #( wire rsp_v, busy; wire [DCR_DATA_W-1:0] rsp_d; - foldsim_token_in #(1) t_rspv (clk, reset, dcr_rsp_valid_valid, dcr_rsp_valid_ready, dcr_rsp_valid_bits, finish, have[0], rsp_v); - foldsim_token_in #(DCR_DATA_W) t_rspd (clk, reset, dcr_rsp_data_valid, dcr_rsp_data_ready, dcr_rsp_data_bits, finish, have[1], rsp_d); - foldsim_token_in #(1) t_busy (clk, reset, busy_valid, busy_ready, busy_bits, finish, have[2], busy); + foldsim_token_in #(1, 1) t_rspv (clk, reset, dcr_rsp_valid_valid, dcr_rsp_valid_ready, dcr_rsp_valid_bits, finish, have[0], rsp_v); + foldsim_token_in #(DCR_DATA_W, 1) t_rspd (clk, reset, dcr_rsp_data_valid, dcr_rsp_data_ready, dcr_rsp_data_bits, finish, have[1], rsp_d); + foldsim_token_in #(1, 1) t_busy (clk, reset, busy_valid, busy_ready, busy_bits, finish, have[2], busy); + // Staged token ends: the cycle finishes from the endpoint's registers, a + // host cycle after its last token. assign finish = run & (&have) & (&done); // ---- The host's registers ---- diff --git a/hw/rtl/afu/foldsim/foldsim_token.sv b/hw/rtl/afu/foldsim/foldsim_token.sv index 9620a31265..b9eecb7634 100644 --- a/hw/rtl/afu/foldsim/foldsim_token.sv +++ b/hw/rtl/afu/foldsim/foldsim_token.sv @@ -17,11 +17,14 @@ // // Valid and ready depend on the endpoint's state alone, never on the // simulator's handshake, so the simulator may connect them to its models -// combinationally. +// combinationally. With STAGED, `have` and `done` do too: a token counts from +// the host cycle after it is taken, so the endpoint's finish starts at its own +// registers rather than at the simulator's valid and ready. // An input channel: the token is kept until the endpoint finishes its cycle. module foldsim_token_in #( - parameter W = 1 + parameter W = 1, + parameter STAGED = 0 ) ( input wire clk, input wire reset, @@ -36,8 +39,8 @@ module foldsim_token_in #( reg [W-1:0] held; assign ready = ~got; - assign have = got | valid; - assign data = got ? held : bits; + assign have = STAGED ? got : (got | valid); + assign data = (STAGED || got) ? held : bits; always @(posedge clk) begin if (reset || finish) begin @@ -52,7 +55,9 @@ module foldsim_token_in #( endmodule // An output channel: the token is offered once `ok` until it is taken. -module foldsim_token_out ( +module foldsim_token_out #( + parameter STAGED = 0 +) ( input wire clk, input wire reset, output wire valid, @@ -64,7 +69,7 @@ module foldsim_token_out ( reg fired; assign valid = ~fired & ok; - assign done = fired | (ok & ready); + assign done = STAGED ? fired : (fired | (ok & ready)); always @(posedge clk) begin if (reset || finish) begin From 7138fb0c3e796c58cb16eedfc4bdd499d9a26b3d Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 02:48:35 -0700 Subject: [PATCH 220/336] dram_timing: the DRAM standards' tables and Ramulator's rounding, shared dram_sim's organization and timing tables move to sw/common/dram_timing.h, with Ramulator's integer JEDEC rounding of nanosecond timings to DRAM clocks, so the simulators, the bank model and the Foldsim runtime all derive the same clocks from the same data-sheet values. DramSim::Type is the shared standard enum. No behavior change: tests/unittest/dram_sim's output and simx demo/sgemm cycles are identical. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 86804808129f38f3e52131737878a32863551509) --- sim/common/dram_sim.cpp | 91 ++++------------- sim/common/dram_sim.h | 9 +- sw/common/dram_timing.h | 215 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+), 79 deletions(-) create mode 100644 sw/common/dram_timing.h diff --git a/sim/common/dram_sim.cpp b/sim/common/dram_sim.cpp index bc899b89fc..81f7aafcc0 100644 --- a/sim/common/dram_sim.cpp +++ b/sim/common/dram_sim.cpp @@ -13,6 +13,7 @@ #include "dram_sim.h" #include "util.h" +#include #include #include #include @@ -36,57 +37,9 @@ using namespace vortex; namespace { -// Bus timings are DRAM clocks, fixed by the burst; array timings are -// nanoseconds, which Ramulator converts at the configured data rate. -struct clock_timing_t { - const char* name; - int clocks; -}; - -struct ns_timing_t { - const char* name; - float ns; -}; - -// A 32 B burst is BL4 on a 64-bit pseudo-channel: two clocks, and the same -// spacing between column commands to another bank group (four within one). -// nRTW only completes the set Ramulator requires. -const clock_timing_t hbm_bus_timing[] = { - {"nBL", 2}, {"nCCDS", 2}, {"nCCDL", 4}, {"nRTW", 3}, -}; - -// JEDEC HBM2 array timing as GPGPU-Sim's SM7_QV100 configuration lists it (its -// JEDEC line, at 1 GHz: RCD=RP=CL=14, RAS=33, RC=47, WR=12, RRD=4, RTPL=4); -// the rest from Ramulator's HBM2_2Gbps preset, whose 1 ns clock makes its -// cycles nanoseconds. The array does not speed up with the interface: HBM2E -// and HBM3 parts keep these timings (GPGPU-Sim's SM80_A100 and SM90_H100 HBM -// lines are the same 14/33/47 ns at their own clocks). Refresh timings complete -// the set Ramulator requires. -const ns_timing_t hbm_array_timing[] = { - {"tCL", 14}, {"tRCDRD", 14}, {"tRCDWR", 14}, {"tRP", 14}, {"tRAS", 33}, {"tRC", 47}, - {"tCWL", 2}, {"tWR", 12}, {"tRTPS", 2}, {"tRTPL", 4}, {"tRRDS", 4}, {"tRRDL", 4}, - {"tWTRS", 3}, {"tWTRL", 4}, {"tFAW", 15}, {"tRFCSB", 160}, {"tREFI", 3900}, {"tRREFD", 8}, -}; - -// Ramulator's GDDR6 clock is half the data rate: a 32 B burst, 16 beats on a -// x16 channel, is eight clocks, and column commands are two (three within a -// bank group) command clocks apart, a command clock being eight data beats. -const clock_timing_t gddr6_bus_timing[] = { - {"nBL", 8}, {"nCCDS", 8}, {"nCCDL", 12}, -}; - -// Ramulator's x16 GDDR6 timing at 2000 (its GDDR6_2000_1350mV_double preset -// and secondary tables) at its 570 ps command clock, in nanoseconds; refresh -// timings complete the set Ramulator requires. -const ns_timing_t gddr6_array_timing[] = { - {"tCL", 13.7f}, {"tRCDRD", 14.8f}, {"tRCDWD", 9.1f}, {"tRP", 14.8f}, {"tRAS", 30.2f}, {"tRC", 45.0f}, - {"tWR", 14.8f}, {"tRTP", 2.3f}, {"tCWL", 3.4f}, {"tRRDS", 2.9f}, {"tRRDL", 3.4f}, - {"tWTRS", 5.1f}, {"tWTRL", 6.3f}, {"tFAW", 16.0f}, - {"tRFC", 120.0f}, {"tRFCpb", 60.0f}, {"tREFI", 1900.0f}, {"tRREFD", 8.0f}, -}; - template -void set_timing(YAML::Node timing, uint32_t data_rate, const clock_timing_t (&bus)[B], const ns_timing_t (&array)[A]) { +void set_timing(YAML::Node timing, uint32_t data_rate, const dram_timing::clock_timing_t (&bus)[B], + const dram_timing::ns_timing_t (&array)[A]) { timing["rate"] = data_rate; for (auto& t : bus) { timing[t.name] = t.clocks; @@ -130,19 +83,15 @@ class DramSim::Impl { uint32_t inflight; }; - // Each standard moves 32 B per column command (prefetch x channel width). - static constexpr uint32_t tx_size_ = 32; - - // Every device here holds 8 Gb per channel; addresses above alias, which - // only affects row hits. - static constexpr uint64_t channel_size_ = uint64_t(1) << 30; + static constexpr uint32_t tx_size_ = dram_timing::kBurstSize; - // Bursts per row visit (minimalist open page, 128 B). - static constexpr uint32_t row_visit_bits_ = 2; + // Addresses above a channel's 8 Gb alias, which only affects row hits. + static constexpr uint64_t channel_size_ = dram_timing::kChannelSize; Config config_; - uint32_t bank_bits_; // pseudo-channel, bank group and bank - uint32_t column_bits_; // bursts per row + uint32_t row_visit_bits_; // bursts per row visit (minimalist open page, 128 B) + uint32_t bank_bits_; // pseudo-channel, bank group and bank + uint32_t column_bits_; // bursts per row std::vector channels_; uint64_t cycle_; // DRAM ticks owed, in units of 1/(2 x core_clock): each core cycle adds the @@ -158,7 +107,7 @@ class DramSim::Impl { dram["org"]["density"] = 8192; dram["org"]["channel"] = 1; dram["org"]["channel_width"] = 128; - set_timing(dram["timing"], config.data_rate, hbm_bus_timing, hbm_array_timing); + set_timing(dram["timing"], config.data_rate, dram_timing::hbm_bus_timing, dram_timing::hbm_array_timing); } static void set_gddr6_device(YAML::Node dram, const Config& config) { @@ -174,7 +123,7 @@ class DramSim::Impl { dram["org"]["column"] = 512; dram["org"]["density"] = 8192; dram["org"]["channel_width"] = 32; - set_timing(dram["timing"], config.data_rate, gddr6_bus_timing, gddr6_array_timing); + set_timing(dram["timing"], config.data_rate, dram_timing::gddr6_bus_timing, dram_timing::gddr6_array_timing); } YAML::Node channel_config(uint32_t ch) const { @@ -278,11 +227,11 @@ class DramSim::Impl { public: Impl(const Config& config) : config_(config) - // HBM: 2 pseudo-channels x 4 bank groups x 4 banks, 32 bursts per 1 KB - // row; GDDR6: 4 x 4 banks, 64 bursts per 2 KB row. - , bank_bits_((config.type == Type::GDDR6) ? 4 : 5) - , column_bits_((config.type == Type::GDDR6) ? 6 : 5) , channels_(config.num_channels) { + auto geometry = dram_timing::geometry(config.type); + row_visit_bits_ = geometry.log2_visit; + bank_bits_ = geometry.bank_bits(); + column_bits_ = geometry.log2_row_bursts; for (uint32_t ch = 0; ch < config.num_channels; ++ch) { auto ramulator_config = this->channel_config(ch); auto& channel = channels_[ch]; @@ -360,13 +309,9 @@ DramSim::Config DramSim::platform_config() { // companion. const std::string type = DRAM_STR(VX_CFG_PLATFORM_MEMORY_TYPE); Config config; - if (type == "HBM2") { - config.type = Type::HBM2; - } else if (type == "HBM3") { - config.type = Type::HBM3; - } else if (type == "GDDR6") { - config.type = Type::GDDR6; - } else { + try { + config.type = dram_timing::parse_standard(type); + } catch (const std::invalid_argument&) { throw std::invalid_argument("dram_sim: unknown VX_CFG_PLATFORM_MEMORY_TYPE " + type); } config.num_channels = VX_CFG_PLATFORM_MEMORY_NUM_BANKS; diff --git a/sim/common/dram_sim.h b/sim/common/dram_sim.h index 565dede377..4a5c4cf21d 100644 --- a/sim/common/dram_sim.h +++ b/sim/common/dram_sim.h @@ -14,6 +14,7 @@ #pragma once #include +#include namespace vortex { @@ -24,12 +25,8 @@ class DramSim { typedef bool (*ResponseCallback)(void *arg); // The JEDEC standards Ramulator models, each at its own organization and - // timing; the data rate picks the speed. - enum class Type { - HBM2, // HBM2/HBM2E: 128-bit channels of two 64-bit pseudo-channels, 32 B bursts - HBM3, // HBM3, a 128-bit channel per pair of 64-bit channels, 32 B bursts - GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages - }; + // timing (sw/common/dram_timing.h); the data rate picks the speed. + using Type = dram_timing::Standard; struct Config { Type type; diff --git a/sw/common/dram_timing.h b/sw/common/dram_timing.h new file mode 100644 index 0000000000..7c75879893 --- /dev/null +++ b/sw/common/dram_timing.h @@ -0,0 +1,215 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The DRAM devices the platform memory models (VX_CFG_PLATFORM_MEMORY_TYPE): +// each JEDEC standard's organization and timing as a data sheet gives it, and +// the conversion of its nanosecond timings to DRAM clocks at a data rate. +// sim/common/dram_sim.cpp hands these to Ramulator; the bank model +// (sim/common/dram_bank_model.h) and the Foldsim runtime convert them with the +// same rounding Ramulator applies, so all of them see the same clocks. + +#pragma once + +#include +#include +#include +#include +#include + +namespace vortex { +namespace dram_timing { + +enum class Standard { + HBM2, // HBM2/HBM2E: 128-bit channels of two 64-bit pseudo-channels, 32 B bursts + HBM3, // HBM3, a 128-bit channel per pair of 64-bit channels, 32 B bursts + GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages +}; + +inline Standard parse_standard(const std::string& name) { + if (name == "HBM2") return Standard::HBM2; + if (name == "HBM3") return Standard::HBM3; + if (name == "GDDR6") return Standard::GDDR6; + throw std::invalid_argument("unknown DRAM standard " + name); +} + +// Bus timings are DRAM clocks, fixed by the burst; array timings are +// nanoseconds, converted at the configured data rate. Names are Ramulator's. +struct clock_timing_t { + const char* name; + int clocks; +}; + +struct ns_timing_t { + const char* name; + float ns; +}; + +// A 32 B burst is BL4 on a 64-bit pseudo-channel: two clocks, and the same +// spacing between column commands to another bank group (four within one). +// nRTW only completes the set Ramulator requires. +inline constexpr clock_timing_t hbm_bus_timing[] = { + {"nBL", 2}, {"nCCDS", 2}, {"nCCDL", 4}, {"nRTW", 3}, +}; + +// JEDEC HBM2 array timing as GPGPU-Sim's SM7_QV100 configuration lists it (its +// JEDEC line, at 1 GHz: RCD=RP=CL=14, RAS=33, RC=47, WR=12, RRD=4, RTPL=4); +// the rest from Ramulator's HBM2_2Gbps preset, whose 1 ns clock makes its +// cycles nanoseconds. The array does not speed up with the interface: HBM2E +// and HBM3 parts keep these timings (GPGPU-Sim's SM80_A100 and SM90_H100 HBM +// lines are the same 14/33/47 ns at their own clocks). Refresh timings complete +// the set Ramulator requires. +inline constexpr ns_timing_t hbm_array_timing[] = { + {"tCL", 14}, {"tRCDRD", 14}, {"tRCDWR", 14}, {"tRP", 14}, {"tRAS", 33}, {"tRC", 47}, + {"tCWL", 2}, {"tWR", 12}, {"tRTPS", 2}, {"tRTPL", 4}, {"tRRDS", 4}, {"tRRDL", 4}, + {"tWTRS", 3}, {"tWTRL", 4}, {"tFAW", 15}, {"tRFCSB", 160}, {"tREFI", 3900}, {"tRREFD", 8}, +}; + +// Ramulator's GDDR6 clock is half the data rate: a 32 B burst, 16 beats on a +// x16 channel, is eight clocks, and column commands are two (three within a +// bank group) command clocks apart, a command clock being eight data beats. +inline constexpr clock_timing_t gddr6_bus_timing[] = { + {"nBL", 8}, {"nCCDS", 8}, {"nCCDL", 12}, +}; + +// Ramulator's x16 GDDR6 timing at 2000 (its GDDR6_2000_1350mV_double preset +// and secondary tables) at its 570 ps command clock, in nanoseconds; refresh +// timings complete the set Ramulator requires. +inline constexpr ns_timing_t gddr6_array_timing[] = { + {"tCL", 13.7f}, {"tRCDRD", 14.8f}, {"tRCDWD", 9.1f}, {"tRP", 14.8f}, {"tRAS", 30.2f}, {"tRC", 45.0f}, + {"tWR", 14.8f}, {"tRTP", 2.3f}, {"tCWL", 3.4f}, {"tRRDS", 2.9f}, {"tRRDL", 3.4f}, + {"tWTRS", 5.1f}, {"tWTRL", 6.3f}, {"tFAW", 16.0f}, + {"tRFC", 120.0f}, {"tRFCpb", 60.0f}, {"tREFI", 1900.0f}, {"tRREFD", 8.0f}, +}; + +// One channel's organization and the address map the simulators lay over it. +struct geometry_t { + uint32_t log2_pcs; // pseudo-channels sharing the channel's commands + uint32_t log2_bgs; // bank groups per pseudo-channel + uint32_t log2_banks; // banks per bank group + uint32_t log2_row_bursts; // 32 B bursts per row + uint32_t log2_visit; // bursts per row visit before the next bank + bool two_clock_act; // an ACT holds the row-command bus for two clocks + uint32_t rtw_extra; // read-to-write turnaround beyond tCL + tBL - tCWL + uint32_t bank_bits() const { return log2_pcs + log2_bgs + log2_banks; } +}; + +// HBM: 2 pseudo-channels x 4 bank groups x 4 banks, 1 KB rows (32 bursts); +// GDDR6 x16: 4 x 4 banks, 2 KB rows (64 bursts). Either way a row visit is four +// bursts (minimalist open page, 128 B). Read-to-write is Ramulator's: HBM's +// assumes a one-clock write preamble, GDDR6's adds the bus turnaround. +inline geometry_t geometry(Standard standard) { + if (standard == Standard::GDDR6) { + return geometry_t{0, 2, 2, 6, 2, false, 4}; + } + return geometry_t{1, 2, 2, 5, 2, true, 2}; +} + +// Every device here holds 8 Gb per channel; addresses above alias. +inline constexpr uint64_t kChannelSize = uint64_t(1) << 30; +// Each standard moves 32 B per column command (prefetch x channel width). +inline constexpr uint32_t kBurstSize = 32; + +// Ramulator's DRAM clock period at a data rate (MT/s): half the data rate, in +// integer picoseconds. +inline int tck_ps(uint32_t data_rate) { + return int(1E6 / (data_rate / 2)); +} + +// Ramulator's JEDEC rounding of a nanosecond timing to clocks (base/utils.cpp), +// in the same float arithmetic. +inline uint32_t jedec_clocks(float t_ns, int tck) { + uint64_t t_ps = t_ns * 1000; + return uint32_t(((t_ps * 1000 / tck) + 974) / 1000); +} + +// The timings a scheduler of one channel needs, in DRAM clocks. +struct clocks_t { + uint32_t BL, CCDS, CCDL; + uint32_t CL, CWL; + uint32_t RCDRD, RCDWR, RP, RAS, RC; + uint32_t RTP, WR; + uint32_t RRDS, RRDL, FAW; + uint32_t WTRS, WTRL; +}; + +namespace detail { + +template +inline bool find_clock(const clock_timing_t (&table)[N], const char* name, uint32_t* out) { + for (auto& t : table) { + if (std::strcmp(t.name, name) == 0) { + *out = uint32_t(t.clocks); + return true; + } + } + return false; +} + +template +inline bool find_ns(const ns_timing_t (&table)[N], const char* name, int tck, uint32_t* out) { + for (auto& t : table) { + if (std::strcmp(t.name, name) == 0) { + *out = jedec_clocks(t.ns, tck); + return true; + } + } + return false; +} + +template +inline uint32_t get(const clock_timing_t (&bus)[B], const ns_timing_t (&array)[A], int tck, + const char* clock_name, const char* ns_name) { + uint32_t v = 0; + if (clock_name && find_clock(bus, clock_name, &v)) return v; + if (ns_name && find_ns(array, ns_name, tck, &v)) return v; + throw std::logic_error(std::string("dram_timing: no timing ") + (ns_name ? ns_name : clock_name)); +} + +} // namespace detail + +// The standard's timings at a data rate. A read may precharge its bank tRTP +// after its column command: HBM's bank-level read-to-precharge is the larger of +// its short and long forms. +inline clocks_t clocks(Standard standard, uint32_t data_rate) { + const int tck = tck_ps(data_rate); + clocks_t c{}; + if (standard == Standard::GDDR6) { + auto g = [&](const char* n, const char* t) { + return detail::get(gddr6_bus_timing, gddr6_array_timing, tck, n, t); + }; + c.BL = g("nBL", nullptr); c.CCDS = g("nCCDS", nullptr); c.CCDL = g("nCCDL", nullptr); + c.CL = g(nullptr, "tCL"); c.CWL = g(nullptr, "tCWL"); + c.RCDRD = g(nullptr, "tRCDRD"); c.RCDWR = g(nullptr, "tRCDWD"); + c.RP = g(nullptr, "tRP"); c.RAS = g(nullptr, "tRAS"); c.RC = g(nullptr, "tRC"); + c.RTP = g(nullptr, "tRTP"); c.WR = g(nullptr, "tWR"); + c.RRDS = g(nullptr, "tRRDS"); c.RRDL = g(nullptr, "tRRDL"); c.FAW = g(nullptr, "tFAW"); + c.WTRS = g(nullptr, "tWTRS"); c.WTRL = g(nullptr, "tWTRL"); + } else { + auto g = [&](const char* n, const char* t) { + return detail::get(hbm_bus_timing, hbm_array_timing, tck, n, t); + }; + c.BL = g("nBL", nullptr); c.CCDS = g("nCCDS", nullptr); c.CCDL = g("nCCDL", nullptr); + c.CL = g(nullptr, "tCL"); c.CWL = g(nullptr, "tCWL"); + c.RCDRD = g(nullptr, "tRCDRD"); c.RCDWR = g(nullptr, "tRCDWR"); + c.RP = g(nullptr, "tRP"); c.RAS = g(nullptr, "tRAS"); c.RC = g(nullptr, "tRC"); + uint32_t rtps = g(nullptr, "tRTPS"), rtpl = g(nullptr, "tRTPL"); + c.RTP = (rtps > rtpl) ? rtps : rtpl; + c.WR = g(nullptr, "tWR"); + c.RRDS = g(nullptr, "tRRDS"); c.RRDL = g(nullptr, "tRRDL"); c.FAW = g(nullptr, "tFAW"); + c.WTRS = g(nullptr, "tWTRS"); c.WTRL = g(nullptr, "tWTRL"); + } + return c; +} + +} // namespace dram_timing +} // namespace vortex From ad2c25f3aee32251a46948116e9b5a904863c474 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 03:24:16 -0700 Subject: [PATCH 221/336] dram_bank_model: the reservation-based DRAM bank model, a DramSim backend sim/common/dram_bank_model.h is the golden model of the Foldsim memory proposal's bank model: one channel's requests reserved once each, their PRE, ACT and column commands placed at the first DRAM clocks every JEDEC constraint allows (tRCD/tRP/tRAS/tRC/tRTP/tWR per bank, tCCD/tRRD/tFAW/ turnarounds per pseudo-channel and bank group, one command per clock, the two-clock HBM ACT), before or after commands reserved earlier; open rows, first come first served per bank; reads complete when their data departs. Writes enter a write buffer, acknowledged there, and drain in batches between dram_sim's watermarks; a batch is placed after every column reserved before it and the reads after it after the batch. Reservations reach at most 448 clocks ahead (Foldsim's ring). DramBankSim puts it behind DramSim's interface, with dram_sim's clock loop, ingress and response drains. DramSim takes it as a backend with VORTEX_DRAM_MODEL=bank (a choice of simulator, not of machine); Ramulator stays the default and unchanged (tests/unittest/dram_sim's output identical), and the bank backend passes the same unit test. tests/unittest/dram_bank_model runs both on the same closed-loop traffic. Its gate: an idle channel's reads take the same latency in both, read for read, on every machine (default, 4 channels, v100, a100, h100, rtx1080, rtx5080): exact. "full" prints the calibration table. On HBM2/HBM3 at up to 16 reads in flight reads are within 1.1% (random) and 0.3% (sequential) of Ramulator, mixed reads and writes within 5.5% (the write drain moved a 50%-write stream from -27% to -4.5%); single-bank row conflicts at high DRAM/core clock ratios stay 18-25% slow, FR-FCFS's reordering of hits within a bank that a per-bank first-come-first-served reservation does not do. simx apps (sgemm, vecadd, sgemv, stencil3d) on the default, a100 and h100 memories are within 1% of Ramulator in cycles. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bb2006a6df61f46f998f615eaee6055105c6bf50) --- sim/common/dram_bank_model.h | 578 ++++++++++++++++++++++++ sim/common/dram_sim.cpp | 64 ++- sim/common/dram_sim.h | 4 +- tests/unittest/Makefile | 2 +- tests/unittest/dram_bank_model/Makefile | 16 + tests/unittest/dram_bank_model/main.cpp | 217 +++++++++ 6 files changed, 872 insertions(+), 9 deletions(-) create mode 100644 sim/common/dram_bank_model.h create mode 100644 tests/unittest/dram_bank_model/Makefile create mode 100644 tests/unittest/dram_bank_model/main.cpp diff --git a/sim/common/dram_bank_model.h b/sim/common/dram_bank_model.h new file mode 100644 index 0000000000..68a628a740 --- /dev/null +++ b/sim/common/dram_bank_model.h @@ -0,0 +1,578 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace vortex { + +// One DRAM channel as a reservation-based bank model: each request is +// scheduled once, by placing its precharge, activate and column commands at +// the first DRAM clocks every JEDEC constraint allows -- before or after +// commands reserved earlier -- and its read data departs tCL + tBL after its +// last column. Banks keep their rows open; a request to a bank waits for the +// bank's earlier requests (first come, first served per bank). The controller +// issues one command per clock; on HBM an ACT holds the row-command bus for +// two. +// +// Requests enter the controller's read or write buffer in order, one per +// request's bursts' worth of clocks, while the buffer has room; a write is +// complete once buffered. The controller serves reads, and turns to writes +// when its write bursts reach the high watermark or no read waits, back once +// they fall below the low one with a read waiting -- dram_sim's write drain. A +// batch of writes is placed after every column reserved before it, and the +// reads after it after the batch, so the data bus turns around once per batch. +// Reservations reach at most `horizon` clocks ahead: a request that would +// reach further waits. +// +// This is the normative model; the face below puts it behind DramSim's +// interface. +class DramBankModel { +public: + struct Config { + dram_timing::geometry_t geom; + dram_timing::clocks_t t; + uint32_t read_queue = 32; // reserved read bursts whose column has not issued + uint32_t write_queue = 32; // the same for writes + // Write bursts buffered or reserved and not issued at which the channel + // turns to writes, and below which it turns back to waiting reads: + // dram_sim's 80 % and 20 % of its 32-entry write buffer. + uint32_t wr_high = 26; + uint32_t wr_low = 7; + // Clocks a reservation may reach past the current one: foldsim's ring of + // 512 clocks, less the word of 64 it is retiring. + uint32_t horizon = 448; + uint32_t bursts = 2; // 32 B bursts per request + + static Config make(dram_timing::Standard standard, uint32_t data_rate, uint32_t block_size) { + Config c; + c.geom = dram_timing::geometry(standard); + c.t = dram_timing::clocks(standard, data_rate); + c.bursts = (block_size > dram_timing::kBurstSize) ? block_size / dram_timing::kBurstSize : 1; + if (c.bursts > (1u << c.geom.log2_visit)) { + throw std::invalid_argument("dram_bank_model: a request spans more than one row visit"); + } + return c; + } + + // Column-to-column spacing on one pseudo-channel, and within one bank + // group, by the directions of the earlier and the later command. + uint32_t rtw() const { + int64_t v = int64_t(t.CL) + t.BL + geom.rtw_extra - t.CWL; + return uint32_t(std::max(t.CCDS, v)); + } + uint32_t wtr_s() const { return t.CWL + t.BL + t.WTRS; } + uint32_t wtr_l() const { return t.CWL + t.BL + t.WTRL; } + uint32_t sep_pc(bool from_w, bool to_w) const { + if (from_w == to_w) return t.CCDS; + return from_w ? wtr_s() : rtw(); + } + uint32_t sep_bg(bool from_w, bool to_w) const { + if (from_w == to_w) return t.CCDL; + return from_w ? wtr_l() : 0; + } + // The furthest clock after a command its constraints on later commands + // reach, which must lie within the horizon. + uint32_t col_reach(bool w) const { + uint32_t r = std::max(t.CCDS, t.CCDL); + r = std::max(r, w ? std::max(wtr_s(), wtr_l()) : rtw()); + return r - 1; + } + uint32_t act_reach() const { + uint32_t r = std::max({t.RRDS, t.RRDL, t.FAW}) - 1; + return std::max(r, geom.two_clock_act ? 1 : 0); + } + uint32_t num_banks() const { return 1u << geom.bank_bits(); } + uint32_t pc_of(uint32_t bank) const { return bank & ((1u << geom.log2_pcs) - 1); } + uint32_t bg_of(uint32_t bank) const { + return (bank >> geom.log2_pcs) & ((1u << geom.log2_bgs) - 1); + } + }; + + struct Stats { + uint64_t reads = 0, writes = 0, hits = 0, misses = 0, conflicts = 0; + uint64_t deferred = 0; // clocks a request waited: it reached past the horizon + }; + + explicit DramBankModel(const Config& cfg) : cfg_(cfg) { this->reset(); } + + const Config& config() const { return cfg_; } + const Stats& stats() const { return stats_; } + + void reset() { + d_ = 0; + ingress_.clear(); + ingress_ready_ = 0; + uint32_t nb = cfg_.num_banks(); + open_.assign(nb, false); + row_.assign(nb, 0); + act_.assign(nb, kNever); + col_ready_.assign(nb, 0); + pre_ready_.assign(nb, 0); + cmd_.clear(); + rowbus_.clear(); + cols_.clear(); + acts_.clear(); + colkind_.clear(); + dep_.clear(); + rq_ = 0; + wq_ = 0; + rbuf_.clear(); + wbuf_.clear(); + write_mode_ = false; + frontier_ = 0; + batch_floor_ = 0; + batch_last_ = 0; + read_floor_ = 0; + stats_ = Stats(); + } + + // The bank and row of a byte address within the channel: a row visit of + // four bursts, then pseudo-channel, bank group and bank from the low end, + // the bank XORed with the row folded to its width (permutation interleave). + void decode(uint64_t addr, uint32_t* bank, uint32_t* row) const { + const auto& g = cfg_.geom; + uint32_t bb = g.bank_bits(); + uint64_t tx = (addr & (dram_timing::kChannelSize - 1)) / dram_timing::kBurstSize; + tx >>= g.log2_visit; + uint64_t b = tx & ((1u << bb) - 1); + tx >>= bb; + uint64_t r = tx >> (g.log2_row_bursts - g.log2_visit); + for (uint64_t x = r; x != 0; x >>= bb) { + b ^= x & ((1u << bb) - 1); + } + *bank = uint32_t(b); + *row = uint32_t(r); + } + + // A request joins the ingress; `tag` comes back when it completes. + void push(uint32_t tag, bool write, uint64_t addr) { + Request r; + r.tag = tag; + r.write = write; + this->decode(addr, &r.bank, &r.row); + ingress_.push_back(r); + } + + bool ingress_empty() const { return ingress_.empty(); } + size_t ingress_size() const { return ingress_.size(); } + uint64_t clock() const { return d_; } + + // One DRAM clock d: columns issued by d leave the controller queues, the + // ingress head enters its buffer if it may, the head of the buffer the + // controller serves is reserved if it can be (its commands no earlier than + // d+1), and a read whose data departs at d+1 completes. + template + void tick(ReadDone&& read_done, WriteDone&& write_done) { + const int64_t d = int64_t(d_); + for (auto it = colkind_.begin(); it != colkind_.end() && it->first <= d;) { + (it->second ? wq_ : rq_) -= 1; + it = colkind_.erase(it); + } + const uint32_t nb = cfg_.bursts; + if (!ingress_.empty() && d >= ingress_ready_) { + const Request& r = ingress_.front(); + if (r.write ? (wq_ + nb * (wbuf_.size() + 1) <= cfg_.write_queue) + : (rq_ + nb * (rbuf_.size() + 1) <= cfg_.read_queue)) { + (r.write ? wbuf_ : rbuf_).push_back(r); + if (r.write) { + write_done(r.tag); + } + ingress_.pop_front(); + ingress_ready_ = d + nb; + } + } + const uint64_t wn = wq_ + nb * wbuf_.size(); + const uint64_t rn = rq_ + nb * rbuf_.size(); + if (!write_mode_) { + if (wn >= cfg_.wr_high || rn == 0) { + write_mode_ = true; + batch_floor_ = std::max(d + 1, frontier_ + 1); + } + } else if (wn < cfg_.wr_low && rn != 0) { + write_mode_ = false; + read_floor_ = batch_last_ + 1; + } + auto& buf = write_mode_ ? wbuf_ : rbuf_; + if (!buf.empty()) { + if (this->reserve(buf.front(), d + 1, d + int64_t(cfg_.horizon), + write_mode_ ? batch_floor_ : read_floor_)) { + buf.pop_front(); + } else { + ++stats_.deferred; + } + } + auto it = dep_.find(d + 1); + if (it != dep_.end()) { + read_done(it->second); + dep_.erase(it); + } + this->prune(d); + ++d_; + } + +private: + static constexpr int64_t kNever = INT64_MIN / 4; + static constexpr int64_t kNone = INT64_MAX / 4; + + struct Request { + uint32_t tag; + bool write; + uint32_t bank; + uint32_t row; + }; + + struct Col { + uint32_t pc, pg; + bool w; + }; + + struct Act { + uint32_t pc, pg; + }; + + bool cmd_free(int64_t t) const { return !cmd_.count(t); } + bool row_free(int64_t t) const { return !rowbus_.count(t); } + + // The first clock in [lb, end] a column of direction `w` to (pc, pg) may + // take, or kNone. A clock inside a reserved column's forbidden span skips + // to its end. + int64_t place_col(int64_t lb, int64_t end, uint32_t pc, uint32_t pg, bool w) const { + const int64_t reach = this->max_col_sep(); + int64_t t = lb; + while (t <= end) { + int64_t next = t; + if (!this->cmd_free(t)) { + next = t + 1; + } + for (auto it = cols_.lower_bound(t - reach); it != cols_.end() && it->first <= t + reach; ++it) { + const int64_t c = it->first; + const Col& col = it->second; + if (col.pc != pc || c == t) { + continue; + } + int64_t need_pc, need_bg, after_pc, after_bg; + after_pc = cfg_.sep_pc(col.w, w); + after_bg = cfg_.sep_bg(col.w, w); + if (c < t) { + need_pc = after_pc; + need_bg = after_bg; + } else { + need_pc = cfg_.sep_pc(w, col.w); + need_bg = cfg_.sep_bg(w, col.w); + } + const int64_t dist = (c < t) ? (t - c) : (c - t); + if (dist < need_pc) { + next = std::max(next, c + std::max(after_pc, 1)); + } + if (col.pg == pg && dist < need_bg) { + next = std::max(next, c + std::max(after_bg, 1)); + } + } + if (next == t) { + return t; + } + t = next; + } + return kNone; + } + + // True if an ACT to (pc, pg) may take clock t: the command and row buses + // are free, tRRD holds against every ACT of the pseudo-channel (tRRD_L + // within the bank group) on either side, and no tFAW window holding t + // holds four other ACTs of the pseudo-channel. + bool act_ok(int64_t t, uint32_t pc, uint32_t pg) const { + if (!this->cmd_free(t) || !this->row_free(t)) return false; + if (cfg_.geom.two_clock_act && !this->row_free(t + 1)) return false; + const int64_t F = cfg_.t.FAW; + const int64_t reach = std::max({F, cfg_.t.RRDS, cfg_.t.RRDL}); + std::vector near; + for (auto it = acts_.lower_bound(t - reach); it != acts_.end() && it->first <= t + reach; ++it) { + const int64_t a = it->first; + const Act& act = it->second; + if (act.pc != pc) continue; + const int64_t dist = (a < t) ? (t - a) : (a - t); + if (dist < int64_t(cfg_.t.RRDS)) return false; + if (act.pg == pg && dist < int64_t(cfg_.t.RRDL)) return false; + if (dist < F) near.push_back(a); + } + near.push_back(t); + std::sort(near.begin(), near.end()); + for (size_t i = 0; i + 4 < near.size(); ++i) { + if (near[i + 4] - near[i] < F) return false; + } + return true; + } + + int64_t place_act(int64_t lb, int64_t end, uint32_t pc, uint32_t pg) const { + for (int64_t t = lb; t <= end; ++t) { + if (this->act_ok(t, pc, pg)) return t; + } + return kNone; + } + + int64_t place_pre(int64_t lb, int64_t end) const { + for (int64_t t = lb; t <= end; ++t) { + if (this->cmd_free(t) && this->row_free(t)) return t; + } + return kNone; + } + + int64_t max_col_sep() const { + return std::max({cfg_.t.CCDS, cfg_.t.CCDL, cfg_.rtw(), cfg_.wtr_s(), cfg_.wtr_l()}); + } + + // Places the request's commands; false (and nothing changed) if its queue + // is full or it does not fit within the horizon. + bool reserve(const Request& r, int64_t e, int64_t end, int64_t floor) { + const auto& T = cfg_.t; + const uint32_t nb = cfg_.bursts; + const uint32_t b = r.bank; + const uint32_t pc = cfg_.pc_of(b); + const uint32_t pg = pc * (1u << cfg_.geom.log2_bgs) + cfg_.bg_of(b); + const bool hit = open_[b] && row_[b] == r.row; + const bool conflict = open_[b] && !hit; + int64_t pre = kNone, act = kNone; + int64_t col_ready = col_ready_[b]; + int64_t pre_ready = pre_ready_[b]; + if (!hit) { + int64_t act_lb = e; + if (conflict) { + pre = this->place_pre(std::max(e, pre_ready_[b]), end); + if (pre == kNone) return false; + act_lb = pre + T.RP; + } + act_lb = std::max(act_lb, act_[b] + int64_t(T.RC)); + act = this->place_act(act_lb, end, pc, pg); + if (act == kNone) return false; + col_ready = act + (r.write ? T.RCDWR : T.RCDRD); + pre_ready = act + T.RAS; + } + int64_t col[8]; + for (uint32_t k = 0; k < nb; ++k) { + int64_t lb = std::max({e + int64_t(k), col_ready, floor}); + if (k != 0) { + lb = std::max(lb, col[k - 1] + int64_t(std::max(T.CCDL, 1))); + } + col[k] = this->place_col(lb, end, pc, pg, r.write); + if (col[k] == kNone) return false; + col_ready = std::max(col_ready, col[k] + 1); + pre_ready = std::max(pre_ready, r.write ? col[k] + T.CWL + T.BL + T.WR : col[k] + T.RTP); + } + const int64_t last_col = col[nb - 1]; + const int64_t departure = last_col + T.CL + T.BL; + int64_t last = last_col + cfg_.col_reach(r.write); + if (!r.write) last = std::max(last, departure); + if (act != kNone) last = std::max(last, act + int64_t(cfg_.act_reach())); + if (last > end) { + return false; + } + + // Commit. + if (pre != kNone) { + cmd_.insert(pre); + rowbus_.insert(pre); + } + if (act != kNone) { + cmd_.insert(act); + rowbus_.insert(act); + if (cfg_.geom.two_clock_act) rowbus_.insert(act + 1); + acts_[act] = Act{pc, pg}; + act_[b] = act; + open_[b] = true; + row_[b] = r.row; + } + for (uint32_t k = 0; k < nb; ++k) { + cmd_.insert(col[k]); + cols_[col[k]] = Col{pc, pg, r.write}; + colkind_[col[k]] = r.write; + frontier_ = std::max(frontier_, col[k]); + if (r.write) batch_last_ = std::max(batch_last_, col[k]); + } + (r.write ? wq_ : rq_) += nb; + col_ready_[b] = col_ready; + pre_ready_[b] = pre_ready; + if (!r.write) { + dep_[departure] = r.tag; + } + if (r.write) ++stats_.writes; else ++stats_.reads; + if (hit) ++stats_.hits; else if (conflict) ++stats_.conflicts; else ++stats_.misses; + return true; + } + + // Drops what can no longer constrain a command at d+1 or later. + void prune(int64_t d) { + const int64_t keep_cols = d + 1 - this->max_col_sep(); + while (!cols_.empty() && cols_.begin()->first < keep_cols) cols_.erase(cols_.begin()); + const int64_t keep_acts = d + 1 - std::max({cfg_.t.FAW, cfg_.t.RRDS, cfg_.t.RRDL}); + while (!acts_.empty() && acts_.begin()->first < keep_acts) acts_.erase(acts_.begin()); + while (!cmd_.empty() && *cmd_.begin() <= d) cmd_.erase(cmd_.begin()); + while (!rowbus_.empty() && *rowbus_.begin() <= d) rowbus_.erase(rowbus_.begin()); + } + + Config cfg_; + uint64_t d_; + std::deque ingress_; + int64_t ingress_ready_; + std::vector open_; + std::vector row_; + std::vector act_, col_ready_, pre_ready_; + std::set cmd_, rowbus_; + std::map cols_; + std::map acts_; + std::map colkind_; + std::map dep_; + uint32_t rq_, wq_; + std::deque rbuf_, wbuf_; + bool write_mode_; + int64_t frontier_, batch_floor_, batch_last_, read_floor_; + Stats stats_; +}; + +/////////////////////////////////////////////////////////////////////////////// + +// The bank model behind DramSim's interface (sim/common/dram_sim.h): one +// channel per memory bank, the same core-clock credit loop, one request +// reserved per bursts' worth of DRAM clocks per channel, the fixed latency +// after a read's data departs, and responses in completion order per channel. +class DramBankSim { +public: + typedef bool (*ResponseCallback)(void* arg); + + struct Config { + dram_timing::Standard type; + uint32_t num_channels; + uint32_t block_size; + uint32_t core_clock; // MHz + uint32_t data_rate; // MT/s + uint32_t latency; // core cycles added after a read's data departs + uint32_t queue_size; // requests in flight per channel; 0 = unbounded + }; + + explicit DramBankSim(const Config& cfg) : DramBankSim(cfg, DramBankModel::Config()) {} + + // `policy` gives the controller's queues, horizon and write policy. + DramBankSim(const Config& cfg, const DramBankModel::Config& policy) + : cfg_(cfg) + , model_cfg_(DramBankModel::Config::make(cfg.type, cfg.data_rate, cfg.block_size)) { + model_cfg_.read_queue = policy.read_queue; + model_cfg_.write_queue = policy.write_queue; + model_cfg_.wr_high = policy.wr_high; + model_cfg_.wr_low = policy.wr_low; + model_cfg_.horizon = policy.horizon; + for (uint32_t i = 0; i < cfg.num_channels; ++i) { + channels_.emplace_back(model_cfg_); + } + this->reset(); + } + + void reset() { + cycle_ = 0; + credit_ = 0; + for (auto& ch : channels_) { + ch.model.reset(); + ch.read_rsps.clear(); + ch.write_rsps.clear(); + ch.inflight = 0; + } + requests_.clear(); + free_.clear(); + } + + void tick() { + ++cycle_; + credit_ += cfg_.data_rate; + for (auto& ch : channels_) { + this->drain(ch.read_rsps, ch); + this->drain(ch.write_rsps, ch); + } + while (credit_ >= 2ull * cfg_.core_clock) { + for (auto& ch : channels_) { + ch.model.tick( + [&](uint32_t tag) { ch.read_rsps.push_back({tag, cycle_ + 1 + cfg_.latency}); }, + [&](uint32_t tag) { ch.write_rsps.push_back({tag, cycle_}); }); + } + credit_ -= 2ull * cfg_.core_clock; + } + } + + bool can_accept(uint32_t channel) const { + return (cfg_.queue_size == 0) || (channels_.at(channel).inflight < cfg_.queue_size); + } + + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback cb, void* arg) { + uint32_t tag; + if (!free_.empty()) { + tag = free_.back(); + free_.pop_back(); + requests_[tag] = {cb, arg}; + } else { + tag = uint32_t(requests_.size()); + requests_.push_back({cb, arg}); + } + auto& ch = channels_.at(channel); + ch.model.push(tag, is_write, addr & ~uint64_t(cfg_.block_size - 1)); + ++ch.inflight; + } + + const DramBankModel& channel(uint32_t i) const { return channels_.at(i).model; } + +private: + struct Rsp { + uint32_t tag; + uint64_t ready; + }; + + struct Channel { + explicit Channel(const DramBankModel::Config& c) : model(c) {} + DramBankModel model; + std::deque read_rsps, write_rsps; + uint32_t inflight = 0; + }; + + struct Req { + ResponseCallback cb; + void* arg; + }; + + void drain(std::deque& rsps, Channel& ch) { + while (!rsps.empty() && rsps.front().ready <= cycle_) { + Req& req = requests_[rsps.front().tag]; + if (req.cb && !req.cb(req.arg)) { + break; + } + free_.push_back(rsps.front().tag); + --ch.inflight; + rsps.pop_front(); + } + } + + Config cfg_; + DramBankModel::Config model_cfg_; + std::vector channels_; + std::vector requests_; + std::vector free_; + uint64_t cycle_; + uint64_t credit_; +}; + +} // namespace vortex diff --git a/sim/common/dram_sim.cpp b/sim/common/dram_sim.cpp index 81f7aafcc0..dbb8a05c3c 100644 --- a/sim/common/dram_sim.cpp +++ b/sim/common/dram_sim.cpp @@ -12,8 +12,11 @@ // limitations under the License. #include "dram_sim.h" +#include "dram_bank_model.h" #include "util.h" #include +#include +#include #include #include #include @@ -51,8 +54,44 @@ void set_timing(YAML::Node timing, uint32_t data_rate, const dram_timing::clock_ } +// A DRAM backend: Ramulator, or the bank model (sim/common/dram_bank_model.h) +// with VORTEX_DRAM_MODEL=bank -- a choice of simulator, not of machine. class DramSim::Impl { +public: + virtual ~Impl() {} + virtual void reset() = 0; + virtual void tick() = 0; + virtual bool can_accept(uint32_t channel) const = 0; + virtual void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback cb, void* arg) = 0; +}; + +namespace { + +class BankDram : public DramSim::Impl { +public: + explicit BankDram(const DramSim::Config& c) + : sim_(DramBankSim::Config{c.type, c.num_channels, c.block_size, c.core_clock, c.data_rate, + c.latency, c.queue_size}) {} + void reset() override { sim_.reset(); } + void tick() override { sim_.tick(); } + bool can_accept(uint32_t channel) const override { return sim_.can_accept(channel); } + void send_request(uint32_t channel, uint64_t addr, bool is_write, DramSim::ResponseCallback cb, + void* arg) override { + sim_.send_request(channel, addr, is_write, cb, arg); + } + +private: + DramBankSim sim_; +}; + +} + +class RamulatorDram : public DramSim::Impl { private: + using Config = DramSim::Config; + using Type = DramSim::Type; + using ResponseCallback = DramSim::ResponseCallback; + // One request (a block); it completes when all its transactions have. struct request_t { ResponseCallback callback; @@ -225,7 +264,7 @@ class DramSim::Impl { } public: - Impl(const Config& config) + RamulatorDram(const Config& config) : config_(config) , channels_(config.num_channels) { auto geometry = dram_timing::geometry(config.type); @@ -243,7 +282,7 @@ class DramSim::Impl { this->reset(); } - ~Impl() { + ~RamulatorDram() override { std::ofstream nullstream("ramulator.stats.log"); auto original_buf = std::cout.rdbuf(); std::cout.rdbuf(nullstream.rdbuf()); @@ -254,7 +293,7 @@ class DramSim::Impl { std::cout.rdbuf(original_buf); } - void reset() { + void reset() override { cycle_ = 0; clock_credit_ = 0; for (auto& channel : channels_) { @@ -265,7 +304,7 @@ class DramSim::Impl { } } - void tick() { + void tick() override { ++cycle_; clock_credit_ += config_.data_rate; for (auto& channel : channels_) { @@ -281,11 +320,11 @@ class DramSim::Impl { } } - bool can_accept(uint32_t channel) const { + bool can_accept(uint32_t channel) const override { return (config_.queue_size == 0) || (channels_.at(channel).inflight < config_.queue_size); } - void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) override { // A block wider than a burst is several transactions. uint32_t n = (config_.block_size > tx_size_) ? (config_.block_size / tx_size_) : 1; uint64_t base = addr & ~uint64_t(config_.block_size - 1); @@ -323,8 +362,19 @@ DramSim::Config DramSim::platform_config() { return config; } +static DramSim::Impl* make_backend(const DramSim::Config& config) { + const char* model = getenv("VORTEX_DRAM_MODEL"); + if (model == nullptr || model[0] == '\0' || !strcmp(model, "ramulator")) { + return new RamulatorDram(config); + } + if (!strcmp(model, "bank")) { + return new BankDram(config); + } + throw std::invalid_argument(std::string("dram_sim: unknown VORTEX_DRAM_MODEL ") + model); +} + DramSim::DramSim(const Config& config) - : impl_(new Impl(config)) + : impl_(make_backend(config)) {} DramSim::~DramSim() { diff --git a/sim/common/dram_sim.h b/sim/common/dram_sim.h index 4a5c4cf21d..cf260c132b 100644 --- a/sim/common/dram_sim.h +++ b/sim/common/dram_sim.h @@ -54,8 +54,10 @@ class DramSim { // addr: byte address within the channel. void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg); -private: + // The backend: Ramulator, or with VORTEX_DRAM_MODEL=bank the bank model. class Impl; + +private: Impl* impl_; }; diff --git a/tests/unittest/Makefile b/tests/unittest/Makefile index 33554509cd..66626ae757 100644 --- a/tests/unittest/Makefile +++ b/tests/unittest/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../..) include $(ROOT_DIR)/config.mk -PROJECTS := vx_malloc linked_list dram_sim dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt +PROJECTS := vx_malloc linked_list dram_sim dram_bank_model dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt all: $(foreach p,$(PROJECTS),$(MAKE) -C $(p) &&) true diff --git a/tests/unittest/dram_bank_model/Makefile b/tests/unittest/dram_bank_model/Makefile new file mode 100644 index 0000000000..243cb3366b --- /dev/null +++ b/tests/unittest/dram_bank_model/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := dram_bank_model + +SRC_DIR := $(VORTEX_HOME)/tests/unittest/$(PROJECT) +RAMULATOR_DIR := $(THIRD_PARTY_DIR)/ramulator + +SRCS := $(SRC_DIR)/main.cpp $(SIM_COMMON_DIR)/dram_sim.cpp + +CXXFLAGS += -I$(SIM_COMMON_DIR) +CXXFLAGS += -isystem $(RAMULATOR_DIR)/ext/spdlog/include -isystem $(RAMULATOR_DIR)/ext/yaml-cpp/include -isystem $(RAMULATOR_DIR)/src +LDFLAGS += -Wl,-rpath,$(RAMULATOR_DIR) -L$(RAMULATOR_DIR) -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state + +include ../common.mk diff --git a/tests/unittest/dram_bank_model/main.cpp b/tests/unittest/dram_bank_model/main.cpp new file mode 100644 index 0000000000..2408dad63f --- /dev/null +++ b/tests/unittest/dram_bank_model/main.cpp @@ -0,0 +1,217 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The bank model (sim/common/dram_bank_model.h) against Ramulator (DramSim) on +// the same closed-loop traffic: per channel, `window` requests outstanding, +// reads until their response, writes until acknowledged. Both take the +// traffic through DramSim's interface, so the clock loop, ingress, fixed +// latency and response drains are the same and only the scheduling differs. +// +// The gate: an idle channel's reads (one outstanding) take the same latency in +// both, read for read, on every configuration -- row hits, misses and +// conflicts. Loaded traffic is reported: bandwidth, mean / p50 / p99 read +// latency, the Kolmogorov-Smirnov distance between the latency distributions, +// and the bank model's row-hit rate. +// +// ./dram_bank_model the gate, on short runs +// ./dram_bank_model full [cfg] the full table (20,000 requests per cell) + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace vortex; + +namespace { + +struct Machine { + const char* name; + DramSim::Config cfg; +}; + +// The default machine and the profiles' channels (profiles/*.json), one +// channel each unless named otherwise. +const Machine kMachines[] = { + {"default", {DramSim::Type::HBM2, 1, 64, 400, 800, 0, 0}}, + {"default4", {DramSim::Type::HBM2, 4, 64, 400, 800, 0, 0}}, + {"v100", {DramSim::Type::HBM2, 1, 64, 1530, 1752, 0, 0}}, + {"a100", {DramSim::Type::HBM2, 1, 64, 1410, 3038, 190, 257}}, + {"h100", {DramSim::Type::HBM3, 1, 64, 1980, 6545, 0, 0}}, + {"rtx1080", {DramSim::Type::GDDR6, 1, 64, 1607, 10000, 0, 0}}, + {"rtx5080", {DramSim::Type::GDDR6, 1, 64, 2295, 15000, 339, 0}}, +}; + +struct Rec { + uint64_t sent, done; + bool w; +}; + +uint64_t g_cycle = 0; + +bool on_rsp(void* arg) { + static_cast(arg)->done = g_cycle; + return true; +} + +struct Pattern { + const char* name; + std::function block; + double write_frac; +}; + +struct Result { + double bw = 0, mean = 0, p50 = 0, p99 = 0; + std::vector lat; +}; + +template +Result run(M& m, const DramSim::Config& cfg, const Pattern& p, uint32_t window, uint32_t n) { + std::mt19937_64 rng(12345), rng_w(777); + std::vector recs(uint64_t(n) * cfg.num_channels); + std::vector sent(cfg.num_channels, 0); + std::vector> live(cfg.num_channels); + const uint64_t total = recs.size(); + uint64_t done = 0; + g_cycle = 0; + std::uniform_real_distribution u(0, 1); + while (done < total) { + for (uint32_t ch = 0; ch < cfg.num_channels; ++ch) { + auto& l = live[ch]; + for (size_t i = 0; i < l.size();) { + if (l[i]->done != ~0ull) { + l[i] = l.back(); + l.pop_back(); + ++done; + } else { + ++i; + } + } + if (sent[ch] < n && l.size() < window && m.can_accept(ch)) { + Rec* r = &recs[uint64_t(ch) * n + sent[ch]]; + r->sent = g_cycle; + r->done = ~0ull; + r->w = u(rng_w) < p.write_frac; + m.send_request(ch, p.block(sent[ch], rng) * cfg.block_size, r->w, on_rsp, r); + l.push_back(r); + ++sent[ch]; + } + } + m.tick(); + ++g_cycle; + } + Result res; + res.bw = double(total) * cfg.block_size / double(g_cycle); + for (auto& r : recs) { + if (!r.w) res.lat.push_back(r.done - r.sent); + } + std::sort(res.lat.begin(), res.lat.end()); + double s = 0; + for (auto v : res.lat) s += double(v); + if (!res.lat.empty()) { + res.mean = s / double(res.lat.size()); + res.p50 = double(res.lat[res.lat.size() / 2]); + res.p99 = double(res.lat[std::min(res.lat.size() - 1, size_t(0.99 * double(res.lat.size())))]); + } + return res; +} + +double ks_distance(const std::vector& a, const std::vector& b) { + if (a.empty() || b.empty()) return 0; + double ks = 0; + size_t i = 0, j = 0; + while (i < a.size() && j < b.size()) { + uint64_t v = std::min(a[i], b[j]); + while (i < a.size() && a[i] <= v) ++i; + while (j < b.size() && b[j] <= v) ++j; + ks = std::max(ks, std::fabs(double(i) / double(a.size()) - double(j) / double(b.size()))); + } + return ks; +} + +DramBankSim::Config bank_config(const DramSim::Config& c) { + return DramBankSim::Config{c.type, c.num_channels, c.block_size, c.core_clock, c.data_rate, + c.latency, c.queue_size}; +} + +std::vector patterns() { + const uint64_t blocks = (256ull << 20) / 64; + return { + {"seq", [](uint64_t i, std::mt19937_64&) { return i; }, 0.0}, + {"rand", [blocks](uint64_t, std::mt19937_64& r) { return r() % blocks; }, 0.0}, + // Four sequential streams 1 MB apart, interleaved. + {"strided4", [](uint64_t i, std::mt19937_64&) { return (i % 4) * (1ull << 14) + i / 4; }, 0.0}, + // Blocks in random rows among 64 that all map to one bank. + {"conflict", [](uint64_t i, std::mt19937_64& r) { + uint64_t row = r() % 64; + return (row * 33 << 15) / 64 + (i % 2); + }, 0.0}, + {"rand_w30", [blocks](uint64_t, std::mt19937_64& r) { return r() % blocks; }, 0.3}, + {"seq_w50", [](uint64_t i, std::mt19937_64&) { return i; }, 0.5}, + }; +} + +} // namespace + +int main(int argc, char** argv) { + const bool full = argc > 1 && !std::strcmp(argv[1], "full"); + const char* only = (full && argc > 2) ? argv[2] : nullptr; + const uint32_t n = full ? 20000 : 2000; + const std::vector windows = full ? std::vector{1, 4, 16, 64} + : std::vector{1, 16}; + int failures = 0; + printf("%-9s %-9s %4s | %7s %7s %6s %6s | %7s %7s %6s %6s | %6s %6s %5s %5s\n", "machine", + "pattern", "win", "ref B/c", "mean", "p50", "p99", "mod B/c", "mean", "p50", "p99", "dBW%", + "dMean%", "KS", "hit%"); + for (const auto& mach : kMachines) { + if (only && std::strcmp(only, mach.name)) continue; + for (const auto& p : patterns()) { + for (uint32_t w : windows) { + DramSim ref(mach.cfg); + ref.reset(); + Result a = run(ref, mach.cfg, p, w, n); + DramBankSim mod(bank_config(mach.cfg)); + Result b = run(mod, mach.cfg, p, w, n); + uint64_t hits = 0, ops = 0; + for (uint32_t ch = 0; ch < mach.cfg.num_channels; ++ch) { + const auto& st = mod.channel(ch).stats(); + hits += st.hits; + ops += st.hits + st.misses + st.conflicts; + } + const double ks = ks_distance(a.lat, b.lat); + printf("%-9s %-9s %4u | %7.2f %7.1f %6.0f %6.0f | %7.2f %7.1f %6.0f %6.0f | %6.1f %6.1f %5.3f %5.1f\n", + mach.name, p.name, w, a.bw, a.mean, a.p50, a.p99, b.bw, b.mean, b.p50, b.p99, + 100.0 * (b.bw - a.bw) / a.bw, a.mean ? 100.0 * (b.mean - a.mean) / a.mean : 0.0, ks, + ops ? 100.0 * double(hits) / double(ops) : 0.0); + // One read outstanding on an idle channel: every latency the same. + if (w == 1 && p.write_frac == 0 && a.lat != b.lat) { + printf("FAIL %s %s: idle read latencies differ\n", mach.name, p.name); + ++failures; + } + } + } + } + if (failures) { + printf("FAILED: %d\n", failures); + return 1; + } + printf("PASSED!\n"); + return 0; +} From 249c77e7a3b91069e8c08121fc3902ea91793e16 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 03:25:47 -0700 Subject: [PATCH 222/336] rtlsim: AXI_MEM_MODEL=bank times the AXI4 ports with the DRAM bank model DramBankAxi puts the bank model behind one AXI4 port in AxiLatencyPipe's style (outputs from registered state, step() after the edge): a read takes the lowest free of 32 slots until R takes it and reaches the channel in the next cycle; an AW and its W form a write, acknowledged on B once the controller buffered it; reads are ready LATENCY + 1 cycles after the cycle whose DRAM clocks their data departs in, and leave in that order; at most QUEUE_SIZE requests are in flight. It is the normative face of Foldsim's memory endpoint (Foldsim's own read and write slots). rtlsim's AXI_BUS=1 keeps the latency-bandwidth pipe by default (AXI_MEM_MODEL=pipe); AXI_MEM_MODEL=bank gives each port a channel configured from VX_CFG_PLATFORM_MEMORY_*, with read data kept per slot. On the Foldsim target configuration (s2w4i2, one port, 4 id bits): demo 4888 instructions / 10748 cycles, sgemm 336928 / 399345, PASSED; the pipe's 8710 / 415462 unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 81db409d8fa11fd3e5867bbf29770a35ec26aa0d) --- sim/common/dram_bank_model.h | 206 ++++++++++++++++++++++++++++++++++- sim/rtlsim/Makefile | 8 +- sim/rtlsim/processor.cpp | 125 ++++++++++++++++++++- 3 files changed, 333 insertions(+), 6 deletions(-) diff --git a/sim/common/dram_bank_model.h b/sim/common/dram_bank_model.h index 68a628a740..26f8bf852b 100644 --- a/sim/common/dram_bank_model.h +++ b/sim/common/dram_bank_model.h @@ -44,8 +44,9 @@ namespace vortex { // Reservations reach at most `horizon` clocks ahead: a request that would // reach further waits. // -// This is the normative model; the face below puts it behind DramSim's -// interface. +// This is the normative model: foldsim's memory endpoint +// (hw/rtl/afu/foldsim/foldsim_mem_bank.sv) implements the same decisions, and +// the faces below put it behind DramSim's interface and behind an AXI4 port. class DramBankModel { public: struct Config { @@ -575,4 +576,205 @@ class DramBankSim { uint64_t credit_; }; +/////////////////////////////////////////////////////////////////////////////// + +// The bank model as one AXI4 memory port, in AxiLatencyPipe's style +// (sim/common/axi_latency_pipe.h): every output is a function of registered +// state, the caller samples the request side before the clock edge and calls +// step() after it. Single-beat transactions; a burst sets the error, and from +// the next cycle on the port is ready for nothing. +// +// A read takes the lowest free read slot from acceptance until R takes it; it +// reaches the channel in the cycle after it was accepted, and is ready to +// drive R LATENCY + 1 cycles after the cycle whose DRAM clocks its data +// departed in. An AW and its W, in either order, form a write, which reaches +// the channel in the same way and is acknowledged on B from the cycle after +// the controller buffered it. Requests reach the channel in the order they +// formed, a write before a read of the same cycle. +// R and B take the oldest ready response. A request counts against `queue` +// from acceptance until it is ready. +class DramBankAxi { +public: + struct Config { + DramBankModel::Config model; + uint32_t core_clock = 400; // MHz + uint32_t data_rate = 800; // MT/s + uint32_t latency = 0; // core cycles + uint32_t queue = 0; // requests in flight; 0 = only the slots bound them + uint32_t read_slots = 32; + uint32_t write_slots = 16; // AWs, and Ws, accepted and not acknowledged on B + }; + + struct Inputs { + bool ar_valid = false; + uint32_t ar_id = 0; + uint64_t ar_addr = 0; + uint32_t ar_len = 0; + bool aw_valid = false; + uint32_t aw_id = 0; + uint64_t aw_addr = 0; + uint32_t aw_len = 0; + bool w_valid = false; + bool w_last = true; + bool r_ready = false; + bool b_ready = false; + }; + + explicit DramBankAxi(const Config& cfg) : cfg_(cfg), model_(cfg.model) { this->reset(); } + + const DramBankModel& model() const { return model_; } + + void reset() { + model_.reset(); + cycle_ = 0; + credit_ = 0; + error_ = false; + inflight_ = 0; + slot_busy_.assign(cfg_.read_slots, false); + slot_id_.assign(cfg_.read_slots, 0); + slots_used_ = 0; + pending_aw_ = 0; + pending_w_ = 0; + aw_queue_.clear(); + ready_reads_.clear(); + uncounted_ = 0; + acks_.clear(); + cur_r_valid_ = false; + cur_b_valid_ = false; + } + + bool ar_ready() const { return !error_ && this->room() && slots_used_ < cfg_.read_slots; } + bool aw_ready() const { return !error_ && this->room() && pending_aw_ < cfg_.write_slots; } + bool w_ready() const { return !error_ && pending_w_ < cfg_.write_slots; } + + bool r_valid() const { return cur_r_valid_; } + uint32_t r_id() const { return cur_r_id_; } + uint32_t r_slot() const { return cur_r_slot_; } + bool b_valid() const { return cur_b_valid_; } + uint32_t b_id() const { return cur_b_id_; } + bool error() const { return error_; } + // The slot a read accepted this cycle takes: the lowest free once R has + // taken its read, if it does. + uint32_t ar_slot(bool r_ready) const { + for (uint32_t s = 0; s < cfg_.read_slots; ++s) { + if (!slot_busy_[s] || (r_ready && cur_r_valid_ && s == cur_r_slot_)) return s; + } + return cfg_.read_slots; + } + // Nothing accepted is unanswered. + bool idle() const { + return slots_used_ == 0 && pending_aw_ == 0 && pending_w_ == 0; + } + + void step(const Inputs& in) { + const bool ar_hs = in.ar_valid && this->ar_ready(); + const bool aw_hs = in.aw_valid && this->aw_ready(); + const bool w_hs = in.w_valid && this->w_ready(); + const bool burst = (ar_hs && in.ar_len != 0) || (aw_hs && in.aw_len != 0) || (w_hs && !in.w_last); + const bool ar_fire = ar_hs && in.ar_len == 0; + const bool aw_fire = aw_hs && in.aw_len == 0; + const bool w_fire = w_hs && in.w_last; + const bool r_fire = cur_r_valid_ && in.r_ready; + const bool b_fire = cur_b_valid_ && in.b_ready; + + // This cycle's DRAM clocks, on what was accepted before it. + credit_ += cfg_.data_rate; + while (credit_ >= 2ull * cfg_.core_clock) { + model_.tick( + [&](uint32_t slot) { ready_reads_.push_back({slot, cycle_ + 1 + cfg_.latency}); }, + [&](uint32_t id) { acks_.push_back(id); ++uncounted_acks_; }); + credit_ -= 2ull * cfg_.core_clock; + } + + // Responses: the stage driving R and B, then what is ready next cycle. + if (r_fire) { + slot_busy_[cur_r_slot_] = false; + --slots_used_; + cur_r_valid_ = false; + } + if (b_fire) { + cur_b_valid_ = false; + } + const uint64_t next = cycle_ + 1; + while (uncounted_ < ready_reads_.size() && ready_reads_[uncounted_].ready <= next) { + ++uncounted_; + --inflight_; + } + inflight_ -= uncounted_acks_; + uncounted_acks_ = 0; + if (!cur_r_valid_ && uncounted_ != 0) { + cur_r_valid_ = true; + cur_r_slot_ = ready_reads_.front().slot; + cur_r_id_ = slot_id_[cur_r_slot_]; + ready_reads_.pop_front(); + --uncounted_; + } + if (!cur_b_valid_ && !acks_.empty()) { + cur_b_valid_ = true; + cur_b_id_ = acks_.front(); + acks_.pop_front(); + } + + // Requests: writes form from an AW and its W, then join the channel ahead + // of this cycle's read. + const bool new_write = (pending_w_ > pending_aw_ && aw_fire) + || (pending_w_ < pending_aw_ && w_fire) + || (aw_fire && w_fire); + if (aw_fire) { + aw_queue_.push_back({in.aw_id, in.aw_addr}); + } + if (new_write) { + auto aw = aw_queue_.front(); + aw_queue_.pop_front(); + model_.push(aw.id, true, aw.addr); + } + if (ar_fire) { + uint32_t s = 0; + while (slot_busy_[s]) ++s; + slot_busy_[s] = true; + slot_id_[s] = in.ar_id; + ++slots_used_; + model_.push(s, false, in.ar_addr); + } + inflight_ += uint32_t(ar_fire) + uint32_t(aw_fire); + pending_aw_ = pending_aw_ + uint32_t(aw_fire) - uint32_t(b_fire && pending_aw_ != 0); + pending_w_ = pending_w_ + uint32_t(w_fire) - uint32_t(b_fire && pending_w_ != 0); + if (burst) { + error_ = true; + } + ++cycle_; + } + +private: + struct Ready { + uint32_t slot; + uint64_t ready; + }; + + struct Aw { + uint32_t id; + uint64_t addr; + }; + + bool room() const { return cfg_.queue == 0 || inflight_ < cfg_.queue; } + + Config cfg_; + DramBankModel model_; + uint64_t cycle_; + uint64_t credit_; + bool error_; + uint32_t inflight_; + std::vector slot_busy_; + std::vector slot_id_; + uint32_t slots_used_; + uint32_t pending_aw_, pending_w_; + std::deque aw_queue_; + std::deque ready_reads_; // reads departed; the first `uncounted_` are ready next cycle + size_t uncounted_; + std::deque acks_; + uint32_t uncounted_acks_ = 0; + bool cur_r_valid_, cur_b_valid_; + uint32_t cur_r_id_ = 0, cur_r_slot_ = 0, cur_b_id_ = 0; +}; + } // namespace vortex diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index 9ece4088e8..f3eaf2bb23 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -134,10 +134,16 @@ SRCS += $(DPI_DIR)/dpi_util.cpp $(DPI_DIR)/dpi_float.cpp SRCS += $(SRC_DIR)/processor.cpp # AXI_BUS=1 simulates Vortex behind its AXI4 adapter, as the FPGA platforms -# instantiate it, against a latency-bandwidth-pipe memory model. +# instantiate it, against a latency-bandwidth-pipe memory model, or with +# AXI_MEM_MODEL=bank the DRAM bank model (sim/common/dram_bank_model.h). ifdef AXI_BUS TOP = rtlsim_axi_shim CXXFLAGS += -DAXI_BUS +ifeq ($(AXI_MEM_MODEL), bank) +CXXFLAGS += -DAXI_MEM_BANK +else ifneq ($(filter-out pipe,$(AXI_MEM_MODEL)),) +$(error AXI_MEM_MODEL must be pipe or bank (got '$(AXI_MEM_MODEL)')) +endif else TOP = rtlsim_shim endif diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index ac4602049e..24d9528bb6 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -17,6 +17,7 @@ #ifdef AXI_BUS #include "Vrtlsim_axi_shim.h" #include +#include typedef Vrtlsim_axi_shim Device; #else #include "Vrtlsim_shim.h" @@ -51,6 +52,7 @@ typedef Vrtlsim_shim Device; #include #include #include +#include #include #include #include @@ -60,7 +62,9 @@ typedef Vrtlsim_shim Device; #include -// AXI_BUS memory timing; the defaults are FASED's. +// AXI_BUS memory timing: a latency-bandwidth pipe, by default with FASED's +// settings, or (AXI_MEM_BANK) a DRAM channel per port as the bank model +// times it, from the platform memory's configuration. #ifndef AXI_MEM_READ_LATENCY #define AXI_MEM_READ_LATENCY 30 #endif @@ -471,7 +475,116 @@ class Processor::Impl { ++timestamp; } -#ifdef AXI_BUS +#if defined(AXI_BUS) && defined(AXI_MEM_BANK) + static constexpr uint32_t AXI_BEAT_SIZE = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + + // The read and write slots of foldsim's memory endpoint, whose timing this + // is cycle for cycle. + static DramBankAxi::Config axi_bank_config() { + auto dram = DramSim::platform_config(); + DramBankAxi::Config c; + c.model = DramBankModel::Config::make(dram.type, dram.data_rate, dram.block_size); + c.core_clock = dram.core_clock; + c.data_rate = dram.data_rate; + c.latency = dram.latency; + c.queue = dram.queue_size; + c.read_slots = 32; + c.write_slots = 16; + return c; + } + + // The address within the port's channel, as dram_sim's banks see it. + static uint64_t axi_channel_addr(uint64_t addr) { + #if VX_CFG_PLATFORM_MEMORY_INTERLEAVE == 1 + return (addr / AXI_BEAT_SIZE / VX_CFG_PLATFORM_MEMORY_NUM_BANKS) * AXI_BEAT_SIZE; + #else + constexpr uint32_t bank_addr_bits = VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + return addr & ((uint64_t(1) << (bank_addr_bits + log2ceil(AXI_BEAT_SIZE))) - 1); + #endif + } + + void mem_bus_reset() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + axi_mem_[b] = std::make_unique(axi_bank_config()); + axi_wr_addr_[b].clear(); + axi_rd_data_[b].assign(32, std::vector(AXI_BEAT_SIZE)); + axi_in_[b] = DramBankAxi::Inputs(); + } + this->axi_drive(); + } + + // Samples each port's request side before the clock edge and performs the + // accepted transactions on memory: writes first, then reads, each read's + // data kept in its slot. + void axi_sample() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& mem = *axi_mem_[b]; + auto& in = axi_in_[b]; + in.ar_valid = device_->m_axi_arvalid[b]; + in.ar_id = device_->m_axi_arid[b]; + in.ar_addr = axi_channel_addr(device_->m_axi_araddr[b]); + in.ar_len = device_->m_axi_arlen[b]; + in.aw_valid = device_->m_axi_awvalid[b]; + in.aw_id = device_->m_axi_awid[b]; + in.aw_addr = axi_channel_addr(device_->m_axi_awaddr[b]); + in.aw_len = device_->m_axi_awlen[b]; + in.w_valid = device_->m_axi_wvalid[b]; + in.w_last = device_->m_axi_wlast[b]; + in.r_ready = device_->m_axi_rready[b]; + in.b_ready = device_->m_axi_bready[b]; + assert(!(in.ar_valid && in.ar_len != 0) && !(in.aw_valid && in.aw_len != 0)); + + if (in.aw_valid && mem.aw_ready()) { + axi_wr_addr_[b].push_back(uint64_t(device_->m_axi_awaddr[b])); + } + if (in.w_valid && mem.w_ready()) { + assert(!axi_wr_addr_[b].empty()); + uint64_t addr = axi_wr_addr_[b].front(); + auto data = VDataCast::get(device_->m_axi_wdata[b]); + uint64_t strb = device_->m_axi_wstrb[b]; + for (uint32_t i = 0; i < AXI_BEAT_SIZE; ++i) { + if ((strb >> i) & 0x1) { + (*ram_)[addr + i] = data[i]; + } + } + axi_wr_addr_[b].pop_front(); + } + if (in.ar_valid && mem.ar_ready()) { + ram_->read(axi_rd_data_[b].at(mem.ar_slot(in.r_ready)).data(), + uint64_t(device_->m_axi_araddr[b]), AXI_BEAT_SIZE); + } + } + } + + void axi_step() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + axi_mem_[b]->step(axi_in_[b]); + } + this->axi_drive(); + } + + void axi_drive() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& mem = *axi_mem_[b]; + device_->m_axi_arready[b] = mem.ar_ready(); + device_->m_axi_awready[b] = mem.aw_ready(); + device_->m_axi_wready[b] = mem.w_ready(); + device_->m_axi_rvalid[b] = mem.r_valid(); + device_->m_axi_rresp[b] = 0; + if (mem.r_valid()) { + device_->m_axi_rid[b] = mem.r_id(); + device_->m_axi_rlast[b] = 1; + memcpy(VDataCast::get(device_->m_axi_rdata[b]), + axi_rd_data_[b].at(mem.r_slot()).data(), AXI_BEAT_SIZE); + } + device_->m_axi_bvalid[b] = mem.b_valid(); + device_->m_axi_bresp[b] = 0; + if (mem.b_valid()) { + device_->m_axi_bid[b] = mem.b_id(); + } + } + } +#elif defined(AXI_BUS) static constexpr uint32_t AXI_BEAT_SIZE = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; void mem_bus_reset() { @@ -729,7 +842,13 @@ class Processor::Impl { std::unordered_map print_bufs_; -#ifdef AXI_BUS +#if defined(AXI_BUS) && defined(AXI_MEM_BANK) + std::array, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_mem_; + std::array axi_in_; + std::array, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_wr_addr_; + // Read data by slot. + std::array>, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_rd_data_; +#elif defined(AXI_BUS) std::array axi_mem_; std::array axi_in_; std::array axi_rlast_fire_; From 60de49790f9b6e46530234b65396da025be4610e Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 03:28:10 -0700 Subject: [PATCH 223/336] foldsim: the memory endpoint times its port as a DRAM channel's banks foldsim_mem keeps the latency-bandwidth pipe as its reset model and adds the bank model (foldsim_mem_bank.sv), cycle for cycle DramBankAxi of sim/common/dram_bank_model.h. A reset command's address chooses the model the target has from that reset on (MEM_MODEL reads it back), so a run on the pipe makes exactly the register accesses and takes exactly the cycles it did: s2w4i2 in the in-process simulator, demo 4888 instructions / 8702 cycles / 15559 target / 125233 host cycles and sgemm 336928 / 415454 / 422288 / 3379066, as before. The bank model: a read holds one of 32 slots, its HBM data kept in the slot, and responses leave in departure order; a target cycle's DRAM clocks run in host cycles while the target evaluates it and the cycle finishes once they have. The calendars are bitmaps over a ring of 512 DRAM clocks in distributed RAM (foldsim_mem_bmap.sv): the command and row buses, and per pseudo-channel and bank group the clocks a read, write or ACT may not take; each reserved command marks what it forbids, so placing one is a first-free search of 64 clocks a host cycle, and a word is cleared by a valid bit as the clock passes it. A clock without a reservation is one host cycle, a reservation about ten. Bank-table times are 16-bit, moved up before they could wrap. tFAW is left to tRRD (exact wherever 4 tRRD_S >= tFAW: HBM2, HBM3); the runtime refuses a memory where it is not (GDDR6). Registers (FOLDSIM_VERSION 3, new blocks fully populated): MEM_MODEL, MEM_CAPS, MEM_CLOCK, MEM_LATENCY, MEM_QUEUE, MEM_GEOM, MEM_WMARK at 0xD8-0xF0, timings at 0x800, counters (reads, writes, row hits, misses, conflicts, summed read latency) at 0xC00. With FOLDSIM_MEM_MODEL=bank the runtime programs them from VX_CFG_PLATFORM_MEMORY_* and dram_timing.h and prints the counters at close. The pipe's write RAM (wp_release) is written from a register the host cycle after the target cycle, reads forwarded (Fmax proposal 6.4), so its write enable no longer ends the path through the write's formation; the token ends stay unstaged (staging adds host cycles). hw/unittest/foldsim_mem now compares every target cycle's outputs with the C++ models (AxiLatencyPipe; DramBankAxi) besides its reset, burst and HBM protocol checks: 2 pipe and 12 bank-model settings (HBM2/HBM3 at default, v100, a100, h100 rates; small queues; hits, random rows, one bank to the horizon, streams, read- and write-heavy), ~346k target cycles, 0 differences. In the in-process simulator the bank model runs demo (4888 / 10740 cycles) and sgemm (336928 / 399337), equal to rtlsim on the same model (AXI_MEM_MODEL=bank) but for rtlsim's 8-cycle launch settle, the same as on the pipe. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8b3a3eaf3889127b505f3ed9e4f111fc4b0ba8b1) --- hw/rtl/afu/foldsim/foldsim_afu.sv | 39 +- hw/rtl/afu/foldsim/foldsim_ctrl.sv | 60 +- hw/rtl/afu/foldsim/foldsim_mem.sv | 284 ++++++-- hw/rtl/afu/foldsim/foldsim_mem_bank.sv | 910 +++++++++++++++++++++++++ hw/rtl/afu/foldsim/foldsim_mem_bmap.sv | 79 +++ hw/rtl/afu/foldsim/foldsim_regs.vh | 13 +- hw/syn/foldsim/Makefile | 3 +- hw/unittest/foldsim_mem/Makefile | 2 +- hw/unittest/foldsim_mem/main.cpp | 430 +++++++++--- sim/foldsim/Makefile | 3 +- sw/common/foldsim_regs.h | 40 +- sw/runtime/foldsim/vortex.cpp | 103 ++- 12 files changed, 1800 insertions(+), 166 deletions(-) create mode 100644 hw/rtl/afu/foldsim/foldsim_mem_bank.sv create mode 100644 hw/rtl/afu/foldsim/foldsim_mem_bmap.sv diff --git a/hw/rtl/afu/foldsim/foldsim_afu.sv b/hw/rtl/afu/foldsim/foldsim_afu.sv index cfe9ff1a95..7464ece020 100644 --- a/hw/rtl/afu/foldsim/foldsim_afu.sv +++ b/hw/rtl/afu/foldsim/foldsim_afu.sv @@ -173,16 +173,30 @@ module foldsim_afu #( .name``_valid (m_axi_mem_0_``name``_valid), .name``_ready (m_axi_mem_0_``name``_ready), \ .name``_bits (m_axi_mem_0_``name``_bits) + // The memory endpoint's bank model: read slots, and a ring of 2^9 DRAM + // clocks (MEM_CAPS reports both). + localparam MEM_SLOTS = 32; + localparam MEM_HLOG2 = 9; + localparam [31:0] MEM_CAPS = {4'd0, 4'd1, 4'd1, 8'(MEM_SLOTS), 4'(MEM_HLOG2 - 4), 8'd1}; + wire [31:0] mem_read_latency, mem_write_latency; wire [7:0] mem_max_reads, mem_max_writes; wire [63:0] mem_base; wire mem_idle, mem_error, mem_stall; wire mem_reset_valid, mem_reset_ready, mem_reset_bits; + wire mem_model_bank; + wire [31:0] mem_clock, mem_queue; + wire [15:0] mem_latency, mem_wmark; + wire [12:0] mem_geom; + wire [18*12-1:0] mem_timing; + wire [6*64-1:0] mem_stats; foldsim_mem #( .ID_W (TARGET_ID_W), .ADDR_W (TARGET_ADDR_W), - .DATA_W (DATA_W) + .DATA_W (DATA_W), + .SLOTS (MEM_SLOTS), + .HLOG2 (MEM_HLOG2) ) mem ( .clk (ap_clk), .reset (reset), @@ -191,6 +205,18 @@ module foldsim_afu #( .max_reads (mem_max_reads), .max_writes (mem_max_writes), .base (mem_base), + .model_bank (mem_model_bank), + .cfg_data_rate (mem_clock[15:0]), + .cfg_core_clock(mem_clock[31:16]), + .cfg_latency (mem_latency), + .cfg_queue (mem_queue[15:0]), + .cfg_rq (mem_queue[23:16]), + .cfg_wq (mem_queue[31:24]), + .cfg_wr_high (mem_wmark[7:0]), + .cfg_wr_low (mem_wmark[15:8]), + .cfg_geom (mem_geom), + .cfg_t (mem_timing), + .stats (mem_stats), .idle (mem_idle), .error (mem_error), .stall (mem_stall), @@ -229,7 +255,8 @@ module foldsim_afu #( foldsim_ctrl #( .DCR_ADDR_W (DCR_ADDR_W), .DCR_DATA_W (DCR_DATA_W), - .NUM_CLASSES (`FOLDSIM_NUM_CLASSES) + .NUM_CLASSES (`FOLDSIM_NUM_CLASSES), + .MEM_CAPS (MEM_CAPS) ) ctrl ( .clk (ap_clk), .reset (reset), @@ -252,6 +279,14 @@ module foldsim_afu #( .mem_max_reads (mem_max_reads), .mem_max_writes (mem_max_writes), .mem_base (mem_base), + .mem_model_bank (mem_model_bank), + .mem_clock (mem_clock), + .mem_latency (mem_latency), + .mem_queue (mem_queue), + .mem_geom (mem_geom), + .mem_wmark (mem_wmark), + .mem_timing (mem_timing), + .mem_stats (mem_stats), .mem_idle (mem_idle), .mem_error (mem_error), .mem_stall (mem_stall), diff --git a/hw/rtl/afu/foldsim/foldsim_ctrl.sv b/hw/rtl/afu/foldsim/foldsim_ctrl.sv index 11f18cabcc..32229a3949 100644 --- a/hw/rtl/afu/foldsim/foldsim_ctrl.sv +++ b/hw/rtl/afu/foldsim/foldsim_ctrl.sv @@ -32,7 +32,9 @@ module foldsim_ctrl #( parameter QUEUE = 16, parameter RESET_CYCLES = 64, parameter MEM_DEPTH = 16, - parameter NUM_CLASSES = 1 + parameter NUM_CLASSES = 1, + // MEM_CAPS: what the memory endpoint's bank model was built with. + parameter MEM_CAPS = 32'd0 ) ( input wire clk, input wire reset, @@ -58,6 +60,15 @@ module foldsim_ctrl #( output reg [7:0] mem_max_reads, output reg [7:0] mem_max_writes, output reg [63:0] mem_base, + // The bank model's settings and counters (foldsim_mem_bank). + output wire mem_model_bank, + output reg [31:0] mem_clock, + output reg [15:0] mem_latency, + output reg [31:0] mem_queue, + output reg [12:0] mem_geom, + output reg [15:0] mem_wmark, + output wire [18*12-1:0] mem_timing, + input wire [6*64-1:0] mem_stats, input wire mem_idle, input wire mem_error, input wire mem_stall, @@ -107,6 +118,23 @@ module foldsim_ctrl #( reg [DCR_DATA_W-1:0] rsp_data; reg [63:0] target_cycles, host_cycles, mem_stalls; reg [47:0] finishes [NUM_CLASSES]; + reg [1:0] mem_model; + reg [11:0] mem_t [18]; + + // The model in effect from a reset on: the memory endpoint takes it at the + // end of every reset cycle, so a reset command's own cycle has it already. + wire [1:0] mem_model_now = (cur_op == `FOLDSIM_CMD_RESET) ? cur_addr[1:0] : mem_model; + assign mem_model_bank = (mem_model_now == 2'd1); + for (genvar i = 0; i < 18; ++i) begin : g_mem_t + assign mem_timing[i*12 +: 12] = mem_t[i]; + end + // The default machine's DRAM (HBM2 at 800 MT/s behind a 400 MHz core), + // until the runtime sets its own: tBL, tCCD_S, tCCD_L, tCL, tCWL, + // tRCD_RD, tRCD_WR, tRP, tRAS, tRC, tRTP, tWR, tRRD_S, tRRD_L, tFAW, + // tWTR_S, tWTR_L, read-to-write extra. + localparam [18*12-1:0] MEM_T_RESET = {12'd2, 12'd2, 12'd2, 12'd6, 12'd2, 12'd2, 12'd5, 12'd2, + 12'd19, 12'd14, 12'd6, 12'd6, 12'd6, 12'd1, 12'd6, 12'd4, + 12'd2, 12'd2}; wire q_empty = (q_count == 0); @@ -186,6 +214,14 @@ module foldsim_ctrl #( mem_max_reads <= 8'(MEM_DEPTH); mem_max_writes <= 8'(MEM_DEPTH); mem_base <= 64'h40_0000_0000; + mem_model <= 2'd0; + mem_clock <= {16'd400, 16'd800}; + mem_latency <= 16'd0; + mem_queue <= {8'd32, 8'd32, 16'd0}; + mem_geom <= {1'b1, 2'd2, 4'd5, 2'd2, 2'd2, 2'd1}; + mem_wmark <= {8'd7, 8'd26}; + for (integer i = 0; i < 18; ++i) + mem_t[i] <= MEM_T_RESET[i*12 +: 12]; aw_got <= 1'b0; w_got <= 1'b0; ar_got <= 1'b0; @@ -211,6 +247,10 @@ module foldsim_ctrl #( rsp_data <= rsp_d; end target_busy <= busy; + // A reset command's address chooses the memory model the + // target has from the reset on. + if (cur_op == `FOLDSIM_CMD_RESET) + mem_model <= cur_addr[1:0]; // A start counts as busy until the target says it is, or plainly // never will. if (do_start) begin @@ -258,8 +298,15 @@ module foldsim_ctrl #( `FOLDSIM_REG_MEM_WLAT: mem_write_latency <= (w_data == 0) ? 32'd1 : w_data; `FOLDSIM_REG_MEM_MAXR: mem_max_reads <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); `FOLDSIM_REG_MEM_MAXW: mem_max_writes <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); + `FOLDSIM_REG_MEM_CLOCK: mem_clock <= w_data; + `FOLDSIM_REG_MEM_LATENCY: mem_latency <= w_data[15:0]; + `FOLDSIM_REG_MEM_QUEUE: mem_queue <= w_data; + `FOLDSIM_REG_MEM_GEOM: mem_geom <= w_data[12:0]; + `FOLDSIM_REG_MEM_WMARK: mem_wmark <= w_data[15:0]; default:; endcase + if (wa >= `FOLDSIM_REG_MEM_TIMING && wa < `FOLDSIM_REG_MEM_TIMING + 12'(4 * 18)) + mem_t[5'((wa - `FOLDSIM_REG_MEM_TIMING) >> 2)] <= w_data[11:0]; end if (s_bvalid & s_bready) s_bvalid <= 1'b0; @@ -293,8 +340,19 @@ module foldsim_ctrl #( `FOLDSIM_REG_MSTALL_LO: s_rdata <= mem_stalls[31:0]; `FOLDSIM_REG_MSTALL_HI: s_rdata <= mem_stalls[63:32]; `FOLDSIM_REG_NUM_CLASSES: s_rdata <= 32'(NUM_CLASSES); + `FOLDSIM_REG_MEM_MODEL: s_rdata <= 32'(mem_model); + `FOLDSIM_REG_MEM_CAPS: s_rdata <= MEM_CAPS; + `FOLDSIM_REG_MEM_CLOCK: s_rdata <= mem_clock; + `FOLDSIM_REG_MEM_LATENCY: s_rdata <= 32'(mem_latency); + `FOLDSIM_REG_MEM_QUEUE: s_rdata <= mem_queue; + `FOLDSIM_REG_MEM_GEOM: s_rdata <= 32'(mem_geom); + `FOLDSIM_REG_MEM_WMARK: s_rdata <= 32'(mem_wmark); default: s_rdata <= 32'd0; endcase + if (ar_addr >= `FOLDSIM_REG_MEM_TIMING && ar_addr < `FOLDSIM_REG_MEM_TIMING + 12'(4 * 18)) + s_rdata <= 32'(mem_t[5'((ar_addr - `FOLDSIM_REG_MEM_TIMING) >> 2)]); + if (ar_addr >= `FOLDSIM_REG_MEM_STATS && ar_addr < `FOLDSIM_REG_MEM_STATS + 12'(8 * 6)) + s_rdata <= mem_stats[32'(ar_addr - `FOLDSIM_REG_MEM_STATS) * 8 +: 32]; if (ar_addr >= `FOLDSIM_REG_FINISH_BASE && ar_addr < `FOLDSIM_REG_FINISH_BASE + 12'(8 * NUM_CLASSES)) begin s_rdata <= ar_addr[2] ? 32'(finishes[(ar_addr - `FOLDSIM_REG_FINISH_BASE) >> 3] >> 32) diff --git a/hw/rtl/afu/foldsim/foldsim_mem.sv b/hw/rtl/afu/foldsim/foldsim_mem.sv index 9e06527bd9..8451391661 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem.sv @@ -13,12 +13,16 @@ // The target's AXI4 memory port, as a foldsim endpoint. // -// Timing is a latency-bandwidth pipe, cycle for cycle the model of -// sim/common/axi_latency_pipe.h: a -// fixed latency per transaction, at most `max_reads` reads and `max_writes` -// writes in flight, one beat per cycle on each response channel, responses in -// the order their requests were accepted. Every output is a function of the -// pipe's state, so all of them are offered at the start of a target cycle. +// Timing is one of two models, chosen by `model_bank` at the target's reset. +// The latency-bandwidth pipe, cycle for cycle the model of +// sim/common/axi_latency_pipe.h: a fixed latency per transaction, at most +// `max_reads` reads and `max_writes` writes in flight, one beat per cycle on +// each response channel, responses in the order their requests were accepted. +// Or a DRAM channel's banks (foldsim_mem_bank, cycle for cycle +// sim/common/dram_bank_model.h's DramBankAxi): each read holds a slot, and +// responses leave in the order their data departs. Every output is a function +// of the model's state, so all of them are offered at the start of a target +// cycle. // // Data lives in device memory at `base` plus the target's address. A read // goes to memory the cycle the target makes it, a write the cycle it has both @@ -40,7 +44,10 @@ module foldsim_mem #( parameter ID_W = 4, parameter ADDR_W = 64, parameter DATA_W = 512, - parameter DEPTH = 16 + parameter DEPTH = 16, + // The bank model: read slots, and a ring of 2^HLOG2 DRAM clocks. + parameter SLOTS = 32, + parameter HLOG2 = 9 ) ( input wire clk, input wire reset, @@ -50,6 +57,20 @@ module foldsim_mem #( input wire [7:0] max_reads, input wire [7:0] max_writes, input wire [63:0] base, + // The bank model and its settings (foldsim_mem_bank). + input wire model_bank, + input wire [15:0] cfg_data_rate, + input wire [15:0] cfg_core_clock, + input wire [15:0] cfg_latency, + input wire [15:0] cfg_queue, + input wire [7:0] cfg_rq, + input wire [7:0] cfg_wq, + input wire [7:0] cfg_wr_high, + input wire [7:0] cfg_wr_low, + input wire [12:0] cfg_geom, + input wire [18*12-1:0] cfg_t, + // Reads, writes, row hits, misses, conflicts, read latency sum. + output wire [6*64-1:0] stats, // No transaction in flight, at the target or in memory. output wire idle, output reg error, @@ -110,8 +131,15 @@ module foldsim_mem #( ); localparam CW = $clog2(DEPTH + 1); localparam PW = $clog2(DEPTH); + // Memory's side holds what either model may have in flight. + localparam HD = (SLOTS > DEPTH) ? SLOTS : DEPTH; + localparam HCW = $clog2(HD + 1); + localparam HPW = $clog2(HD); + localparam SW = $clog2(SLOTS); wire finish; + // The bank model is the port's timing. + wire bank; // ---- The target's request side, one token per channel per cycle ---- wire awvalid, wvalid, wlast, arvalid, rready, bready, treset; @@ -150,9 +178,10 @@ module foldsim_mem #( reg cur_r_valid; reg [ID_W-1:0] cur_r_id; - // AW ids waiting for their write data, writes formed and not yet driving - // B, and the one that is. + // AW ids (and addresses) waiting for their write data, writes formed and + // not yet driving B, and the one that is. reg [ID_W-1:0] awq_id [DEPTH]; + reg [29:0] awq_addr [DEPTH]; reg [PW-1:0] awq_head, awq_tail; reg [CW-1:0] awq_count; reg [31:0] wp_release [DEPTH]; @@ -168,15 +197,26 @@ module foldsim_mem #( reg [CW-1:0] rd_count; reg [7:0] b_acked, b_given; // Reads memory has not answered, and writes it has not acknowledged. - reg [CW-1:0] rd_out, wr_out; + reg [HCW-1:0] rd_out, wr_out; wire drained = (rd_out == 0) & (wr_out == 0); - // ---- The pipe's outputs: functions of its state ---- - wire ar_ready = ~error & (32'(pending_reads) < 32'(max_reads)); - wire aw_ready = ~error & (32'(pending_aw) < 32'(max_writes)); - wire w_ready = ~error & (32'(pending_w) < 32'(max_writes)); + // The bank model's state: the port's readiness, R and B. + wire bk_room, bk_slot_free, bk_r_valid, bk_b_valid, bk_no_reads, bk_done; + wire [SW-1:0] bk_alloc_slot, bk_r_slot; + wire [ID_W-1:0] bk_r_id, bk_b_id; + // Read data by slot, and which slots memory has answered. + reg [DATA_W-1:0] slot_data [SLOTS]; + reg [SLOTS-1:0] arrived; + + // ---- The model's outputs: functions of its state ---- + wire ar_ready = ~error & (bank ? (bk_room & bk_slot_free) : (32'(pending_reads) < 32'(max_reads))); + wire aw_ready = ~error & (bank ? (bk_room & (32'(pending_aw) < DEPTH)) : (32'(pending_aw) < 32'(max_writes))); + wire w_ready = ~error & (bank ? (32'(pending_w) < DEPTH) : (32'(pending_w) < 32'(max_writes))); + wire r_valid = bank ? bk_r_valid : cur_r_valid; + wire b_valid = bank ? bk_b_valid : cur_w_valid; // R needs its data back from memory, and B its write acknowledged. - wire ok = (~cur_r_valid | (rd_count != 0)) & (~cur_w_valid | (b_acked != b_given)); + wire ok = bank ? ((~bk_r_valid | arrived[bk_r_slot]) & (~bk_b_valid | (b_acked != b_given))) + : ((~cur_r_valid | (rd_count != 0)) & (~cur_w_valid | (b_acked != b_given))); wire [10:0] done; foldsim_token_out o_awready (clk, reset, awready_valid, awready_ready, ok, finish, done[0]); @@ -194,20 +234,22 @@ module foldsim_mem #( assign awready_bits = aw_ready; assign wready_bits = w_ready; assign arready_bits = ar_ready; - assign rvalid_bits = cur_r_valid; - assign rdata_bits = rd_data[rd_head]; + assign rvalid_bits = r_valid; + assign rdata_bits = bank ? slot_data[bk_r_slot] : rd_data[rd_head]; assign rlast_bits = 1'b1; - assign rid_bits = cur_r_id; + assign rid_bits = bank ? bk_r_id : cur_r_id; assign rresp_bits = 2'b00; - assign bvalid_bits = cur_w_valid; + assign bvalid_bits = b_valid; assign bresp_bits = 2'b00; - assign bid_bits = cur_w_id; + assign bid_bits = bank ? bk_b_id : cur_w_id; - // A reset cycle waits for memory to answer everything it has. + // A reset cycle waits for memory to answer everything it has, and every + // cycle for the bank model's DRAM clocks of it. wire drain_wait = have[14] & treset & ~drained; - assign finish = (&have) & (&done) & ~drain_wait; - assign stall = ~ok | drain_wait; - assign idle = (pending_reads == 0) & (pending_aw == 0) & (pending_w == 0) & drained; + wire tokens_in = (&have) & (&done); + assign finish = tokens_in & ~drain_wait & bk_done; + assign stall = ~ok | drain_wait | (tokens_in & ~bk_done); + assign idle = (pending_reads == 0) & bk_no_reads & (pending_aw == 0) & (pending_w == 0) & drained; // ---- One target cycle of the pipe, on the request side's tokens ---- wire ar_hs = arvalid & ar_ready; @@ -217,8 +259,8 @@ module foldsim_mem #( wire ar_fire = ar_hs & (arlen == 0); wire aw_fire = aw_hs & (awlen == 0); wire w_fire = w_hs & wlast; - wire r_fire = cur_r_valid & rready; - wire b_fire = cur_w_valid & bready; + wire r_fire = r_valid & rready; + wire b_fire = b_valid & bready; // Reads: the latency pipe, into the stage driving R; a read released the // cycle it arrives skips the pipe. @@ -239,18 +281,43 @@ module foldsim_mem #( | (aw_fire & w_fire); wire awq_empty = (awq_count == 0); wire [ID_W-1:0] w_form_id = awq_empty ? awid : awq_id[awq_head]; + wire [29:0] w_form_addr = awq_empty ? awaddr[29:0] : awq_addr[awq_head]; wire awq_pop = new_write & ~awq_empty; wire awq_push = aw_fire & ~(new_write & awq_empty); wire [31:0] w_release = cycle + write_latency - 32'd1; wire wp_empty = (wp_count == 0); wire w_head = ~wp_empty | new_write; - wire [31:0] w_head_rel = wp_empty ? w_release : wp_release[wp_head]; - wire [ID_W-1:0] w_head_id = wp_empty ? w_form_id : wp_id[wp_head]; + // A write enters the pipe's RAM the host cycle after the target cycle + // that formed it, from a register, so the RAM's write enable does not end + // the path through the write's formation; a read of it in between is + // forwarded. + reg wp_wr_pend; + reg [PW-1:0] wp_wr_idx; + reg [31:0] wp_wr_rel; + reg [ID_W-1:0] wp_wr_id; + wire wp_fwd = wp_wr_pend & (wp_wr_idx == wp_head); + wire [31:0] w_head_rel = wp_empty ? w_release : (wp_fwd ? wp_wr_rel : wp_release[wp_head]); + wire [ID_W-1:0] w_head_id = wp_empty ? w_form_id : (wp_fwd ? wp_wr_id : wp_id[wp_head]); wire w_rel = w_head & ($signed(cycle - w_head_rel) >= 0); wire w_take = w_rel & (~cur_w_valid | b_fire); wire wp_push = new_write & ~(w_take & wp_empty); wire wp_pop = w_take & ~wp_empty; + always @(posedge clk) begin + if (wp_wr_pend) begin + wp_release[wp_wr_idx] <= wp_wr_rel; + wp_id[wp_wr_idx] <= wp_wr_id; + end + if (reset) begin + wp_wr_pend <= 1'b0; + end else begin + wp_wr_pend <= finish & ~treset & ~bank & wp_push; + end + wp_wr_idx <= wp_tail; + wp_wr_rel <= w_release; + wp_wr_id <= w_form_id; + end + function automatic [CW-1:0] count(input [CW-1:0] value, input inc, input dec); count = (inc & ~dec) ? value + 1'b1 : ((dec & ~inc & (value != 0)) ? value - 1'b1 : value); endfunction @@ -292,44 +359,45 @@ module foldsim_mem #( cur_w_valid <= 1'b0; error <= 1'b0; end else begin - if (rp_push) begin - rp_release[rp_tail] <= ar_release; - rp_id[rp_tail] <= arid; - rp_tail <= rp_tail + 1'b1; - end - if (rp_pop) - rp_head <= rp_head + 1'b1; - rp_count <= rp_count + CW'(rp_push) - CW'(rp_pop); - if (r_take) begin - cur_r_valid <= 1'b1; - cur_r_id <= r_head_id; - end else if (r_fire) begin - cur_r_valid <= 1'b0; + // The pipe's own state stays as reset under the bank model. + if (~bank) begin + if (rp_push) begin + rp_release[rp_tail] <= ar_release; + rp_id[rp_tail] <= arid; + rp_tail <= rp_tail + 1'b1; + end + if (rp_pop) + rp_head <= rp_head + 1'b1; + rp_count <= rp_count + CW'(rp_push) - CW'(rp_pop); + if (r_take) begin + cur_r_valid <= 1'b1; + cur_r_id <= r_head_id; + end else if (r_fire) begin + cur_r_valid <= 1'b0; + end + if (wp_push) + wp_tail <= wp_tail + 1'b1; + if (wp_pop) + wp_head <= wp_head + 1'b1; + wp_count <= wp_count + CW'(wp_push) - CW'(wp_pop); + if (w_take) begin + cur_w_valid <= 1'b1; + cur_w_id <= w_head_id; + end else if (b_fire) begin + cur_w_valid <= 1'b0; + end + pending_reads <= count(pending_reads, ar_fire, r_fire); end if (awq_push) begin - awq_id[awq_tail] <= awid; - awq_tail <= awq_tail + 1'b1; + awq_id[awq_tail] <= awid; + awq_addr[awq_tail] <= awaddr[29:0]; + awq_tail <= awq_tail + 1'b1; end if (awq_pop) awq_head <= awq_head + 1'b1; awq_count <= awq_count + CW'(awq_push) - CW'(awq_pop); - if (wp_push) begin - wp_release[wp_tail] <= w_release; - wp_id[wp_tail] <= w_form_id; - wp_tail <= wp_tail + 1'b1; - end - if (wp_pop) - wp_head <= wp_head + 1'b1; - wp_count <= wp_count + CW'(wp_push) - CW'(wp_pop); - if (w_take) begin - cur_w_valid <= 1'b1; - cur_w_id <= w_head_id; - end else if (b_fire) begin - cur_w_valid <= 1'b0; - end - pending_reads <= count(pending_reads, ar_fire, r_fire); pending_aw <= count(pending_aw, aw_fire, b_fire); pending_w <= count(pending_w, w_fire, b_fire); if (burst) @@ -341,9 +409,10 @@ module foldsim_mem #( // ---- Memory: requests as the target makes them, in order ---- // A write goes to memory once both its halves are in: only whole writes // are offered, so a lone half can be dropped without breaking AXI. - reg [63:0] arq_addr [DEPTH]; - reg [PW-1:0] arq_head, arq_tail; - reg [CW-1:0] arq_count; + reg [63:0] arq_addr [HD]; + reg [SW-1:0] arq_slot [HD]; + reg [HPW-1:0] arq_head, arq_tail, rs_head; + reg [HCW-1:0] arq_count; reg [63:0] awm_addr [DEPTH]; reg [PW-1:0] awm_head, awm_tail; reg [DATA_W-1:0] wm_data [DEPTH]; @@ -379,16 +448,22 @@ module foldsim_mem #( assign m_bready = 1'b1; always @(posedge clk) begin - if (arq_push) + if (arq_push) begin arq_addr[arq_tail] <= base + 64'(araddr); + arq_slot[arq_tail] <= bk_alloc_slot; + end if (awm_push) awm_addr[awm_tail] <= base + 64'(awaddr); if (wm_push) begin wm_data[wm_tail] <= wdata; wm_strb[wm_tail] <= wstrb; end - if (rd_push) + if (rd_push & ~bank) rd_data[rd_tail] <= m_rdata; + // Memory answers reads in the order they were made: the bank model's + // data goes to the read's slot. + if (rd_push & bank) + slot_data[arq_slot[rs_head]] <= m_rdata; end // A flush comes once memory has answered everything: nothing is offered @@ -399,6 +474,8 @@ module foldsim_mem #( arq_head <= 0; arq_tail <= 0; arq_count <= 0; + rs_head <= 0; + arrived <= '0; awm_head <= 0; awm_tail <= 0; wm_head <= 0; @@ -417,7 +494,19 @@ module foldsim_mem #( arq_tail <= arq_tail + 1'b1; if (arq_pop) arq_head <= arq_head + 1'b1; - arq_count <= arq_count + CW'(arq_push) - CW'(arq_pop); + arq_count <= arq_count + HCW'(arq_push) - HCW'(arq_pop); + if (rd_push) + rs_head <= rs_head + 1'b1; + // A slot's data has not arrived from the cycle its read is taken. + begin : arrived_ + reg [SLOTS-1:0] a; + a = arrived; + if (arq_push & bank) + a[bk_alloc_slot] = 1'b0; + if (rd_push & bank) + a[arq_slot[rs_head]] = 1'b1; + arrived <= a; + end if (awm_push) awm_tail <= awm_tail + 1'b1; @@ -430,14 +519,16 @@ module foldsim_mem #( aw_formed <= aw_formed + CW'(form) - CW'(awm_pop); w_formed <= w_formed + CW'(form) - CW'(wm_pop); - if (rd_push) - rd_tail <= rd_tail + 1'b1; - if (rd_pop) - rd_head <= rd_head + 1'b1; - rd_count <= rd_count + CW'(rd_push) - CW'(rd_pop); + if (~bank) begin + if (rd_push) + rd_tail <= rd_tail + 1'b1; + if (rd_pop) + rd_head <= rd_head + 1'b1; + rd_count <= rd_count + CW'(rd_push) - CW'(rd_pop); + end - rd_out <= rd_out + CW'(arq_push) - CW'(rd_push); - wr_out <= wr_out + CW'(form) - CW'(b_push); + rd_out <= rd_out + HCW'(arq_push) - HCW'(rd_push); + wr_out <= wr_out + HCW'(form) - HCW'(b_push); if (b_push) b_acked <= b_acked + 8'd1; if (take & b_fire) @@ -445,4 +536,55 @@ module foldsim_mem #( end end + // ---- The bank model ---- + foldsim_mem_bank #( + .ID_W (ID_W), + .SLOTS (SLOTS), + .WSLOTS (DEPTH), + .HLOG2 (HLOG2), + .BURSTS (DATA_W / 256) + ) bank_model ( + .clk (clk), + .reset (reset), + .enable (model_bank), + .cfg_data_rate (cfg_data_rate), + .cfg_core_clock (cfg_core_clock), + .cfg_latency (cfg_latency), + .cfg_queue (cfg_queue), + .cfg_rq (cfg_rq), + .cfg_wq (cfg_wq), + .cfg_wr_high (cfg_wr_high), + .cfg_wr_low (cfg_wr_low), + .cfg_geom (cfg_geom), + .cfg_t (cfg_t), + .finish (finish), + .treset (treset), + .ar_fire (ar_fire), + .ar_id (arid), + .ar_addr (araddr[29:0]), + .aw_fire (aw_fire), + .new_write (new_write), + .nw_id (w_form_id), + .nw_addr (w_form_addr), + .r_fire (r_fire), + .b_fire (b_fire), + .room (bk_room), + .slot_free (bk_slot_free), + .alloc_slot (bk_alloc_slot), + .r_valid (bk_r_valid), + .r_id (bk_r_id), + .r_slot (bk_r_slot), + .b_valid (bk_b_valid), + .b_id (bk_b_id), + .no_reads (bk_no_reads), + .done (bk_done), + .active (bank), + .stat_reads (stats[0*64 +: 64]), + .stat_writes (stats[1*64 +: 64]), + .stat_hits (stats[2*64 +: 64]), + .stat_misses (stats[3*64 +: 64]), + .stat_conflicts (stats[4*64 +: 64]), + .stat_latency (stats[5*64 +: 64]) + ); + endmodule diff --git a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv new file mode 100644 index 0000000000..372811ca15 --- /dev/null +++ b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv @@ -0,0 +1,910 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The timing of foldsim's memory endpoint as a DRAM bank model: the model of +// sim/common/dram_bank_model.h (DramBankAxi over DramBankModel), decision for +// decision, for one AXI4 port and one DRAM channel. +// +// foldsim_mem gives it each target cycle's requests when the cycle finishes; +// it answers with the port's state for the next one: whether a read or write +// may be taken, and the read and write acknowledgement R and B present. The +// DRAM clocks of a target cycle run in host cycles while the target evaluates +// it, and the cycle does not finish before they have (`done`): requests of a +// cycle reach the channel in the next one, and a read departs at least +// tCL + tBL clocks after it was reserved, so a cycle's clocks depend only on +// what earlier cycles gave. +// +// The calendars are bitmaps over a ring of 2^HLOG2 DRAM clocks in distributed +// RAM (foldsim_mem_bmap): which clocks hold a command, the row-command bus, +// and per pseudo-channel and bank group the clocks a read, a write or an ACT +// may not take -- each command, once reserved, marks the clocks its +// constraints forbid to later ones, so placing a command is a first-free +// search over 64 clocks a host cycle. A reservation reaches at most +// 2^HLOG2 - 64 clocks ahead: the ring's last word is the one being retired. +// Times of the bank table and the controller are 16-bit, modulo; a time too +// old to constrain anything is moved up before it could wrap. +module foldsim_mem_bank #( + parameter ID_W = 4, + parameter SLOTS = 32, + parameter WSLOTS = 16, + parameter HLOG2 = 9, + parameter BURSTS = 2, + parameter ROW_W = 20, + parameter INQ = 64, + parameter SW = $clog2(SLOTS) +) ( + input wire clk, + input wire reset, + + // The bank model is the port's timing from the next target reset on. + input wire enable, + + input wire [15:0] cfg_data_rate, + input wire [15:0] cfg_core_clock, + input wire [15:0] cfg_latency, + input wire [15:0] cfg_queue, + input wire [7:0] cfg_rq, + input wire [7:0] cfg_wq, + input wire [7:0] cfg_wr_high, + input wire [7:0] cfg_wr_low, + // [1:0] log2 pseudo-channels, [3:2] log2 bank groups, [5:4] log2 banks per + // group, [9:6] log2 bursts per row, [11:10] log2 bursts per row visit, + // [12] an ACT holds the row bus for two clocks. + input wire [12:0] cfg_geom, + // tBL, tCCD_S, tCCD_L, tCL, tCWL, tRCD_RD, tRCD_WR, tRP, tRAS, tRC, tRTP, + // tWR, tRRD_S, tRRD_L, tFAW, tWTR_S, tWTR_L, read-to-write extra: DRAM + // clocks, 12 bits each. + input wire [18*12-1:0] cfg_t, + + // The target cycle finishes; a reset cycle drops everything. + input wire finish, + input wire treset, + // What the cycle takes: a read, a write formed of an AW and its W, and the + // responses R and B give. + input wire ar_fire, + input wire [ID_W-1:0] ar_id, + input wire [29:0] ar_addr, + input wire aw_fire, + input wire new_write, + input wire [ID_W-1:0] nw_id, + input wire [29:0] nw_addr, + input wire r_fire, + input wire b_fire, + + // The port's state for the target cycle. + output wire room, + output wire slot_free, + output wire [SW-1:0] alloc_slot, + output wire r_valid, + output wire [ID_W-1:0] r_id, + output wire [SW-1:0] r_slot, + output wire b_valid, + output wire [ID_W-1:0] b_id, + output wire no_reads, + // The target cycle's DRAM clocks are done. + output wire done, + // The bank model is the port's timing, since the last target reset. + output reg active, + + // Reads, writes, row hits, misses, conflicts, and the sum of read + // latencies (target cycles from AR to R). + output reg [63:0] stat_reads, + output reg [63:0] stat_writes, + output reg [63:0] stat_hits, + output reg [63:0] stat_misses, + output reg [63:0] stat_conflicts, + output reg [63:0] stat_latency +); + localparam H = 1 << HLOG2; + localparam HE = H - 64; + localparam WORDS = H / 64; + localparam WW = HLOG2 - 6; + localparam NB = BURSTS; + localparam TAG_W = (SW > ID_W) ? SW : ID_W; + localparam BKW = 5; + localparam IQW = $clog2(INQ); + localparam BQ = 32; + localparam BQW = $clog2(BQ); + + // ---- Configuration and its derived constants ---- + wire [11:0] T [18]; + for (genvar i = 0; i < 18; ++i) begin : g_t + assign T[i] = cfg_t[i*12 +: 12]; + end + localparam I_BL = 0, I_CCDS = 1, I_CCDL = 2, I_CL = 3, I_CWL = 4, I_RCDRD = 5, I_RCDWR = 6, + I_RP = 7, I_RAS = 8, I_RC = 9, I_RTP = 10, I_WR = 11, I_RRDS = 12, I_RRDL = 13, + I_FAW = 14, I_WTRS = 15, I_WTRL = 16, I_RTWX = 17; + + wire [1:0] g_lpcs = cfg_geom[1:0]; + wire [1:0] g_lbgs = cfg_geom[3:2]; + wire [1:0] g_lbanks = cfg_geom[5:4]; + wire [3:0] g_lrow = cfg_geom[9:6]; + wire [1:0] g_lvisit = cfg_geom[11:10]; + wire g_act2 = cfg_geom[12]; + + function automatic signed [15:0] smax(input signed [15:0] a, input signed [15:0] b); + smax = (a > b) ? a : b; + endfunction + + reg signed [15:0] k_ccds, k_ccdl, k_rtw, k_wtrs, k_wtrl, k_reach_r, k_reach_w, k_reach_a; + reg signed [15:0] k_wrpre, k_dep, k_ccdl1, k_rcdrd, k_rcdwr, k_rp, k_ras, k_rc, k_rtp; + reg signed [15:0] k_rrds, k_rrdl; + reg [16:0] k_twoclk; + reg [2:0] k_bb; + always @(posedge clk) begin + k_ccds <= 16'(T[I_CCDS]); + k_ccdl <= 16'(T[I_CCDL]); + k_rtw <= smax(16'(T[I_CCDS]), 16'(T[I_CL]) + 16'(T[I_BL]) + 16'(T[I_RTWX]) - 16'(T[I_CWL])); + k_wtrs <= 16'(T[I_CWL]) + 16'(T[I_BL]) + 16'(T[I_WTRS]); + k_wtrl <= 16'(T[I_CWL]) + 16'(T[I_BL]) + 16'(T[I_WTRL]); + k_reach_r <= smax(smax(k_ccds, k_ccdl), k_rtw) - 16'sd1; + k_reach_w <= smax(smax(k_ccds, k_ccdl), smax(k_wtrs, k_wtrl)) - 16'sd1; + k_reach_a <= smax(smax(smax(16'(T[I_RRDS]), 16'(T[I_RRDL])), 16'(T[I_FAW])) - 16'sd1, + g_act2 ? 16'sd1 : 16'sd0); + k_wrpre <= 16'(T[I_CWL]) + 16'(T[I_BL]) + 16'(T[I_WR]); + k_dep <= 16'(T[I_CL]) + 16'(T[I_BL]); + k_ccdl1 <= smax(16'(T[I_CCDL]), 16'sd1); + k_rcdrd <= 16'(T[I_RCDRD]); + k_rcdwr <= 16'(T[I_RCDWR]); + k_rp <= 16'(T[I_RP]); + k_ras <= 16'(T[I_RAS]); + k_rc <= 16'(T[I_RC]); + k_rtp <= 16'(T[I_RTP]); + k_rrds <= 16'(T[I_RRDS]); + k_rrdl <= 16'(T[I_RRDL]); + k_twoclk <= {cfg_core_clock, 1'b0}; + k_bb <= 3'(g_lpcs) + 3'(g_lbgs) + 3'(g_lbanks); + end + + // The bank and row of a channel address (sim/common/dram_bank_model.h's + // decode): a row visit, then the bank bits, the row above the column, and + // the bank XORed with the row folded to its width. + function automatic [BKW+ROW_W-1:0] decode(input [29:5] a); + reg [24:0] tx; + reg [24:0] b; + reg [ROW_W-1:0] r; + reg [24:0] mask; + tx = a[29:5]; + mask = (25'd1 << k_bb) - 25'd1; + b = (tx >> g_lvisit) & mask; + r = ROW_W'(tx >> (32'(k_bb) + 32'(g_lrow))); + for (integer k = 0; k < ROW_W; ++k) begin + if (k_bb != 0 && 32'(k) * 32'(k_bb) < ROW_W) + b = b ^ ((25'(r) >> (32'(k) * 32'(k_bb))) & mask); + end + decode = {BKW'(b), r}; + endfunction + + // ---- The port: slots, responses, requests in flight ---- + reg [SLOTS-1:0] slot_busy; + reg [ID_W-1:0] slot_id [SLOTS]; + reg [SW:0] slots_used; + reg [15:0] inflight; + reg cur_r_valid, cur_b_valid; + reg [SW-1:0] cur_r_slot; + reg [ID_W-1:0] cur_r_id, cur_b_id; + reg [31:0] tcyc; + + // Reads departed, in order, with the target cycle they are ready in; the + // first `rf_ready` are ready in the next cycle. + reg [SW-1:0] rf_slot [SLOTS]; + reg [31:0] rf_cyc [SLOTS]; + reg [SW-1:0] rf_head, rf_tail; + reg [SW:0] rf_count, rf_ready; + // Writes the controller has buffered, to acknowledge in order. + localparam AQW = $clog2(WSLOTS); + reg [ID_W-1:0] af_id [WSLOTS]; + reg [AQW-1:0] af_head, af_tail; + reg [AQW:0] af_count; + // What this target cycle's DRAM clocks made ready by the next one. + reg [SW:0] new_ready; + reg [AQW:0] new_acks; + + assign room = (cfg_queue == 16'd0) | (inflight < cfg_queue); + assign slot_free = (32'(slots_used) < SLOTS); + assign r_valid = cur_r_valid; + assign r_id = cur_r_id; + assign r_slot = cur_r_slot; + assign b_valid = cur_b_valid; + assign b_id = cur_b_id; + assign no_reads = (slots_used == 0); + + // The lowest free slot once R has taken its read. + wire [SLOTS-1:0] free_slots = ~slot_busy | ((r_fire ? SLOTS'(1) : SLOTS'(0)) << cur_r_slot); + reg [SW-1:0] lowest_free; + always @(*) begin + lowest_free = '0; + for (integer s = SLOTS - 1; s >= 0; --s) + if (free_slots[s]) + lowest_free = SW'(s); + end + assign alloc_slot = lowest_free; + + // This cycle's requests, staged for the channel: a write, then a read. + reg st_w_valid, st_r_valid; + reg [ID_W-1:0] st_w_id; + reg [SW-1:0] st_r_slot; + reg [29:0] st_w_addr, st_r_addr; + + // ---- The channel ---- + // Requests in the order they formed; the controller's read and write + // buffers. + reg iq_w [INQ]; + reg [TAG_W-1:0] iq_tag [INQ]; + reg [29:0] iq_addr[INQ]; + reg [IQW-1:0] iq_head, iq_tail; + reg [IQW:0] iq_count; + reg [TAG_W-1:0] rb_tag [BQ], wb_tag [BQ]; + reg [BKW-1:0] rb_bank[BQ], wb_bank[BQ]; + reg [ROW_W-1:0] rb_row [BQ], wb_row [BQ]; + reg [BQW-1:0] rb_head, rb_tail, wb_head, wb_tail; + reg [BQW:0] rb_count, wb_count; + + reg [15:0] d; // the DRAM clock, modulo + reg [31:0] credit; + reg [15:0] ingress_ready; + reg [7:0] rq_cnt, wq_cnt; // reserved bursts whose column has not issued + reg wmode; + reg [15:0] frontier, batch_floor, batch_last, read_floor; + + // The bank table. A fresh bank has no row open and no time that matters. + reg [31:0] bt_open, bt_fresh; + reg [ROW_W-1:0] bt_row [32]; + reg [15:0] bt_act [32]; + reg [15:0] bt_colr [32]; + reg [15:0] bt_prer [32]; + reg [4:0] scrub_b; + + wire [15:0] e = d + 16'd1; // the first clock a reservation may take + wire [HLOG2-1:0] base = HLOG2'(e); // its place on the ring + function automatic signed [15:0] rel(input [15:0] t); + rel = $signed(t - e); + endfunction + + // ---- Calendars ---- + // The command bus is two calendars, row and column commands, so a + // reservation marks both at once. + localparam B_CMDR = 0, B_ROW = 1, B_KR = 2, B_KW = 3, B_DEPV = 4, B_FRP = 5, B_FWP = 6, + B_FRG = 7, B_FWG = 8, B_FAP = 9, B_FAG = 10, B_CMDC = 11, NBM = 12; + + reg [WW-1:0] s_word; // the word the search reads + wire [WW-1:0] s_word1 = s_word + WW'(1); + reg op_w; + reg [TAG_W-1:0] op_tag; + reg [BKW-1:0] op_bank; + reg [ROW_W-1:0] op_row; + reg op_pc; + reg [2:0] op_pg; + + wire [WW-1:0] end_word = WW'(base >> 6); + reg bm_clear; + reg [WW-1:0] bm_clear_word; + // A target reset (or the port leaving the bank model) clears the + // calendars at the cycle's end, before the next cycle's first clock. + wire bm_clear_all = finish & (treset | ~active); + reg [NBM-1:0] wr_en; + reg [WW-1:0] wr_word [NBM]; + reg [63:0] wr_mask [NBM]; + wire [63:0] rd_cmdr, rd_cmdc, rd_row, rd_kr, rd_kw, rd_depv; + wire [63:0] rd_cmd = rd_cmdr | rd_cmdc; + /* verilator lint_off UNUSEDSIGNAL */ + wire [63:0] rd_row1; + /* verilator lint_on UNUSEDSIGNAL */ + wire [63:0] rd_frp, rd_fwp, rd_frg, rd_fwg, rd_fap, rd_fag; + + foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_cmdr ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(1'b0), .rd_word(s_word), .rd_data(rd_cmdr), + .we(wr_en[B_CMDR]), .wr_inst(1'b0), .wr_word(wr_word[B_CMDR]), .wr_mask(wr_mask[B_CMDR])); + foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_cmdc ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(1'b0), .rd_word(s_word), .rd_data(rd_cmdc), + .we(wr_en[B_CMDC]), .wr_inst(1'b0), .wr_word(wr_word[B_CMDC]), .wr_mask(wr_mask[B_CMDC])); + foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(2)) m_row ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(2'b00), .rd_word({s_word1, s_word}), .rd_data({rd_row1, rd_row}), + .we(wr_en[B_ROW]), .wr_inst(1'b0), .wr_word(wr_word[B_ROW]), .wr_mask(wr_mask[B_ROW])); + foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_kr ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(1'b0), .rd_word(end_word), .rd_data(rd_kr), + .we(wr_en[B_KR]), .wr_inst(1'b0), .wr_word(wr_word[B_KR]), .wr_mask(wr_mask[B_KR])); + foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_kw ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(1'b0), .rd_word(end_word), .rd_data(rd_kw), + .we(wr_en[B_KW]), .wr_inst(1'b0), .wr_word(wr_word[B_KW]), .wr_mask(wr_mask[B_KW])); + foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_depv ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(1'b0), .rd_word(end_word), .rd_data(rd_depv), + .we(wr_en[B_DEPV]), .wr_inst(1'b0), .wr_word(wr_word[B_DEPV]), .wr_mask(wr_mask[B_DEPV])); + foldsim_mem_bmap #(.INST(2), .WORDS(WORDS), .NREAD(1)) m_frp ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(op_pc), .rd_word(s_word), .rd_data(rd_frp), + .we(wr_en[B_FRP]), .wr_inst(op_pc), .wr_word(wr_word[B_FRP]), .wr_mask(wr_mask[B_FRP])); + foldsim_mem_bmap #(.INST(2), .WORDS(WORDS), .NREAD(1)) m_fwp ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(op_pc), .rd_word(s_word), .rd_data(rd_fwp), + .we(wr_en[B_FWP]), .wr_inst(op_pc), .wr_word(wr_word[B_FWP]), .wr_mask(wr_mask[B_FWP])); + foldsim_mem_bmap #(.INST(8), .WORDS(WORDS), .NREAD(1)) m_frg ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(op_pg), .rd_word(s_word), .rd_data(rd_frg), + .we(wr_en[B_FRG]), .wr_inst(op_pg), .wr_word(wr_word[B_FRG]), .wr_mask(wr_mask[B_FRG])); + foldsim_mem_bmap #(.INST(8), .WORDS(WORDS), .NREAD(1)) m_fwg ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(op_pg), .rd_word(s_word), .rd_data(rd_fwg), + .we(wr_en[B_FWG]), .wr_inst(op_pg), .wr_word(wr_word[B_FWG]), .wr_mask(wr_mask[B_FWG])); + foldsim_mem_bmap #(.INST(2), .WORDS(WORDS), .NREAD(1)) m_fap ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(op_pc), .rd_word(s_word), .rd_data(rd_fap), + .we(wr_en[B_FAP]), .wr_inst(op_pc), .wr_word(wr_word[B_FAP]), .wr_mask(wr_mask[B_FAP])); + foldsim_mem_bmap #(.INST(8), .WORDS(WORDS), .NREAD(1)) m_fag ( + .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), + .rd_inst(op_pg), .rd_word(s_word), .rd_data(rd_fag), + .we(wr_en[B_FAG]), .wr_inst(op_pg), .wr_word(wr_word[B_FAG]), .wr_mask(wr_mask[B_FAG])); + + // The read each departure completes: a slot per clock of the ring. + reg [SW-1:0] dep_slot [H]; + + // ---- The engine ---- + localparam S_IDLE = 4'd0, S_PUSH = 4'd1, S_TICK = 4'd3, S_BANK = 4'd4, + S_SETUP = 4'd5, S_SRCH = 4'd6, S_CHECK = 4'd7, S_COMMIT = 4'd8, S_END = 4'd9, + S_WALK = 4'd10; + localparam K_PRE = 2'd0, K_ACT = 2'd1, K_C0 = 2'd2, K_C1 = 2'd3; + reg [3:0] state; + + assign done = (state == S_IDLE); + + // The operation: the request being reserved, and its commands, as clocks + // relative to e. + reg op_hit, op_conf; + reg signed [15:0] r_act, r_colr, r_prer, r_floor; + reg signed [15:0] c_pre, c_act, c_col0, c_col1; + reg [1:0] kind; + + // The search: the next word to read (as the ring position and the clocks + // left to the horizon) and the one read last cycle, to decide on. + reg [HLOG2-1:0] s_pos; + reg signed [15:0] s_left; + reg s_fetch; + reg sb_valid, b_last; + reg [63:0] b_cand; + reg [WW-1:0] b_word; + + function automatic [63:0] span_mask(input [5:0] off, input signed [15:0] left); + reg [6:0] top; + top = (left >= 16'sd64 - 16'(off)) ? 7'd64 : 7'(7'(off) + 7'(left)); + span_mask = (~64'd0 << off) & ((top == 7'd64) ? ~64'd0 : ~(~64'd0 << top)); + endfunction + + wire [5:0] s_off = s_pos[5:0]; + wire [63:0] s_mask = span_mask(s_off, s_left); + wire signed [15:0] s_take = (s_left >= 16'sd64 - 16'(s_off)) ? 16'sd64 - 16'(s_off) : s_left; + wire [63:0] f_col = rd_cmd | (op_w ? (rd_fwp | rd_fwg) : (rd_frp | rd_frg)); + wire [63:0] f_act = rd_cmd | rd_row | (g_act2 ? {rd_row1[0], rd_row[63:1]} : 64'd0) | rd_fap | rd_fag; + wire [63:0] f_pre = rd_cmd | rd_row; + wire [63:0] forb = (kind == K_PRE) ? f_pre : ((kind == K_ACT) ? f_act : f_col); + + reg [5:0] b_ffs; + always @(*) begin + b_ffs = '0; + for (integer j = 63; j >= 0; --j) + if (b_cand[j]) + b_ffs = 6'(j); + end + wire [HLOG2-1:0] b_pos = HLOG2'({b_word, b_ffs}); + wire signed [15:0] b_rel = 16'(HLOG2'(b_pos - base)); + + // ---- Commit: per calendar, up to two ranges of clocks to mark ---- + reg cm_v [NBM][2]; + reg signed [15:0] cm_lo [NBM][2]; + reg signed [15:0] cm_hi [NBM][2]; + reg [HLOG2-1:0] cm_pos [NBM]; + reg signed [15:0] cm_left[NBM]; + reg cm_busy [NBM]; + reg cm_next [NBM]; // the second range is still to mark + reg cm_start; + + reg signed [15:0] ck_colr, ck_prer, ck_lastcol; + reg cm_any; + always @(*) begin + cm_any = 1'b0; + for (integer m = 0; m < NBM; ++m) + cm_any = cm_any | cm_busy[m]; + end + + // The read data the cycle's departure completes, and the clock's columns. + wire end_depv = rd_depv[base[5:0]]; + wire end_kr = rd_kr[base[5:0]]; + wire end_kw = rd_kw[base[5:0]]; + + // Ingress head, decoded, and whether it enters its buffer this tick. + wire ih_w = iq_w[iq_head]; + wire [TAG_W-1:0] ih_tag = iq_tag[iq_head]; + wire [BKW+ROW_W-1:0] ih_dec = decode(iq_addr[iq_head][29:5]); + wire [8:0] wb_after = 9'(wq_cnt) + 9'(NB) * (9'(wb_count) + 9'd1); + wire [8:0] rb_after = 9'(rq_cnt) + 9'(NB) * (9'(rb_count) + 9'd1); + wire ih_time = ($signed(d - ingress_ready) >= 0); + wire ih_room = ih_w ? (wb_after <= 9'(cfg_wq)) : (rb_after <= 9'(cfg_rq)); + wire move = (iq_count != 0) & ih_time & ih_room; + wire mv_w = move & ih_w; + wire mv_r = move & ~ih_w; + // The buffers' bursts after the move, and the mode they put the channel in. + wire [8:0] wn = 9'(wq_cnt) + 9'(NB) * (9'(wb_count) + 9'(mv_w)); + wire [8:0] rn = 9'(rq_cnt) + 9'(NB) * (9'(rb_count) + 9'(mv_r)); + wire to_write = ~wmode & ((wn >= 9'(cfg_wr_high)) | (rn == 0)); + wire to_read = wmode & (wn < 9'(cfg_wr_low)) & (rn != 0); + wire wmode_n = to_write | (wmode & ~to_read); + wire [BQW:0] sel_count = wmode_n ? (wb_count + (BQW+1)'(mv_w)) + : (rb_count + (BQW+1)'(mv_r)); + + // The end of DRAM clock d: clock d+1's departing read and the columns it + // issues leave the controller; then the next clock. + task automatic end_clock(); + if (end_depv) begin + rf_slot[rf_tail] <= dep_slot[base]; + rf_cyc[rf_tail] <= tcyc + 32'd1 + 32'(cfg_latency); + rf_tail <= rf_tail + 1'b1; + rf_count <= rf_count + 1'b1; + end + rq_cnt <= rq_cnt - (end_kr ? 8'd1 : 8'd0); + wq_cnt <= wq_cnt - (end_kw ? 8'd1 : 8'd0); + // The ring's word ends with this clock: it is all past. + if (base[5:0] == 6'd63) begin + bm_clear <= 1'b1; + bm_clear_word <= end_word; + end + // Times older than any constraint is long move up, so none wraps. + if (~bt_fresh[scrub_b]) begin + if ($signed(bt_act[scrub_b] - (e + 16'd1)) < -16'sd16384) + bt_act[scrub_b] <= e + 16'd1 - 16'd16384; + if ($signed(bt_colr[scrub_b] - (e + 16'd1)) < -16'sd16384) + bt_colr[scrub_b] <= e + 16'd1 - 16'd16384; + if ($signed(bt_prer[scrub_b] - (e + 16'd1)) < -16'sd16384) + bt_prer[scrub_b] <= e + 16'd1 - 16'd16384; + end + scrub_b <= scrub_b + 1'b1; + if ($signed(frontier - e) < 0) frontier <= e; + if ($signed(batch_last - e) < 0) batch_last <= e; + if ($signed(batch_floor - e) < 0) batch_floor <= e; + if ($signed(read_floor - e) < 0) read_floor <= e; + if ($signed(ingress_ready - e) < 0) ingress_ready <= e; + d <= d + 16'd1; + endtask + + always @(posedge clk) begin + bm_clear <= 1'b0; + cm_start <= 1'b0; + + if (reset) begin + state <= S_IDLE; + active <= 1'b0; + stat_reads <= '0; + stat_writes <= '0; + stat_hits <= '0; + stat_misses <= '0; + stat_conflicts <= '0; + stat_latency <= '0; + end + if (reset || finish) begin + // ---- The target cycle ends: the port's state for the next ---- + if (finish && treset) + active <= enable; + if (reset || treset || ~active) begin + slot_busy <= '0; + slots_used <= '0; + inflight <= '0; + cur_r_valid <= 1'b0; + cur_b_valid <= 1'b0; + tcyc <= '0; + rf_head <= '0; + rf_tail <= '0; + rf_count <= '0; + rf_ready <= '0; + af_head <= '0; + af_tail <= '0; + af_count <= '0; + st_w_valid <= 1'b0; + st_r_valid <= 1'b0; + iq_head <= '0; + iq_tail <= '0; + iq_count <= '0; + rb_head <= '0; + rb_tail <= '0; + rb_count <= '0; + wb_head <= '0; + wb_tail <= '0; + wb_count <= '0; + d <= '0; + credit <= 32'(cfg_data_rate); + ingress_ready <= '0; + rq_cnt <= '0; + wq_cnt <= '0; + wmode <= 1'b0; + frontier <= '0; + batch_floor <= '0; + batch_last <= '0; + read_floor <= '0; + bt_open <= '0; + bt_fresh <= '1; + scrub_b <= '0; + end else begin + tcyc <= tcyc + 32'd1; + // R and B, then what is ready next cycle. + begin : release_ + reg r_load, b_load; + r_load = (~cur_r_valid | r_fire) & (rf_ready != 0); + b_load = (~cur_b_valid | b_fire) & (af_count != 0); + if (r_load) begin + cur_r_valid <= 1'b1; + cur_r_slot <= rf_slot[rf_head]; + cur_r_id <= slot_id[rf_slot[rf_head]]; + rf_head <= rf_head + 1'b1; + rf_count <= rf_count - 1'b1; + rf_ready <= rf_ready - 1'b1; + end else if (r_fire) begin + cur_r_valid <= 1'b0; + end + if (b_load) begin + cur_b_valid <= 1'b1; + cur_b_id <= af_id[af_head]; + af_head <= af_head + 1'b1; + af_count <= af_count - 1'b1; + end else if (b_fire) begin + cur_b_valid <= 1'b0; + end + if (r_fire) + slot_busy[cur_r_slot] <= 1'b0; + if (ar_fire) begin + slot_busy[alloc_slot] <= 1'b1; + slot_id[alloc_slot] <= ar_id; + end + slots_used <= slots_used + (SW+1)'(ar_fire) - (SW+1)'(r_fire); + inflight <= inflight + 16'(ar_fire) + 16'(aw_fire) - 16'(new_ready) - 16'(new_acks); + if (ar_fire) + stat_latency <= stat_latency - 64'(tcyc); + if (r_fire) + stat_latency <= stat_latency + 64'(tcyc) - (ar_fire ? 64'(tcyc) : 64'd0); + end + st_w_valid <= new_write; + st_w_id <= nw_id; + st_w_addr <= nw_addr; + st_r_valid <= ar_fire; + st_r_slot <= alloc_slot; + st_r_addr <= ar_addr; + credit <= credit + 32'(cfg_data_rate); + end + if (~reset) + state <= ~(treset ? enable : active) ? S_IDLE + : ((~treset & active & (new_write | ar_fire)) ? S_PUSH : S_TICK); + new_ready <= '0; + new_acks <= '0; + end else begin + case (state) + S_PUSH: begin + // A cycle's write reaches the channel before its read. + if (st_w_valid) begin + iq_w[iq_tail] <= 1'b1; + iq_tag[iq_tail] <= TAG_W'(st_w_id); + iq_addr[iq_tail] <= st_w_addr; + iq_tail <= iq_tail + 1'b1; + iq_count <= iq_count + 1'b1; + st_w_valid <= 1'b0; + end else begin + if (st_r_valid) begin + iq_w[iq_tail] <= 1'b0; + iq_tag[iq_tail] <= TAG_W'(st_r_slot); + iq_addr[iq_tail] <= st_r_addr; + iq_tail <= iq_tail + 1'b1; + iq_count <= iq_count + 1'b1; + st_r_valid <= 1'b0; + end + state <= S_TICK; + end + end + S_TICK: begin + if (credit >= 32'(k_twoclk)) begin + // One DRAM clock d: the ingress head enters its buffer if + // it may, the channel picks reads or writes, and the head + // of that buffer is reserved if it can be. + credit <= credit - 32'(k_twoclk); + // With nothing to reserve the clock ends now; what the + // tick sets below wins over the clock's end. + if (sel_count == 0) + end_clock(); + if (move) begin + iq_head <= iq_head + 1'b1; + iq_count <= iq_count - 1'b1; + ingress_ready <= d + 16'(NB); + if (ih_w) begin + wb_tag[wb_tail] <= ih_tag; + wb_bank[wb_tail] <= ih_dec[BKW+ROW_W-1:ROW_W]; + wb_row[wb_tail] <= ih_dec[ROW_W-1:0]; + wb_tail <= wb_tail + 1'b1; + af_id[af_tail] <= ID_W'(ih_tag); + af_tail <= af_tail + 1'b1; + af_count <= af_count + 1'b1; + new_acks <= new_acks + 1'b1; + end else begin + rb_tag[rb_tail] <= ih_tag; + rb_bank[rb_tail] <= ih_dec[BKW+ROW_W-1:ROW_W]; + rb_row[rb_tail] <= ih_dec[ROW_W-1:0]; + rb_tail <= rb_tail + 1'b1; + end + end + if (move & ih_w) + wb_count <= wb_count + 1'b1; + if (move & ~ih_w) + rb_count <= rb_count + 1'b1; + wmode <= wmode_n; + if (to_write) + batch_floor <= ($signed(frontier + 16'd1 - e) > 0) ? frontier + 16'd1 : e; + if (to_read) + read_floor <= batch_last + 16'd1; + state <= (sel_count != 0) ? S_BANK : S_TICK; + end else begin + state <= S_WALK; + end + end + S_BANK: begin + // The request at the head of the buffer the channel serves, + // against its bank. + begin : bank_ + reg [BKW-1:0] b; + reg [ROW_W-1:0] row; + reg w; + w = wmode; + b = w ? wb_bank[wb_head] : rb_bank[rb_head]; + row = w ? wb_row[wb_head] : rb_row[rb_head]; + op_w <= w; + op_tag <= w ? wb_tag[wb_head] : rb_tag[rb_head]; + op_bank <= b; + op_row <= row; + op_pc <= (g_lpcs != 0) ? b[0] : 1'b0; + op_pg <= 3'((32'(b) & ((32'd1 << (32'(g_lpcs) + 32'(g_lbgs))) - 32'd1))); + op_hit <= bt_open[b] & (bt_row[b] == row); + op_conf <= bt_open[b] & (bt_row[b] != row); + r_act <= bt_fresh[b] ? -16'sd16384 : rel(bt_act[b]); + r_colr <= bt_fresh[b] ? -16'sd1 : rel(bt_colr[b]); + r_prer <= bt_fresh[b] ? -16'sd1 : rel(bt_prer[b]); + r_floor <= rel(w ? batch_floor : read_floor); + kind <= (bt_open[b] & (bt_row[b] == row)) ? K_C0 + : (bt_open[b] ? K_PRE : K_ACT); + end + state <= S_SETUP; + end + S_SETUP: begin + // The search's lower bound, the first clock it may place at. + begin : setup_ + reg signed [15:0] lb; + case (kind) + K_PRE: lb = smax(16'sd0, r_prer); + K_ACT: lb = smax(op_conf ? c_pre + k_rp : 16'sd0, r_act + k_rc); + K_C0: lb = smax(smax(16'sd0, r_floor), + op_hit ? r_colr : c_act + (op_w ? k_rcdwr : k_rcdrd)); + default: lb = c_col0 + k_ccdl1; + endcase + if (lb >= 16'(HE)) begin + state <= S_END; + end else begin + s_pos <= HLOG2'(base + HLOG2'(lb)); + s_left <= 16'(HE) - lb; + s_word <= WW'(HLOG2'(base + HLOG2'(lb)) >> 6); + s_fetch <= 1'b1; + sb_valid <= 1'b0; + state <= S_SRCH; + end + end + end + S_SRCH: begin + // A word a cycle: the word read now, masked to the clocks + // left; and last cycle's word, whose first free clock wins. + if (s_fetch) begin + b_cand <= ~forb & s_mask; + b_word <= s_word; + sb_valid <= 1'b1; + b_last <= (s_left - s_take == 0); + s_pos <= HLOG2'(s_pos + HLOG2'(s_take)); + s_word <= WW'(HLOG2'(s_pos + HLOG2'(s_take)) >> 6); + s_left <= s_left - s_take; + s_fetch <= (s_left - s_take != 0); + end + if (sb_valid) begin + if (b_cand != 0) begin + case (kind) + K_PRE: begin c_pre <= b_rel; kind <= K_ACT; state <= S_SETUP; end + K_ACT: begin c_act <= b_rel; kind <= K_C0; state <= S_SETUP; end + K_C0: begin + c_col0 <= b_rel; + if (NB == 2) begin + kind <= K_C1; + state <= S_SETUP; + end else begin + state <= S_CHECK; + end + end + default: begin c_col1 <= b_rel; state <= S_CHECK; end + endcase + end else if (b_last) begin + state <= S_END; + end + end + end + S_CHECK: begin + // The reservation must reach no further than the horizon. + begin : check_ + reg signed [15:0] lastcol, last, colr, prer; + reg act; + act = ~op_hit; + lastcol = (NB == 2) ? c_col1 : c_col0; + last = lastcol + (op_w ? k_reach_w : k_reach_r); + if (~op_w) + last = smax(last, lastcol + k_dep); + if (act) + last = smax(last, c_act + k_reach_a); + colr = op_hit ? r_colr : c_act + (op_w ? k_rcdwr : k_rcdrd); + prer = op_hit ? r_prer : c_act + k_ras; + ck_lastcol <= lastcol; + ck_colr <= smax(colr, lastcol + 16'sd1); + ck_prer <= smax(prer, lastcol + (op_w ? k_wrpre : k_rtp)); + state <= (last < 16'(HE)) ? S_COMMIT : S_END; + cm_start <= (last < 16'(HE)); + end + end + S_COMMIT: begin + if (cm_start) begin + // The bank, the controller, the read's departure. + if (~op_hit) begin + bt_open[op_bank] <= 1'b1; + bt_row[op_bank] <= op_row; + bt_act[op_bank] <= e + c_act; + end + bt_fresh[op_bank] <= 1'b0; + bt_colr[op_bank] <= e + ck_colr; + bt_prer[op_bank] <= e + ck_prer; + if (op_w) begin + wq_cnt <= wq_cnt + 8'(NB); + wb_head <= wb_head + 1'b1; + wb_count <= wb_count - 1'b1; + batch_last <= ($signed(e + ck_lastcol - batch_last) > 0) ? e + ck_lastcol : batch_last; + stat_writes <= stat_writes + 64'd1; + end else begin + rq_cnt <= rq_cnt + 8'(NB); + rb_head <= rb_head + 1'b1; + rb_count <= rb_count - 1'b1; + dep_slot[HLOG2'(base + HLOG2'(ck_lastcol + k_dep))] <= SW'(op_tag); + stat_reads <= stat_reads + 64'd1; + end + frontier <= ($signed(e + ck_lastcol - frontier) > 0) ? e + ck_lastcol : frontier; + if (op_hit) + stat_hits <= stat_hits + 64'd1; + else if (op_conf) + stat_conflicts <= stat_conflicts + 64'd1; + else + stat_misses <= stat_misses + 64'd1; + end else if (!cm_any) begin + state <= S_END; + end + end + S_END: begin + end_clock(); + state <= S_TICK; + end + S_WALK: begin + // Reads that are ready in the next target cycle. + if (rf_ready != rf_count + && $signed(rf_cyc[SW'(rf_head + SW'(rf_ready))] - (tcyc + 32'd1)) <= 0) begin + rf_ready <= rf_ready + 1'b1; + new_ready <= new_ready + 1'b1; + end else begin + state <= S_IDLE; + end + end + default:; + endcase + end + end + + // ---- Commit: ranges of clocks marked in each calendar, a word a cycle ---- + function automatic signed [15:0] clamp0(input signed [15:0] v); + clamp0 = (v < 0) ? 16'sd0 : v; + endfunction + + // The ranges each calendar marks, from the reservation; an empty one is + // dropped. + always @(*) begin + for (integer m = 0; m < NBM; ++m) begin + for (integer k = 0; k < 2; ++k) begin + cm_v[m][k] = 1'b0; + cm_lo[m][k] = 16'sd0; + cm_hi[m][k] = 16'sd0; + end + end + // Row commands: PRE, ACT; the row bus for PRE, and ACT's one or two + // clocks. Column commands, and their directions for the controller's + // queues. + cm_v[B_CMDR][0] = op_conf; cm_lo[B_CMDR][0] = c_pre; cm_hi[B_CMDR][0] = c_pre; + cm_v[B_CMDR][1] = ~op_hit; cm_lo[B_CMDR][1] = c_act; cm_hi[B_CMDR][1] = c_act; + cm_v[B_ROW][0] = op_conf; cm_lo[B_ROW][0] = c_pre; cm_hi[B_ROW][0] = c_pre; + cm_v[B_ROW][1] = ~op_hit; cm_lo[B_ROW][1] = c_act; + cm_hi[B_ROW][1] = c_act + (g_act2 ? 16'sd1 : 16'sd0); + cm_v[B_CMDC][0] = 1'b1; cm_lo[B_CMDC][0] = c_col0; cm_hi[B_CMDC][0] = c_col0; + cm_v[B_CMDC][1] = (NB == 2); cm_lo[B_CMDC][1] = c_col1; cm_hi[B_CMDC][1] = c_col1; + cm_v[op_w ? B_KW : B_KR][0] = 1'b1; + cm_lo[op_w ? B_KW : B_KR][0] = c_col0; cm_hi[op_w ? B_KW : B_KR][0] = c_col0; + cm_v[op_w ? B_KW : B_KR][1] = (NB == 2); + cm_lo[op_w ? B_KW : B_KR][1] = c_col1; cm_hi[op_w ? B_KW : B_KR][1] = c_col1; + // The read's departure. + cm_v[B_DEPV][0] = ~op_w; + cm_lo[B_DEPV][0] = ck_lastcol + k_dep; cm_hi[B_DEPV][0] = ck_lastcol + k_dep; + // What each column forbids to later columns of its pseudo-channel and + // bank group, by direction. + for (integer k = 0; k < NB; ++k) begin + reg signed [15:0] c; + c = (k == 0) ? c_col0 : c_col1; + if (~op_w) begin + cm_v[B_FRP][k] = 1'b1; cm_lo[B_FRP][k] = clamp0(c - k_ccds + 16'sd1); cm_hi[B_FRP][k] = c + k_ccds - 16'sd1; + cm_v[B_FWP][k] = 1'b1; cm_lo[B_FWP][k] = clamp0(c - k_wtrs + 16'sd1); cm_hi[B_FWP][k] = c + k_rtw - 16'sd1; + cm_v[B_FRG][k] = 1'b1; cm_lo[B_FRG][k] = clamp0(c - k_ccdl + 16'sd1); cm_hi[B_FRG][k] = c + k_ccdl - 16'sd1; + cm_v[B_FWG][k] = 1'b1; cm_lo[B_FWG][k] = clamp0(c - k_wtrl + 16'sd1); cm_hi[B_FWG][k] = c; + end else begin + cm_v[B_FWP][k] = 1'b1; cm_lo[B_FWP][k] = clamp0(c - k_ccds + 16'sd1); cm_hi[B_FWP][k] = c + k_ccds - 16'sd1; + cm_v[B_FRP][k] = 1'b1; cm_lo[B_FRP][k] = clamp0(c - k_rtw + 16'sd1); cm_hi[B_FRP][k] = c + k_wtrs - 16'sd1; + cm_v[B_FWG][k] = 1'b1; cm_lo[B_FWG][k] = clamp0(c - k_ccdl + 16'sd1); cm_hi[B_FWG][k] = c + k_ccdl - 16'sd1; + cm_v[B_FRG][k] = 1'b1; cm_lo[B_FRG][k] = c; cm_hi[B_FRG][k] = c + k_wtrl - 16'sd1; + end + end + // What the ACT forbids to later ACTs: tRRD within the pseudo-channel, + // tRRD_L within the bank group. + cm_v[B_FAP][0] = ~op_hit; + cm_lo[B_FAP][0] = clamp0(c_act - k_rrds + 16'sd1); cm_hi[B_FAP][0] = c_act + k_rrds - 16'sd1; + cm_v[B_FAG][0] = ~op_hit; + cm_lo[B_FAG][0] = clamp0(c_act - k_rrdl + 16'sd1); cm_hi[B_FAG][0] = c_act + k_rrdl - 16'sd1; + for (integer m = 0; m < NBM; ++m) + for (integer k = 0; k < 2; ++k) + if (cm_hi[m][k] < cm_lo[m][k]) + cm_v[m][k] = 1'b0; + end + + // A word a cycle per calendar; a range's last word starts the next. + always @(posedge clk) begin + wr_en <= '0; + for (integer m = 0; m < NBM; ++m) begin + if (reset) begin + cm_busy[m] <= 1'b0; + end else if (cm_start) begin + cm_busy[m] <= cm_v[m][0] | cm_v[m][1]; + cm_next[m] <= cm_v[m][0] & cm_v[m][1]; + cm_pos[m] <= HLOG2'(base + HLOG2'(cm_v[m][0] ? cm_lo[m][0] : cm_lo[m][1])); + cm_left[m] <= cm_v[m][0] ? cm_hi[m][0] - cm_lo[m][0] + 16'sd1 + : cm_hi[m][1] - cm_lo[m][1] + 16'sd1; + end else if (cm_busy[m]) begin + reg signed [15:0] take; + reg [5:0] off; + off = cm_pos[m][5:0]; + take = (cm_left[m] >= 16'sd64 - 16'(off)) ? 16'sd64 - 16'(off) : cm_left[m]; + wr_en[m] <= 1'b1; + wr_word[m] <= WW'(cm_pos[m] >> 6); + wr_mask[m] <= span_mask(off, cm_left[m]); + if (cm_left[m] == take) begin + cm_busy[m] <= cm_next[m]; + cm_next[m] <= 1'b0; + cm_pos[m] <= HLOG2'(base + HLOG2'(cm_lo[m][1])); + cm_left[m] <= cm_hi[m][1] - cm_lo[m][1] + 16'sd1; + end else begin + cm_pos[m] <= HLOG2'(cm_pos[m] + HLOG2'(take)); + cm_left[m] <= cm_left[m] - take; + end + end + end + end + +endmodule diff --git a/hw/rtl/afu/foldsim/foldsim_mem_bmap.sv b/hw/rtl/afu/foldsim/foldsim_mem_bmap.sv new file mode 100644 index 0000000000..efe7aef633 --- /dev/null +++ b/hw/rtl/afu/foldsim/foldsim_mem_bmap.sv @@ -0,0 +1,79 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// `INST` bitmaps over the bank model's ring of DRAM clocks (foldsim_mem_bank), +// each WORDS words of 64 clocks, in distributed RAM. A word with no write +// since the last `clear` of it reads zero: a valid bit per word stands for +// the clearing, so a word of every bitmap is cleared in one cycle. A write +// ORs a mask into a word. +module foldsim_mem_bmap #( + parameter INST = 1, + parameter WORDS = 8, + parameter NREAD = 1, + parameter IW = (INST > 1) ? $clog2(INST) : 1, + parameter WW = $clog2(WORDS) +) ( + input wire clk, + input wire reset, + // Every word of every bitmap reads zero from the next cycle on. + input wire clear_all, + // Word `clear_word` of every bitmap reads zero from the next cycle on. + input wire clear, + input wire [WW-1:0] clear_word, + + input wire [NREAD*IW-1:0] rd_inst, + input wire [NREAD*WW-1:0] rd_word, + output wire [NREAD*64-1:0] rd_data, + + input wire we, + input wire [IW-1:0] wr_inst, + input wire [WW-1:0] wr_word, + input wire [63:0] wr_mask +); + localparam N = INST * WORDS; + localparam AW = (N > 1) ? $clog2(N) : 1; + + reg [63:0] mem [N]; + reg [N-1:0] valid; + + function automatic [AW-1:0] addr(input [IW-1:0] inst, input [WW-1:0] word); + addr = AW'(32'(inst) * WORDS + 32'(word)); + endfunction + + for (genvar i = 0; i < NREAD; ++i) begin : g_rd + wire [AW-1:0] a = addr(rd_inst[i*IW +: IW], rd_word[i*WW +: WW]); + assign rd_data[i*64 +: 64] = valid[a] ? mem[a] : 64'd0; + end + + wire [AW-1:0] wa = addr(wr_inst, wr_word); + wire [63:0] old = valid[wa] ? mem[wa] : 64'd0; + + always @(posedge clk) begin + if (we) + mem[wa] <= old | wr_mask; + end + + always @(posedge clk) begin + if (reset || clear_all) begin + valid <= '0; + end else begin + if (clear) begin + for (integer k = 0; k < INST; ++k) + valid[k * WORDS + 32'(clear_word)] <= 1'b0; + end + if (we) + valid[wa] <= 1'b1; + end + end + +endmodule diff --git a/hw/rtl/afu/foldsim/foldsim_regs.vh b/hw/rtl/afu/foldsim/foldsim_regs.vh index 916c03bd43..1f84e3db39 100644 --- a/hw/rtl/afu/foldsim/foldsim_regs.vh +++ b/hw/rtl/afu/foldsim/foldsim_regs.vh @@ -22,7 +22,7 @@ `define FOLDSIM_REGS_VH `define FOLDSIM_ID 32'h464F4C44 -`define FOLDSIM_VERSION 32'd2 +`define FOLDSIM_VERSION 32'd3 `define FOLDSIM_REG_ID 12'h000 `define FOLDSIM_REG_VERSION 12'h004 @@ -45,8 +45,19 @@ `define FOLDSIM_REG_MEM_WLAT 12'h0CC `define FOLDSIM_REG_MEM_MAXR 12'h0D0 `define FOLDSIM_REG_MEM_MAXW 12'h0D4 +// The memory model (sw/common/foldsim_regs.h has the fields). +`define FOLDSIM_REG_MEM_MODEL 12'h0D8 +`define FOLDSIM_REG_MEM_CAPS 12'h0DC +`define FOLDSIM_REG_MEM_CLOCK 12'h0E0 +`define FOLDSIM_REG_MEM_LATENCY 12'h0E4 +`define FOLDSIM_REG_MEM_QUEUE 12'h0E8 +`define FOLDSIM_REG_MEM_GEOM 12'h0EC +`define FOLDSIM_REG_MEM_WMARK 12'h0F0 // Per class c: the host cycles a thread of it finished its target cycle. `define FOLDSIM_REG_FINISH_BASE 12'h100 +// The bank model's DRAM timings, a word each, and its counters, 64-bit each. +`define FOLDSIM_REG_MEM_TIMING 12'h800 +`define FOLDSIM_REG_MEM_STATS 12'hC00 `define FOLDSIM_CMD_NONE 3'd0 `define FOLDSIM_CMD_DCR_WRITE 3'd1 diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index 2ca729a549..72422448ed 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -162,7 +162,8 @@ $(GEN_SV): $(TARGET_SRCS) $(FOLDSIM_TOOLS) $(CLASSES_STAMP) $(KERNEL_SRCS): $(GEN_SV) @{ echo "+incdir+$(FOLDSIM_RTL_DIR)"; echo "+incdir+$(GEN_DIR)"; \ echo "$(GEN_SV)"; \ - for f in foldsim_token.sv foldsim_mem.sv foldsim_ctrl.sv foldsim_afu.sv; do echo "$(FOLDSIM_RTL_DIR)/$$f"; done; \ + for f in foldsim_token.sv foldsim_mem_bmap.sv foldsim_mem_bank.sv foldsim_mem.sv foldsim_ctrl.sv \ + foldsim_afu.sv; do echo "$(FOLDSIM_RTL_DIR)/$$f"; done; \ } > $@ # The kernel's clock at FOLDSIM_FREQ, for its out-of-context synthesis. diff --git a/hw/unittest/foldsim_mem/Makefile b/hw/unittest/foldsim_mem/Makefile index 74e830f037..d1c66fb359 100644 --- a/hw/unittest/foldsim_mem/Makefile +++ b/hw/unittest/foldsim_mem/Makefile @@ -8,7 +8,7 @@ FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) -CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) SRCS := $(SRC_DIR)/main.cpp diff --git a/hw/unittest/foldsim_mem/main.cpp b/hw/unittest/foldsim_mem/main.cpp index 0afda874b5..4fcc87d338 100644 --- a/hw/unittest/foldsim_mem/main.cpp +++ b/hw/unittest/foldsim_mem/main.cpp @@ -13,41 +13,106 @@ // foldsim's memory endpoint (hw/rtl/afu/foldsim/foldsim_mem.sv), driven as the // simulator and the control endpoint drive it -- one token per channel per -// target cycle -- by a model of an AXI4 master, against a model of HBM that -// checks the AXI4 protocol on the device side. The properties under test: -// * read data, write acknowledgements and ids are right, and no response -// comes before its latency; +// target cycle -- by a model of an AXI4 master, against a model of HBM with +// random ready and latency that checks the AXI4 protocol on the device side. +// Under each timing model and a range of its settings: +// * every target cycle's outputs equal the C++ model's +// (sim/common/axi_latency_pipe.h for the pipe, sim/common/dram_bank_model.h +// for the bank model), the model stepped on the same requests; +// * read data, write acknowledgements and ids are right; // * a reset with HBM transactions in flight finishes only once HBM has // answered them all, takes no request, and delivers nothing from before // it to the target after it; // * a burst sets `error` and reaches HBM in no part, and a reset clears it. +// +// ./foldsim_mem every setup +// ./foldsim_mem one setup #include #include #include #include #include +#include #include #include +#include #include #include #include +#include +#include + #include "Vfoldsim_mem.h" #include "verilated.h" +using namespace vortex; + namespace { constexpr int kWords = 16; // DATA_W = 512 -constexpr uint32_t kReadLatency = 10; -constexpr uint32_t kWriteLatency = 12; constexpr uint64_t kBase = 0x10000000ull; -constexpr uint64_t kLines = 64; // target addresses in use -constexpr uint64_t kMagic = 0x7f000ull; // offered only in reset cycles -constexpr uint64_t kBurstAddr = 0x7e000ull; // offered only as a burst +constexpr uint64_t kMagic = 0x3ff7f000ull; // offered only in reset cycles +constexpr uint64_t kBurstAddr = 0x3ff7e000ull; // offered only as a burst using Line = std::array; +// The traffic a setup makes: the addresses its requests take. +enum class Traffic { + Small, // 64 lines, 4 KB: row hits + Random, // anywhere in 64 MB + Rows, // random rows 32 KB apart: row conflicts + Stream, // sequential, a few streams + OneBank, // random rows of one bank: conflicts back to back, to the horizon +}; + +struct Setup { + const char* name; + bool bank; + // The pipe. + uint32_t rlat, wlat, maxr, maxw; + // The bank model. + dram_timing::Standard type; + uint32_t core_clock, data_rate, latency, queue; + uint32_t rq, wq, wr_high, wr_low; + Traffic traffic; + double p_read, p_write, p_ready; + uint64_t cycles; +}; + +const Setup kSetups[] = { + {"pipe", false, 10, 12, 16, 16, dram_timing::Standard::HBM2, 0, 0, 0, 0, 0, 0, 0, 0, + Traffic::Small, 0.3, 0.25, 1.0, 3000}, + {"pipe_slow", false, 37, 5, 6, 3, dram_timing::Standard::HBM2, 0, 0, 0, 0, 0, 0, 0, 0, + Traffic::Random, 0.5, 0.3, 0.7, 3000}, + {"bank", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, + Traffic::Small, 0.3, 0.25, 1.0, 3000}, + {"bank_random", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, + Traffic::Random, 0.5, 0.3, 0.8, 4000}, + {"bank_rows", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 3, 0, 32, 32, 26, 7, + Traffic::Rows, 0.6, 0.2, 0.9, 4000}, + {"bank_stream", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, + Traffic::Stream, 0.7, 0.4, 0.9, 4000}, + {"bank_a100", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 1410, 3038, 190, 257, 32, 32, 26, 7, + Traffic::Random, 0.5, 0.3, 0.8, 4000}, + {"bank_h100", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + Traffic::Rows, 0.5, 0.3, 0.8, 4000}, + {"bank_h100_s", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + Traffic::Stream, 0.8, 0.5, 0.95, 4000}, + {"bank_queues", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 700, 3200, 7, 5, 6, 4, 3, 2, + Traffic::Random, 0.6, 0.5, 0.7, 4000}, + {"bank_reads", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, + Traffic::Random, 0.9, 0.05, 0.9, 4000}, + {"bank_onebank", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + Traffic::OneBank, 0.9, 0.1, 0.9, 4000}, + {"bank_onebank2", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 11, 3, 32, 32, 26, 7, + Traffic::OneBank, 0.7, 0.3, 0.8, 4000}, + {"bank_v100", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 1530, 1752, 0, 0, 32, 32, 26, 7, + Traffic::Rows, 0.8, 0.2, 0.9, 4000}, +}; + +const Setup* setup = nullptr; Vfoldsim_mem* dut = nullptr; std::mt19937 rng(7); uint64_t host_cycle = 0; @@ -58,8 +123,8 @@ int failures = 0; do { \ if (!(cond)) { \ if (failures++ < 20) { \ - printf("FAIL host %lu target %lu: ", (unsigned long)host_cycle, \ - (unsigned long)tcycle); \ + printf("FAIL %s host %lu target %lu: ", setup->name, \ + (unsigned long)host_cycle, (unsigned long)tcycle); \ printf(__VA_ARGS__); \ printf("\n"); \ } \ @@ -76,16 +141,37 @@ Line make_line(uint64_t addr, uint32_t version) { return l; } +// ---- The C++ models, stepped on what the target offered ---- +std::unique_ptr golden_pipe; +std::unique_ptr golden_bank; +bool golden_valid = true; // the pipe models no bursts: void until a reset + +DramBankAxi::Config bank_config(const Setup& s) { + DramBankAxi::Config c; + c.model = DramBankModel::Config::make(s.type, s.data_rate, 64); + c.model.read_queue = s.rq; + c.model.write_queue = s.wq; + c.model.wr_high = s.wr_high; + c.model.wr_low = s.wr_low; + c.core_clock = s.core_clock; + c.data_rate = s.data_rate; + c.latency = s.latency; + c.queue = s.queue; + c.read_slots = 32; + c.write_slots = 16; + return c; +} + // ---- HBM: an AXI4 slave with random ready and latency ---- struct Hbm { std::map mem; double ready_p = 0.7; int lat_lo = 1, lat_hi = 40; struct Pending { uint64_t at; Line data; }; - std::deque r; // reads accepted, not returned - std::deque b; // writes done, not acknowledged - std::deque aw; // AWs waiting for their W - std::deque> w; // Ws waiting for their AW + std::deque r; + std::deque b; + std::deque aw; + std::deque> w; std::set ar_seen; uint64_t n_ar = 0, n_aw = 0, n_w = 0; bool arready = false, awready = false, wready = false; @@ -181,55 +267,91 @@ struct Hbm { Hbm hbm; // ---- The target: an AXI4 master issuing single-beat reads and writes ---- +// Like Vortex's AXI adapter, it has no two reads with one id outstanding. struct Op { uint64_t addr; int id; Line data; }; struct Master { bool generate = false; - bool reset = false; // this target cycle is a reset cycle + bool reset = false; bool offer_magic = false; - // A burst to offer: 0 none, 1 AR, 2 AW, 3 a W without WLAST. - int burst = 0; + int burst = 0; // 0 none, 1 AR, 2 AW, 3 a W without WLAST + bool rready = true, bready = true; - // What each channel offers this cycle, held until it fires. bool ar_v = false, aw_v = false, w_v = false; Op ar{}, aw{}, w{}; int ar_len = 0, aw_len = 0; bool w_last = true; - std::deque ar_q, aw_q, w_q; // queued, not yet offered + std::deque ar_q, aw_q, w_q; struct ReadExp { Op op; bool known; uint64_t fired; }; - std::deque reads; // fired, awaiting R - std::deque aw_only, w_only; // fired halves not yet formed + std::map reads; // fired, awaiting R, by id + std::deque aw_only, w_only; struct WriteExp { Op op; uint64_t formed; }; - std::deque writes; // formed, awaiting B + std::deque writes; - std::map known; // data of every acknowledged write - std::set unknown; // writes a reset left undecided - std::map busy; // reads or writes in flight per address + std::map known; + std::set unknown; + std::map busy; + std::set pending_writes; + std::set free_rids; uint32_t version = 1; - int next_id = 0; + int next_wid = 0; + uint64_t stream = 0; uint64_t n_r = 0, n_b = 0, stale = 0; + Master() { this->clear_ids(); } + + void clear_ids() { + free_rids.clear(); + for (int i = 0; i < 16; ++i) + free_rids.insert(i); + } + bool idle() const { return ar_q.empty() && aw_q.empty() && w_q.empty() && reads.empty() && aw_only.empty() && w_only.empty() && writes.empty() && !ar_v && !aw_v && !w_v; } + uint64_t pick_addr() { + switch (setup->traffic) { + case Traffic::Small: return uint64_t(rand_int(0, 63)) * 64; + case Traffic::Random: return uint64_t(rand_int(0, (1 << 20) - 1)) * 64; + case Traffic::Rows: return uint64_t(rand_int(0, 63)) * 32768 + uint64_t(rand_int(0, 3)) * 64; + case Traffic::Stream: { + uint64_t s = uint64_t(rand_int(0, 3)); + return (s << 22) + ((stream++ & 0xfff) * 64); + } + case Traffic::OneBank: { + // A random row, and the block of it the bank model maps to bank 5. + for (;;) { + uint64_t a = uint64_t(rand_int(0, 255)) * 32768 + uint64_t(rand_int(0, 511)) * 64; + uint32_t bank, row; + golden_bank->model().decode(a, &bank, &row); + if (bank == 5) + return a; + } + } + } + return 0; + } + void make_requests() { if (!generate || reset) return; - if (ar_q.size() + reads.size() < 24 && chance(0.3)) { - const uint64_t a = uint64_t(rand_int(0, kLines - 1)) * 64; - if (busy[a] == 0 || (writes_to(a) == 0)) { + if (ar_q.size() + reads.size() < 16 && !free_rids.empty() && chance(setup->p_read)) { + const uint64_t a = this->pick_addr(); + if (!pending_writes.count(a)) { ++busy[a]; - ar_q.push_back({a, next_id++ & 15, {}}); + const int id = *free_rids.begin(); + free_rids.erase(free_rids.begin()); + ar_q.push_back({a, id, {}}); } } - if (aw_q.size() + writes.size() < 24 && chance(0.25)) { - const uint64_t a = uint64_t(rand_int(0, kLines - 1)) * 64; + if (aw_q.size() + writes.size() < 24 && chance(setup->p_write)) { + const uint64_t a = this->pick_addr(); if (busy[a] == 0) { ++busy[a]; - Op op{a, next_id++ & 15, make_line(a, version++)}; + Op op{a, next_wid++ & 15, make_line(a, version++)}; aw_q.push_back(op); w_q.push_back(op); pending_writes.insert(a); @@ -237,11 +359,9 @@ struct Master { } } - std::set pending_writes; - int writes_to(uint64_t a) const { return pending_writes.count(a) ? 1 : 0; } - - // The channels' tokens for this cycle. void offer() { + rready = chance(setup->p_ready); + bready = chance(setup->p_ready); if (reset) { ar_v = offer_magic; ar = {kMagic, 0, {}}; @@ -287,21 +407,18 @@ struct Master { busy.erase(a); } - // One target cycle, given the endpoint's tokens for it. void cycle(bool arready, bool awready, bool wready, bool rvalid, const Line& rdata, int rid, bool bvalid, int bid) { if (reset) { - // The target is in reset: it ignores its inputs and loses its state. - for (auto& r : reads) - read_done(r.op.addr); + for (auto& kv : reads) + read_done(kv.second.op.addr); for (auto* q : {&aw_only, &w_only}) for (auto& op : *q) unknown.insert(op.addr); for (auto& wr : writes) unknown.insert(wr.op.addr); - for (auto* q : {&ar_q}) - for (auto& op : *q) - read_done(op.addr); + for (auto& op : ar_q) + read_done(op.addr); for (auto& op : aw_q) unknown.insert(op.addr); for (auto& op : w_q) @@ -319,25 +436,28 @@ struct Master { ar_v = aw_v = w_v = false; busy.clear(); pending_writes.clear(); + this->clear_ids(); return; } - if (rvalid) { + if (rvalid && rready) { ++n_r; - if (reads.empty()) { + auto it = reads.find(rid); + if (it == reads.end()) { ++stale; - CHECK(false, "R with no read outstanding"); + CHECK(false, "R id %d with no read outstanding", rid); } else { - const ReadExp e = reads.front(); - reads.pop_front(); - CHECK(rid == e.op.id, "R id %d, expected %d", rid, e.op.id); - CHECK(tcycle >= e.fired + kReadLatency, "R at %lu for a read fired at %lu", - (unsigned long)tcycle, (unsigned long)e.fired); + const ReadExp e = it->second; + reads.erase(it); + free_rids.insert(rid); + if (!setup->bank) + CHECK(tcycle >= e.fired + setup->rlat, "R at %lu for a read fired at %lu", + (unsigned long)tcycle, (unsigned long)e.fired); if (e.known) CHECK(rdata == e.op.data, "R data wrong at 0x%lx", (unsigned long)e.op.addr); read_done(e.op.addr); } } - if (bvalid) { + if (bvalid && bready) { ++n_b; if (writes.empty()) { ++stale; @@ -346,8 +466,9 @@ struct Master { const WriteExp e = writes.front(); writes.pop_front(); CHECK(bid == e.op.id, "B id %d, expected %d", bid, e.op.id); - CHECK(tcycle >= e.formed + kWriteLatency, "B at %lu for a write formed at %lu", - (unsigned long)tcycle, (unsigned long)e.formed); + if (!setup->bank) + CHECK(tcycle >= e.formed + setup->wlat, "B at %lu for a write formed at %lu", + (unsigned long)tcycle, (unsigned long)e.formed); known[e.op.addr] = e.op.data; unknown.erase(e.op.addr); pending_writes.erase(e.op.addr); @@ -361,7 +482,7 @@ struct Master { const bool k = !unknown.count(ar.addr); ReadExp e{ar, k, tcycle}; e.op.data = (it != known.end()) ? it->second : Line{}; - reads.push_back(e); + reads[ar.id] = e; } else { burst = 0; } @@ -402,10 +523,8 @@ bool taken[NUM_IN], got[NUM_OUT]; bool o_awready, o_wready, o_arready, o_rvalid, o_bvalid; int o_rid, o_bid; Line o_rdata; -// Set when a reset cycle's tokens are all exchanged: the endpoint's next -// output token says it has finished that cycle. bool after_reset = false; -uint64_t reset_cycles_done = 0, drained_at_reset = 0; +uint64_t reset_cycles_done = 0, drained_at_reset = 0, compared = 0; void drive_tokens() { const Master& m = master; @@ -426,8 +545,8 @@ void drive_tokens() { IN(I_ARADDR, araddr, m.ar.addr); IN(I_ARID, arid, m.ar.id); IN(I_ARLEN, arlen, m.ar_len); - IN(I_RREADY, rready, 1); - IN(I_BREADY, bready, 1); + IN(I_RREADY, rready, m.rready); + IN(I_BREADY, bready, m.bready); IN(I_TRESET, treset, m.reset); #undef IN #define OUT(ch, name) dut->name##_ready = !got[ch] @@ -475,8 +594,6 @@ void host_step() { for (int j = 0; j < NUM_OUT; ++j) any_out |= out_take[j]; if (after_reset && any_out) { - // The first token of the cycle after a reset cycle: that cycle has - // finished, so HBM has answered everything and nothing is offered to it. after_reset = false; ++reset_cycles_done; CHECK(hbm.outstanding() == 0, "reset cycle finished with %zu HBM transactions open", @@ -494,7 +611,75 @@ void host_step() { got[j] |= out_take[j]; } -// Runs target cycles until `n` have completed. +// The cycle's outputs against the C++ model's; then the model takes the +// cycle's requests. +void golden_cycle() { + const Master& m = master; + if (m.reset) { + if (golden_pipe) golden_pipe->reset(); + if (golden_bank) golden_bank->reset(); + golden_valid = true; + return; + } + if (golden_valid) { + ++compared; + if (golden_bank) { + auto& g = *golden_bank; + CHECK(o_arready == g.ar_ready(), "ARREADY %d, model %d", o_arready, g.ar_ready()); + CHECK(o_awready == g.aw_ready(), "AWREADY %d, model %d", o_awready, g.aw_ready()); + CHECK(o_wready == g.w_ready(), "WREADY %d, model %d", o_wready, g.w_ready()); + CHECK(o_rvalid == g.r_valid(), "RVALID %d, model %d", o_rvalid, g.r_valid()); + if (o_rvalid && g.r_valid()) + CHECK(uint32_t(o_rid) == g.r_id(), "RID %d, model %u", o_rid, g.r_id()); + CHECK(o_bvalid == g.b_valid(), "BVALID %d, model %d", o_bvalid, g.b_valid()); + if (o_bvalid && g.b_valid()) + CHECK(uint32_t(o_bid) == g.b_id(), "BID %d, model %u", o_bid, g.b_id()); + DramBankAxi::Inputs in; + in.ar_valid = m.ar_v; + in.ar_id = m.ar.id; + in.ar_addr = m.ar.addr; + in.ar_len = m.ar_len; + in.aw_valid = m.aw_v; + in.aw_id = m.aw.id; + in.aw_addr = m.aw.addr; + in.aw_len = m.aw_len; + in.w_valid = m.w_v; + in.w_last = m.w_last; + in.r_ready = m.rready; + in.b_ready = m.bready; + g.step(in); + } else { + auto& g = *golden_pipe; + CHECK(o_arready == g.ar_ready(), "ARREADY %d, model %d", o_arready, g.ar_ready()); + CHECK(o_awready == g.aw_ready(), "AWREADY %d, model %d", o_awready, g.aw_ready()); + CHECK(o_wready == g.w_ready(), "WREADY %d, model %d", o_wready, g.w_ready()); + CHECK(o_rvalid == g.r_valid(), "RVALID %d, model %d", o_rvalid, g.r_valid()); + if (o_rvalid && g.r_valid()) + CHECK(uint32_t(o_rid) == g.r_id(), "RID %d, model %u", o_rid, g.r_id()); + CHECK(o_bvalid == g.b_valid(), "BVALID %d, model %d", o_bvalid, g.b_valid()); + if (o_bvalid && g.b_valid()) + CHECK(uint32_t(o_bid) == g.b_id(), "BID %d, model %u", o_bid, g.b_id()); + AxiLatencyPipe::Inputs in; + in.ar_valid = m.ar_v; + in.ar_id = m.ar.id; + in.ar_len = m.ar_len; + in.aw_valid = m.aw_v; + in.aw_id = m.aw.id; + in.w_valid = m.w_v; + in.w_last = m.w_last; + in.r_ready = m.rready; + in.b_ready = m.bready; + // The pipe has no error: a burst it takes ends the comparison until a + // reset. + if ((in.ar_valid && g.ar_ready() && in.ar_len != 0) + || (in.aw_valid && g.aw_ready() && m.aw_len != 0) + || (in.w_valid && g.w_ready() && !in.w_last)) + golden_valid = false; + g.step(in); + } + } +} + void run(uint64_t n) { for (uint64_t k = 0; k < n; ++k) { master.make_requests(); @@ -516,12 +701,13 @@ void run(uint64_t n) { break; host_step(); if (++spins > 200000) { - printf("FAIL: target cycle %lu never completes\n", (unsigned long)tcycle); + printf("FAIL %s: target cycle %lu never completes\n", setup->name, (unsigned long)tcycle); exit(1); } } if (master.reset) after_reset = true; + golden_cycle(); master.cycle(o_arready, o_awready, o_wready, o_rvalid, o_rdata, o_rid, o_bvalid, o_bid); ++tcycle; } @@ -553,7 +739,6 @@ void burst_case(int kind, const char* what) { run(2); CHECK(dut->error, "%s: no error", what); CHECK(!o_arready && !o_awready && !o_wready, "%s: still ready after the error", what); - // Requests now wait; the endpoint stays stopped. master.generate = true; run(50); master.generate = false; @@ -565,16 +750,64 @@ void burst_case(int kind, const char* what) { run(800); } -} // namespace +void configure() { + const Setup& s = *setup; + dut->read_latency = s.rlat ? s.rlat : 30; + dut->write_latency = s.wlat ? s.wlat : 30; + dut->max_reads = s.maxr ? s.maxr : 16; + dut->max_writes = s.maxw ? s.maxw : 16; + dut->base = kBase; + dut->model_bank = s.bank; + if (!s.bank) { + AxiLatencyPipe::Config c; + c.read_latency = s.rlat; + c.write_latency = s.wlat; + c.max_reads = s.maxr; + c.max_writes = s.maxw; + golden_pipe = std::make_unique(c); + golden_bank.reset(); + return; + } + const auto cfg = bank_config(s); + const auto& t = cfg.model.t; + const auto& g = cfg.model.geom; + dut->cfg_data_rate = s.data_rate; + dut->cfg_core_clock = s.core_clock; + dut->cfg_latency = s.latency; + dut->cfg_queue = s.queue; + dut->cfg_rq = s.rq; + dut->cfg_wq = s.wq; + dut->cfg_wr_high = s.wr_high; + dut->cfg_wr_low = s.wr_low; + dut->cfg_geom = g.log2_pcs | (g.log2_bgs << 2) | (g.log2_banks << 4) | (g.log2_row_bursts << 6) + | (g.log2_visit << 10) | (uint32_t(g.two_clock_act) << 12); + const uint32_t v[18] = {t.BL, t.CCDS, t.CCDL, t.CL, t.CWL, t.RCDRD, t.RCDWR, t.RP, t.RAS, t.RC, + t.RTP, t.WR, t.RRDS, t.RRDL, t.FAW, t.WTRS, t.WTRL, g.rtw_extra}; + for (int w = 0; w < 7; ++w) + dut->cfg_t[w] = 0; + for (int i = 0; i < 18; ++i) + for (int b = 0; b < 12; ++b) + if ((v[i] >> b) & 1) + dut->cfg_t[(i * 12 + b) / 32] |= 1u << ((i * 12 + b) % 32); + golden_bank = std::make_unique(cfg); + golden_pipe.reset(); +} + +int run_setup(const Setup& s) { + setup = &s; + rng.seed(7); + host_cycle = tcycle = 0; + reset_cycles_done = drained_at_reset = compared = 0; + hbm = Hbm(); + master = Master(); + golden_valid = true; + for (int i = 0; i < NUM_IN; ++i) taken[i] = false; + for (int j = 0; j < NUM_OUT; ++j) got[j] = false; + after_reset = false; + const int failures_before = failures; -int main(int argc, char** argv) { - Verilated::commandArgs(argc, argv); dut = new Vfoldsim_mem(); - dut->read_latency = kReadLatency; - dut->write_latency = kWriteLatency; - dut->max_reads = 16; - dut->max_writes = 16; - dut->base = kBase; + configure(); dut->reset = 1; for (int i = 0; i < 8; ++i) { hbm.drive(); @@ -587,7 +820,7 @@ int main(int argc, char** argv) { reset_cycles(4, true); master.generate = true; - run(3000); + run(s.cycles); // Resets with HBM slow and busy, at random points. for (int k = 0; k < 8; ++k) { @@ -599,7 +832,7 @@ int main(int argc, char** argv) { hbm.lat_hi = 40; hbm.ready_p = 0.7; reset_cycles(rand_int(1, 4), true); - run(1500); + run(s.cycles / 2); } burst_case(1, "AR burst"); @@ -614,14 +847,43 @@ int main(int argc, char** argv) { CHECK(!hbm.ar_seen.count(kBase + kBurstAddr), "a burst reached HBM"); CHECK(dut->idle, "endpoint not idle at the end"); CHECK(master.stale == 0, "%lu stale responses", (unsigned long)master.stale); - CHECK(drained_at_reset >= 4, "only %lu resets found HBM busy", (unsigned long)drained_at_reset); - - printf("target cycles %lu, host cycles %lu; %lu reads, %lu writes acknowledged; " - "%lu reset cycles, %lu of them with HBM busy\n", - (unsigned long)tcycle, (unsigned long)host_cycle, (unsigned long)master.n_r, - (unsigned long)master.n_b, (unsigned long)reset_cycles_done, - (unsigned long)drained_at_reset); + // One bank's conflicts leave too little in flight for every reset to meet. + if (s.traffic != Traffic::OneBank) + CHECK(drained_at_reset >= 4, "only %lu resets found HBM busy", (unsigned long)drained_at_reset); + + std::string extra; + if (golden_bank) { + const auto& st = golden_bank->model().stats(); + char buf[200]; + snprintf(buf, sizeof(buf), "; model since last reset: %lu reads, %lu writes, %lu hits, %lu misses, %lu conflicts, %lu deferred", + (unsigned long)st.reads, (unsigned long)st.writes, (unsigned long)st.hits, + (unsigned long)st.misses, (unsigned long)st.conflicts, (unsigned long)st.deferred); + extra = buf; + } + printf("%-12s target cycles %lu, host cycles %lu, %lu compared; %lu reads, %lu writes; " + "%lu reset cycles, %lu with HBM busy%s: %s\n", + s.name, (unsigned long)tcycle, (unsigned long)host_cycle, (unsigned long)compared, + (unsigned long)master.n_r, (unsigned long)master.n_b, (unsigned long)reset_cycles_done, + (unsigned long)drained_at_reset, extra.c_str(), + failures == failures_before ? "ok" : "FAILED"); delete dut; + dut = nullptr; + return failures - failures_before; +} + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + const char* only = nullptr; + for (int i = 1; i < argc; ++i) + if (argv[i][0] != '+') + only = argv[i]; + for (const auto& s : kSetups) { + if (only && std::strcmp(only, s.name)) + continue; + run_setup(s); + } if (failures != 0) { printf("FAILED: %d check(s)\n", failures); return 1; diff --git a/sim/foldsim/Makefile b/sim/foldsim/Makefile index e2cd0266c9..503f936e95 100644 --- a/sim/foldsim/Makefile +++ b/sim/foldsim/Makefile @@ -25,7 +25,8 @@ THREADS ?= $(shell python3 -c 'import multiprocessing as mp; print(min(32, mp.cp VL_FLAGS += -j $(THREADS) TOP := foldsim_afu -RTL_SRCS := $(GEN_SV) $(FOLDSIM_RTL_DIR)/foldsim_token.sv $(FOLDSIM_RTL_DIR)/foldsim_mem.sv \ +RTL_SRCS := $(GEN_SV) $(FOLDSIM_RTL_DIR)/foldsim_token.sv $(FOLDSIM_RTL_DIR)/foldsim_mem_bmap.sv \ + $(FOLDSIM_RTL_DIR)/foldsim_mem_bank.sv $(FOLDSIM_RTL_DIR)/foldsim_mem.sv \ $(FOLDSIM_RTL_DIR)/foldsim_ctrl.sv $(FOLDSIM_RTL_DIR)/foldsim_afu.sv SRCS := $(SRC_DIR)/foldsim_emul.cpp diff --git a/sw/common/foldsim_regs.h b/sw/common/foldsim_regs.h index 9d7870785a..e83ba3fb97 100644 --- a/sw/common/foldsim_regs.h +++ b/sw/common/foldsim_regs.h @@ -19,7 +19,7 @@ #pragma once #define FOLDSIM_ID 0x464F4C44u -#define FOLDSIM_VERSION 2u +#define FOLDSIM_VERSION 3u #define FOLDSIM_REG_ID 0x000u #define FOLDSIM_REG_VERSION 0x004u @@ -44,6 +44,44 @@ #define FOLDSIM_REG_MEM_MAXW 0x0D4u // most writes in flight #define FOLDSIM_REG_FINISH(c) (0x100u + 8u * (c)) // threads of class c finished, lo/hi +// The memory model: the latency-bandwidth pipe above, or the bank model +// (hw/rtl/afu/foldsim/foldsim_mem_bank.sv), the memory port's timing as a DRAM +// channel's banks. A reset command's CMD_ADDR chooses it, a FOLDSIM_MEM_MODEL_* +// the target has from that reset on (the kernel's own reset: the pipe). Times +// are DRAM clocks. +#define FOLDSIM_REG_MEM_MODEL 0x0D8u // read only: [1:0] the FOLDSIM_MEM_MODEL_* chosen +#define FOLDSIM_REG_MEM_CAPS 0x0DCu // read only: FOLDSIM_MEM_CAPS_* +#define FOLDSIM_REG_MEM_CLOCK 0x0E0u // [15:0] data rate (MT/s), [31:16] core clock (MHz) +#define FOLDSIM_REG_MEM_LATENCY 0x0E4u // fixed read latency, target cycles +#define FOLDSIM_REG_MEM_QUEUE 0x0E8u // [15:0] requests in flight (0: slots only), + // [23:16] read queue, [31:24] write queue (bursts) +#define FOLDSIM_REG_MEM_GEOM 0x0ECu // log2 of: [1:0] pseudo-channels, [3:2] bank groups, + // [5:4] banks per group, [9:6] bursts per row, + // [11:10] bursts per row visit; [12] two-clock ACT +#define FOLDSIM_REG_MEM_WMARK 0x0F0u // write bursts: [7:0] to writes at, [15:8] back below +#define FOLDSIM_REG_MEM_TIMING(i) (0x800u + 4u * (i)) // FOLDSIM_MEM_T_*, 12 bits each +#define FOLDSIM_REG_MEM_STATS(i) (0xC00u + 8u * (i)) // FOLDSIM_MEM_STAT_*, lo/hi + +#define FOLDSIM_MEM_MODEL_PIPE 0u +#define FOLDSIM_MEM_MODEL_BANK 1u + +#define FOLDSIM_MEM_CAPS_PORTS(c) ((c) & 0xffu) +#define FOLDSIM_MEM_CAPS_HORIZON(c) (1u << ((((c) >> 8) & 0xfu) + 4u)) // ring, DRAM clocks +#define FOLDSIM_MEM_CAPS_SLOTS(c) (((c) >> 12) & 0xffu) // read slots per port +#define FOLDSIM_MEM_CAPS_TIME_W(c) ((((c) >> 28) & 0xfu) + 12u) // timing bits + +enum { + FOLDSIM_MEM_T_BL, FOLDSIM_MEM_T_CCDS, FOLDSIM_MEM_T_CCDL, FOLDSIM_MEM_T_CL, FOLDSIM_MEM_T_CWL, + FOLDSIM_MEM_T_RCDRD, FOLDSIM_MEM_T_RCDWR, FOLDSIM_MEM_T_RP, FOLDSIM_MEM_T_RAS, FOLDSIM_MEM_T_RC, + FOLDSIM_MEM_T_RTP, FOLDSIM_MEM_T_WR, FOLDSIM_MEM_T_RRDS, FOLDSIM_MEM_T_RRDL, FOLDSIM_MEM_T_FAW, + FOLDSIM_MEM_T_WTRS, FOLDSIM_MEM_T_WTRL, FOLDSIM_MEM_T_RTW_EXTRA, FOLDSIM_MEM_NUM_TIMINGS +}; + +enum { + FOLDSIM_MEM_STAT_READS, FOLDSIM_MEM_STAT_WRITES, FOLDSIM_MEM_STAT_HITS, FOLDSIM_MEM_STAT_MISSES, + FOLDSIM_MEM_STAT_CONFLICTS, FOLDSIM_MEM_STAT_LATENCY, FOLDSIM_MEM_NUM_STATS +}; + #define FOLDSIM_STATUS_BUSY 0x1u // queued, running, or memory in flight #define FOLDSIM_STATUS_FULL 0x2u // the command queue is full #define FOLDSIM_STATUS_RSP 0x4u // a DCR read has its response diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp index c5249196ed..1bee1f0888 100644 --- a/sw/runtime/foldsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -329,7 +330,18 @@ class vx_device { return -1; } // The memory model: a latency-bandwidth pipe, 30-cycle latency and 16 - // transactions in flight each way unless set. + // transactions in flight each way unless set; or, with + // FOLDSIM_MEM_MODEL=bank, the platform memory's DRAM as the bank model + // times it. + const char* model = getenv("FOLDSIM_MEM_MODEL"); + const std::string model_name = (model != nullptr) ? model : ""; + if (!model_name.empty() && model_name != "pipe" && model_name != "bank") { + fprintf(stderr, "[VXDRV] FOLDSIM_MEM_MODEL must be pipe or bank (got '%s')\n", model); + return -1; + } + bank_model_ = (model_name == "bank"); + if (bank_model_ && this->setup_bank_model() != 0) + return -1; const uint32_t rlat = env_or("FOLDSIM_MEM_READ_LATENCY", 30); const uint32_t wlat = env_or("FOLDSIM_MEM_WRITE_LATENCY", 30); const uint32_t maxr = env_or("FOLDSIM_MEM_MAX_READS", 16); @@ -345,9 +357,11 @@ class vx_device { // A target left running by an earlier process is reset, as a card's // native driver resets its device on open: Vortex counts VX_CFG_RESET_DELAY // cycles of reset before it takes a request. The reset also clears a - // memory error that process left, so only the state after it counts. + // memory error that process left, so only the state after it counts, and + // chooses the memory model. uint32_t s = 0; - if (command(FOLDSIM_CMD_RESET, 0, 64) != 0 || wait_idle(false) != 0 || status(&s) != 0) + const uint32_t mem_model = bank_model_ ? FOLDSIM_MEM_MODEL_BANK : FOLDSIM_MEM_MODEL_PIPE; + if (command(FOLDSIM_CMD_RESET, mem_model, 64) != 0 || wait_idle(false) != 0 || status(&s) != 0) return -1; if (mem_error(s)) return -1; @@ -407,6 +421,75 @@ class vx_device { private: int reg_write(uint32_t off, uint32_t v) { return transport_.reg_write(off, v); } +#define FOLDSIM_STR_(x) #x +#define FOLDSIM_STR(x) FOLDSIM_STR_(x) + + // The bank model's settings: the platform memory this runtime was built for + // (VX_CFG_PLATFORM_MEMORY_*), its data-sheet timings in DRAM clocks as + // dram_sim derives them, and dram_sim's controller queues and write + // watermarks. They take effect at the target reset that follows. The image + // fixes the structure: its port is one channel, whatever + // VX_CFG_PLATFORM_MEMORY_NUM_BANKS says. + int setup_bank_model() { + uint32_t caps = 0; + if (transport_.reg_read(FOLDSIM_REG_MEM_CAPS, &caps) != 0) + return -1; + if (FOLDSIM_MEM_CAPS_PORTS(caps) == 0) { + fprintf(stderr, "[VXDRV] this foldsim image has no bank model\n"); + return -1; + } + dram_timing::Standard standard; + try { + standard = dram_timing::parse_standard(FOLDSIM_STR(VX_CFG_PLATFORM_MEMORY_TYPE)); + } catch (const std::exception& e) { + fprintf(stderr, "[VXDRV] %s\n", e.what()); + return -1; + } + const uint32_t rate = VX_CFG_PLATFORM_MEMORY_DATA_RATE; + const uint32_t clock = VX_CFG_PLATFORM_CLOCK_RATE; + const auto t = dram_timing::clocks(standard, rate); + const auto g = dram_timing::geometry(standard); + const uint32_t v[FOLDSIM_MEM_NUM_TIMINGS] = { + t.BL, t.CCDS, t.CCDL, t.CL, t.CWL, t.RCDRD, t.RCDWR, t.RP, t.RAS, t.RC, + t.RTP, t.WR, t.RRDS, t.RRDL, t.FAW, t.WTRS, t.WTRL, g.rtw_extra}; + const uint32_t time_max = (1u << FOLDSIM_MEM_CAPS_TIME_W(caps)) - 1; + for (uint32_t x : v) { + if (x > time_max) { + fprintf(stderr, "[VXDRV] a DRAM timing of %u clocks exceeds the image's %u\n", x, time_max); + return -1; + } + } + // The image places ACTs under tRRD and the per-bank timings; tFAW it + // leaves to tRRD, which covers it wherever four tRRD_S span a tFAW. + if (4 * t.RRDS < t.FAW) { + fprintf(stderr, "[VXDRV] the bank model of this image does not model tFAW, which %s at " + "%u MT/s needs (tRRD_S %u, tFAW %u clocks)\n", + FOLDSIM_STR(VX_CFG_PLATFORM_MEMORY_TYPE), rate, t.RRDS, t.FAW); + return -1; + } + if (rate > 0xffff || clock > 0xffff || VX_CFG_PLATFORM_MEMORY_LATENCY > 0xffff + || VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE > 0xffff) { + fprintf(stderr, "[VXDRV] the platform memory's settings exceed the bank model's registers\n"); + return -1; + } + const uint32_t geom = g.log2_pcs | (g.log2_bgs << 2) | (g.log2_banks << 4) + | (g.log2_row_bursts << 6) | (g.log2_visit << 10) + | (uint32_t(g.two_clock_act) << 12); + const uint32_t queue = VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE | (32u << 16) | (32u << 24); + const uint32_t wmark = 26u | (7u << 8); + if (reg_write(FOLDSIM_REG_MEM_CLOCK, rate | (clock << 16)) != 0 || + reg_write(FOLDSIM_REG_MEM_LATENCY, VX_CFG_PLATFORM_MEMORY_LATENCY) != 0 || + reg_write(FOLDSIM_REG_MEM_QUEUE, queue) != 0 || + reg_write(FOLDSIM_REG_MEM_GEOM, geom) != 0 || + reg_write(FOLDSIM_REG_MEM_WMARK, wmark) != 0) + return -1; + for (uint32_t i = 0; i < FOLDSIM_MEM_NUM_TIMINGS; ++i) { + if (reg_write(FOLDSIM_REG_MEM_TIMING(i), v[i]) != 0) + return -1; + } + return 0; + } + uint64_t read64(uint32_t lo) { uint32_t l = 0, h = 0, h2 = 0; do { @@ -440,6 +523,19 @@ class vx_device { } if (!line.empty()) fprintf(stderr, "[foldsim] class busy:%s\n", line.c_str()); + if (bank_model_) { + uint64_t st[FOLDSIM_MEM_NUM_STATS]; + for (uint32_t i = 0; i < FOLDSIM_MEM_NUM_STATS; ++i) + st[i] = read64(FOLDSIM_REG_MEM_STATS(i)); + const uint64_t reads = st[FOLDSIM_MEM_STAT_READS]; + fprintf(stderr, + "[foldsim] bank model: %lu reads, %lu writes; row hits %lu, misses %lu, conflicts %lu; " + "mean read latency %.1f cycles\n", + (unsigned long)reads, (unsigned long)st[FOLDSIM_MEM_STAT_WRITES], + (unsigned long)st[FOLDSIM_MEM_STAT_HITS], (unsigned long)st[FOLDSIM_MEM_STAT_MISSES], + (unsigned long)st[FOLDSIM_MEM_STAT_CONFLICTS], + reads ? double(st[FOLDSIM_MEM_STAT_LATENCY]) / double(reads) : 0.0); + } const auto& st = transport_.stats; fprintf(stderr, "[foldsim] host: %lu register reads, %lu writes in %.3f s; %lu bytes of device memory in %.3f s\n", @@ -644,6 +740,7 @@ class vx_device { std::chrono::steady_clock::time_point launch_start_; bool failed_ = false; bool open_ = false; + bool bank_model_ = false; vortex::CommandProcessor cp_; std::mutex host_mu_; std::map host_regions_; From aa34bd3f5907f85aee286ed86822e7a5722c1b1a Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 03:43:24 -0700 Subject: [PATCH 224/336] docs: Foldsim's bank memory model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit foldsim.md describes the memory endpoint's two models (§4.1): the latency-bandwidth pipe, still the default and unchanged, and the bank model -- port slots, the controller's buffers and write drain, the reservation of PRE, ACT and column commands, completion in departure order -- and how the hardware keeps its calendars (bitmaps over a ring of 512 DRAM clocks in distributed RAM, a first-free search of 64 clocks a host cycle, a reservation reaching 448 clocks ahead), with a figure (docs/assets/img/foldsim_mem_bank.svg). The register table, runtime, validation and limitations follow; §7.1 gives the measured results: Foldsim (Verilator) equal to rtlsim on the same model but for the 8-cycle launch settle, and within 0.8 % of SimX on Ramulator for sgemm and vecadd on the default, a100 and h100 memories (the store-dominated demo +5 %). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 45361909a3cae6cd3fe3432ab229880cdb20935c) --- docs/assets/img/foldsim_mem_bank.svg | 73 +++++++++++++++ docs/designs/foldsim.md | 125 ++++++++++++++++++++++--- hw/rtl/afu/foldsim/foldsim_mem_bank.sv | 2 +- 3 files changed, 186 insertions(+), 14 deletions(-) create mode 100644 docs/assets/img/foldsim_mem_bank.svg diff --git a/docs/assets/img/foldsim_mem_bank.svg b/docs/assets/img/foldsim_mem_bank.svg new file mode 100644 index 0000000000..257cd41d39 --- /dev/null +++ b/docs/assets/img/foldsim_mem_bank.svg @@ -0,0 +1,73 @@ + + foldsim memory endpoint: the bank model + + + + + + + + + per target cycle (at finish) + + AR / AW + W tokens + read takes a slot (32) + AW+W form a write + + + R / B stages + R: oldest ready read + B: oldest buffered write + data: slot buffer (HBM) + + + per DRAM clock (host cycles, before the cycle finishes) + + ingress + in order, from the + next target cycle + + + read buffer + + write buffer (acked) + write drain: 26 / 7 bursts + + + reservation (one request) + bank table (32 banks): open row, ACT, column-, PRE-ready + PRE, ACT, column, column: each a first-free search, + 64 clocks a host cycle, from its lower bound + horizon check, then mark what each command forbids + + + ring calendars, 512 clocks (LUTRAM) + command bus: row, column commands + row bus (PRE, two-clock ACT) + forbidden to a read / write column: + per pseudo-channel (tCCD_S, turnarounds) + per bank group (tCCD_L, tWTR_L) + forbidden to an ACT: tRRD_S, tRRD_L + column directions (controller queues) + departures: read slot per clock + a word of 64 clocks cleared as it passes; + a reservation reaches at most 448 clocks + + + departure at clock d+1 + read ready LATENCY + 1 target cycles + after its clock's target cycle + (ready FIFO, in departure order) + + + + + + + + + + + + write acknowledged + diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index d97ed4434d..1f1272f4f9 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -149,10 +149,11 @@ the host-memory master: `s_axi_control` and `m_axi_mem_0` (linked to the models of the target's surroundings that take one token per channel per target cycle (`foldsim_token.sv`): -- **`foldsim_mem`** is the target's AXI4 port, timed as a latency-bandwidth - pipe — cycle for cycle `sim/common/axi_latency_pipe.h`: a fixed latency per - transaction, bounded reads and writes in flight, one beat per cycle, - responses in request order. A read goes to HBM (`base` + target address) +- **`foldsim_mem`** is the target's AXI4 port, timed by one of two models + (§4.1): a latency-bandwidth pipe — cycle for cycle + `sim/common/axi_latency_pipe.h`: a fixed latency per transaction, bounded + reads and writes in flight, one beat per cycle, responses in request + order — or a DRAM channel's banks. A read goes to HBM (`base` + target address) the cycle the target makes it, a write the cycle it has both its AW and its W, so HBM only ever sees whole single-beat writes; a target cycle that should present a response memory has not returned yet does not finish until @@ -165,6 +166,60 @@ target cycle (`foldsim_token.sv`): applied for one target cycle (a reset for its count). Kernel reset holds the target in reset for 64 cycles. +### 4.1 The memory models + +The kernel's own reset gives the pipe; a reset command chooses the model the +target has from that reset on (§5). The pipe is the default, and with it the +endpoint makes exactly the transactions and takes exactly the cycles it did +before the bank model existed. + +**The bank model** (`foldsim_mem_bank.sv`) is cycle for cycle +`sim/common/dram_bank_model.h` (`DramBankAxi` over `DramBankModel`), the +reservation-based model of dram_sim's DRAM, configured with dram_sim's own +knobs (`VX_CFG_PLATFORM_MEMORY_{TYPE,DATA_RATE,LATENCY,QUEUE_SIZE}`, +`VX_CFG_PLATFORM_CLOCK_RATE`) and its data-sheet timings +(`sw/common/dram_timing.h`, converted to DRAM clocks with Ramulator's +rounding). The same model is a `DramSim` backend in the simulators +(`VORTEX_DRAM_MODEL=bank`) and rtlsim's AXI4 memory (`AXI_BUS=1 +AXI_MEM_MODEL=bank`). + +![the bank model](../assets/img/foldsim_mem_bank.svg) + +- *Port.* A read holds one of 32 slots from acceptance until R takes it; its + data, from HBM, waits in the slot. An AW and its W form a write. Reads and + writes reach the channel in the order they formed, from the next target + cycle; at most `QUEUE_SIZE` requests are in flight (0: the slots only). +- *Channel.* One request a request's bursts' worth of DRAM clocks enters the + controller's read or write buffer (32 bursts each); a write is acknowledged + there. The controller serves reads and drains writes in batches between + dram_sim's watermarks (26 and 7 bursts). +- *Reservation.* The request at the head of the buffer being served is + scheduled once: its PRE, ACT and column commands take the first DRAM clocks + every JEDEC constraint allows (tRCD, tRP, tRAS, tRC, tRTP, tWR per bank; + tCCD, tRRD, the read/write turnarounds per pseudo-channel and bank group; + one command per clock; HBM's two-clock ACT), before or after commands + reserved earlier. Rows stay open; a bank serves its requests in order. A + batch of writes goes after every column reserved before it, the reads after + it after the batch. +- *Completion.* A read is ready `LATENCY` + 1 target cycles after the target + cycle whose DRAM clocks its data departs in (tCL + tBL after its last + column); R and B take the oldest ready response, so reads complete out of + order — legal, since Vortex never has two reads of one id outstanding. + +In hardware the calendars are bitmaps over a ring of 512 DRAM clocks in +distributed RAM: per clock, whether it holds a command, the row bus, and per +pseudo-channel and bank group the clocks a read, a write or an ACT may not +take. Every command marks, once reserved, the clocks its constraints forbid +to later ones, so placing one is a first-free search over 64 clocks a host +cycle; a word of the ring is cleared, by a valid bit, as the DRAM clock +passes it, and a reservation reaches at most 448 clocks ahead (the model's +horizon). A target cycle's DRAM clocks run in host cycles while the target +evaluates it — each is one host cycle, a reservation about ten — and the +cycle finishes only once they have; the `memory waits` counter includes +the host cycles a finished cycle waits for them. tFAW is left to tRRD, +which covers it wherever four tRRD_S span a tFAW (HBM2, HBM3); the runtime +refuses a memory where it does not (GDDR6). + **Reset.** The control endpoint also gives the memory endpoint a token each target cycle: the target's reset in that cycle. A reset command thereby resets the target's whole memory port, not only the target. A target cycle in @@ -202,8 +257,12 @@ for the RTL): | `0x1C` | the last DCR read's response | | `0x20`–`0x2C` | target cycles, kernel-clock cycles | | `0x30`, `0x38` | kernel-clock cycles a target cycle waited for HBM; the number of classes | -| `0xC0`–`0xD4` | memory model: device base, read and write latency, most reads and writes in flight | +| `0xC0`–`0xD4` | memory model: device base, the pipe's read and write latency, most reads and writes in flight | +| `0xD8`, `0xDC` | the memory model in effect (read only); the bank model's capacities (read only) | +| `0xE0`–`0xF0` | bank model: data rate and core clock, fixed latency, requests in flight and controller queues, geometry, write watermarks | | `0x100 + 8c` | per class, threads finished | +| `0x800`–`0x844` | bank model: DRAM timings, a word each (tBL … tWTR_L, read-to-write extra) | +| `0xC00`–`0xC2C` | bank model: reads, writes, row hits, misses, conflicts, summed read latency (64-bit) | `0x40`–`0xBF` is empty: on the V80 the PCIe block keeps its MSI-X table at that offset of the BAR the first kernel is mapped at, and answers those @@ -216,7 +275,12 @@ HBM, so a host that sees the target idle has every effect of what it asked for. The runtime resets the target on open, as a card's native driver resets its -device; that also clears a memory error an earlier process left. The command +device; that also clears a memory error an earlier process left, and the reset +command's address chooses the memory model. With `FOLDSIM_MEM_MODEL=bank` the +runtime first writes the bank model's registers from the platform memory it +was built for (its `CONFIGS`), refusing what the image cannot hold, and +`--perf`-style counters print at close; otherwise it makes exactly the +register accesses it always has. The command processor model's hooks return no status, so the runtime latches a device error instead — a memory error, a failed register or memory access, or a wait that timed out — and from then on every register access the runtime core @@ -237,11 +301,15 @@ advancing for that time (60 s when it is unset). - **The memory endpoint against `AxiLatencyPipe`**, in Verilator: random requests and device-memory latency, every target cycle's outputs and every read's data compared; five latency and limit settings, 0 differences. -- **The memory endpoint's reset and bursts** (`hw/unittest/foldsim_mem`, in - Verilator): an AXI4 master as the target, HBM with random ready and - latency checking the device-side protocol, resets at random points with - HBM busy, and each kind of burst; every response and id checked, none from - before a reset delivered after it. +- **The memory endpoint** (`hw/unittest/foldsim_mem`, in Verilator): an + AXI4 master as the target, HBM with random ready and latency checking the + device-side protocol, resets at random points with HBM busy, and each kind + of burst; every target cycle's outputs compared with the C++ model's + (`AxiLatencyPipe`, `DramBankAxi`), every response, id and read's data + checked, none from before a reset delivered after it. Two pipe settings and + twelve bank-model settings (HBM2 and HBM3 at the default, v100, a100 and + h100 rates; small queues; row hits, random rows, one bank to the horizon, + streams, read- and write-heavy): 0 differences. - **Applications** in the in-process simulator and on the V80, against rtlsim behind the AXI4 adapter on `AxiLatencyPipe` (`AXI_BUS=1`) with the same memory settings: retired instructions and cycles identical (§7). @@ -283,7 +351,37 @@ cycles after reset, while each L1 bank still invalidates one set per cycle counters see; a host's launch comes thousands of cycles later. Given that settle time, rtlsim's cycles are Foldsim's exactly. -### 7.1 tinygpu: Foldsim and the firesim backend +### 7.1 The bank model + +s2w4i2 in the in-process simulator (Verilator) with `FOLDSIM_MEM_MODEL=bank`, +against SimX on the same configuration (one memory bank, Ramulator behind +dram_sim) and rtlsim behind the AXI4 adapter on the same model +(`AXI_MEM_MODEL=bank`). Memories: the default (HBM2, 800 MT/s, 400 MHz core) +and the a100 and h100 profiles' (`VX_CFG_PLATFORM_MEMORY_*` from +`profiles/*.json`). + +| app | memory | Foldsim, bank model | rtlsim, bank model | SimX, Ramulator | SimX, bank model | +|---|---|---:|---:|---:|---:| +| demo | default | 10,740 | 10,748 | 10,190 | 9,148 | +| sgemm | default | 399,337 | 399,345 | 401,434 | 401,421 | +| sgemm | a100 | 691,372 | | 690,935 | 690,780 | +| sgemm | h100 | 422,294 | | 425,547 | 425,541 | +| vecadd `-n16384` | default | 194,242 | | 193,267 | 192,901 | +| vecadd `-n16384` | a100 | 311,917 | | 312,529 | 312,587 | +| vecadd `-n16384` | h100 | 223,875 | | 224,921 | 224,921 | + +Instruction counts are equal everywhere. rtlsim's cycles are Foldsim's plus +the 8-cycle launch settle it has on the pipe too (§7). Against SimX on +Ramulator Foldsim is within 0.8 % on sgemm and vecadd at all three memories; +the store-dominated demo (2,048 writes to 268 reads) is 5 % slower, and +SimX's own bank backend 10 % faster: the reservation model drains a write +stream differently from Ramulator's per-clock scheduler (a +mixed-traffic residual). The pipe, by contrast, has no memory to configure: +sgemm takes 415,454 cycles at every profile. A bank-model run costs host +cycles the pipe does not (memory waits): 7.6–8.7 host cycles per target cycle +against the pipe's 8.0. + +### 7.2 tinygpu: Foldsim and the firesim backend The same Vortex tree, configuration and V80 on both flows: 1 core, 2 warps, 2 threads, no L1 caches or local memory, STD FPU, one memory bank, kernel @@ -324,7 +422,8 @@ folds (s2w4i2 above). is target cycle k. 3. **The memory endpoint takes single-beat transactions**, as Vortex issues them; a burst sets the status register's memory error, stops the - endpoint, and only a reset clears it. + endpoint, and only a reset clears it. Under the bank model, no two reads + of one id may be outstanding (Vortex's AXI adapter never has them). 4. **One memory port**: the target is built with one platform memory bank. 5. **Classes are named, not found**: `FOLDSIM_CLASSES` lists them. 6. **No kernel register in `0x40`–`0xBF`.** The V80 shell configures the diff --git a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv index 372811ca15..506c6991da 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv @@ -462,7 +462,7 @@ module foldsim_mem_bank #( bm_clear <= 1'b1; bm_clear_word <= end_word; end - // Times older than any constraint is long move up, so none wraps. + // A time too old to constrain anything is moved up before it could wrap. if (~bt_fresh[scrub_b]) begin if ($signed(bt_act[scrub_b] - (e + 16'd1)) < -16'sd16384) bt_act[scrub_b] <= e + 16'd1 - 16'd16384; From b74160565bcc8f3d7a87cdb985b5937982b7dc9a Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 09:44:58 -0700 Subject: [PATCH 225/336] foldsim_mem: the bank model and the pipe time for 250 MHz and more Out-of-context on the V80 part the endpoint missed 4.0 ns (WNS -0.792 ns, 4256 failing endpoints): the clock-end of an idle DRAM clock hung off the ingress-move -> mode -> select chain into the departure FIFO's and the bank table's write enables, the bank table (written by the reservation and the scrub) and the departure FIFO had become flip-flops, and the target cycle's finish gated every register of the engine. None of the model's decisions change, only the host cycles they take: - the engine's state machine no longer sits under the finish (a cycle finishes only with the engine idle); - a DRAM clock is a move cycle and a mode cycle (the clock without a reservation ending in the second); the clock's departure and retiring columns are read when it starts, a reservation's own column in it added; - the departure FIFO is written the host cycle after its clock, the bank table through one port (the scrub moved to the once-a-target-cycle walk), so both are distributed RAM again; the calendars clear the host cycle after a target reset, in a cycle that takes no DRAM clock; - a pipeline cycle each in the bank lookup, the search's setup and the horizon check; addresses decoded when they join the ingress, the bank fold precomputed for each bank width; - the pipe's release compare needs no adder in front: an empty pipe's arrival is due iff 1 - latency >= 0, exactly the old result. hw/unittest/foldsim_mem: 0 differences on all 14 settings, the pipe's host cycles unchanged; s2w4i2 in the in-process simulator: pipe demo/sgemm 4888/8702/15559/125233 and 336928/415454/422288/3379066, unchanged; bank runs' target behavior unchanged, about 25% more host cycles per target cycle in the unit test. OOC at 4.0 ns: WNS +0.279 ns (266 MHz), 0 failing endpoints, 11323 LUTs, 0 BRAM. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 9b6b9f3feefb8724ee5ddb706f507e4989ef3cbc) --- hw/rtl/afu/foldsim/foldsim_mem.sv | 21 +- hw/rtl/afu/foldsim/foldsim_mem_bank.sv | 657 ++++++++++++++----------- 2 files changed, 369 insertions(+), 309 deletions(-) diff --git a/hw/rtl/afu/foldsim/foldsim_mem.sv b/hw/rtl/afu/foldsim/foldsim_mem.sv index 8451391661..7527599735 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem.sv @@ -181,7 +181,7 @@ module foldsim_mem #( // AW ids (and addresses) waiting for their write data, writes formed and // not yet driving B, and the one that is. reg [ID_W-1:0] awq_id [DEPTH]; - reg [29:0] awq_addr [DEPTH]; + reg [29:5] awq_addr [DEPTH]; reg [PW-1:0] awq_head, awq_tail; reg [CW-1:0] awq_count; reg [31:0] wp_release [DEPTH]; @@ -264,12 +264,16 @@ module foldsim_mem #( // Reads: the latency pipe, into the stage driving R; a read released the // cycle it arrives skips the pipe. + // A read arriving at an empty pipe is due at once if its release, + // cycle + latency - 1, is not after the cycle: a function of the latency + // alone, so the release's adder is not in front of the compare. wire [31:0] ar_release = cycle + read_latency - 32'd1; wire rp_empty = (rp_count == 0); wire r_head = ~rp_empty | ar_fire; - wire [31:0] r_head_rel = rp_empty ? ar_release : rp_release[rp_head]; wire [ID_W-1:0] r_head_id = rp_empty ? arid : rp_id[rp_head]; - wire r_release = r_head & ($signed(cycle - r_head_rel) >= 0); + wire r_due = rp_empty ? ($signed(32'd1 - read_latency) >= 0) + : ($signed(cycle - rp_release[rp_head]) >= 0); + wire r_release = r_head & r_due; wire r_take = r_release & (~cur_r_valid | r_fire); wire rp_push = ar_fire & ~(r_take & rp_empty); wire rp_pop = r_take & ~rp_empty; @@ -281,7 +285,7 @@ module foldsim_mem #( | (aw_fire & w_fire); wire awq_empty = (awq_count == 0); wire [ID_W-1:0] w_form_id = awq_empty ? awid : awq_id[awq_head]; - wire [29:0] w_form_addr = awq_empty ? awaddr[29:0] : awq_addr[awq_head]; + wire [29:5] w_form_addr = awq_empty ? awaddr[29:5] : awq_addr[awq_head]; wire awq_pop = new_write & ~awq_empty; wire awq_push = aw_fire & ~(new_write & awq_empty); wire [31:0] w_release = cycle + write_latency - 32'd1; @@ -296,9 +300,10 @@ module foldsim_mem #( reg [31:0] wp_wr_rel; reg [ID_W-1:0] wp_wr_id; wire wp_fwd = wp_wr_pend & (wp_wr_idx == wp_head); - wire [31:0] w_head_rel = wp_empty ? w_release : (wp_fwd ? wp_wr_rel : wp_release[wp_head]); wire [ID_W-1:0] w_head_id = wp_empty ? w_form_id : (wp_fwd ? wp_wr_id : wp_id[wp_head]); - wire w_rel = w_head & ($signed(cycle - w_head_rel) >= 0); + wire w_due = wp_empty ? ($signed(32'd1 - write_latency) >= 0) + : ($signed(cycle - (wp_fwd ? wp_wr_rel : wp_release[wp_head])) >= 0); + wire w_rel = w_head & w_due; wire w_take = w_rel & (~cur_w_valid | b_fire); wire wp_push = new_write & ~(w_take & wp_empty); wire wp_pop = w_take & ~wp_empty; @@ -391,7 +396,7 @@ module foldsim_mem #( if (awq_push) begin awq_id[awq_tail] <= awid; - awq_addr[awq_tail] <= awaddr[29:0]; + awq_addr[awq_tail] <= awaddr[29:5]; awq_tail <= awq_tail + 1'b1; end if (awq_pop) @@ -561,7 +566,7 @@ module foldsim_mem #( .treset (treset), .ar_fire (ar_fire), .ar_id (arid), - .ar_addr (araddr[29:0]), + .ar_addr (araddr[29:5]), .aw_fire (aw_fire), .new_write (new_write), .nw_id (w_form_id), diff --git a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv index 506c6991da..da1121be07 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv @@ -73,11 +73,11 @@ module foldsim_mem_bank #( // responses R and B give. input wire ar_fire, input wire [ID_W-1:0] ar_id, - input wire [29:0] ar_addr, + input wire [29:5] ar_addr, input wire aw_fire, input wire new_write, input wire [ID_W-1:0] nw_id, - input wire [29:0] nw_addr, + input wire [29:5] nw_addr, input wire r_fire, input wire b_fire, @@ -141,6 +141,8 @@ module foldsim_mem_bank #( reg signed [15:0] k_rrds, k_rrdl; reg [16:0] k_twoclk; reg [2:0] k_bb; + reg [4:0] k_rowsh; + reg [24:0] k_bmask; always @(posedge clk) begin k_ccds <= 16'(T[I_CCDS]); k_ccdl <= 16'(T[I_CCDL]); @@ -164,25 +166,38 @@ module foldsim_mem_bank #( k_rrdl <= 16'(T[I_RRDL]); k_twoclk <= {cfg_core_clock, 1'b0}; k_bb <= 3'(g_lpcs) + 3'(g_lbgs) + 3'(g_lbanks); + k_rowsh <= 5'(g_lpcs) + 5'(g_lbgs) + 5'(g_lbanks) + 5'(g_lrow); + k_bmask <= (25'd1 << (3'(g_lpcs) + 3'(g_lbgs) + 3'(g_lbanks))) - 25'd1; end // The bank and row of a channel address (sim/common/dram_bank_model.h's // decode): a row visit, then the bank bits, the row above the column, and // the bank XORed with the row folded to its width. + // The row folded onto a bank field of each width a geometry may have, so + // the bank's width only selects among them. + function automatic [5:0] fold(input [ROW_W-1:0] r, input integer bb); + fold = '0; + for (integer k = 0; k < ROW_W; k += bb) + fold = fold ^ 6'((32'(r) >> k) & ((32'd1 << bb) - 32'd1)); + endfunction + function automatic [BKW+ROW_W-1:0] decode(input [29:5] a); reg [24:0] tx; reg [24:0] b; reg [ROW_W-1:0] r; - reg [24:0] mask; + reg [5:0] f; tx = a[29:5]; - mask = (25'd1 << k_bb) - 25'd1; - b = (tx >> g_lvisit) & mask; - r = ROW_W'(tx >> (32'(k_bb) + 32'(g_lrow))); - for (integer k = 0; k < ROW_W; ++k) begin - if (k_bb != 0 && 32'(k) * 32'(k_bb) < ROW_W) - b = b ^ ((25'(r) >> (32'(k) * 32'(k_bb))) & mask); - end - decode = {BKW'(b), r}; + b = (tx >> g_lvisit) & k_bmask; + r = ROW_W'(tx >> k_rowsh); + case (k_bb) + 3'd1: f = fold(r, 1); + 3'd2: f = fold(r, 2); + 3'd3: f = fold(r, 3); + 3'd4: f = fold(r, 4); + 3'd5: f = fold(r, 5); + default: f = fold(r, 6); + endcase + decode = {BKW'(b ^ 25'(f)), r}; endfunction // ---- The port: slots, responses, requests in flight ---- @@ -193,14 +208,18 @@ module foldsim_mem_bank #( reg cur_r_valid, cur_b_valid; reg [SW-1:0] cur_r_slot; reg [ID_W-1:0] cur_r_id, cur_b_id; - reg [31:0] tcyc; + reg [31:0] tcyc, tcyc1; // the target cycle, and the next // Reads departed, in order, with the target cycle they are ready in; the - // first `rf_ready` are ready in the next cycle. + // first `rf_ready` are ready in the next cycle. A departure enters the + // host cycle after its DRAM clock ends. reg [SW-1:0] rf_slot [SLOTS]; reg [31:0] rf_cyc [SLOTS]; reg [SW-1:0] rf_head, rf_tail; reg [SW:0] rf_count, rf_ready; + reg rfp_valid; + reg [SW-1:0] rfp_slot; + reg [31:0] rfp_cyc; // Writes the controller has buffered, to acknowledge in order. localparam AQW = $clog2(WSLOTS); reg [ID_W-1:0] af_id [WSLOTS]; @@ -209,6 +228,9 @@ module foldsim_mem_bank #( // What this target cycle's DRAM clocks made ready by the next one. reg [SW:0] new_ready; reg [AQW:0] new_acks; + // The last cycle's reads taken by AR and by R, for the latency sum. + reg lat_ar, lat_r; + reg [31:0] lat_t; assign room = (cfg_queue == 16'd0) | (inflight < cfg_queue); assign slot_free = (32'(slots_used) < SLOTS); @@ -234,14 +256,15 @@ module foldsim_mem_bank #( reg st_w_valid, st_r_valid; reg [ID_W-1:0] st_w_id; reg [SW-1:0] st_r_slot; - reg [29:0] st_w_addr, st_r_addr; + reg [29:5] st_w_addr, st_r_addr; // ---- The channel ---- - // Requests in the order they formed; the controller's read and write - // buffers. + // Requests in the order they formed, decoded; the controller's read and + // write buffers. reg iq_w [INQ]; reg [TAG_W-1:0] iq_tag [INQ]; - reg [29:0] iq_addr[INQ]; + reg [BKW-1:0] iq_bank[INQ]; + reg [ROW_W-1:0] iq_row [INQ]; reg [IQW-1:0] iq_head, iq_tail; reg [IQW:0] iq_count; reg [TAG_W-1:0] rb_tag [BQ], wb_tag [BQ]; @@ -257,7 +280,8 @@ module foldsim_mem_bank #( reg wmode; reg [15:0] frontier, batch_floor, batch_last, read_floor; - // The bank table. A fresh bank has no row open and no time that matters. + // The bank table, written at one port. A fresh bank has no row open and + // no time that matters. reg [31:0] bt_open, bt_fresh; reg [ROW_W-1:0] bt_row [32]; reg [15:0] bt_act [32]; @@ -290,8 +314,8 @@ module foldsim_mem_bank #( reg bm_clear; reg [WW-1:0] bm_clear_word; // A target reset (or the port leaving the bank model) clears the - // calendars at the cycle's end, before the next cycle's first clock. - wire bm_clear_all = finish & (treset | ~active); + // calendars in the host cycle after it; that cycle takes no DRAM clock. + reg bm_clear_all; reg [NBM-1:0] wr_en; reg [WW-1:0] wr_word [NBM]; reg [63:0] wr_mask [NBM]; @@ -355,9 +379,9 @@ module foldsim_mem_bank #( reg [SW-1:0] dep_slot [H]; // ---- The engine ---- - localparam S_IDLE = 4'd0, S_PUSH = 4'd1, S_TICK = 4'd3, S_BANK = 4'd4, - S_SETUP = 4'd5, S_SRCH = 4'd6, S_CHECK = 4'd7, S_COMMIT = 4'd8, S_END = 4'd9, - S_WALK = 4'd10; + localparam S_IDLE = 4'd0, S_PUSH = 4'd1, S_TICK = 4'd2, S_MODE = 4'd3, S_BANK = 4'd4, + S_BANK2 = 4'd5, S_SETUP = 4'd6, S_SETUP2 = 4'd7, S_SRCH = 4'd8, S_CHECK = 4'd9, + S_CHECK2 = 4'd10, S_COMMIT = 4'd11, S_END = 4'd12, S_WALK = 4'd13; localparam K_PRE = 2'd0, K_ACT = 2'd1, K_C0 = 2'd2, K_C1 = 2'd3; reg [3:0] state; @@ -370,8 +394,10 @@ module foldsim_mem_bank #( reg signed [15:0] c_pre, c_act, c_col0, c_col1; reg [1:0] kind; - // The search: the next word to read (as the ring position and the clocks - // left to the horizon) and the one read last cycle, to decide on. + // The search: its lower bound; the next word to read (as the ring + // position and the clocks left to the horizon) and the one read last + // cycle, to decide on. + reg signed [15:0] s_lb; reg [HLOG2-1:0] s_pos; reg signed [15:0] s_left; reg s_fetch; @@ -413,7 +439,7 @@ module foldsim_mem_bank #( reg cm_next [NBM]; // the second range is still to mark reg cm_start; - reg signed [15:0] ck_colr, ck_prer, ck_lastcol; + reg signed [15:0] ck_colr, ck_prer, ck_lastcol, ck_tcol, ck_tdep, ck_tact; reg cm_any; always @(*) begin cm_any = 1'b0; @@ -421,79 +447,315 @@ module foldsim_mem_bank #( cm_any = cm_any | cm_busy[m]; end - // The read data the cycle's departure completes, and the clock's columns. - wire end_depv = rd_depv[base[5:0]]; - wire end_kr = rd_kr[base[5:0]]; - wire end_kw = rd_kw[base[5:0]]; + // Clock d+1, read when its DRAM clock starts: the read departing in it and + // the columns issued in it -- and the columns the clock's own reservation + // places in it. + reg ed_depv, ed_kr, ed_kw; + reg [SW-1:0] ed_slot; - // Ingress head, decoded, and whether it enters its buffer this tick. + // Ingress head, and whether it enters its buffer this tick. wire ih_w = iq_w[iq_head]; - wire [TAG_W-1:0] ih_tag = iq_tag[iq_head]; - wire [BKW+ROW_W-1:0] ih_dec = decode(iq_addr[iq_head][29:5]); wire [8:0] wb_after = 9'(wq_cnt) + 9'(NB) * (9'(wb_count) + 9'd1); wire [8:0] rb_after = 9'(rq_cnt) + 9'(NB) * (9'(rb_count) + 9'd1); wire ih_time = ($signed(d - ingress_ready) >= 0); wire ih_room = ih_w ? (wb_after <= 9'(cfg_wq)) : (rb_after <= 9'(cfg_rq)); wire move = (iq_count != 0) & ih_time & ih_room; - wire mv_w = move & ih_w; - wire mv_r = move & ~ih_w; - // The buffers' bursts after the move, and the mode they put the channel in. - wire [8:0] wn = 9'(wq_cnt) + 9'(NB) * (9'(wb_count) + 9'(mv_w)); - wire [8:0] rn = 9'(rq_cnt) + 9'(NB) * (9'(rb_count) + 9'(mv_r)); + // The buffers' bursts, and the mode they put the channel in. + wire [8:0] wn = 9'(wq_cnt) + 9'(NB) * 9'(wb_count); + wire [8:0] rn = 9'(rq_cnt) + 9'(NB) * 9'(rb_count); wire to_write = ~wmode & ((wn >= 9'(cfg_wr_high)) | (rn == 0)); wire to_read = wmode & (wn < 9'(cfg_wr_low)) & (rn != 0); wire wmode_n = to_write | (wmode & ~to_read); - wire [BQW:0] sel_count = wmode_n ? (wb_count + (BQW+1)'(mv_w)) - : (rb_count + (BQW+1)'(mv_r)); + wire sel_any = wmode_n ? (wb_count != 0) : (rb_count != 0); + + // The bank table's port: a reservation's bank, or the scrub's. + wire bt_commit = (state == S_COMMIT) & cm_start; + wire bt_scrub = (state == S_TICK) & (credit < 32'(k_twoclk)) & ~bt_fresh[scrub_b]; + wire [4:0] bt_wa = bt_commit ? op_bank : scrub_b; + function automatic [15:0] scrubbed(input [15:0] t); + scrubbed = ($signed(t - (e + 16'd1)) < -16'sd16384) ? e + 16'd1 - 16'd16384 : t; + endfunction + always @(posedge clk) begin + if (bt_commit | bt_scrub) begin + bt_row[bt_wa] <= (bt_commit & ~op_hit) ? op_row : bt_row[bt_wa]; + bt_act[bt_wa] <= bt_commit ? (op_hit ? bt_act[bt_wa] : e + c_act) : scrubbed(bt_act[bt_wa]); + bt_colr[bt_wa] <= bt_commit ? e + ck_colr : scrubbed(bt_colr[bt_wa]); + bt_prer[bt_wa] <= bt_commit ? e + ck_prer : scrubbed(bt_prer[bt_wa]); + end + end // The end of DRAM clock d: clock d+1's departing read and the columns it // issues leave the controller; then the next clock. task automatic end_clock(); - if (end_depv) begin - rf_slot[rf_tail] <= dep_slot[base]; - rf_cyc[rf_tail] <= tcyc + 32'd1 + 32'(cfg_latency); - rf_tail <= rf_tail + 1'b1; - rf_count <= rf_count + 1'b1; - end - rq_cnt <= rq_cnt - (end_kr ? 8'd1 : 8'd0); - wq_cnt <= wq_cnt - (end_kw ? 8'd1 : 8'd0); + rfp_valid <= ed_depv; + rfp_slot <= ed_slot; + rfp_cyc <= tcyc1 + 32'(cfg_latency); + rq_cnt <= rq_cnt - (ed_kr ? 8'd1 : 8'd0); + wq_cnt <= wq_cnt - (ed_kw ? 8'd1 : 8'd0); // The ring's word ends with this clock: it is all past. if (base[5:0] == 6'd63) begin bm_clear <= 1'b1; bm_clear_word <= end_word; end - // A time too old to constrain anything is moved up before it could wrap. - if (~bt_fresh[scrub_b]) begin - if ($signed(bt_act[scrub_b] - (e + 16'd1)) < -16'sd16384) - bt_act[scrub_b] <= e + 16'd1 - 16'd16384; - if ($signed(bt_colr[scrub_b] - (e + 16'd1)) < -16'sd16384) - bt_colr[scrub_b] <= e + 16'd1 - 16'd16384; - if ($signed(bt_prer[scrub_b] - (e + 16'd1)) < -16'sd16384) - bt_prer[scrub_b] <= e + 16'd1 - 16'd16384; - end - scrub_b <= scrub_b + 1'b1; if ($signed(frontier - e) < 0) frontier <= e; if ($signed(batch_last - e) < 0) batch_last <= e; if ($signed(batch_floor - e) < 0) batch_floor <= e; if ($signed(read_floor - e) < 0) read_floor <= e; if ($signed(ingress_ready - e) < 0) ingress_ready <= e; - d <= d + 16'd1; + d <= d + 16'd1; endtask always @(posedge clk) begin bm_clear <= 1'b0; + bm_clear_all <= 1'b0; cm_start <= 1'b0; + rfp_valid <= 1'b0; - if (reset) begin - state <= S_IDLE; - active <= 1'b0; - stat_reads <= '0; - stat_writes <= '0; - stat_hits <= '0; - stat_misses <= '0; - stat_conflicts <= '0; - stat_latency <= '0; + if (rfp_valid) begin + rf_slot[rf_tail] <= rfp_slot; + rf_cyc[rf_tail] <= rfp_cyc; + rf_tail <= rf_tail + 1'b1; + rf_count <= rf_count + 1'b1; + end + lat_ar <= 1'b0; + lat_r <= 1'b0; + if (lat_ar | lat_r) + stat_latency <= stat_latency + (lat_r ? 64'(lat_t) : 64'd0) - (lat_ar ? 64'(lat_t) : 64'd0); + + // The engine runs while the target evaluates its cycle; the cycle + // finishes only once the engine is idle. + case (state) + S_PUSH: begin + // A cycle's write reaches the channel before its read. + if (st_w_valid) begin + iq_w[iq_tail] <= 1'b1; + iq_tag[iq_tail] <= TAG_W'(st_w_id); + {iq_bank[iq_tail], iq_row[iq_tail]} <= decode(st_w_addr[29:5]); + iq_tail <= iq_tail + 1'b1; + iq_count <= iq_count + 1'b1; + st_w_valid <= 1'b0; + end else begin + if (st_r_valid) begin + iq_w[iq_tail] <= 1'b0; + iq_tag[iq_tail] <= TAG_W'(st_r_slot); + {iq_bank[iq_tail], iq_row[iq_tail]} <= decode(st_r_addr[29:5]); + iq_tail <= iq_tail + 1'b1; + iq_count <= iq_count + 1'b1; + st_r_valid <= 1'b0; + end + state <= S_TICK; + end + end + S_TICK: begin + if (credit >= 32'(k_twoclk)) begin + // One DRAM clock d: the ingress head enters its buffer if it + // may. + credit <= credit - 32'(k_twoclk); + if (move) begin + iq_head <= iq_head + 1'b1; + iq_count <= iq_count - 1'b1; + ingress_ready <= d + 16'(NB); + if (ih_w) begin + wb_tag[wb_tail] <= iq_tag[iq_head]; + wb_bank[wb_tail] <= iq_bank[iq_head]; + wb_row[wb_tail] <= iq_row[iq_head]; + wb_tail <= wb_tail + 1'b1; + wb_count <= wb_count + 1'b1; + af_id[af_tail] <= ID_W'(iq_tag[iq_head]); + af_tail <= af_tail + 1'b1; + af_count <= af_count + 1'b1; + new_acks <= new_acks + 1'b1; + end else begin + rb_tag[rb_tail] <= iq_tag[iq_head]; + rb_bank[rb_tail] <= iq_bank[iq_head]; + rb_row[rb_tail] <= iq_row[iq_head]; + rb_tail <= rb_tail + 1'b1; + rb_count <= rb_count + 1'b1; + end + end + ed_depv <= rd_depv[base[5:0]]; + ed_kr <= rd_kr[base[5:0]]; + ed_kw <= rd_kw[base[5:0]]; + ed_slot <= dep_slot[base]; + state <= S_MODE; + end else begin + // The cycle's clocks are done; a bank's times move up (the + // table's port is free), and the ready reads are counted. + scrub_b <= scrub_b + 1'b1; + state <= S_WALK; + end + end + S_MODE: begin + // The channel picks reads or writes; the head of that buffer is + // reserved if it can be, and with nothing to reserve the clock + // ends now. + wmode <= wmode_n; + if (to_write) + batch_floor <= ($signed(frontier + 16'd1 - e) > 0) ? frontier + 16'd1 : e; + if (to_read) + read_floor <= batch_last + 16'd1; + if (sel_any) begin + state <= S_BANK; + end else begin + end_clock(); + state <= S_TICK; + end + end + S_BANK: begin + // The request at the head of the buffer the channel serves. + op_w <= wmode; + op_tag <= wmode ? wb_tag[wb_head] : rb_tag[rb_head]; + op_bank <= wmode ? wb_bank[wb_head] : rb_bank[rb_head]; + op_row <= wmode ? wb_row[wb_head] : rb_row[rb_head]; + r_floor <= rel(wmode ? batch_floor : read_floor); + state <= S_BANK2; + end + S_BANK2: begin + // Against its bank. + op_pc <= (g_lpcs != 0) ? op_bank[0] : 1'b0; + op_pg <= 3'((32'(op_bank) & ((32'd1 << (32'(g_lpcs) + 32'(g_lbgs))) - 32'd1))); + op_hit <= bt_open[op_bank] & (bt_row[op_bank] == op_row); + op_conf <= bt_open[op_bank] & (bt_row[op_bank] != op_row); + r_act <= bt_fresh[op_bank] ? -16'sd16384 : rel(bt_act[op_bank]); + r_colr <= bt_fresh[op_bank] ? -16'sd1 : rel(bt_colr[op_bank]); + r_prer <= bt_fresh[op_bank] ? -16'sd1 : rel(bt_prer[op_bank]); + kind <= (bt_open[op_bank] & (bt_row[op_bank] == op_row)) ? K_C0 + : (bt_open[op_bank] ? K_PRE : K_ACT); + state <= S_SETUP; + end + S_SETUP: begin + // The search's lower bound, the first clock it may place at. + case (kind) + K_PRE: s_lb <= smax(16'sd0, r_prer); + K_ACT: s_lb <= smax(op_conf ? c_pre + k_rp : 16'sd0, r_act + k_rc); + K_C0: s_lb <= smax(smax(16'sd0, r_floor), + op_hit ? r_colr : c_act + (op_w ? k_rcdwr : k_rcdrd)); + default: s_lb <= c_col0 + k_ccdl1; + endcase + state <= S_SETUP2; + end + S_SETUP2: begin + if (s_lb >= 16'(HE)) begin + state <= S_END; + end else begin + s_pos <= HLOG2'(base + HLOG2'(s_lb)); + s_left <= 16'(HE) - s_lb; + s_word <= WW'(HLOG2'(base + HLOG2'(s_lb)) >> 6); + s_fetch <= 1'b1; + sb_valid <= 1'b0; + state <= S_SRCH; + end + end + S_SRCH: begin + // A word a cycle: the word read now, masked to the clocks left; + // and last cycle's word, whose first free clock wins. + if (s_fetch) begin + b_cand <= ~forb & s_mask; + b_word <= s_word; + sb_valid <= 1'b1; + b_last <= (s_left - s_take == 0); + s_pos <= HLOG2'(s_pos + HLOG2'(s_take)); + s_word <= WW'(HLOG2'(s_pos + HLOG2'(s_take)) >> 6); + s_left <= s_left - s_take; + s_fetch <= (s_left - s_take != 0); + end + if (sb_valid) begin + if (b_cand != 0) begin + case (kind) + K_PRE: begin c_pre <= b_rel; kind <= K_ACT; state <= S_SETUP; end + K_ACT: begin c_act <= b_rel; kind <= K_C0; state <= S_SETUP; end + K_C0: begin + c_col0 <= b_rel; + if (NB == 2) begin + kind <= K_C1; + state <= S_SETUP; + end else begin + state <= S_CHECK; + end + end + default: begin c_col1 <= b_rel; state <= S_CHECK; end + endcase + end else if (b_last) begin + state <= S_END; + end + end + end + S_CHECK: begin + // How far the reservation reaches, and the bank's new times. + begin : check_ + reg signed [15:0] lastcol, colr, prer; + lastcol = (NB == 2) ? c_col1 : c_col0; + colr = op_hit ? r_colr : c_act + (op_w ? k_rcdwr : k_rcdrd); + prer = op_hit ? r_prer : c_act + k_ras; + ck_lastcol <= lastcol; + ck_colr <= smax(colr, lastcol + 16'sd1); + ck_prer <= smax(prer, lastcol + (op_w ? k_wrpre : k_rtp)); + ck_tcol <= lastcol + (op_w ? k_reach_w : k_reach_r); + ck_tdep <= op_w ? -16'sd16384 : lastcol + k_dep; + ck_tact <= op_hit ? -16'sd16384 : c_act + k_reach_a; + end + state <= S_CHECK2; + end + S_CHECK2: begin + // The reservation must reach no further than the horizon. + begin : check2_ + reg ok; + ok = (ck_tcol < 16'(HE)) & (ck_tdep < 16'(HE)) & (ck_tact < 16'(HE)); + state <= ok ? S_COMMIT : S_END; + cm_start <= ok; + end end + S_COMMIT: begin + if (cm_start) begin + // The bank (its times through the table's port), the + // controller, the read's departure, and a column in clock d+1 + // for the clock's end. + if (~op_hit) + bt_open[op_bank] <= 1'b1; + bt_fresh[op_bank] <= 1'b0; + if (op_w) begin + wq_cnt <= wq_cnt + 8'(NB); + wb_head <= wb_head + 1'b1; + wb_count <= wb_count - 1'b1; + batch_last <= ($signed(e + ck_lastcol - batch_last) > 0) ? e + ck_lastcol : batch_last; + stat_writes <= stat_writes + 64'd1; + if (c_col0 == 0) ed_kw <= 1'b1; + end else begin + rq_cnt <= rq_cnt + 8'(NB); + rb_head <= rb_head + 1'b1; + rb_count <= rb_count - 1'b1; + dep_slot[HLOG2'(base + HLOG2'(ck_lastcol + k_dep))] <= SW'(op_tag); + stat_reads <= stat_reads + 64'd1; + if (c_col0 == 0) ed_kr <= 1'b1; + end + frontier <= ($signed(e + ck_lastcol - frontier) > 0) ? e + ck_lastcol : frontier; + if (op_hit) + stat_hits <= stat_hits + 64'd1; + else if (op_conf) + stat_conflicts <= stat_conflicts + 64'd1; + else + stat_misses <= stat_misses + 64'd1; + end else if (!cm_any) begin + state <= S_END; + end + end + S_END: begin + end_clock(); + state <= S_TICK; + end + S_WALK: begin + // Reads that are ready in the next target cycle. + if (rf_ready != rf_count + && $signed(rf_cyc[SW'(rf_head + SW'(rf_ready))] - tcyc1) <= 0) begin + rf_ready <= rf_ready + 1'b1; + new_ready <= new_ready + 1'b1; + end else begin + state <= S_IDLE; + end + end + default:; + endcase + if (reset || finish) begin // ---- The target cycle ends: the port's state for the next ---- if (finish && treset) @@ -505,6 +767,7 @@ module foldsim_mem_bank #( cur_r_valid <= 1'b0; cur_b_valid <= 1'b0; tcyc <= '0; + tcyc1 <= 32'd1; rf_head <= '0; rf_tail <= '0; rf_count <= '0; @@ -536,8 +799,10 @@ module foldsim_mem_bank #( bt_open <= '0; bt_fresh <= '1; scrub_b <= '0; + bm_clear_all <= 1'b1; end else begin - tcyc <= tcyc + 32'd1; + tcyc <= tcyc1; + tcyc1 <= tcyc1 + 32'd1; // R and B, then what is ready next cycle. begin : release_ reg r_load, b_load; @@ -569,247 +834,37 @@ module foldsim_mem_bank #( end slots_used <= slots_used + (SW+1)'(ar_fire) - (SW+1)'(r_fire); inflight <= inflight + 16'(ar_fire) + 16'(aw_fire) - 16'(new_ready) - 16'(new_acks); - if (ar_fire) - stat_latency <= stat_latency - 64'(tcyc); - if (r_fire) - stat_latency <= stat_latency + 64'(tcyc) - (ar_fire ? 64'(tcyc) : 64'd0); + lat_ar <= ar_fire; + lat_r <= r_fire; + lat_t <= tcyc; end st_w_valid <= new_write; st_w_id <= nw_id; - st_w_addr <= nw_addr; + st_w_addr <= nw_addr[29:5]; st_r_valid <= ar_fire; st_r_slot <= alloc_slot; - st_r_addr <= ar_addr; + st_r_addr <= ar_addr[29:5]; credit <= credit + 32'(cfg_data_rate); end - if (~reset) - state <= ~(treset ? enable : active) ? S_IDLE - : ((~treset & active & (new_write | ar_fire)) ? S_PUSH : S_TICK); + // A clearing cycle's next host cycle clears the calendars, so the + // first DRAM clock waits a host cycle (in S_PUSH). + state <= ~(treset ? enable : active) ? S_IDLE + : (((treset | ~active) | new_write | ar_fire) ? S_PUSH : S_TICK); new_ready <= '0; new_acks <= '0; - end else begin - case (state) - S_PUSH: begin - // A cycle's write reaches the channel before its read. - if (st_w_valid) begin - iq_w[iq_tail] <= 1'b1; - iq_tag[iq_tail] <= TAG_W'(st_w_id); - iq_addr[iq_tail] <= st_w_addr; - iq_tail <= iq_tail + 1'b1; - iq_count <= iq_count + 1'b1; - st_w_valid <= 1'b0; - end else begin - if (st_r_valid) begin - iq_w[iq_tail] <= 1'b0; - iq_tag[iq_tail] <= TAG_W'(st_r_slot); - iq_addr[iq_tail] <= st_r_addr; - iq_tail <= iq_tail + 1'b1; - iq_count <= iq_count + 1'b1; - st_r_valid <= 1'b0; - end - state <= S_TICK; - end - end - S_TICK: begin - if (credit >= 32'(k_twoclk)) begin - // One DRAM clock d: the ingress head enters its buffer if - // it may, the channel picks reads or writes, and the head - // of that buffer is reserved if it can be. - credit <= credit - 32'(k_twoclk); - // With nothing to reserve the clock ends now; what the - // tick sets below wins over the clock's end. - if (sel_count == 0) - end_clock(); - if (move) begin - iq_head <= iq_head + 1'b1; - iq_count <= iq_count - 1'b1; - ingress_ready <= d + 16'(NB); - if (ih_w) begin - wb_tag[wb_tail] <= ih_tag; - wb_bank[wb_tail] <= ih_dec[BKW+ROW_W-1:ROW_W]; - wb_row[wb_tail] <= ih_dec[ROW_W-1:0]; - wb_tail <= wb_tail + 1'b1; - af_id[af_tail] <= ID_W'(ih_tag); - af_tail <= af_tail + 1'b1; - af_count <= af_count + 1'b1; - new_acks <= new_acks + 1'b1; - end else begin - rb_tag[rb_tail] <= ih_tag; - rb_bank[rb_tail] <= ih_dec[BKW+ROW_W-1:ROW_W]; - rb_row[rb_tail] <= ih_dec[ROW_W-1:0]; - rb_tail <= rb_tail + 1'b1; - end - end - if (move & ih_w) - wb_count <= wb_count + 1'b1; - if (move & ~ih_w) - rb_count <= rb_count + 1'b1; - wmode <= wmode_n; - if (to_write) - batch_floor <= ($signed(frontier + 16'd1 - e) > 0) ? frontier + 16'd1 : e; - if (to_read) - read_floor <= batch_last + 16'd1; - state <= (sel_count != 0) ? S_BANK : S_TICK; - end else begin - state <= S_WALK; - end - end - S_BANK: begin - // The request at the head of the buffer the channel serves, - // against its bank. - begin : bank_ - reg [BKW-1:0] b; - reg [ROW_W-1:0] row; - reg w; - w = wmode; - b = w ? wb_bank[wb_head] : rb_bank[rb_head]; - row = w ? wb_row[wb_head] : rb_row[rb_head]; - op_w <= w; - op_tag <= w ? wb_tag[wb_head] : rb_tag[rb_head]; - op_bank <= b; - op_row <= row; - op_pc <= (g_lpcs != 0) ? b[0] : 1'b0; - op_pg <= 3'((32'(b) & ((32'd1 << (32'(g_lpcs) + 32'(g_lbgs))) - 32'd1))); - op_hit <= bt_open[b] & (bt_row[b] == row); - op_conf <= bt_open[b] & (bt_row[b] != row); - r_act <= bt_fresh[b] ? -16'sd16384 : rel(bt_act[b]); - r_colr <= bt_fresh[b] ? -16'sd1 : rel(bt_colr[b]); - r_prer <= bt_fresh[b] ? -16'sd1 : rel(bt_prer[b]); - r_floor <= rel(w ? batch_floor : read_floor); - kind <= (bt_open[b] & (bt_row[b] == row)) ? K_C0 - : (bt_open[b] ? K_PRE : K_ACT); - end - state <= S_SETUP; - end - S_SETUP: begin - // The search's lower bound, the first clock it may place at. - begin : setup_ - reg signed [15:0] lb; - case (kind) - K_PRE: lb = smax(16'sd0, r_prer); - K_ACT: lb = smax(op_conf ? c_pre + k_rp : 16'sd0, r_act + k_rc); - K_C0: lb = smax(smax(16'sd0, r_floor), - op_hit ? r_colr : c_act + (op_w ? k_rcdwr : k_rcdrd)); - default: lb = c_col0 + k_ccdl1; - endcase - if (lb >= 16'(HE)) begin - state <= S_END; - end else begin - s_pos <= HLOG2'(base + HLOG2'(lb)); - s_left <= 16'(HE) - lb; - s_word <= WW'(HLOG2'(base + HLOG2'(lb)) >> 6); - s_fetch <= 1'b1; - sb_valid <= 1'b0; - state <= S_SRCH; - end - end - end - S_SRCH: begin - // A word a cycle: the word read now, masked to the clocks - // left; and last cycle's word, whose first free clock wins. - if (s_fetch) begin - b_cand <= ~forb & s_mask; - b_word <= s_word; - sb_valid <= 1'b1; - b_last <= (s_left - s_take == 0); - s_pos <= HLOG2'(s_pos + HLOG2'(s_take)); - s_word <= WW'(HLOG2'(s_pos + HLOG2'(s_take)) >> 6); - s_left <= s_left - s_take; - s_fetch <= (s_left - s_take != 0); - end - if (sb_valid) begin - if (b_cand != 0) begin - case (kind) - K_PRE: begin c_pre <= b_rel; kind <= K_ACT; state <= S_SETUP; end - K_ACT: begin c_act <= b_rel; kind <= K_C0; state <= S_SETUP; end - K_C0: begin - c_col0 <= b_rel; - if (NB == 2) begin - kind <= K_C1; - state <= S_SETUP; - end else begin - state <= S_CHECK; - end - end - default: begin c_col1 <= b_rel; state <= S_CHECK; end - endcase - end else if (b_last) begin - state <= S_END; - end - end - end - S_CHECK: begin - // The reservation must reach no further than the horizon. - begin : check_ - reg signed [15:0] lastcol, last, colr, prer; - reg act; - act = ~op_hit; - lastcol = (NB == 2) ? c_col1 : c_col0; - last = lastcol + (op_w ? k_reach_w : k_reach_r); - if (~op_w) - last = smax(last, lastcol + k_dep); - if (act) - last = smax(last, c_act + k_reach_a); - colr = op_hit ? r_colr : c_act + (op_w ? k_rcdwr : k_rcdrd); - prer = op_hit ? r_prer : c_act + k_ras; - ck_lastcol <= lastcol; - ck_colr <= smax(colr, lastcol + 16'sd1); - ck_prer <= smax(prer, lastcol + (op_w ? k_wrpre : k_rtp)); - state <= (last < 16'(HE)) ? S_COMMIT : S_END; - cm_start <= (last < 16'(HE)); - end - end - S_COMMIT: begin - if (cm_start) begin - // The bank, the controller, the read's departure. - if (~op_hit) begin - bt_open[op_bank] <= 1'b1; - bt_row[op_bank] <= op_row; - bt_act[op_bank] <= e + c_act; - end - bt_fresh[op_bank] <= 1'b0; - bt_colr[op_bank] <= e + ck_colr; - bt_prer[op_bank] <= e + ck_prer; - if (op_w) begin - wq_cnt <= wq_cnt + 8'(NB); - wb_head <= wb_head + 1'b1; - wb_count <= wb_count - 1'b1; - batch_last <= ($signed(e + ck_lastcol - batch_last) > 0) ? e + ck_lastcol : batch_last; - stat_writes <= stat_writes + 64'd1; - end else begin - rq_cnt <= rq_cnt + 8'(NB); - rb_head <= rb_head + 1'b1; - rb_count <= rb_count - 1'b1; - dep_slot[HLOG2'(base + HLOG2'(ck_lastcol + k_dep))] <= SW'(op_tag); - stat_reads <= stat_reads + 64'd1; - end - frontier <= ($signed(e + ck_lastcol - frontier) > 0) ? e + ck_lastcol : frontier; - if (op_hit) - stat_hits <= stat_hits + 64'd1; - else if (op_conf) - stat_conflicts <= stat_conflicts + 64'd1; - else - stat_misses <= stat_misses + 64'd1; - end else if (!cm_any) begin - state <= S_END; - end - end - S_END: begin - end_clock(); - state <= S_TICK; - end - S_WALK: begin - // Reads that are ready in the next target cycle. - if (rf_ready != rf_count - && $signed(rf_cyc[SW'(rf_head + SW'(rf_ready))] - (tcyc + 32'd1)) <= 0) begin - rf_ready <= rf_ready + 1'b1; - new_ready <= new_ready + 1'b1; - end else begin - state <= S_IDLE; - end - end - default:; - endcase + end + if (reset) begin + state <= S_IDLE; + active <= 1'b0; + stat_reads <= '0; + stat_writes <= '0; + stat_hits <= '0; + stat_misses <= '0; + stat_conflicts <= '0; + stat_latency <= '0; + lat_ar <= 1'b0; + lat_r <= 1'b0; + rfp_valid <= 1'b0; end end From 7d24a308ded550265ba089d23b2dc5a14389ea2f Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 09:57:18 -0700 Subject: [PATCH 226/336] foldsim_mem: staged token ends (Fmax proposal 6.4) Depends on 9a5fc9066 (cherry-picked c1b30f880 from exp/domains-flow, "foldsim: the control endpoint's token ends are staged"), which adds foldsim_token's STAGED parameter. foldsim_mem takes STAGED and passes it to its token ends: have and done are registers, so the endpoint's finish -- and every register it enables -- starts at the endpoint's own state instead of the simulator's valid and ready, the flow-through the Fmax runs found on the s2w4i2 kernel's worst paths. foldsim_afu builds the endpoint staged (MEM_STAGED). A token counts a host cycle later; target behavior is unchanged. Measured, s2w4i2 in the in-process simulator (Verilator): pipe demo and sgemm unchanged in every count (4888 / 8702 / 15559 target / 125233 host; 336928 / 415454 / 422288 / 3379066): the threaded models' eight host cycles a target cycle hide the extra one. Bank model: demo 173809 host cycles (+0.4 %), sgemm 2972748 (-0.7 %), target behavior unchanged. In hw/unittest/foldsim_mem's harness, whose master answers at once, the pipe settings take 3-14 % more host cycles, the bank settings within 4 %. hw/unittest/foldsim_mem_staged runs the endpoint test on the staged endpoint: 0 differences against the C++ models on all 14 settings, as the unstaged one. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 059f239df024c8721667b579a884c5af06290053) --- hw/rtl/afu/foldsim/foldsim_afu.sv | 7 ++- hw/rtl/afu/foldsim/foldsim_mem.sv | 57 +++++++++++++------------ hw/unittest/Makefile | 3 ++ hw/unittest/foldsim_mem_staged/Makefile | 27 ++++++++++++ 4 files changed, 66 insertions(+), 28 deletions(-) create mode 100644 hw/unittest/foldsim_mem_staged/Makefile diff --git a/hw/rtl/afu/foldsim/foldsim_afu.sv b/hw/rtl/afu/foldsim/foldsim_afu.sv index 7464ece020..b7b2cad3cd 100644 --- a/hw/rtl/afu/foldsim/foldsim_afu.sv +++ b/hw/rtl/afu/foldsim/foldsim_afu.sv @@ -177,6 +177,10 @@ module foldsim_afu #( // clocks (MEM_CAPS reports both). localparam MEM_SLOTS = 32; localparam MEM_HLOG2 = 9; + // The memory endpoint's token ends register have and done, so its finish + // starts at its own state, not at the simulator's handshake. A token then + // counts a host cycle later; s2w4i2's host cycles do not move. + localparam MEM_STAGED = 1; localparam [31:0] MEM_CAPS = {4'd0, 4'd1, 4'd1, 8'(MEM_SLOTS), 4'(MEM_HLOG2 - 4), 8'd1}; wire [31:0] mem_read_latency, mem_write_latency; @@ -196,7 +200,8 @@ module foldsim_afu #( .ADDR_W (TARGET_ADDR_W), .DATA_W (DATA_W), .SLOTS (MEM_SLOTS), - .HLOG2 (MEM_HLOG2) + .HLOG2 (MEM_HLOG2), + .STAGED (MEM_STAGED) ) mem ( .clk (ap_clk), .reset (reset), diff --git a/hw/rtl/afu/foldsim/foldsim_mem.sv b/hw/rtl/afu/foldsim/foldsim_mem.sv index 7527599735..7237908105 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem.sv @@ -47,7 +47,10 @@ module foldsim_mem #( parameter DEPTH = 16, // The bank model: read slots, and a ring of 2^HLOG2 DRAM clocks. parameter SLOTS = 32, - parameter HLOG2 = 9 + parameter HLOG2 = 9, + // Token ends with registered have and done (foldsim_token): the cycle's + // finish starts at the endpoint's own registers, a host cycle later. + parameter STAGED = 0 ) ( input wire clk, input wire reset, @@ -150,21 +153,21 @@ module foldsim_mem #( wire [DATA_W/8-1:0] wstrb; wire [14:0] have; - foldsim_token_in #(1) t_awvalid (clk, reset, awvalid_valid, awvalid_ready, awvalid_bits, finish, have[0], awvalid); - foldsim_token_in #(ADDR_W) t_awaddr (clk, reset, awaddr_valid, awaddr_ready, awaddr_bits, finish, have[1], awaddr); - foldsim_token_in #(ID_W) t_awid (clk, reset, awid_valid, awid_ready, awid_bits, finish, have[2], awid); - foldsim_token_in #(8) t_awlen (clk, reset, awlen_valid, awlen_ready, awlen_bits, finish, have[3], awlen); - foldsim_token_in #(1) t_wvalid (clk, reset, wvalid_valid, wvalid_ready, wvalid_bits, finish, have[4], wvalid); - foldsim_token_in #(DATA_W) t_wdata (clk, reset, wdata_valid, wdata_ready, wdata_bits, finish, have[5], wdata); - foldsim_token_in #(DATA_W/8) t_wstrb (clk, reset, wstrb_valid, wstrb_ready, wstrb_bits, finish, have[6], wstrb); - foldsim_token_in #(1) t_wlast (clk, reset, wlast_valid, wlast_ready, wlast_bits, finish, have[7], wlast); - foldsim_token_in #(1) t_arvalid (clk, reset, arvalid_valid, arvalid_ready, arvalid_bits, finish, have[8], arvalid); - foldsim_token_in #(ADDR_W) t_araddr (clk, reset, araddr_valid, araddr_ready, araddr_bits, finish, have[9], araddr); - foldsim_token_in #(ID_W) t_arid (clk, reset, arid_valid, arid_ready, arid_bits, finish, have[10], arid); - foldsim_token_in #(8) t_arlen (clk, reset, arlen_valid, arlen_ready, arlen_bits, finish, have[11], arlen); - foldsim_token_in #(1) t_rready (clk, reset, rready_valid, rready_ready, rready_bits, finish, have[12], rready); - foldsim_token_in #(1) t_bready (clk, reset, bready_valid, bready_ready, bready_bits, finish, have[13], bready); - foldsim_token_in #(1) t_treset (clk, reset, treset_valid, treset_ready, treset_bits, finish, have[14], treset); + foldsim_token_in #(1, STAGED) t_awvalid (clk, reset, awvalid_valid, awvalid_ready, awvalid_bits, finish, have[0], awvalid); + foldsim_token_in #(ADDR_W, STAGED) t_awaddr (clk, reset, awaddr_valid, awaddr_ready, awaddr_bits, finish, have[1], awaddr); + foldsim_token_in #(ID_W, STAGED) t_awid (clk, reset, awid_valid, awid_ready, awid_bits, finish, have[2], awid); + foldsim_token_in #(8, STAGED) t_awlen (clk, reset, awlen_valid, awlen_ready, awlen_bits, finish, have[3], awlen); + foldsim_token_in #(1, STAGED) t_wvalid (clk, reset, wvalid_valid, wvalid_ready, wvalid_bits, finish, have[4], wvalid); + foldsim_token_in #(DATA_W, STAGED) t_wdata (clk, reset, wdata_valid, wdata_ready, wdata_bits, finish, have[5], wdata); + foldsim_token_in #(DATA_W/8, STAGED) t_wstrb (clk, reset, wstrb_valid, wstrb_ready, wstrb_bits, finish, have[6], wstrb); + foldsim_token_in #(1, STAGED) t_wlast (clk, reset, wlast_valid, wlast_ready, wlast_bits, finish, have[7], wlast); + foldsim_token_in #(1, STAGED) t_arvalid (clk, reset, arvalid_valid, arvalid_ready, arvalid_bits, finish, have[8], arvalid); + foldsim_token_in #(ADDR_W, STAGED) t_araddr (clk, reset, araddr_valid, araddr_ready, araddr_bits, finish, have[9], araddr); + foldsim_token_in #(ID_W, STAGED) t_arid (clk, reset, arid_valid, arid_ready, arid_bits, finish, have[10], arid); + foldsim_token_in #(8, STAGED) t_arlen (clk, reset, arlen_valid, arlen_ready, arlen_bits, finish, have[11], arlen); + foldsim_token_in #(1, STAGED) t_rready (clk, reset, rready_valid, rready_ready, rready_bits, finish, have[12], rready); + foldsim_token_in #(1, STAGED) t_bready (clk, reset, bready_valid, bready_ready, bready_bits, finish, have[13], bready); + foldsim_token_in #(1, STAGED) t_treset (clk, reset, treset_valid, treset_ready, treset_bits, finish, have[14], treset); // ---- The pipe's state ---- reg [31:0] cycle; @@ -219,17 +222,17 @@ module foldsim_mem #( : ((~cur_r_valid | (rd_count != 0)) & (~cur_w_valid | (b_acked != b_given))); wire [10:0] done; - foldsim_token_out o_awready (clk, reset, awready_valid, awready_ready, ok, finish, done[0]); - foldsim_token_out o_wready (clk, reset, wready_valid, wready_ready, ok, finish, done[1]); - foldsim_token_out o_arready (clk, reset, arready_valid, arready_ready, ok, finish, done[2]); - foldsim_token_out o_rvalid (clk, reset, rvalid_valid, rvalid_ready, ok, finish, done[3]); - foldsim_token_out o_rdata (clk, reset, rdata_valid, rdata_ready, ok, finish, done[4]); - foldsim_token_out o_rlast (clk, reset, rlast_valid, rlast_ready, ok, finish, done[5]); - foldsim_token_out o_rid (clk, reset, rid_valid, rid_ready, ok, finish, done[6]); - foldsim_token_out o_rresp (clk, reset, rresp_valid, rresp_ready, ok, finish, done[7]); - foldsim_token_out o_bvalid (clk, reset, bvalid_valid, bvalid_ready, ok, finish, done[8]); - foldsim_token_out o_bresp (clk, reset, bresp_valid, bresp_ready, ok, finish, done[9]); - foldsim_token_out o_bid (clk, reset, bid_valid, bid_ready, ok, finish, done[10]); + foldsim_token_out #(STAGED) o_awready (clk, reset, awready_valid, awready_ready, ok, finish, done[0]); + foldsim_token_out #(STAGED) o_wready (clk, reset, wready_valid, wready_ready, ok, finish, done[1]); + foldsim_token_out #(STAGED) o_arready (clk, reset, arready_valid, arready_ready, ok, finish, done[2]); + foldsim_token_out #(STAGED) o_rvalid (clk, reset, rvalid_valid, rvalid_ready, ok, finish, done[3]); + foldsim_token_out #(STAGED) o_rdata (clk, reset, rdata_valid, rdata_ready, ok, finish, done[4]); + foldsim_token_out #(STAGED) o_rlast (clk, reset, rlast_valid, rlast_ready, ok, finish, done[5]); + foldsim_token_out #(STAGED) o_rid (clk, reset, rid_valid, rid_ready, ok, finish, done[6]); + foldsim_token_out #(STAGED) o_rresp (clk, reset, rresp_valid, rresp_ready, ok, finish, done[7]); + foldsim_token_out #(STAGED) o_bvalid (clk, reset, bvalid_valid, bvalid_ready, ok, finish, done[8]); + foldsim_token_out #(STAGED) o_bresp (clk, reset, bresp_valid, bresp_ready, ok, finish, done[9]); + foldsim_token_out #(STAGED) o_bid (clk, reset, bid_valid, bid_ready, ok, finish, done[10]); assign awready_bits = aw_ready; assign wready_bits = w_ready; diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index c1531a435f..23baf30e94 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -21,6 +21,7 @@ all: $(MAKE) -C afu_reset_seq $(MAKE) -C mm_axi_arb $(MAKE) -C foldsim_mem + $(MAKE) -C foldsim_mem_staged $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma $(MAKE) -C cp_llc_port @@ -57,6 +58,7 @@ run: $(MAKE) -C afu_reset_seq run $(MAKE) -C mm_axi_arb run $(MAKE) -C foldsim_mem run + $(MAKE) -C foldsim_mem_staged run $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run $(MAKE) -C cp_llc_port run @@ -210,6 +212,7 @@ clean: $(MAKE) -C mm_axi_arb clean $(MAKE) -C cp_core clean $(MAKE) -C foldsim_mem clean + $(MAKE) -C foldsim_mem_staged clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean $(MAKE) -C fsqrt_unit clean diff --git a/hw/unittest/foldsim_mem_staged/Makefile b/hw/unittest/foldsim_mem_staged/Makefile new file mode 100644 index 0000000000..ea68da7673 --- /dev/null +++ b/hw/unittest/foldsim_mem_staged/Makefile @@ -0,0 +1,27 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +# hw/unittest/foldsim_mem's test, on the endpoint with staged token ends (the +# kernel's). +PROJECT := foldsim_mem + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/foldsim_mem + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_INCLUDE := -I$(FOLDSIM_RTL_DIR) +PARAMS += -GSTAGED=1 + +# The token modules are not named after their file. +VL_FLAGS += $(FOLDSIM_RTL_DIR)/foldsim_token.sv + +TOP := foldsim_mem + +include ../common.mk From 454a4729077b325fcb8be3b1886e620a5c1b62a6 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 11:05:10 -0700 Subject: [PATCH 227/336] foldsim_mem_bank: close 3.33 ns Out-of-context (V80 part, endpoint staged as the kernel builds it) 3.33 ns missed by 0.036 ns on the walk's ready compare and the bank table's write data. The walk reads the departure FIFO at a running scan index and compares 24-bit ready cycles (an entry not yet counted ready is at most LATENCY + 1 cycles ahead); the scrub reads its bank in one host cycle and writes it in the next; the reservation's absolute bank times and the commit's ranges are registered as the horizon is checked. hw/unittest/foldsim_mem and foldsim_mem_staged: 0 differences on all 14 settings. s2w4i2 in the in-process simulator: pipe demo/sgemm unchanged in every count; bank-model runs unchanged in target behavior. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 9865f2abe75af94a1213d36416f9950158f88fb4) --- hw/rtl/afu/foldsim/foldsim_mem_bank.sv | 118 ++++++++++++++++--------- 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv index da1121be07..1522b81891 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv @@ -213,13 +213,15 @@ module foldsim_mem_bank #( // Reads departed, in order, with the target cycle they are ready in; the // first `rf_ready` are ready in the next cycle. A departure enters the // host cycle after its DRAM clock ends. + // Ready cycles are kept modulo 2^24: an entry not yet counted ready is at + // most LATENCY + 1 cycles ahead. reg [SW-1:0] rf_slot [SLOTS]; - reg [31:0] rf_cyc [SLOTS]; - reg [SW-1:0] rf_head, rf_tail; + reg [23:0] rf_cyc [SLOTS]; + reg [SW-1:0] rf_head, rf_tail, rf_scan; // rf_scan = rf_head + rf_ready reg [SW:0] rf_count, rf_ready; reg rfp_valid; reg [SW-1:0] rfp_slot; - reg [31:0] rfp_cyc; + reg [23:0] rfp_cyc; // Writes the controller has buffered, to acknowledge in order. localparam AQW = $clog2(WSLOTS); reg [ID_W-1:0] af_id [WSLOTS]; @@ -430,6 +432,10 @@ module foldsim_mem_bank #( wire signed [15:0] b_rel = 16'(HLOG2'(b_pos - base)); // ---- Commit: per calendar, up to two ranges of clocks to mark ---- + // (computed from the reservation, registered as the horizon is checked) + reg cx_v [NBM][2]; + reg signed [15:0] cx_lo [NBM][2]; + reg signed [15:0] cx_hi [NBM][2]; reg cm_v [NBM][2]; reg signed [15:0] cm_lo [NBM][2]; reg signed [15:0] cm_hi [NBM][2]; @@ -469,18 +475,24 @@ module foldsim_mem_bank #( wire sel_any = wmode_n ? (wb_count != 0) : (rb_count != 0); // The bank table's port: a reservation's bank, or the scrub's. + // The scrub reads a bank in one host cycle and writes it in the next. + reg sc_pend; + reg [4:0] sc_b; + reg [15:0] sc_act, sc_colr, sc_prer; + reg [ROW_W-1:0] sc_row; + reg [15:0] cw_act, cw_colr, cw_prer; // the reservation's, absolute + reg [ROW_W-1:0] cw_row; wire bt_commit = (state == S_COMMIT) & cm_start; - wire bt_scrub = (state == S_TICK) & (credit < 32'(k_twoclk)) & ~bt_fresh[scrub_b]; - wire [4:0] bt_wa = bt_commit ? op_bank : scrub_b; + wire [4:0] bt_wa = bt_commit ? op_bank : sc_b; function automatic [15:0] scrubbed(input [15:0] t); scrubbed = ($signed(t - (e + 16'd1)) < -16'sd16384) ? e + 16'd1 - 16'd16384 : t; endfunction always @(posedge clk) begin - if (bt_commit | bt_scrub) begin - bt_row[bt_wa] <= (bt_commit & ~op_hit) ? op_row : bt_row[bt_wa]; - bt_act[bt_wa] <= bt_commit ? (op_hit ? bt_act[bt_wa] : e + c_act) : scrubbed(bt_act[bt_wa]); - bt_colr[bt_wa] <= bt_commit ? e + ck_colr : scrubbed(bt_colr[bt_wa]); - bt_prer[bt_wa] <= bt_commit ? e + ck_prer : scrubbed(bt_prer[bt_wa]); + if (bt_commit | sc_pend) begin + bt_row[bt_wa] <= bt_commit ? cw_row : sc_row; + bt_act[bt_wa] <= bt_commit ? cw_act : sc_act; + bt_colr[bt_wa] <= bt_commit ? cw_colr : sc_colr; + bt_prer[bt_wa] <= bt_commit ? cw_prer : sc_prer; end end @@ -489,7 +501,7 @@ module foldsim_mem_bank #( task automatic end_clock(); rfp_valid <= ed_depv; rfp_slot <= ed_slot; - rfp_cyc <= tcyc1 + 32'(cfg_latency); + rfp_cyc <= 24'(tcyc1 + 32'(cfg_latency)); rq_cnt <= rq_cnt - (ed_kr ? 8'd1 : 8'd0); wq_cnt <= wq_cnt - (ed_kw ? 8'd1 : 8'd0); // The ring's word ends with this clock: it is all past. @@ -510,6 +522,7 @@ module foldsim_mem_bank #( bm_clear_all <= 1'b0; cm_start <= 1'b0; rfp_valid <= 1'b0; + sc_pend <= 1'b0; if (rfp_valid) begin rf_slot[rf_tail] <= rfp_slot; @@ -581,6 +594,12 @@ module foldsim_mem_bank #( end else begin // The cycle's clocks are done; a bank's times move up (the // table's port is free), and the ready reads are counted. + sc_pend <= ~bt_fresh[scrub_b]; + sc_b <= scrub_b; + sc_row <= bt_row[scrub_b]; + sc_act <= scrubbed(bt_act[scrub_b]); + sc_colr <= scrubbed(bt_colr[scrub_b]); + sc_prer <= scrubbed(bt_prer[scrub_b]); scrub_b <= scrub_b + 1'b1; state <= S_WALK; end @@ -698,6 +717,17 @@ module foldsim_mem_bank #( end S_CHECK2: begin // The reservation must reach no further than the horizon. + for (integer m = 0; m < NBM; ++m) begin + for (integer k = 0; k < 2; ++k) begin + cm_v[m][k] <= cx_v[m][k]; + cm_lo[m][k] <= cx_lo[m][k]; + cm_hi[m][k] <= cx_hi[m][k]; + end + end + cw_row <= op_hit ? bt_row[op_bank] : op_row; + cw_act <= op_hit ? bt_act[op_bank] : e + c_act; + cw_colr <= e + ck_colr; + cw_prer <= e + ck_prer; begin : check2_ reg ok; ok = (ck_tcol < 16'(HE)) & (ck_tdep < 16'(HE)) & (ck_tact < 16'(HE)); @@ -745,9 +775,9 @@ module foldsim_mem_bank #( end S_WALK: begin // Reads that are ready in the next target cycle. - if (rf_ready != rf_count - && $signed(rf_cyc[SW'(rf_head + SW'(rf_ready))] - tcyc1) <= 0) begin + if (rf_ready != rf_count && $signed(rf_cyc[rf_scan] - tcyc1[23:0]) <= 0) begin rf_ready <= rf_ready + 1'b1; + rf_scan <= rf_scan + 1'b1; new_ready <= new_ready + 1'b1; end else begin state <= S_IDLE; @@ -772,6 +802,7 @@ module foldsim_mem_bank #( rf_tail <= '0; rf_count <= '0; rf_ready <= '0; + rf_scan <= '0; af_head <= '0; af_tail <= '0; af_count <= '0; @@ -865,6 +896,7 @@ module foldsim_mem_bank #( lat_ar <= 1'b0; lat_r <= 1'b0; rfp_valid <= 1'b0; + sc_pend <= 1'b0; end end @@ -878,55 +910,55 @@ module foldsim_mem_bank #( always @(*) begin for (integer m = 0; m < NBM; ++m) begin for (integer k = 0; k < 2; ++k) begin - cm_v[m][k] = 1'b0; - cm_lo[m][k] = 16'sd0; - cm_hi[m][k] = 16'sd0; + cx_v[m][k] = 1'b0; + cx_lo[m][k] = 16'sd0; + cx_hi[m][k] = 16'sd0; end end // Row commands: PRE, ACT; the row bus for PRE, and ACT's one or two // clocks. Column commands, and their directions for the controller's // queues. - cm_v[B_CMDR][0] = op_conf; cm_lo[B_CMDR][0] = c_pre; cm_hi[B_CMDR][0] = c_pre; - cm_v[B_CMDR][1] = ~op_hit; cm_lo[B_CMDR][1] = c_act; cm_hi[B_CMDR][1] = c_act; - cm_v[B_ROW][0] = op_conf; cm_lo[B_ROW][0] = c_pre; cm_hi[B_ROW][0] = c_pre; - cm_v[B_ROW][1] = ~op_hit; cm_lo[B_ROW][1] = c_act; - cm_hi[B_ROW][1] = c_act + (g_act2 ? 16'sd1 : 16'sd0); - cm_v[B_CMDC][0] = 1'b1; cm_lo[B_CMDC][0] = c_col0; cm_hi[B_CMDC][0] = c_col0; - cm_v[B_CMDC][1] = (NB == 2); cm_lo[B_CMDC][1] = c_col1; cm_hi[B_CMDC][1] = c_col1; - cm_v[op_w ? B_KW : B_KR][0] = 1'b1; - cm_lo[op_w ? B_KW : B_KR][0] = c_col0; cm_hi[op_w ? B_KW : B_KR][0] = c_col0; - cm_v[op_w ? B_KW : B_KR][1] = (NB == 2); - cm_lo[op_w ? B_KW : B_KR][1] = c_col1; cm_hi[op_w ? B_KW : B_KR][1] = c_col1; + cx_v[B_CMDR][0] = op_conf; cx_lo[B_CMDR][0] = c_pre; cx_hi[B_CMDR][0] = c_pre; + cx_v[B_CMDR][1] = ~op_hit; cx_lo[B_CMDR][1] = c_act; cx_hi[B_CMDR][1] = c_act; + cx_v[B_ROW][0] = op_conf; cx_lo[B_ROW][0] = c_pre; cx_hi[B_ROW][0] = c_pre; + cx_v[B_ROW][1] = ~op_hit; cx_lo[B_ROW][1] = c_act; + cx_hi[B_ROW][1] = c_act + (g_act2 ? 16'sd1 : 16'sd0); + cx_v[B_CMDC][0] = 1'b1; cx_lo[B_CMDC][0] = c_col0; cx_hi[B_CMDC][0] = c_col0; + cx_v[B_CMDC][1] = (NB == 2); cx_lo[B_CMDC][1] = c_col1; cx_hi[B_CMDC][1] = c_col1; + cx_v[op_w ? B_KW : B_KR][0] = 1'b1; + cx_lo[op_w ? B_KW : B_KR][0] = c_col0; cx_hi[op_w ? B_KW : B_KR][0] = c_col0; + cx_v[op_w ? B_KW : B_KR][1] = (NB == 2); + cx_lo[op_w ? B_KW : B_KR][1] = c_col1; cx_hi[op_w ? B_KW : B_KR][1] = c_col1; // The read's departure. - cm_v[B_DEPV][0] = ~op_w; - cm_lo[B_DEPV][0] = ck_lastcol + k_dep; cm_hi[B_DEPV][0] = ck_lastcol + k_dep; + cx_v[B_DEPV][0] = ~op_w; + cx_lo[B_DEPV][0] = ck_lastcol + k_dep; cx_hi[B_DEPV][0] = ck_lastcol + k_dep; // What each column forbids to later columns of its pseudo-channel and // bank group, by direction. for (integer k = 0; k < NB; ++k) begin reg signed [15:0] c; c = (k == 0) ? c_col0 : c_col1; if (~op_w) begin - cm_v[B_FRP][k] = 1'b1; cm_lo[B_FRP][k] = clamp0(c - k_ccds + 16'sd1); cm_hi[B_FRP][k] = c + k_ccds - 16'sd1; - cm_v[B_FWP][k] = 1'b1; cm_lo[B_FWP][k] = clamp0(c - k_wtrs + 16'sd1); cm_hi[B_FWP][k] = c + k_rtw - 16'sd1; - cm_v[B_FRG][k] = 1'b1; cm_lo[B_FRG][k] = clamp0(c - k_ccdl + 16'sd1); cm_hi[B_FRG][k] = c + k_ccdl - 16'sd1; - cm_v[B_FWG][k] = 1'b1; cm_lo[B_FWG][k] = clamp0(c - k_wtrl + 16'sd1); cm_hi[B_FWG][k] = c; + cx_v[B_FRP][k] = 1'b1; cx_lo[B_FRP][k] = clamp0(c - k_ccds + 16'sd1); cx_hi[B_FRP][k] = c + k_ccds - 16'sd1; + cx_v[B_FWP][k] = 1'b1; cx_lo[B_FWP][k] = clamp0(c - k_wtrs + 16'sd1); cx_hi[B_FWP][k] = c + k_rtw - 16'sd1; + cx_v[B_FRG][k] = 1'b1; cx_lo[B_FRG][k] = clamp0(c - k_ccdl + 16'sd1); cx_hi[B_FRG][k] = c + k_ccdl - 16'sd1; + cx_v[B_FWG][k] = 1'b1; cx_lo[B_FWG][k] = clamp0(c - k_wtrl + 16'sd1); cx_hi[B_FWG][k] = c; end else begin - cm_v[B_FWP][k] = 1'b1; cm_lo[B_FWP][k] = clamp0(c - k_ccds + 16'sd1); cm_hi[B_FWP][k] = c + k_ccds - 16'sd1; - cm_v[B_FRP][k] = 1'b1; cm_lo[B_FRP][k] = clamp0(c - k_rtw + 16'sd1); cm_hi[B_FRP][k] = c + k_wtrs - 16'sd1; - cm_v[B_FWG][k] = 1'b1; cm_lo[B_FWG][k] = clamp0(c - k_ccdl + 16'sd1); cm_hi[B_FWG][k] = c + k_ccdl - 16'sd1; - cm_v[B_FRG][k] = 1'b1; cm_lo[B_FRG][k] = c; cm_hi[B_FRG][k] = c + k_wtrl - 16'sd1; + cx_v[B_FWP][k] = 1'b1; cx_lo[B_FWP][k] = clamp0(c - k_ccds + 16'sd1); cx_hi[B_FWP][k] = c + k_ccds - 16'sd1; + cx_v[B_FRP][k] = 1'b1; cx_lo[B_FRP][k] = clamp0(c - k_rtw + 16'sd1); cx_hi[B_FRP][k] = c + k_wtrs - 16'sd1; + cx_v[B_FWG][k] = 1'b1; cx_lo[B_FWG][k] = clamp0(c - k_ccdl + 16'sd1); cx_hi[B_FWG][k] = c + k_ccdl - 16'sd1; + cx_v[B_FRG][k] = 1'b1; cx_lo[B_FRG][k] = c; cx_hi[B_FRG][k] = c + k_wtrl - 16'sd1; end end // What the ACT forbids to later ACTs: tRRD within the pseudo-channel, // tRRD_L within the bank group. - cm_v[B_FAP][0] = ~op_hit; - cm_lo[B_FAP][0] = clamp0(c_act - k_rrds + 16'sd1); cm_hi[B_FAP][0] = c_act + k_rrds - 16'sd1; - cm_v[B_FAG][0] = ~op_hit; - cm_lo[B_FAG][0] = clamp0(c_act - k_rrdl + 16'sd1); cm_hi[B_FAG][0] = c_act + k_rrdl - 16'sd1; + cx_v[B_FAP][0] = ~op_hit; + cx_lo[B_FAP][0] = clamp0(c_act - k_rrds + 16'sd1); cx_hi[B_FAP][0] = c_act + k_rrds - 16'sd1; + cx_v[B_FAG][0] = ~op_hit; + cx_lo[B_FAG][0] = clamp0(c_act - k_rrdl + 16'sd1); cx_hi[B_FAG][0] = c_act + k_rrdl - 16'sd1; for (integer m = 0; m < NBM; ++m) for (integer k = 0; k < 2; ++k) - if (cm_hi[m][k] < cm_lo[m][k]) - cm_v[m][k] = 1'b0; + if (cx_hi[m][k] < cx_lo[m][k]) + cx_v[m][k] = 1'b0; end // A word a cycle per calendar; a range's last word starts the next. From 49d3389f4c99851d5a306304a476d6ecfc04a06d Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 15 Oct 2026 11:05:10 -0700 Subject: [PATCH 228/336] docs: Foldsim's staged endpoints and the bank model's host cost Both endpoints' token ends are staged; the bank model takes two host cycles a DRAM clock and about fifteen more a reservation, 7.3-9.5 host cycles per target cycle on s2w4i2. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0ef73fae4451ab3d9ec0ab465290db0d690438fb) --- docs/designs/foldsim.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 1f1272f4f9..ed3a357f2b 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -214,12 +214,18 @@ to later ones, so placing one is a first-free search over 64 clocks a host cycle; a word of the ring is cleared, by a valid bit, as the DRAM clock passes it, and a reservation reaches at most 448 clocks ahead (the model's horizon). A target cycle's DRAM clocks run in host cycles while the target -evaluates it — each is one host cycle, a reservation about ten — and the -cycle finishes only once they have; the `memory waits` counter includes -the host cycles a finished cycle waits for them. tFAW is left to tRRD, +evaluates it — two host cycles a clock, about fifteen more for a +reservation — and the cycle finishes only once they have; the `memory +waits` counter includes the host cycles a finished cycle waits for them. tFAW is left to tRRD, which covers it wherever four tRRD_S span a tFAW (HBM2, HBM3); the runtime refuses a memory where it does not (GDDR6). +Both endpoints' token ends are staged (`foldsim_token`'s `STAGED`): `have` +and `done` are registers, so an endpoint's finish starts at its own state, +not at the simulator's valid and ready. A token counts a host cycle later, +which s2w4i2's eight host cycles a target cycle hide (its counts are +unchanged). + **Reset.** The control endpoint also gives the memory endpoint a token each target cycle: the target's reset in that cycle. A reset command thereby resets the target's whole memory port, not only the target. A target cycle in @@ -378,7 +384,7 @@ SimX's own bank backend 10 % faster: the reservation model drains a write stream differently from Ramulator's per-clock scheduler (a mixed-traffic residual). The pipe, by contrast, has no memory to configure: sgemm takes 415,454 cycles at every profile. A bank-model run costs host -cycles the pipe does not (memory waits): 7.6–8.7 host cycles per target cycle +cycles the pipe does not (memory waits): 7.3–9.5 host cycles per target cycle against the pipe's 8.0. ### 7.2 tinygpu: Foldsim and the firesim backend From 9baaf7a5cbb973a28303cec7a0fc2c338fe5353a Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 16 Oct 2026 17:50:35 -0700 Subject: [PATCH 229/336] foldsim sim: simulation models of the Xilinx floating-point cores The in-process simulator (Verilator) cannot run the cores' encrypted netlists, so sim/foldsim/xilinx_fp_models.sv models each core (xil_fma/fdiv/fsqrt/fmul/fadd) with its ports, its C_Latency and its clock enable: the result of Vortex's DPI floating point on the inputs, delayed by that many enabled cycles. Numerics as PG060 documents them for these configurations: round to nearest even, denormal inputs taken as zero and denormal results flushed to zero with underflow, the cores' flags on tuser; NaN encoding and signaling-NaN flags are softfloat's, unconfirmed against the cores. Simulation only: sim/foldsim's Makefile builds it, with the DPI sources and softfloat, into libfoldsim_emul.so; no synthesis flow reads it. Verified: s2w4i2 with VX_fpu_dsp in Verilator, demo, sgemm and dogfood's FP tests (fadd, fmadd, fdiv, fdiv2, fsqrt, ftoi, itof, trig) pass. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 19a80325258c0b96939b001bea1b825cbd5ab49a) --- sim/foldsim/Makefile | 19 +++- sim/foldsim/xilinx_fp_models.sv | 194 ++++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 sim/foldsim/xilinx_fp_models.sv diff --git a/sim/foldsim/Makefile b/sim/foldsim/Makefile index 503f936e95..d1f82490d3 100644 --- a/sim/foldsim/Makefile +++ b/sim/foldsim/Makefile @@ -15,9 +15,13 @@ CONFIG_FILE := $(DESTDIR)/foldsim_emul_config.stamp CXXFLAGS += -std=c++17 -fPIC -O2 -DNDEBUG -I$(SRC_DIR) LDFLAGS += -shared -pthread +# The Xilinx floating-point cores' simulation models compute with Vortex's DPI +# floating point. +CXXFLAGS += -I$(SW_COMMON_DIR) -I$(THIRD_PARTY_DIR)/softfloat/source/include +LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a VL_FLAGS += -O3 -Wno-fatal -Wno-lint -Wno-style -Wno-MULTIDRIVEN --x-initial 0 --x-assign 0 -VL_FLAGS += -I$(FOLDSIM_RTL_DIR) -I$(WORK_DIR)/gen +VL_FLAGS += -I$(FOLDSIM_RTL_DIR) -I$(WORK_DIR)/gen -I$(DPI_DIR) ifdef VCD VL_FLAGS += --trace --trace-structs endif @@ -25,10 +29,13 @@ THREADS ?= $(shell python3 -c 'import multiprocessing as mp; print(min(32, mp.cp VL_FLAGS += -j $(THREADS) TOP := foldsim_afu -RTL_SRCS := $(GEN_SV) $(FOLDSIM_RTL_DIR)/foldsim_token.sv $(FOLDSIM_RTL_DIR)/foldsim_mem_bmap.sv \ - $(FOLDSIM_RTL_DIR)/foldsim_mem_bank.sv $(FOLDSIM_RTL_DIR)/foldsim_mem.sv \ - $(FOLDSIM_RTL_DIR)/foldsim_ctrl.sv $(FOLDSIM_RTL_DIR)/foldsim_afu.sv -SRCS := $(SRC_DIR)/foldsim_emul.cpp +# The simulation models of the Xilinx floating-point cores the simulator may +# instantiate (its FPU of type DSP): Verilator cannot run their netlists. +RTL_SRCS := $(SRC_DIR)/xilinx_fp_models.sv $(GEN_SV) $(FOLDSIM_RTL_DIR)/foldsim_token.sv \ + $(FOLDSIM_RTL_DIR)/foldsim_mem_bmap.sv $(FOLDSIM_RTL_DIR)/foldsim_mem_bank.sv \ + $(FOLDSIM_RTL_DIR)/foldsim_mem.sv $(FOLDSIM_RTL_DIR)/foldsim_ctrl.sv $(FOLDSIM_RTL_DIR)/foldsim_afu.sv +SRCS := $(SRC_DIR)/foldsim_emul.cpp $(DPI_DIR)/dpi_float.cpp $(DPI_DIR)/dpi_util.cpp \ + $(SW_COMMON_DIR)/softfloat_ext.cpp $(SW_COMMON_DIR)/rvfloats.cpp PROJECT := libfoldsim_emul.so @@ -45,7 +52,7 @@ $(GEN_SV): force $(if $(FOLDSIM_FLAGS),FOLDSIM_FLAGS='$(FOLDSIM_FLAGS)') \ $(if $(LIFT_PYTHON),LIFT_PYTHON=$(LIFT_PYTHON)) $(if $(FOLDSIM),FOLDSIM=$(FOLDSIM)) -$(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(wildcard $(FOLDSIM_RTL_DIR)/*) $(CONFIG_FILE) +$(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(SRC_DIR)/xilinx_fp_models.sv $(wildcard $(FOLDSIM_RTL_DIR)/*) $(CONFIG_FILE) $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(RTL_SRCS) --top-module $(TOP) \ $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ diff --git a/sim/foldsim/xilinx_fp_models.sv b/sim/foldsim/xilinx_fp_models.sv new file mode 100644 index 0000000000..096678222a --- /dev/null +++ b/sim/foldsim/xilinx_fp_models.sv @@ -0,0 +1,194 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Simulation models of the Xilinx floating-point cores (floating_point 7.1) +// that hw/scripts/xilinx_ip_gen.tcl generates, for Verilator, which cannot run +// their encrypted netlists. Simulation only: never synthesized, never read by +// the Foldsim compiler (which sees the black boxes of xilinx_ip_stubs.sv). +// +// Each has its core's ports, latency and clock enable: its result is computed +// from its inputs by Vortex's DPI floating point (softfloat) and delayed by +// the core's C_Latency cycles of aclk with aclken high, the pipeline its +// contract declares. The numerics follow what PG060 documents for these +// configurations: round to nearest even only; denormal inputs taken as zero +// and denormal results flushed to zero, keeping the sign, with underflow +// raised; the flags of each core on m_axis_result_tuser. What the documents +// leave open is softfloat's (RISC-V) choice here: a NaN result is the +// canonical quiet NaN 0x7fc00000, and invalid is raised for signaling NaN +// inputs as for the invalid operations. Bit-exactness to the cores can only be +// confirmed on silicon. + +`include "dpi_float.vh" + +package xilinx_fp_model_pkg; + // A denormal input, as the cores read it: zero of its sign. + function automatic logic [31:0] ftz_in(input logic [31:0] x); + return (x[30:23] == 8'h0) ? {x[31], 31'h0} : x; + endfunction + // A result as the cores give it: a denormal flushed to zero of its sign. + function automatic logic [31:0] ftz_out(input logic [31:0] x); + return (x[30:23] == 8'h0) ? {x[31], 31'h0} : x; + endfunction + function automatic logic is_denormal(input logic [31:0] x); + return (x[30:23] == 8'h0) && (x[22:0] != 23'h0); + endfunction + function automatic logic [63:0] box(input logic [31:0] x); + return {32'hffffffff, x}; + endfunction +endpackage + +// The cores' pipeline: `D` bits taken on aclk with aclken high, given LATENCY +// such cycles later. +module xilinx_fp_model_pipe #( + parameter LATENCY = 1, + parameter D = 1 +) ( + input wire aclk, + input wire aclken, + input wire [D-1:0] d, + output wire [D-1:0] q +); + logic [D-1:0] stage [LATENCY]; + always @(posedge aclk) begin + if (aclken) begin + stage[0] <= d; + for (int i = 1; i < LATENCY; ++i) + stage[i] <= stage[i - 1]; + end + end + assign q = stage[LATENCY - 1]; +endmodule + +module xil_fma ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + input wire s_axis_c_tvalid, + input wire [31:0] s_axis_c_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [2:0] m_axis_result_tuser // underflow, overflow, invalid +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + always @(*) begin + dpi_fmadd(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), + box(ftz_in(s_axis_c_tdata)), 3'b000, r, f); + end + wire tiny = is_denormal(r[31:0]); + // tuser = {invalid, overflow, underflow}; fflags = {NV, DZ, OF, UF, NX}. + xilinx_fp_model_pipe #(.LATENCY(16), .D(1 + 32 + 3)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid & s_axis_b_tvalid & s_axis_c_tvalid, ftz_out(r[31:0]), f[4], f[2], f[1] | tiny}), + .q({m_axis_result_tvalid, m_axis_result_tdata, m_axis_result_tuser})); +endmodule + +module xil_fdiv ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [3:0] m_axis_result_tuser // underflow, overflow, invalid, divide by zero +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + always @(*) begin + dpi_fdiv(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), 3'b000, r, f); + end + wire tiny = is_denormal(r[31:0]); + // tuser = {divide by zero, invalid, overflow, underflow}. + xilinx_fp_model_pipe #(.LATENCY(28), .D(1 + 32 + 4)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid & s_axis_b_tvalid, ftz_out(r[31:0]), f[3], f[4], f[2], f[1] | tiny}), + .q({m_axis_result_tvalid, m_axis_result_tdata, m_axis_result_tuser})); +endmodule + +module xil_fsqrt ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [0:0] m_axis_result_tuser // invalid +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + always @(*) begin + dpi_fsqrt(1'b1, 0, box(ftz_in(s_axis_a_tdata)), 3'b000, r, f); + end + xilinx_fp_model_pipe #(.LATENCY(28), .D(1 + 32 + 1)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid, ftz_out(r[31:0]), f[4]}), + .q({m_axis_result_tvalid, m_axis_result_tdata, m_axis_result_tuser})); +endmodule + +module xil_fmul ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + always @(*) begin + dpi_fmul(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), 3'b000, r, f); + end + xilinx_fp_model_pipe #(.LATENCY(8), .D(1 + 32)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid & s_axis_b_tvalid, ftz_out(r[31:0])}), + .q({m_axis_result_tvalid, m_axis_result_tdata})); +endmodule + +module xil_fadd ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + input wire s_axis_operation_tvalid, + input wire [7:0] s_axis_operation_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + // Operation bit 0: subtract. + always @(*) begin + if (s_axis_operation_tdata[0]) + dpi_fsub(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), 3'b000, r, f); + else + dpi_fadd(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), 3'b000, r, f); + end + xilinx_fp_model_pipe #(.LATENCY(11), .D(1 + 32)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid & s_axis_b_tvalid & s_axis_operation_tvalid, ftz_out(r[31:0])}), + .q({m_axis_result_tvalid, m_axis_result_tdata})); +endmodule From 629ba43ea9a1ca17c2628eaa199434d1dba2cfef Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 16 Oct 2026 17:50:35 -0700 Subject: [PATCH 230/336] xilinx IP: the floating-point cores' pipeline contracts, and their check Each black box in xilinx_ip_stubs.sv declares the contract the RTL uses it under, as Verilog attributes: fixed_latency, clock, clock_enable, data_in, data_out, tie (inputs driven with constants) and pure_pipeline (no other state). xilinx_ip_gen.tcl now states xil_fmul's and xil_fadd's C_Latency (8 and 11, the latency Vivado gives them by default, which VX_tcu_fedp_dsp assumes). check_ip_contracts.py checks each contract latency against C_Latency, the latency the RTL asserts or assumes, the core's simulation model and, with --cflags, the configuration's VX_CFG_*_LATENCY. Verified: Vivado 2025.2 resolves C_Latency 28/28/16/8/11 for xil_fdiv/fsqrt/fma/fmul/fadd as generated on the V80 part; the check passes, and fails on -DVX_CFG_FMA_LATENCY=12 with VIVADO; circt-verilog keeps the attributes on the stub modules. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b527755dbdfbdbc5616cc65ef7a0b2944a965afa) --- hw/scripts/check_ip_contracts.py | 135 +++++++++++++++++++++++++++++++ hw/scripts/xilinx_ip_gen.tcl | 4 +- hw/scripts/xilinx_ip_stubs.sv | 33 ++++++-- 3 files changed, 165 insertions(+), 7 deletions(-) create mode 100755 hw/scripts/check_ip_contracts.py diff --git a/hw/scripts/check_ip_contracts.py b/hw/scripts/check_ip_contracts.py new file mode 100755 index 0000000000..9218021c50 --- /dev/null +++ b/hw/scripts/check_ip_contracts.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check the pipeline contracts of the Xilinx floating-point cores. + +Each black box in xilinx_ip_stubs.sv declares the latency it is used at +(`fixed_latency`). That latency must be the core's C_Latency in +xilinx_ip_gen.tcl, the latency the RTL around it asserts, its simulation +model's (sim/foldsim/xilinx_fp_models.sv) and, with --cflags, the +VX_CFG_*_LATENCY the configuration resolves to for it. + +usage: check_ip_contracts.py [--cflags '<-D flags of the build>'] +""" +import argparse +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.realpath(__file__)) +ROOT = os.path.realpath(os.path.join(HERE, '..', '..')) + +# Where the RTL states the latency it uses each core at. +RTL_ASSERTS = { + 'xil_fma': ('hw/rtl/fpu/VX_fma_unit.sv', r'USE_VENDOR_IP\s*\|\|\s*\(LATENCY\s*==\s*(\d+)\)'), + 'xil_fdiv': ('hw/rtl/fpu/VX_fdiv_unit.sv', r'USE_VENDOR_IP\s*\|\|\s*\(LATENCY\s*==\s*(\d+)\)'), + 'xil_fsqrt': ('hw/rtl/fpu/VX_fsqrt_unit.sv', r'USE_VENDOR_IP\s*\|\|\s*\(LATENCY\s*==\s*(\d+)\)'), + 'xil_fmul': ('hw/rtl/tcu/dsp/VX_tcu_fedp_dsp.sv', r'localparam\s+FMUL_LATENCY\s*=\s*(\d+)'), + 'xil_fadd': ('hw/rtl/tcu/dsp/VX_tcu_fedp_dsp.sv', r'localparam\s+FADD_LATENCY\s*=\s*(\d+)'), +} + +# The configuration's latency of each core, when the configuration selects it. +CONFIG_LATENCIES = { + 'xil_fma': [('VX_CFG_FMA_LATENCY', 'VX_CFG_FPU_TYPE_DSP')], + 'xil_fdiv': [('VX_CFG_FDIV_LATENCY', 'VX_CFG_FPU_TYPE_DSP'), ('VX_CFG_RTU_FDIV_LATENCY', None)], + 'xil_fsqrt': [('VX_CFG_FSQRT_LATENCY', 'VX_CFG_FPU_TYPE_DSP')], +} + + +def stub_contracts(path): + """module -> its attributes, from the `(* ... *)` before each module.""" + text = open(path).read() + contracts = {} + for attrs, name in re.findall(r'\(\*(.*?)\*\)\s*module\s+(\w+)', text, re.S): + values = {} + for item in re.findall(r'(\w+)(?:\s*=\s*("[^"]*"|\w+))?', attrs): + values[item[0]] = item[1].strip('"') if item[1] else '' + contracts[name] = values + return contracts + + +def tcl_latencies(path): + """module -> C_Latency, from each core's set_property dictionary.""" + text = open(path).read() + latencies = {} + for body, name in re.findall(r'set_property -dict \[list (.*?)\] \[get_ips (\w+)\]', text, re.S): + m = re.search(r'CONFIG\.C_Latency \{(\d+)\}', body) + latencies[name] = int(m.group(1)) if m else None + return latencies + + +def model_latencies(path): + """module -> the latency of its simulation model's pipeline.""" + text = open(path).read() + return {name: int(lat) for name, lat in + re.findall(r'module (\w+) \(.*?xilinx_fp_model_pipe #\(\.LATENCY\((\d+)\)', text, re.S)} + + +def config_values(cflags): + out = subprocess.run([sys.executable, os.path.join(ROOT, 'ci', 'gen_config.py'), + '--config', os.path.join(ROOT, 'VX_config.toml'), '--cflags', cflags], + check=True, capture_output=True, text=True).stdout + values = {} + for flag in out.split(): + if flag.startswith('-D'): + name, _, value = flag[2:].partition('=') + values[name] = value + return values + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + ap.add_argument('--cflags', help="the build's -D flags, to check its resolved latencies") + a = ap.parse_args() + errors = [] + contracts = stub_contracts(os.path.join(HERE, 'xilinx_ip_stubs.sv')) + tcl = tcl_latencies(os.path.join(HERE, 'xilinx_ip_gen.tcl')) + config = config_values(a.cflags) if a.cflags is not None else None + models = model_latencies(os.path.join(ROOT, 'sim', 'foldsim', 'xilinx_fp_models.sv')) + for name, rtl in RTL_ASSERTS.items(): + c = contracts.get(name) + if c is None or 'fixed_latency' not in c: + errors.append(f'{name}: no fixed_latency contract in xilinx_ip_stubs.sv') + continue + latency = int(c['fixed_latency']) + if tcl.get(name) != latency: + errors.append(f'{name}: contract latency {latency}, C_Latency {tcl.get(name)} in xilinx_ip_gen.tcl') + if models.get(name) != latency: + errors.append(f'{name}: contract latency {latency}, sim/foldsim/xilinx_fp_models.sv models {models.get(name)}') + path, pattern = rtl + m = re.search(pattern, open(os.path.join(ROOT, path)).read()) + if not m: + errors.append(f'{name}: {path} no longer states the latency it uses the core at') + elif int(m.group(1)) != latency: + errors.append(f'{name}: contract latency {latency}, {path} uses {m.group(1)}') + if config is None: + continue + for knob, selector in CONFIG_LATENCIES.get(name, []): + if selector and selector not in config: + continue + # The RTU uses the vendor divider on the Vivado flow only. + if knob == 'VX_CFG_RTU_FDIV_LATENCY' and not re.search(r'-DVIVADO\b', a.cflags): + continue + if knob in config and int(config[knob]) != latency: + errors.append(f'{name}: contract latency {latency}, the configuration resolves {knob}={config[knob]}') + for e in errors: + print(f'ERROR: {e}', file=sys.stderr) + if errors: + sys.exit(1) + print(f'xilinx IP contracts: {", ".join(f"{n} {contracts[n]["fixed_latency"]}" for n in RTL_ASSERTS)} consistent') + + +if __name__ == '__main__': + main() diff --git a/hw/scripts/xilinx_ip_gen.tcl b/hw/scripts/xilinx_ip_gen.tcl index 16bb3541a4..119cd8fec3 100644 --- a/hw/scripts/xilinx_ip_gen.tcl +++ b/hw/scripts/xilinx_ip_gen.tcl @@ -41,10 +41,10 @@ create_ip -name floating_point -vendor xilinx.com -library ip -version 7.1 -modu set_property -dict [list CONFIG.Component_Name {xil_fma} CONFIG.Operation_Type {FMA} CONFIG.Add_Sub_Value {Add} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Has_UNDERFLOW {true} CONFIG.C_Has_OVERFLOW {true} CONFIG.C_Has_INVALID_OP {true} CONFIG.Has_A_TUSER {false} CONFIG.A_Precision_Type {Single} CONFIG.C_A_Exponent_Width {8} CONFIG.C_A_Fraction_Width {24} CONFIG.Result_Precision_Type {Single} CONFIG.C_Result_Exponent_Width {8} CONFIG.C_Result_Fraction_Width {24} CONFIG.C_Mult_Usage {Medium_Usage} CONFIG.Has_RESULT_TREADY {false} CONFIG.C_Latency {16} CONFIG.C_Rate {1} CONFIG.A_TUSER_Width {1}] [get_ips xil_fma] create_ip -name floating_point -vendor xilinx.com -library ip -version 7.1 -module_name xil_fmul -dir ${ip_dir} -set_property -dict [list CONFIG.Operation_Type {Multiply} CONFIG.A_Precision_Type {Single} CONFIG.Result_Precision_Type {Single} CONFIG.Has_RESULT_TREADY {false} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Rate {1} CONFIG.C_Mult_Usage {Full_Usage}] [get_ips xil_fmul] +set_property -dict [list CONFIG.Operation_Type {Multiply} CONFIG.A_Precision_Type {Single} CONFIG.Result_Precision_Type {Single} CONFIG.Has_RESULT_TREADY {false} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Rate {1} CONFIG.C_Mult_Usage {Full_Usage} CONFIG.C_Latency {8}] [get_ips xil_fmul] create_ip -name floating_point -vendor xilinx.com -library ip -version 7.1 -module_name xil_fadd -dir ${ip_dir} -set_property -dict [list CONFIG.Operation_Type {Add_Subtract} CONFIG.A_Precision_Type {Single} CONFIG.Result_Precision_Type {Single} CONFIG.Has_RESULT_TREADY {false} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Rate {1} CONFIG.C_Mult_Usage {Full_Usage}] [get_ips xil_fadd] +set_property -dict [list CONFIG.Operation_Type {Add_Subtract} CONFIG.A_Precision_Type {Single} CONFIG.Result_Precision_Type {Single} CONFIG.Has_RESULT_TREADY {false} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Rate {1} CONFIG.C_Mult_Usage {Full_Usage} CONFIG.C_Latency {11}] [get_ips xil_fadd] generate_target all [get_ips] diff --git a/hw/scripts/xilinx_ip_stubs.sv b/hw/scripts/xilinx_ip_stubs.sv index f1ecb14186..4b5860da1a 100644 --- a/hw/scripts/xilinx_ip_stubs.sv +++ b/hw/scripts/xilinx_ip_stubs.sv @@ -16,8 +16,19 @@ // The CIRCT elaboration reads them to know each core's interface and keeps // every instance as an instance; Vivado binds it to the generated core. // Synthesis never reads this file. Keep it in step with xilinx_ip_gen.tcl. +// +// Each declares the contract the RTL uses it under: a fixed-latency +// pipeline. Its data_out outputs are a function of its data_in inputs of +// exactly fixed_latency cycles of `clock` with `clock_enable` high before, +// and of nothing else (pure_pipeline: non-blocking flow control, no reset, no other state); the `tie` +// inputs are driven with those constants, its other outputs are unused. +// fixed_latency is the core's C_Latency in xilinx_ip_gen.tcl and the latency +// the RTL asserts (hw/scripts/check_ip_contracts.py checks both). -(* black_box *) +(* black_box, fixed_latency = 28, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata s_axis_b_tdata", + data_out = "m_axis_result_tdata m_axis_result_tuser", + tie = "s_axis_a_tvalid=1 s_axis_b_tvalid=1", pure_pipeline *) module xil_fdiv ( input wire aclk, input wire aclken, @@ -31,7 +42,10 @@ module xil_fdiv ( ); endmodule -(* black_box *) +(* black_box, fixed_latency = 28, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata", + data_out = "m_axis_result_tdata m_axis_result_tuser", + tie = "s_axis_a_tvalid=1", pure_pipeline *) module xil_fsqrt ( input wire aclk, input wire aclken, @@ -43,7 +57,10 @@ module xil_fsqrt ( ); endmodule -(* black_box *) +(* black_box, fixed_latency = 16, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata s_axis_b_tdata s_axis_c_tdata", + data_out = "m_axis_result_tdata m_axis_result_tuser", + tie = "s_axis_a_tvalid=1 s_axis_b_tvalid=1 s_axis_c_tvalid=1", pure_pipeline *) module xil_fma ( input wire aclk, input wire aclken, @@ -59,7 +76,10 @@ module xil_fma ( ); endmodule -(* black_box *) +(* black_box, fixed_latency = 8, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata s_axis_b_tdata", + data_out = "m_axis_result_tdata", + tie = "s_axis_a_tvalid=1 s_axis_b_tvalid=1", pure_pipeline *) module xil_fmul ( input wire aclk, input wire aclken, @@ -72,7 +92,10 @@ module xil_fmul ( ); endmodule -(* black_box *) +(* black_box, fixed_latency = 11, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata s_axis_b_tdata s_axis_operation_tdata", + data_out = "m_axis_result_tdata", + tie = "s_axis_a_tvalid=1 s_axis_b_tvalid=1 s_axis_operation_tvalid=1", pure_pipeline *) module xil_fadd ( input wire aclk, input wire aclken, From f6c9b78c7985f7804846203c4791609fd62b0196 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 16 Oct 2026 17:50:35 -0700 Subject: [PATCH 231/336] foldsim flow: the configuration's FPU, with the Xilinx cores folded The flow no longer forces the soft FPU (VX_CFG_FPU_TYPE_STD, the RTU's soft divider latency) nor rewrites the units' VENDOR_OK: on Vivado the configuration's DSP FPU selects the Xilinx floating-point cores, as in the native flows. The sources take xilinx_ip_stubs.sv, the cores' black boxes with their contracts, after check_ip_contracts.py checks them against the build's flags; foldsim runs with --fold-pipelines, which simulates each core by an unthreaded copy and a per-thread store of its results; the kernel is packaged with the cores when the simulator instantiates them, generated as the aved flow generates them. -DVX_CFG_FPU_TYPE_STD in CONFIGS selects the soft FPU. Verified: s2w4i2 generates with VX_fpu_dsp a class, its 12 cores folded; in Verilator, the threaded simulator (CPRT's options + fold-pipelines) and the unthreaded one give the same instructions and kernel cycles on demo, sgemm and dogfood's FP tests. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5c869f714d7f7b3bbffef169c225a1a93915aca2) --- docs/designs/foldsim.md | 15 ++++++++---- hw/syn/foldsim/Makefile | 40 +++++++++++++++---------------- hw/syn/xilinx/aved/package_ip.tcl | 2 +- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index ed3a357f2b..5242119593 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -60,10 +60,17 @@ scheduler's `ready` on it, the slice's `ready` on that): fused, tinygpu runs at 3.00 host cycles per target cycle instead of 4.00 (V80), with the same threads and copies and fewer LUTs. -The flow reads Vortex as source, so it builds the configuration whose units -are source: the STD FPU (`-DVX_CFG_FPU_TYPE_STD`), and the RTU's soft FMAs and -dividers, which the flow's copy of the floating-point units selects in place of -the Xilinx cores. A RAM that asks for block RAM but reads asynchronously reads +The flow builds the configuration's FPU, as the native Vivado flows do: of +type DSP, the Xilinx floating-point cores. Their netlists are encrypted, so +their state cannot be threaded; each core declares a pipeline contract +(`hw/scripts/xilinx_ip_stubs.sv`: latency, clock enable, data ports, no other +state), and the compiler simulates each by one unthreaded copy of the core and +a per-thread store of its results (`foldsim --fold-pipelines`, see the +compiler's `foldsim-find-pipelines`). `hw/scripts/check_ip_contracts.py` checks +each contract's latency against the core's generation, the RTL and the +configuration; the in-process simulator runs the cores' models +(`sim/foldsim/xilinx_fp_models.sv`). `-DVX_CFG_FPU_TYPE_STD` in `CONFIGS` +selects the soft FPU. A RAM that asks for block RAM but reads asynchronously reads synchronously once the target is flattened (CIRCT `seq-retime-firmem-reads`, see the hardware library's "Block RAM and asynchronous reads"); `gen/memories.rpt` lists each such RAM and the decision. diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index 72422448ed..0252c78de0 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -54,7 +54,7 @@ LIFT_PYTHON ?= python3 # The kernel clock (MHz): the image is timed at it and the shell programs it. FOLDSIM_FREQ ?= 250 # Options of the foldsim driver, e.g. --region-loads=2048 --registered-state. -FOLDSIM_FLAGS ?= +FOLDSIM_FLAGS ?= --fold-pipelines # The target: Vortex with its AXI4 port, DCR, start and busy as plain ports. TARGET_TOP := foldsim_target @@ -66,13 +66,14 @@ RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(FOLDSIM_RTL_DIR)/target SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG # One memory port, as the memory endpoint models; 4 AXI id bits, what Vortex -# drives with its tag buffer engaged. The FPU and RTU use Vortex's own -# datapaths (the STD FPU, the RTU's soft cores with their DSP multiplies): a -# vendor netlist is not source the flow can read, nor state it can thread. The -# staged sources therefore never select the floating-point cores (see -# VENDOR_FP_UNITS), and the RTU divider has the soft one's latency. -PLATFORM_DEFINES := -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DPLATFORM_MEMORY_ID_WIDTH=4 \ - -DVX_CFG_RTU_FDIV_LATENCY=17 -DVX_CFG_FPU_TYPE_STD +# drives with its tag buffer engaged. The FPU type is the configuration's, as +# in the native Vivado flows: DSP selects the Xilinx floating-point cores, which +# the simulator keeps as instances: their netlists are encrypted, so their +# state cannot be threaded, and each is simulated by one unthreaded copy of +# itself and a per-thread store of its results (foldsim --fold-pipelines, by +# the pipeline contract xilinx_ip_stubs.sv declares). -DVX_CFG_FPU_TYPE_STD in +# CONFIGS selects the soft FPU instead. +PLATFORM_DEFINES := -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DPLATFORM_MEMORY_ID_WIDTH=4 ifdef PERF PLATFORM_DEFINES += -DPERF_ENABLE endif @@ -120,22 +121,18 @@ SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv) $(wildcard $(FO all: $(VBIN) -# The units that would wrap a Xilinx floating-point core take their soft path, -# the one they take wherever no vendor core exists. -VENDOR_FP_UNITS := VX_fma_unit.sv VX_fdiv_unit.sv VX_fsqrt_unit.sv +# The Xilinx floating-point cores, declared as black boxes with their pipeline +# contracts; their contracts agree with the cores' generation and the RTL. +IP_STUBS := $(SCRIPT_DIR)/xilinx_ip_stubs.sv # The list is written to a temporary and moved into place last, so a failing # step leaves no list that looks current. sources: $(SRCS_LIST) -$(SRCS_LIST): $(RTL_SRCS) +$(SRCS_LIST): $(RTL_SRCS) $(IP_STUBS) $(SCRIPT_DIR)/xilinx_ip_gen.tcl rm -rf $(SRCS_DIR) $@ $@.tmp + python3 $(SCRIPT_DIR)/check_ip_contracts.py --cflags '$(GEN_CFLAGS)' cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) -T$(TARGET_TOP) -C$(SRCS_NAME) \ -O$(SRCS_NAME).f.tmp $(RTL_PKGS) - @for f in $(VENDOR_FP_UNITS); do \ - [ -f $(SRCS_DIR)/$$f ] || continue; \ - grep -q 'localparam VENDOR_OK = 1;' $(SRCS_DIR)/$$f || \ - { echo "ERROR: $$f no longer selects its vendor core by VENDOR_OK"; exit 1; }; \ - sed -i 's/localparam VENDOR_OK = 1;/localparam VENDOR_OK = 0;/' $(SRCS_DIR)/$$f; \ - done + cp $(IP_STUBS) $(SRCS_DIR)/ && echo "$(SRCS_DIR)/$(notdir $(IP_STUBS))" >> $@.tmp @for f in $(SHELL_MODULES); do \ rm -f $(SRCS_DIR)/$$f; \ grep -v "/$$f$$" $@.tmp > $@.tmp2 && mv $@.tmp2 $@.tmp; \ @@ -175,10 +172,13 @@ $(OOC_XDC): FORCE @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ package-ip: $(IP_DIR)/component.xml -$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(OOC_XDC) $(wildcard $(FOLDSIM_RTL_DIR)/*) +# The kernel is packaged with the floating-point cores the simulator +# instantiates, generated as the aved flow generates them. +$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(OOC_XDC) $(wildcard $(FOLDSIM_RTL_DIR)/*) $(SCRIPT_DIR)/xilinx_ip_gen.tcl cd $(BUILD_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -nojournal -nolog \ -source $(VORTEX_HOME)/hw/syn/xilinx/aved/package_ip.tcl \ - -tclargs foldsim_afu $(KERNEL_SRCS) . $(DEVICE) no-vendor-ip ooc-xdc=$(OOC_XDC) + -tclargs foldsim_afu $(KERNEL_SRCS) . $(DEVICE) ooc-xdc=$(OOC_XDC) \ + $$(grep -qE '^ *xil_f(ma|div|sqrt|mul|add) ' $(GEN_SV) || echo no-vendor-ip) $(CFG): FORCE @python3 $(SCRIPT_DIR)/gen_platform.py --flow=aved --device=$(DEVICE) --emit=cfg --kernel=foldsim_afu \ diff --git a/hw/syn/xilinx/aved/package_ip.tcl b/hw/syn/xilinx/aved/package_ip.tcl index 5c1749a974..ada3047e57 100644 --- a/hw/syn/xilinx/aved/package_ip.tcl +++ b/hw/syn/xilinx/aved/package_ip.tcl @@ -25,7 +25,7 @@ set vcs_file [lindex $::argv 1] set build_dir [lindex $::argv 2] set part [lindex $::argv 3] # A kernel whose RTL instantiates no vendor floating-point core (a foldsim -# simulator, whose FPU is soft logic) is packaged without them. +# simulator of a soft FPU) is packaged without them. set options [lrange $::argv 4 end] set vendor_ip [expr {[lsearch -exact $options no-vendor-ip] < 0}] # The kernel's clock for its out-of-context synthesis, so that synthesis From 4242c7f5e51f6974aa84188aca714742e8e34774 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sat, 17 Oct 2026 23:23:53 -0700 Subject: [PATCH 232/336] foldsim: a memory endpoint per memory port, and the bank model's refresh Several memory ports. The kernel has one memory endpoint and one device-memory master per memory port of the target, which has as many as its configuration's platform memory banks (VX_CFG_PLATFORM_MEMORY_NUM_BANKS in CONFIGS, 1 when unset: 1, 2, 4 or 8). Each port is a DRAM channel with its own bank model, as dram_sim gives each bank its own channel; the model sees the block within the port's channel. The flow writes the port count, the channel shift and the AXI id width (4 + log2 ports: Vortex's AXI adapter puts the GPU memory port above its tag buffer's index) into foldsim_kernel.vh; the linker config binds every master to MEM. The control endpoint gives each endpoint the target's reset and reads each port's counters at 0xC00 + 0x40p; the runtime prints them per channel; the emulator models one HBM master per port. Refresh. The bank model (sim/common/dram_bank_model.h, and foldsim_mem_bank decision for decision) refreshes per bank or all banks at once when VORTEX_DRAM_REFRESH asks for it, in the DramSim backend, rtlsim's AXI port and Foldsim alike: when due, a refresh takes a DRAM clock's turn to reserve, precharges and refreshes its bank (every bank), which may not activate for tRFCsb (tRFC). HBM's all-bank tRFC joins the timing tables (Ramulator's value for the 8 Gb channel; dram_sim's results are unchanged). Counters add refreshes and the target cycles a port could take no read. Register map version 4. Validated: the endpoint co-simulation, with five refresh setups and as one of two ports, 0 differences; s2w4i2 with two ports in Foldsim equals rtlsim behind the AXI4 adapter (plus its 8-cycle launch settle) on demo and sgemm, pipe and bank model; one port unchanged; refresh one per interval, and all-bank refresh costs a stream at least tRFC/tREFI. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 79018d0d2bdaa7a833e6197a6df923e037d3c3ba) --- docs/designs/foldsim.md | 82 ++++- hw/rtl/afu/foldsim/foldsim_afu.sv | 370 +++++++++++--------- hw/rtl/afu/foldsim/foldsim_ctrl.sv | 48 ++- hw/rtl/afu/foldsim/foldsim_mem.sv | 33 +- hw/rtl/afu/foldsim/foldsim_mem_bank.sv | 138 +++++++- hw/rtl/afu/foldsim/foldsim_ports.vh | 34 ++ hw/rtl/afu/foldsim/foldsim_regs.vh | 7 +- hw/rtl/afu/foldsim/target/foldsim_target.sv | 9 +- hw/scripts/gen_platform.py | 12 +- hw/syn/foldsim/Makefile | 43 ++- hw/unittest/Makefile | 3 + hw/unittest/foldsim_mem/main.cpp | 41 ++- hw/unittest/foldsim_mem_ch2/Makefile | 28 ++ sim/common/dram_bank_model.h | 103 +++++- sim/common/dram_sim.cpp | 9 +- sim/foldsim/Makefile | 5 +- sim/foldsim/foldsim_emul.cpp | 206 +++++++---- sim/rtlsim/processor.cpp | 5 +- sw/common/dram_timing.h | 28 +- sw/common/foldsim_regs.h | 17 +- sw/runtime/foldsim/vortex.cpp | 87 ++++- tests/unittest/dram_bank_model/main.cpp | 67 +++- 22 files changed, 1039 insertions(+), 336 deletions(-) create mode 100644 hw/rtl/afu/foldsim/foldsim_ports.vh create mode 100644 hw/unittest/foldsim_mem_ch2/Makefile diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 5242119593..926bab6b45 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -151,12 +151,19 @@ directly, since the shell's valid and ready depend on its own state alone. ## 4. The kernel `hw/rtl/afu/foldsim/foldsim_afu.sv` has the native V80 kernel's ports but for -the host-memory master: `s_axi_control` and `m_axi_mem_0` (linked to the -`MEM` HBM aperture). It instantiates `foldsim_sim` and two endpoints, both -models of the target's surroundings that take one token per channel per -target cycle (`foldsim_token.sv`): - -- **`foldsim_mem`** is the target's AXI4 port, timed by one of two models +the host-memory master: `s_axi_control` and one device-memory master +`m_axi_mem_

` per memory port of the target, each linked to the `MEM` HBM +aperture (the linker gives each its own way in). The target has as many +memory ports as its configuration has platform memory banks +(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS` in `CONFIGS`: 1, 2, 4 or 8; one when +unset); the flow writes their number, the shift to a port's channel and their +AXI id width into `gen/foldsim_kernel.vh`, and the kernel lists its ports with +`FOLDSIM_REPEAT` (`foldsim_ports.vh`). It instantiates `foldsim_sim`, a +memory endpoint per port and the control endpoint, all models of the target's +surroundings that take one token per channel per target cycle +(`foldsim_token.sv`): + +- **`foldsim_mem`** is one of the target's AXI4 ports, timed by one of two models (§4.1): a latency-bandwidth pipe — cycle for cycle `sim/common/axi_latency_pipe.h`: a fixed latency per transaction, bounded reads and writes in flight, one beat per cycle, responses in request @@ -173,6 +180,11 @@ target cycle (`foldsim_token.sv`): applied for one target cycle (a reset for its count). Kernel reset holds the target in reset for 64 cycles. +With several ports, Vortex's AXI adapter puts the index of the GPU memory +port a read came from above its tag buffer's 4 index bits, so the target's +AXI ids are 4 + log2(ports) bits wide; the flow sets +`PLATFORM_MEMORY_ID_WIDTH` and the endpoints' id width to match. + ### 4.1 The memory models The kernel's own reset gives the pipe; a reset command chooses the model the @@ -212,6 +224,20 @@ AXI_MEM_MODEL=bank`). cycle whose DRAM clocks its data departs in (tCL + tBL after its last column); R and B take the oldest ready response, so reads complete out of order — legal, since Vortex never has two reads of one id outstanding. +- *Channels.* Each memory port is a channel of its own, with its own endpoint + and engine, as dram_sim gives each memory bank its own channel. Ports + interleave the target's memory by block, so a port's channel sees the + block index without the port's bits (`PLATFORM_MEMORY_INTERLEAVE`); device + memory takes the target's address on every port. +- *Refresh* (off unless `VORTEX_DRAM_REFRESH` asks for it; dram_sim models + none). Every interval a refresh takes the DRAM clock's turn to reserve. + Per bank (HBM REFsb, GDDR6 REFpb; the banks in turn, an interval of + tREFI / banks): the bank is precharged if open, once its reserved commands + allow, then refreshed once the precharge and its last activate allow, and + may not activate for tRFCsb. All banks (an interval of tREFI): one + precharge after every open bank's commands, one REF, and no bank activates + for tRFC. Both commands take the row-command bus as a precharge does; a + refresh that would reach past the horizon waits, and the requests with it. In hardware the calendars are bitmaps over a ring of 512 DRAM clocks in distributed RAM: per clock, whether it holds a command, the row bus, and per @@ -220,7 +246,10 @@ take. Every command marks, once reserved, the clocks its constraints forbid to later ones, so placing one is a first-free search over 64 clocks a host cycle; a word of the ring is cleared, by a valid bit, as the DRAM clock passes it, and a reservation reaches at most 448 clocks ahead (the model's -horizon). A target cycle's DRAM clocks run in host cycles while the target +horizon). A refresh goes through the same search and commit as a request's +precharge; for all-bank refresh the channel keeps the latest time a +precharge and an activate are allowed over the banks opened since the last +one, so it never reads the whole bank table. A target cycle's DRAM clocks run in host cycles while the target evaluates it — two host cycles a clock, about fifteen more for a reservation — and the cycle finishes only once they have; the `memory waits` counter includes the host cycles a finished cycle waits for them. tFAW is left to tRRD, @@ -271,11 +300,11 @@ for the RTL): | `0x20`–`0x2C` | target cycles, kernel-clock cycles | | `0x30`, `0x38` | kernel-clock cycles a target cycle waited for HBM; the number of classes | | `0xC0`–`0xD4` | memory model: device base, the pipe's read and write latency, most reads and writes in flight | -| `0xD8`, `0xDC` | the memory model in effect (read only); the bank model's capacities (read only) | -| `0xE0`–`0xF0` | bank model: data rate and core clock, fixed latency, requests in flight and controller queues, geometry, write watermarks | +| `0xD8`, `0xDC` | the memory model and refresh in effect (read only); the bank model's capacities: ports, engines and masters, slots, horizon (read only) | +| `0xE0`–`0xF8` | bank model: data rate and core clock, fixed latency, requests in flight and controller queues, geometry, write watermarks, refresh interval and busy time | | `0x100 + 8c` | per class, threads finished | | `0x800`–`0x844` | bank model: DRAM timings, a word each (tBL … tWTR_L, read-to-write extra) | -| `0xC00`–`0xC2C` | bank model: reads, writes, row hits, misses, conflicts, summed read latency (64-bit) | +| `0xC00 + 0x40p` | memory port p's bank model: reads, writes, row hits, misses, conflicts, summed read latency, refreshes, target cycles it could take no read (64-bit each) | `0x40`–`0xBF` is empty: on the V80 the PCIe block keeps its MSI-X table at that offset of the BAR the first kernel is mapped at, and answers those @@ -289,10 +318,13 @@ for. The runtime resets the target on open, as a card's native driver resets its device; that also clears a memory error an earlier process left, and the reset -command's address chooses the memory model. With `FOLDSIM_MEM_MODEL=bank` the -runtime first writes the bank model's registers from the platform memory it -was built for (its `CONFIGS`), refusing what the image cannot hold, and -`--perf`-style counters print at close; otherwise it makes exactly the +command's address chooses the memory model and, with +`VORTEX_DRAM_REFRESH=bank|all`, the bank model's refresh. With +`FOLDSIM_MEM_MODEL=bank` the runtime first writes the bank model's registers +from the platform memory it was built for (its `CONFIGS`), every channel +alike, refusing what the image cannot hold and warning when a refresh keeps +a bank past the image's horizon; the counters print at close, per channel +and in all; otherwise it makes exactly the register accesses it always has. The command processor model's hooks return no status, so the runtime latches a device error instead — a memory error, a failed register or memory access, or a wait @@ -319,10 +351,17 @@ advancing for that time (60 s when it is unset). device-side protocol, resets at random points with HBM busy, and each kind of burst; every target cycle's outputs compared with the C++ model's (`AxiLatencyPipe`, `DramBankAxi`), every response, id and read's data - checked, none from before a reset delivered after it. Two pipe settings and + checked, none from before a reset delivered after it. Two pipe settings, twelve bank-model settings (HBM2 and HBM3 at the default, v100, a100 and h100 rates; small queues; row hits, random rows, one bank to the horizon, - streams, read- and write-heavy): 0 differences. + streams, read- and write-heavy) and five with refresh (per bank and all + banks, at the standard intervals and faster), on the endpoint alone, with + staged token ends, and as one of two ports (`foldsim_mem_ch2`): 0 + differences. +- **Refresh against its interval** (`tests/unittest/dram_bank_model + refresh`): one refresh per interval, to the operation, at every memory and + pattern; all-bank refresh costs a stream at least tRFC / tREFI of its + bandwidth (9.3–12 %, against 9.0 %). - **Applications** in the in-process simulator and on the V80, against rtlsim behind the AXI4 adapter on `AxiLatencyPipe` (`AXI_BUS=1`) with the same memory settings: retired instructions and cycles identical (§7). @@ -437,7 +476,16 @@ folds (s2w4i2 above). them; a burst sets the status register's memory error, stops the endpoint, and only a reset clears it. Under the bank model, no two reads of one id may be outstanding (Vortex's AXI adapter never has them). -4. **One memory port**: the target is built with one platform memory bank. +4. **Memory ports**: 1, 2, 4 or 8, one per platform memory bank, each a + channel with its own engine (about 11k LUTs). A refresh that keeps its + bank longer than the horizon (per-bank refresh at HBM3's clock: tRFCsb is + 524 clocks against 448) holds the requests behind one to that bank at the + channel's ingress, which overstates refresh's cost; the runtime warns. + With refresh on, a run's cycles depend on where the refresh schedule + stands at launch, so on the host's timing before it: such runs equal + another simulator's only from a common reset (the endpoint + co-simulation), not across hosts. With refresh off they do not depend on + host timing. 5. **Classes are named, not found**: `FOLDSIM_CLASSES` lists them. 6. **No kernel register in `0x40`–`0xBF`.** The V80 shell configures the PCIe block's MSI-X table internally, at offset `0x40` of the BAR whose diff --git a/hw/rtl/afu/foldsim/foldsim_afu.sv b/hw/rtl/afu/foldsim/foldsim_afu.sv index b7b2cad3cd..a1222f9bb3 100644 --- a/hw/rtl/afu/foldsim/foldsim_afu.sv +++ b/hw/rtl/afu/foldsim/foldsim_afu.sv @@ -12,19 +12,24 @@ // limitations under the License. `include "foldsim_sim.vh" +`include "foldsim_kernel.vh" +`include "foldsim_ports.vh" // The foldsim kernel: the simulator CIRCT composes of Vortex (foldsim_sim, of -// the target foldsim_target), the memory endpoint on its AXI4 port and the -// control endpoint on its DCR, start and busy signals. Its ports are the -// native V80 kernel's, but for the host-memory master it does not need. +// the target foldsim_target), a memory endpoint on each of its AXI4 ports and +// the control endpoint on its DCR, start and busy signals. Its ports are the +// native V80 kernel's, but for the host-memory master it does not need: one +// device-memory master per memory port of the target (FOLDSIM_MEM_PORTS, from +// foldsim_kernel.vh, with the ports' id width), each its own way into device +// memory. module foldsim_afu #( parameter C_S_AXI_CONTROL_ADDR_WIDTH = 16, parameter C_S_AXI_CONTROL_DATA_WIDTH = 32, parameter C_M_AXI_MEM_ID_WIDTH = 1, parameter C_M_AXI_MEM_DATA_WIDTH = 512, parameter C_M_AXI_MEM_ADDR_WIDTH = 64, - // The target's AXI4 port. - parameter TARGET_ID_W = 4, + // The target's AXI4 ports. + parameter TARGET_ID_W = `FOLDSIM_MEM_ID_W, parameter TARGET_ADDR_W = 64, parameter DCR_ADDR_W = 12, parameter DCR_DATA_W = 32 @@ -32,31 +37,34 @@ module foldsim_afu #( input wire ap_clk, input wire ap_rst_n, - output wire m_axi_mem_0_awvalid, - input wire m_axi_mem_0_awready, - output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_0_awaddr, - output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_0_awid, - output wire [7:0] m_axi_mem_0_awlen, - output wire m_axi_mem_0_wvalid, - input wire m_axi_mem_0_wready, - output wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_0_wdata, - output wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_0_wstrb, - output wire m_axi_mem_0_wlast, - output wire m_axi_mem_0_arvalid, - input wire m_axi_mem_0_arready, - output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_0_araddr, - output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_0_arid, - output wire [7:0] m_axi_mem_0_arlen, - input wire m_axi_mem_0_rvalid, - output wire m_axi_mem_0_rready, - input wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_0_rdata, - input wire m_axi_mem_0_rlast, - input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_0_rid, - input wire [1:0] m_axi_mem_0_rresp, - input wire m_axi_mem_0_bvalid, - output wire m_axi_mem_0_bready, - input wire [1:0] m_axi_mem_0_bresp, - input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_0_bid, +`define FOLDSIM_AXI_MEM(i) \ + output wire m_axi_mem_``i``_awvalid, \ + input wire m_axi_mem_``i``_awready, \ + output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_``i``_awaddr, \ + output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_``i``_awid, \ + output wire [7:0] m_axi_mem_``i``_awlen, \ + output wire m_axi_mem_``i``_wvalid, \ + input wire m_axi_mem_``i``_wready, \ + output wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_``i``_wdata, \ + output wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_``i``_wstrb, \ + output wire m_axi_mem_``i``_wlast, \ + output wire m_axi_mem_``i``_arvalid, \ + input wire m_axi_mem_``i``_arready, \ + output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_``i``_araddr, \ + output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_``i``_arid, \ + output wire [7:0] m_axi_mem_``i``_arlen, \ + input wire m_axi_mem_``i``_rvalid, \ + output wire m_axi_mem_``i``_rready, \ + input wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_``i``_rdata, \ + input wire m_axi_mem_``i``_rlast, \ + input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_``i``_rid, \ + input wire [1:0] m_axi_mem_``i``_rresp, \ + input wire m_axi_mem_``i``_bvalid, \ + output wire m_axi_mem_``i``_bready, \ + input wire [1:0] m_axi_mem_``i``_bresp, \ + input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_``i``_bid + + `FOLDSIM_REPEAT(`FOLDSIM_MEM_PORTS, FOLDSIM_AXI_MEM, FOLDSIM_COMMA), input wire s_axi_control_awvalid, output wire s_axi_control_awready, @@ -76,7 +84,9 @@ module foldsim_afu #( input wire s_axi_control_bready, output wire [1:0] s_axi_control_bresp ); +`undef FOLDSIM_AXI_MEM localparam DATA_W = C_M_AXI_MEM_DATA_WIDTH; + localparam PORTS = `FOLDSIM_MEM_PORTS; // A registered reset, so it reaches the whole kernel in one cycle's time. reg reset; @@ -89,43 +99,102 @@ module foldsim_afu #( wire [w-1:0] name``_bits; `FOLDSIM_CH(reset, 1) - `FOLDSIM_CH(m_axi_mem_0_awready, 1) - `FOLDSIM_CH(m_axi_mem_0_wready, 1) - `FOLDSIM_CH(m_axi_mem_0_arready, 1) - `FOLDSIM_CH(m_axi_mem_0_rvalid, 1) - `FOLDSIM_CH(m_axi_mem_0_rdata, DATA_W) - `FOLDSIM_CH(m_axi_mem_0_rlast, 1) - `FOLDSIM_CH(m_axi_mem_0_rid, TARGET_ID_W) - `FOLDSIM_CH(m_axi_mem_0_rresp, 2) - `FOLDSIM_CH(m_axi_mem_0_bvalid, 1) - `FOLDSIM_CH(m_axi_mem_0_bresp, 2) - `FOLDSIM_CH(m_axi_mem_0_bid, TARGET_ID_W) `FOLDSIM_CH(dcr_req_valid, 1) `FOLDSIM_CH(dcr_req_rw, 1) `FOLDSIM_CH(dcr_req_addr, DCR_ADDR_W) `FOLDSIM_CH(dcr_req_data, DCR_DATA_W) `FOLDSIM_CH(start, 1) - `FOLDSIM_CH(m_axi_mem_0_awvalid, 1) - `FOLDSIM_CH(m_axi_mem_0_awaddr, TARGET_ADDR_W) - `FOLDSIM_CH(m_axi_mem_0_awid, TARGET_ID_W) - `FOLDSIM_CH(m_axi_mem_0_awlen, 8) - `FOLDSIM_CH(m_axi_mem_0_wvalid, 1) - `FOLDSIM_CH(m_axi_mem_0_wdata, DATA_W) - `FOLDSIM_CH(m_axi_mem_0_wstrb, DATA_W/8) - `FOLDSIM_CH(m_axi_mem_0_wlast, 1) - `FOLDSIM_CH(m_axi_mem_0_arvalid, 1) - `FOLDSIM_CH(m_axi_mem_0_araddr, TARGET_ADDR_W) - `FOLDSIM_CH(m_axi_mem_0_arid, TARGET_ID_W) - `FOLDSIM_CH(m_axi_mem_0_arlen, 8) - `FOLDSIM_CH(m_axi_mem_0_rready, 1) - `FOLDSIM_CH(m_axi_mem_0_bready, 1) `FOLDSIM_CH(dcr_rsp_valid, 1) `FOLDSIM_CH(dcr_rsp_data, DCR_DATA_W) `FOLDSIM_CH(busy, 1) `undef FOLDSIM_CH + // A memory port's channels, by port. +`define FOLDSIM_MCH(name, w) \ + wire mem_``name``_valid [PORTS]; \ + wire mem_``name``_ready [PORTS]; \ + wire [w-1:0] mem_``name``_bits [PORTS]; + + `FOLDSIM_MCH(awready, 1) + `FOLDSIM_MCH(wready, 1) + `FOLDSIM_MCH(arready, 1) + `FOLDSIM_MCH(rvalid, 1) + `FOLDSIM_MCH(rdata, DATA_W) + `FOLDSIM_MCH(rlast, 1) + `FOLDSIM_MCH(rid, TARGET_ID_W) + `FOLDSIM_MCH(rresp, 2) + `FOLDSIM_MCH(bvalid, 1) + `FOLDSIM_MCH(bresp, 2) + `FOLDSIM_MCH(bid, TARGET_ID_W) + `FOLDSIM_MCH(awvalid, 1) + `FOLDSIM_MCH(awaddr, TARGET_ADDR_W) + `FOLDSIM_MCH(awid, TARGET_ID_W) + `FOLDSIM_MCH(awlen, 8) + `FOLDSIM_MCH(wvalid, 1) + `FOLDSIM_MCH(wdata, DATA_W) + `FOLDSIM_MCH(wstrb, DATA_W/8) + `FOLDSIM_MCH(wlast, 1) + `FOLDSIM_MCH(arvalid, 1) + `FOLDSIM_MCH(araddr, TARGET_ADDR_W) + `FOLDSIM_MCH(arid, TARGET_ID_W) + `FOLDSIM_MCH(arlen, 8) + `FOLDSIM_MCH(rready, 1) + `FOLDSIM_MCH(bready, 1) +`undef FOLDSIM_MCH + + // Device memory, by port. + wire hbm_awvalid [PORTS], hbm_awready [PORTS]; + wire [63:0] hbm_awaddr [PORTS]; + wire [7:0] hbm_awlen [PORTS]; + wire hbm_wvalid [PORTS], hbm_wready [PORTS], hbm_wlast [PORTS]; + wire [DATA_W-1:0] hbm_wdata [PORTS]; + wire [DATA_W/8-1:0] hbm_wstrb [PORTS]; + wire hbm_bvalid [PORTS], hbm_bready [PORTS]; + wire hbm_arvalid [PORTS], hbm_arready [PORTS]; + wire [63:0] hbm_araddr [PORTS]; + wire [7:0] hbm_arlen [PORTS]; + wire hbm_rvalid [PORTS], hbm_rready [PORTS]; + wire [DATA_W-1:0] hbm_rdata [PORTS]; + +`define FOLDSIM_HBM(i) \ + assign m_axi_mem_``i``_awvalid = hbm_awvalid[i]; \ + assign hbm_awready[i] = m_axi_mem_``i``_awready; \ + assign m_axi_mem_``i``_awaddr = hbm_awaddr[i]; \ + assign m_axi_mem_``i``_awid = '0; \ + assign m_axi_mem_``i``_awlen = hbm_awlen[i]; \ + assign m_axi_mem_``i``_wvalid = hbm_wvalid[i]; \ + assign hbm_wready[i] = m_axi_mem_``i``_wready; \ + assign m_axi_mem_``i``_wdata = hbm_wdata[i]; \ + assign m_axi_mem_``i``_wstrb = hbm_wstrb[i]; \ + assign m_axi_mem_``i``_wlast = hbm_wlast[i]; \ + assign hbm_bvalid[i] = m_axi_mem_``i``_bvalid; \ + assign m_axi_mem_``i``_bready = hbm_bready[i]; \ + assign m_axi_mem_``i``_arvalid = hbm_arvalid[i]; \ + assign hbm_arready[i] = m_axi_mem_``i``_arready; \ + assign m_axi_mem_``i``_araddr = hbm_araddr[i]; \ + assign m_axi_mem_``i``_arid = '0; \ + assign m_axi_mem_``i``_arlen = hbm_arlen[i]; \ + assign hbm_rvalid[i] = m_axi_mem_``i``_rvalid; \ + assign m_axi_mem_``i``_rready = hbm_rready[i]; \ + assign hbm_rdata[i] = m_axi_mem_``i``_rdata + + `FOLDSIM_REPEAT(`FOLDSIM_MEM_PORTS, FOLDSIM_HBM, FOLDSIM_SEMI); +`undef FOLDSIM_HBM + `define FOLDSIM_IN(name) \ .name``_valid (name``_valid), .name``_ready (name``_ready), .name``_bits (name``_bits) +`define FOLDSIM_MIN(i, name) \ + .m_axi_mem_``i``_``name``_valid (mem_``name``_valid[i]), \ + .m_axi_mem_``i``_``name``_ready (mem_``name``_ready[i]), \ + .m_axi_mem_``i``_``name``_bits (mem_``name``_bits[i]) +`define FOLDSIM_SIM_MEM(i) \ + `FOLDSIM_MIN(i, awready), `FOLDSIM_MIN(i, wready), `FOLDSIM_MIN(i, arready), \ + `FOLDSIM_MIN(i, rvalid), `FOLDSIM_MIN(i, rdata), `FOLDSIM_MIN(i, rlast), `FOLDSIM_MIN(i, rid), \ + `FOLDSIM_MIN(i, rresp), `FOLDSIM_MIN(i, bvalid), `FOLDSIM_MIN(i, bresp), `FOLDSIM_MIN(i, bid), \ + `FOLDSIM_MIN(i, awvalid), `FOLDSIM_MIN(i, awaddr), `FOLDSIM_MIN(i, awid), `FOLDSIM_MIN(i, awlen), \ + `FOLDSIM_MIN(i, wvalid), `FOLDSIM_MIN(i, wdata), `FOLDSIM_MIN(i, wstrb), `FOLDSIM_MIN(i, wlast), \ + `FOLDSIM_MIN(i, arvalid), `FOLDSIM_MIN(i, araddr), `FOLDSIM_MIN(i, arid), `FOLDSIM_MIN(i, arlen), \ + `FOLDSIM_MIN(i, rready), `FOLDSIM_MIN(i, bready) wire [`FOLDSIM_NUM_CLASSES-1:0] profile_finish /*verilator public_flat_rd*/; @@ -133,134 +202,121 @@ module foldsim_afu #( .hostClock (ap_clk), .hostReset (reset), .profile_finish (profile_finish), + `FOLDSIM_REPEAT(`FOLDSIM_MEM_PORTS, FOLDSIM_SIM_MEM, FOLDSIM_COMMA), `FOLDSIM_IN(reset), - `FOLDSIM_IN(m_axi_mem_0_awready), - `FOLDSIM_IN(m_axi_mem_0_wready), - `FOLDSIM_IN(m_axi_mem_0_arready), - `FOLDSIM_IN(m_axi_mem_0_rvalid), - `FOLDSIM_IN(m_axi_mem_0_rdata), - `FOLDSIM_IN(m_axi_mem_0_rlast), - `FOLDSIM_IN(m_axi_mem_0_rid), - `FOLDSIM_IN(m_axi_mem_0_rresp), - `FOLDSIM_IN(m_axi_mem_0_bvalid), - `FOLDSIM_IN(m_axi_mem_0_bresp), - `FOLDSIM_IN(m_axi_mem_0_bid), `FOLDSIM_IN(dcr_req_valid), `FOLDSIM_IN(dcr_req_rw), `FOLDSIM_IN(dcr_req_addr), `FOLDSIM_IN(dcr_req_data), `FOLDSIM_IN(start), - `FOLDSIM_IN(m_axi_mem_0_awvalid), - `FOLDSIM_IN(m_axi_mem_0_awaddr), - `FOLDSIM_IN(m_axi_mem_0_awid), - `FOLDSIM_IN(m_axi_mem_0_awlen), - `FOLDSIM_IN(m_axi_mem_0_wvalid), - `FOLDSIM_IN(m_axi_mem_0_wdata), - `FOLDSIM_IN(m_axi_mem_0_wstrb), - `FOLDSIM_IN(m_axi_mem_0_wlast), - `FOLDSIM_IN(m_axi_mem_0_arvalid), - `FOLDSIM_IN(m_axi_mem_0_araddr), - `FOLDSIM_IN(m_axi_mem_0_arid), - `FOLDSIM_IN(m_axi_mem_0_arlen), - `FOLDSIM_IN(m_axi_mem_0_rready), - `FOLDSIM_IN(m_axi_mem_0_bready), `FOLDSIM_IN(dcr_rsp_valid), `FOLDSIM_IN(dcr_rsp_data), `FOLDSIM_IN(busy) ); +`undef FOLDSIM_SIM_MEM +`undef FOLDSIM_MIN -`define FOLDSIM_MEM(name) \ - .name``_valid (m_axi_mem_0_``name``_valid), .name``_ready (m_axi_mem_0_``name``_ready), \ - .name``_bits (m_axi_mem_0_``name``_bits) - - // The memory endpoint's bank model: read slots, and a ring of 2^9 DRAM + // Each memory endpoint's bank model: read slots, and a ring of 2^9 DRAM // clocks (MEM_CAPS reports both). localparam MEM_SLOTS = 32; localparam MEM_HLOG2 = 9; - // The memory endpoint's token ends register have and done, so its finish - // starts at its own state, not at the simulator's handshake. A token then + // The memory endpoints' token ends register have and done, so their finish + // starts at their own state, not at the simulator's handshake. A token then // counts a host cycle later; s2w4i2's host cycles do not move. localparam MEM_STAGED = 1; - localparam [31:0] MEM_CAPS = {4'd0, 4'd1, 4'd1, 8'(MEM_SLOTS), 4'(MEM_HLOG2 - 4), 8'd1}; + // Ports, engines and device-memory masters: one of each per port. + localparam [31:0] MEM_CAPS = {4'd0, 4'(PORTS), 4'(PORTS), 8'(MEM_SLOTS), 4'(MEM_HLOG2 - 4), 8'(PORTS)}; wire [31:0] mem_read_latency, mem_write_latency; wire [7:0] mem_max_reads, mem_max_writes; wire [63:0] mem_base; - wire mem_idle, mem_error, mem_stall; - wire mem_reset_valid, mem_reset_ready, mem_reset_bits; wire mem_model_bank; wire [31:0] mem_clock, mem_queue; wire [15:0] mem_latency, mem_wmark; wire [12:0] mem_geom; wire [18*12-1:0] mem_timing; - wire [6*64-1:0] mem_stats; + wire [PORTS*8*64-1:0] mem_stats; + wire [1:0] mem_refresh; + wire [15:0] mem_refi, mem_rfc; + wire [PORTS-1:0] mem_idle, mem_error, mem_stall; + wire [PORTS-1:0] mem_reset_valid, mem_reset_ready, mem_reset_bits; - foldsim_mem #( - .ID_W (TARGET_ID_W), - .ADDR_W (TARGET_ADDR_W), - .DATA_W (DATA_W), - .SLOTS (MEM_SLOTS), - .HLOG2 (MEM_HLOG2), - .STAGED (MEM_STAGED) - ) mem ( - .clk (ap_clk), - .reset (reset), - .read_latency (mem_read_latency), - .write_latency (mem_write_latency), - .max_reads (mem_max_reads), - .max_writes (mem_max_writes), - .base (mem_base), - .model_bank (mem_model_bank), - .cfg_data_rate (mem_clock[15:0]), - .cfg_core_clock(mem_clock[31:16]), - .cfg_latency (mem_latency), - .cfg_queue (mem_queue[15:0]), - .cfg_rq (mem_queue[23:16]), - .cfg_wq (mem_queue[31:24]), - .cfg_wr_high (mem_wmark[7:0]), - .cfg_wr_low (mem_wmark[15:8]), - .cfg_geom (mem_geom), - .cfg_t (mem_timing), - .stats (mem_stats), - .idle (mem_idle), - .error (mem_error), - .stall (mem_stall), - .treset_valid (mem_reset_valid), - .treset_ready (mem_reset_ready), - .treset_bits (mem_reset_bits), - `FOLDSIM_MEM(awvalid), `FOLDSIM_MEM(awaddr), `FOLDSIM_MEM(awid), `FOLDSIM_MEM(awlen), - `FOLDSIM_MEM(wvalid), `FOLDSIM_MEM(wdata), `FOLDSIM_MEM(wstrb), `FOLDSIM_MEM(wlast), - `FOLDSIM_MEM(arvalid), `FOLDSIM_MEM(araddr), `FOLDSIM_MEM(arid), `FOLDSIM_MEM(arlen), - `FOLDSIM_MEM(rready), `FOLDSIM_MEM(bready), - `FOLDSIM_MEM(awready), `FOLDSIM_MEM(wready), `FOLDSIM_MEM(arready), - `FOLDSIM_MEM(rvalid), `FOLDSIM_MEM(rdata), `FOLDSIM_MEM(rlast), `FOLDSIM_MEM(rid), `FOLDSIM_MEM(rresp), - `FOLDSIM_MEM(bvalid), `FOLDSIM_MEM(bresp), `FOLDSIM_MEM(bid), - .m_awvalid (m_axi_mem_0_awvalid), - .m_awready (m_axi_mem_0_awready), - .m_awaddr (m_axi_mem_0_awaddr), - .m_awlen (m_axi_mem_0_awlen), - .m_wvalid (m_axi_mem_0_wvalid), - .m_wready (m_axi_mem_0_wready), - .m_wdata (m_axi_mem_0_wdata), - .m_wstrb (m_axi_mem_0_wstrb), - .m_wlast (m_axi_mem_0_wlast), - .m_bvalid (m_axi_mem_0_bvalid), - .m_bready (m_axi_mem_0_bready), - .m_arvalid (m_axi_mem_0_arvalid), - .m_arready (m_axi_mem_0_arready), - .m_araddr (m_axi_mem_0_araddr), - .m_arlen (m_axi_mem_0_arlen), - .m_rvalid (m_axi_mem_0_rvalid), - .m_rready (m_axi_mem_0_rready), - .m_rdata (m_axi_mem_0_rdata) - ); - assign m_axi_mem_0_awid = '0; - assign m_axi_mem_0_arid = '0; +`define FOLDSIM_MEM(name) \ + .name``_valid (mem_``name``_valid[p]), .name``_ready (mem_``name``_ready[p]), \ + .name``_bits (mem_``name``_bits[p]) + + for (genvar p = 0; p < PORTS; ++p) begin : g_mem + foldsim_mem #( + .ID_W (TARGET_ID_W), + .ADDR_W (TARGET_ADDR_W), + .DATA_W (DATA_W), + .SLOTS (MEM_SLOTS), + .HLOG2 (MEM_HLOG2), + .STAGED (MEM_STAGED), + .CH_SHIFT (`FOLDSIM_MEM_CH_SHIFT) + ) mem ( + .clk (ap_clk), + .reset (reset), + .read_latency (mem_read_latency), + .write_latency (mem_write_latency), + .max_reads (mem_max_reads), + .max_writes (mem_max_writes), + .base (mem_base), + .model_bank (mem_model_bank), + .cfg_data_rate (mem_clock[15:0]), + .cfg_core_clock(mem_clock[31:16]), + .cfg_latency (mem_latency), + .cfg_queue (mem_queue[15:0]), + .cfg_rq (mem_queue[23:16]), + .cfg_wq (mem_queue[31:24]), + .cfg_wr_high (mem_wmark[7:0]), + .cfg_wr_low (mem_wmark[15:8]), + .cfg_geom (mem_geom), + .cfg_t (mem_timing), + .cfg_refresh (mem_refresh), + .cfg_refi (mem_refi), + .cfg_rfc (mem_rfc), + .stats (mem_stats[p*8*64 +: 8*64]), + .idle (mem_idle[p]), + .error (mem_error[p]), + .stall (mem_stall[p]), + .treset_valid (mem_reset_valid[p]), + .treset_ready (mem_reset_ready[p]), + .treset_bits (mem_reset_bits[p]), + `FOLDSIM_MEM(awvalid), `FOLDSIM_MEM(awaddr), `FOLDSIM_MEM(awid), `FOLDSIM_MEM(awlen), + `FOLDSIM_MEM(wvalid), `FOLDSIM_MEM(wdata), `FOLDSIM_MEM(wstrb), `FOLDSIM_MEM(wlast), + `FOLDSIM_MEM(arvalid), `FOLDSIM_MEM(araddr), `FOLDSIM_MEM(arid), `FOLDSIM_MEM(arlen), + `FOLDSIM_MEM(rready), `FOLDSIM_MEM(bready), + `FOLDSIM_MEM(awready), `FOLDSIM_MEM(wready), `FOLDSIM_MEM(arready), + `FOLDSIM_MEM(rvalid), `FOLDSIM_MEM(rdata), `FOLDSIM_MEM(rlast), `FOLDSIM_MEM(rid), `FOLDSIM_MEM(rresp), + `FOLDSIM_MEM(bvalid), `FOLDSIM_MEM(bresp), `FOLDSIM_MEM(bid), + .m_awvalid (hbm_awvalid[p]), + .m_awready (hbm_awready[p]), + .m_awaddr (hbm_awaddr[p]), + .m_awlen (hbm_awlen[p]), + .m_wvalid (hbm_wvalid[p]), + .m_wready (hbm_wready[p]), + .m_wdata (hbm_wdata[p]), + .m_wstrb (hbm_wstrb[p]), + .m_wlast (hbm_wlast[p]), + .m_bvalid (hbm_bvalid[p]), + .m_bready (hbm_bready[p]), + .m_arvalid (hbm_arvalid[p]), + .m_arready (hbm_arready[p]), + .m_araddr (hbm_araddr[p]), + .m_arlen (hbm_arlen[p]), + .m_rvalid (hbm_rvalid[p]), + .m_rready (hbm_rready[p]), + .m_rdata (hbm_rdata[p]) + ); + end +`undef FOLDSIM_MEM foldsim_ctrl #( .DCR_ADDR_W (DCR_ADDR_W), .DCR_DATA_W (DCR_DATA_W), .NUM_CLASSES (`FOLDSIM_NUM_CLASSES), + .MEM_PORTS (PORTS), .MEM_CAPS (MEM_CAPS) ) ctrl ( .clk (ap_clk), @@ -291,10 +347,13 @@ module foldsim_afu #( .mem_geom (mem_geom), .mem_wmark (mem_wmark), .mem_timing (mem_timing), + .mem_refresh (mem_refresh), + .mem_refi (mem_refi), + .mem_rfc (mem_rfc), .mem_stats (mem_stats), - .mem_idle (mem_idle), - .mem_error (mem_error), - .mem_stall (mem_stall), + .mem_idle (&mem_idle), + .mem_error (|mem_error), + .mem_stall (|mem_stall), .profile_finish (profile_finish), .mem_reset_valid (mem_reset_valid), .mem_reset_ready (mem_reset_ready), @@ -313,6 +372,5 @@ module foldsim_afu #( assign s_axi_control_bresp = 2'b00; `undef FOLDSIM_IN -`undef FOLDSIM_MEM endmodule diff --git a/hw/rtl/afu/foldsim/foldsim_ctrl.sv b/hw/rtl/afu/foldsim/foldsim_ctrl.sv index 32229a3949..510c47970a 100644 --- a/hw/rtl/afu/foldsim/foldsim_ctrl.sv +++ b/hw/rtl/afu/foldsim/foldsim_ctrl.sv @@ -33,7 +33,9 @@ module foldsim_ctrl #( parameter RESET_CYCLES = 64, parameter MEM_DEPTH = 16, parameter NUM_CLASSES = 1, - // MEM_CAPS: what the memory endpoint's bank model was built with. + // The memory endpoints, one per memory port of the target. + parameter MEM_PORTS = 1, + // MEM_CAPS: what the memory endpoints' bank models were built with. parameter MEM_CAPS = 32'd0 ) ( input wire clk, @@ -68,14 +70,20 @@ module foldsim_ctrl #( output reg [12:0] mem_geom, output reg [15:0] mem_wmark, output wire [18*12-1:0] mem_timing, - input wire [6*64-1:0] mem_stats, + // Refresh: [0] on, [1] all banks; its interval and busy time (clocks). + output wire [1:0] mem_refresh, + output reg [15:0] mem_refi, + output reg [15:0] mem_rfc, + // Per memory port, its counters. + input wire [MEM_PORTS*8*64-1:0] mem_stats, input wire mem_idle, input wire mem_error, input wire mem_stall, // Which classes of the simulator's models finish a thread this cycle. input wire [NUM_CLASSES-1:0] profile_finish, - // The target's reset, a token per target cycle for the memory endpoint. - output wire mem_reset_valid, input wire mem_reset_ready, output wire mem_reset_bits, + // The target's reset, a token per target cycle for each memory endpoint. + output wire [MEM_PORTS-1:0] mem_reset_valid, input wire [MEM_PORTS-1:0] mem_reset_ready, + output wire [MEM_PORTS-1:0] mem_reset_bits, // Target inputs, as channels of the simulator: the endpoint's outputs. output wire reset_valid, input wire reset_ready, output wire reset_bits, @@ -118,13 +126,15 @@ module foldsim_ctrl #( reg [DCR_DATA_W-1:0] rsp_data; reg [63:0] target_cycles, host_cycles, mem_stalls; reg [47:0] finishes [NUM_CLASSES]; - reg [1:0] mem_model; + reg [5:0] mem_model; reg [11:0] mem_t [18]; // The model in effect from a reset on: the memory endpoint takes it at the // end of every reset cycle, so a reset command's own cycle has it already. - wire [1:0] mem_model_now = (cur_op == `FOLDSIM_CMD_RESET) ? cur_addr[1:0] : mem_model; - assign mem_model_bank = (mem_model_now == 2'd1); + // [1:0] the model, [4] refresh, [5] all banks at once. + wire [5:0] mem_model_now = (cur_op == `FOLDSIM_CMD_RESET) ? cur_addr[5:0] : mem_model; + assign mem_model_bank = (mem_model_now[1:0] == 2'd1); + assign mem_refresh = mem_model_now[5:4]; for (genvar i = 0; i < 18; ++i) begin : g_mem_t assign mem_timing[i*12 +: 12] = mem_t[i]; end @@ -150,7 +160,7 @@ module foldsim_ctrl #( : 32'd0; wire finish; - wire [6:0] done; + wire [6+MEM_PORTS-1:0] done; wire [2:0] have; foldsim_token_out #(1) o_reset (clk, reset, reset_valid, reset_ready, run, finish, done[0]); foldsim_token_out #(1) o_dvalid (clk, reset, dcr_req_valid_valid, dcr_req_valid_ready, run, finish, done[1]); @@ -158,14 +168,16 @@ module foldsim_ctrl #( foldsim_token_out #(1) o_daddr (clk, reset, dcr_req_addr_valid, dcr_req_addr_ready, run, finish, done[3]); foldsim_token_out #(1) o_ddata (clk, reset, dcr_req_data_valid, dcr_req_data_ready, run, finish, done[4]); foldsim_token_out #(1) o_start (clk, reset, start_valid, start_ready, run, finish, done[5]); - foldsim_token_out #(1) o_mreset (clk, reset, mem_reset_valid, mem_reset_ready, run, finish, done[6]); + for (genvar p = 0; p < MEM_PORTS; ++p) begin : g_mreset + foldsim_token_out #(1) o_mreset (clk, reset, mem_reset_valid[p], mem_reset_ready[p], run, finish, done[6+p]); + assign mem_reset_bits[p] = in_reset; + end assign reset_bits = in_reset; assign dcr_req_valid_bits = dcr_valid; assign dcr_req_rw_bits = dcr_rw; assign dcr_req_addr_bits = cur_addr; assign dcr_req_data_bits = cur_data; assign start_bits = do_start; - assign mem_reset_bits = in_reset; wire rsp_v, busy; wire [DCR_DATA_W-1:0] rsp_d; @@ -214,7 +226,9 @@ module foldsim_ctrl #( mem_max_reads <= 8'(MEM_DEPTH); mem_max_writes <= 8'(MEM_DEPTH); mem_base <= 64'h40_0000_0000; - mem_model <= 2'd0; + mem_model <= 6'd0; + mem_refi <= 16'd0; + mem_rfc <= 16'd0; mem_clock <= {16'd400, 16'd800}; mem_latency <= 16'd0; mem_queue <= {8'd32, 8'd32, 16'd0}; @@ -250,7 +264,7 @@ module foldsim_ctrl #( // A reset command's address chooses the memory model the // target has from the reset on. if (cur_op == `FOLDSIM_CMD_RESET) - mem_model <= cur_addr[1:0]; + mem_model <= cur_addr[5:0]; // A start counts as busy until the target says it is, or plainly // never will. if (do_start) begin @@ -303,6 +317,8 @@ module foldsim_ctrl #( `FOLDSIM_REG_MEM_QUEUE: mem_queue <= w_data; `FOLDSIM_REG_MEM_GEOM: mem_geom <= w_data[12:0]; `FOLDSIM_REG_MEM_WMARK: mem_wmark <= w_data[15:0]; + `FOLDSIM_REG_MEM_REFI: mem_refi <= w_data[15:0]; + `FOLDSIM_REG_MEM_RFC: mem_rfc <= w_data[15:0]; default:; endcase if (wa >= `FOLDSIM_REG_MEM_TIMING && wa < `FOLDSIM_REG_MEM_TIMING + 12'(4 * 18)) @@ -347,12 +363,16 @@ module foldsim_ctrl #( `FOLDSIM_REG_MEM_QUEUE: s_rdata <= mem_queue; `FOLDSIM_REG_MEM_GEOM: s_rdata <= 32'(mem_geom); `FOLDSIM_REG_MEM_WMARK: s_rdata <= 32'(mem_wmark); + `FOLDSIM_REG_MEM_REFI: s_rdata <= 32'(mem_refi); + `FOLDSIM_REG_MEM_RFC: s_rdata <= 32'(mem_rfc); default: s_rdata <= 32'd0; endcase if (ar_addr >= `FOLDSIM_REG_MEM_TIMING && ar_addr < `FOLDSIM_REG_MEM_TIMING + 12'(4 * 18)) s_rdata <= 32'(mem_t[5'((ar_addr - `FOLDSIM_REG_MEM_TIMING) >> 2)]); - if (ar_addr >= `FOLDSIM_REG_MEM_STATS && ar_addr < `FOLDSIM_REG_MEM_STATS + 12'(8 * 6)) - s_rdata <= mem_stats[32'(ar_addr - `FOLDSIM_REG_MEM_STATS) * 8 +: 32]; + // A port's counters at FOLDSIM_REG_MEM_STATS + 0x40 per port. + if (ar_addr >= `FOLDSIM_REG_MEM_STATS && ar_addr < `FOLDSIM_REG_MEM_STATS + 12'(64 * MEM_PORTS)) + s_rdata <= mem_stats[32'((ar_addr - `FOLDSIM_REG_MEM_STATS) >> 6) * (8 * 64) + + 32'(ar_addr[5:0]) * 8 +: 32]; if (ar_addr >= `FOLDSIM_REG_FINISH_BASE && ar_addr < `FOLDSIM_REG_FINISH_BASE + 12'(8 * NUM_CLASSES)) begin s_rdata <= ar_addr[2] ? 32'(finishes[(ar_addr - `FOLDSIM_REG_FINISH_BASE) >> 3] >> 32) diff --git a/hw/rtl/afu/foldsim/foldsim_mem.sv b/hw/rtl/afu/foldsim/foldsim_mem.sv index 7237908105..3c7f497d5b 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem.sv @@ -50,7 +50,11 @@ module foldsim_mem #( parameter HLOG2 = 9, // Token ends with registered have and done (foldsim_token): the cycle's // finish starts at the endpoint's own registers, a host cycle later. - parameter STAGED = 0 + parameter STAGED = 0, + // The target's ports interleave its address space by block, 2^CH_SHIFT of + // them: the bank model sees the block within the port's channel, as + // dram_sim's banks do. Device memory still takes the target's address. + parameter CH_SHIFT = 0 ) ( input wire clk, input wire reset, @@ -72,8 +76,12 @@ module foldsim_mem #( input wire [7:0] cfg_wr_low, input wire [12:0] cfg_geom, input wire [18*12-1:0] cfg_t, - // Reads, writes, row hits, misses, conflicts, read latency sum. - output wire [6*64-1:0] stats, + input wire [1:0] cfg_refresh, + input wire [15:0] cfg_refi, + input wire [15:0] cfg_rfc, + // Reads, writes, row hits, misses, conflicts, read latency sum, + // refreshes, target cycles the port could take no read. + output wire [8*64-1:0] stats, // No transaction in flight, at the target or in memory. output wire idle, output reg error, @@ -139,6 +147,7 @@ module foldsim_mem #( localparam HCW = $clog2(HD + 1); localparam HPW = $clog2(HD); localparam SW = $clog2(SLOTS); + localparam LB = $clog2(DATA_W / 8); wire finish; // The bank model is the port's timing. @@ -169,6 +178,11 @@ module foldsim_mem #( foldsim_token_in #(1, STAGED) t_bready (clk, reset, bready_valid, bready_ready, bready_bits, finish, have[13], bready); foldsim_token_in #(1, STAGED) t_treset (clk, reset, treset_valid, treset_ready, treset_bits, finish, have[14], treset); + // The requests' addresses within the port's channel, the bits the bank + // model takes: the block index drops the port's. + wire [29:5] ar_ch = {araddr[29 + CH_SHIFT -: 30 - LB], araddr[LB-1:5]}; + wire [29:5] aw_ch = {awaddr[29 + CH_SHIFT -: 30 - LB], awaddr[LB-1:5]}; + // ---- The pipe's state ---- reg [31:0] cycle; reg [CW-1:0] pending_reads, pending_aw, pending_w; @@ -288,7 +302,7 @@ module foldsim_mem #( | (aw_fire & w_fire); wire awq_empty = (awq_count == 0); wire [ID_W-1:0] w_form_id = awq_empty ? awid : awq_id[awq_head]; - wire [29:5] w_form_addr = awq_empty ? awaddr[29:5] : awq_addr[awq_head]; + wire [29:5] w_form_addr = awq_empty ? aw_ch : awq_addr[awq_head]; wire awq_pop = new_write & ~awq_empty; wire awq_push = aw_fire & ~(new_write & awq_empty); wire [31:0] w_release = cycle + write_latency - 32'd1; @@ -399,7 +413,7 @@ module foldsim_mem #( if (awq_push) begin awq_id[awq_tail] <= awid; - awq_addr[awq_tail] <= awaddr[29:5]; + awq_addr[awq_tail] <= aw_ch; awq_tail <= awq_tail + 1'b1; end if (awq_pop) @@ -565,11 +579,14 @@ module foldsim_mem #( .cfg_wr_low (cfg_wr_low), .cfg_geom (cfg_geom), .cfg_t (cfg_t), + .cfg_refresh (cfg_refresh), + .cfg_refi (cfg_refi), + .cfg_rfc (cfg_rfc), .finish (finish), .treset (treset), .ar_fire (ar_fire), .ar_id (arid), - .ar_addr (araddr[29:5]), + .ar_addr (ar_ch), .aw_fire (aw_fire), .new_write (new_write), .nw_id (w_form_id), @@ -592,7 +609,9 @@ module foldsim_mem #( .stat_hits (stats[2*64 +: 64]), .stat_misses (stats[3*64 +: 64]), .stat_conflicts (stats[4*64 +: 64]), - .stat_latency (stats[5*64 +: 64]) + .stat_latency (stats[5*64 +: 64]), + .stat_refreshes (stats[6*64 +: 64]), + .stat_full (stats[7*64 +: 64]) ); endmodule diff --git a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv index 1522b81891..5d4dd0b25f 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv @@ -33,6 +33,12 @@ // 2^HLOG2 - 64 clocks ahead: the ring's last word is the one being retired. // Times of the bank table and the controller are 16-bit, modulo; a time too // old to constrain anything is moved up before it could wrap. +// +// Refresh, when on, takes a DRAM clock's turn to reserve every `cfg_refi` +// clocks: a precharge and a REF, placed as a precharge is, on the next bank in +// turn or (all-bank) on every bank. For all-bank refresh the channel keeps the +// latest time a precharge and an activate allow it, over the banks opened +// since the last one, instead of reading the bank table. module foldsim_mem_bank #( parameter ID_W = 4, parameter SLOTS = 32, @@ -65,6 +71,11 @@ module foldsim_mem_bank #( // tWR, tRRD_S, tRRD_L, tFAW, tWTR_S, tWTR_L, read-to-write extra: DRAM // clocks, 12 bits each. input wire [18*12-1:0] cfg_t, + // Refresh: [0] on, [1] all banks at once; clocks between refreshes, and + // clocks a refreshed bank may not activate. + input wire [1:0] cfg_refresh, + input wire [15:0] cfg_refi, + input wire [15:0] cfg_rfc, // The target cycle finishes; a reset cycle drops everything. input wire finish, @@ -96,14 +107,17 @@ module foldsim_mem_bank #( // The bank model is the port's timing, since the last target reset. output reg active, - // Reads, writes, row hits, misses, conflicts, and the sum of read - // latencies (target cycles from AR to R). + // Reads, writes, row hits, misses, conflicts, the sum of read latencies + // (target cycles from AR to R), refreshes, and target cycles the port + // could take no read. output reg [63:0] stat_reads, output reg [63:0] stat_writes, output reg [63:0] stat_hits, output reg [63:0] stat_misses, output reg [63:0] stat_conflicts, - output reg [63:0] stat_latency + output reg [63:0] stat_latency, + output reg [63:0] stat_refreshes, + output reg [63:0] stat_full ); localparam H = 1 << HLOG2; localparam HE = H - 64; @@ -138,7 +152,7 @@ module foldsim_mem_bank #( reg signed [15:0] k_ccds, k_ccdl, k_rtw, k_wtrs, k_wtrl, k_reach_r, k_reach_w, k_reach_a; reg signed [15:0] k_wrpre, k_dep, k_ccdl1, k_rcdrd, k_rcdwr, k_rp, k_ras, k_rc, k_rtp; - reg signed [15:0] k_rrds, k_rrdl; + reg signed [15:0] k_rrds, k_rrdl, k_rfc; reg [16:0] k_twoclk; reg [2:0] k_bb; reg [4:0] k_rowsh; @@ -164,6 +178,7 @@ module foldsim_mem_bank #( k_rtp <= 16'(T[I_RTP]); k_rrds <= 16'(T[I_RRDS]); k_rrdl <= 16'(T[I_RRDL]); + k_rfc <= $signed(cfg_rfc); k_twoclk <= {cfg_core_clock, 1'b0}; k_bb <= 3'(g_lpcs) + 3'(g_lbgs) + 3'(g_lbanks); k_rowsh <= 5'(g_lpcs) + 5'(g_lbgs) + 5'(g_lbanks) + 5'(g_lrow); @@ -281,6 +296,13 @@ module foldsim_mem_bank #( reg [7:0] rq_cnt, wq_cnt; // reserved bursts whose column has not issued reg wmode; reg [15:0] frontier, batch_floor, batch_last, read_floor; + // Refresh: when the next is due, and the bank it is for. All-bank: a bank + // was opened since the last; the latest precharge and activate + tRC any + // bank allows; and no bank activates before ab_floor. + reg [15:0] ref_due; + reg [4:0] ref_bank; + reg ab_open; + reg [15:0] ab_prer, ab_actrc, ab_floor; // The bank table, written at one port. A fresh bank has no row open and // no time that matters. @@ -306,6 +328,7 @@ module foldsim_mem_bank #( reg [WW-1:0] s_word; // the word the search reads wire [WW-1:0] s_word1 = s_word + WW'(1); reg op_w; + reg op_ref; // the operation is a refresh reg [TAG_W-1:0] op_tag; reg [BKW-1:0] op_bank; reg [ROW_W-1:0] op_row; @@ -383,8 +406,9 @@ module foldsim_mem_bank #( // ---- The engine ---- localparam S_IDLE = 4'd0, S_PUSH = 4'd1, S_TICK = 4'd2, S_MODE = 4'd3, S_BANK = 4'd4, S_BANK2 = 4'd5, S_SETUP = 4'd6, S_SETUP2 = 4'd7, S_SRCH = 4'd8, S_CHECK = 4'd9, - S_CHECK2 = 4'd10, S_COMMIT = 4'd11, S_END = 4'd12, S_WALK = 4'd13; - localparam K_PRE = 2'd0, K_ACT = 2'd1, K_C0 = 2'd2, K_C1 = 2'd3; + S_CHECK2 = 4'd10, S_COMMIT = 4'd11, S_END = 4'd12, S_WALK = 4'd13, + S_RBANK = 4'd14, S_RBANK2 = 4'd15; + localparam K_PRE = 3'd0, K_ACT = 3'd1, K_C0 = 3'd2, K_C1 = 3'd3, K_REF = 3'd4; reg [3:0] state; assign done = (state == S_IDLE); @@ -393,8 +417,8 @@ module foldsim_mem_bank #( // relative to e. reg op_hit, op_conf; reg signed [15:0] r_act, r_colr, r_prer, r_floor; - reg signed [15:0] c_pre, c_act, c_col0, c_col1; - reg [1:0] kind; + reg signed [15:0] c_pre, c_act, c_col0, c_col1, c_ref; + reg [2:0] kind; // The search: its lower bound; the next word to read (as the ring // position and the clocks left to the horizon) and the one read last @@ -419,7 +443,7 @@ module foldsim_mem_bank #( wire [63:0] f_col = rd_cmd | (op_w ? (rd_fwp | rd_fwg) : (rd_frp | rd_frg)); wire [63:0] f_act = rd_cmd | rd_row | (g_act2 ? {rd_row1[0], rd_row[63:1]} : 64'd0) | rd_fap | rd_fag; wire [63:0] f_pre = rd_cmd | rd_row; - wire [63:0] forb = (kind == K_PRE) ? f_pre : ((kind == K_ACT) ? f_act : f_col); + wire [63:0] forb = ((kind == K_PRE) | (kind == K_REF)) ? f_pre : ((kind == K_ACT) ? f_act : f_col); reg [5:0] b_ffs; always @(*) begin @@ -473,6 +497,7 @@ module foldsim_mem_bank #( wire to_read = wmode & (wn < 9'(cfg_wr_low)) & (rn != 0); wire wmode_n = to_write | (wmode & ~to_read); wire sel_any = wmode_n ? (wb_count != 0) : (rb_count != 0); + wire ref_now = cfg_refresh[0] & (cfg_refi != 16'd0) & ($signed(e - ref_due) >= 0); // The bank table's port: a reservation's bank, or the scrub's. // The scrub reads a bank in one host cycle and writes it in the next. @@ -482,7 +507,7 @@ module foldsim_mem_bank #( reg [ROW_W-1:0] sc_row; reg [15:0] cw_act, cw_colr, cw_prer; // the reservation's, absolute reg [ROW_W-1:0] cw_row; - wire bt_commit = (state == S_COMMIT) & cm_start; + wire bt_commit = (state == S_COMMIT) & cm_start & ~(op_ref & cfg_refresh[1]); wire [4:0] bt_wa = bt_commit ? op_bank : sc_b; function automatic [15:0] scrubbed(input [15:0] t); scrubbed = ($signed(t - (e + 16'd1)) < -16'sd16384) ? e + 16'd1 - 16'd16384 : t; @@ -514,6 +539,9 @@ module foldsim_mem_bank #( if ($signed(batch_floor - e) < 0) batch_floor <= e; if ($signed(read_floor - e) < 0) read_floor <= e; if ($signed(ingress_ready - e) < 0) ingress_ready <= e; + if ($signed(ab_prer - e) < 0) ab_prer <= e; + if ($signed(ab_actrc - e) < 0) ab_actrc <= e; + if ($signed(ab_floor - e) < 0) ab_floor <= e; d <= d + 16'd1; endtask @@ -613,7 +641,9 @@ module foldsim_mem_bank #( batch_floor <= ($signed(frontier + 16'd1 - e) > 0) ? frontier + 16'd1 : e; if (to_read) read_floor <= batch_last + 16'd1; - if (sel_any) begin + if (ref_now) begin + state <= S_RBANK; + end else if (sel_any) begin state <= S_BANK; end else begin end_clock(); @@ -622,6 +652,7 @@ module foldsim_mem_bank #( end S_BANK: begin // The request at the head of the buffer the channel serves. + op_ref <= 1'b0; op_w <= wmode; op_tag <= wmode ? wb_tag[wb_head] : rb_tag[rb_head]; op_bank <= wmode ? wb_bank[wb_head] : rb_bank[rb_head]; @@ -646,7 +677,8 @@ module foldsim_mem_bank #( // The search's lower bound, the first clock it may place at. case (kind) K_PRE: s_lb <= smax(16'sd0, r_prer); - K_ACT: s_lb <= smax(op_conf ? c_pre + k_rp : 16'sd0, r_act + k_rc); + K_ACT: s_lb <= smax(smax(op_conf ? c_pre + k_rp : 16'sd0, r_act + k_rc), rel(ab_floor)); + K_REF: s_lb <= smax(op_conf ? c_pre + k_rp : 16'sd0, r_act + k_rc); K_C0: s_lb <= smax(smax(16'sd0, r_floor), op_hit ? r_colr : c_act + (op_w ? k_rcdwr : k_rcdrd)); default: s_lb <= c_col0 + k_ccdl1; @@ -681,7 +713,8 @@ module foldsim_mem_bank #( if (sb_valid) begin if (b_cand != 0) begin case (kind) - K_PRE: begin c_pre <= b_rel; kind <= K_ACT; state <= S_SETUP; end + K_PRE: begin c_pre <= b_rel; kind <= op_ref ? K_REF : K_ACT; state <= S_SETUP; end + K_REF: begin c_ref <= b_rel; state <= S_CHECK; end K_ACT: begin c_act <= b_rel; kind <= K_C0; state <= S_SETUP; end K_C0: begin c_col0 <= b_rel; @@ -712,6 +745,12 @@ module foldsim_mem_bank #( ck_tcol <= lastcol + (op_w ? k_reach_w : k_reach_r); ck_tdep <= op_w ? -16'sd16384 : lastcol + k_dep; ck_tact <= op_hit ? -16'sd16384 : c_act + k_reach_a; + // A refresh's commands are within the horizon by its search. + if (op_ref) begin + ck_tcol <= -16'sd16384; + ck_tdep <= -16'sd16384; + ck_tact <= -16'sd16384; + end end state <= S_CHECK2; end @@ -724,10 +763,10 @@ module foldsim_mem_bank #( cm_hi[m][k] <= cx_hi[m][k]; end end - cw_row <= op_hit ? bt_row[op_bank] : op_row; - cw_act <= op_hit ? bt_act[op_bank] : e + c_act; - cw_colr <= e + ck_colr; - cw_prer <= e + ck_prer; + cw_row <= (op_hit | op_ref) ? bt_row[op_bank] : op_row; + cw_act <= op_ref ? e + c_ref + k_rfc - k_rc : (op_hit ? bt_act[op_bank] : e + c_act); + cw_colr <= op_ref ? bt_colr[op_bank] : e + ck_colr; + cw_prer <= op_ref ? bt_prer[op_bank] : e + ck_prer; begin : check2_ reg ok; ok = (ck_tcol < 16'(HE)) & (ck_tdep < 16'(HE)) & (ck_tact < 16'(HE)); @@ -736,13 +775,32 @@ module foldsim_mem_bank #( end end S_COMMIT: begin - if (cm_start) begin + if (cm_start & op_ref) begin + // A refresh closes its bank (every bank), and the next is due. + if (cfg_refresh[1]) begin + bt_open <= '0; + ab_open <= 1'b0; + ab_floor <= e + c_ref + k_rfc; + ab_actrc <= e + c_ref + k_rfc; + end else begin + bt_open[op_bank] <= 1'b0; + bt_fresh[op_bank] <= 1'b0; + ref_bank <= (ref_bank + 5'd1) & k_bmask[4:0]; + end + ref_due <= ref_due + cfg_refi; + stat_refreshes <= stat_refreshes + 64'd1; + end else if (cm_start) begin // The bank (its times through the table's port), the // controller, the read's departure, and a column in clock d+1 // for the clock's end. if (~op_hit) bt_open[op_bank] <= 1'b1; bt_fresh[op_bank] <= 1'b0; + ab_open <= 1'b1; + if ($signed(cw_prer - ab_prer) > 0) + ab_prer <= cw_prer; + if (~op_hit && $signed(e + c_act + k_rc - ab_actrc) > 0) + ab_actrc <= e + c_act + k_rc; if (op_w) begin wq_cnt <= wq_cnt + 8'(NB); wb_head <= wb_head + 1'b1; @@ -773,6 +831,30 @@ module foldsim_mem_bank #( end_clock(); state <= S_TICK; end + S_RBANK: begin + // A refresh: the bank in turn. + op_ref <= 1'b1; + op_w <= 1'b0; + op_bank <= ref_bank; + state <= S_RBANK2; + end + S_RBANK2: begin + // Precharged first if open, then REF: against the bank, or the + // channel's latest times for all banks. + op_hit <= 1'b0; + if (cfg_refresh[1]) begin + op_conf <= ab_open; + r_prer <= rel(ab_prer); + r_act <= rel(ab_actrc) - k_rc; + kind <= ab_open ? K_PRE : K_REF; + end else begin + op_conf <= bt_open[op_bank]; + r_prer <= bt_fresh[op_bank] ? -16'sd1 : rel(bt_prer[op_bank]); + r_act <= bt_fresh[op_bank] ? -16'sd16384 : rel(bt_act[op_bank]); + kind <= bt_open[op_bank] ? K_PRE : K_REF; + end + state <= S_SETUP; + end S_WALK: begin // Reads that are ready in the next target cycle. if (rf_ready != rf_count && $signed(rf_cyc[rf_scan] - tcyc1[23:0]) <= 0) begin @@ -831,9 +913,17 @@ module foldsim_mem_bank #( bt_fresh <= '1; scrub_b <= '0; bm_clear_all <= 1'b1; + ref_due <= cfg_refi; + ref_bank <= '0; + ab_open <= 1'b0; + ab_prer <= '0; + ab_actrc <= '0; + ab_floor <= '0; end else begin tcyc <= tcyc1; tcyc1 <= tcyc1 + 32'd1; + if (~(room & slot_free)) + stat_full <= stat_full + 64'd1; // R and B, then what is ready next cycle. begin : release_ reg r_load, b_load; @@ -893,6 +983,8 @@ module foldsim_mem_bank #( stat_misses <= '0; stat_conflicts <= '0; stat_latency <= '0; + stat_refreshes <= '0; + stat_full <= '0; lat_ar <= 1'b0; lat_r <= 1'b0; rfp_valid <= 1'b0; @@ -955,6 +1047,16 @@ module foldsim_mem_bank #( cx_lo[B_FAP][0] = clamp0(c_act - k_rrds + 16'sd1); cx_hi[B_FAP][0] = c_act + k_rrds - 16'sd1; cx_v[B_FAG][0] = ~op_hit; cx_lo[B_FAG][0] = clamp0(c_act - k_rrdl + 16'sd1); cx_hi[B_FAG][0] = c_act + k_rrdl - 16'sd1; + // A refresh: its precharge, if any, and REF, on the row-command bus. + if (op_ref) begin + for (integer m = 0; m < NBM; ++m) + for (integer k = 0; k < 2; ++k) + cx_v[m][k] = 1'b0; + cx_v[B_CMDR][0] = op_conf; cx_lo[B_CMDR][0] = c_pre; cx_hi[B_CMDR][0] = c_pre; + cx_v[B_CMDR][1] = 1'b1; cx_lo[B_CMDR][1] = c_ref; cx_hi[B_CMDR][1] = c_ref; + cx_v[B_ROW][0] = op_conf; cx_lo[B_ROW][0] = c_pre; cx_hi[B_ROW][0] = c_pre; + cx_v[B_ROW][1] = 1'b1; cx_lo[B_ROW][1] = c_ref; cx_hi[B_ROW][1] = c_ref; + end for (integer m = 0; m < NBM; ++m) for (integer k = 0; k < 2; ++k) if (cx_hi[m][k] < cx_lo[m][k]) diff --git a/hw/rtl/afu/foldsim/foldsim_ports.vh b/hw/rtl/afu/foldsim/foldsim_ports.vh new file mode 100644 index 0000000000..c4c8d90819 --- /dev/null +++ b/hw/rtl/afu/foldsim/foldsim_ports.vh @@ -0,0 +1,34 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The kernel's memory ports carry their index in their names, so they are +// listed by FOLDSIM_REPEAT, as VX_platform.vh's MP_REPEAT lists the native +// shells' ports. + +`ifndef FOLDSIM_PORTS_VH +`define FOLDSIM_PORTS_VH + +`define FOLDSIM_REPEAT(n,f,s) `_FOLDSIM_REPEAT_``n(f,s) +`define _FOLDSIM_REPEAT_1(f,s) `f(0) +`define _FOLDSIM_REPEAT_2(f,s) `_FOLDSIM_REPEAT_1(f,s) `s `f(1) +`define _FOLDSIM_REPEAT_3(f,s) `_FOLDSIM_REPEAT_2(f,s) `s `f(2) +`define _FOLDSIM_REPEAT_4(f,s) `_FOLDSIM_REPEAT_3(f,s) `s `f(3) +`define _FOLDSIM_REPEAT_5(f,s) `_FOLDSIM_REPEAT_4(f,s) `s `f(4) +`define _FOLDSIM_REPEAT_6(f,s) `_FOLDSIM_REPEAT_5(f,s) `s `f(5) +`define _FOLDSIM_REPEAT_7(f,s) `_FOLDSIM_REPEAT_6(f,s) `s `f(6) +`define _FOLDSIM_REPEAT_8(f,s) `_FOLDSIM_REPEAT_7(f,s) `s `f(7) + +`define FOLDSIM_COMMA , +`define FOLDSIM_SEMI ; + +`endif // FOLDSIM_PORTS_VH diff --git a/hw/rtl/afu/foldsim/foldsim_regs.vh b/hw/rtl/afu/foldsim/foldsim_regs.vh index 1f84e3db39..ef5edee3e5 100644 --- a/hw/rtl/afu/foldsim/foldsim_regs.vh +++ b/hw/rtl/afu/foldsim/foldsim_regs.vh @@ -22,7 +22,7 @@ `define FOLDSIM_REGS_VH `define FOLDSIM_ID 32'h464F4C44 -`define FOLDSIM_VERSION 32'd3 +`define FOLDSIM_VERSION 32'd4 `define FOLDSIM_REG_ID 12'h000 `define FOLDSIM_REG_VERSION 12'h004 @@ -53,9 +53,12 @@ `define FOLDSIM_REG_MEM_QUEUE 12'h0E8 `define FOLDSIM_REG_MEM_GEOM 12'h0EC `define FOLDSIM_REG_MEM_WMARK 12'h0F0 +`define FOLDSIM_REG_MEM_REFI 12'h0F4 +`define FOLDSIM_REG_MEM_RFC 12'h0F8 // Per class c: the host cycles a thread of it finished its target cycle. `define FOLDSIM_REG_FINISH_BASE 12'h100 -// The bank model's DRAM timings, a word each, and its counters, 64-bit each. +// The bank model's DRAM timings, a word each, and per memory port p its +// counters, 64-bit each, from FOLDSIM_REG_MEM_STATS + 0x40 * p. `define FOLDSIM_REG_MEM_TIMING 12'h800 `define FOLDSIM_REG_MEM_STATS 12'hC00 diff --git a/hw/rtl/afu/foldsim/target/foldsim_target.sv b/hw/rtl/afu/foldsim/target/foldsim_target.sv index aa82553683..271d97ad3b 100644 --- a/hw/rtl/afu/foldsim/target/foldsim_target.sv +++ b/hw/rtl/afu/foldsim/target/foldsim_target.sv @@ -17,7 +17,8 @@ // scalar ports, the boundary foldsim channelizes. // // Vortex_axi declares its AXI channels as SystemVerilog unpacked arrays indexed -// by bank; each bank is broken out into discrete m_axi_mem__* signals. The +// by bank; each of its VX_CFG_PLATFORM_MEMORY_NUM_BANKS banks is broken out into +// discrete m_axi_mem__* signals. The // port shape and the array conversion are the same ones the XRT AFU uses, taken // from its macros so the two cannot drift apart. // @@ -28,12 +29,12 @@ module foldsim_target import VX_gpu_pkg::*; #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = `VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8, parameter C_M_AXI_MEM_ADDR_WIDTH = 64, - parameter C_M_AXI_MEM_NUM_BANKS = 1 + parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS ) ( input wire clk, input wire reset, - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), + `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), // DCR write request input wire dcr_req_valid, @@ -81,7 +82,7 @@ module foldsim_target import VX_gpu_pkg::*; #( wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS]; wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS]; - `AXI_MEM_TO_ARRAY (0); + `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_TO_ARRAY, MP_SEMI); Vortex_axi #( .AXI_DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), diff --git a/hw/scripts/gen_platform.py b/hw/scripts/gen_platform.py index 579906267d..5645152c67 100644 --- a/hw/scripts/gen_platform.py +++ b/hw/scripts/gen_platform.py @@ -219,8 +219,10 @@ def emit_mk(c: Card) -> str: def emit_cfg(c: Card, mem_tag: str, host_tag: str, kernel_mhz: int, - kernel: str = "vortex_afu") -> str: - """The linker config of `kernel`; one with no host-memory port has no host tag.""" + kernel: str = "vortex_afu", mem_ports: int = 1) -> str: + """The linker config of `kernel`; one with no host-memory port has no host tag. + A kernel with several device-memory masters (m_axi_mem_0..mem_ports-1) binds + each to the memory tag: the linker gives each its own way into it.""" if c.flow != "aved": raise DescriptorError(f"card '{c.name}' is an {c.flow} card; cfg is for the aved flow") if host_tag: @@ -233,7 +235,7 @@ def emit_cfg(c: Card, mem_tag: str, host_tag: str, kernel_mhz: int, "# ethernet stack the AFU has no use for.", "shell=compute", f"nk={kernel}:1:{inst}", - f"sp={inst}.m_axi_mem_0:{mem_tag}", + ] + [f"sp={inst}.m_axi_mem_{i}:{mem_tag}" for i in range(mem_ports) ] + ([f"sp={inst}.m_axi_host:{host_tag}"] if host_tag else []) + [ "", "# Kernel clock, in Hz rather than via the linker's --clock-hz flag, whose", @@ -310,6 +312,8 @@ def main(argv: Optional[List[str]] = None) -> int: ap.add_argument("--kernel-mhz", type=int, default=0) ap.add_argument("--kernel", default="vortex_afu", help="kernel the cfg links (aved flow)") ap.add_argument("--no-host", action="store_true", help="the kernel has no host-memory port") + ap.add_argument("--mem-ports", type=int, default=1, + help="device-memory masters of the kernel the cfg links (aved flow)") ap.add_argument("--platform-dir", default=PLATFORM_DIR) ap.add_argument("-o", "--output", default="-") a = ap.parse_args(argv) @@ -324,7 +328,7 @@ def main(argv: Optional[List[str]] = None) -> int: elif a.emit == "cfg": out = emit_cfg(c, a.mem_tag or c.mem_tag, "" if a.no_host else (a.host_tag or c.host_tag), - a.kernel_mhz or c.kernel_mhz, a.kernel) + a.kernel_mhz or c.kernel_mhz, a.kernel, a.mem_ports) elif a.emit == "firesim-mk": out = emit_firesim_mk(c) elif a.emit == "scala": diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index 0252c78de0..9ca5fe302f 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -65,21 +65,40 @@ RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm \ -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(FOLDSIM_RTL_DIR)/target SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG -# One memory port, as the memory endpoint models; 4 AXI id bits, what Vortex -# drives with its tag buffer engaged. The FPU type is the configuration's, as +# One memory port per memory bank of the configuration (one unless CONFIGS +# sets VX_CFG_PLATFORM_MEMORY_NUM_BANKS), each its own memory endpoint and DRAM +# channel. AXI ids are what Vortex drives with its tag buffer engaged: the +# buffer's 4 index bits, and above them the index of the GPU's memory port the +# read came from, of which there are at most as many as memory ports. The +# FPU type is the configuration's, as # in the native Vivado flows: DSP selects the Xilinx floating-point cores, which # the simulator keeps as instances: their netlists are encrypted, so their # state cannot be threaded, and each is simulated by one unthreaded copy of # itself and a per-thread store of its results (foldsim --fold-pipelines, by # the pipeline contract xilinx_ip_stubs.sv declares). -DVX_CFG_FPU_TYPE_STD in # CONFIGS selects the soft FPU instead. -PLATFORM_DEFINES := -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DPLATFORM_MEMORY_ID_WIDTH=4 +MEM_PORTS := $(or $(lastword $(patsubst -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=%,%, \ + $(filter -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=%,$(CONFIGS)))),1) +ifeq ($(filter 1 2 4 8,$(MEM_PORTS)),) +$(error the foldsim kernel has 1, 2, 4 or 8 memory ports (VX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(MEM_PORTS))) +endif +MEM_ID_W := $(shell python3 -c 'print(4 + ($(MEM_PORTS) - 1).bit_length())') +PLATFORM_DEFINES := -DPLATFORM_MEMORY_ID_WIDTH=$(MEM_ID_W) +ifeq ($(MEM_PORTS),1) +PLATFORM_DEFINES += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 +endif ifdef PERF PLATFORM_DEFINES += -DPERF_ENABLE endif XCONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py \ --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN) $(SYNTH_DEFINES) $(PLATFORM_DEFINES)') TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) +# The shift that takes an address to its memory port's channel: the ports +# interleave by block, or (INTERLEAVE=0) split the space, where every port's +# channel holds the address as it is. +cfg_value = $(lastword $(patsubst -D$(1)=%,%,$(filter -D$(1)=%,$(XCONFIGS)))) +MEM_CH_SHIFT := $(if $(filter 1,$(call cfg_value,VX_CFG_PLATFORM_MEMORY_INTERLEAVE)), \ + $(shell python3 -c 'print(($(MEM_PORTS) - 1).bit_length())'),0) # The extensions' packages and include paths, as every synthesis flow finds # them. include $(VORTEX_HOME)/hw/syn/extensions.mk @@ -102,6 +121,7 @@ SRCS_LIST := $(BUILD_DIR)/$(SRCS_NAME).f LIFT_LIST := $(BUILD_DIR)/lift/srcs.f TARGET_SRCS := $(if $(strip $(FOLDSIM_LIFT)),$(LIFT_LIST),$(SRCS_LIST)) GEN_SV := $(GEN_DIR)/foldsim_sim.sv +KERNEL_VH := $(GEN_DIR)/foldsim_kernel.vh KERNEL_SRCS := $(BUILD_DIR)/kernel_srcs.f IP_DIR := $(BUILD_DIR)/foldsim_afu_ip CFG := $(BUILD_DIR)/config.cfg @@ -147,7 +167,7 @@ $(LIFT_LIST): $(SRCS_LIST) $(SRC_DIR)/lift_boundaries.py $(CLASSES_STAMP) $(LIFT_PYTHON) $(SRC_DIR)/lift_boundaries.py --srcs $(SRCS_LIST) --top $(TARGET_TOP) \ --out $(BUILD_DIR)/lift $(foreach u,$(FOLDSIM_LIFT),--unit '$(u)') -generate: $(GEN_SV) +generate: $(GEN_SV) $(KERNEL_VH) # The compiler is the driver and the tools it runs: foldsim-opt beside it and # CIRCT's, so a rebuilt tool regenerates the simulator. FOLDSIM_TOOLS := $(FOLDSIM) $(wildcard $(dir $(FOLDSIM))foldsim-opt \ @@ -156,7 +176,16 @@ $(GEN_SV): $(TARGET_SRCS) $(FOLDSIM_TOOLS) $(CLASSES_STAMP) $(FOLDSIM) --srcs $(TARGET_SRCS) --top $(TARGET_TOP) --out $(GEN_DIR) \ $(if $(CIRCT_BIN),--circt-bin $(CIRCT_BIN)) $(FOLDSIM_FLAGS) $(FOLDSIM_CLASSES) -$(KERNEL_SRCS): $(GEN_SV) +# What the kernel's top needs of the configuration: its memory ports and their +# id width. +$(KERNEL_VH): FORCE + @mkdir -p $(@D) + @printf '%s\n' '// Generated by hw/syn/foldsim. Do not edit.' \ + '`define FOLDSIM_MEM_PORTS $(MEM_PORTS)' '`define FOLDSIM_MEM_CH_SHIFT $(strip $(MEM_CH_SHIFT))' \ + '`define FOLDSIM_MEM_ID_W $(MEM_ID_W)' > $@.tmp + @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ + +$(KERNEL_SRCS): $(GEN_SV) $(KERNEL_VH) @{ echo "+incdir+$(FOLDSIM_RTL_DIR)"; echo "+incdir+$(GEN_DIR)"; \ echo "$(GEN_SV)"; \ for f in foldsim_token.sv foldsim_mem_bmap.sv foldsim_mem_bank.sv foldsim_mem.sv foldsim_ctrl.sv \ @@ -174,7 +203,7 @@ $(OOC_XDC): FORCE package-ip: $(IP_DIR)/component.xml # The kernel is packaged with the floating-point cores the simulator # instantiates, generated as the aved flow generates them. -$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(OOC_XDC) $(wildcard $(FOLDSIM_RTL_DIR)/*) $(SCRIPT_DIR)/xilinx_ip_gen.tcl +$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(KERNEL_VH) $(OOC_XDC) $(wildcard $(FOLDSIM_RTL_DIR)/*) $(SCRIPT_DIR)/xilinx_ip_gen.tcl cd $(BUILD_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -nojournal -nolog \ -source $(VORTEX_HOME)/hw/syn/xilinx/aved/package_ip.tcl \ -tclargs foldsim_afu $(KERNEL_SRCS) . $(DEVICE) ooc-xdc=$(OOC_XDC) \ @@ -182,7 +211,7 @@ $(IP_DIR)/component.xml: $(KERNEL_SRCS) $(OOC_XDC) $(wildcard $(FOLDSIM_RTL_DIR) $(CFG): FORCE @python3 $(SCRIPT_DIR)/gen_platform.py --flow=aved --device=$(DEVICE) --emit=cfg --kernel=foldsim_afu \ - --no-host --kernel-mhz=$(FOLDSIM_FREQ) -o $@.tmp + --no-host --mem-ports=$(MEM_PORTS) --kernel-mhz=$(FOLDSIM_FREQ) -o $@.tmp @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ FORCE: diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 23baf30e94..29aed69f98 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -22,6 +22,7 @@ all: $(MAKE) -C mm_axi_arb $(MAKE) -C foldsim_mem $(MAKE) -C foldsim_mem_staged + $(MAKE) -C foldsim_mem_ch2 $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma $(MAKE) -C cp_llc_port @@ -59,6 +60,7 @@ run: $(MAKE) -C mm_axi_arb run $(MAKE) -C foldsim_mem run $(MAKE) -C foldsim_mem_staged run + $(MAKE) -C foldsim_mem_ch2 run $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run $(MAKE) -C cp_llc_port run @@ -213,6 +215,7 @@ clean: $(MAKE) -C cp_core clean $(MAKE) -C foldsim_mem clean $(MAKE) -C foldsim_mem_staged clean + $(MAKE) -C foldsim_mem_ch2 clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean $(MAKE) -C fsqrt_unit clean diff --git a/hw/unittest/foldsim_mem/main.cpp b/hw/unittest/foldsim_mem/main.cpp index 4fcc87d338..c0ad6c2af8 100644 --- a/hw/unittest/foldsim_mem/main.cpp +++ b/hw/unittest/foldsim_mem/main.cpp @@ -58,6 +58,15 @@ constexpr uint64_t kBurstAddr = 0x3ff7e000ull; // offered only as a burst using Line = std::array; +// The endpoint's CH_SHIFT: the port is one of 2^shift, and the bank model +// sees the block within the port's channel. +#ifndef FOLDSIM_MEM_CH_SHIFT +#define FOLDSIM_MEM_CH_SHIFT 0 +#endif +uint64_t channel_addr(uint64_t a) { + return ((a >> (6 + FOLDSIM_MEM_CH_SHIFT)) << 6) | (a & 63); +} + // The traffic a setup makes: the addresses its requests take. enum class Traffic { Small, // 64 lines, 4 KB: row hits @@ -79,6 +88,9 @@ struct Setup { Traffic traffic; double p_read, p_write, p_ready; uint64_t cycles; + // Refresh, and its interval and busy time if not the standard's (0). + DramBankModel::Refresh refresh = DramBankModel::Refresh::Off; + uint32_t refi = 0, rfc = 0; }; const Setup kSetups[] = { @@ -110,6 +122,16 @@ const Setup kSetups[] = { Traffic::OneBank, 0.7, 0.3, 0.8, 4000}, {"bank_v100", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 1530, 1752, 0, 0, 32, 32, 26, 7, Traffic::Rows, 0.8, 0.2, 0.9, 4000}, + {"ref_bank", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, + Traffic::Random, 0.5, 0.3, 0.8, 4000, DramBankModel::Refresh::PerBank}, + {"ref_bank_s", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 1410, 3038, 190, 257, 32, 32, 26, 7, + Traffic::Stream, 0.8, 0.4, 0.9, 4000, DramBankModel::Refresh::PerBank}, + {"ref_all", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + Traffic::Rows, 0.5, 0.3, 0.8, 4000, DramBankModel::Refresh::AllBank}, + {"ref_all_fast", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 3, 0, 32, 32, 26, 7, + Traffic::Random, 0.6, 0.3, 0.9, 4000, DramBankModel::Refresh::AllBank, 600, 0}, + {"ref_bank_fast", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + Traffic::Rows, 0.8, 0.2, 0.9, 4000, DramBankModel::Refresh::PerBank, 60, 0}, }; const Setup* setup = nullptr; @@ -148,7 +170,9 @@ bool golden_valid = true; // the pipe models no bursts: void until a reset DramBankAxi::Config bank_config(const Setup& s) { DramBankAxi::Config c; - c.model = DramBankModel::Config::make(s.type, s.data_rate, 64); + c.model = DramBankModel::Config::make(s.type, s.data_rate, 64, s.refresh); + if (s.refi) c.model.refi = s.refi; + if (s.rfc) c.model.rfc = s.rfc; c.model.read_queue = s.rq; c.model.write_queue = s.wq; c.model.wr_high = s.wr_high; @@ -637,11 +661,11 @@ void golden_cycle() { DramBankAxi::Inputs in; in.ar_valid = m.ar_v; in.ar_id = m.ar.id; - in.ar_addr = m.ar.addr; + in.ar_addr = channel_addr(m.ar.addr); in.ar_len = m.ar_len; in.aw_valid = m.aw_v; in.aw_id = m.aw.id; - in.aw_addr = m.aw.addr; + in.aw_addr = channel_addr(m.aw.addr); in.aw_len = m.aw_len; in.w_valid = m.w_v; in.w_last = m.w_last; @@ -781,6 +805,10 @@ void configure() { dut->cfg_wr_low = s.wr_low; dut->cfg_geom = g.log2_pcs | (g.log2_bgs << 2) | (g.log2_banks << 4) | (g.log2_row_bursts << 6) | (g.log2_visit << 10) | (uint32_t(g.two_clock_act) << 12); + dut->cfg_refresh = (cfg.model.refresh != DramBankModel::Refresh::Off) + | (uint32_t(cfg.model.refresh == DramBankModel::Refresh::AllBank) << 1); + dut->cfg_refi = cfg.model.refi; + dut->cfg_rfc = cfg.model.rfc; const uint32_t v[18] = {t.BL, t.CCDS, t.CCDL, t.CL, t.CWL, t.RCDRD, t.RCDWR, t.RP, t.RAS, t.RC, t.RTP, t.WR, t.RRDS, t.RRDL, t.FAW, t.WTRS, t.WTRL, g.rtw_extra}; for (int w = 0; w < 7; ++w) @@ -854,10 +882,11 @@ int run_setup(const Setup& s) { std::string extra; if (golden_bank) { const auto& st = golden_bank->model().stats(); - char buf[200]; - snprintf(buf, sizeof(buf), "; model since last reset: %lu reads, %lu writes, %lu hits, %lu misses, %lu conflicts, %lu deferred", + char buf[256]; + snprintf(buf, sizeof(buf), "; model since last reset: %lu reads, %lu writes, %lu hits, %lu misses, %lu conflicts, %lu refreshes, %lu deferred", (unsigned long)st.reads, (unsigned long)st.writes, (unsigned long)st.hits, - (unsigned long)st.misses, (unsigned long)st.conflicts, (unsigned long)st.deferred); + (unsigned long)st.misses, (unsigned long)st.conflicts, (unsigned long)st.refreshes, + (unsigned long)st.deferred); extra = buf; } printf("%-12s target cycles %lu, host cycles %lu, %lu compared; %lu reads, %lu writes; " diff --git a/hw/unittest/foldsim_mem_ch2/Makefile b/hw/unittest/foldsim_mem_ch2/Makefile new file mode 100644 index 0000000000..3509bb338e --- /dev/null +++ b/hw/unittest/foldsim_mem_ch2/Makefile @@ -0,0 +1,28 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +# hw/unittest/foldsim_mem's test, on the kernel's endpoint (staged token ends) +# as one of two ports: the bank model sees the port's channel. +PROJECT := foldsim_mem + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/foldsim_mem + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) +CXXFLAGS += -DFOLDSIM_MEM_CH_SHIFT=1 + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_INCLUDE := -I$(FOLDSIM_RTL_DIR) +PARAMS += -GSTAGED=1 -GCH_SHIFT=1 + +# The token modules are not named after their file. +VL_FLAGS += $(FOLDSIM_RTL_DIR)/foldsim_token.sv + +TOP := foldsim_mem + +include ../common.mk diff --git a/sim/common/dram_bank_model.h b/sim/common/dram_bank_model.h index 26f8bf852b..e678152e85 100644 --- a/sim/common/dram_bank_model.h +++ b/sim/common/dram_bank_model.h @@ -21,6 +21,7 @@ #include #include #include +#include #include namespace vortex { @@ -44,14 +45,26 @@ namespace vortex { // Reservations reach at most `horizon` clocks ahead: a request that would // reach further waits. // +// Refresh, when on, is an operation of its own: every `refi` clocks it takes +// the clock's turn to reserve. A per-bank refresh closes the next bank in +// turn -- a precharge once the bank's commands allow it, then REF -- and keeps +// it from activating for `rfc` clocks; an all-bank refresh does the same to +// every bank with one precharge and one REF. A refresh that would reach past +// the horizon waits, and the requests with it. +// // This is the normative model: foldsim's memory endpoint // (hw/rtl/afu/foldsim/foldsim_mem_bank.sv) implements the same decisions, and // the faces below put it behind DramSim's interface and behind an AXI4 port. class DramBankModel { public: + enum class Refresh { Off, PerBank, AllBank }; + struct Config { dram_timing::geometry_t geom; dram_timing::clocks_t t; + Refresh refresh = Refresh::Off; // dram_sim models none + uint32_t refi = 0; // clocks from one refresh operation to the next + uint32_t rfc = 0; // clocks a refreshed bank may not activate uint32_t read_queue = 32; // reserved read bursts whose column has not issued uint32_t write_queue = 32; // the same for writes // Write bursts buffered or reserved and not issued at which the channel @@ -64,7 +77,8 @@ class DramBankModel { uint32_t horizon = 448; uint32_t bursts = 2; // 32 B bursts per request - static Config make(dram_timing::Standard standard, uint32_t data_rate, uint32_t block_size) { + static Config make(dram_timing::Standard standard, uint32_t data_rate, uint32_t block_size, + Refresh refresh = Refresh::Off) { Config c; c.geom = dram_timing::geometry(standard); c.t = dram_timing::clocks(standard, data_rate); @@ -72,9 +86,25 @@ class DramBankModel { if (c.bursts > (1u << c.geom.log2_visit)) { throw std::invalid_argument("dram_bank_model: a request spans more than one row visit"); } + c.set_refresh(refresh, dram_timing::refresh(standard, data_rate)); return c; } + // Per bank, the banks take turns within tREFI; all-bank, the channel + // refreshes once per tREFI. + void set_refresh(Refresh mode, const dram_timing::refresh_t& r) { + refresh = mode; + refi = (mode == Refresh::Off) ? 0 : (mode == Refresh::PerBank) ? r.REFI / this->num_banks() : r.REFI; + rfc = (mode == Refresh::Off) ? 0 : (mode == Refresh::PerBank) ? r.RFCPB : r.RFC; + } + + static Refresh parse_refresh(const std::string& name) { + if (name.empty() || name == "off") return Refresh::Off; + if (name == "bank") return Refresh::PerBank; + if (name == "all") return Refresh::AllBank; + throw std::invalid_argument("dram_bank_model: refresh must be off, bank or all (got '" + name + "')"); + } + // Column-to-column spacing on one pseudo-channel, and within one bank // group, by the directions of the earlier and the later command. uint32_t rtw() const { @@ -110,8 +140,8 @@ class DramBankModel { }; struct Stats { - uint64_t reads = 0, writes = 0, hits = 0, misses = 0, conflicts = 0; - uint64_t deferred = 0; // clocks a request waited: it reached past the horizon + uint64_t reads = 0, writes = 0, hits = 0, misses = 0, conflicts = 0, refreshes = 0; + uint64_t deferred = 0; // clocks a request or refresh waited: it reached past the horizon }; explicit DramBankModel(const Config& cfg) : cfg_(cfg) { this->reset(); } @@ -144,6 +174,8 @@ class DramBankModel { batch_floor_ = 0; batch_last_ = 0; read_floor_ = 0; + ref_due_ = cfg_.refi; + ref_bank_ = 0; stats_ = Stats(); } @@ -214,7 +246,14 @@ class DramBankModel { read_floor_ = batch_last_ + 1; } auto& buf = write_mode_ ? wbuf_ : rbuf_; - if (!buf.empty()) { + if (cfg_.refresh != Refresh::Off && cfg_.refi != 0 && d + 1 >= ref_due_) { + if (this->refresh(d + 1, d + int64_t(cfg_.horizon))) { + ref_due_ += cfg_.refi; + ++stats_.refreshes; + } else { + ++stats_.deferred; + } + } else if (!buf.empty()) { if (this->reserve(buf.front(), d + 1, d + int64_t(cfg_.horizon), write_mode_ ? batch_floor_ : read_floor_)) { buf.pop_front(); @@ -420,6 +459,50 @@ class DramBankModel { return true; } + // A refresh operation: the bank in turn (all banks), precharged if open -- + // after every command reserved to it allows -- then REF, once the precharge + // and the last activate allow; until rfc after it the bank may not + // activate. Both commands take the row-command bus as a precharge does. + // False (and nothing changed) if they do not fit within the horizon. + bool refresh(int64_t e, int64_t end) { + const auto& T = cfg_.t; + const uint32_t nb = cfg_.num_banks(); + const bool all = (cfg_.refresh == Refresh::AllBank); + bool open = false; + int64_t pre_lb = e, act_lb = e; + for (uint32_t b = 0; b < nb; ++b) { + if (!all && b != ref_bank_) continue; + if (open_[b]) { + open = true; + pre_lb = std::max(pre_lb, pre_ready_[b]); + } + act_lb = std::max(act_lb, act_[b] + int64_t(T.RC)); + } + int64_t pre = kNone, lb = act_lb; + if (open) { + pre = this->place_pre(pre_lb, end); + if (pre == kNone) return false; + lb = std::max(lb, pre + int64_t(T.RP)); + } + const int64_t ref = this->place_pre(lb, end); + if (ref == kNone) return false; + + // Commit. + if (pre != kNone) { + cmd_.insert(pre); + rowbus_.insert(pre); + } + cmd_.insert(ref); + rowbus_.insert(ref); + for (uint32_t b = 0; b < nb; ++b) { + if (!all && b != ref_bank_) continue; + open_[b] = false; + act_[b] = ref + int64_t(cfg_.rfc) - int64_t(T.RC); + } + if (!all) ref_bank_ = (ref_bank_ + 1) % nb; + return true; + } + // Drops what can no longer constrain a command at d+1 or later. void prune(int64_t d) { const int64_t keep_cols = d + 1 - this->max_col_sep(); @@ -446,6 +529,8 @@ class DramBankModel { std::deque rbuf_, wbuf_; bool write_mode_; int64_t frontier_, batch_floor_, batch_last_, read_floor_; + int64_t ref_due_; + uint32_t ref_bank_; Stats stats_; }; @@ -467,6 +552,7 @@ class DramBankSim { uint32_t data_rate; // MT/s uint32_t latency; // core cycles added after a read's data departs uint32_t queue_size; // requests in flight per channel; 0 = unbounded + DramBankModel::Refresh refresh = DramBankModel::Refresh::Off; }; explicit DramBankSim(const Config& cfg) : DramBankSim(cfg, DramBankModel::Config()) {} @@ -474,7 +560,7 @@ class DramBankSim { // `policy` gives the controller's queues, horizon and write policy. DramBankSim(const Config& cfg, const DramBankModel::Config& policy) : cfg_(cfg) - , model_cfg_(DramBankModel::Config::make(cfg.type, cfg.data_rate, cfg.block_size)) { + , model_cfg_(DramBankModel::Config::make(cfg.type, cfg.data_rate, cfg.block_size, cfg.refresh)) { model_cfg_.read_queue = policy.read_queue; model_cfg_.write_queue = policy.write_queue; model_cfg_.wr_high = policy.wr_high; @@ -641,6 +727,7 @@ class DramBankAxi { acks_.clear(); cur_r_valid_ = false; cur_b_valid_ = false; + full_cycles_ = 0; } bool ar_ready() const { return !error_ && this->room() && slots_used_ < cfg_.read_slots; } @@ -653,6 +740,8 @@ class DramBankAxi { bool b_valid() const { return cur_b_valid_; } uint32_t b_id() const { return cur_b_id_; } bool error() const { return error_; } + // Target cycles the port could take no read: no slot, or no room. + uint64_t full_cycles() const { return full_cycles_; } // The slot a read accepted this cycle takes: the lowest free once R has // taken its read, if it does. uint32_t ar_slot(bool r_ready) const { @@ -676,6 +765,9 @@ class DramBankAxi { const bool w_fire = w_hs && in.w_last; const bool r_fire = cur_r_valid_ && in.r_ready; const bool b_fire = cur_b_valid_ && in.b_ready; + if (!(this->room() && slots_used_ < cfg_.read_slots)) { + ++full_cycles_; + } // This cycle's DRAM clocks, on what was accepted before it. credit_ += cfg_.data_rate; @@ -775,6 +867,7 @@ class DramBankAxi { uint32_t uncounted_acks_ = 0; bool cur_r_valid_, cur_b_valid_; uint32_t cur_r_id_ = 0, cur_r_slot_ = 0, cur_b_id_ = 0; + uint64_t full_cycles_ = 0; }; } // namespace vortex diff --git a/sim/common/dram_sim.cpp b/sim/common/dram_sim.cpp index dbb8a05c3c..4c019d38ce 100644 --- a/sim/common/dram_sim.cpp +++ b/sim/common/dram_sim.cpp @@ -71,7 +71,7 @@ class BankDram : public DramSim::Impl { public: explicit BankDram(const DramSim::Config& c) : sim_(DramBankSim::Config{c.type, c.num_channels, c.block_size, c.core_clock, c.data_rate, - c.latency, c.queue_size}) {} + c.latency, c.queue_size, refresh_mode()}) {} void reset() override { sim_.reset(); } void tick() override { sim_.tick(); } bool can_accept(uint32_t channel) const override { return sim_.can_accept(channel); } @@ -81,6 +81,13 @@ class BankDram : public DramSim::Impl { } private: + // VORTEX_DRAM_REFRESH=off|bank|all: refresh, which Ramulator here does not + // model, per bank or all banks at once. + static DramBankModel::Refresh refresh_mode() { + const char* s = getenv("VORTEX_DRAM_REFRESH"); + return DramBankModel::Config::parse_refresh(s ? s : ""); + } + DramBankSim sim_; }; diff --git a/sim/foldsim/Makefile b/sim/foldsim/Makefile index d1f82490d3..c56e81be69 100644 --- a/sim/foldsim/Makefile +++ b/sim/foldsim/Makefile @@ -10,6 +10,8 @@ SRC_DIR := $(VORTEX_HOME)/sim/foldsim FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim WORK_DIR := $(DESTDIR)/foldsim_emul GEN_SV := $(WORK_DIR)/gen/foldsim_sim.sv +# The kernel's memory ports, as the image's flow derived them. +KERNEL_VH := $(WORK_DIR)/gen/foldsim_kernel.vh CONFIG_FILE := $(DESTDIR)/foldsim_emul_config.stamp @@ -54,7 +56,8 @@ $(GEN_SV): force $(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(SRC_DIR)/xilinx_fp_models.sv $(wildcard $(FOLDSIM_RTL_DIR)/*) $(CONFIG_FILE) $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(RTL_SRCS) --top-module $(TOP) \ - $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + $(SRCS) -CFLAGS '$(CXXFLAGS) -DFOLDSIM_MEM_PORTS='$$(sed -n 's/^`define FOLDSIM_MEM_PORTS //p' $(KERNEL_VH)) \ + -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ $(CONFIG_FILE): force @mkdir -p $(@D) diff --git a/sim/foldsim/foldsim_emul.cpp b/sim/foldsim/foldsim_emul.cpp index 45e0b509f0..570ccd527b 100644 --- a/sim/foldsim/foldsim_emul.cpp +++ b/sim/foldsim/foldsim_emul.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include using namespace vortex; @@ -43,6 +44,11 @@ uint64_t env_or(const char* name, uint64_t value) { constexpr uint64_t kPage = 4096; constexpr uint32_t kBeat = 64; +#ifndef FOLDSIM_MEM_PORTS +#define FOLDSIM_MEM_PORTS 1 +#endif +static_assert(FOLDSIM_MEM_PORTS >= 1 && FOLDSIM_MEM_PORTS <= 8, "the kernel has 1 to 8 memory ports"); + // Device memory: sparse, zero where never written. class memory_t { public: @@ -104,6 +110,7 @@ class foldsim_emul::Impl { reg_trace_ = fopen(path, "w"); context_ = std::make_unique(); top_ = std::make_unique(context_.get()); + bind_ports(); #ifdef VM_TRACE // FOLDSIM_EMUL_VCD=: a waveform of the kernel, in a build with --trace, // from kernel cycle FOLDSIM_EMUL_VCD_FROM on, FOLDSIM_EMUL_VCD_DEPTH levels @@ -197,6 +204,52 @@ class foldsim_emul::Impl { std::array data; }; + // One of the kernel's device-memory masters: its signals, and what memory + // has in flight on it. + struct port_t { + CData *awvalid, *awready, *wvalid, *wready, *arvalid, *arready; + CData *rvalid, *rready, *rlast, *rid, *rresp, *bvalid, *bready, *bid, *bresp; + QData *awaddr, *araddr, *wstrb; + VlWide<16> *wdata, *rdata; + std::deque reads; + std::deque aw; + std::deque> w; + std::deque writes; + }; + +#define FOLDSIM_PORT(i) \ + { \ + port_t& p = ports_[i]; \ + p.awvalid = &top_->m_axi_mem_##i##_awvalid; p.awready = &top_->m_axi_mem_##i##_awready; \ + p.wvalid = &top_->m_axi_mem_##i##_wvalid; p.wready = &top_->m_axi_mem_##i##_wready; \ + p.arvalid = &top_->m_axi_mem_##i##_arvalid; p.arready = &top_->m_axi_mem_##i##_arready; \ + p.rvalid = &top_->m_axi_mem_##i##_rvalid; p.rready = &top_->m_axi_mem_##i##_rready; \ + p.rlast = &top_->m_axi_mem_##i##_rlast; p.rid = &top_->m_axi_mem_##i##_rid; \ + p.rresp = &top_->m_axi_mem_##i##_rresp; p.bvalid = &top_->m_axi_mem_##i##_bvalid; \ + p.bready = &top_->m_axi_mem_##i##_bready; p.bid = &top_->m_axi_mem_##i##_bid; \ + p.bresp = &top_->m_axi_mem_##i##_bresp; p.awaddr = &top_->m_axi_mem_##i##_awaddr; \ + p.araddr = &top_->m_axi_mem_##i##_araddr; p.wstrb = &top_->m_axi_mem_##i##_wstrb; \ + p.wdata = &top_->m_axi_mem_##i##_wdata; p.rdata = &top_->m_axi_mem_##i##_rdata; \ + } + + void bind_ports() { + FOLDSIM_PORT(0) +#if FOLDSIM_MEM_PORTS > 1 + FOLDSIM_PORT(1) +#endif +#if FOLDSIM_MEM_PORTS > 2 + FOLDSIM_PORT(2) + FOLDSIM_PORT(3) +#endif +#if FOLDSIM_MEM_PORTS > 4 + FOLDSIM_PORT(4) + FOLDSIM_PORT(5) + FOLDSIM_PORT(6) + FOLDSIM_PORT(7) +#endif + } +#undef FOLDSIM_PORT + void idle_master() { top_->s_axi_control_awvalid = 0; top_->s_axi_control_wvalid = 0; @@ -216,43 +269,56 @@ class foldsim_emul::Impl { edge(); } - // The memory's side of the kernel's AXI4 port: every request accepted, and - // answered in order once its latency has passed. + // The memory's side of the kernel's AXI4 ports: every request accepted, and + // answered in order on its port once its latency has passed. void drive_memory() { - top_->m_axi_mem_0_awready = 1; - top_->m_axi_mem_0_wready = 1; - top_->m_axi_mem_0_arready = 1; - bool r = !reads_.empty() && reads_.front().at <= cycle_; - top_->m_axi_mem_0_rvalid = r; - if (r) - std::memcpy(top_->m_axi_mem_0_rdata.data(), reads_.front().data.data(), kBeat); - top_->m_axi_mem_0_rlast = 1; - top_->m_axi_mem_0_rid = 0; - top_->m_axi_mem_0_rresp = 0; - top_->m_axi_mem_0_bvalid = !writes_.empty() && writes_.front() <= cycle_; - top_->m_axi_mem_0_bid = 0; - top_->m_axi_mem_0_bresp = 0; + for (auto& p : ports_) { + *p.awready = 1; + *p.wready = 1; + *p.arready = 1; + bool r = !p.reads.empty() && p.reads.front().at <= cycle_; + *p.rvalid = r; + if (r) + std::memcpy(p.rdata->data(), p.reads.front().data.data(), kBeat); + *p.rlast = 1; + *p.rid = 0; + *p.rresp = 0; + *p.bvalid = !p.writes.empty() && p.writes.front() <= cycle_; + *p.bid = 0; + *p.bresp = 0; + } } + // What a port did on the edge, as the kernel offered it before the edge. + struct sample_t { + bool ar, aw, w, r_fire, b_fire; + uint64_t araddr, awaddr; + uint8_t wdata[kBeat], strobe[kBeat]; + }; + // The rising edge: the handshakes as the kernel saw them before it, then // the edge itself. void edge() { drive_memory(); top_->eval(); - bool ar = top_->m_axi_mem_0_arvalid; - uint64_t araddr = top_->m_axi_mem_0_araddr; - bool aw = top_->m_axi_mem_0_awvalid; - uint64_t awaddr = top_->m_axi_mem_0_awaddr; - bool w = top_->m_axi_mem_0_wvalid; - uint8_t wdata[kBeat], strobe[kBeat]; - if (w) { - std::memcpy(wdata, top_->m_axi_mem_0_wdata.data(), kBeat); - uint64_t s = top_->m_axi_mem_0_wstrb; - for (uint32_t i = 0; i < kBeat; ++i) - strobe[i] = (s >> i) & 1; + std::array smp; + for (int k = 0; k < FOLDSIM_MEM_PORTS; ++k) { + auto& p = ports_[k]; + auto& x = smp[k]; + x.ar = *p.arvalid; + x.araddr = *p.araddr; + x.aw = *p.awvalid; + x.awaddr = *p.awaddr; + x.w = *p.wvalid; + if (x.w) { + std::memcpy(x.wdata, p.wdata->data(), kBeat); + uint64_t s = *p.wstrb; + for (uint32_t i = 0; i < kBeat; ++i) + x.strobe[i] = (s >> i) & 1; + } + x.r_fire = *p.rvalid && *p.rready; + x.b_fire = *p.bvalid && *p.bready; } - bool r_fire = top_->m_axi_mem_0_rvalid && top_->m_axi_mem_0_rready; - bool b_fire = top_->m_axi_mem_0_bvalid && top_->m_axi_mem_0_bready; if (trace_ != nullptr) fputc(top_->rootp->foldsim_afu__DOT__profile_finish & 0xff, trace_); @@ -264,42 +330,50 @@ class foldsim_emul::Impl { #endif ++cycle_; - if (axi_trace_ != nullptr) { - if (ar) - fprintf(axi_trace_, "%lu AR %lx\n", (unsigned long)cycle_, (unsigned long)araddr); - if (aw) - fprintf(axi_trace_, "%lu AW %lx\n", (unsigned long)cycle_, (unsigned long)awaddr); - if (w) - fprintf(axi_trace_, "%lu W\n", (unsigned long)cycle_); - if (r_fire) - fprintf(axi_trace_, "%lu R\n", (unsigned long)cycle_); - if (b_fire) - fprintf(axi_trace_, "%lu B\n", (unsigned long)cycle_); - } - if (ar) { - request_t req; - req.at = cycle_ + mem_latency_; - memory_.read(araddr, req.data.data(), kBeat); - reads_.push_back(req); - } - if (r_fire) - reads_.pop_front(); - if (aw) - aw_.push_back(awaddr); - if (w) { - std::array beat; - std::memcpy(beat.data(), wdata, kBeat); - std::memcpy(beat.data() + kBeat, strobe, kBeat); - w_.push_back(beat); - } - while (!aw_.empty() && !w_.empty()) { - memory_.write(aw_.front(), w_.front().data(), kBeat, w_.front().data() + kBeat); - aw_.pop_front(); - w_.pop_front(); - writes_.push_back(cycle_ + mem_latency_); + // Each block of the target's memory belongs to one port, so the order + // the ports are served in is not visible. + for (int k = 0; k < FOLDSIM_MEM_PORTS; ++k) { + auto& p = ports_[k]; + const auto& x = smp[k]; + if (axi_trace_ != nullptr) { + // Port 0's lines keep the single-port format. + const char* tag = (k == 0) ? "" : port_tag_[k].c_str(); + if (x.ar) + fprintf(axi_trace_, "%lu %sAR %lx\n", (unsigned long)cycle_, tag, (unsigned long)x.araddr); + if (x.aw) + fprintf(axi_trace_, "%lu %sAW %lx\n", (unsigned long)cycle_, tag, (unsigned long)x.awaddr); + if (x.w) + fprintf(axi_trace_, "%lu %sW\n", (unsigned long)cycle_, tag); + if (x.r_fire) + fprintf(axi_trace_, "%lu %sR\n", (unsigned long)cycle_, tag); + if (x.b_fire) + fprintf(axi_trace_, "%lu %sB\n", (unsigned long)cycle_, tag); + } + if (x.ar) { + request_t req; + req.at = cycle_ + mem_latency_; + memory_.read(x.araddr, req.data.data(), kBeat); + p.reads.push_back(req); + } + if (x.r_fire) + p.reads.pop_front(); + if (x.aw) + p.aw.push_back(x.awaddr); + if (x.w) { + std::array beat; + std::memcpy(beat.data(), x.wdata, kBeat); + std::memcpy(beat.data() + kBeat, x.strobe, kBeat); + p.w.push_back(beat); + } + while (!p.aw.empty() && !p.w.empty()) { + memory_.write(p.aw.front(), p.w.front().data(), kBeat, p.w.front().data() + kBeat); + p.aw.pop_front(); + p.w.pop_front(); + p.writes.push_back(cycle_ + mem_latency_); + } + if (x.b_fire) + p.writes.pop_front(); } - if (b_fire) - writes_.pop_front(); } std::unique_ptr context_; @@ -315,10 +389,8 @@ class foldsim_emul::Impl { FILE* trace_ = nullptr; FILE* axi_trace_ = nullptr; FILE* reg_trace_ = nullptr; - std::deque reads_; - std::deque aw_; - std::deque> w_; - std::deque writes_; + std::array ports_; + const std::array port_tag_ = {"", "1:", "2:", "3:", "4:", "5:", "6:", "7:"}; }; foldsim_emul::foldsim_emul() : impl_(new Impl()) {} diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index 24d9528bb6..70accd2a22 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -483,7 +483,10 @@ class Processor::Impl { static DramBankAxi::Config axi_bank_config() { auto dram = DramSim::platform_config(); DramBankAxi::Config c; - c.model = DramBankModel::Config::make(dram.type, dram.data_rate, dram.block_size); + // VORTEX_DRAM_REFRESH=off|bank|all, as the bank model's DramSim backend. + const char* refresh = getenv("VORTEX_DRAM_REFRESH"); + c.model = DramBankModel::Config::make(dram.type, dram.data_rate, dram.block_size, + DramBankModel::Config::parse_refresh(refresh ? refresh : "")); c.core_clock = dram.core_clock; c.data_rate = dram.data_rate; c.latency = dram.latency; diff --git a/sw/common/dram_timing.h b/sw/common/dram_timing.h index 7c75879893..f409ec383d 100644 --- a/sw/common/dram_timing.h +++ b/sw/common/dram_timing.h @@ -66,12 +66,14 @@ inline constexpr clock_timing_t hbm_bus_timing[] = { // the rest from Ramulator's HBM2_2Gbps preset, whose 1 ns clock makes its // cycles nanoseconds. The array does not speed up with the interface: HBM2E // and HBM3 parts keep these timings (GPGPU-Sim's SM80_A100 and SM90_H100 HBM -// lines are the same 14/33/47 ns at their own clocks). Refresh timings complete -// the set Ramulator requires. +// lines are the same 14/33/47 ns at their own clocks). Refresh: the per-bank +// tRFCSB and tREFI of that preset, and the all-bank tRFC of Ramulator's table +// for the 8 Gb channel dram_sim configures. inline constexpr ns_timing_t hbm_array_timing[] = { {"tCL", 14}, {"tRCDRD", 14}, {"tRCDWR", 14}, {"tRP", 14}, {"tRAS", 33}, {"tRC", 47}, {"tCWL", 2}, {"tWR", 12}, {"tRTPS", 2}, {"tRTPL", 4}, {"tRRDS", 4}, {"tRRDL", 4}, - {"tWTRS", 3}, {"tWTRL", 4}, {"tFAW", 15}, {"tRFCSB", 160}, {"tREFI", 3900}, {"tRREFD", 8}, + {"tWTRS", 3}, {"tWTRL", 4}, {"tFAW", 15}, {"tRFC", 350}, {"tRFCSB", 160}, {"tREFI", 3900}, + {"tRREFD", 8}, }; // Ramulator's GDDR6 clock is half the data rate: a 32 B burst, 16 beats on a @@ -211,5 +213,25 @@ inline clocks_t clocks(Standard standard, uint32_t data_rate) { return c; } +// Refresh, in DRAM clocks: the interval in which every bank is refreshed once +// (tREFI), and how long a refresh keeps its banks from activating, all of them +// (tRFC) or one (per-bank refresh: HBM's REFsb, GDDR6's REFpb). +struct refresh_t { + uint32_t REFI, RFC, RFCPB; +}; + +inline refresh_t refresh(Standard standard, uint32_t data_rate) { + const int tck = tck_ps(data_rate); + refresh_t r{}; + if (standard == Standard::GDDR6) { + auto g = [&](const char* t) { return detail::get(gddr6_bus_timing, gddr6_array_timing, tck, nullptr, t); }; + r.REFI = g("tREFI"); r.RFC = g("tRFC"); r.RFCPB = g("tRFCpb"); + } else { + auto g = [&](const char* t) { return detail::get(hbm_bus_timing, hbm_array_timing, tck, nullptr, t); }; + r.REFI = g("tREFI"); r.RFC = g("tRFC"); r.RFCPB = g("tRFCSB"); + } + return r; +} + } // namespace dram_timing } // namespace vortex diff --git a/sw/common/foldsim_regs.h b/sw/common/foldsim_regs.h index e83ba3fb97..95f44728a2 100644 --- a/sw/common/foldsim_regs.h +++ b/sw/common/foldsim_regs.h @@ -19,7 +19,7 @@ #pragma once #define FOLDSIM_ID 0x464F4C44u -#define FOLDSIM_VERSION 3u +#define FOLDSIM_VERSION 4u #define FOLDSIM_REG_ID 0x000u #define FOLDSIM_REG_VERSION 0x004u @@ -49,7 +49,8 @@ // channel's banks. A reset command's CMD_ADDR chooses it, a FOLDSIM_MEM_MODEL_* // the target has from that reset on (the kernel's own reset: the pipe). Times // are DRAM clocks. -#define FOLDSIM_REG_MEM_MODEL 0x0D8u // read only: [1:0] the FOLDSIM_MEM_MODEL_* chosen +#define FOLDSIM_REG_MEM_MODEL 0x0D8u // read only: [1:0] the FOLDSIM_MEM_MODEL_* chosen, + // [5:4] the FOLDSIM_MEM_REFRESH_* #define FOLDSIM_REG_MEM_CAPS 0x0DCu // read only: FOLDSIM_MEM_CAPS_* #define FOLDSIM_REG_MEM_CLOCK 0x0E0u // [15:0] data rate (MT/s), [31:16] core clock (MHz) #define FOLDSIM_REG_MEM_LATENCY 0x0E4u // fixed read latency, target cycles @@ -59,13 +60,18 @@ // [5:4] banks per group, [9:6] bursts per row, // [11:10] bursts per row visit; [12] two-clock ACT #define FOLDSIM_REG_MEM_WMARK 0x0F0u // write bursts: [7:0] to writes at, [15:8] back below +#define FOLDSIM_REG_MEM_REFI 0x0F4u // refresh: DRAM clocks from one to the next +#define FOLDSIM_REG_MEM_RFC 0x0F8u // refresh: DRAM clocks a refreshed bank may not activate #define FOLDSIM_REG_MEM_TIMING(i) (0x800u + 4u * (i)) // FOLDSIM_MEM_T_*, 12 bits each -#define FOLDSIM_REG_MEM_STATS(i) (0xC00u + 8u * (i)) // FOLDSIM_MEM_STAT_*, lo/hi +#define FOLDSIM_REG_MEM_STATS(p, i) (0xC00u + 0x40u * (p) + 8u * (i)) // port p's FOLDSIM_MEM_STAT_*, lo/hi #define FOLDSIM_MEM_MODEL_PIPE 0u #define FOLDSIM_MEM_MODEL_BANK 1u +// A reset command's address: the model, and the bank model's refresh. +#define FOLDSIM_MEM_REFRESH_BANK 0x10u // per bank, the banks in turn +#define FOLDSIM_MEM_REFRESH_ALL 0x30u // all banks at once -#define FOLDSIM_MEM_CAPS_PORTS(c) ((c) & 0xffu) +#define FOLDSIM_MEM_CAPS_PORTS(c) ((c) & 0xffu) // memory ports, a channel each #define FOLDSIM_MEM_CAPS_HORIZON(c) (1u << ((((c) >> 8) & 0xfu) + 4u)) // ring, DRAM clocks #define FOLDSIM_MEM_CAPS_SLOTS(c) (((c) >> 12) & 0xffu) // read slots per port #define FOLDSIM_MEM_CAPS_TIME_W(c) ((((c) >> 28) & 0xfu) + 12u) // timing bits @@ -79,7 +85,8 @@ enum { enum { FOLDSIM_MEM_STAT_READS, FOLDSIM_MEM_STAT_WRITES, FOLDSIM_MEM_STAT_HITS, FOLDSIM_MEM_STAT_MISSES, - FOLDSIM_MEM_STAT_CONFLICTS, FOLDSIM_MEM_STAT_LATENCY, FOLDSIM_MEM_NUM_STATS + FOLDSIM_MEM_STAT_CONFLICTS, FOLDSIM_MEM_STAT_LATENCY, FOLDSIM_MEM_STAT_REFRESHES, + FOLDSIM_MEM_STAT_FULL, FOLDSIM_MEM_NUM_STATS }; #define FOLDSIM_STATUS_BUSY 0x1u // queued, running, or memory in flight diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp index 1bee1f0888..ae313d10f4 100644 --- a/sw/runtime/foldsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -340,6 +340,22 @@ class vx_device { return -1; } bank_model_ = (model_name == "bank"); + // VORTEX_DRAM_REFRESH=off|bank|all: the bank model's refresh, as the + // simulators' bank model takes it. + const char* refresh = getenv("VORTEX_DRAM_REFRESH"); + const std::string refresh_name = (refresh != nullptr) ? refresh : ""; + if (refresh_name == "bank") { + refresh_ = FOLDSIM_MEM_REFRESH_BANK; + } else if (refresh_name == "all") { + refresh_ = FOLDSIM_MEM_REFRESH_ALL; + } else if (!refresh_name.empty() && refresh_name != "off") { + fprintf(stderr, "[VXDRV] VORTEX_DRAM_REFRESH must be off, bank or all (got '%s')\n", refresh); + return -1; + } + if (refresh_ != 0 && !bank_model_) { + fprintf(stderr, "[VXDRV] VORTEX_DRAM_REFRESH needs FOLDSIM_MEM_MODEL=bank\n"); + return -1; + } if (bank_model_ && this->setup_bank_model() != 0) return -1; const uint32_t rlat = env_or("FOLDSIM_MEM_READ_LATENCY", 30); @@ -360,7 +376,7 @@ class vx_device { // memory error that process left, so only the state after it counts, and // chooses the memory model. uint32_t s = 0; - const uint32_t mem_model = bank_model_ ? FOLDSIM_MEM_MODEL_BANK : FOLDSIM_MEM_MODEL_PIPE; + const uint32_t mem_model = (bank_model_ ? FOLDSIM_MEM_MODEL_BANK : FOLDSIM_MEM_MODEL_PIPE) | refresh_; if (command(FOLDSIM_CMD_RESET, mem_model, 64) != 0 || wait_idle(false) != 0 || status(&s) != 0) return -1; if (mem_error(s)) @@ -427,14 +443,15 @@ class vx_device { // The bank model's settings: the platform memory this runtime was built for // (VX_CFG_PLATFORM_MEMORY_*), its data-sheet timings in DRAM clocks as // dram_sim derives them, and dram_sim's controller queues and write - // watermarks. They take effect at the target reset that follows. The image - // fixes the structure: its port is one channel, whatever - // VX_CFG_PLATFORM_MEMORY_NUM_BANKS says. + // watermarks; every port's channel takes the same. They take effect at the + // target reset that follows. The image fixes the structure: a channel per + // memory port of the target it was built for. int setup_bank_model() { uint32_t caps = 0; if (transport_.reg_read(FOLDSIM_REG_MEM_CAPS, &caps) != 0) return -1; - if (FOLDSIM_MEM_CAPS_PORTS(caps) == 0) { + mem_ports_ = FOLDSIM_MEM_CAPS_PORTS(caps); + if (mem_ports_ == 0) { fprintf(stderr, "[VXDRV] this foldsim image has no bank model\n"); return -1; } @@ -487,6 +504,29 @@ class vx_device { if (reg_write(FOLDSIM_REG_MEM_TIMING(i), v[i]) != 0) return -1; } + // Refresh as the bank model takes it (DramBankModel::Config::set_refresh). + if (refresh_ != 0) { + const auto r = dram_timing::refresh(standard, rate); + const uint32_t banks = 1u << (g.log2_pcs + g.log2_bgs + g.log2_banks); + const bool all = (refresh_ == FOLDSIM_MEM_REFRESH_ALL); + const uint32_t refi = all ? r.REFI : r.REFI / banks; + const uint32_t rfc = all ? r.RFC : r.RFCPB; + if (refi > 0xffff || rfc > 0xffff) { + fprintf(stderr, "[VXDRV] the refresh timings exceed the bank model's registers\n"); + return -1; + } + if (reg_write(FOLDSIM_REG_MEM_REFI, refi) != 0 || reg_write(FOLDSIM_REG_MEM_RFC, rfc) != 0) + return -1; + // A bank refreshing beyond the reservations' reach holds every request + // behind one to it at the channel's ingress: refresh then costs more + // than it would with the reach to place them. + const uint32_t horizon = FOLDSIM_MEM_CAPS_HORIZON(caps) - 64; + if (rfc >= horizon) { + fprintf(stderr, "[VXDRV] warning: a refresh keeps its bank %u DRAM clocks, past this image's " + "%u-clock horizon: refresh costs more here than it would in a channel that could " + "place the requests behind it\n", rfc, horizon); + } + } return 0; } @@ -523,18 +563,31 @@ class vx_device { } if (!line.empty()) fprintf(stderr, "[foldsim] class busy:%s\n", line.c_str()); + // The bank model's counters, per channel and, with several, in all. if (bank_model_) { - uint64_t st[FOLDSIM_MEM_NUM_STATS]; - for (uint32_t i = 0; i < FOLDSIM_MEM_NUM_STATS; ++i) - st[i] = read64(FOLDSIM_REG_MEM_STATS(i)); - const uint64_t reads = st[FOLDSIM_MEM_STAT_READS]; - fprintf(stderr, - "[foldsim] bank model: %lu reads, %lu writes; row hits %lu, misses %lu, conflicts %lu; " - "mean read latency %.1f cycles\n", - (unsigned long)reads, (unsigned long)st[FOLDSIM_MEM_STAT_WRITES], - (unsigned long)st[FOLDSIM_MEM_STAT_HITS], (unsigned long)st[FOLDSIM_MEM_STAT_MISSES], - (unsigned long)st[FOLDSIM_MEM_STAT_CONFLICTS], - reads ? double(st[FOLDSIM_MEM_STAT_LATENCY]) / double(reads) : 0.0); + uint64_t total[FOLDSIM_MEM_NUM_STATS] = {}; + for (uint32_t p = 0; p <= mem_ports_; ++p) { + uint64_t st[FOLDSIM_MEM_NUM_STATS]; + const bool all = (p == mem_ports_); + if (all && mem_ports_ == 1) + break; + for (uint32_t i = 0; i < FOLDSIM_MEM_NUM_STATS; ++i) { + st[i] = all ? total[i] : read64(FOLDSIM_REG_MEM_STATS(p, i)); + total[i] += all ? 0 : st[i]; + } + char who[32] = "bank model"; + if (!all && mem_ports_ > 1) + snprintf(who, sizeof(who), "bank model, channel %u", p); + const uint64_t reads = st[FOLDSIM_MEM_STAT_READS]; + fprintf(stderr, + "[foldsim] %s: %lu reads, %lu writes; row hits %lu, misses %lu, conflicts %lu; " + "mean read latency %.1f cycles; %lu refreshes; %lu cycles full\n", + who, (unsigned long)reads, (unsigned long)st[FOLDSIM_MEM_STAT_WRITES], + (unsigned long)st[FOLDSIM_MEM_STAT_HITS], (unsigned long)st[FOLDSIM_MEM_STAT_MISSES], + (unsigned long)st[FOLDSIM_MEM_STAT_CONFLICTS], + reads ? double(st[FOLDSIM_MEM_STAT_LATENCY]) / double(reads) : 0.0, + (unsigned long)st[FOLDSIM_MEM_STAT_REFRESHES], (unsigned long)st[FOLDSIM_MEM_STAT_FULL]); + } } const auto& st = transport_.stats; fprintf(stderr, @@ -741,6 +794,8 @@ class vx_device { bool failed_ = false; bool open_ = false; bool bank_model_ = false; + uint32_t refresh_ = 0; + uint32_t mem_ports_ = 1; vortex::CommandProcessor cp_; std::mutex host_mu_; std::map host_regions_; diff --git a/tests/unittest/dram_bank_model/main.cpp b/tests/unittest/dram_bank_model/main.cpp index 2408dad63f..c52421105a 100644 --- a/tests/unittest/dram_bank_model/main.cpp +++ b/tests/unittest/dram_bank_model/main.cpp @@ -25,6 +25,13 @@ // // ./dram_bank_model the gate, on short runs // ./dram_bank_model full [cfg] the full table (20,000 requests per cell) +// ./dram_bank_model refresh the bandwidth refresh costs the bank model, +// per bank and all banks, at 64 outstanding +// (dram_sim models no refresh). The gate: one +// refresh per interval, to the operation; and a +// stream, which moves data every clock it may, +// loses at least tRFC / tREFI to all-bank +// refresh, the time no bank is open #include #include @@ -146,9 +153,10 @@ double ks_distance(const std::vector& a, const std::vector& return ks; } -DramBankSim::Config bank_config(const DramSim::Config& c) { +DramBankSim::Config bank_config(const DramSim::Config& c, + DramBankModel::Refresh refresh = DramBankModel::Refresh::Off) { return DramBankSim::Config{c.type, c.num_channels, c.block_size, c.core_clock, c.data_rate, - c.latency, c.queue_size}; + c.latency, c.queue_size, refresh}; } std::vector patterns() { @@ -170,7 +178,62 @@ std::vector patterns() { } // namespace +// Bandwidth with refresh off, per bank and all banks. +int refresh_table() { + using R = DramBankModel::Refresh; + int failures = 0; + const uint32_t n = 60000, w = 64; + printf("%-9s %-9s | %7s | %7s %6s %6s | %7s %6s %6s %6s\n", "machine", "pattern", "off B/c", + "bank", "loss%", "refs", "all", "loss%", "refs", "tRFC/tREFI%"); + for (const auto& mach : kMachines) { + if (mach.cfg.type == DramSim::Type::GDDR6) continue; + const auto std = (mach.cfg.type == DramSim::Type::HBM3) ? dram_timing::Standard::HBM3 + : dram_timing::Standard::HBM2; + const auto r = dram_timing::refresh(std, mach.cfg.data_rate); + const double analytic = 100.0 * double(r.RFC) / double(r.REFI); + for (const auto& p : patterns()) { + if (p.write_frac != 0 || !std::strcmp(p.name, "conflict")) continue; + double bw[3]; + uint64_t refs[3]; + const R modes[3] = {R::Off, R::PerBank, R::AllBank}; + for (int k = 0; k < 3; ++k) { + DramBankSim mod(bank_config(mach.cfg, modes[k])); + bw[k] = run(mod, mach.cfg, p, w, n).bw; + refs[k] = 0; + for (uint32_t ch = 0; ch < mach.cfg.num_channels; ++ch) { + const auto& m = mod.channel(ch); + refs[k] += m.stats().refreshes; + // The refreshes due by the last clock, but one in progress. + const uint32_t refi = m.config().refi; + const uint64_t due = refi ? m.clock() / refi : 0; + if (m.stats().refreshes + 1 < due || m.stats().refreshes > due) { + printf("FAIL %s %s: %lu refreshes in %lu clocks of %u\n", mach.name, p.name, + (unsigned long)m.stats().refreshes, (unsigned long)m.clock(), refi); + ++failures; + } + } + } + const double lb = 100.0 * (1 - bw[1] / bw[0]), la = 100.0 * (1 - bw[2] / bw[0]); + printf("%-9s %-9s | %7.2f | %7.2f %6.2f %6lu | %7.2f %6.2f %6lu %6.2f\n", mach.name, p.name, + bw[0], bw[1], lb, (unsigned long)refs[1], bw[2], la, (unsigned long)refs[2], analytic); + if (!std::strcmp(p.name, "seq") && la < analytic) { + printf("FAIL %s %s: all-bank refresh costs %.2f%%, less than tRFC/tREFI (%.2f%%)\n", mach.name, + p.name, la, analytic); + ++failures; + } + } + } + if (failures) { + printf("FAILED: %d\n", failures); + return 1; + } + printf("PASSED!\n"); + return 0; +} + int main(int argc, char** argv) { + if (argc > 1 && !std::strcmp(argv[1], "refresh")) + return refresh_table(); const bool full = argc > 1 && !std::strcmp(argv[1], "full"); const char* only = (full && argc > 2) ? argv[2] : nullptr; const uint32_t n = full ? 20000 : 2000; From e4d9769b73cee5cde9406baa2b6b50e8ff737b12 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 18 Oct 2026 02:39:11 -0700 Subject: [PATCH 233/336] hw/syn/foldsim: the 250 MHz option set by default, and an ooc target The image flow's default compiler options are now the set that times s2w4i2 at 250 MHz: regions of 2048 loads with their own control, registered state committed by select over two host cycles, pipelined multipliers, staged memory writes and tokens, and folded fixed-latency pipelines. Every simulator stays cycle-exact: tinygpu and s2w4i2 (soft FPU) in the in-process simulator equal rtlsim behind the AXI4 adapter with the pipe and the bank model, and s2w4i2 with the Xilinx cores its earlier counts. FOLDSIM_FLAGS= gives the plain simulator, which a small target may prefer: tinygpu runs at 4.00 host cycles per target cycle with the default set, 3.00 without. `make ooc` synthesizes, places and routes the kernel alone at FOLDSIM_FREQ, with global retiming and the Xilinx cores it instantiates, for a kernel's Fmax in about half an image build's time. docs/designs/foldsim.md: the timing options and their measured Fmax and area, and the domain and relay options. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 93fc100f15155a954b0fcd23d578a9c8c29a548f) --- docs/designs/foldsim.md | 64 +++++++++++++++++++++++++++++++-- hw/syn/foldsim/Makefile | 25 +++++++++++-- hw/syn/foldsim/ooc.tcl | 80 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 hw/syn/foldsim/ooc.tcl diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 926bab6b45..be5150d86f 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -10,7 +10,7 @@ on it, and runs orders of magnitude faster than Verilator. |---|---| | [1](#1-overview) | Overview and how to run it | | [2](#2-the-compiler) | The compiler: netlists, threading, composition | -| [3](#3-the-simulator) | The simulator: threads, channels, the token fabric | +| [3](#3-the-simulator) | The simulator: threads, channels, the token fabric; timing options; domains and relays | | [4](#4-the-kernel) | The kernel: memory and control endpoints | | [5](#5-the-runtime) | The runtime and its register contract | | [6](#6-validation) | Validation | @@ -146,6 +146,62 @@ reader's thread and slot (LUTRAM); any other, in registers. A target port that is exactly one port of a model of one thread is wired to the shell directly, since the shell's valid and ready depend on its own state alone. +### 3.1 Timing: the options the image flow uses + +A threaded model's critical paths run from its thread index, through every +state memory's read and the target logic, into every state memory's write. +The image flow's default options (`FOLDSIM_FLAGS` in `hw/syn/foldsim`) cut +those paths without changing what the target computes; each keeps the +simulator cycle-exact (§6). + +| option | what it does | +|---|---| +| `--region-loads 2048` | splits a model's state into regions of at most 2,048 loads, each with its own copy of the thread index and of the edge, written a host cycle after the target edge | +| `--registered-state` | reads the next thread's state a host cycle ahead into registers, forwarding the write landing in between; threads then go in turn (`--schedule`) | +| `--commit-select` | a model of one thread commits its state by selecting a registered candidate, not by writing it in the edge's cycle | +| `--state-stages 2` | writes the state two host cycles after its edge in models of three threads or more, so synthesis can retime into the logic before it | +| `--pipeline-multipliers` | with two stages, a state computed by a multiplier takes its first stage at the multiplier's inputs | +| `--staged-ram-writes` | with regions, memories with a registered read are written a host cycle late too, their reads forwarded | +| `--staged-tokens` | a channel writes each token into its slot a host cycle after it is taken, forwarded to its readers until then | +| `--fold-pipelines` | simulates a fixed-latency pipeline by one unthreaded copy and a per-thread store of its results (the Xilinx FP cores, §1) | + +`--compact-staging` (staging placed at the state it stages; for large images, +s2w4i2's Verilog 34 → 16 MB) and `--channel-depth` are not in the default. + +Out-of-context results for s2w4i2 (`make -C hw/syn/foldsim ooc`, xcv80, +4.0 ns unless noted; Vivado retiming moves a run by about ±20 MHz): + +| options | FPU | MHz | LUTs | FFs | +|---|---|---:|---:|---:| +| none | soft | 186.8 | 130.1k | 26.8k | +| regions of 2,048 loads | soft | 214.3 | 136.9k | 83.6k | +| the default set but folding | soft | 250.0 | 211.8k | 174.7k | +| the default set | soft | 235.3 | 194.6k | 157.0k | +| the default set | Xilinx cores | 250.0 (262.7 at 3.33 ns) | 182.8k | 162.4k | + +The soft FPU's own pipeline stages limit it: the same options with ready +scheduling instead of threads in turn reached 250.0 MHz, the difference being +which stages retiming balanced. + +Small targets pay area for these options that they do not need; +`FOLDSIM_FLAGS=` (empty) builds the plain simulator. + +### 3.2 Domains and relays + +`--domains` emits the simulator as a tree of modules, one per class, nested as +the classes are, with every domain boundary a set of module ports; +`--relay ` gives the channels of the named models (or +`:` pairs) a relay stage, so a channel crossing an SLR costs a +host cycle of latency instead of a long wire. Both are off by default. +Measured on s2w4i2 (out of context): with the default options, domains and +relays reach the same 250 MHz at 4.0 ns; at 3.33 ns 273.6 against 262.7 MHz +with the Xilinx cores (equal slack after synthesis), but 225.7 against +238.9 MHz with the soft FPU; without the state-staging options +(regions and folding only) 231.8 against 226.2 MHz at 4.0 ns, for 1.4 % more +LUTs. Both differences are inside the retiming spread. A relay costs host +cycles where a loop runs through it (up to 22 % on img6b), so relays are +chosen per image from a Verilator run. + --- ## 4. The kernel @@ -451,7 +507,11 @@ this compares the two simulators' fidelity and overheads, not threading. | `sgemm -n16` cycles, V80 (= Verilator) | 301,923 | 317,163 | | `sgemm -n64` cycles, V80 | 13,264,035 | 13,937,835 | | `sgemm -n64` kernel time, V80 | 278 ms | 178 ms | -| host cycles per target cycle, `sgemm -n64` | 3.00 | 1.84 (FMR) | +| host cycles per target cycle, `sgemm -n64` | 3.00 ¹ | 1.84 (FMR) | + +¹ With the plain flow (`FOLDSIM_FLAGS=`). The flow's default options (§3.1) +run tinygpu at 4.00 host cycles per target cycle (Verilator), cycles +unchanged: they buy Fmax for large targets with host cycles a small one pays. Instruction counts are identical on both flows, in Verilator and on the V80. Foldsim's cycles equal rtlsim's behind the AXI4 adapter on `AxiLatencyPipe` diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index 9ca5fe302f..b35fe4661b 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -53,8 +53,14 @@ CIRCT_BIN ?= $(CIRCT_PATH)/bin LIFT_PYTHON ?= python3 # The kernel clock (MHz): the image is timed at it and the shell programs it. FOLDSIM_FREQ ?= 250 -# Options of the foldsim driver, e.g. --region-loads=2048 --registered-state. -FOLDSIM_FLAGS ?= --fold-pipelines +# Options of the foldsim driver: by default the set that times s2w4i2 at +# 250 MHz (regions of 2048 loads with their own control, registered state +# committed by select over two host cycles, pipelined multipliers, staged +# memory writes and tokens, folded fixed-latency pipelines); docs/designs/ +# foldsim.md lists each. Large images add --compact-staging; multi-SLR ones +# --domains and --relay. +FOLDSIM_FLAGS ?= --region-loads 2048 --registered-state --commit-select --state-stages 2 \ + --pipeline-multipliers --staged-ram-writes --staged-tokens --fold-pipelines # The target: Vortex with its AXI4 port, DCR, start and busy as plain ports. TARGET_TOP := foldsim_target @@ -137,7 +143,7 @@ RTL_SRCS := $(filter-out $(KERNEL_RTL),$(shell find $(RTL_DIR) -name '*.sv' -o - # compiled apart from the target. SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv) $(wildcard $(FOLDSIM_RTL_DIR)/*.sv)) -.PHONY: all sources generate package-ip clean FORCE +.PHONY: all sources generate package-ip ooc clean FORCE all: $(VBIN) @@ -209,6 +215,19 @@ $(IP_DIR)/component.xml: $(KERNEL_SRCS) $(KERNEL_VH) $(OOC_XDC) $(wildcard $(FOL -tclargs foldsim_afu $(KERNEL_SRCS) . $(DEVICE) ooc-xdc=$(OOC_XDC) \ $$(grep -qE '^ *xil_f(ma|div|sqrt|mul|add) ' $(GEN_SV) || echo no-vendor-ip) +# The kernel alone, out of context at FOLDSIM_FREQ: synthesis with global +# retiming, placement and routing, without the shell, its reports in +# $(BUILD_DIR)/ooc. It answers what a change does to the kernel's Fmax in +# about half an image build's time. +OOC_DIR := $(BUILD_DIR)/ooc +ooc: $(KERNEL_SRCS) + rm -rf $(OOC_DIR) && mkdir -p $(OOC_DIR) + cd $(OOC_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -nojournal -log vivado.log \ + -source $(SRC_DIR)/ooc.tcl -tclargs $(KERNEL_SRCS) $(DEVICE) \ + $$(python3 -c 'print(round(1000 / $(FOLDSIM_FREQ), 3))') \ + $$(grep -qE '^ *xil_f(ma|div|sqrt|mul|add) ' $(GEN_SV) || echo no-vendor-ip) + @grep -A2 'WNS(ns)' $(OOC_DIR)/routed_timing_summary.rpt | tail -1 + $(CFG): FORCE @python3 $(SCRIPT_DIR)/gen_platform.py --flow=aved --device=$(DEVICE) --emit=cfg --kernel=foldsim_afu \ --no-host --mem-ports=$(MEM_PORTS) --kernel-mhz=$(FOLDSIM_FREQ) -o $@.tmp diff --git a/hw/syn/foldsim/ooc.tcl b/hw/syn/foldsim/ooc.tcl new file mode 100644 index 0000000000..e6436114e3 --- /dev/null +++ b/hw/syn/foldsim/ooc.tcl @@ -0,0 +1,80 @@ +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The foldsim kernel alone, out of context: synthesis with global retiming, +# placement and routing at the kernel clock, as the image's own run times it +# but without the shell around it. Reports land in the current directory. +# +# vivado -mode batch -source ooc.tcl -tclargs [no-vendor-ip] +# +# The Xilinx floating-point cores a simulator instantiates are generated as +# the aved flow generates them (TOOL_DIR's xilinx_ip_gen.tcl) and synthesized +# out of context; no-vendor-ip skips them for a kernel without any. + +if {$argc < 3} { + puts "Usage: $::argv0 \[no-vendor-ip\]" + exit 1 +} +lassign $argv srcs part period options +set vendor_ip [expr {$options ne "no-vendor-ip"}] + +if {$vendor_ip} { + set tool_dir $::env(TOOL_DIR) + set argv [list [pwd]/ip $part] + set argc 2 + source ${tool_dir}/xilinx_ip_gen.tcl +} + +set incdirs {} +set fp [open $srcs r] +foreach line [split [read $fp] "\n"] { + set line [string trim $line] + if {$line eq "" || [string match "#*" $line]} continue + if {[string match "+incdir+*" $line]} { lappend incdirs [string range $line 8 end]; continue } + read_verilog -sv $line +} +close $fp +if {$vendor_ip} { + foreach xci [glob [pwd]/ip/*/*.xci] { read_ip $xci } + generate_target all [get_ips] + synth_ip [get_ips] +} + +# The clock with the uncertainty the image's runs see. +set f [open clk.xdc w] +puts $f "create_clock -name ap_clk -period $period \[get_ports ap_clk\]" +puts $f "set_clock_uncertainty -setup 0.200 \[get_clocks ap_clk\]" +close $f +read_xdc clk.xdc + +set t0 [clock seconds] +synth_design -top foldsim_afu -part $part -mode out_of_context -include_dirs $incdirs -global_retiming on +set t1 [clock seconds] +report_utilization -file synth_util.rpt +report_timing_summary -max_paths 20 -file synth_timing_summary.rpt +opt_design +place_design -directive Explore +phys_opt_design -directive AggressiveExplore +set t2 [clock seconds] +route_design -directive Explore +phys_opt_design -directive AggressiveExplore +set t3 [clock seconds] +report_utilization -file routed_util.rpt +report_utilization -hierarchical -hierarchical_depth 6 -file routed_util_hier.rpt +report_timing_summary -max_paths 50 -file routed_timing_summary.rpt +report_design_analysis -timing -max_paths 100 -file routed_design_analysis.rpt +report_design_analysis -logic_level_distribution -logic_level_dist_paths 5000 -file routed_lld.rpt +report_high_fanout_nets -max_nets 40 -timing -file routed_hfn.rpt +set f [open times.txt w] +puts $f "synth [expr {$t1-$t0}] place [expr {$t2-$t1}] route [expr {$t3-$t2}] total [expr {$t3-$t0}]" +close $f From e2ea771d801ef8ec10d1d3f41c41da91b58d26ea Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 18 Oct 2026 12:08:12 -0700 Subject: [PATCH 234/336] mem_to_axi: no write tag through the request crossbar Every write leaves with AXI id 0 since the write tracker orders stores against fills, so the tag a write request carried through the request crossbar was never read. With the tag buffer engaged (the platform's AXI id narrower than the requester's tag), the crossbar's tag field kept the write tag's width, wider than the read tag it now only carries, and Verilator's lint stopped on the unused bits (rtlsim AXI_BUS=1 with one input port, two banks and a 5-bit id). The crossbar now carries the read tag alone: fewer crossbar bits wherever the write tag was the wider, and no change in cycles (rtlsim AXI_BUS=1, bank model: demo and sgemm equal before and after at id widths 4, 5 and 32). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c70681f3ec163196051e6792ca1600d79ee558b5) --- hw/rtl/libs/VX_mem_to_axi.sv | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/hw/rtl/libs/VX_mem_to_axi.sv b/hw/rtl/libs/VX_mem_to_axi.sv index 9d0070eb87..4a3f156e59 100644 --- a/hw/rtl/libs/VX_mem_to_axi.sv +++ b/hw/rtl/libs/VX_mem_to_axi.sv @@ -121,14 +121,12 @@ module VX_mem_to_axi #( localparam NEEDED_TAG_WIDTH = TAG_WIDTH_IN + NUM_PORTS_IN_BITS; localparam READ_TAG_WIDTH = (NEEDED_TAG_WIDTH > TAG_WIDTH_OUT) ? TAG_BUFFER_ADDRW : TAG_WIDTH_IN; localparam READ_FULL_TAG_WIDTH = READ_TAG_WIDTH + NUM_PORTS_IN_BITS; - localparam WRITE_TAG_WIDTH = `MIN(TAG_WIDTH_IN, TAG_WIDTH_OUT); - localparam DST_TAG_WIDTH = `MAX(READ_FULL_TAG_WIDTH, WRITE_TAG_WIDTH); - localparam XBAR_TAG_WIDTH = `MAX(READ_TAG_WIDTH, WRITE_TAG_WIDTH); - localparam REQ_XBAR_DATAW = 1 + BANK_ADDR_WIDTH + DATA_SIZE + DATA_WIDTH + XBAR_TAG_WIDTH; + // Writes carry no tag: every write leaves with AXI id 0 (the write tracker). + localparam REQ_XBAR_DATAW = 1 + BANK_ADDR_WIDTH + DATA_SIZE + DATA_WIDTH + READ_TAG_WIDTH; localparam RSP_XBAR_DATAW = DATA_WIDTH + READ_TAG_WIDTH; `STATIC_ASSERT ((DST_ADDR_WDITH >= ADDR_WIDTH_IN), ("invalid address width: current=%0d, expected=%0d", DST_ADDR_WDITH, ADDR_WIDTH_IN)) - `STATIC_ASSERT ((TAG_WIDTH_OUT >= DST_TAG_WIDTH), ("invalid output tag width: current=%0d, expected=%0d", TAG_WIDTH_OUT, DST_TAG_WIDTH)) + `STATIC_ASSERT ((TAG_WIDTH_OUT >= READ_FULL_TAG_WIDTH), ("invalid output tag width: current=%0d, expected=%0d", TAG_WIDTH_OUT, READ_FULL_TAG_WIDTH)) `STATIC_ASSERT ((WRITE_TRACK_DEPTH >= 2) && `IS_POW2(WRITE_TRACK_DEPTH), ("WRITE_TRACK_DEPTH must be a power of 2 >= 2: current=%0d", WRITE_TRACK_DEPTH)) // Bank selection @@ -202,7 +200,7 @@ module VX_mem_to_axi #( for (genvar i = 0; i < NUM_PORTS_IN; ++i) begin : g_req_xbar_data_in wire tag_ready = mem_req_rw[i] || mem_rd_req_tag_ready[i]; - wire [XBAR_TAG_WIDTH-1:0] tag_value = mem_req_rw[i] ? XBAR_TAG_WIDTH'(mem_req_tag[i]) : XBAR_TAG_WIDTH'(mem_rd_req_tag[i]); + wire [READ_TAG_WIDTH-1:0] tag_value = mem_req_rw[i] ? '0 : mem_rd_req_tag[i]; assign req_xbar_valid_in[i] = mem_req_valid[i] && tag_ready; assign req_xbar_data_in[i] = {mem_req_rw[i], req_bank_addr[i], mem_req_byteen[i], mem_req_data[i], tag_value}; assign mem_req_ready[i] = req_xbar_ready_in[i] && tag_ready; @@ -232,7 +230,7 @@ module VX_mem_to_axi #( wire xbar_rw_out; wire [BANK_ADDR_WIDTH-1:0] xbar_addr_out; - wire [XBAR_TAG_WIDTH-1:0] xbar_tag_out; + wire [READ_TAG_WIDTH-1:0] xbar_tag_out; wire [DATA_WIDTH-1:0] xbar_data_out; wire [DATA_SIZE-1:0] xbar_byteen_out; @@ -352,7 +350,7 @@ module VX_mem_to_axi #( assign xbar_tag_r_out = READ_FULL_TAG_WIDTH'({xbar_tag_out, req_xbar_sel_out[i]}); end else begin : g_no_input_sel `UNUSED_VAR (req_xbar_sel_out) - assign xbar_tag_r_out = READ_TAG_WIDTH'(xbar_tag_out); + assign xbar_tag_r_out = xbar_tag_out; end assign m_axi_arvalid[i] = req_xbar_valid_out[i] && ~xbar_rw_out && ~rd_hazard; From 6779cf64711a999be8790e674ab41103339e003e Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 18 Oct 2026 12:08:30 -0700 Subject: [PATCH 235/336] foldsim: the bank model is the memory model, ports from the configuration The memory endpoint keeps one timing model: the DRAM channel's banks (foldsim_mem_bank, cycle for cycle DramBankAxi). The fixed-latency pipe leaves the endpoint, the control block, the register map (version 5: the latency and in-flight registers are gone, and a reset command's address is the refresh in effect), the runtime (no FOLDSIM_MEM_MODEL; it always writes the platform memory's settings) and the endpoint's co-simulation. rtlsim keeps AxiLatencyPipe, FireSim FASED's model, as the FireSim backend's reference; Foldsim compares with rtlsim on the bank model (AXI_MEM_MODEL=bank) at the platform's AXI id width. Every default is the configuration's: the image flow takes the number of memory ports from VX_config.toml's VX_CFG_PLATFORM_MEMORY_NUM_BANKS (2) instead of defaulting to one and forcing it, and the emulator has no fallback count. The flow now spells out FOLDSIM_MEM_REPEAT(f,s), the list of the kernel's indexed memory ports, in the generated foldsim_kernel.vh, replacing foldsim_ports.vh: Vivado's IP packager does not paste a macro's expanded argument into a macro name, so packaging failed for every image. Verilator: the endpoint co-simulation is exact on its 17 bank-model settings (alone, staged, as one of two ports); tinygpu with the configuration's defaults equals rtlsim on the bank model (demo 16,375, sgemm -n16 195,152 cycles), and s2w4i2 equals it plus rtlsim's 8-cycle launch settle (demo 6,616, sgemm 399,646 with the soft FPU; 404,896 with the Xilinx cores). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 39d6c0efe82fea2b2b532c671d036db1a55e16e9) --- docs/designs/foldsim.md | 146 ++++++++--------- docs/simulation.md | 6 +- hw/rtl/afu/foldsim/foldsim_afu.sv | 26 ++-- hw/rtl/afu/foldsim/foldsim_ctrl.sv | 44 ++---- hw/rtl/afu/foldsim/foldsim_mem.sv | 207 ++++--------------------- hw/rtl/afu/foldsim/foldsim_mem_bank.sv | 13 +- hw/rtl/afu/foldsim/foldsim_ports.vh | 34 ---- hw/rtl/afu/foldsim/foldsim_regs.vh | 8 +- hw/syn/foldsim/Makefile | 52 +++---- hw/unittest/foldsim_mem/main.cpp | 178 +++++++-------------- sim/foldsim/foldsim_emul.cpp | 2 +- sw/common/foldsim_regs.h | 26 ++-- sw/runtime/foldsim/vortex.cpp | 43 +---- 13 files changed, 230 insertions(+), 555 deletions(-) delete mode 100644 hw/rtl/afu/foldsim/foldsim_ports.vh diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index be5150d86f..f0d247ab36 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -211,19 +211,18 @@ the host-memory master: `s_axi_control` and one device-memory master `m_axi_mem_

` per memory port of the target, each linked to the `MEM` HBM aperture (the linker gives each its own way in). The target has as many memory ports as its configuration has platform memory banks -(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS` in `CONFIGS`: 1, 2, 4 or 8; one when -unset); the flow writes their number, the shift to a port's channel and their -AXI id width into `gen/foldsim_kernel.vh`, and the kernel lists its ports with -`FOLDSIM_REPEAT` (`foldsim_ports.vh`). It instantiates `foldsim_sim`, a +(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS`: 1, 2, 4 or 8; `VX_config.toml`'s 2 unless +`CONFIGS` sets it); the flow writes their number, the shift to a port's +channel, their AXI id width and `FOLDSIM_MEM_REPEAT`, which lists the ports +whose names carry their index, into `gen/foldsim_kernel.vh` (spelled out, as +Vivado's IP packager does not paste an expanded macro argument into a macro +name). It instantiates `foldsim_sim`, a memory endpoint per port and the control endpoint, all models of the target's surroundings that take one token per channel per target cycle (`foldsim_token.sv`): -- **`foldsim_mem`** is one of the target's AXI4 ports, timed by one of two models - (§4.1): a latency-bandwidth pipe — cycle for cycle - `sim/common/axi_latency_pipe.h`: a fixed latency per transaction, bounded - reads and writes in flight, one beat per cycle, responses in request - order — or a DRAM channel's banks. A read goes to HBM (`base` + target address) +- **`foldsim_mem`** is one of the target's AXI4 ports, timed as a DRAM + channel's banks (§4.1). A read goes to HBM (`base` + target address) the cycle the target makes it, a write the cycle it has both its AW and its W, so HBM only ever sees whole single-beat writes; a target cycle that should present a response memory has not returned yet does not finish until @@ -241,12 +240,12 @@ port a read came from above its tag buffer's 4 index bits, so the target's AXI ids are 4 + log2(ports) bits wide; the flow sets `PLATFORM_MEMORY_ID_WIDTH` and the endpoints' id width to match. -### 4.1 The memory models +### 4.1 The memory model -The kernel's own reset gives the pipe; a reset command chooses the model the -target has from that reset on (§5). The pipe is the default, and with it the -endpoint makes exactly the transactions and takes exactly the cycles it did -before the bank model existed. +The memory model takes its settings at a target reset: the kernel's own reset +gives it the default machine's DRAM (HBM2 at 800 MT/s behind a 400 MHz core), +and the runtime writes the configuration's before the reset it makes on open +(§5). **The bank model** (`foldsim_mem_bank.sv`) is cycle for cycle `sim/common/dram_bank_model.h` (`DramBankAxi` over `DramBankModel`), the @@ -324,8 +323,8 @@ resets the target's whole memory port, not only the target. A target cycle in reset takes no request, and finishes only once HBM has answered every transaction it has from before the reset — reads returned, writes acknowledged, nothing left offered to it — so the AXI4 port to HBM stays in -protocol. The cycle then drops the pipe: requests the target made and was not -answered, data not delivered to it, the lone half of a write, and the memory +protocol. The cycle then drops the model's state: requests the target made +and was not answered, data not delivered to it, the lone half of a write, and the memory error. No response from before a reset reaches the target after it; writes HBM already took stay written, as they would behind a real memory controller. @@ -355,8 +354,8 @@ for the RTL): | `0x1C` | the last DCR read's response | | `0x20`–`0x2C` | target cycles, kernel-clock cycles | | `0x30`, `0x38` | kernel-clock cycles a target cycle waited for HBM; the number of classes | -| `0xC0`–`0xD4` | memory model: device base, the pipe's read and write latency, most reads and writes in flight | -| `0xD8`, `0xDC` | the memory model and refresh in effect (read only); the bank model's capacities: ports, engines and masters, slots, horizon (read only) | +| `0xC0`, `0xC4` | device base of the target's memory | +| `0xD8`, `0xDC` | the refresh in effect (read only); the bank model's capacities: ports, engines and masters, slots, horizon (read only) | | `0xE0`–`0xF8` | bank model: data rate and core clock, fixed latency, requests in flight and controller queues, geometry, write watermarks, refresh interval and busy time | | `0x100 + 8c` | per class, threads finished | | `0x800`–`0x844` | bank model: DRAM timings, a word each (tBL … tWTR_L, read-to-write extra) | @@ -374,14 +373,12 @@ for. The runtime resets the target on open, as a card's native driver resets its device; that also clears a memory error an earlier process left, and the reset -command's address chooses the memory model and, with -`VORTEX_DRAM_REFRESH=bank|all`, the bank model's refresh. With -`FOLDSIM_MEM_MODEL=bank` the runtime first writes the bank model's registers -from the platform memory it was built for (its `CONFIGS`), every channel -alike, refusing what the image cannot hold and warning when a refresh keeps -a bank past the image's horizon; the counters print at close, per channel -and in all; otherwise it makes exactly the -register accesses it always has. The command +command's address chooses, with `VORTEX_DRAM_REFRESH=bank|all`, the bank +model's refresh. Before it the runtime writes the bank model's registers from +the platform memory it was built for (`VX_CFG_PLATFORM_MEMORY_*`), every +channel alike, refusing what the image cannot hold and warning when a refresh +keeps a bank past the image's horizon; the counters print at close, per +channel and in all. The command processor model's hooks return no status, so the runtime latches a device error instead — a memory error, a failed register or memory access, or a wait that timed out — and from then on every register access the runtime core @@ -399,16 +396,12 @@ advancing for that time (60 s when it is unset). in Verilator): random target inputs, every channel stalled at random, every output token compared. s2w4i2: 3,000 target cycles, 51,000 tokens, 0 differences. -- **The memory endpoint against `AxiLatencyPipe`**, in Verilator: random - requests and device-memory latency, every target cycle's outputs and every - read's data compared; five latency and limit settings, 0 differences. - **The memory endpoint** (`hw/unittest/foldsim_mem`, in Verilator): an AXI4 master as the target, HBM with random ready and latency checking the device-side protocol, resets at random points with HBM busy, and each kind of burst; every target cycle's outputs compared with the C++ model's - (`AxiLatencyPipe`, `DramBankAxi`), every response, id and read's data - checked, none from before a reset delivered after it. Two pipe settings, - twelve bank-model settings (HBM2 and HBM3 at the default, v100, a100 and + (`DramBankAxi`), every response, id and read's data checked, none from + before a reset delivered after it. Twelve bank-model settings (HBM2 and HBM3 at the default, v100, a100 and h100 rates; small queues; row hits, random rows, one bank to the horizon, streams, read- and write-heavy) and five with refresh (per bank and all banks, at the standard intervals and faster), on the endpoint alone, with @@ -419,8 +412,9 @@ advancing for that time (60 s when it is unset). pattern; all-bank refresh costs a stream at least tRFC / tREFI of its bandwidth (9.3–12 %, against 9.0 %). - **Applications** in the in-process simulator and on the V80, against rtlsim - behind the AXI4 adapter on `AxiLatencyPipe` (`AXI_BUS=1`) with the same - memory settings: retired instructions and cycles identical (§7). + behind the AXI4 adapter on the same bank model (`AXI_BUS=1 + AXI_MEM_MODEL=bank`) with the same memory settings: retired instructions and + cycles identical (§7). - **Foldsim's own tests** (`check-foldsim` in the Foldsim repository): lit tests of every pass and of the driver, and small threaded simulators run in Verilator against their expected tokens — one and several threads, @@ -433,37 +427,38 @@ advancing for that time (60 s when it is unset). Configuration s2w4i2 (2 cores, 4 warps, 4 threads, issue width 2), seven classes: one rest-of-Vortex model, the socket (2 threads) and six unit -classes (4 threads each). +classes (4 threads each). Every setting is the configuration's +(`VX_config.toml`): 2 memory channels on the default machine's DRAM, and the +Xilinx floating-point cores, which Vivado synthesis selects; the flow's +default options (§3.1). -| | where | demo cycles | sgemm cycles | sgemm n=128 cycles | host / target cycle | -|---|---|---:|---:|---:|---:| -| Foldsim, threads in turn, in-process | Verilator | 8,702 | 415,454 | | 12.06 | -| Foldsim, ready scheduling, in-process | Verilator | 8,702 | 415,454 | | 8.00 | -| Foldsim, threads in turn | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 or 12.00 | -| Foldsim, ready scheduling | V80 | 8,702 | 415,454 | 3,497,994 | 8.00 | -| rtlsim, AXI4 with `AxiLatencyPipe` | Verilator | 8,710 | 415,462 | | | -| rtlsim, same, launched after the caches' reset sweep | Verilator | 8,702 | 415,454 | | | - -On the V80 the kernel closes at 149.8 MHz (run at 143 MHz) in 138k routed -LUTs, against 219k for the firesim backend's image of the same configuration. -sgemm n=128 takes 196 ms with ready scheduling: 50× SimX on the same host, -1.7× the firesim backend's image. - -Instruction counts equal the firesim backend's and rtlsim's. Cycle counts are -1.5–6 % below the firesim backend's, whose memory model has the same -latency-bandwidth settings but adds channel cycles `AxiLatencyPipe` does not; -Foldsim agrees with `AxiLatencyPipe` (rtlsim) instead: every memory -transaction in the same target cycle. rtlsim's harness starts the kernel 8 -cycles after reset, while each L1 bank still invalidates one set per cycle -(64 for the instruction cache), so its first fetch waits 8 cycles the -counters see; a host's launch comes thousands of cycles later. Given that -settle time, rtlsim's cycles are Foldsim's exactly. +| | where | demo cycles | sgemm cycles | host / target cycle | +|---|---|---:|---:|---:| +| Foldsim, Xilinx cores | Verilator | 6,616 (`-n64`) | 404,896 | 12.04 / 8.18 | +| Foldsim, soft FPU (`-DVX_CFG_FPU_TYPE_STD`) | Verilator | 6,616 | 399,646 | 12.04 / 8.17 | +| rtlsim, soft FPU, AXI4 adapter on the bank model (`AXI_BUS=1 AXI_MEM_MODEL=bank`, `PLATFORM_MEMORY_ID_WIDTH=5`) | Verilator | 6,624 | 399,654 | | + +rtlsim's harness starts the kernel 8 cycles after reset, while each L1 bank +still invalidates one set per cycle (64 for the instruction cache), so its +first fetch waits 8 cycles the counters see; a host's launch comes thousands +of cycles later. Given that settle time, rtlsim's cycles are Foldsim's +exactly. rtlsim must take the platform's AXI id width (4 + log2 of the +channels): at its own default of 32 the AXI adapter needs no tag buffer and +times reads differently (demo 6,552). + +**On the V80** the same configuration and options with one memory channel, +domains and issue/LSU relays (§3.2) close **250 MHz** in the image flow and run +demo, sgemm and eight FP tests (`dogfood`) at the instruction and cycle counts +of Verilator, as does the bank model on that image (demo 10,740, sgemm 404,513 +cycles; one channel). The plain flow's s2w4i2 image had closed at 149.8 MHz in +138k routed LUTs (219k for the firesim backend's image of the same +configuration). ### 7.1 The bank model -s2w4i2 in the in-process simulator (Verilator) with `FOLDSIM_MEM_MODEL=bank`, -against SimX on the same configuration (one memory bank, Ramulator behind -dram_sim) and rtlsim behind the AXI4 adapter on the same model +s2w4i2 in the in-process simulator (Verilator), against SimX on the same +configuration (one memory bank, Ramulator behind dram_sim) and rtlsim behind +the AXI4 adapter on the same model (`AXI_MEM_MODEL=bank`). Memories: the default (HBM2, 800 MT/s, 400 MHz core) and the a100 and h100 profiles' (`VX_CFG_PLATFORM_MEMORY_*` from `profiles/*.json`). @@ -478,23 +473,27 @@ and the a100 and h100 profiles' (`VX_CFG_PLATFORM_MEMORY_*` from | vecadd `-n16384` | a100 | 311,917 | | 312,529 | 312,587 | | vecadd `-n16384` | h100 | 223,875 | | 224,921 | 224,921 | +One memory channel (`-DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1`), soft FPU. Instruction counts are equal everywhere. rtlsim's cycles are Foldsim's plus -the 8-cycle launch settle it has on the pipe too (§7). Against SimX on +the 8-cycle launch settle (§7). Against SimX on Ramulator Foldsim is within 0.8 % on sgemm and vecadd at all three memories; the store-dominated demo (2,048 writes to 268 reads) is 5 % slower, and SimX's own bank backend 10 % faster: the reservation model drains a write stream differently from Ramulator's per-clock scheduler (a -mixed-traffic residual). The pipe, by contrast, has no memory to configure: -sgemm takes 415,454 cycles at every profile. A bank-model run costs host -cycles the pipe does not (memory waits): 7.3–9.5 host cycles per target cycle -against the pipe's 8.0. +mixed-traffic residual). A fixed-latency memory model, which Foldsim had +before, gave sgemm 415,454 cycles at every profile. The bank model's DRAM +clocks cost host cycles (memory waits): 7.3–12 host cycles per target cycle +on s2w4i2. ### 7.2 tinygpu: Foldsim and the firesim backend The same Vortex tree, configuration and V80 on both flows: 1 core, 2 warps, 2 threads, no L1 caches or local memory, STD FPU, one memory bank, kernel -clock 143 MHz. Every class has one instance, so Foldsim folds nothing here: -this compares the two simulators' fidelity and overheads, not threading. +clock 143 MHz, measured 2026-10-13 with the plain flow and the fixed-latency +memory model Foldsim had then (FASED's latency-bandwidth pipe, which rtlsim +keeps as `AxiLatencyPipe`). Every class has one instance, so Foldsim folds +nothing here: this compares the two simulators' fidelity and overheads, not +threading. | | Foldsim | firesim backend | |---|---:|---:| @@ -509,12 +508,15 @@ this compares the two simulators' fidelity and overheads, not threading. | `sgemm -n64` kernel time, V80 | 278 ms | 178 ms | | host cycles per target cycle, `sgemm -n64` | 3.00 ¹ | 1.84 (FMR) | -¹ With the plain flow (`FOLDSIM_FLAGS=`). The flow's default options (§3.1) -run tinygpu at 4.00 host cycles per target cycle (Verilator), cycles -unchanged: they buy Fmax for large targets with host cycles a small one pays. +¹ With the plain flow (`FOLDSIM_FLAGS=`) and the fixed-latency memory. The +flow's default options (§3.1) took it to 4.00, cycles unchanged: they buy Fmax +for large targets with host cycles a small one pays. With today's defaults (2 +channels, the bank model) tinygpu takes demo 16,375 and `sgemm -n16` 195,152 +cycles, equal to rtlsim's on the bank model, at 7.1–7.2 host cycles per target +cycle (Verilator). Instruction counts are identical on both flows, in Verilator and on the V80. -Foldsim's cycles equal rtlsim's behind the AXI4 adapter on `AxiLatencyPipe` +Foldsim's cycles equalled rtlsim's behind the AXI4 adapter on `AxiLatencyPipe` with the same memory settings (`AXI_BUS=1`) exactly; the firesim backend's are 5.0–5.1 % above, from its memory model's channel mechanics. With nothing to fold, a Foldsim target cycle costs three host cycles (the token exchange diff --git a/docs/simulation.md b/docs/simulation.md index 260d4fe3f3..5b9426ccb9 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -116,9 +116,9 @@ The flow needs CIRCT (`circt-verilog`, `circt-opt`, `firtool`) and Foldsim (the `foldsim` driver and `foldsim-opt`) from the toolchain (`./ci/toolchain_install.sh --circt --foldsim`; `CIRCT_BIN=

` and `FOLDSIM=` override them), and pyslang for the boundary lifting -(`LIFT_PYTHON`). The memory model's settings are runtime -knobs: `FOLDSIM_MEM_READ_LATENCY`, `FOLDSIM_MEM_WRITE_LATENCY` (target cycles, -30 by default) and `FOLDSIM_MEM_MAX_READS`, `FOLDSIM_MEM_MAX_WRITES` (16). +(`LIFT_PYTHON`). The memory is timed as the configuration's DRAM +(`VX_CFG_PLATFORM_MEMORY_*`), a channel per memory bank, as `dram_sim` times +it; `VORTEX_DRAM_REFRESH=bank|all` adds refresh. `VORTEX_FOLDSIM_TIMEOUT_S=` fails a launch that runs longer; a command, DCR read or reset the target leaves unanswered fails too once the target has advanced no cycle for that long, or for 60 s when it is unset. At exit the runtime reports the target and kernel-clock cycles the simulator ran diff --git a/hw/rtl/afu/foldsim/foldsim_afu.sv b/hw/rtl/afu/foldsim/foldsim_afu.sv index a1222f9bb3..db92addcd5 100644 --- a/hw/rtl/afu/foldsim/foldsim_afu.sv +++ b/hw/rtl/afu/foldsim/foldsim_afu.sv @@ -13,7 +13,9 @@ `include "foldsim_sim.vh" `include "foldsim_kernel.vh" -`include "foldsim_ports.vh" + +`define FOLDSIM_COMMA , +`define FOLDSIM_SEMI ; // The foldsim kernel: the simulator CIRCT composes of Vortex (foldsim_sim, of // the target foldsim_target), a memory endpoint on each of its AXI4 ports and @@ -64,7 +66,7 @@ module foldsim_afu #( input wire [1:0] m_axi_mem_``i``_bresp, \ input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_``i``_bid - `FOLDSIM_REPEAT(`FOLDSIM_MEM_PORTS, FOLDSIM_AXI_MEM, FOLDSIM_COMMA), + `FOLDSIM_MEM_REPEAT(FOLDSIM_AXI_MEM, FOLDSIM_COMMA), input wire s_axi_control_awvalid, output wire s_axi_control_awready, @@ -178,7 +180,7 @@ module foldsim_afu #( assign m_axi_mem_``i``_rready = hbm_rready[i]; \ assign hbm_rdata[i] = m_axi_mem_``i``_rdata - `FOLDSIM_REPEAT(`FOLDSIM_MEM_PORTS, FOLDSIM_HBM, FOLDSIM_SEMI); + `FOLDSIM_MEM_REPEAT(FOLDSIM_HBM, FOLDSIM_SEMI); `undef FOLDSIM_HBM `define FOLDSIM_IN(name) \ @@ -202,7 +204,7 @@ module foldsim_afu #( .hostClock (ap_clk), .hostReset (reset), .profile_finish (profile_finish), - `FOLDSIM_REPEAT(`FOLDSIM_MEM_PORTS, FOLDSIM_SIM_MEM, FOLDSIM_COMMA), + `FOLDSIM_MEM_REPEAT(FOLDSIM_SIM_MEM, FOLDSIM_COMMA), `FOLDSIM_IN(reset), `FOLDSIM_IN(dcr_req_valid), `FOLDSIM_IN(dcr_req_rw), @@ -227,10 +229,7 @@ module foldsim_afu #( // Ports, engines and device-memory masters: one of each per port. localparam [31:0] MEM_CAPS = {4'd0, 4'(PORTS), 4'(PORTS), 8'(MEM_SLOTS), 4'(MEM_HLOG2 - 4), 8'(PORTS)}; - wire [31:0] mem_read_latency, mem_write_latency; - wire [7:0] mem_max_reads, mem_max_writes; wire [63:0] mem_base; - wire mem_model_bank; wire [31:0] mem_clock, mem_queue; wire [15:0] mem_latency, mem_wmark; wire [12:0] mem_geom; @@ -257,12 +256,7 @@ module foldsim_afu #( ) mem ( .clk (ap_clk), .reset (reset), - .read_latency (mem_read_latency), - .write_latency (mem_write_latency), - .max_reads (mem_max_reads), - .max_writes (mem_max_writes), .base (mem_base), - .model_bank (mem_model_bank), .cfg_data_rate (mem_clock[15:0]), .cfg_core_clock(mem_clock[31:16]), .cfg_latency (mem_latency), @@ -335,12 +329,7 @@ module foldsim_afu #( .s_rvalid (s_axi_control_rvalid), .s_rready (s_axi_control_rready), .s_rdata (s_axi_control_rdata), - .mem_read_latency (mem_read_latency), - .mem_write_latency (mem_write_latency), - .mem_max_reads (mem_max_reads), - .mem_max_writes (mem_max_writes), .mem_base (mem_base), - .mem_model_bank (mem_model_bank), .mem_clock (mem_clock), .mem_latency (mem_latency), .mem_queue (mem_queue), @@ -374,3 +363,6 @@ module foldsim_afu #( `undef FOLDSIM_IN endmodule + +`undef FOLDSIM_COMMA +`undef FOLDSIM_SEMI diff --git a/hw/rtl/afu/foldsim/foldsim_ctrl.sv b/hw/rtl/afu/foldsim/foldsim_ctrl.sv index 510c47970a..ee865f9251 100644 --- a/hw/rtl/afu/foldsim/foldsim_ctrl.sv +++ b/hw/rtl/afu/foldsim/foldsim_ctrl.sv @@ -31,7 +31,6 @@ module foldsim_ctrl #( parameter DCR_DATA_W = 32, parameter QUEUE = 16, parameter RESET_CYCLES = 64, - parameter MEM_DEPTH = 16, parameter NUM_CLASSES = 1, // The memory endpoints, one per memory port of the target. parameter MEM_PORTS = 1, @@ -56,14 +55,9 @@ module foldsim_ctrl #( input wire s_rready, output reg [31:0] s_rdata, - // The memory endpoint's settings and state. - output reg [31:0] mem_read_latency, - output reg [31:0] mem_write_latency, - output reg [7:0] mem_max_reads, - output reg [7:0] mem_max_writes, + // The memory endpoints' settings and state: where the target's memory + // is, and the bank model's settings and counters (foldsim_mem_bank). output reg [63:0] mem_base, - // The bank model's settings and counters (foldsim_mem_bank). - output wire mem_model_bank, output reg [31:0] mem_clock, output reg [15:0] mem_latency, output reg [31:0] mem_queue, @@ -126,15 +120,13 @@ module foldsim_ctrl #( reg [DCR_DATA_W-1:0] rsp_data; reg [63:0] target_cycles, host_cycles, mem_stalls; reg [47:0] finishes [NUM_CLASSES]; - reg [5:0] mem_model; + reg [1:0] mem_refresh_r; reg [11:0] mem_t [18]; - // The model in effect from a reset on: the memory endpoint takes it at the - // end of every reset cycle, so a reset command's own cycle has it already. - // [1:0] the model, [4] refresh, [5] all banks at once. - wire [5:0] mem_model_now = (cur_op == `FOLDSIM_CMD_RESET) ? cur_addr[5:0] : mem_model; - assign mem_model_bank = (mem_model_now[1:0] == 2'd1); - assign mem_refresh = mem_model_now[5:4]; + // The refresh in effect from a reset on: the memory endpoint takes it at + // the end of every reset cycle, so a reset command's own cycle has it + // already. + assign mem_refresh = (cur_op == `FOLDSIM_CMD_RESET) ? cur_addr[1:0] : mem_refresh_r; for (genvar i = 0; i < 18; ++i) begin : g_mem_t assign mem_timing[i*12 +: 12] = mem_t[i]; end @@ -221,12 +213,8 @@ module foldsim_ctrl #( mem_stalls <= 0; for (integer c = 0; c < NUM_CLASSES; ++c) finishes[c] <= 0; - mem_read_latency <= 32'd30; - mem_write_latency <= 32'd30; - mem_max_reads <= 8'(MEM_DEPTH); - mem_max_writes <= 8'(MEM_DEPTH); mem_base <= 64'h40_0000_0000; - mem_model <= 6'd0; + mem_refresh_r <= 2'd0; mem_refi <= 16'd0; mem_rfc <= 16'd0; mem_clock <= {16'd400, 16'd800}; @@ -261,10 +249,10 @@ module foldsim_ctrl #( rsp_data <= rsp_d; end target_busy <= busy; - // A reset command's address chooses the memory model the - // target has from the reset on. + // A reset command's address chooses the refresh the target + // has from the reset on. if (cur_op == `FOLDSIM_CMD_RESET) - mem_model <= cur_addr[5:0]; + mem_refresh_r <= cur_addr[1:0]; // A start counts as busy until the target says it is, or plainly // never will. if (do_start) begin @@ -308,10 +296,6 @@ module foldsim_ctrl #( `FOLDSIM_REG_CMD_DATA: cmd_data <= DCR_DATA_W'(w_data); `FOLDSIM_REG_MEM_BASE_LO: mem_base[31:0] <= w_data; `FOLDSIM_REG_MEM_BASE_HI: mem_base[63:32] <= w_data; - `FOLDSIM_REG_MEM_RLAT: mem_read_latency <= (w_data == 0) ? 32'd1 : w_data; - `FOLDSIM_REG_MEM_WLAT: mem_write_latency <= (w_data == 0) ? 32'd1 : w_data; - `FOLDSIM_REG_MEM_MAXR: mem_max_reads <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); - `FOLDSIM_REG_MEM_MAXW: mem_max_writes <= (w_data == 0 || w_data > MEM_DEPTH) ? 8'(MEM_DEPTH) : 8'(w_data); `FOLDSIM_REG_MEM_CLOCK: mem_clock <= w_data; `FOLDSIM_REG_MEM_LATENCY: mem_latency <= w_data[15:0]; `FOLDSIM_REG_MEM_QUEUE: mem_queue <= w_data; @@ -349,14 +333,10 @@ module foldsim_ctrl #( `FOLDSIM_REG_HCYCLES_HI: s_rdata <= host_cycles[63:32]; `FOLDSIM_REG_MEM_BASE_LO: s_rdata <= mem_base[31:0]; `FOLDSIM_REG_MEM_BASE_HI: s_rdata <= mem_base[63:32]; - `FOLDSIM_REG_MEM_RLAT: s_rdata <= mem_read_latency; - `FOLDSIM_REG_MEM_WLAT: s_rdata <= mem_write_latency; - `FOLDSIM_REG_MEM_MAXR: s_rdata <= 32'(mem_max_reads); - `FOLDSIM_REG_MEM_MAXW: s_rdata <= 32'(mem_max_writes); `FOLDSIM_REG_MSTALL_LO: s_rdata <= mem_stalls[31:0]; `FOLDSIM_REG_MSTALL_HI: s_rdata <= mem_stalls[63:32]; `FOLDSIM_REG_NUM_CLASSES: s_rdata <= 32'(NUM_CLASSES); - `FOLDSIM_REG_MEM_MODEL: s_rdata <= 32'(mem_model); + `FOLDSIM_REG_MEM_REFRESH: s_rdata <= 32'(mem_refresh_r); `FOLDSIM_REG_MEM_CAPS: s_rdata <= MEM_CAPS; `FOLDSIM_REG_MEM_CLOCK: s_rdata <= mem_clock; `FOLDSIM_REG_MEM_LATENCY: s_rdata <= 32'(mem_latency); diff --git a/hw/rtl/afu/foldsim/foldsim_mem.sv b/hw/rtl/afu/foldsim/foldsim_mem.sv index 3c7f497d5b..5928849aca 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem.sv @@ -13,12 +13,7 @@ // The target's AXI4 memory port, as a foldsim endpoint. // -// Timing is one of two models, chosen by `model_bank` at the target's reset. -// The latency-bandwidth pipe, cycle for cycle the model of -// sim/common/axi_latency_pipe.h: a fixed latency per transaction, at most -// `max_reads` reads and `max_writes` writes in flight, one beat per cycle on -// each response channel, responses in the order their requests were accepted. -// Or a DRAM channel's banks (foldsim_mem_bank, cycle for cycle +// Timing is a DRAM channel's banks (foldsim_mem_bank, cycle for cycle // sim/common/dram_bank_model.h's DramBankAxi): each read holds a slot, and // responses leave in the order their data departs. Every output is a function // of the model's state, so all of them are offered at the start of a target @@ -27,7 +22,7 @@ // Data lives in device memory at `base` plus the target's address. A read // goes to memory the cycle the target makes it, a write the cycle it has both // its AW and its W; the response is offered to -// the target when the pipe releases it, and a target cycle that would offer +// the target when the model releases it, and a target cycle that would offer // one memory has not returned yet simply does not finish until it has. A // write is acknowledged to the target only once memory has acknowledged it, // so any later request of the target sees it. @@ -37,13 +32,14 @@ // // The target's reset is a token of the control endpoint's. A target cycle in // reset takes no request; it finishes once every transaction memory has from -// before it has completed, and drops the pipe -- requests not answered, data -// not delivered, a write's lone half -- and `error`. A response from before a +// before it has completed, and drops the model's state -- requests not +// answered, data not delivered, a write's lone half -- and `error`. A response from before a // reset never reaches the target after it. module foldsim_mem #( parameter ID_W = 4, parameter ADDR_W = 64, parameter DATA_W = 512, + // Writes in flight. parameter DEPTH = 16, // The bank model: read slots, and a ring of 2^HLOG2 DRAM clocks. parameter SLOTS = 32, @@ -59,13 +55,8 @@ module foldsim_mem #( input wire clk, input wire reset, - input wire [31:0] read_latency, - input wire [31:0] write_latency, - input wire [7:0] max_reads, - input wire [7:0] max_writes, input wire [63:0] base, - // The bank model and its settings (foldsim_mem_bank). - input wire model_bank, + // The bank model's settings (foldsim_mem_bank). input wire [15:0] cfg_data_rate, input wire [15:0] cfg_core_clock, input wire [15:0] cfg_latency, @@ -142,7 +133,7 @@ module foldsim_mem #( ); localparam CW = $clog2(DEPTH + 1); localparam PW = $clog2(DEPTH); - // Memory's side holds what either model may have in flight. + // Memory's side holds every read and write the model may have in flight. localparam HD = (SLOTS > DEPTH) ? SLOTS : DEPTH; localparam HCW = $clog2(HD + 1); localparam HPW = $clog2(HD); @@ -150,8 +141,6 @@ module foldsim_mem #( localparam LB = $clog2(DATA_W / 8); wire finish; - // The bank model is the port's timing. - wire bank; // ---- The target's request side, one token per channel per cycle ---- wire awvalid, wvalid, wlast, arvalid, rready, bready, treset; @@ -183,35 +172,17 @@ module foldsim_mem #( wire [29:5] ar_ch = {araddr[29 + CH_SHIFT -: 30 - LB], araddr[LB-1:5]}; wire [29:5] aw_ch = {awaddr[29 + CH_SHIFT -: 30 - LB], awaddr[LB-1:5]}; - // ---- The pipe's state ---- - reg [31:0] cycle; - reg [CW-1:0] pending_reads, pending_aw, pending_w; + // ---- The endpoint's state ---- + // Requests in flight: writes' halves taken and not yet acknowledged. + reg [CW-1:0] pending_aw, pending_w; - // Reads accepted and not yet driving R, and the one that is. - reg [31:0] rp_release [DEPTH]; - reg [ID_W-1:0] rp_id [DEPTH]; - reg [PW-1:0] rp_head, rp_tail; - reg [CW-1:0] rp_count; - reg cur_r_valid; - reg [ID_W-1:0] cur_r_id; - - // AW ids (and addresses) waiting for their write data, writes formed and - // not yet driving B, and the one that is. + // AW ids and channel addresses waiting for their write data. reg [ID_W-1:0] awq_id [DEPTH]; reg [29:5] awq_addr [DEPTH]; reg [PW-1:0] awq_head, awq_tail; reg [CW-1:0] awq_count; - reg [31:0] wp_release [DEPTH]; - reg [ID_W-1:0] wp_id [DEPTH]; - reg [PW-1:0] wp_head, wp_tail; - reg [CW-1:0] wp_count; - reg cur_w_valid; - reg [ID_W-1:0] cur_w_id; - - // Memory's side: read data in order, and writes acknowledged. - reg [DATA_W-1:0] rd_data [DEPTH]; - reg [PW-1:0] rd_head, rd_tail; - reg [CW-1:0] rd_count; + + // Writes memory has acknowledged, and the target has been given. reg [7:0] b_acked, b_given; // Reads memory has not answered, and writes it has not acknowledged. reg [HCW-1:0] rd_out, wr_out; @@ -226,14 +197,13 @@ module foldsim_mem #( reg [SLOTS-1:0] arrived; // ---- The model's outputs: functions of its state ---- - wire ar_ready = ~error & (bank ? (bk_room & bk_slot_free) : (32'(pending_reads) < 32'(max_reads))); - wire aw_ready = ~error & (bank ? (bk_room & (32'(pending_aw) < DEPTH)) : (32'(pending_aw) < 32'(max_writes))); - wire w_ready = ~error & (bank ? (32'(pending_w) < DEPTH) : (32'(pending_w) < 32'(max_writes))); - wire r_valid = bank ? bk_r_valid : cur_r_valid; - wire b_valid = bank ? bk_b_valid : cur_w_valid; + wire ar_ready = ~error & bk_room & bk_slot_free; + wire aw_ready = ~error & bk_room & (32'(pending_aw) < DEPTH); + wire w_ready = ~error & (32'(pending_w) < DEPTH); + wire r_valid = bk_r_valid; + wire b_valid = bk_b_valid; // R needs its data back from memory, and B its write acknowledged. - wire ok = bank ? ((~bk_r_valid | arrived[bk_r_slot]) & (~bk_b_valid | (b_acked != b_given))) - : ((~cur_r_valid | (rd_count != 0)) & (~cur_w_valid | (b_acked != b_given))); + wire ok = (~bk_r_valid | arrived[bk_r_slot]) & (~bk_b_valid | (b_acked != b_given)); wire [10:0] done; foldsim_token_out #(STAGED) o_awready (clk, reset, awready_valid, awready_ready, ok, finish, done[0]); @@ -252,13 +222,13 @@ module foldsim_mem #( assign wready_bits = w_ready; assign arready_bits = ar_ready; assign rvalid_bits = r_valid; - assign rdata_bits = bank ? slot_data[bk_r_slot] : rd_data[rd_head]; + assign rdata_bits = slot_data[bk_r_slot]; assign rlast_bits = 1'b1; - assign rid_bits = bank ? bk_r_id : cur_r_id; + assign rid_bits = bk_r_id; assign rresp_bits = 2'b00; assign bvalid_bits = b_valid; assign bresp_bits = 2'b00; - assign bid_bits = bank ? bk_b_id : cur_w_id; + assign bid_bits = bk_b_id; // A reset cycle waits for memory to answer everything it has, and every // cycle for the bank model's DRAM clocks of it. @@ -266,9 +236,9 @@ module foldsim_mem #( wire tokens_in = (&have) & (&done); assign finish = tokens_in & ~drain_wait & bk_done; assign stall = ~ok | drain_wait | (tokens_in & ~bk_done); - assign idle = (pending_reads == 0) & bk_no_reads & (pending_aw == 0) & (pending_w == 0) & drained; + assign idle = bk_no_reads & (pending_aw == 0) & (pending_w == 0) & drained; - // ---- One target cycle of the pipe, on the request side's tokens ---- + // ---- One target cycle, on the request side's tokens ---- wire ar_hs = arvalid & ar_ready; wire aw_hs = awvalid & aw_ready; wire w_hs = wvalid & w_ready; @@ -279,24 +249,7 @@ module foldsim_mem #( wire r_fire = r_valid & rready; wire b_fire = b_valid & bready; - // Reads: the latency pipe, into the stage driving R; a read released the - // cycle it arrives skips the pipe. - // A read arriving at an empty pipe is due at once if its release, - // cycle + latency - 1, is not after the cycle: a function of the latency - // alone, so the release's adder is not in front of the compare. - wire [31:0] ar_release = cycle + read_latency - 32'd1; - wire rp_empty = (rp_count == 0); - wire r_head = ~rp_empty | ar_fire; - wire [ID_W-1:0] r_head_id = rp_empty ? arid : rp_id[rp_head]; - wire r_due = rp_empty ? ($signed(32'd1 - read_latency) >= 0) - : ($signed(cycle - rp_release[rp_head]) >= 0); - wire r_release = r_head & r_due; - wire r_take = r_release & (~cur_r_valid | r_fire); - wire rp_push = ar_fire & ~(r_take & rp_empty); - wire rp_pop = r_take & ~rp_empty; - - // Writes: an AW and its W, in either order, form one write, its latency - // counted from the cycle both are in. + // Writes: an AW and its W, in either order, form one write. wire new_write = ((pending_w > pending_aw) & aw_fire) | ((pending_w < pending_aw) & w_fire) | (aw_fire & w_fire); @@ -305,40 +258,6 @@ module foldsim_mem #( wire [29:5] w_form_addr = awq_empty ? aw_ch : awq_addr[awq_head]; wire awq_pop = new_write & ~awq_empty; wire awq_push = aw_fire & ~(new_write & awq_empty); - wire [31:0] w_release = cycle + write_latency - 32'd1; - wire wp_empty = (wp_count == 0); - wire w_head = ~wp_empty | new_write; - // A write enters the pipe's RAM the host cycle after the target cycle - // that formed it, from a register, so the RAM's write enable does not end - // the path through the write's formation; a read of it in between is - // forwarded. - reg wp_wr_pend; - reg [PW-1:0] wp_wr_idx; - reg [31:0] wp_wr_rel; - reg [ID_W-1:0] wp_wr_id; - wire wp_fwd = wp_wr_pend & (wp_wr_idx == wp_head); - wire [ID_W-1:0] w_head_id = wp_empty ? w_form_id : (wp_fwd ? wp_wr_id : wp_id[wp_head]); - wire w_due = wp_empty ? ($signed(32'd1 - write_latency) >= 0) - : ($signed(cycle - (wp_fwd ? wp_wr_rel : wp_release[wp_head])) >= 0); - wire w_rel = w_head & w_due; - wire w_take = w_rel & (~cur_w_valid | b_fire); - wire wp_push = new_write & ~(w_take & wp_empty); - wire wp_pop = w_take & ~wp_empty; - - always @(posedge clk) begin - if (wp_wr_pend) begin - wp_release[wp_wr_idx] <= wp_wr_rel; - wp_id[wp_wr_idx] <= wp_wr_id; - end - if (reset) begin - wp_wr_pend <= 1'b0; - end else begin - wp_wr_pend <= finish & ~treset & ~bank & wp_push; - end - wp_wr_idx <= wp_tail; - wp_wr_rel <= w_release; - wp_wr_id <= w_form_id; - end function automatic [CW-1:0] count(input [CW-1:0] value, input inc, input dec); count = (inc & ~dec) ? value + 1'b1 : ((dec & ~inc & (value != 0)) ? value - 1'b1 : value); @@ -346,71 +265,21 @@ module foldsim_mem #( always @(posedge clk) begin if (reset) begin - cycle <= 0; - pending_reads <= 0; pending_aw <= 0; pending_w <= 0; - rp_head <= 0; - rp_tail <= 0; - rp_count <= 0; - cur_r_valid <= 1'b0; awq_head <= 0; awq_tail <= 0; awq_count <= 0; - wp_head <= 0; - wp_tail <= 0; - wp_count <= 0; - cur_w_valid <= 1'b0; error <= 1'b0; end else if (finish) begin - cycle <= cycle + 32'd1; if (treset) begin - pending_reads <= 0; pending_aw <= 0; pending_w <= 0; - rp_head <= 0; - rp_tail <= 0; - rp_count <= 0; - cur_r_valid <= 1'b0; awq_head <= 0; awq_tail <= 0; awq_count <= 0; - wp_head <= 0; - wp_tail <= 0; - wp_count <= 0; - cur_w_valid <= 1'b0; error <= 1'b0; end else begin - // The pipe's own state stays as reset under the bank model. - if (~bank) begin - if (rp_push) begin - rp_release[rp_tail] <= ar_release; - rp_id[rp_tail] <= arid; - rp_tail <= rp_tail + 1'b1; - end - if (rp_pop) - rp_head <= rp_head + 1'b1; - rp_count <= rp_count + CW'(rp_push) - CW'(rp_pop); - if (r_take) begin - cur_r_valid <= 1'b1; - cur_r_id <= r_head_id; - end else if (r_fire) begin - cur_r_valid <= 1'b0; - end - if (wp_push) - wp_tail <= wp_tail + 1'b1; - if (wp_pop) - wp_head <= wp_head + 1'b1; - wp_count <= wp_count + CW'(wp_push) - CW'(wp_pop); - if (w_take) begin - cur_w_valid <= 1'b1; - cur_w_id <= w_head_id; - end else if (b_fire) begin - cur_w_valid <= 1'b0; - end - pending_reads <= count(pending_reads, ar_fire, r_fire); - end - if (awq_push) begin awq_id[awq_tail] <= awid; awq_addr[awq_tail] <= aw_ch; @@ -453,7 +322,6 @@ module foldsim_mem #( wire wm_pop = m_wvalid & m_wready; wire form = take & new_write; wire rd_push = m_rvalid & m_rready; - wire rd_pop = take & r_fire; wire b_push = m_bvalid & m_bready; assign m_arvalid = (arq_count != 0); @@ -480,11 +348,9 @@ module foldsim_mem #( wm_data[wm_tail] <= wdata; wm_strb[wm_tail] <= wstrb; end - if (rd_push & ~bank) - rd_data[rd_tail] <= m_rdata; - // Memory answers reads in the order they were made: the bank model's - // data goes to the read's slot. - if (rd_push & bank) + // Memory answers reads in the order they were made: the data goes + // to the read's slot. + if (rd_push) slot_data[arq_slot[rs_head]] <= m_rdata; end @@ -504,9 +370,6 @@ module foldsim_mem #( wm_tail <= 0; aw_formed <= 0; w_formed <= 0; - rd_head <= 0; - rd_tail <= 0; - rd_count <= 0; rd_out <= 0; wr_out <= 0; b_acked <= 0; @@ -523,9 +386,9 @@ module foldsim_mem #( begin : arrived_ reg [SLOTS-1:0] a; a = arrived; - if (arq_push & bank) + if (arq_push) a[bk_alloc_slot] = 1'b0; - if (rd_push & bank) + if (rd_push) a[arq_slot[rs_head]] = 1'b1; arrived <= a; end @@ -541,14 +404,6 @@ module foldsim_mem #( aw_formed <= aw_formed + CW'(form) - CW'(awm_pop); w_formed <= w_formed + CW'(form) - CW'(wm_pop); - if (~bank) begin - if (rd_push) - rd_tail <= rd_tail + 1'b1; - if (rd_pop) - rd_head <= rd_head + 1'b1; - rd_count <= rd_count + CW'(rd_push) - CW'(rd_pop); - end - rd_out <= rd_out + HCW'(arq_push) - HCW'(rd_push); wr_out <= wr_out + HCW'(form) - HCW'(b_push); if (b_push) @@ -568,7 +423,6 @@ module foldsim_mem #( ) bank_model ( .clk (clk), .reset (reset), - .enable (model_bank), .cfg_data_rate (cfg_data_rate), .cfg_core_clock (cfg_core_clock), .cfg_latency (cfg_latency), @@ -603,7 +457,6 @@ module foldsim_mem #( .b_id (bk_b_id), .no_reads (bk_no_reads), .done (bk_done), - .active (bank), .stat_reads (stats[0*64 +: 64]), .stat_writes (stats[1*64 +: 64]), .stat_hits (stats[2*64 +: 64]), diff --git a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv index 5d4dd0b25f..a530f2f8e2 100644 --- a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv +++ b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv @@ -52,9 +52,6 @@ module foldsim_mem_bank #( input wire clk, input wire reset, - // The bank model is the port's timing from the next target reset on. - input wire enable, - input wire [15:0] cfg_data_rate, input wire [15:0] cfg_core_clock, input wire [15:0] cfg_latency, @@ -104,8 +101,6 @@ module foldsim_mem_bank #( output wire no_reads, // The target cycle's DRAM clocks are done. output wire done, - // The bank model is the port's timing, since the last target reset. - output reg active, // Reads, writes, row hits, misses, conflicts, the sum of read latencies // (target cycles from AR to R), refreshes, and target cycles the port @@ -130,6 +125,10 @@ module foldsim_mem_bank #( localparam BQ = 32; localparam BQW = $clog2(BQ); + // The model has its settings: it starts at the first target reset, after + // the runtime has written them. + reg active; + // ---- Configuration and its derived constants ---- wire [11:0] T [18]; for (genvar i = 0; i < 18; ++i) begin : g_t @@ -871,7 +870,7 @@ module foldsim_mem_bank #( if (reset || finish) begin // ---- The target cycle ends: the port's state for the next ---- if (finish && treset) - active <= enable; + active <= 1'b1; if (reset || treset || ~active) begin slot_busy <= '0; slots_used <= '0; @@ -969,7 +968,7 @@ module foldsim_mem_bank #( end // A clearing cycle's next host cycle clears the calendars, so the // first DRAM clock waits a host cycle (in S_PUSH). - state <= ~(treset ? enable : active) ? S_IDLE + state <= ~(treset | active) ? S_IDLE : (((treset | ~active) | new_write | ar_fire) ? S_PUSH : S_TICK); new_ready <= '0; new_acks <= '0; diff --git a/hw/rtl/afu/foldsim/foldsim_ports.vh b/hw/rtl/afu/foldsim/foldsim_ports.vh deleted file mode 100644 index c4c8d90819..0000000000 --- a/hw/rtl/afu/foldsim/foldsim_ports.vh +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The kernel's memory ports carry their index in their names, so they are -// listed by FOLDSIM_REPEAT, as VX_platform.vh's MP_REPEAT lists the native -// shells' ports. - -`ifndef FOLDSIM_PORTS_VH -`define FOLDSIM_PORTS_VH - -`define FOLDSIM_REPEAT(n,f,s) `_FOLDSIM_REPEAT_``n(f,s) -`define _FOLDSIM_REPEAT_1(f,s) `f(0) -`define _FOLDSIM_REPEAT_2(f,s) `_FOLDSIM_REPEAT_1(f,s) `s `f(1) -`define _FOLDSIM_REPEAT_3(f,s) `_FOLDSIM_REPEAT_2(f,s) `s `f(2) -`define _FOLDSIM_REPEAT_4(f,s) `_FOLDSIM_REPEAT_3(f,s) `s `f(3) -`define _FOLDSIM_REPEAT_5(f,s) `_FOLDSIM_REPEAT_4(f,s) `s `f(4) -`define _FOLDSIM_REPEAT_6(f,s) `_FOLDSIM_REPEAT_5(f,s) `s `f(5) -`define _FOLDSIM_REPEAT_7(f,s) `_FOLDSIM_REPEAT_6(f,s) `s `f(6) -`define _FOLDSIM_REPEAT_8(f,s) `_FOLDSIM_REPEAT_7(f,s) `s `f(7) - -`define FOLDSIM_COMMA , -`define FOLDSIM_SEMI ; - -`endif // FOLDSIM_PORTS_VH diff --git a/hw/rtl/afu/foldsim/foldsim_regs.vh b/hw/rtl/afu/foldsim/foldsim_regs.vh index ef5edee3e5..dcf1ba7c3c 100644 --- a/hw/rtl/afu/foldsim/foldsim_regs.vh +++ b/hw/rtl/afu/foldsim/foldsim_regs.vh @@ -22,7 +22,7 @@ `define FOLDSIM_REGS_VH `define FOLDSIM_ID 32'h464F4C44 -`define FOLDSIM_VERSION 32'd4 +`define FOLDSIM_VERSION 32'd5 `define FOLDSIM_REG_ID 12'h000 `define FOLDSIM_REG_VERSION 12'h004 @@ -41,12 +41,8 @@ `define FOLDSIM_REG_NUM_CLASSES 12'h038 `define FOLDSIM_REG_MEM_BASE_LO 12'h0C0 `define FOLDSIM_REG_MEM_BASE_HI 12'h0C4 -`define FOLDSIM_REG_MEM_RLAT 12'h0C8 -`define FOLDSIM_REG_MEM_WLAT 12'h0CC -`define FOLDSIM_REG_MEM_MAXR 12'h0D0 -`define FOLDSIM_REG_MEM_MAXW 12'h0D4 // The memory model (sw/common/foldsim_regs.h has the fields). -`define FOLDSIM_REG_MEM_MODEL 12'h0D8 +`define FOLDSIM_REG_MEM_REFRESH 12'h0D8 `define FOLDSIM_REG_MEM_CAPS 12'h0DC `define FOLDSIM_REG_MEM_CLOCK 12'h0E0 `define FOLDSIM_REG_MEM_LATENCY 12'h0E4 diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index b35fe4661b..f7a0f2736b 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -71,38 +71,33 @@ RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm \ -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(FOLDSIM_RTL_DIR)/target SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG -# One memory port per memory bank of the configuration (one unless CONFIGS -# sets VX_CFG_PLATFORM_MEMORY_NUM_BANKS), each its own memory endpoint and DRAM -# channel. AXI ids are what Vortex drives with its tag buffer engaged: the -# buffer's 4 index bits, and above them the index of the GPU's memory port the -# read came from, of which there are at most as many as memory ports. The -# FPU type is the configuration's, as -# in the native Vivado flows: DSP selects the Xilinx floating-point cores, which -# the simulator keeps as instances: their netlists are encrypted, so their -# state cannot be threaded, and each is simulated by one unthreaded copy of -# itself and a per-thread store of its results (foldsim --fold-pipelines, by -# the pipeline contract xilinx_ip_stubs.sv declares). -DVX_CFG_FPU_TYPE_STD in -# CONFIGS selects the soft FPU instead. -MEM_PORTS := $(or $(lastword $(patsubst -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=%,%, \ - $(filter -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=%,$(CONFIGS)))),1) -ifeq ($(filter 1 2 4 8,$(MEM_PORTS)),) -$(error the foldsim kernel has 1, 2, 4 or 8 memory ports (VX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(MEM_PORTS))) -endif -MEM_ID_W := $(shell python3 -c 'print(4 + ($(MEM_PORTS) - 1).bit_length())') -PLATFORM_DEFINES := -DPLATFORM_MEMORY_ID_WIDTH=$(MEM_ID_W) -ifeq ($(MEM_PORTS),1) -PLATFORM_DEFINES += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -endif ifdef PERF -PLATFORM_DEFINES += -DPERF_ENABLE +PLATFORM_DEFINES := -DPERF_ENABLE endif +# Every default is the configuration's (VX_config.toml with CONFIGS). The FPU +# type is chosen as in the native Vivado flows: DSP selects the Xilinx +# floating-point cores, which the simulator keeps as instances: their netlists +# are encrypted, so their state cannot be threaded, and each is simulated by +# one unthreaded copy of itself and a per-thread store of its results (foldsim +# --fold-pipelines, by the pipeline contract xilinx_ip_stubs.sv declares). XCONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py \ --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN) $(SYNTH_DEFINES) $(PLATFORM_DEFINES)') TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) +cfg_value = $(lastword $(patsubst -D$(1)=%,%,$(filter -D$(1)=%,$(XCONFIGS)))) +# One memory port per memory bank of the configuration, each its own memory +# endpoint and DRAM channel. AXI ids are what Vortex drives with its tag +# buffer engaged: the buffer's 4 index bits, and above them the index of the +# GPU's memory port the read came from, of which there are at most as many as +# memory ports. +MEM_PORTS := $(call cfg_value,VX_CFG_PLATFORM_MEMORY_NUM_BANKS) +ifeq ($(filter 1 2 4 8,$(MEM_PORTS)),) +$(error the foldsim kernel has 1, 2, 4 or 8 memory ports (VX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(MEM_PORTS))) +endif +MEM_ID_W := $(shell python3 -c 'print(4 + ($(MEM_PORTS) - 1).bit_length())') +PLATFORM_DEFINES += -DPLATFORM_MEMORY_ID_WIDTH=$(MEM_ID_W) # The shift that takes an address to its memory port's channel: the ports # interleave by block, or (INTERLEAVE=0) split the space, where every port's # channel holds the address as it is. -cfg_value = $(lastword $(patsubst -D$(1)=%,%,$(filter -D$(1)=%,$(XCONFIGS)))) MEM_CH_SHIFT := $(if $(filter 1,$(call cfg_value,VX_CFG_PLATFORM_MEMORY_INTERLEAVE)), \ $(shell python3 -c 'print(($(MEM_PORTS) - 1).bit_length())'),0) # The extensions' packages and include paths, as every synthesis flow finds @@ -182,13 +177,16 @@ $(GEN_SV): $(TARGET_SRCS) $(FOLDSIM_TOOLS) $(CLASSES_STAMP) $(FOLDSIM) --srcs $(TARGET_SRCS) --top $(TARGET_TOP) --out $(GEN_DIR) \ $(if $(CIRCT_BIN),--circt-bin $(CIRCT_BIN)) $(FOLDSIM_FLAGS) $(FOLDSIM_CLASSES) -# What the kernel's top needs of the configuration: its memory ports and their -# id width. +# What the kernel's top needs of the configuration: its memory ports, their id +# width, and FOLDSIM_MEM_REPEAT(f,s), `f(p) for each port p joined by `s, which +# lists the ports whose names carry their index. Spelled out here: Vivado's IP +# packager does not paste a macro's expanded argument into a macro name. +MEM_REPEAT := $(shell python3 -c 'print(" `s ".join("`f(%d)" % p for p in range($(MEM_PORTS))))') $(KERNEL_VH): FORCE @mkdir -p $(@D) @printf '%s\n' '// Generated by hw/syn/foldsim. Do not edit.' \ '`define FOLDSIM_MEM_PORTS $(MEM_PORTS)' '`define FOLDSIM_MEM_CH_SHIFT $(strip $(MEM_CH_SHIFT))' \ - '`define FOLDSIM_MEM_ID_W $(MEM_ID_W)' > $@.tmp + '`define FOLDSIM_MEM_ID_W $(MEM_ID_W)' '`define FOLDSIM_MEM_REPEAT(f,s) $(MEM_REPEAT)' > $@.tmp @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ $(KERNEL_SRCS): $(GEN_SV) $(KERNEL_VH) diff --git a/hw/unittest/foldsim_mem/main.cpp b/hw/unittest/foldsim_mem/main.cpp index c0ad6c2af8..392bbb3cc4 100644 --- a/hw/unittest/foldsim_mem/main.cpp +++ b/hw/unittest/foldsim_mem/main.cpp @@ -15,10 +15,10 @@ // simulator and the control endpoint drive it -- one token per channel per // target cycle -- by a model of an AXI4 master, against a model of HBM with // random ready and latency that checks the AXI4 protocol on the device side. -// Under each timing model and a range of its settings: +// Under a range of the bank model's settings: // * every target cycle's outputs equal the C++ model's -// (sim/common/axi_latency_pipe.h for the pipe, sim/common/dram_bank_model.h -// for the bank model), the model stepped on the same requests; +// (sim/common/dram_bank_model.h's DramBankAxi), the model stepped on the +// same requests; // * read data, write acknowledgements and ids are right; // * a reset with HBM transactions in flight finishes only once HBM has // answered them all, takes no request, and delivers nothing from before @@ -41,7 +41,6 @@ #include #include -#include #include #include "Vfoldsim_mem.h" @@ -78,10 +77,6 @@ enum class Traffic { struct Setup { const char* name; - bool bank; - // The pipe. - uint32_t rlat, wlat, maxr, maxw; - // The bank model. dram_timing::Standard type; uint32_t core_clock, data_rate, latency, queue; uint32_t rq, wq, wr_high, wr_low; @@ -94,43 +89,39 @@ struct Setup { }; const Setup kSetups[] = { - {"pipe", false, 10, 12, 16, 16, dram_timing::Standard::HBM2, 0, 0, 0, 0, 0, 0, 0, 0, + {"bank", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, Traffic::Small, 0.3, 0.25, 1.0, 3000}, - {"pipe_slow", false, 37, 5, 6, 3, dram_timing::Standard::HBM2, 0, 0, 0, 0, 0, 0, 0, 0, - Traffic::Random, 0.5, 0.3, 0.7, 3000}, - {"bank", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, - Traffic::Small, 0.3, 0.25, 1.0, 3000}, - {"bank_random", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, + {"bank_random", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, Traffic::Random, 0.5, 0.3, 0.8, 4000}, - {"bank_rows", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 3, 0, 32, 32, 26, 7, + {"bank_rows", dram_timing::Standard::HBM2, 400, 800, 3, 0, 32, 32, 26, 7, Traffic::Rows, 0.6, 0.2, 0.9, 4000}, - {"bank_stream", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, + {"bank_stream", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, Traffic::Stream, 0.7, 0.4, 0.9, 4000}, - {"bank_a100", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 1410, 3038, 190, 257, 32, 32, 26, 7, + {"bank_a100", dram_timing::Standard::HBM2, 1410, 3038, 190, 257, 32, 32, 26, 7, Traffic::Random, 0.5, 0.3, 0.8, 4000}, - {"bank_h100", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + {"bank_h100", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, Traffic::Rows, 0.5, 0.3, 0.8, 4000}, - {"bank_h100_s", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + {"bank_h100_s", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, Traffic::Stream, 0.8, 0.5, 0.95, 4000}, - {"bank_queues", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 700, 3200, 7, 5, 6, 4, 3, 2, + {"bank_queues", dram_timing::Standard::HBM2, 700, 3200, 7, 5, 6, 4, 3, 2, Traffic::Random, 0.6, 0.5, 0.7, 4000}, - {"bank_reads", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, + {"bank_reads", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, Traffic::Random, 0.9, 0.05, 0.9, 4000}, - {"bank_onebank", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + {"bank_onebank", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, Traffic::OneBank, 0.9, 0.1, 0.9, 4000}, - {"bank_onebank2", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 11, 3, 32, 32, 26, 7, + {"bank_onebank2", dram_timing::Standard::HBM2, 400, 800, 11, 3, 32, 32, 26, 7, Traffic::OneBank, 0.7, 0.3, 0.8, 4000}, - {"bank_v100", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 1530, 1752, 0, 0, 32, 32, 26, 7, + {"bank_v100", dram_timing::Standard::HBM2, 1530, 1752, 0, 0, 32, 32, 26, 7, Traffic::Rows, 0.8, 0.2, 0.9, 4000}, - {"ref_bank", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, + {"ref_bank", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, Traffic::Random, 0.5, 0.3, 0.8, 4000, DramBankModel::Refresh::PerBank}, - {"ref_bank_s", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 1410, 3038, 190, 257, 32, 32, 26, 7, + {"ref_bank_s", dram_timing::Standard::HBM2, 1410, 3038, 190, 257, 32, 32, 26, 7, Traffic::Stream, 0.8, 0.4, 0.9, 4000, DramBankModel::Refresh::PerBank}, - {"ref_all", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + {"ref_all", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, Traffic::Rows, 0.5, 0.3, 0.8, 4000, DramBankModel::Refresh::AllBank}, - {"ref_all_fast", true, 0, 0, 0, 0, dram_timing::Standard::HBM2, 400, 800, 3, 0, 32, 32, 26, 7, + {"ref_all_fast", dram_timing::Standard::HBM2, 400, 800, 3, 0, 32, 32, 26, 7, Traffic::Random, 0.6, 0.3, 0.9, 4000, DramBankModel::Refresh::AllBank, 600, 0}, - {"ref_bank_fast", true, 0, 0, 0, 0, dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, + {"ref_bank_fast", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, Traffic::Rows, 0.8, 0.2, 0.9, 4000, DramBankModel::Refresh::PerBank, 60, 0}, }; @@ -164,9 +155,7 @@ Line make_line(uint64_t addr, uint32_t version) { } // ---- The C++ models, stepped on what the target offered ---- -std::unique_ptr golden_pipe; std::unique_ptr golden_bank; -bool golden_valid = true; // the pipe models no bursts: void until a reset DramBankAxi::Config bank_config(const Setup& s) { DramBankAxi::Config c; @@ -473,9 +462,6 @@ struct Master { const ReadExp e = it->second; reads.erase(it); free_rids.insert(rid); - if (!setup->bank) - CHECK(tcycle >= e.fired + setup->rlat, "R at %lu for a read fired at %lu", - (unsigned long)tcycle, (unsigned long)e.fired); if (e.known) CHECK(rdata == e.op.data, "R data wrong at 0x%lx", (unsigned long)e.op.addr); read_done(e.op.addr); @@ -490,9 +476,6 @@ struct Master { const WriteExp e = writes.front(); writes.pop_front(); CHECK(bid == e.op.id, "B id %d, expected %d", bid, e.op.id); - if (!setup->bank) - CHECK(tcycle >= e.formed + setup->wlat, "B at %lu for a write formed at %lu", - (unsigned long)tcycle, (unsigned long)e.formed); known[e.op.addr] = e.op.data; unknown.erase(e.op.addr); pending_writes.erase(e.op.addr); @@ -639,69 +622,35 @@ void host_step() { // cycle's requests. void golden_cycle() { const Master& m = master; + auto& g = *golden_bank; if (m.reset) { - if (golden_pipe) golden_pipe->reset(); - if (golden_bank) golden_bank->reset(); - golden_valid = true; + g.reset(); return; } - if (golden_valid) { - ++compared; - if (golden_bank) { - auto& g = *golden_bank; - CHECK(o_arready == g.ar_ready(), "ARREADY %d, model %d", o_arready, g.ar_ready()); - CHECK(o_awready == g.aw_ready(), "AWREADY %d, model %d", o_awready, g.aw_ready()); - CHECK(o_wready == g.w_ready(), "WREADY %d, model %d", o_wready, g.w_ready()); - CHECK(o_rvalid == g.r_valid(), "RVALID %d, model %d", o_rvalid, g.r_valid()); - if (o_rvalid && g.r_valid()) - CHECK(uint32_t(o_rid) == g.r_id(), "RID %d, model %u", o_rid, g.r_id()); - CHECK(o_bvalid == g.b_valid(), "BVALID %d, model %d", o_bvalid, g.b_valid()); - if (o_bvalid && g.b_valid()) - CHECK(uint32_t(o_bid) == g.b_id(), "BID %d, model %u", o_bid, g.b_id()); - DramBankAxi::Inputs in; - in.ar_valid = m.ar_v; - in.ar_id = m.ar.id; - in.ar_addr = channel_addr(m.ar.addr); - in.ar_len = m.ar_len; - in.aw_valid = m.aw_v; - in.aw_id = m.aw.id; - in.aw_addr = channel_addr(m.aw.addr); - in.aw_len = m.aw_len; - in.w_valid = m.w_v; - in.w_last = m.w_last; - in.r_ready = m.rready; - in.b_ready = m.bready; - g.step(in); - } else { - auto& g = *golden_pipe; - CHECK(o_arready == g.ar_ready(), "ARREADY %d, model %d", o_arready, g.ar_ready()); - CHECK(o_awready == g.aw_ready(), "AWREADY %d, model %d", o_awready, g.aw_ready()); - CHECK(o_wready == g.w_ready(), "WREADY %d, model %d", o_wready, g.w_ready()); - CHECK(o_rvalid == g.r_valid(), "RVALID %d, model %d", o_rvalid, g.r_valid()); - if (o_rvalid && g.r_valid()) - CHECK(uint32_t(o_rid) == g.r_id(), "RID %d, model %u", o_rid, g.r_id()); - CHECK(o_bvalid == g.b_valid(), "BVALID %d, model %d", o_bvalid, g.b_valid()); - if (o_bvalid && g.b_valid()) - CHECK(uint32_t(o_bid) == g.b_id(), "BID %d, model %u", o_bid, g.b_id()); - AxiLatencyPipe::Inputs in; - in.ar_valid = m.ar_v; - in.ar_id = m.ar.id; - in.ar_len = m.ar_len; - in.aw_valid = m.aw_v; - in.aw_id = m.aw.id; - in.w_valid = m.w_v; - in.w_last = m.w_last; - in.r_ready = m.rready; - in.b_ready = m.bready; - // The pipe has no error: a burst it takes ends the comparison until a - // reset. - if ((in.ar_valid && g.ar_ready() && in.ar_len != 0) - || (in.aw_valid && g.aw_ready() && m.aw_len != 0) - || (in.w_valid && g.w_ready() && !in.w_last)) - golden_valid = false; - g.step(in); - } - } + ++compared; + CHECK(o_arready == g.ar_ready(), "ARREADY %d, model %d", o_arready, g.ar_ready()); + CHECK(o_awready == g.aw_ready(), "AWREADY %d, model %d", o_awready, g.aw_ready()); + CHECK(o_wready == g.w_ready(), "WREADY %d, model %d", o_wready, g.w_ready()); + CHECK(o_rvalid == g.r_valid(), "RVALID %d, model %d", o_rvalid, g.r_valid()); + if (o_rvalid && g.r_valid()) + CHECK(uint32_t(o_rid) == g.r_id(), "RID %d, model %u", o_rid, g.r_id()); + CHECK(o_bvalid == g.b_valid(), "BVALID %d, model %d", o_bvalid, g.b_valid()); + if (o_bvalid && g.b_valid()) + CHECK(uint32_t(o_bid) == g.b_id(), "BID %d, model %u", o_bid, g.b_id()); + DramBankAxi::Inputs in; + in.ar_valid = m.ar_v; + in.ar_id = m.ar.id; + in.ar_addr = channel_addr(m.ar.addr); + in.ar_len = m.ar_len; + in.aw_valid = m.aw_v; + in.aw_id = m.aw.id; + in.aw_addr = channel_addr(m.aw.addr); + in.aw_len = m.aw_len; + in.w_valid = m.w_v; + in.w_last = m.w_last; + in.r_ready = m.rready; + in.b_ready = m.bready; + g.step(in); } void run(uint64_t n) { @@ -776,22 +725,7 @@ void burst_case(int kind, const char* what) { void configure() { const Setup& s = *setup; - dut->read_latency = s.rlat ? s.rlat : 30; - dut->write_latency = s.wlat ? s.wlat : 30; - dut->max_reads = s.maxr ? s.maxr : 16; - dut->max_writes = s.maxw ? s.maxw : 16; dut->base = kBase; - dut->model_bank = s.bank; - if (!s.bank) { - AxiLatencyPipe::Config c; - c.read_latency = s.rlat; - c.write_latency = s.wlat; - c.max_reads = s.maxr; - c.max_writes = s.maxw; - golden_pipe = std::make_unique(c); - golden_bank.reset(); - return; - } const auto cfg = bank_config(s); const auto& t = cfg.model.t; const auto& g = cfg.model.geom; @@ -818,7 +752,6 @@ void configure() { if ((v[i] >> b) & 1) dut->cfg_t[(i * 12 + b) / 32] |= 1u << ((i * 12 + b) % 32); golden_bank = std::make_unique(cfg); - golden_pipe.reset(); } int run_setup(const Setup& s) { @@ -828,7 +761,6 @@ int run_setup(const Setup& s) { reset_cycles_done = drained_at_reset = compared = 0; hbm = Hbm(); master = Master(); - golden_valid = true; for (int i = 0; i < NUM_IN; ++i) taken[i] = false; for (int j = 0; j < NUM_OUT; ++j) got[j] = false; after_reset = false; @@ -879,21 +811,15 @@ int run_setup(const Setup& s) { if (s.traffic != Traffic::OneBank) CHECK(drained_at_reset >= 4, "only %lu resets found HBM busy", (unsigned long)drained_at_reset); - std::string extra; - if (golden_bank) { - const auto& st = golden_bank->model().stats(); - char buf[256]; - snprintf(buf, sizeof(buf), "; model since last reset: %lu reads, %lu writes, %lu hits, %lu misses, %lu conflicts, %lu refreshes, %lu deferred", - (unsigned long)st.reads, (unsigned long)st.writes, (unsigned long)st.hits, - (unsigned long)st.misses, (unsigned long)st.conflicts, (unsigned long)st.refreshes, - (unsigned long)st.deferred); - extra = buf; - } + const auto& st = golden_bank->model().stats(); printf("%-12s target cycles %lu, host cycles %lu, %lu compared; %lu reads, %lu writes; " - "%lu reset cycles, %lu with HBM busy%s: %s\n", + "%lu reset cycles, %lu with HBM busy; model since last reset: %lu reads, %lu writes, " + "%lu hits, %lu misses, %lu conflicts, %lu refreshes, %lu deferred: %s\n", s.name, (unsigned long)tcycle, (unsigned long)host_cycle, (unsigned long)compared, (unsigned long)master.n_r, (unsigned long)master.n_b, (unsigned long)reset_cycles_done, - (unsigned long)drained_at_reset, extra.c_str(), + (unsigned long)drained_at_reset, (unsigned long)st.reads, (unsigned long)st.writes, + (unsigned long)st.hits, (unsigned long)st.misses, (unsigned long)st.conflicts, + (unsigned long)st.refreshes, (unsigned long)st.deferred, failures == failures_before ? "ok" : "FAILED"); delete dut; dut = nullptr; diff --git a/sim/foldsim/foldsim_emul.cpp b/sim/foldsim/foldsim_emul.cpp index 570ccd527b..35c3d63079 100644 --- a/sim/foldsim/foldsim_emul.cpp +++ b/sim/foldsim/foldsim_emul.cpp @@ -45,7 +45,7 @@ constexpr uint64_t kPage = 4096; constexpr uint32_t kBeat = 64; #ifndef FOLDSIM_MEM_PORTS -#define FOLDSIM_MEM_PORTS 1 +#error "FOLDSIM_MEM_PORTS comes from the generated foldsim_kernel.vh" #endif static_assert(FOLDSIM_MEM_PORTS >= 1 && FOLDSIM_MEM_PORTS <= 8, "the kernel has 1 to 8 memory ports"); diff --git a/sw/common/foldsim_regs.h b/sw/common/foldsim_regs.h index 95f44728a2..e9949bd499 100644 --- a/sw/common/foldsim_regs.h +++ b/sw/common/foldsim_regs.h @@ -19,7 +19,7 @@ #pragma once #define FOLDSIM_ID 0x464F4C44u -#define FOLDSIM_VERSION 4u +#define FOLDSIM_VERSION 5u #define FOLDSIM_REG_ID 0x000u #define FOLDSIM_REG_VERSION 0x004u @@ -38,19 +38,13 @@ #define FOLDSIM_REG_NUM_CLASSES 0x038u // classes of threaded models #define FOLDSIM_REG_MEM_BASE_LO 0x0C0u // device address of target address 0 #define FOLDSIM_REG_MEM_BASE_HI 0x0C4u -#define FOLDSIM_REG_MEM_RLAT 0x0C8u // read latency, target cycles -#define FOLDSIM_REG_MEM_WLAT 0x0CCu // write latency, target cycles -#define FOLDSIM_REG_MEM_MAXR 0x0D0u // most reads in flight -#define FOLDSIM_REG_MEM_MAXW 0x0D4u // most writes in flight #define FOLDSIM_REG_FINISH(c) (0x100u + 8u * (c)) // threads of class c finished, lo/hi -// The memory model: the latency-bandwidth pipe above, or the bank model -// (hw/rtl/afu/foldsim/foldsim_mem_bank.sv), the memory port's timing as a DRAM -// channel's banks. A reset command's CMD_ADDR chooses it, a FOLDSIM_MEM_MODEL_* -// the target has from that reset on (the kernel's own reset: the pipe). Times -// are DRAM clocks. -#define FOLDSIM_REG_MEM_MODEL 0x0D8u // read only: [1:0] the FOLDSIM_MEM_MODEL_* chosen, - // [5:4] the FOLDSIM_MEM_REFRESH_* +// The memory model: each memory port's timing as a DRAM channel's banks +// (hw/rtl/afu/foldsim/foldsim_mem_bank.sv). Its settings take effect at the +// next target reset; a reset command's CMD_ADDR is the FOLDSIM_MEM_REFRESH_* +// the target has from that reset on. Times are DRAM clocks. +#define FOLDSIM_REG_MEM_REFRESH 0x0D8u // read only: the FOLDSIM_MEM_REFRESH_* in effect #define FOLDSIM_REG_MEM_CAPS 0x0DCu // read only: FOLDSIM_MEM_CAPS_* #define FOLDSIM_REG_MEM_CLOCK 0x0E0u // [15:0] data rate (MT/s), [31:16] core clock (MHz) #define FOLDSIM_REG_MEM_LATENCY 0x0E4u // fixed read latency, target cycles @@ -65,11 +59,9 @@ #define FOLDSIM_REG_MEM_TIMING(i) (0x800u + 4u * (i)) // FOLDSIM_MEM_T_*, 12 bits each #define FOLDSIM_REG_MEM_STATS(p, i) (0xC00u + 0x40u * (p) + 8u * (i)) // port p's FOLDSIM_MEM_STAT_*, lo/hi -#define FOLDSIM_MEM_MODEL_PIPE 0u -#define FOLDSIM_MEM_MODEL_BANK 1u -// A reset command's address: the model, and the bank model's refresh. -#define FOLDSIM_MEM_REFRESH_BANK 0x10u // per bank, the banks in turn -#define FOLDSIM_MEM_REFRESH_ALL 0x30u // all banks at once +#define FOLDSIM_MEM_REFRESH_OFF 0u +#define FOLDSIM_MEM_REFRESH_BANK 1u // per bank, the banks in turn +#define FOLDSIM_MEM_REFRESH_ALL 3u // all banks at once #define FOLDSIM_MEM_CAPS_PORTS(c) ((c) & 0xffu) // memory ports, a channel each #define FOLDSIM_MEM_CAPS_HORIZON(c) (1u << ((((c) >> 8) & 0xfu) + 4u)) // ring, DRAM clocks diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp index ae313d10f4..33eb72e087 100644 --- a/sw/runtime/foldsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -252,11 +252,6 @@ class transport_t { #endif -uint64_t env_or(const char* name, uint64_t value) { - const char* s = getenv(name); - return (s != nullptr && s[0] != '\0') ? strtoull(s, nullptr, 0) : value; -} - // What the host spent on the transport: register accesses, and device-memory // transfers, with their wall time. struct transport_stats_t { @@ -329,18 +324,8 @@ class vx_device { version, FOLDSIM_VERSION); return -1; } - // The memory model: a latency-bandwidth pipe, 30-cycle latency and 16 - // transactions in flight each way unless set; or, with - // FOLDSIM_MEM_MODEL=bank, the platform memory's DRAM as the bank model - // times it. - const char* model = getenv("FOLDSIM_MEM_MODEL"); - const std::string model_name = (model != nullptr) ? model : ""; - if (!model_name.empty() && model_name != "pipe" && model_name != "bank") { - fprintf(stderr, "[VXDRV] FOLDSIM_MEM_MODEL must be pipe or bank (got '%s')\n", model); - return -1; - } - bank_model_ = (model_name == "bank"); - // VORTEX_DRAM_REFRESH=off|bank|all: the bank model's refresh, as the + // The memory model: the platform memory's DRAM, as the bank model times + // it. VORTEX_DRAM_REFRESH=off|bank|all: its refresh, as the // simulators' bank model takes it. const char* refresh = getenv("VORTEX_DRAM_REFRESH"); const std::string refresh_name = (refresh != nullptr) ? refresh : ""; @@ -352,32 +337,19 @@ class vx_device { fprintf(stderr, "[VXDRV] VORTEX_DRAM_REFRESH must be off, bank or all (got '%s')\n", refresh); return -1; } - if (refresh_ != 0 && !bank_model_) { - fprintf(stderr, "[VXDRV] VORTEX_DRAM_REFRESH needs FOLDSIM_MEM_MODEL=bank\n"); + if (this->setup_bank_model() != 0) return -1; - } - if (bank_model_ && this->setup_bank_model() != 0) - return -1; - const uint32_t rlat = env_or("FOLDSIM_MEM_READ_LATENCY", 30); - const uint32_t wlat = env_or("FOLDSIM_MEM_WRITE_LATENCY", 30); - const uint32_t maxr = env_or("FOLDSIM_MEM_MAX_READS", 16); - const uint32_t maxw = env_or("FOLDSIM_MEM_MAX_WRITES", 16); if (reg_write(FOLDSIM_REG_MEM_BASE_LO, uint32_t(kMemBase)) != 0 || reg_write(FOLDSIM_REG_MEM_BASE_HI, uint32_t(kMemBase >> 32)) != 0 || - reg_write(FOLDSIM_REG_MEM_RLAT, rlat) != 0 || - reg_write(FOLDSIM_REG_MEM_WLAT, wlat) != 0 || - reg_write(FOLDSIM_REG_MEM_MAXR, maxr) != 0 || - reg_write(FOLDSIM_REG_MEM_MAXW, maxw) != 0 || reg_write(FOLDSIM_REG_CTRL, 1) != 0) return -1; // A target left running by an earlier process is reset, as a card's // native driver resets its device on open: Vortex counts VX_CFG_RESET_DELAY // cycles of reset before it takes a request. The reset also clears a // memory error that process left, so only the state after it counts, and - // chooses the memory model. + // starts the memory model with its settings and refresh. uint32_t s = 0; - const uint32_t mem_model = (bank_model_ ? FOLDSIM_MEM_MODEL_BANK : FOLDSIM_MEM_MODEL_PIPE) | refresh_; - if (command(FOLDSIM_CMD_RESET, mem_model, 64) != 0 || wait_idle(false) != 0 || status(&s) != 0) + if (command(FOLDSIM_CMD_RESET, refresh_, 64) != 0 || wait_idle(false) != 0 || status(&s) != 0) return -1; if (mem_error(s)) return -1; @@ -452,7 +424,7 @@ class vx_device { return -1; mem_ports_ = FOLDSIM_MEM_CAPS_PORTS(caps); if (mem_ports_ == 0) { - fprintf(stderr, "[VXDRV] this foldsim image has no bank model\n"); + fprintf(stderr, "[VXDRV] this foldsim image reports no memory ports\n"); return -1; } dram_timing::Standard standard; @@ -564,7 +536,7 @@ class vx_device { if (!line.empty()) fprintf(stderr, "[foldsim] class busy:%s\n", line.c_str()); // The bank model's counters, per channel and, with several, in all. - if (bank_model_) { + { uint64_t total[FOLDSIM_MEM_NUM_STATS] = {}; for (uint32_t p = 0; p <= mem_ports_; ++p) { uint64_t st[FOLDSIM_MEM_NUM_STATS]; @@ -793,7 +765,6 @@ class vx_device { std::chrono::steady_clock::time_point launch_start_; bool failed_ = false; bool open_ = false; - bool bank_model_ = false; uint32_t refresh_ = 0; uint32_t mem_ports_ = 1; vortex::CommandProcessor cp_; From 26b8dad1440f21e52e70bddfb42aa57c28dea303 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 18 Oct 2026 12:49:24 -0700 Subject: [PATCH 236/336] ci: Foldsim parity against rtlsim (tinygpu demo and sgemm, exact) A model_parity case now names its two drivers in `legs:` (simx and rtlsim by default). ci/testcases/foldsim.yaml runs [foldsim, rtlsim] at tolerance 0: tinygpu demo and sgemm -n16 in the foldsim driver's in-process simulator against rtlsim behind the AXI4 adapter on the same DRAM bank model, at the AXI id width the Foldsim kernel drives (4 + log2 of the toml's 2 banks). The soft FPU, as the Xilinx cores' simulation models need Vivado. The planner's per-cell `needs` covered only the cases the event's filters selected, while the cell's -m expression runs all of them; they now cover every case the cell runs. ci.yml installs the foldsim prebuilt and pyslang for a cell that needs foldsim (the model_parity cell). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit de99936e6fadfecae22d49235279ea2d91d3a542) Integration (phase 2): the parity legs keep phase 2's counter comparison (parity_counters, against the model and reference legs' output), the per-check shard budget, and the release_parity check; a cell's needs follow this commit (every case its -m expression runs). --- .github/workflows/ci.yml | 7 ++++ ci/test_runner.py | 37 +++++++++--------- ci/testcase.py | 35 ++++++++++++----- ci/testcases/foldsim.yaml | 52 ++++++++++++++++++++++++++ docs/designs/continuous_integration.md | 16 ++++++-- docs/designs/foldsim.md | 4 +- 6 files changed, 120 insertions(+), 31 deletions(-) create mode 100644 ci/testcases/foldsim.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a16733cf0e..5744db4ed4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,6 +232,13 @@ jobs: mpi) sudo bash ./ci/install_dependencies.sh --mpi ;; gem5) sudo bash ./ci/install_dependencies.sh --gem5 ;; esac + # A check's cell gathers cases from every category, so per-case tools are + # keyed on the cell's needs, not its name. + - name: Per-case tools + if: contains(matrix.needs, 'foldsim') + run: | + (cd build${{ matrix.xlen }} && bash ci/toolchain_install.sh --foldsim) + pip install --quiet pyslang==12.0.0 - name: Run cell run: | cd build${{ matrix.xlen }} diff --git a/ci/test_runner.py b/ci/test_runner.py index 69f9eefbde..446d12c1bb 100644 --- a/ci/test_runner.py +++ b/ci/test_runner.py @@ -11,11 +11,12 @@ its failure is expected and does not fail CI (and an unexpected pass shows as XPASS). Use it only for triaged, documented breakage. -A `check: model_parity` case runs the same app/args/configs on simx and rtlsim -and compares the runtime's final `PERF: instrs=…, cycles=…` summary: retired -instructions must match exactly (both are deterministic ISA-level executions, -so any delta is functional divergence) and cycles must agree within the case's -`tolerance` (SimX is the timing model of the RTL, not just a functional oracle). +A `check: model_parity` case runs the same app/args/configs on its two `legs` +(simx and rtlsim by default) and compares the runtime's final `PERF: instrs=…, +cycles=…` summary: retired instructions must match exactly (both are +deterministic ISA-level executions, so any delta is functional divergence) and +cycles must agree within the case's `tolerance` (SimX is the timing model of the +RTL, not just a functional oracle; Foldsim is cycle-exact, so its tolerance is 0). A case with `parity_counters` also compares that MPM class's counters. """ @@ -126,21 +127,23 @@ def _release_parity(case, xlen): def _model_parity(case, xlen): - simx_instrs, simx_cycles, simx_out = _run_one(case, xlen, "simx") - rtl_instrs, rtl_cycles, rtl_out = _run_one(case, xlen, "rtlsim") - gap = abs(rtl_cycles - simx_cycles) / float(rtl_cycles) - print("PARITY: {}: instrs simx={} rtlsim={}, cycles simx={} rtlsim={}, " + model, ref = case.legs + model_instrs, model_cycles, model_out = _run_one(case, xlen, model) + ref_instrs, ref_cycles, ref_out = _run_one(case, xlen, ref) + gap = abs(ref_cycles - model_cycles) / float(ref_cycles) + print("PARITY: {}: instrs {m}={} {r}={}, cycles {m}={} {r}={}, " "gap={:.2%} (tolerance {:.0%})".format( - case.id, simx_instrs, rtl_instrs, simx_cycles, rtl_cycles, - gap, case.tolerance)) - assert simx_instrs == rtl_instrs, \ - "{}: retired-instruction mismatch (simx={}, rtlsim={}) — functional " \ - "divergence, not a timing gap".format(case.id, simx_instrs, rtl_instrs) + case.id, model_instrs, ref_instrs, model_cycles, ref_cycles, + gap, case.tolerance, m=model, r=ref)) + assert model_instrs == ref_instrs, \ + "{}: retired-instruction mismatch ({m}={}, {r}={}) — functional " \ + "divergence, not a timing gap".format(case.id, model_instrs, ref_instrs, + m=model, r=ref) assert gap <= case.tolerance, \ - "{}: cycle gap {:.2%} exceeds tolerance {:.0%} (simx={}, rtlsim={})".format( - case.id, gap, case.tolerance, simx_cycles, rtl_cycles) + "{}: cycle gap {:.2%} exceeds tolerance {:.0%} ({m}={}, {r}={})".format( + case.id, gap, case.tolerance, model_cycles, ref_cycles, m=model, r=ref) if case.parity_counters: - _compare_counters(case, simx_out, rtl_out, simx_cycles, rtl_cycles) + _compare_counters(case, model_out, ref_out, model_cycles, ref_cycles) def _perf_gate(case, xlen, update): diff --git a/ci/testcase.py b/ci/testcase.py index 3998238844..1f577d0741 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -54,6 +54,10 @@ # simx<->rtlsim cycle-parity default: both timing models must agree within 5%. DEFAULT_PARITY_TOLERANCE = 0.05 +DEFAULT_PARITY_LEGS = ["simx", "rtlsim"] +# A parity leg runs through blackbox. foldsim is a leg but not a cell driver: it +# generates and builds its simulator inside the run, so there is no sim build. +PARITY_DRIVERS = VALID_DRIVERS | {"foldsim"} # Per-case wall times (seconds, per xlen) measured by a CI run, recorded by # ci/record_runtimes.py from the run's junit reports. A cell whose recorded total @@ -121,6 +125,9 @@ def __init__(self, category, entry, driver, defaults): # must pass the class with --perf=N): `exact` counters must match, and # `tolerance` counters must agree within the case tolerance. self.parity_counters = dict(entry.get("parity_counters", {})) + # The two drivers a model_parity case runs: the model, then the RTL it + # is checked against. + self.legs = list(entry.get("legs", defaults.get("legs", DEFAULT_PARITY_LEGS))) self.authored_drivers = ("driver" in entry) or ("drivers" in entry) # make-run / script fields self.dir = entry.get("dir", "") @@ -487,20 +494,22 @@ def cmd_matrix(args): if xfilter and xlen not in xfilter: continue name = cell_name(c) - key = (name, drv, xlen) - cell = cells.setdefault(key, { - "category": name, "driver": drv, "xlen": xlen, "needs": set(), + cells.setdefault((name, drv, xlen), { + "category": name, "driver": drv, "xlen": xlen, }) - cell["needs"].update(c.needs) - members = {} + # A cell's needs are those of every case its -m expression runs, not only of + # the cases this event's filters planned it for. + members, needs = {}, {} for c in cases: if c.tier in OPT_IN_TIERS: continue for xlen in c.xlens: - members.setdefault((cell_name(c), c.marker_driver or "host", xlen), []).append(c.id) + key = (cell_name(c), c.marker_driver or "host", xlen) + members.setdefault(key, []).append(c.id) + needs.setdefault(key, set()).update(c.needs) out = [] for key, cell in cells.items(): - cell["needs"] = sorted(cell["needs"]) + cell["needs"] = sorted(needs.get(key, ())) shards, _ = shard_plan(members.get(key, []), key[2], budget=CELL_SHARD_BUDGET_S.get(key[0], SHARD_BUDGET_S)) if shards == 1: @@ -603,9 +612,15 @@ def cmd_lint(args): errors.append("{}: check cases must be via blackbox".format(c.id)) if c.authored_drivers: errors.append("{}: check cases must not set driver/drivers " - "(the runner drives simx and rtlsim)".format(c.id)) - if not (0.0 < c.tolerance < 1.0): - errors.append("{}: tolerance must be in (0, 1)".format(c.id)) + "(the runner drives the legs itself)".format(c.id)) + if len(c.legs) != 2 or len(set(c.legs)) != 2 or \ + not set(c.legs) <= PARITY_DRIVERS: + errors.append("{}: legs must be two distinct drivers of {}".format( + c.id, sorted(PARITY_DRIVERS))) + # 0 asks for exact cycles, from a model that is cycle-exact by + # construction. + if not (0.0 <= c.tolerance < 1.0): + errors.append("{}: tolerance must be in [0, 1)".format(c.id)) if c.parity_counters: pc = c.parity_counters if c.check != "model_parity": diff --git a/ci/testcases/foldsim.yaml b/ci/testcases/foldsim.yaml new file mode 100644 index 0000000000..c1d37f73a5 --- /dev/null +++ b/ci/testcases/foldsim.yaml @@ -0,0 +1,52 @@ +# Foldsim suite: the threaded simulator hw/syn/foldsim compiles from Vortex +# (docs/designs/foldsim.md), checked against the RTL it was compiled from. +# +# check: model_parity with legs [foldsim, rtlsim] and tolerance 0. Foldsim is +# cycle-exact by construction, so retired instructions AND cycles must be +# identical. The reference is rtlsim behind the AXI4 adapter on the same +# DRAM bank model (AXI_BUS=1 AXI_MEM_MODEL=bank); the env reaches only +# rtlsim's build. Exactness holds at any workload size, so the configuration +# is the smallest: tinygpu, generated and simulated in Verilator by the +# foldsim driver's in-process target. +# +# The configuration is the toml's but for two choices of the test: +# - FPU_TYPE_STD: the toml's FPU under Vivado synthesis is the Xilinx cores, +# whose simulation models need a Vivado install. +# - PLATFORM_MEMORY_ID_WIDTH=5 for rtlsim: the AXI id width the Foldsim kernel +# drives with Vortex's tag buffer engaged, 4 + log2(memory banks), which the +# flow derives from the toml's 2 banks. At rtlsim's default of 32 there is no +# tag buffer and cycles differ. +# +# needs: foldsim -- the foldsim prebuilt and pyslang (the boundary lifting); ci.yml +# installs them in a cell whose cases need them. circt is in the lite toolchain. +category: foldsim +defaults: + xlen: + - 32 + tier: full + needs: + - foldsim + legs: + - foldsim + - rtlsim + tolerance: 0 + configs: >- + -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=1 -DVX_CFG_NUM_WARPS=2 + -DVX_CFG_NUM_THREADS=2 -DVX_CFG_ICACHE_DISABLE -DVX_CFG_DCACHE_DISABLE + -DVX_CFG_LMEM_DISABLE -DVX_CFG_FPU_TYPE_STD -DPLATFORM_MEMORY_ID_WIDTH=5 +tests: +- id: parity-demo + check: model_parity + via: blackbox + app: demo + env: + AXI_BUS: 1 + AXI_MEM_MODEL: bank +- id: parity-sgemm + check: model_parity + via: blackbox + app: sgemm + args: -n16 + env: + AXI_BUS: 1 + AXI_MEM_MODEL: bank diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index e8c5591ea8..aee40a5df2 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -133,7 +133,10 @@ A case with `known_issue:` still builds and runs. `conftest.py` marks it `xfail`, so its failure does not fail the run; if it starts passing it surfaces as `XPASS`. Reserve it for triaged, documented breakage. `needs:` does not skip either — it records what a cell must provision, and a -missing dependency is a real failure. +missing dependency is a real failure. A cell's `needs` cover every case its +`-m` expression selects, and the workflow provisions per-case tools (the +foldsim prebuilt) from them rather than from the cell's name, because a +check's cell gathers cases from every category. #### Lint rules @@ -195,9 +198,9 @@ A check is an assertion across runs rather than a clean exit. | | `model_parity` | `perf_gate` | |---|---|---| -| compares | SimX against rtlsim, same case | rtlsim against a checked-in baseline | +| compares | a model against rtlsim, same case (SimX by default; Foldsim) | rtlsim against a checked-in baseline | | asserts | instructions equal; cycles within tolerance | cycles within tolerance; instructions equal | -| tolerance | 5 %, overridable per case or per category | 2 %, one constant | +| tolerance | 5 %, overridable per case or per category (0 for Foldsim) | 2 %, one constant | | golden data | none | `ci/baselines/perf/.json` | | tier | `full` | `full` | | pinned to | the rtlsim driver, which elaborates the RTL | the rtlsim driver | @@ -227,6 +230,13 @@ its own parity case with only that extension enabled, so a regression is attributable. Workloads are sized so that steady state dominates — a tiny kernel is all boot and dispatch skew, and its gap ratio is noise. +A case's `legs:` names its two drivers, the model and then the reference; +the default is `[simx, rtlsim]`. `ci/testcases/foldsim.yaml` runs +`[foldsim, rtlsim]` at tolerance 0: Foldsim is compiled from the RTL and is +cycle-exact by construction, so any cycle difference is a defect. A tiny +configuration is enough there. Its reference is rtlsim behind the AXI4 adapter +on the same DRAM bank model (`AXI_BUS=1 AXI_MEM_MODEL=bank`). + **Never widen a tolerance to absorb a divergence.** Model the behavior, or mark the case `known_issue` while it is investigated. diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index f0d247ab36..81e8305ba8 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -414,7 +414,9 @@ advancing for that time (60 s when it is unset). - **Applications** in the in-process simulator and on the V80, against rtlsim behind the AXI4 adapter on the same bank model (`AXI_BUS=1 AXI_MEM_MODEL=bank`) with the same memory settings: retired instructions and - cycles identical (§7). + cycles identical (§7). CI holds this for tinygpu demo and sgemm in the + in-process simulator (`ci/testcases/foldsim.yaml`, a `model_parity` check at + tolerance 0). - **Foldsim's own tests** (`check-foldsim` in the Foldsim repository): lit tests of every pass and of the driver, and small threaded simulators run in Verilator against their expected tokens — one and several threads, From 304b6776c5189b318e56525554c1b61cca0e4af2 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 18 Oct 2026 14:09:23 -0700 Subject: [PATCH 237/336] docs/designs/foldsim: the toml-default s2w4i2 image on the V80 at 250 MHz Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 46a7785a64b85a6cc967a14c6166aff8929d17ed) --- docs/designs/foldsim.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 81e8305ba8..0b96ad504b 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -448,11 +448,13 @@ exactly. rtlsim must take the platform's AXI id width (4 + log2 of the channels): at its own default of 32 the AXI adapter needs no tag buffer and times reads differently (demo 6,552). -**On the V80** the same configuration and options with one memory channel, -domains and issue/LSU relays (§3.2) close **250 MHz** in the image flow and run -demo, sgemm and eight FP tests (`dogfood`) at the instruction and cycle counts -of Verilator, as does the bank model on that image (demo 10,740, sgemm 404,513 -cycles; one channel). The plain flow's s2w4i2 image had closed at 149.8 MHz in +**On the V80** this configuration, with every setting the toml's and the +flow's default options, closes **250 MHz** in the image flow (226k LUTs, 195k +FFs and 20 DSPs in the kernel). It runs demo, sgemm and eight FP tests +(`dogfood`) at Verilator's instruction and cycle counts: demo 6,616, sgemm +404,896, and dogfood 21 1,657,630 cycles, at 8.1–8.2 host cycles per target +cycle. A one-channel variant with domains and issue/LSU relays (§3.2) also +closed 250 MHz and ran the same tests exactly. The plain flow's s2w4i2 image had closed at 149.8 MHz in 138k routed LUTs (219k for the firesim backend's image of the same configuration). From 9cb96c76b7bbaabcbfd23ce2bd65750d69b086e1 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 21 Oct 2026 18:42:14 -0700 Subject: [PATCH 238/336] rtl: no same-cycle input path to what a slice, core or socket exports With compute slices (FETCH_WIDTH = ISSUE_WIDTH = every execution-unit width), no forward output of a slice, of the core or of a socket now depends on a same-cycle input. Each was a combinational path across a module boundary that the module's owner should register, and each is fixed by moving logic to the consumer or to the issuer, with no register, no cycle and no new state added: - busy: a level covers only the requests it issues. VX_cta_dispatch no longer ORs the accept cycle of a launch; the socket ORs its cores' launch accepts and its MMU's translations into its own registered busy, and the cluster ORs the launch and DCR beats it presents to its sockets. The scheduler's cycle counter still counts the accept cycle. busy at the AFU is unchanged, cycle for cycle; the top level's DCR term stays, as the AFU's own request. - retire: with several warp-control requesters, a compute slice retires an exiting warp from the grant of its own exit request (SLICE_RETIRE), the same cycle the core's CTA accounting takes it, rather than through the core's arbiter. With a sole requester the core's retirement is already registered and is kept. - I-cache flush response: the frontend slice drops a response whose flush select bit is set, so the core passes port 0's response through without its demux; the other ports clear that bit as a constant. - all_ibuf_full: each slice exports whether all its warps' ibuffers are full (its registered flags), and every slice ANDs the slices' flags into the core-wide view the core used to register from their next state: the same value in every cycle. The scheduler's idle counter takes the two candidate views a slice computes and selects by the same AND. Checked cycle-exact against the base on rtlsim, demo and sgemm with every perf counter identical, in six configurations: the default, stage-sliced, compute slices with 1 and 2 cores per socket, backend slices with a fetch crossbar, and compute slices with VM. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0c6105b769c985ffd8407982dc8092061de9456a) Integration (phase 2): phase 2 moved the LSU memory paths into the blocks (lsu_busy, mem_unit_busy). The core's busy drops only ~mmu_empty, which is the socket's MMU drain (mmu_drained) seen through an input; the socket ORs its MMU drain into its own busy, alongside phase 2's per-core lsu_busy. --- hw/rtl/VX_cluster.sv | 9 ++++- hw/rtl/VX_gpu_pkg.sv | 4 ++ hw/rtl/VX_socket.sv | 23 ++++++++--- hw/rtl/core/VX_compute_slice.sv | 55 +++++++++++++++++++++++++- hw/rtl/core/VX_core.sv | 53 +++++++++++++++++-------- hw/rtl/core/VX_cta_dispatch.sv | 14 +++---- hw/rtl/core/VX_frontend_slice.sv | 26 +++++++++++- hw/rtl/core/VX_scheduler.sv | 38 ++++++++++-------- hw/rtl/core/VX_warp_sched.sv | 12 ++++-- hw/rtl/interfaces/VX_slice_sched_if.sv | 27 +++++++------ 10 files changed, 194 insertions(+), 67 deletions(-) diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 6121d36926..fb2fe0adce 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -488,10 +488,15 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; // combinationally -- the beat presented at the cluster input and any beat resident // in a per-socket output skid -- so a launch in transit is never invisible on the // presented cycle. Child (per_socket_busy) stays registered. + // A socket's busy has no path from its inputs: the launch and DCR beats this + // level presents to the sockets join the registered aggregation here. wire [NUM_SOCKETS-1:0] per_socket_kmu_valid; + wire [NUM_SOCKETS-1:0] per_socket_dcr_valid; for (genvar i = 0; i < NUM_SOCKETS; ++i) begin : g_kmu_link_valid assign per_socket_kmu_valid[i] = per_socket_kmu_bus_if[i].valid; + assign per_socket_dcr_valid[i] = per_socket_dcr_bus_if[i].req_valid; end + wire socket_issued_busy = (| per_socket_kmu_valid) | (| per_socket_dcr_valid); `ifdef VX_CFG_EXT_RASTER_ENABLE // The merge->distribute link is a registered cluster-internal launch bus; a beat // resident in it is covered here just like the input and per-socket links. @@ -502,9 +507,9 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; wire busy_r; `ifdef EXT_GFX_ANY_ENABLE - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | gfx_busy | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | socket_issued_busy | gfx_busy | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); `else - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | socket_issued_busy | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); `endif assign busy = busy_r | dcr_bus_if.req_valid | launch_link_busy; diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 4dee25d424..c1bd73d290 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -757,6 +757,10 @@ package VX_gpu_pkg; && (!`VX_CFG_EXT_F_ENABLED || (`VX_CFG_NUM_FPU_BLOCKS == `VX_CFG_ISSUE_WIDTH)) && (!`VX_CFG_EXT_TCU_ENABLED || (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH)); localparam COMPUTE_SLICES = BACKEND_SLICES && (`VX_CFG_FETCH_WIDTH == `VX_CFG_ISSUE_WIDTH); + // A compute slice whose warp-control requests are arbitrated with other + // slices' retires its own warps when its exit request is granted. With a + // sole requester, the core retires them from its registered request. + localparam SLICE_RETIRE = COMPUTE_SLICES && (`VX_CFG_NUM_SFU_BLOCKS > 1); /////////////////////////////// TENSOR UNIT /////////////////////////////// diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 3b360dfee8..b5bd01799a 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -976,14 +976,23 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; ); end - // Launch liveness: fold this level's launch-link `valid` at both ends into busy - // combinationally -- the beat presented at the socket input and any beat resident - // in a per-core output skid -- so an in-transit launch stays visible on the - // presented cycle. The child (per_core_busy) aggregation stays registered. + // Launch liveness: a request is covered by its issuer, so busy has no path from + // the socket's inputs. The cluster covers the launch and DCR beats it presents + // here; this socket covers the beats resident in its per-core output skids and + // the cycle a core accepts one, which the core leaves out of its busy. With a + // single core, the skid is a wire from the input and the cluster covers it. + localparam BUSY_REG = (`VX_CFG_SOCKET_SIZE > 1); wire [`VX_CFG_SOCKET_SIZE-1:0] per_core_kmu_valid; + wire [`VX_CFG_SOCKET_SIZE-1:0] per_core_kmu_fire; for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE; ++i) begin : g_kmu_link_valid assign per_core_kmu_valid[i] = per_core_kmu_bus_if[i].valid; + assign per_core_kmu_fire[i] = per_core_kmu_bus_if[i].valid && per_core_kmu_bus_if[i].ready; end +`ifdef VX_CFG_VM_ENABLE + wire mmu_busy = ~socket_mmu_drained; +`else + wire mmu_busy = 1'b0; +`endif // Stores are ack-less, so the cores go idle with their last requests still // on the way out; the socket stays busy until none is held in its caches' // uncached paths or arbiters. @@ -995,8 +1004,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif wire mem_req_empty = icache_req_empty && dcache_req_empty && tcache_req_empty && rtcache_req_empty && mem_arb_empty; + wire issued_busy = BUSY_REG ? (dcr_bus_if.req_valid | (| per_core_kmu_fire)) : 1'b0; + wire skid_busy = BUSY_REG ? (| per_core_kmu_valid) : 1'b0; wire busy_r; - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy) | (|per_core_lsu_busy) | ~mem_req_empty, 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); - assign busy = busy_r | dcr_bus_if.req_valid | kmu_bus_if[0].valid | (|per_core_kmu_valid); + `BUFFER_EX(busy_r, (|per_core_busy) | (|per_core_lsu_busy) | ~mem_req_empty | mmu_busy | issued_busy, 1'b1, 1, BUSY_REG); + assign busy = busy_r | skid_busy; endmodule diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index 9eb31133da..cbfce89ffd 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -131,6 +131,59 @@ import VX_tcu_pkg::*; wire [SW-1:0] issued; wire [SW-1:0] pending_alm_empty; + // The core-wide accounting retires an exiting warp when the shared unit grants + // its exit request. With several requesters, the slice sees that grant on its + // own request (registered toward the unit) and retires the warp in the same + // cycle, so the retirement does not cross the core's arbiter; with a sole + // requester, the core's retirement is already registered. + VX_slice_sched_if fs_sched_if(); + + assign fs_sched_if.cta_fire = slice_sched_if.cta_fire; + assign fs_sched_if.cta_lid = slice_sched_if.cta_lid; + assign fs_sched_if.cta_PC = slice_sched_if.cta_PC; + assign fs_sched_if.cta_tmask = slice_sched_if.cta_tmask; + assign fs_sched_if.cta_param = slice_sched_if.cta_param; + assign fs_sched_if.cta_init = slice_sched_if.cta_init; + assign fs_sched_if.warp_write = slice_sched_if.warp_write; + assign fs_sched_if.warp_waddr = slice_sched_if.warp_waddr; + assign fs_sched_if.warp_wdata = slice_sched_if.warp_wdata; + assign fs_sched_if.id_write = slice_sched_if.id_write; + assign fs_sched_if.id_waddr = slice_sched_if.id_waddr; + assign fs_sched_if.id_wdata = slice_sched_if.id_wdata; + assign fs_sched_if.wspawn_apply = slice_sched_if.wspawn_apply; + assign fs_sched_if.wspawn_wmask = slice_sched_if.wspawn_wmask; + assign fs_sched_if.wspawn_pc = slice_sched_if.wspawn_pc; + assign fs_sched_if.wspawn_src = slice_sched_if.wspawn_src; + assign fs_sched_if.wspawn_src_lid = slice_sched_if.wspawn_src_lid; + assign fs_sched_if.wspawn_mscratch = slice_sched_if.wspawn_mscratch; + assign fs_sched_if.bar_unlock_valid = slice_sched_if.bar_unlock_valid; + assign fs_sched_if.bar_unlock_mask = slice_sched_if.bar_unlock_mask; + assign fs_sched_if.slice_ibuf_full = slice_sched_if.slice_ibuf_full; +`ifdef EXT_SCHED_TIMEOUT_DUMP + assign fs_sched_if.timeout_dump = slice_sched_if.timeout_dump; +`endif + + if (SLICE_RETIRE) begin : g_retire + assign fs_sched_if.retire_valid = warp_shared_if.req_valid && warp_shared_if.req_ready + && (warp_shared_if.req_data.kind == WSH_EXIT); + assign fs_sched_if.retire_lid = wid_to_wfs(warp_shared_if.req_data.wid); + `UNUSED_VAR ({slice_sched_if.retire_valid, slice_sched_if.retire_lid}) + end else begin : g_retire_core + assign fs_sched_if.retire_valid = slice_sched_if.retire_valid; + assign fs_sched_if.retire_lid = slice_sched_if.retire_lid; + end + + assign slice_sched_if.active = fs_sched_if.active; + assign slice_sched_if.stalled = fs_sched_if.stalled; + assign slice_sched_if.ibuf_full = fs_sched_if.ibuf_full; + assign slice_sched_if.pending_empty = fs_sched_if.pending_empty; + assign slice_sched_if.branch_valid = fs_sched_if.branch_valid; + assign slice_sched_if.divergence = fs_sched_if.divergence; + assign slice_sched_if.sched_valid_room = fs_sched_if.sched_valid_room; + assign slice_sched_if.sched_valid_full = fs_sched_if.sched_valid_full; + assign slice_sched_if.sched_fire = fs_sched_if.sched_fire; + assign slice_sched_if.sched_tmask = fs_sched_if.sched_tmask; + VX_frontend_slice #( .INSTANCE_ID (INSTANCE_ID), .SLICE_STR ((`VX_CFG_FETCH_WIDTH > 1) ? SLICE_STR : "") @@ -143,7 +196,7 @@ import VX_tcu_pkg::*; `ifdef PERF_ENABLE .fetch_perf (fetch_perf), `endif - .slice_sched_if (slice_sched_if), + .slice_sched_if (fs_sched_if), .icache_bus_if (icache_bus_if), .decode_if (decode_if), .branch (branch), diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index c018683547..e7bf27571d 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -101,10 +101,11 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses; `endif - // frontend-slice icache requests; port 0 also carries the injected flush + // frontend-slice icache ports, tagged with the flush arbiter's select bit (the + // LSB); port 0 also carries the injected flush VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), - .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) + .TAG_WIDTH (ICACHE_TAG_WIDTH) ) fetch_icache_if[ICACHE_NUM_REQS](); `ifdef VX_CFG_TCU_WGMMA_ENABLE @@ -205,7 +206,31 @@ import VX_tcu_pkg::*; `SCOPE_IO_SWITCH (3); // Port 0 injects the cache flush into its request stream; the cache flushes - // every bank. The merged request leaves the core registered. + // every bank. The merged request leaves the core registered. The response + // goes to the frontend slice as the cache returns it, and the slice drops the + // flush's own (select bit set), so no logic lies between the cache's response + // register and the slice. The injector observes the response for its done. + VX_mem_bus_if #( + .DATA_SIZE (ICACHE_WORD_SIZE), + .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) + ) flush_core_if(); + + assign flush_core_if.req_valid = fetch_icache_if[0].req_valid; + assign flush_core_if.req_data.rw = fetch_icache_if[0].req_data.rw; + assign flush_core_if.req_data.addr = fetch_icache_if[0].req_data.addr; + assign flush_core_if.req_data.data = fetch_icache_if[0].req_data.data; + assign flush_core_if.req_data.byteen = fetch_icache_if[0].req_data.byteen; + assign flush_core_if.req_data.attr = fetch_icache_if[0].req_data.attr; + assign flush_core_if.req_data.tag = fetch_icache_if[0].req_data.tag[ICACHE_TAG_WIDTH-1:1]; + assign fetch_icache_if[0].req_ready = flush_core_if.req_ready; + `UNUSED_VAR (fetch_icache_if[0].req_data.tag[0]) + + assign fetch_icache_if[0].rsp_valid = icache_bus_if[0].rsp_valid; + assign fetch_icache_if[0].rsp_data = icache_bus_if[0].rsp_data; + assign flush_core_if.rsp_ready = fetch_icache_if[0].rsp_ready; + `UNUSED_VAR (flush_core_if.rsp_valid) + `UNUSED_VAR (flush_core_if.rsp_data) + VX_dcr_flush #( .WORD_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH), @@ -215,25 +240,21 @@ import VX_tcu_pkg::*; .reset (reset), .dcr_flush_if (dcr_flush_icache_if), .l1_only (dcr_flush_l1_only), - .core_bus_if (fetch_icache_if[0]), + .core_bus_if (flush_core_if), .cache_bus_if (icache_bus_if[0]) ); - // The other ports carry their frontend slice's registered requests, with the - // flush arbiter's select bit, the tag's LSB, clear. + // The other ports carry their frontend slice's registered requests, whose + // select bit is clear; a response's is cleared as a constant, so the slice's + // flush check folds away. for (genvar i = 1; i < ICACHE_NUM_REQS; ++i) begin : g_icache_ports assign icache_bus_if[i].req_valid = fetch_icache_if[i].req_valid; - assign icache_bus_if[i].req_data.rw = fetch_icache_if[i].req_data.rw; - assign icache_bus_if[i].req_data.addr = fetch_icache_if[i].req_data.addr; - assign icache_bus_if[i].req_data.data = fetch_icache_if[i].req_data.data; - assign icache_bus_if[i].req_data.byteen = fetch_icache_if[i].req_data.byteen; - assign icache_bus_if[i].req_data.attr = fetch_icache_if[i].req_data.attr; - assign icache_bus_if[i].req_data.tag = {fetch_icache_if[i].req_data.tag, 1'b0}; + assign icache_bus_if[i].req_data = fetch_icache_if[i].req_data; assign fetch_icache_if[i].req_ready = icache_bus_if[i].req_ready; assign fetch_icache_if[i].rsp_valid = icache_bus_if[i].rsp_valid; assign fetch_icache_if[i].rsp_data.data = icache_bus_if[i].rsp_data.data; - assign fetch_icache_if[i].rsp_data.tag = icache_bus_if[i].rsp_data.tag[ICACHE_TAG_WIDTH-1:1]; + assign fetch_icache_if[i].rsp_data.tag = {icache_bus_if[i].rsp_data.tag[ICACHE_TAG_WIDTH-1:1], 1'b0}; assign icache_bus_if[i].rsp_ready = fetch_icache_if[i].rsp_ready; `UNUSED_VAR (icache_bus_if[i].rsp_data.tag[0]) end @@ -810,11 +831,11 @@ import VX_tcu_pkg::*; // Fragment work drains at the producer (VX_raster_core.busy), not here. // DCR reads (cache flush, MPM readback) order on their own DCR response, so // busy tracks only kernel execution: warps resident or stores not yet at L1. - // ~mmu_empty also keeps the core busy while a translation or parked store is - // still in flight, so kernel completion cannot race un-drained accesses. // A store still in an LSU block's memory path shows on lsu_busy, which the // parent ORs in; one past a flush injector's buffer shows on mem_unit_busy. - assign busy = sched_busy || mem_unit_busy || ~mmu_empty; + // busy depends on no input: the socket covers the launches it sends and its + // MMU's translations. + assign busy = sched_busy || mem_unit_busy; `ifdef PERF_ENABLE diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index 8dd1147587..508557fd05 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -701,14 +701,12 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( | `VX_CFG_MEM_ADDR_WIDTH'(cur_lmem_base_r); assign cta_csrs.cluster_size = 32'(cluster_size_r); - // Busy from the accept cycle (kmu_bus_if_fire), not just while in DISPATCH: - // the accept->DISPATCH transition is registered, so gating on state alone - // leaves the accept cycle un-busy. With SOCKET_SIZE>1 the socket busy - // aggregation is registered, so on the final CTA kmu_busy drops before the - // buffered per-core busy rises, opening a 1-cycle device-busy gap the host's - // edge-sensitive idle-wait latches as premature completion (cores>1 launch - // failure). Covering the accept cycle closes the gap. - assign busy = (state == DISPATCH) || kmu_bus_if_fire; + // The accept cycle (kmu_bus_if_fire) is covered by the issuer of the launch, + // the socket, so busy has no path from this unit's inputs. Without it, a + // registered busy aggregation would leave a 1-cycle device-busy gap on the + // final CTA, which the host's edge-sensitive idle-wait latches as premature + // completion. + assign busy = (state == DISPATCH); // ------------------------------------------------------------------------- // CTA context storage + per-thread coordinate expansion diff --git a/hw/rtl/core/VX_frontend_slice.sv b/hw/rtl/core/VX_frontend_slice.sv index 9bc2eed602..4c66ef0adb 100644 --- a/hw/rtl/core/VX_frontend_slice.sv +++ b/hw/rtl/core/VX_frontend_slice.sv @@ -36,7 +36,7 @@ module VX_frontend_slice import VX_gpu_pkg::*; #( // core scheduler VX_slice_sched_if.slave slice_sched_if, - // instruction fetch + // instruction fetch, tagged with the core's flush select bit (the LSB) VX_mem_bus_if.master icache_bus_if, // decoded instructions @@ -73,6 +73,28 @@ module VX_frontend_slice import VX_gpu_pkg::*; #( VX_decode_if fe_decode_if(); VX_decode_sched_if decode_sched_if(); + // A response with the select bit set answers the core's cache flush on this + // port, and the core takes it: it is dropped here, on the cache's registered + // response, so the core passes the response through without logic. + VX_mem_bus_if #( + .DATA_SIZE (ICACHE_WORD_SIZE), + .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) + ) fetch_bus_if(); + + assign icache_bus_if.req_valid = fetch_bus_if.req_valid; + assign icache_bus_if.req_data.rw = fetch_bus_if.req_data.rw; + assign icache_bus_if.req_data.addr = fetch_bus_if.req_data.addr; + assign icache_bus_if.req_data.data = fetch_bus_if.req_data.data; + assign icache_bus_if.req_data.byteen = fetch_bus_if.req_data.byteen; + assign icache_bus_if.req_data.attr = fetch_bus_if.req_data.attr; + assign icache_bus_if.req_data.tag = {fetch_bus_if.req_data.tag, 1'b0}; + assign fetch_bus_if.req_ready = icache_bus_if.req_ready; + + assign fetch_bus_if.rsp_valid = icache_bus_if.rsp_valid && ~icache_bus_if.rsp_data.tag[0]; + assign fetch_bus_if.rsp_data.data = icache_bus_if.rsp_data.data; + assign fetch_bus_if.rsp_data.tag = icache_bus_if.rsp_data.tag[ICACHE_TAG_WIDTH-1:1]; + assign icache_bus_if.rsp_ready = fetch_bus_if.rsp_ready; + VX_warp_sched #( .INSTANCE_ID (`SFORMATF(("%s-scheduler%s", INSTANCE_ID, SLICE_STR))) ) warp_sched ( @@ -111,7 +133,7 @@ module VX_frontend_slice import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE .fetch_perf (fetch_perf), `endif - .icache_bus_if (icache_bus_if), + .icache_bus_if (fetch_bus_if), .schedule_if (schedule_if), .fetch_if (fetch_if) ); diff --git a/hw/rtl/core/VX_scheduler.sv b/hw/rtl/core/VX_scheduler.sv index efcc981018..2e92beac9e 100644 --- a/hw/rtl/core/VX_scheduler.sv +++ b/hw/rtl/core/VX_scheduler.sv @@ -61,14 +61,18 @@ module VX_scheduler import VX_gpu_pkg::*; #( // slice status in wid order (wid = {wfs, fsw}) wire [`VX_CFG_NUM_WARPS-1:0] stalled_warps; - wire [`VX_CFG_NUM_WARPS-1:0] ibuf_full_n, pending_warp_empty; + wire [`VX_CFG_NUM_WARPS-1:0] pending_warp_empty; for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_interleave assign active_warps[w] = slice_if[w % FW].active[w / FW]; assign stalled_warps[w] = slice_if[w % FW].stalled[w / FW]; - assign ibuf_full_n[w] = slice_if[w % FW].ibuf_full_n[w / FW]; assign pending_warp_empty[w] = slice_if[w % FW].pending_empty[w / FW]; end + wire [FW-1:0] slice_ibuf_full; + for (genvar p = 0; p < FW; ++p) begin : g_slice_ibuf_full + assign slice_ibuf_full[p] = slice_if[p].ibuf_full; + end + // CTA dispatcher: lane p launches frontend slice p's warps wire [FW-1:0] cta_fire; wire [FW-1:0][NW_WIDTH-1:0] cta_wid; @@ -138,6 +142,9 @@ module VX_scheduler import VX_gpu_pkg::*; #( // counters + // a CTA counts from its accept cycle, which busy leaves to the issuer + wire kmu_fire = kmu_bus_if.valid && kmu_bus_if.ready; + reg [PERF_CTR_BITS-1:0] cycles_r; reg [PERF_CTR_BITS-1:0] instret_r; wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] committed_warps_cnt; @@ -147,7 +154,7 @@ module VX_scheduler import VX_gpu_pkg::*; #( cycles_r <= '0; instret_r <= '0; end else begin - if (busy) begin + if (busy || kmu_fire) begin cycles_r <= cycles_r + 1; end instret_r <= instret_r + PERF_CTR_BITS'(committed_warps_cnt); @@ -156,16 +163,6 @@ module VX_scheduler import VX_gpu_pkg::*; #( assign cycles = cycles_r; assign instret = instret_r; - // from the slices' next-state full, so it is valid the same cycle as their full flags - reg all_ibuf_full; - always @(posedge clk) begin - if (reset) begin - all_ibuf_full <= 1'b0; - end else begin - all_ibuf_full <= (& ibuf_full_n); - end - end - `ifdef VX_CFG_VM_ENABLE // Per-core SATP CSR, written by any SFU block's CSR unit. Accesses in one // cycle take effect in block order: each block reads the value the lower @@ -216,21 +213,28 @@ module VX_scheduler import VX_gpu_pkg::*; #( assign slice_if[p].bar_unlock_valid = bar_unlock_valid; - assign slice_if[p].retire_valid = cta_warp_done && (wid_to_fsw(cta_warp_done_wid) == FSW); - assign slice_if[p].retire_lid = wid_to_wfs(cta_warp_done_wid); + if (SLICE_RETIRE) begin : g_retire_in_slice + assign slice_if[p].retire_valid = 1'b0; + assign slice_if[p].retire_lid = '0; + end else begin : g_retire + assign slice_if[p].retire_valid = cta_warp_done && (wid_to_fsw(cta_warp_done_wid) == FSW); + assign slice_if[p].retire_lid = wid_to_wfs(cta_warp_done_wid); + end for (genvar l = 0; l < SW; ++l) begin : g_lw assign slice_if[p].wspawn_wmask[l] = wspawn_wmask[l * FW + p]; assign slice_if[p].bar_unlock_mask[l] = bar_unlock_mask[l * FW + p]; end - assign slice_if[p].all_ibuf_full = all_ibuf_full; + assign slice_if[p].slice_ibuf_full = slice_ibuf_full; end + // the slices schedule from the core-wide view: every ibuffer of the core is full + wire all_ibuf_full = (& slice_ibuf_full); wire [FW-1:0] sched_valid, sched_fire; wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] sched_tmask; for (genvar p = 0; p < FW; ++p) begin : g_sched_status - assign sched_valid[p] = slice_if[p].sched_valid; + assign sched_valid[p] = all_ibuf_full ? slice_if[p].sched_valid_full : slice_if[p].sched_valid_room; assign sched_fire[p] = slice_if[p].sched_fire; assign sched_tmask[p] = slice_if[p].sched_tmask; end diff --git a/hw/rtl/core/VX_warp_sched.sv b/hw/rtl/core/VX_warp_sched.sv index 70cdac2baa..fca7e336b1 100644 --- a/hw/rtl/core/VX_warp_sched.sv +++ b/hw/rtl/core/VX_warp_sched.sv @@ -378,10 +378,13 @@ module VX_warp_sched import VX_gpu_pkg::*; #( wire [SW-1:0] preferred_warps = ready_warps & ~ibuf_full; `ifndef L1_ENABLE // without L1, the icache must never stall: it shares the bus with dcache responses + wire [SW-1:0] full_warps = preferred_warps; wire [SW-1:0] schedule_warps = preferred_warps; - `UNUSED_VAR (sched_if.all_ibuf_full) + `UNUSED_VAR (sched_if.slice_ibuf_full) `else - wire [SW-1:0] schedule_warps = sched_if.all_ibuf_full ? ready_warps : preferred_warps; + // once every ibuffer of the core is full, any ready warp may fetch + wire [SW-1:0] full_warps = ready_warps; + wire [SW-1:0] schedule_warps = (& sched_if.slice_ibuf_full) ? full_warps : preferred_warps; `endif wire [`UP(FETCH_WFS_BITS)-1:0] cand_index; @@ -477,7 +480,7 @@ module VX_warp_sched import VX_gpu_pkg::*; #( // status assign sched_if.active = active_warps; assign sched_if.stalled = stalled_warps; - assign sched_if.ibuf_full_n = ibuf_full_n; + assign sched_if.ibuf_full = (& ibuf_full); assign sched_if.pending_empty = pending_empty; // performance events @@ -487,7 +490,8 @@ module VX_warp_sched import VX_gpu_pkg::*; #( for (genvar q = 0; q < NPS; ++q) begin : g_divergence assign sched_if.divergence[q] = wctl[q].split_valid && wctl[q].split.is_dvg; end - assign sched_if.sched_valid = cand_valid; + assign sched_if.sched_valid_room = (| preferred_warps); + assign sched_if.sched_valid_full = (| full_warps); assign sched_if.sched_fire = schedule_if_fire; assign sched_if.sched_tmask = schedule_if.data.tmask; diff --git a/hw/rtl/interfaces/VX_slice_sched_if.sv b/hw/rtl/interfaces/VX_slice_sched_if.sv index 4f66b97f24..95d8c79d6d 100644 --- a/hw/rtl/interfaces/VX_slice_sched_if.sv +++ b/hw/rtl/interfaces/VX_slice_sched_if.sv @@ -20,6 +20,7 @@ interface VX_slice_sched_if import VX_gpu_pkg::*; (); localparam SW = PER_FETCH_WARPS; localparam LW = FETCH_WFS_W; + localparam FW = `VX_CFG_FETCH_WIDTH; // CTA launch logic cta_fire; @@ -53,8 +54,8 @@ interface VX_slice_sched_if import VX_gpu_pkg::*; (); logic retire_valid; logic [LW-1:0] retire_lid; - // every warp of the core has a full ibuffer - logic all_ibuf_full; + // every slice's ibuf_full; the slice ANDs them into its core-wide view + logic [FW-1:0] slice_ibuf_full; `ifdef EXT_SCHED_TIMEOUT_DUMP // the core-wide stall timeout is about to fire logic timeout_dump; @@ -64,11 +65,13 @@ interface VX_slice_sched_if import VX_gpu_pkg::*; (); logic [SW-1:0] active; logic [SW-1:0] stalled; logic [SW-1:0] pending_empty; - // next-state ibuffer full, registered core-wide into all_ibuf_full - logic [SW-1:0] ibuf_full_n; + // every warp of the slice has a full ibuffer + logic ibuf_full; logic [FETCH_ALU_PORTS-1:0] branch_valid; logic [FETCH_SFU_PORTS-1:0] divergence; - logic sched_valid; + // a warp is schedulable, with the core's ibuffers not all full and all full + logic sched_valid_room; + logic sched_valid_full; logic sched_fire; logic [`VX_CFG_NUM_THREADS-1:0] sched_tmask; @@ -95,17 +98,18 @@ interface VX_slice_sched_if import VX_gpu_pkg::*; (); output bar_unlock_mask, output retire_valid, output retire_lid, - output all_ibuf_full, + output slice_ibuf_full, `ifdef EXT_SCHED_TIMEOUT_DUMP output timeout_dump, `endif input active, input stalled, - input ibuf_full_n, + input ibuf_full, input pending_empty, input branch_valid, input divergence, - input sched_valid, + input sched_valid_room, + input sched_valid_full, input sched_fire, input sched_tmask ); @@ -133,17 +137,18 @@ interface VX_slice_sched_if import VX_gpu_pkg::*; (); input bar_unlock_mask, input retire_valid, input retire_lid, - input all_ibuf_full, + input slice_ibuf_full, `ifdef EXT_SCHED_TIMEOUT_DUMP input timeout_dump, `endif output active, output stalled, - output ibuf_full_n, + output ibuf_full, output pending_empty, output branch_valid, output divergence, - output sched_valid, + output sched_valid_room, + output sched_valid_full, output sched_fire, output sched_tmask ); From dffd76dca89dd174bdfeba4a762c94910a508f55 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 22 Oct 2026 14:13:50 -0700 Subject: [PATCH 239/336] cache: drain test from the crossbar's buffer valids, not an occupancy counter The flush tokens wait until the memory-request crossbar has drained. That was an occupancy counter that pop-counted valid & ready of every bank input each cycle. With the bypass enabled the cache's crossbar stages are pipe buffers, whose ready is combinational, so the count depended on a ready chain through every omega stage and the bypass arbiter: 22 LUT levels on the D-cache. VX_stream_omega now reports empty. With one or two entries per switch output and no arbiter tree, a stage buffer holds an item exactly when its registered valid is set, so empty is the NOR of the stage valids: registered inputs only, and equal to the counter's registered empty on every cycle (checked in rtlsim, demo and sgemm, against the counter kept as a reference). No storage (OUT_BUF 0) is always empty; deeper output buffers keep the counter. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 47b38109d5ea6856734c2baff84d4eb55b328a74) --- hw/rtl/cache/VX_cache.sv | 40 ++++++----------------------- hw/rtl/libs/VX_stream_omega.sv | 47 +++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 2b3c81cfa7..51b8311a11 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -273,7 +273,8 @@ module VX_cache import VX_gpu_pkg::*; #( .data_out (per_bank_mem_rsp_pdata), `UNUSED_PIN (sel_out), .ready_out (per_bank_mem_rsp_ready), - `UNUSED_PIN (collisions) + `UNUSED_PIN (collisions), + `UNUSED_PIN (empty) ); wire [NUM_BANKS-1:0][`CS_SECTOR_WIDTH-1:0] per_bank_mem_rsp_data; wire [NUM_BANKS-1:0][BANK_MEM_TAG_WIDTH-1:0] per_bank_mem_rsp_tag; @@ -659,7 +660,8 @@ module VX_cache import VX_gpu_pkg::*; #( .data_out (core_rsp_pdata), `UNUSED_PIN (sel_out), .ready_out (core_rsp_ready), - `UNUSED_PIN (collisions) + `UNUSED_PIN (collisions), + `UNUSED_PIN (empty) ); end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_rsp @@ -757,35 +759,8 @@ module VX_cache import VX_gpu_pkg::*; #( end // Write-backs from different banks are not ordered with each other inside // the crossbar, so the tokens wait until it has drained. - if (FLUSH_FORWARD != 0) begin : g_mem_req_xbar_pending - localparam XBAR_IN_W = `CLOG2(NUM_BANKS+1); - localparam XBAR_OUT_W = `CLOG2(MEM_PORTS+1); - // generous bound on what the crossbar can hold: buffered stages plus outputs - localparam XBAR_CAPACITY = 4 * NUM_BANKS * (`CLOG2(NUM_BANKS) + 1); - wire [NUM_BANKS-1:0] xbar_in_fire = mem_req_xbar_valid & mem_req_xbar_ready; - wire [MEM_PORTS-1:0] xbar_out_fire = mem_req_tmp_valid & mem_req_tmp_ready; - wire [XBAR_IN_W-1:0] xbar_in_cnt; - wire [XBAR_OUT_W-1:0] xbar_out_cnt; - `POP_COUNT(xbar_in_cnt, xbar_in_fire); - `POP_COUNT(xbar_out_cnt, xbar_out_fire); - VX_pending_size #( - .SIZE (XBAR_CAPACITY), - .INCRW (XBAR_IN_W), - .DECRW (XBAR_OUT_W) - ) xbar_pending ( - .clk (clk), - .reset (reset), - .incr (xbar_in_cnt), - .decr (xbar_out_cnt), - .empty (mem_req_empty), - `UNUSED_PIN (alm_empty), - `UNUSED_PIN (full), - `UNUSED_PIN (alm_full), - `UNUSED_PIN (size) - ); - end else begin : g_no_mem_req_xbar_pending - assign mem_req_empty = 1'b1; - end + wire mem_req_xbar_empty; + assign mem_req_empty = (FLUSH_FORWARD != 0) ? mem_req_xbar_empty : 1'b1; VX_stream_omega #( .NUM_INPUTS (NUM_BANKS), .NUM_OUTPUTS (MEM_PORTS), @@ -803,7 +778,8 @@ module VX_cache import VX_gpu_pkg::*; #( .data_out (mem_req_pdata), .sel_out (mem_req_tmp_idx), .ready_out (mem_req_tmp_ready), - `UNUSED_PIN (collisions) + `UNUSED_PIN (collisions), + .empty (mem_req_xbar_empty) ); for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_tmp_if wire [`UP(BANK_SEL_BITS)-1:0] mem_req_raw_bid_w; diff --git a/hw/rtl/libs/VX_stream_omega.sv b/hw/rtl/libs/VX_stream_omega.sv index 1723a21e4b..ca4b0cdedb 100644 --- a/hw/rtl/libs/VX_stream_omega.sv +++ b/hw/rtl/libs/VX_stream_omega.sv @@ -39,10 +39,20 @@ module VX_stream_omega #( output wire [NUM_OUTPUTS-1:0][IN_WIDTH-1:0] sel_out, input wire [NUM_OUTPUTS-1:0] ready_out, - output wire [PERF_CTR_BITS-1:0] collisions + output wire [PERF_CTR_BITS-1:0] collisions, + output wire empty ); `STATIC_ASSERT (`IS_POW2(RADIX), ("inavlid parameters")) + // With one or two entries per switch output and no arbiter tree, an + // output buffer holds an item exactly when its registered valid is set, + // so the network is empty when no switch output is valid. + localparam BUF_KIND = OUT_BUF & 7; + localparam VALID_HOLDS = (BUF_KIND >= 1 && BUF_KIND <= 3) + && (MAX_FANOUT == 0 || RADIX <= (MAX_FANOUT + MAX_FANOUT / 2)); + + wire buf_any_valid; + // If network size smaller than radix, simply use a crossbar. if (NUM_INPUTS <= RADIX && NUM_OUTPUTS <= RADIX) begin : g_fallback VX_stream_xbar #( @@ -66,6 +76,7 @@ module VX_stream_omega #( .ready_out, .collisions ); + assign buf_any_valid = (| valid_out); end else begin : g_omega localparam RADIX_LG = `LOG2UP(RADIX); localparam N_INPUTS_M = `MAX(NUM_INPUTS, NUM_OUTPUTS); @@ -207,6 +218,40 @@ module VX_stream_omega #( end assign collisions = collisions_r; + assign buf_any_valid = (| switch_valid_out); + end + + if (BUF_KIND == 0) begin : g_empty_passthru + `UNUSED_VAR (buf_any_valid) + assign empty = 1'b1; + end else if (VALID_HOLDS) begin : g_empty_valid + assign empty = ~buf_any_valid; + end else begin : g_empty_count + // Deeper output buffers hide items behind an invalid output, so the + // network counts what it takes and gives. + localparam IN_W = `CLOG2(NUM_INPUTS+1); + localparam OUT_W = `CLOG2(NUM_OUTPUTS+1); + localparam CAPACITY = 4 * `MAX(NUM_INPUTS, NUM_OUTPUTS) * (`CLOG2(`MAX(NUM_INPUTS, NUM_OUTPUTS)) + 1); + wire [IN_W-1:0] in_cnt; + wire [OUT_W-1:0] out_cnt; + `POP_COUNT(in_cnt, valid_in & ready_in); + `POP_COUNT(out_cnt, valid_out & ready_out); + `UNUSED_VAR (buf_any_valid) + VX_pending_size #( + .SIZE (CAPACITY), + .INCRW (IN_W), + .DECRW (OUT_W) + ) pending ( + .clk (clk), + .reset (reset), + .incr (in_cnt), + .decr (out_cnt), + .empty (empty), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); end endmodule From 0dd2982525415c49def61d27a58575b4566e8d8c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 22 Oct 2026 14:42:35 -0700 Subject: [PATCH 240/336] mem: register the handshakes a request-pending counter counts VX_mem_req_pending counted valid & ready of a section's inputs and outputs, then added and compared for empty, in the cycle of the handshake. Those handshakes end long ready chains (a cache cluster's core arbiters wait on the cache's request crossbar), so the whole count sat at the end of them. The handshakes are now registered first: the count lags a cycle, and empty adds the registered handshakes back. It is the same value on every cycle (checked in rtlsim, demo and sgemm, against the previous counter kept as a reference), with the ready chain ending at a flop. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7ba6c3b50e35f7be5dd21a985efe2020f568441c) --- hw/rtl/mem/VX_mem_req_pending.sv | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/hw/rtl/mem/VX_mem_req_pending.sv b/hw/rtl/mem/VX_mem_req_pending.sv index b92e287920..90ed7e620e 100644 --- a/hw/rtl/mem/VX_mem_req_pending.sv +++ b/hw/rtl/mem/VX_mem_req_pending.sv @@ -28,12 +28,29 @@ module VX_mem_req_pending #( ); localparam INCRW = `CLOG2(NUM_INPUTS+1); localparam DECRW = `CLOG2(NUM_OUTPUTS+1); + localparam SIZEW = `CLOG2(SIZE+1); + + // The handshakes usually end long ready chains, so they are registered + // before counting; the count then lags a cycle and empty adds back the + // registered handshakes, the same value on every cycle. + reg [NUM_INPUTS-1:0] in_fire_r; + reg [NUM_OUTPUTS-1:0] out_fire_r; + always @(posedge clk) begin + if (reset) begin + in_fire_r <= '0; + out_fire_r <= '0; + end else begin + in_fire_r <= in_fire; + out_fire_r <= out_fire; + end + end wire [INCRW-1:0] in_cnt; wire [DECRW-1:0] out_cnt; - `POP_COUNT(in_cnt, in_fire); - `POP_COUNT(out_cnt, out_fire); + `POP_COUNT(in_cnt, in_fire_r); + `POP_COUNT(out_cnt, out_fire_r); + wire [SIZEW-1:0] size_lag; VX_pending_size #( .SIZE (SIZE), .INCRW (INCRW), @@ -43,11 +60,13 @@ module VX_mem_req_pending #( .reset (reset), .incr (in_cnt), .decr (out_cnt), - .empty (empty), + `UNUSED_PIN (empty), `UNUSED_PIN (alm_empty), `UNUSED_PIN (full), `UNUSED_PIN (alm_full), - `UNUSED_PIN (size) + .size (size_lag) ); + assign empty = ((SIZEW+1)'(size_lag) + (SIZEW+1)'(in_cnt)) == (SIZEW+1)'(out_cnt); + endmodule From 45a587de6bc9780c43131d8632b137ec881b43d7 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Thu, 22 Oct 2026 18:57:23 -0700 Subject: [PATCH 241/336] cache: decode a response's target ports as it enters the bank queue Under read merging, the core-response multicast decoded each bank lane's targets (its port, plus the merged ports its tag marks in the port's group) from the bank queue's head, in front of the response arbiters whose grants pop that same head: a shift and an OR at the start of the socket's response loop. The bank now decodes the targets as a response enters its queue and keeps them in the entry, so the multicast starts from registers. The flush answer on bank 0's lane still targets its one port. Cycle-exact. Checked: rtlsim demo and sgemm on the small compute-slice GPU, identical instructions and cycles (demo 10,921, sgemm 325,140); the 12-run lint clean with the request stage on top. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e622c90a5c4959d2a3a6f29e2dbc14a4a25b560d) Integration (phase 2): the bank's response queue entry carries the targets beside phase 2's RSP_WORDS words (line merging); line merging still reads the port index for the leader's tag. --- hw/rtl/cache/VX_cache.sv | 21 ++++++++++++--------- hw/rtl/cache/VX_cache_bank.sv | 24 ++++++++++++++++++++---- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 51b8311a11..def3addae7 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -298,6 +298,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_rsp_data; wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_core_rsp_tag; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_core_rsp_idx; + wire [NUM_BANKS-1:0][NUM_REQS-1:0] per_bank_core_rsp_targets; wire [NUM_BANKS-1:0] per_bank_core_rsp_ready; wire [NUM_BANKS-1:0] per_bank_mem_req_valid; wire [NUM_BANKS-1:0][`CS_LINE_SECTOR_ADDR_WIDTH-1:0] per_bank_mem_req_addr; @@ -494,6 +495,7 @@ module VX_cache import VX_gpu_pkg::*; #( .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), + .MERGE_GROUP (DO_READ_MERGE ? MERGE_GROUP : 0), .TAG_WIDTH (BANK_TAG_WIDTH), .RSP_WORDS (RSP_WORDS), .RSP_WSEL_OFFS (DO_READ_MERGE ? MERGE_GROUP : 0), @@ -525,6 +527,7 @@ module VX_cache import VX_gpu_pkg::*; #( .core_rsp_data (per_bank_core_rsp_data[bank_id]), .core_rsp_tag (per_bank_core_rsp_tag[bank_id]), .core_rsp_idx (per_bank_core_rsp_idx[bank_id]), + .core_rsp_targets (per_bank_core_rsp_targets[bank_id]), .core_rsp_ready (per_bank_core_rsp_ready[bank_id]), .mem_req_valid (per_bank_mem_req_valid[bank_id]), .mem_req_addr (per_bank_mem_req_addr[bank_id]), @@ -577,7 +580,8 @@ module VX_cache import VX_gpu_pkg::*; #( // A merged read's response goes to its leader's port and to every // merged port: each output takes it when its arbiter grants it, and the // bank's response is released once all its targets have taken it. - localparam GROUP_BITS = `CLOG2(MERGE_GROUP); + // the targets carry the port index; only line merging reads it, for the leader + `UNUSED_VAR (core_rsp_xbar_idx) wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_pending; wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_taken; wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_valid; @@ -599,15 +603,13 @@ module VX_cache import VX_gpu_pkg::*; #( end end for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_lanes - wire [MERGE_GROUP-1:0] merged_mask; - if (i == 0) begin : g_flush_mask - assign merged_mask = per_bank_core_rsp_valid[0] ? per_bank_core_rsp_tag[0][MERGE_GROUP-1:0] : '0; - end else begin : g_bank_mask - assign merged_mask = per_bank_core_rsp_tag[i][MERGE_GROUP-1:0]; + wire [NUM_REQS-1:0] targets; + if (i == 0) begin : g_flush_targets + assign targets = per_bank_core_rsp_valid[0] ? per_bank_core_rsp_targets[0] + : (NUM_REQS'(1) << flush_rsp_idx); + end else begin : g_bank_targets + assign targets = per_bank_core_rsp_targets[i]; end - wire [REQ_SEL_WIDTH-1:0] group_base = (core_rsp_xbar_idx[i] >> GROUP_BITS) << GROUP_BITS; - wire [NUM_REQS-1:0] targets = (NUM_REQS'(1) << core_rsp_xbar_idx[i]) - | (NUM_REQS'(merged_mask) << group_base); reg [NUM_REQS-1:0] sent_r; assign lane_pending[i] = targets & ~sent_r & {NUM_REQS{core_rsp_xbar_valid[i]}}; for (genvar j = 0; j < NUM_REQS; ++j) begin : g_taken @@ -643,6 +645,7 @@ module VX_cache import VX_gpu_pkg::*; #( ); end end else begin : g_core_rsp_xbar + `UNUSED_VAR (per_bank_core_rsp_targets) VX_stream_omega #( .NUM_INPUTS (NUM_BANKS), .NUM_OUTPUTS (NUM_REQS), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 5a3b5041cf..a823225984 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -45,6 +45,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter RSP_WORDS = 1, parameter RSP_WSEL_OFFS = 0, parameter RSP_WSEL_STRIDE = 0, + parameter MERGE_GROUP = 0, // read-merge group: the tag's low bits mark the merged ports (0: none) parameter MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE), parameter MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH, parameter REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS), @@ -77,6 +78,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( output wire [RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] core_rsp_data, output wire [TAG_WIDTH-1:0] core_rsp_tag, output wire [REQ_SEL_WIDTH-1:0] core_rsp_idx, + output wire [NUM_REQS-1:0] core_rsp_targets, // the ports the response goes to input wire core_rsp_ready, // Memory request (sector-granular; = line when 1 sector/line) @@ -1386,8 +1388,23 @@ module VX_cache_bank import VX_gpu_pkg::*; #( `RUNTIME_ASSERT (~fwd_fire || (replay_addr == fbuf_addr_r), ("%t: %s fill-forward address mismatch: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(replay_addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fbuf_addr_r, BANK_ID))) `RUNTIME_ASSERT (~(flush_fire && fwd_pending), ("%t: %s flush during fill-forward drain", $time, INSTANCE_ID)) + // The ports a response goes to, decoded as it enters the queue so that + // the response arbiters start from registers: its own port and, under + // read merging, the merged ports of its group the tag's low bits mark. + wire [TAG_WIDTH-1:0] crsp_in_tag = crsp_queue_valid ? stC.req.tag : replay_tag; + wire [REQ_SEL_WIDTH-1:0] crsp_in_idx = crsp_queue_valid ? stC.req.req_idx : replay_idx; + wire [NUM_REQS-1:0] crsp_in_targets; + if (MERGE_GROUP > 1) begin : g_merge_targets + localparam GROUP_BITS = `CLOG2(MERGE_GROUP); + wire [REQ_SEL_WIDTH-1:0] group_base = (crsp_in_idx >> GROUP_BITS) << GROUP_BITS; + assign crsp_in_targets = (NUM_REQS'(1) << crsp_in_idx) + | (NUM_REQS'(crsp_in_tag[MERGE_GROUP-1:0]) << group_base); + end else begin : g_port_target + assign crsp_in_targets = NUM_REQS'(1) << crsp_in_idx; + end + VX_elastic_buffer #( - .DATAW (TAG_WIDTH + RSP_WORDS * `CS_WORD_WIDTH + REQ_SEL_WIDTH), + .DATAW (NUM_REQS + TAG_WIDTH + RSP_WORDS * `CS_WORD_WIDTH + REQ_SEL_WIDTH), .SIZE (CRSQ_QUEUE_SIZE), .OUT_REG (`TO_OUT_BUF_REG(CORE_OUT_BUF)) ) core_rsp_queue ( @@ -1395,9 +1412,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .reset (reset), .valid_in (crsp_queue_valid || fwd_head), .ready_in (crsp_queue_ready), - .data_in (crsp_queue_valid ? {stC.req.tag, crsp_words, stC.req.req_idx} - : {replay_tag, fwd_words, replay_idx}), - .data_out ({core_rsp_tag, core_rsp_data, core_rsp_idx}), + .data_in ({crsp_in_targets, crsp_in_tag, (crsp_queue_valid ? crsp_words : fwd_words), crsp_in_idx}), + .data_out ({core_rsp_targets, core_rsp_tag, core_rsp_data, core_rsp_idx}), .valid_out (core_rsp_valid), .ready_out (core_rsp_ready) ); From 48afd9315beea11b62fafaee1cfdcbec31215ee7 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 04:23:24 -0700 Subject: [PATCH 242/336] cta_dispatch: a thread-index step wraps by equality The per-lane thread-index expansion steps each lane +1 along X with one wrap into Y then Z, two lanes per pipeline stage. Each step compared against the block dimension with >= and subtracted it on a wrap: two magnitude compares and two subtractors in ripple, 13-15 levels per stage, the native core's critical paths at 250 MHz. The coordinates entering the expansion are reduced (the warp bases come from tid_advance), so a +1 step reaches the dimension exactly when it wraps, and wraps to 0: equality compares and a clear replace the compares and the subtractors. Fragment warps take their lane records from the stamps, not these coordinates. Cycle-exact. Timing (native core class, out of context, V80 at 4.0 ns, placed): WNS -0.473 -> +0.005 ns, failing endpoints 3,496 -> 0. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a65c9426fe3fcc660da4c50b55a27e203b71443d) --- hw/rtl/core/VX_cta_dispatch.sv | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index 508557fd05..b7efc4d3c3 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -734,14 +734,17 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( input logic [CTA_TID_WIDTH:0] bd_x, input logic [CTA_TID_WIDTH:0] bd_y ); + // prev is a reduced coordinate (prev[0] < bd_x, prev[1] < bd_y: the + // warp bases come from tid_advance), so a +1 step wraps exactly when + // it reaches the dimension, and wraps to 0. logic [CTA_TID_WIDTH:0] nx, ny; logic wx, wy; nx = {1'b0, prev[0]} + (CTA_TID_WIDTH+1)'(1); - wx = (nx >= bd_x); + wx = (nx == bd_x); ny = {1'b0, prev[1]} + (CTA_TID_WIDTH+1)'(wx); - wy = wx && (ny >= bd_y); - tid_next[0] = wx ? CTA_TID_WIDTH'(nx - bd_x) : CTA_TID_WIDTH'(nx); - tid_next[1] = wy ? CTA_TID_WIDTH'(ny - bd_y) : CTA_TID_WIDTH'(ny); + wy = wx && (ny == bd_y); + tid_next[0] = wx ? '0 : CTA_TID_WIDTH'(nx); + tid_next[1] = wy ? '0 : CTA_TID_WIDTH'(ny); tid_next[2] = prev[2] + CTA_TID_WIDTH'(wy); endfunction From 6ef6fa13c4a6338e8cdf92296f51050e718d3d5e Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 11:07:27 -0700 Subject: [PATCH 243/336] cta_dispatch: one thread-index step per stage over more than 8 threads The per-lane thread-index expansion runs two serial steps per stage. With 16 threads that ripple is still 10-12 levels at 300 MHz. Over more than 8 threads it now takes one step per stage; the added write latency lands in cta_warp_ram long before a warp reads it. Smaller configurations keep two steps and their cycles. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a789b3e528fe8484e70d7f97989741ce0cac363c) --- hw/rtl/core/VX_cta_dispatch.sv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index b7efc4d3c3..04fbef2e21 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -726,7 +726,8 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // TID_STEP lanes/cycle to meet timing; each launch lane has its own pipeline. // cta_warp_ram is read many cycles after a warp launches (fetch/decode/issue // latency >> TID_STAGES), so the added write latency is hidden. - localparam TID_STEP = 2; + // one step per stage once the ripple across many threads is the long path + localparam TID_STEP = (`VX_CFG_NUM_THREADS > 8) ? 1 : 2; localparam TID_STAGES = (`VX_CFG_NUM_THREADS - 1 + TID_STEP - 1) / TID_STEP; function automatic logic [2:0][CTA_TID_WIDTH-1:0] tid_next ( From 9e4fd082a7c8be4549d8378a163dfd3df2cc0e95 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 13:07:22 -0700 Subject: [PATCH 244/336] local_mem: request arbiter slices of four over more than 12 lanes At 16 lanes the request crossbar's per-bank arbiter splits into a tree of two 8-input slices, and the slice grant feeding each lane's ready was among the core's longest paths at 300 MHz. Slices of four keep the tree two levels deep (same added latency, which SimX already models as one tree level) while halving the grant logic each request waits on. Narrower configurations keep the default fan-out and are unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 12667eecf6dec9534d4a0bc5ef86ca6cf2751201) --- hw/rtl/mem/VX_local_mem.sv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hw/rtl/mem/VX_local_mem.sv b/hw/rtl/mem/VX_local_mem.sv index 26a798537a..82059d8454 100644 --- a/hw/rtl/mem/VX_local_mem.sv +++ b/hw/rtl/mem/VX_local_mem.sv @@ -234,6 +234,9 @@ module VX_local_mem import VX_gpu_pkg::*; #( .DATAW (REQ_DATAW), .PERF_CTR_BITS (PERF_CTR_BITS), .ARBITER ("P"), + // over more than 12 ports the arbiters are a tree; slices of four keep + // its depth and shorten the grant each request waits on + .MAX_FANOUT ((NUM_REQS > 12) ? 4 : `MAX_FANOUT), .OUT_BUF (3) // output should be registered for the data_store addressing ) req_xbar ( .clk (clk), From 653569ef3e159866639f0c9a32739c2c88029b28 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 14:34:07 -0700 Subject: [PATCH 245/336] ci: timing_gate, a per-unit logic-depth gate on every RTL change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fpga_gate and asic_gate find a deeper path the next night at best, and their Fmax does not say which unit got deeper. timing_gate screens the same DUT catalog with foldsim-estimate-timing, in seconds per DUT and without a Vivado licence, and asserts the depth of every RTL unit against a recorded baseline. - hw/syn/xilinx/dut: target `screen` (make -screen). It elaborates the DUT's sources with circt_elaborate.sh into a CIRCT netlist, maps it with foldsim-timing-map and screens it; screen_summary.py writes the summary row and the per-unit depths (generate indices folded). - circt_elaborate.sh: -f mlir writes the lowered HW/Comb/Seq netlist and stops before firtool, for tools built on CIRCT. - synth_gate.py: a Screen backend beside Xilinx and Yosys, with per-tool summary file, key metric and default tolerance, and a compare() hook in gate() through which the screen fails a unit that got deeper or a new unit at or above the fix threshold (12 levels at 300 MHz). Max levels is gated with zero tolerance: the screen is deterministic. - timing_gate.py, testcases/timing_gate.yaml (fpga_gate's builds except top, whose VX_cp_core keeps LLHD ref ports), tier `timing` opt-in, .github/workflows/timing_gate.yml on pull requests and pushes that touch the RTL, synth_report.py columns, continuous_integration.md §3.5 and §4.6. The screen does not reuse gen_sources.sh -E's netlist: gen_sources.sh parses options with getopts, which stops at the first positional argument, and every DUT passes $(RTL_PKGS) first, so -E has never taken effect in this flow. That is left as found, since fixing it changes what fpga_gate builds. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7cc67347f73ab49556f5983dc83526cf0ffd672e) Integration (phase 2): synth_gate keeps phase 2's one-sided gates (Fmax against the target clock, area against the catalog budget); the screen has no area budget, so the catalog requires a budget only from a tool that names one. The CI design doc keeps the target/budget wording and adds the screen. --- .github/workflows/timing_gate.yml | 125 ++++++++++++++++ AGENTS.md | 2 +- ci/synth_gate.py | 189 +++++++++++++++++++++++-- ci/synth_report.py | 19 +-- ci/testcase.py | 5 +- ci/testcases/timing_gate.yaml | 174 +++++++++++++++++++++++ ci/timing_gate.py | 41 ++++++ docs/designs/continuous_integration.md | 78 +++++++--- hw/scripts/circt_elaborate.sh | 17 ++- hw/scripts/screen_summary.py | 126 +++++++++++++++++ hw/syn/common.mk | 3 +- hw/syn/xilinx/dut/Makefile | 6 + hw/syn/xilinx/dut/common.mk | 21 ++- 13 files changed, 754 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/timing_gate.yml create mode 100644 ci/testcases/timing_gate.yaml create mode 100755 ci/timing_gate.py create mode 100644 hw/scripts/screen_summary.py diff --git a/.github/workflows/timing_gate.yml b/.github/workflows/timing_gate.yml new file mode 100644 index 0000000000..2bfefa934b --- /dev/null +++ b/.github/workflows/timing_gate.yml @@ -0,0 +1,125 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# +# timing_gate — logic-level regression gate on hosted runners. +# Screens the DUT catalog (ci/testcases/timing_gate.yaml) with +# foldsim-estimate-timing and asserts the depth of every RTL unit against the +# golden results in ci/baselines/synthesis/screen/, so an RTL change that makes +# a unit deeper is named on the change that does it, not the next night. +# +# Why its own workflow rather than a cell in ci.yml: it needs foldsim, which the +# base toolchain does not install (ci/toolchain_install.sh --foldsim), and it +# reads the synthesis gates' DUT flow rather than a pytest build tree. Unlike +# the synthesis gates it is cheap -- seconds to minutes per DUT, no licence -- +# so it runs on every pull request and push that touches the RTL, in one job. +# +# See docs/designs/continuous_integration.md §3.5. + +name: Timing Gate + +on: + pull_request: + paths: + - 'hw/rtl/**' + - 'hw/unittest/**' + - 'hw/scripts/**' + - 'hw/syn/xilinx/dut/**' + - 'ci/synth_gate.py' + - 'ci/timing_gate.py' + - 'ci/testcases/timing_gate.yaml' + - 'ci/baselines/synthesis/screen/**' + push: + branches: [master] + paths: + - 'hw/rtl/**' + - 'hw/unittest/**' + - 'hw/scripts/**' + - 'hw/syn/xilinx/dut/**' + - 'ci/synth_gate.py' + - 'ci/timing_gate.py' + - 'ci/testcases/timing_gate.yaml' + - 'ci/baselines/synthesis/screen/**' + workflow_dispatch: + inputs: + builds: + description: "build ids/groups, space separated (blank = all)" + default: "" + +concurrency: + group: timing-gate-${{ github.ref }} + cancel-in-progress: true + +env: + CCACHE_DISABLE: 1 + +jobs: + screen: + runs-on: ubuntu-22.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + # Restores the prebuilt toolchain (CIRCT among it) and exports TOOLDIR; + # prepare=true installs it on a cache miss. + - name: Setup Vortex + uses: ./.github/actions/setup-vortex + with: + prepare: true + + # foldsim is not in the base toolchain. Its prebuilt is a small fetch. + - name: Install foldsim + run: | + set -euo pipefail + mkdir -p tc_foldsim && cd tc_foldsim + ../configure --tooldir=$TOOLDIR + ci/toolchain_install.sh --foldsim + + # The command is the one the catalog declares (the single `run:` case of + # ci/testcases/timing_gate.yaml), so what CI runs and what the catalog + # says cannot drift apart. + - name: Gate + id: run + env: + IN_BUILDS: ${{ github.event.inputs.builds }} + run: | + set -uo pipefail + CMD=$(python3 -c "import yaml; print(yaml.safe_load(open('ci/testcases/timing_gate.yaml'))['tests'][0]['run'])") + # Dispatch inputs reach the shell through an env var, never + # interpolated into the script. + ARGS=() + for b in ${IN_BUILDS:-}; do ARGS+=(-b "$b"); done + rc=0 + $CMD "${ARGS[@]}" --timeout 3600 --report timing_gate.json || rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Summarize + if: always() + run: | + set -uo pipefail + python3 ci/synth_report.py --annotate error timing_gate.json \ + > timing_gate_report.md || true + cat timing_gate_report.md >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: timing-gate + path: | + timing_gate.json + timing_gate_report.md + build32_timing_gate/hw/syn/xilinx/dut/timing_gate_*/build.log + build32_timing_gate/hw/syn/xilinx/dut/timing_gate_*/screen_* + build32_timing_gate/hw/syn/xilinx/dut/timing_gate_*/screen/report.txt + if-no-files-found: warn + + - name: Verdict + run: | + case "${{ steps.run.outputs.rc }}" in + 0) echo "timing_gate passed" ;; + 1) echo "::error::timing_gate FAILED — a unit got deeper than its baseline"; exit 1 ;; + *) echo "::error::timing_gate build error (see logs)"; exit 1 ;; + esac diff --git a/AGENTS.md b/AGENTS.md index b3807145aa..54f54ad480 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,7 +86,7 @@ See [docs/testing.md](docs/testing.md) and [docs/debugging.md](docs/debugging.md - **RTL coverage path is `xrt`, not `rtlsim`.** When discussing or planning RTL verification, `xrt` is the canonical path — `rtlsim` bypasses the AFU surface. `rtlsim` remains useful for fast iteration on processor RTL; `xrt` is what proves the full integration. - **`ci/regression.sh` is the canonical source of tested configurations.** Use it to discover supported parameter combinations before inventing ad hoc ones. - **Perf-regression baselines (`ci/baselines/perf/*.json`) are golden data — never hand-edit them, and never "fix" a red perf gate by bumping the number.** They are regenerated only by `pytest ci -m perf_gate --update-baselines` (a human-run, reviewed step), and CI must never pass that flag. A `perf_gate` failure means real cycles moved: root-cause it, or — if the change is intended — regenerate the baseline so the diff shows the perf delta for review. Same rule as image goldens and `known_issue:`. -- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). +- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,screen}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/timing_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `timing_gate` means a unit got deeper, and its reasons name the unit. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). - **SimX is the RTL's timing model — keep them in lockstep.** Any change that moves RTL cycles (pipeline structure, arbitration, queue depths, cache/memory behavior) must land together with the matching SimX timing-model update, and vice-versa. The `model_parity` CI gate enforces this: a `check: model_parity` case (`ci/testcases/core.yaml` + per-extension parity categories) runs the same app/args/configs on simx and rtlsim and asserts exact retired-instruction match plus cycle agreement within the case tolerance (default 5%). Never widen a tolerance to absorb a divergence — model the behavior. When adding a hardware feature, add or extend a parity case that exercises it. - **When RTL debugging stalls, switch to the SimX-as-oracle pattern.** For numerical bugs, deep pipeline races, or any failure mode where rtlsim is "close but wrong": (1) build/extend the SimX C++ model so it mirrors the *new* RTL architecture and gets to PASS; (2) add matching trace dumps to both SimX and RTL (cycle, FU events, SRAM addresses+data, hazards) — same CSV format on both sides; (3) diff trace files — the first divergence is the bug. Don't keep guessing from output values; localize via trace diff. See [docs/debugging.md](docs/debugging.md#simx-as-oracle-for-rtl-debug). diff --git a/ci/synth_gate.py b/ci/synth_gate.py index 92dabb07d3..a8d8e82086 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -7,15 +7,17 @@ post-synthesis timing and area against the recorded numbers, so an RTL change that quietly costs timing closure or area cannot land green. -Two backends, one gate. Everything that is not "how do I drive the tool and what -does it call its numbers" is shared: +Three backends, one gate. Everything that is not "how do I drive the tool and +what does it call its numbers" is shared: --tool=xilinx Vivado synthesis + place-and-route -> ci/fpga_gate.py --tool=yosys Yosys + OpenSTA on an ASIC PDK -> ci/asic_gate.py + --tool=screen logic levels per RTL unit, no synthesis -> ci/timing_gate.py Spec : ci/testcases/.yaml (what to build) Baselines : ci/baselines/synthesis//.json (last accepted metrics) - Metrics : /synth_summary.csv, written by the flow itself + Metrics : /synth_summary.csv (screen_summary.csv for the screen), + written by the flow itself Same division as every other check in the catalog: the yaml is hand-authored, commented and reviewed; the baseline is machine-written and never hand-edited. @@ -94,6 +96,12 @@ class Tool: gated = () # asserted by default (--gate overrides) area = "" # the gated area metric, held to the catalog's budget budget_key = "" # the catalog field holding that budget + # The metric whose presence in a baseline entry means "recorded". + key_metric = "fmax_mhz" + # The flow's one-row metric file, in the build's work directory. + summary = "synth_summary.csv" + # Default gate tolerance, overridden by --threshold and the catalog. + tolerance = TOLERANCE integral = () # recorded as ints; every other metric is a rounded float columns = () # (header, metric, format, width) for the report table @@ -148,6 +156,12 @@ def extras(self, work): """Diagnostics recorded next to the metrics, never gated.""" return {} + def compare(self, metrics, base): + """Reasons beyond the per-metric tolerance check, as gate() words them: + a reason containing "improvement" asks for a re-record, any other + fails.""" + return [] + def incomplete(self, metrics): """Whether a cached result predates something this tool now records. @@ -439,7 +453,144 @@ def describe_baseline(self, base): base.get("cell_area_um2") or 0) -TOOLS = {t.name: t for t in (Xilinx(), Yosys())} +class Screen(Tool): + """The timing screen: logic levels per RTL unit (hw/syn/xilinx/dut, target + `screen`). + + foldsim-estimate-timing ranks every register-to-register path of the CIRCT + netlist the Xilinx flow elaborates by its logic levels, counted as Vivado + counts them on the V80. It needs no Vivado and no licence and takes seconds + per DUT, so it can run on every change where the Fmax gate runs nightly. + + Depth is deterministic -- the same RTL and tools give the same levels -- so + the gate has no tolerance: a unit that gets deeper fails, by name, until it + is fixed or a human re-records it. What it gates is depth, not Fmax: the + screen does not see wires (docs/designs/continuous_integration.md 3.5). + """ + + name = "screen" + gate = "timing_gate" + flow = "hw/syn/xilinx/dut" + + metrics = ("max_levels", "deep_endpoints", "watch_endpoints", "endpoints", + "build_time_s") + gated = ("max_levels",) + integral = metrics + key_metric = "max_levels" + summary = "screen_summary.csv" + tolerance = 0.0 + columns = (("max_lv", "max_levels", "%d", 7), + ("deep", "deep_endpoints", "%d", 9), + ("watch", "watch_endpoints", "%d", 9), + ("ends", "endpoints", "%d", 9)) + + # Elaboration and mapping of the widest DUTs; the screen itself is seconds. + weights = {"top": 600, "vortex": 600, "gfx": 600, "core": 300, + "tensor": 300, "tcu": 300} + fallback_weight = 120 + job_label = "make" + + # The netlist exists once foldsim-opt starts mapping it; a typo in a define + # or an include dies in gen_sources.sh or circt_elaborate.sh before that. + elab_done = "foldsim-timing-map" + elab_label = "elaboration" + phases = ( + ("gen_sources.sh", "sources"), + ("circt_elaborate.sh", "elaboration"), + (elab_done, "mapping"), + ("foldsim-estimate-timing", "screen"), + ("screen_summary.py", "reporting"), + ) + error_re = re.compile(r"^(?:ERROR|FATAL|Error)\b|: error:|^\S+: \*\*\* ") + + UNITS = "screen_units.json" + + def env(self, args): + return {"circt": tool_version("circt-verilog", "--version", + package="circt", line_prefix="CIRCT"), + "foldsim": tool_version("foldsim-opt", "--version", + package="foldsim", + line_prefix="foldsim"), + "xlen": args.xlen} + + def hash_key(self, env): + # The levels are the model's: a new foldsim or CIRCT can map the same + # RTL to other levels, which must read as STALE, not as a regression. + return [env["circt"], env["foldsim"], str(env["xlen"])] + + def env_warning(self, build, env): + for key, label in (("foldsim", "foldsim"), ("circt", "CIRCT")): + ref = build.get("baseline_env", {}).get(key) + if ref and env.get(key) and ref != env[key]: + return ("baseline was recorded on %s %s, running %s -- this " + "reads as STALE until re-recorded" % (label, ref, env[key])) + return None + + def work_dir(self, dut_root, prefix, build): + return os.path.join(dut_root, "%s_%s" % (prefix, build["dut"])) + + def clean(self, dut_root, work, build, benv): + subprocess.run(["make", "-C", work, "clean"], env=benv, + stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, + check=True) + + def build_command(self, dut_root, work, build, env, args): + os.makedirs(work, exist_ok=True) + shutil.copy2(os.path.join(dut_root, "build.mk"), + os.path.join(work, "Makefile")) + return ["make", "-C", work, "screen"], dict(DUT=build["dut"], + CONFIGS=build["configs"]) + + def extras(self, work): + try: + with open(os.path.join(work, self.UNITS)) as fh: + doc = json.load(fh) + except (OSError, ValueError): + return {} + return {"units": doc.get("units", {}), + "deepest_paths": doc.get("deepest_paths", []), + "fix_levels": doc.get("fix_levels"), + "watch_levels": doc.get("watch_levels")} + + def incomplete(self, metrics): + return metrics.get("units") is None + + def compare(self, metrics, base): + """Per unit: deeper than its baseline fails; a new unit at the fix + threshold fails; shallower, or gone from the watch list, is an + improvement to lock in.""" + cur, ref = metrics.get("units"), base.get("units") + if cur is None or ref is None: + return [] + fix = metrics.get("fix_levels") or 0 + reasons = [] + for unit in sorted(set(cur) | set(ref)): + now, was = cur.get(unit), ref.get(unit) + if was is None: + if now >= fix: + reasons.append("unit %s: new at %d levels (fix threshold " + "%d) -- REGRESSION" % (unit, now, fix)) + elif now is None: + reasons.append("unit %s: %d levels -> below the watch threshold " + "-- improvement, update the baseline to lock it " + "in" % (unit, was)) + elif now != was: + reasons.append("unit %s: %d -> %d levels -- %s" + % (unit, was, now, + "REGRESSION" if now > was else + "improvement, update the baseline to lock " + "it in")) + return reasons + + def describe(self, m): + return "max %d levels, %d end points >= %s" % ( + m["max_levels"], m["deep_endpoints"], m.get("fix_levels")) + + def describe_baseline(self, base): + return "max %d levels" % base["max_levels"] + + +TOOLS = {t.name: t for t in (Xilinx(), Yosys(), Screen())} def tool_version(name, flag, package=None, line_prefix=None): @@ -450,7 +601,11 @@ def tool_version(name, flag, package=None, line_prefix=None): copy of yosys would make the baseline's tool env a lie. """ tooldir = os.environ.get("TOOLDIR") or os.path.expanduser("~/tools") - binary = os.path.join(tooldir, package or name, "bin", name) + package = package or name + # _PATH overrides the toolchain copy, as in hw/syn/common.mk. + root = os.environ.get(package.upper() + "_PATH") or \ + os.path.join(tooldir, package) + binary = os.path.join(root, "bin", name) if not os.path.exists(binary): binary = name try: @@ -534,7 +689,8 @@ def load_catalog(root, tool): if bid in builds: sys.exit("ERROR: duplicate build id '%s' in %s" % (bid, spec_path(root, tool.gate))) - missing = {"dut", "clock_mhz", "configs", tool.budget_key} - set(entry) + required = {"dut", "clock_mhz", "configs"} | ({tool.budget_key} if tool.budget_key else set()) + missing = required - set(entry) if missing: sys.exit("ERROR: %s: missing %s" % (bid, ", ".join(sorted(missing)))) group = entry.get("group", "default") @@ -549,7 +705,8 @@ def load_catalog(root, tool): result = recorded.get("result") or {} # An all-null skeleton means "never recorded" -- the file ships with the # shape visible, not with numbers. - build["baseline"] = result if result.get("fmax_mhz") is not None else None + build["baseline"] = result if result.get(tool.key_metric) is not None \ + else None build["baseline_env"] = result.get("env") or {} build["config_hash"] = recorded.get("config_hash") builds[bid] = build @@ -918,10 +1075,10 @@ def run_build(build, build_dir, env, args, tool): rc, elapsed // 60, "".join("\n | " + e for e in errors), log)) - summary = os.path.join(work, "synth_summary.csv") + summary = os.path.join(work, tool.summary) if not os.path.exists(summary): - raise BuildError("no synth_summary.csv (the flow did not reach report), " - "see %s" % log) + raise BuildError("no %s (the flow did not reach report), see %s" + % (tool.summary, log)) metrics = read_summary(summary, tool) metrics["build_time_s"] = elapsed metrics.update(tool.extras(work)) @@ -1046,6 +1203,8 @@ def verdict(name, reasons): "improvement, update the baseline to lock it in" if better else "REGRESSION")) + reasons += tool.compare(result["metrics"], base) + # Target-frequency gate: Fmax must be within tolerance of the clock the design # was BUILT for, never of the last recording. One-sided: a faster build passes, # since placement noise alone moves Fmax by more than the tolerance. @@ -1160,8 +1319,9 @@ def main(argv=None, default_tool="xilinx"): "ci/testcases/.yaml and the golden file tree " "ci/baselines/synthesis// (default: %s)" % default_tool) - ap.add_argument("--threshold", type=float, default=TOLERANCE, metavar="F", - help="global gate tolerance, fraction (default: %.2f). " + ap.add_argument("--threshold", type=float, default=None, metavar="F", + help="global gate tolerance, fraction (default: the " + "tool's, %.2f for synthesis, 0 for the screen). " "Overridden per-metric by --metric-threshold and " "per-build by the catalog's \"thresholds\"." % TOLERANCE) @@ -1205,6 +1365,8 @@ def main(argv=None, default_tool="xilinx"): help="print the schedule without running the tool") args = ap.parse_args(argv) tool = TOOLS[args.tool] + if args.threshold is None: + args.threshold = tool.tolerance # Line-buffer stdout: progress is the point of the elaboration watch, and it # is worthless if it sits in a block buffer until the run ends (which is what @@ -1241,7 +1403,8 @@ def main(argv=None, default_tool="xilinx"): print("%-10s [%s] dut=%-8s %4d MHz %s%s" % (b["id"], b["group"], b["dut"], b["clock_mhz"], "baseline: " + tool.describe_baseline(base) - if base.get("fmax_mhz") else "no baseline", + if base.get(tool.key_metric) is not None + else "no baseline", " KNOWN ISSUE: " + b["known_issue"] if b.get("known_issue") else "")) print(" %s" % b["configs"]) diff --git a/ci/synth_report.py b/ci/synth_report.py index 03356ac238..84b3513307 100755 --- a/ci/synth_report.py +++ b/ci/synth_report.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """synth_report — render synthesis-gate reports as one Markdown summary. -Both gates end a run with `--report` JSON: asic_gate.yml one per DUT job, -fpga_gate.yml one for the whole sweep. This joins them into a single Markdown -document for the workflow summary -- every build's metrics against its -baseline, why each build that did not pass did not, and where each design is -tight -- so a run is diagnosed from its GitHub page rather than from the machine -that built it. It exits with the worst outcome so the caller can decide whether +Every gate ends a run with `--report` JSON: asic_gate.yml one per DUT job, +fpga_gate.yml and timing_gate.yml one for the whole sweep. This joins them +into a single Markdown document for the workflow summary -- every build's +metrics against its baseline, why each build that did not pass did not, and +where each design is tight -- so a run is diagnosed from its GitHub page +rather than from the machine that built it. It exits with the worst outcome so the caller can decide whether the commit was actually gated: 0 every build passed (or is a tracked known issue) @@ -28,7 +28,7 @@ # pass are reported, not failed. PASSING = ("PASS", "RECORDED", "KNOWN-ISSUE", "XPASS") -# Every metric either tool records, shown in this order when a build reports it. +# Every metric any tool records, shown in this order when a build reports it. COLUMNS = (("Fmax (MHz)", "fmax_mhz", "%.1f"), ("WNS (ns)", "wns_ns", "%.3f"), ("TNS (ns)", "tns_ns", "%.1f"), @@ -42,7 +42,10 @@ ("LUTRAM", "lutram", "%d"), ("BRAM", "bram", "%d"), ("URAM", "uram", "%d"), - ("DSP", "dsp", "%d")) + ("DSP", "dsp", "%d"), + ("Max levels", "max_levels", "%d"), + ("End points >= 12", "deep_endpoints", "%d"), + ("End points >= 8", "watch_endpoints", "%d")) # Slack sits near zero by design, so a relative change of it says nothing. NO_DELTA = ("wns_ns", "tns_ns") diff --git a/ci/testcase.py b/ci/testcase.py index 1f577d0741..1b501445b5 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -48,9 +48,10 @@ # nightly), and everything is what ci.yml's matrix can run as a CELL -- these # cannot be. `fpga` needs a licensed Vivado and hours of a whole machine; # `asic` needs 1-2 hours PER DUT, which is a fan-out of standalone jobs, not one -# cell. Each has its own workflow (.github/workflows/{fpga,asic}_gate.yml). +# cell; `timing` needs foldsim, which the base toolchain does not install. Each +# has its own workflow (.github/workflows/{fpga,asic,timing}_gate.yml). # Opting in is explicit, never implied. -OPT_IN_TIERS = {"fpga", "asic"} +OPT_IN_TIERS = {"fpga", "asic", "timing"} # simx<->rtlsim cycle-parity default: both timing models must agree within 5%. DEFAULT_PARITY_TOLERANCE = 0.05 diff --git a/ci/testcases/timing_gate.yaml b/ci/testcases/timing_gate.yaml new file mode 100644 index 0000000000..c2b01aa0d3 --- /dev/null +++ b/ci/testcases/timing_gate.yaml @@ -0,0 +1,174 @@ +# Logic-level regression gate (the timing screen): the deepest end point of +# every RTL unit must not get deeper than a checked-in golden result +# (ci/baselines/synthesis/screen/.json). See ci/timing_gate.py. +# +# Sibling of fpga_gate (fpga_gate.yaml) and asic_gate (asic_gate.yaml): same +# discipline -- golden baseline, results only ever re-recorded by a human -- +# over foldsim-estimate-timing instead of a synthesis tool. It counts logic +# levels as Vivado counts them on the V80 (LUTs, CARRY8s, distributed-RAM +# reads) on the CIRCT netlist the Xilinx flow elaborates, so it gates depth, +# not Fmax: wires are not in it. +# +# - Per unit. A unit is an end point's instance path with generate indices +# folded (g_banks_3_bank -> g_banks_*_bank). Every unit at or above the +# watch threshold (8 levels at 300 MHz) is recorded; a unit that gets deeper +# fails, and so does a new unit at or above the fix threshold (12). +# - No tolerance. The screen is deterministic, so any change in depth is a +# real one; a shallower unit reads IMPROVED, to be locked in by a re-record. +# - The tool versions (CIRCT, foldsim) are part of config_hash: a new model +# reads STALE, not as a regression. +# +# --------------------------------------------------------------------------- +# Two blocks, two consumers, as in the synthesis gates: +# +# tests: what pytest/CI sees -- ONE case, whose `run:` is literally the +# command CI executes. +# builds: what ci/timing_gate.py reads -- the build list. The pytest loader +# ignores this key. +# +# The builds are fpga_gate.yaml's, DUT for DUT and config for config (all but +# `top`, see below), so where both gates run a DUT they measure the same RTL, and a unit this gate flags +# can be read against the Fmax gate's critical paths. clock_mhz is the Fmax +# gate's target, recorded for that comparison; the screen's thresholds are +# 300 MHz ones. +# +# Tier `timing` is OPT-IN (OPT_IN_TIERS in testcase.py): it needs foldsim, +# which the base toolchain does not install. .github/workflows/timing_gate.yml +# installs it and runs the gate. +# +# Run one by name: ci/timing_gate.py -b tcu +# By hand, no gate: make -C hw/syn/xilinx/dut tcu-screen +# Re-record: ci/timing_gate.py --update-baseline (human-reviewed, never CI) +# --------------------------------------------------------------------------- +category: timing_gate +defaults: + # 32-bit only, as the synthesis gates. + xlen: + - 32 + tier: timing + via: script + # Any RTL change can move depth. + touches: + - hw/rtl/ +tests: +# Seconds to minutes per DUT: the whole catalog runs in one job. +- id: gate + run: ci/timing_gate.py -j 2 + +# --------------------------------------------------------------------------- +# builds — read by ci/timing_gate.py, ignored by the pytest catalog loader. +# +# dut hw/syn/xilinx/dut/catalog.mk target to screen +# clock_mhz the Fmax gate's target for this DUT (recorded, not asserted) +# configs EXTRA defines, appended to the catalog's +# group which golden file the results land in +# known_issue tracked expected failure: still screens and reports, does not +# fail the run +# --------------------------------------------------------------------------- +builds: + +# --- core: the base pipeline and its memory system --------------------------- + +# L1/LLC datapath with atomics: tag/data pipeline, MSHR, AMO unit. +- id: cache + group: core + dut: cache + clock_mhz: 300 + configs: -DVX_CFG_EXT_A_ENABLE + +# Wide-SIMD core: issue/operand-collect/LSU/FPU at NT=NW=16, plus the RVC +# decompressor and atomics. +- id: core + group: core + dut: core + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE + +# fpga_gate's `top` is not screened: CIRCT's import leaves the command +# processor's AXI-Lite response ports (VX_cp_core) as LLHD references, which +# no HW/Comb/Seq tool reads. It joins this list when the import lowers them. + +# --- dxa --------------------------------------------------------------------- + +# DXA engine: descriptor queue, address generation, multicast distribution. +- id: dxa + group: dxa + dut: dxa + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# --- tensor ------------------------------------------------------------------ + +# TFR tensor core with every datatype plus sparsity and WGMMA. +- id: tcu + group: tensor + dut: tcu + clock_mhz: 300 + configs: >- + -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_TCU_TYPE_TFR + -DVX_CFG_TCU_FP16_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DVX_CFG_TCU_INT8_ENABLE + -DVX_CFG_TCU_INT4_ENABLE -DVX_CFG_TCU_TF32_ENABLE + -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + +# --- graphics ---------------------------------------------------------------- + +# Ray-tracing unit: CW-BVH4 walker, box/triangle PEs, short stack. +- id: rtu + group: graphics + dut: rtu + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# Rasterizer: bin/block/quad walker, edge evaluators, rcache port. +- id: raster + group: graphics + dut: raster + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# Output merger: depth/stencil test, blend, ocache port. +- id: om + group: graphics + dut: om + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# Texture unit: address generation, LOD, bilinear filter, tcache port. +- id: tex + group: graphics + dut: tex + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# --- vm ---------------------------------------------------------------------- + +# Standalone MMU network: dTLB + iTLB, pooled L1 TLB MSHR, L2 TLB, PTW walker +# hierarchy, fed by a shared L2. The memory-translation bellwether; the rest of +# the extension config comes from hw/syn/xilinx/dut/vm/Makefile. +- id: vm + group: vm + dut: vm + clock_mhz: 250 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_CORES=2 + +# --- graphics (combined subsystem) ------------------------------------------- + +# Full graphics subsystem in one wrapper: raster + om + tex + rtu + dxa behind a +# shared L2, 2 cores / socket. Closes the graphics cross-unit boundaries that the +# per-unit DUTs above do not. Subsystem enables come from +# hw/syn/xilinx/dut/gfx/Makefile. +- id: gfx + group: graphics + dut: gfx + clock_mhz: 250 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# --- tensor (combined subsystem) --------------------------------------------- + +# Full tensor subsystem in one wrapper: TCU (WGMMA + MX) + dxa behind a shared L2, +# 2 cores / socket. Enables come from hw/syn/xilinx/dut/tensor/Makefile. +- id: tensor + group: tensor + dut: tensor + clock_mhz: 250 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 diff --git a/ci/timing_gate.py b/ci/timing_gate.py new file mode 100755 index 0000000000..539ada474b --- /dev/null +++ b/ci/timing_gate.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""timing_gate — logic-level regression gate (the timing screen). + +Gates the depth of every RTL unit of the DUT catalog: foldsim-estimate-timing +counts the logic levels of every register-to-register path of the CIRCT netlist +the Xilinx flow elaborates, as Vivado counts them on the V80, without running +Vivado. A unit that gets one level deeper than its baseline fails, by name, in +seconds -- where ci/fpga_gate.py finds the Fmax it costs the next night. The +gating machinery is shared with the synthesis gates and lives in +ci/synth_gate.py; this entry point only pins the tool. + + Spec : ci/testcases/timing_gate.yaml (what to screen) + Baselines : ci/baselines/synthesis/screen/.json (last accepted levels) + Builds : hw/syn/xilinx/dut/timing_gate_ (no Vivado) + Metrics : /screen_summary.csv, screen_units.json + (written by hw/syn/xilinx/dut's `screen` target) + +It needs CIRCT and foldsim (ci/toolchain_install.sh --circt --foldsim); the +Xilinx environment is not used. + +Usage (from a source checkout): + ci/timing_gate.py --list + ci/timing_gate.py # all builds, gate against baselines + ci/timing_gate.py -b cache -b tcu # explicit builds + ci/timing_gate.py --update-baseline # re-record (human-reviewed, never in CI) + +The same screen runs by hand through the flow's own entry point -- +`make -C hw/syn/xilinx/dut cache-screen` -- so the gate adds no second way to +build. + +See docs/designs/continuous_integration.md 3.5. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import synth_gate # noqa: E402 + +if __name__ == "__main__": + sys.exit(synth_gate.main(default_tool="screen")) diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index aee40a5df2..184d25d77a 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -277,22 +277,25 @@ so the parity and perf views of one workload are two cases. `perf_gate` catches a change that costs cycles. The synthesis gates catch one that costs **timing closure or area**: they synthesize a catalog of designs and assert the results against the target clock and area budget each build -declares, recording every run's metrics in golden baselines. - -| | `fpga_gate` | `asic_gate` | -|---|---|---| -| flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | -| design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | -| gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | -| also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | -| comparable when | same device, optimization level, xlen | same PDK, VT, corner, xlen | -| runner | self-hosted, licensed Vivado | hosted, one job per design | -| default tolerance | Fmax 2 %, LUT 5 % | Fmax 2 %, cell area 5 % | - -Neither is a pytest cell. A cell is a build tree plus a driver; a synthesis -build is neither, and both gates are hours long. They are standalone scripts +declares, recording every run's metrics in golden baselines. The timing gate +catches one that makes a unit **deeper**, in seconds and without synthesis. + +| | `fpga_gate` | `asic_gate` | `timing_gate` | +|---|---|---|---| +| flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | CIRCT → `foldsim-estimate-timing` (logic levels as Vivado counts them on the V80) | +| design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | `hw/syn/xilinx/dut`, target `screen` | +| gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | `max_levels` against the baseline, and the deepest end point of every RTL unit | +| also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | end point bits at ≥ 12 and ≥ 8 levels, the deepest paths | +| comparable when | same device, optimization level, xlen | same PDK, VT, corner, xlen | same CIRCT, foldsim, xlen | +| runner | self-hosted, licensed Vivado | hosted, one job per design | hosted, one job for the catalog | +| default tolerance | Fmax 2 %, LUT 5 % | Fmax 2 %, cell area 5 % | none: the screen is deterministic | + +None is a pytest cell. A cell is a build tree plus a driver; a gate build is +neither, and the synthesis gates are hours long. They are standalone scripts driven by their own workflows, and their tiers are **opt-in**: an empty tier -selection means "everything a cell can run", which excludes them. +selection means "everything a cell can run", which excludes them. The timing +gate's tier is opt-in because it needs foldsim, which the base toolchain does +not install. #### One implementation, two adapters @@ -300,21 +303,23 @@ selection means "everything a cell can run", which excludes them. tool-specific — catalog loading, the config hash, threshold resolution, `known_issue`, the resumable session, the scheduler, the gate and the report. A tool adapter supplies only what differs: metric names, how the flow is -invoked, what its log looks like, and which environment fields make two runs -comparable. `fpga_gate.py` and `asic_gate.py` are entry points that pin the -tool. +invoked, what its log looks like, which environment fields make two runs +comparable, and any assertion beyond the per-metric check (the timing gate's +per-unit depths). `fpga_gate.py`, `asic_gate.py` and `timing_gate.py` are +entry points that pin the tool. #### Spec and baseline | file | written by | holds | |---|---|---| -| `ci/testcases/{fpga,asic}_gate.yaml` | a person | per build: a design, a target clock, an area budget, `CONFIGS`, a group, optional `thresholds` and `known_issue` | -| `ci/baselines/synthesis/{xilinx,yosys}/.json` | the gate | measured metrics, the config hash, the tool versions and environment | +| `ci/testcases/{fpga,asic,timing}_gate.yaml` | a person | per build: a design, a target clock, an area budget (synthesis gates), `CONFIGS`, a group, optional `thresholds` and `known_issue` | +| `ci/baselines/synthesis/{xilinx,yosys,screen}/.json` | the gate | measured metrics, the config hash, the tool versions and environment | Groups are `core`, `tensor`, `graphics`, `dxa` and `vm`. `asic_gate` uses the same group names and gates a subset of the builds (§4.5), so where the two gates share a design, a divergence between the tools on the same module is a -finding, not noise. +finding, not noise. `timing_gate` screens exactly `fpga_gate`'s builds, so a +unit it flags can be read against the Fmax gate's critical paths. The config hash ties the two files together: edit the spec, and the gate refuses to compare against numbers recorded for the old one. It matters only @@ -360,6 +365,26 @@ so the flow reports the signed worst slack instead. | `XPASS` | a known issue stopped reproducing; clear the flag | no | | `RECORDED` | a baseline was written | no | +#### What the timing gate asserts + +The screen counts logic levels, not slack: it does not see wires, so it gates +depth, which it predicts (Spearman 0.49–0.93 against Vivado per register on the +V80 gates), and leaves Fmax to `fpga_gate`. A **unit** is an end point's +instance path with generate indices folded (`g_banks_3_bank` → +`g_banks_*_bank`). + +| assertion | reason | +|---|---| +| no unit deeper than its baseline | the change that adds a level is named, by unit | +| no new unit at the fix threshold (12 levels at 300 MHz) | a new deep cone is a regression even with no baseline to compare | +| a shallower unit, or one that drops below the watch threshold (8), reads `IMPROVED` | the gain must be recorded | + +There is no tolerance: the same RTL and tools give the same levels. A new CIRCT +or foldsim can map the same RTL differently, so both versions are in the config +hash and a tool change reads `STALE`. `make -C hw/syn/xilinx/dut -screen` +runs one screen by hand; its `screen/report.txt` prints the deepest paths op by +op. + #### Running a multi-hour sweep | mechanism | what it does | @@ -507,6 +532,17 @@ The largest designs — `top`, `gfx`, `tensor`, `vm` — are therefore gated on `fpga_gate` only, and `core` and `tex` are gated at a reduced width. Measure a design's peak memory before adding it: an over-budget design costs a silent job kill, not a red gate. +### 4.6 `timing_gate.yml` + +On every pull request and push to master that touches the RTL, the DUT +wrappers, the screen's scripts, the gate or its baselines; and by manual +dispatch. One hosted job screens the whole catalog: it restores the toolchain, +installs foldsim (`ci/toolchain_install.sh --foldsim`), runs the catalog's +`run:` command with `--report`, and renders the report with +[`synth_report.py`](../../ci/synth_report.py) into the workflow summary. The +artifacts are the report and, per design, the build log, the summary and the +screen's deepest paths. + --- ## 5. Running locally and coverage diff --git a/hw/scripts/circt_elaborate.sh b/hw/scripts/circt_elaborate.sh index c6e07b1590..7049613065 100755 --- a/hw/scripts/circt_elaborate.sh +++ b/hw/scripts/circt_elaborate.sh @@ -33,7 +33,9 @@ # colliding with a write undefined in block RAM; other tools ignore it; # - -f sv (the default) writes SystemVerilog, for Vivado and Quartus; -f # v2005 writes Verilog-2005 for tools with a Verilog frontend (yosys): no -# packed arrays or structs, no local variables, no size casts. +# packed arrays or structs, no local variables, no size casts; -f mlir +# writes the HW/Comb/Seq netlist itself, for tools built on CIRCT (the +# timing screen), and exports nothing. # # The sources are a filelist (-l, as gen_sources.sh writes it: +define+, # +incdir+ and files in compile order), or include directories whose sources @@ -85,7 +87,7 @@ while getopts "t:l:o:I:D:b:x:f:m:r:h" arg; do x) # file of black-box declarations (vendor IP) stubs+=("${OPTARG}") ;; - f) # output format: sv or v2005 + f) # output format: sv, v2005 or mlir format=${OPTARG} ;; m) # RAMs without a hint: off or auto @@ -106,8 +108,8 @@ if [ -z "$top" ]; then exit 1 fi case "$format" in - sv|v2005) ;; - *) echo "circt_elaborate.sh: error: format must be sv or v2005, not '$format'" >&2 + sv|v2005|mlir) ;; + *) echo "circt_elaborate.sh: error: format must be sv, v2005 or mlir, not '$format'" >&2 exit 1 ;; esac [ -z "$report" ] && report="${output_file%.*}.memories.rpt" @@ -138,6 +140,8 @@ for stub in "${stubs[@]}"; do done stem="${output_file%.*}" +# The netlist is the output itself in mlir format; keep the import beside it. +[ "$format" = mlir ] && stem="$stem.import" "$CIRCT_VERILOG" --ir-hw --top="$top" --detect-memories --allow-use-before-declare \ --allow-hierarchical-const --ignore-uninstantiated-modules "${blackboxes[@]}" \ "${defines[@]}" "${args[@]}" "${files[@]}" -o "$stem.mlir" @@ -154,6 +158,11 @@ else fi # A module body is a graph region, where CSE cannot replace a duplicate whose # users come first; sorting the ops first lets it merge every duplicate. +if [ "$format" = mlir ]; then + "$CIRCT_OPT" "$stem.mlir" "${passes[@]}" --canonicalize --topological-sort --cse \ + -o "$output_file" + exit 0 +fi "$CIRCT_OPT" "$stem.mlir" "${passes[@]}" --canonicalize --topological-sort --cse \ -o "$stem.lowered.mlir" "$FIRTOOL" --format=mlir "$stem.lowered.mlir" --disable-all-randomization \ diff --git a/hw/scripts/screen_summary.py b/hw/scripts/screen_summary.py new file mode 100644 index 0000000000..b90b4a667e --- /dev/null +++ b/hw/scripts/screen_summary.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Summarize a timing screen (foldsim-estimate-timing) for the timing gate. + +The screen writes one row per end point bit: the logic levels of its longest +path, its start and its end. ci/timing_gate.py gates depth per RTL unit, so +this collapses the table into what it compares: + + screen_summary.csv one row: the deepest end point, the end point bits at or + above the fix and watch thresholds, and all end points + screen_units.json the deepest end point of every unit at or above the + watch threshold, and the deepest paths (diagnostics) + +A unit is an end point's instance path with generate indices folded +(g_banks_3_bank -> g_banks_*_bank), so one deep bank reads as one unit. + + usage: screen_summary.py --top [--output-dir ] +""" + +import argparse +import collections +import csv +import json +import os +import re + +# 300 MHz on the V80: failing end points have a median of 7-8 levels, and +# about 55-60% of the registers at 12 or more fail (timing_screen_proposal.md). +FIX_LEVELS = 12 +WATCH_LEVELS = 8 +NUM_PATHS = 10 + +COLUMNS = ("top", "max_levels", "deep_endpoints", "watch_endpoints", + "endpoints") + +INDEX = re.compile(r"_\d+(?=_|/|$)") + + +def norm(name): + return name.replace(".", "_").replace("[", "_").replace("]", "") + + +def instance_paths(path): + """Instance id -> path from the top, from the screen's instances.csv.""" + rows = {} + with open(path) as fh: + for row in csv.DictReader(fh): + rows[int(row["id"])] = (int(row["parent"]), row["instance"]) + paths = {0: ""} + + def get(ctx): + if ctx not in paths: + parent, name = rows[ctx] + p = get(parent) + paths[ctx] = "%s/%s" % (p, name) if p else name + return paths[ctx] + + for ctx in rows: + get(ctx) + return paths + + +def unit(path): + """The unit of an end point: its directory, generate indices folded.""" + return INDEX.sub("_*", norm(path).rpartition("/")[0]) or "" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("screen") + ap.add_argument("--top", required=True) + ap.add_argument("--output-dir", default=".") + args = ap.parse_args() + + paths = instance_paths(os.path.join(args.screen, "instances.csv")) + units = collections.defaultdict(int) + counts = collections.Counter() + deepest = [] + with open(os.path.join(args.screen, "endpoints.csv")) as fh: + for row in csv.DictReader(fh): + levels = int(row["levels"]) + counts["endpoints"] += 1 + if levels < WATCH_LEVELS: + continue + counts["watch_endpoints"] += 1 + counts["deep_endpoints"] += levels >= FIX_LEVELS + ip = paths[int(row["end_inst"])] + end = "%s/%s" % (ip, row["end"]) if ip else row["end"] + u = unit(end) + units[u] = max(units[u], levels) + sp = paths[int(row["start_inst"])] + deepest.append((levels, "%s/%s" % (sp, row["start"]) if sp + else row["start"], end)) + # Keep the list short: one sort at the end of a few thousand rows. + if len(deepest) > 50 * NUM_PATHS: + deepest = sorted(deepest, key=lambda d: -d[0])[:NUM_PATHS] + deepest = sorted(deepest, key=lambda d: -d[0])[:NUM_PATHS] + + summary = {"top": args.top, + "max_levels": max(units.values(), default=0), + "deep_endpoints": counts["deep_endpoints"], + "watch_endpoints": counts["watch_endpoints"], + "endpoints": counts["endpoints"]} + if not units: + # Nothing reaches the watch threshold; the deepest is still recorded. + with open(os.path.join(args.screen, "endpoints.csv")) as fh: + summary["max_levels"] = max( + (int(r["levels"]) for r in csv.DictReader(fh)), default=0) + + with open(os.path.join(args.output_dir, "screen_summary.csv"), "w", + newline="") as fh: + w = csv.DictWriter(fh, fieldnames=COLUMNS) + w.writeheader() + w.writerow(summary) + with open(os.path.join(args.output_dir, "screen_units.json"), "w") as fh: + json.dump({"fix_levels": FIX_LEVELS, "watch_levels": WATCH_LEVELS, + "units": dict(sorted(units.items())), + "deepest_paths": ["%d %s -> %s" % d for d in deepest]}, + fh, indent=2) + fh.write("\n") + print("screen: %s max %d levels, %d end point bits >= %d, %d units >= %d" + % (args.top, summary["max_levels"], summary["deep_endpoints"], + FIX_LEVELS, len(units), WATCH_LEVELS)) + + +if __name__ == "__main__": + main() diff --git a/hw/syn/common.mk b/hw/syn/common.mk index b02c8519df..d2726d79dd 100644 --- a/hw/syn/common.mk +++ b/hw/syn/common.mk @@ -38,8 +38,9 @@ CIRCT_VERILOG ?= $(CIRCT_PATH)/bin/circt-verilog CIRCT_OPT ?= $(CIRCT_PATH)/bin/circt-opt FIRTOOL ?= $(CIRCT_PATH)/bin/firtool FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim +FOLDSIM_OPT ?= $(FOLDSIM_PATH)/bin/foldsim-opt YOSYS ?= $(YOSYS_PATH)/bin/yosys STA ?= $(STA_PATH)/bin/sta VERILATOR ?= $(VERILATOR_PATH)/bin/verilator -export CIRCT_PATH FOLDSIM_PATH YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL FOLDSIM YOSYS STA VERILATOR +export CIRCT_PATH FOLDSIM_PATH YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL FOLDSIM FOLDSIM_OPT YOSYS STA VERILATOR diff --git a/hw/syn/xilinx/dut/Makefile b/hw/syn/xilinx/dut/Makefile index 802263a1ad..e72a36bb5f 100644 --- a/hw/syn/xilinx/dut/Makefile +++ b/hw/syn/xilinx/dut/Makefile @@ -23,6 +23,12 @@ $(DUTS): cp build.mk $(PREFIX)_$@/Makefile $(MAKE) -C $(PREFIX)_$@ DUT=$@ clean && $(MAKE) -C $(PREFIX)_$@ DUT=$@ > $(PREFIX)_$@/build.log 2>&1 & +# The timing screen of one DUT, in the foreground: make tcu-screen +%-screen: + mkdir -p $(PREFIX)_$* + cp build.mk $(PREFIX)_$*/Makefile + $(MAKE) -C $(PREFIX)_$* DUT=$* clean screen + # Re-run power analysis on an existing build without rebuilding the design. # Usage: make -power SAIF_FILE= [SAIF_INST=] %-power: diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index 8ab286ed21..199af7ddff 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -84,9 +84,26 @@ power: TOOL_DIR=$(SCRIPT_DIR) SAIF_FILE=$(SAIF_FILE) SAIF_INST=$(SAIF_INST) \ $(VIVADO) -mode batch -source $(SCRIPT_DIR)/xilinx_power_analysis.tcl +# The timing screen (foldsim-estimate-timing): the logic levels of every +# register-to-register path of the design, counted as Vivado counts them, by +# RTL unit, in seconds and without Vivado. It elaborates the same sources and +# defines as gen-sources into a CIRCT netlist of its own, with the Xilinx +# floating-point cores as black boxes. ci/timing_gate.py gates +# screen_summary.csv and screen_units.json; screen/report.txt has the deepest +# paths op by op. +screen: + mkdir -p screen + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -Cscreen/src -Oscreen/sources.f $(RTL_PKGS) + $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t $(TOP_LEVEL_ENTITY) -l screen/sources.f \ + -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r screen/memories.rpt -o screen/netlist.mlir + $(FOLDSIM_OPT) screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-timing-map)' \ + --emit-bytecode -o screen/mapped.mlirbc + $(FOLDSIM_OPT) screen/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) output=screen/endpoints.csv instances=screen/instances.csv report=screen/report.txt})' + python3 $(SCRIPT_DIR)/screen_summary.py screen --top $(TOP_LEVEL_ENTITY) + clean: ifndef RESUME - rm -rf project_1 + rm -rf project_1 screen screen_summary.csv screen_units.json rm -rf .Xil rm -f *.rpt rm -f *.log @@ -96,4 +113,4 @@ else @echo "RESUME is defined, skipping clean." endif -.PHONY: all gen-sources build clean power +.PHONY: all gen-sources build screen clean power From f87734f9cc7189365e9f363aefc8a3f86c883594 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 14:58:00 -0700 Subject: [PATCH 246/336] timing_gate: count levels for the DUT flow's device The Xilinx DUT flow builds for DEVICE (the U55C by default), and the screen counted its levels with the V80's DSPs. The screen target now asks gen_platform.py for the family of DEVICE (--emit=family, from the card descriptors whose part or match_device it is) and passes it to foldsim-estimate-timing, which has profiles for the V80 and the U55C and fails on any other device. The device joins the screen's baseline fingerprint, as it is part of fpga_gate's, so a baseline recorded for another device reads as STALE. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5c07bedfb1de8c9c8de5a505391fadf5687138b6) (cherry picked from commit 108196dbb059aacd140879cb47268c8d8753d61e) Integration (phase 2): the linker-config emitter reads the descriptor's single memory tag (phase 2 descriptor schema). --- ci/synth_gate.py | 11 ++++--- ci/testcases/timing_gate.yaml | 6 ++-- ci/timing_gate.py | 4 +-- docs/designs/continuous_integration.md | 13 +++++--- hw/scripts/gen_platform.py | 45 ++++++++++++++++++-------- hw/syn/xilinx/dut/common.mk | 8 +++-- 6 files changed, 57 insertions(+), 30 deletions(-) diff --git a/ci/synth_gate.py b/ci/synth_gate.py index a8d8e82086..1604039761 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -459,8 +459,9 @@ class Screen(Tool): foldsim-estimate-timing ranks every register-to-register path of the CIRCT netlist the Xilinx flow elaborates by its logic levels, counted as Vivado - counts them on the V80. It needs no Vivado and no licence and takes seconds - per DUT, so it can run on every change where the Fmax gate runs nightly. + counts them on DEVICE's family. It needs no Vivado and no licence and + takes seconds per DUT, so it can run on every change where the Fmax gate + runs nightly. Depth is deterministic -- the same RTL and tools give the same levels -- so the gate has no tolerance: a unit that gets deeper fails, by name, until it @@ -511,12 +512,14 @@ def env(self, args): "foldsim": tool_version("foldsim-opt", "--version", package="foldsim", line_prefix="foldsim"), + "device": os.environ.get("DEVICE", "xcu55c-fsvh2892-2L-e"), "xlen": args.xlen} def hash_key(self, env): # The levels are the model's: a new foldsim or CIRCT can map the same - # RTL to other levels, which must read as STALE, not as a regression. - return [env["circt"], env["foldsim"], str(env["xlen"])] + # RTL to other levels, and another device's family counts them + # differently; either must read as STALE, not as a regression. + return [env["circt"], env["foldsim"], env["device"], str(env["xlen"])] def env_warning(self, build, env): for key, label in (("foldsim", "foldsim"), ("circt", "CIRCT")): diff --git a/ci/testcases/timing_gate.yaml b/ci/testcases/timing_gate.yaml index c2b01aa0d3..660b7a2a0f 100644 --- a/ci/testcases/timing_gate.yaml +++ b/ci/testcases/timing_gate.yaml @@ -5,9 +5,9 @@ # Sibling of fpga_gate (fpga_gate.yaml) and asic_gate (asic_gate.yaml): same # discipline -- golden baseline, results only ever re-recorded by a human -- # over foldsim-estimate-timing instead of a synthesis tool. It counts logic -# levels as Vivado counts them on the V80 (LUTs, CARRY8s, distributed-RAM -# reads) on the CIRCT netlist the Xilinx flow elaborates, so it gates depth, -# not Fmax: wires are not in it. +# levels as Vivado counts them on DEVICE's family (LUTs, CARRY8s, DSP cells, +# distributed-RAM reads) on the CIRCT netlist the Xilinx flow elaborates, so +# it gates depth, not Fmax: wires are not in it. # # - Per unit. A unit is an end point's instance path with generate indices # folded (g_banks_3_bank -> g_banks_*_bank). Every unit at or above the diff --git a/ci/timing_gate.py b/ci/timing_gate.py index 539ada474b..f6d6fa2321 100755 --- a/ci/timing_gate.py +++ b/ci/timing_gate.py @@ -3,8 +3,8 @@ Gates the depth of every RTL unit of the DUT catalog: foldsim-estimate-timing counts the logic levels of every register-to-register path of the CIRCT netlist -the Xilinx flow elaborates, as Vivado counts them on the V80, without running -Vivado. A unit that gets one level deeper than its baseline fails, by name, in +the Xilinx flow elaborates, as Vivado counts them on DEVICE's family, without +running Vivado. A unit that gets one level deeper than its baseline fails, by name, in seconds -- where ci/fpga_gate.py finds the Fmax it costs the next night. The gating machinery is shared with the synthesis gates and lives in ci/synth_gate.py; this entry point only pins the tool. diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index 184d25d77a..84fc0fbc70 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -282,7 +282,7 @@ catches one that makes a unit **deeper**, in seconds and without synthesis. | | `fpga_gate` | `asic_gate` | `timing_gate` | |---|---|---|---| -| flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | CIRCT → `foldsim-estimate-timing` (logic levels as Vivado counts them on the V80) | +| flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | CIRCT → `foldsim-estimate-timing` (logic levels as Vivado counts them on `DEVICE`'s family) | | design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | `hw/syn/xilinx/dut`, target `screen` | | gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | `max_levels` against the baseline, and the deepest end point of every RTL unit | | also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | end point bits at ≥ 12 and ≥ 8 levels, the deepest paths | @@ -369,9 +369,14 @@ so the flow reports the signed worst slack instead. The screen counts logic levels, not slack: it does not see wires, so it gates depth, which it predicts (Spearman 0.49–0.93 against Vivado per register on the -V80 gates), and leaves Fmax to `fpga_gate`. A **unit** is an end point's -instance path with generate indices folded (`g_banks_3_bank` → -`g_banks_*_bank`). +V80 gates), and leaves Fmax to `fpga_gate`. It counts them for the family of +the DUT flow's `DEVICE` (the U55C by default), which +`gen_platform.py --emit=family` reads from `hw/syn/xilinx/platforms`; the +device is part of the baseline fingerprint, as it is for `fpga_gate`. The +screen has profiles for the V80 and the U55C only, and fails on any other +`DEVICE`. A +**unit** is an end point's instance path with generate indices folded +(`g_banks_3_bank` → `g_banks_*_bank`). | assertion | reason | |---|---| diff --git a/hw/scripts/gen_platform.py b/hw/scripts/gen_platform.py index 5645152c67..84dbd31a0b 100644 --- a/hw/scripts/gen_platform.py +++ b/hw/scripts/gen_platform.py @@ -24,6 +24,8 @@ firesim-mk make fragment for hw/syn/firesim and sim/firesim scala the card's FireSim platform config class card the resolved card name + family the FPGA family of a device part, from any card on it (the DUT + synthesis flow, which builds for a part rather than a card) """ from __future__ import annotations @@ -181,6 +183,16 @@ def matches(c: Card) -> bool: return top[0] +def family_of(cards: List[Card], device: str) -> str: + families = {c.family for c in cards + if c.family and (device == c.part or + any(fnmatch.fnmatchcase(device, p) for p in c.match_device))} + if len(families) != 1: + raise DescriptorError(f"no single family for device '{device}': " + f"{sorted(families) or 'no card has this part'}") + return families.pop() + + def emit_mk(c: Card) -> str: # override: a command-line CONFIGS=... must not drop the card settings. ov = "override " @@ -306,7 +318,7 @@ def main(argv: Optional[List[str]] = None) -> int: ap.add_argument("--family", default="", help="FPGA family reported by platforminfo (xrt flow)") ap.add_argument("--device", default="", help="device part (aved flow)") ap.add_argument("--card", default="", help="select a card by name instead of matching") - ap.add_argument("--emit", required=True, choices=("mk", "cfg", "firesim-mk", "scala", "card")) + ap.add_argument("--emit", required=True, choices=("mk", "cfg", "firesim-mk", "scala", "card", "family")) ap.add_argument("--mem-tag", default="") ap.add_argument("--host-tag", default="") ap.add_argument("--kernel-mhz", type=int, default=0) @@ -318,23 +330,28 @@ def main(argv: Optional[List[str]] = None) -> int: ap.add_argument("-o", "--output", default="-") a = ap.parse_args(argv) - if not a.flow and not a.card: + if a.emit == "family" and not a.device: + ap.error("--emit=family takes --device") + if not a.flow and not a.card and a.emit != "family": ap.error("--flow is required unless --card names the card") try: cards = load_all(a.platform_dir) - c = resolve(cards, a.flow, a.xsa, a.family.strip(), a.device, a.card) - if a.emit == "mk": - out = emit_mk(c) - elif a.emit == "cfg": - out = emit_cfg(c, a.mem_tag or c.mem_tag, - "" if a.no_host else (a.host_tag or c.host_tag), - a.kernel_mhz or c.kernel_mhz, a.kernel, a.mem_ports) - elif a.emit == "firesim-mk": - out = emit_firesim_mk(c) - elif a.emit == "scala": - out = emit_scala(c) + if a.emit == "family": + out = family_of(cards, a.device) + "\n" else: - out = c.name + "\n" + c = resolve(cards, a.flow, a.xsa, a.family.strip(), a.device, a.card) + if a.emit == "mk": + out = emit_mk(c) + elif a.emit == "cfg": + out = emit_cfg(c, a.mem_tag or c.mem_tag, + "" if a.no_host else (a.host_tag or c.host_tag), + a.kernel_mhz or c.kernel_mhz, a.kernel, a.mem_ports) + elif a.emit == "firesim-mk": + out = emit_firesim_mk(c) + elif a.emit == "scala": + out = emit_scala(c) + else: + out = c.name + "\n" except DescriptorError as e: print(f"gen_platform: error: {e}", file=sys.stderr) return 1 diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index 199af7ddff..9082178e1e 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -85,8 +85,9 @@ power: $(VIVADO) -mode batch -source $(SCRIPT_DIR)/xilinx_power_analysis.tcl # The timing screen (foldsim-estimate-timing): the logic levels of every -# register-to-register path of the design, counted as Vivado counts them, by -# RTL unit, in seconds and without Vivado. It elaborates the same sources and +# register-to-register path of the design, counted as Vivado counts them on +# DEVICE's family (hw/syn/xilinx/platforms), by RTL unit, in seconds and +# without Vivado. It elaborates the same sources and # defines as gen-sources into a CIRCT netlist of its own, with the Xilinx # floating-point cores as black boxes. ci/timing_gate.py gates # screen_summary.csv and screen_units.json; screen/report.txt has the deepest @@ -98,7 +99,8 @@ screen: -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r screen/memories.rpt -o screen/netlist.mlir $(FOLDSIM_OPT) screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-timing-map)' \ --emit-bytecode -o screen/mapped.mlirbc - $(FOLDSIM_OPT) screen/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) output=screen/endpoints.csv instances=screen/instances.csv report=screen/report.txt})' + family=$$(python3 $(SCRIPT_DIR)/gen_platform.py --device=$(DEVICE) --emit=family) && \ + $(FOLDSIM_OPT) screen/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$$family output=screen/endpoints.csv instances=screen/instances.csv report=screen/report.txt})" python3 $(SCRIPT_DIR)/screen_summary.py screen --top $(TOP_LEVEL_ENTITY) clean: From 913e6b51e29efcf91d1dcb39c2e081680893c936 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 15:04:25 -0700 Subject: [PATCH 247/336] cta_dispatch: shallower occupancy, window and startup-owed logic Three equivalent rewrites of paths that failed 300 MHz in the core: - The occupancy bound tested m*stride <= LMEM_SIZE for every slot count m, which Vivado built from DSP multipliers on the path into the slot pointer. It is now stride <= floor(LMEM_SIZE/m), a constant per m: the same predicate over positive integers, with no multiplier. - The cluster window was built at the pre-wrapped base, so the window mask waited on the pre-wrap compare. Both candidate bases (the tail and zero) are tested in parallel, the tail one by shifting the registered slot map, and the pre-wrap only selects. A single-slot admission reads the tail directly; pre-wrap applies only to a cluster's first CTA. - Whether a launched warp still owes its startup was looked up through the lane's full/partial thread mask, the deepest input of the launch selection. Both answers are now looked up per lane and the full/partial decision selects between them. All three are cycle-exact. With the registered admission, the core gate at 3.333 ns placed goes from -0.528 ns / 6057 failing endpoints to the barrier-store loop alone (fixed separately). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a5a13d125cbe5d0769fac24e156bb198b8e21812) --- hw/rtl/core/VX_cta_dispatch.sv | 39 ++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index 04fbef2e21..b33f1241e7 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -291,17 +291,24 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // A warp is full unless it is the CTA's last and the block size is not a // multiple of the warp width. wire [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] lane_tmask; + wire [NUM_LANES-1:0] lane_full; for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_tmask - wire is_full_warp = ((CTA_TID_WIDTH+1-NT_BITS)'(lane_order[l]) < full_warps_cur); - assign lane_tmask[l] = is_full_warp ? {`VX_CFG_NUM_THREADS{1'b1}} : partial_tmask; + assign lane_full[l] = ((CTA_TID_WIDTH+1-NT_BITS)'(lane_order[l]) < full_warps_cur); + assign lane_tmask[l] = lane_full[l] ? {`VX_CFG_NUM_THREADS{1'b1}} : partial_tmask; end // Does slot w still owe the startup for any lane a full launch activates? A - // partial last warp re-runs the check on its own mask. - wire [`VX_CFG_NUM_WARPS-1:0] slot_needs_init; + // partial last warp re-runs the check on its own mask. Both answers are + // looked up per lane, and the lane's full/partial decision -- the deepest + // input -- only selects between them. + wire [`VX_CFG_NUM_WARPS-1:0] slot_owes_full, slot_owes_part; for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_needs_init - localparam l = w % NUM_LANES; - assign slot_needs_init[w] = |(lane_tmask[l] & ~warp_init_lanes_r[w]); + assign slot_owes_full[w] = ~(&warp_init_lanes_r[w]); + assign slot_owes_part[w] = |(partial_tmask & ~warp_init_lanes_r[w]); + end + wire [NUM_LANES-1:0] lane_needs_init; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_needs_init + assign lane_needs_init[l] = lane_full[l] ? slot_owes_full[lane_wid[l]] : slot_owes_part[lane_wid[l]]; end // One thread-index advance by `step`, with a single wrap per dimension: every @@ -415,8 +422,9 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // Occupancy bound, registered (off the ready path). usable = largest m in // [1, NUM_WARPS] with m*stride <= LMEM_SIZE; all slots when stride == 0. - // The product m*stride is a constant-times-variable (shift/add), so the - // NUM_WARPS-wide comparator tree has no divider. + // m*stride <= LMEM_SIZE is tested as stride <= floor(LMEM_SIZE/m), a constant + // per m, so the NUM_WARPS-wide comparator tree has neither a multiplier nor + // a divider. localparam PROD_W = LMEM_LOG + NW_WIDTH + 2; always_ff @(posedge clk) begin if (reset) begin @@ -430,7 +438,7 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // write past its slot. usable_slots_r <= (NW_WIDTH+1)'(1); for (integer m = 1; m <= NUM_CTA_SLOTS; m = m + 1) begin - if (PROD_W'(m) * PROD_W'(stride) <= PROD_W'(LMEM_SIZE)) begin + if (PROD_W'(stride) <= PROD_W'(LMEM_SIZE / m)) begin usable_slots_r <= (NW_WIDTH+1)'(m); end end @@ -468,10 +476,15 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // base_slot+K <= usable_slots_r <= NUM_CTA_SLOTS), so the mask is built at // NUM_CTA_SLOTS width directly. K == NUM_CTA_SLOTS wraps to all-ones, which // is the intended full mask. + // The window is tested at both candidate bases in parallel: shifting the + // registered slot map by the tail leaves only the pre-wrap select after the + // window size. wire [NUM_CTA_SLOTS-1:0] window_ones = (NUM_CTA_SLOTS'(1) << cluster_k) - NUM_CTA_SLOTS'(1); - wire [NUM_CTA_SLOTS-1:0] cluster_window = window_ones << base_slot; - wire cluster_window_free = ((slot_valid_r & cluster_window) == '0); - wire admit_ok = is_first_of_cluster ? cluster_window_free : ~slot_valid_r[base_slot]; + wire [NUM_CTA_SLOTS-1:0] slot_valid_tail = slot_valid_r >> base_tail; + wire window_free_tail = ((slot_valid_tail & window_ones) == '0); + wire window_free_zero = ((slot_valid_r & window_ones) == '0); + wire cluster_window_free = cluster_prewrap ? window_free_zero : window_free_tail; + wire admit_ok = is_first_of_cluster ? cluster_window_free : ~slot_valid_r[base_tail]; // A fragment launch's stamps land in the warp record TID_STAGES cycles after the // warp fires (the thread-index ripple), but frag_stamps_r is a single bank. So a // new launch must not overwrite it while an earlier fragment warp is still in that @@ -625,7 +638,7 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( warp_id_r[l] <= lane_wid[l]; warp_tmask_r[l] <= lane_tmask[l]; warp_order_r[l] <= lane_order[l]; - warp_skip_init_r[l] <= ~slot_needs_init[lane_wid[l]]; + warp_skip_init_r[l] <= ~lane_needs_init[l]; dispatched_warps[lane_wid[l]] <= 1'b1; end end From 88a47ace8cd9a512c1f9e42789241cd90626260f Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 15:04:25 -0700 Subject: [PATCH 248/336] bar_unit: start the barrier-state update loop from a flop The barrier state is read, updated (arrival count, wait mask, the global barrier's all-arrived compare) and written back every cycle. Two things lengthened that loop: the state's valid bit was a store_valids[] lookup at the registered address, and the store sat in block RAM, so the loop began at a BRAM clock-to-out. The valid bit is now registered with the read address, forwarding a same-cycle write, and the store is distributed RAM with the same write-first behavior. Both are cycle-exact. Core gate at 3.333 ns, placed: the loop's -0.056 ns (9 endpoints, the last failures in the core) -> 0 failing endpoints. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a7957345c195c49fdbada717b96ec81292dea494) --- hw/rtl/core/VX_bar_unit.sv | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/hw/rtl/core/VX_bar_unit.sv b/hw/rtl/core/VX_bar_unit.sv index 1ab924331a..95f0e80c98 100644 --- a/hw/rtl/core/VX_bar_unit.sv +++ b/hw/rtl/core/VX_bar_unit.sv @@ -205,11 +205,15 @@ module VX_bar_unit import VX_gpu_pkg::*; #( wire store_phase_wdata = gbar_rsp_apply ? ~gbar_pending_phase_r[gbar_bus_if.rsp_data.id] : phase_n; wire store_phase_write = req_valid || gbar_rsp_apply; + // The state is read, updated and written back every cycle: in distributed + // RAM the loop starts from a fabric read rather than a block RAM's + // clock-to-out. VX_dp_ram #( .DATAW (BAR_STATEW), .SIZE (1 << BAR_ADDR_BITS), .RDW_MODE ("W"), - .OUT_REG (1) + .OUT_REG (1), + .LUTRAM (1) ) barrier_state_store ( .clk (clk), .reset (reset), @@ -245,20 +249,27 @@ module VX_bar_unit import VX_gpu_pkg::*; #( wire store_phase_rdata_v = store_valids[store_raddr] ? store_phase_rdata : '0; + // The state read's valid bit is registered with its address (forwarding a + // same-cycle write), so the state's consumers start from a flop rather than + // a store_valids lookup. + reg store_state_valid; + always @(posedge clk) begin if (reset) begin store_valids <= '0; + store_state_valid <= 0; phase_r <= '0; end else begin if (store_state_write) begin store_valids[store_waddr] <= 1'b1; end + store_state_valid <= store_valids[store_raddr] || (store_state_write && (store_waddr == store_raddr)); phase_r <= is_phase_rdw_hazard ? store_phase_wdata : store_phase_rdata_v; end store_waddr <= store_raddr; end - assign {mask_r, count_r, events_r} = store_valids[store_waddr] ? store_state_rdata : '0; + assign {mask_r, count_r, events_r} = store_state_valid ? store_state_rdata : '0; wire phase_async = is_phase_rdw_hazard ? store_phase_wdata : store_phase_rdata_v; From 19db9d41f8f87f597fb21c7d2cf0fbcc4e371fec Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 15:16:17 -0700 Subject: [PATCH 249/336] docs: the timing screen's design docs/designs/timing_screen.md: what the screen answers and how to run it (DUT flow, gate, foldsim scripts, the passes and their options), where each piece lives, the two passes, the level model and the V80 and U55C profiles, the outputs, the integration with the DUT flow and timing_gate, the validation against Vivado, the Vivado probes, and the limits. Two figures: the system and the compiler. AGENTS.md gains the rule to screen an RTL change's depth before committing it; foldsim.md and continuous_integration.md link the doc. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit fa49ba2f8258cda140f34588a270ec03a02261f5) --- AGENTS.md | 1 + docs/assets/img/timing_screen_compiler.svg | 131 ++++++++ docs/assets/img/timing_screen_system.svg | 134 +++++++++ docs/designs/continuous_integration.md | 7 +- docs/designs/foldsim.md | 4 +- docs/designs/timing_screen.md | 331 +++++++++++++++++++++ 6 files changed, 604 insertions(+), 4 deletions(-) create mode 100644 docs/assets/img/timing_screen_compiler.svg create mode 100644 docs/assets/img/timing_screen_system.svg create mode 100644 docs/designs/timing_screen.md diff --git a/AGENTS.md b/AGENTS.md index 54f54ad480..fb147f7a11 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,6 +87,7 @@ See [docs/testing.md](docs/testing.md) and [docs/debugging.md](docs/debugging.md - **`ci/regression.sh` is the canonical source of tested configurations.** Use it to discover supported parameter combinations before inventing ad hoc ones. - **Perf-regression baselines (`ci/baselines/perf/*.json`) are golden data — never hand-edit them, and never "fix" a red perf gate by bumping the number.** They are regenerated only by `pytest ci -m perf_gate --update-baselines` (a human-run, reviewed step), and CI must never pass that flag. A `perf_gate` failure means real cycles moved: root-cause it, or — if the change is intended — regenerate the baseline so the diff shows the perf delta for review. Same rule as image goldens and `known_issue:`. - **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,screen}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/timing_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `timing_gate` means a unit got deeper, and its reasons name the unit. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). +- **Screen the logic depth of an RTL change before committing it.** A change to a pipeline, arbiter, queue or datapath can add logic levels that only the nightly `fpga_gate` would find. Run `make -C hw/syn/xilinx/dut -screen`, or `ci/timing_gate.py -b `, for the DUTs it touches. A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. See [docs/designs/timing_screen.md](docs/designs/timing_screen.md). - **SimX is the RTL's timing model — keep them in lockstep.** Any change that moves RTL cycles (pipeline structure, arbitration, queue depths, cache/memory behavior) must land together with the matching SimX timing-model update, and vice-versa. The `model_parity` CI gate enforces this: a `check: model_parity` case (`ci/testcases/core.yaml` + per-extension parity categories) runs the same app/args/configs on simx and rtlsim and asserts exact retired-instruction match plus cycle agreement within the case tolerance (default 5%). Never widen a tolerance to absorb a divergence — model the behavior. When adding a hardware feature, add or extend a parity case that exercises it. - **When RTL debugging stalls, switch to the SimX-as-oracle pattern.** For numerical bugs, deep pipeline races, or any failure mode where rtlsim is "close but wrong": (1) build/extend the SimX C++ model so it mirrors the *new* RTL architecture and gets to PASS; (2) add matching trace dumps to both SimX and RTL (cycle, FU events, SRAM addresses+data, hazards) — same CSV format on both sides; (3) diff trace files — the first divergence is the bug. Don't keep guessing from output values; localize via trace diff. See [docs/debugging.md](docs/debugging.md#simx-as-oracle-for-rtl-debug). diff --git a/docs/assets/img/timing_screen_compiler.svg b/docs/assets/img/timing_screen_compiler.svg new file mode 100644 index 0000000000..89e7e609f7 --- /dev/null +++ b/docs/assets/img/timing_screen_compiler.svg @@ -0,0 +1,131 @@ + + + + + + + + + + + Timing screen — the two passes + + + + foldsim-timing-map (per module) + + + 1. hw-flatten-modules: inline modules < 300 ops + LUTs pack across buffers and arbiters, as Vivado flattens + + + 2. canonicalize, CSE, comb-balance-mux + mux chains become balanced trees + + + 3. foldsim passes + lower-equality lower-constant-division + fold-zero-width + + + 4. convert-comb-to-synth + to an and-inverter graph; add, sub, icmp, mul, + div and mod stay word-level ops + + + 5. lower-word-to-bits, timing-aware variadic + lowering, maximum-and-cover, SOP balancing + depth-optimal AIG, structurally hashed + + + 6. synth-generic-lut-mapper (6 inputs, timing) + comb.truth_table ops: one LUT6 each + + + Result: mapped.mlirbc + LUT6s, word-level arithmetic, registers (seq.firreg, + seq.compreg), memories (seq.firmem: distributed RAM + if read latency 0, block RAM otherwise), instances of + the modules not inlined, external modules (vendor IP) + CIRCT upstream passes + three foldsim passes + + + + + + + + + + + foldsim-estimate-timing (whole design, elaborated) + + + Collect end points + walk the instance tree from top=; one context + per instance. End points: register D, reset and + enable; memory write ports and registered read + address; inputs of vendor IP and black boxes; + top outputs + + + Resolve a bit + see through wires, extracts, concats, replicates + and instance ports to the op that drives it: one + node per (instance, value, bit). Start points: + registers, block-RAM reads, vendor IP outputs, + black-box outputs, top inputs + + + Evaluate: iterative depth-first, one worst predecessor per bit + levels(node) = max over inputs ( levels(input) + edge(input → node) ) + edge levels per op kind, for the device's family: + LUT 1 + add/sub bit j→i 2 + ⌊i/8⌋ − ⌊j/8⌋ (1 if the bit reads ≤ 6 inputs) + compare as an adder, to the top non-constant bit + multiply ≤ 10 variable bits: a table; else DSP cascade + divide ≤ 10 support bits: a table; else a ripple + LUTRAM read 1 + doublings past 64 words + a combinational loop's closing edge is left out and counted; time and memory are + linear in the design's bits (full GPU: 16-21 s, 2.7 GB) + + + End point levels + the input node's levels, plus a DSP's output + stage unless the end point is a register the + DSP absorbs (with its enable logic) + optional retime-levels=: one-hop balancing + + + Report + walk worst predecessors back to the start: + start, largest fanout, module crossings; + histogram; top-paths and trace= op by op + endpoints.csv, instances.csv, report.txt + + + Device profiles (family=): only the DSPs differ + versal DSP58 27×24 in 1 out 6 cascade 3 + virtexuplusHBM DSP48E2 27×18 in 1 out 5 cascade 2 + + + + + + + mapped + diff --git a/docs/assets/img/timing_screen_system.svg b/docs/assets/img/timing_screen_system.svg new file mode 100644 index 0000000000..a0b550836a --- /dev/null +++ b/docs/assets/img/timing_screen_system.svg @@ -0,0 +1,134 @@ + + + + + + + + + + + + + + Timing screen — where each piece lives, and what flows between them + + + + Vortex repository + + + RTL + VX_config.toml + hw/rtl/**, hw/unittest/*/VX_*_top.sv + the DUT catalog: hw/syn/xilinx/dut/catalog.mk + + + hw/syn/xilinx/dut — target screen + gen_sources.sh -P (defines from the toml) + circt_elaborate.sh -f mlir -x xilinx_ip_stubs.sv + gen_platform.py --device=$(DEVICE) --emit=family + + + hw/scripts/screen_summary.py + screen_summary.csv, screen_units.json + units = instance paths, generate indices folded + + + ci/timing_gate.py (synth_gate.py Screen backend) + ci/testcases/timing_gate.yaml + ci/baselines/synthesis/screen/*.json + .github/workflows/timing_gate.yml + PASS / FAIL per unit, zero tolerance on depth + + + + Toolchain ($(TOOLDIR), prebuilt) + + + CIRCT: circt-verilog, circt-opt + SystemVerilog → HW / Comb / Seq netlist + (the same import foldsim and the Yosys flow use) + + + foldsim-opt (foldsim, on CIRCT) + + foldsim-timing-map + inline small modules, balance, lower to AIG, + map to 6-input LUTs; adders, compares, + multipliers and dividers stay word-level + + foldsim-estimate-timing + per-bit arrival over the elaborated instances, + levels as Vivado counts them on family= + versal (V80) | virtexuplusHBM (U55C) + optional: retime-levels=, trace=, black-boxes= + + + Outputs (per DUT, screen/) + endpoints.csv one row per end point bit + instances.csv the elaborated instance tree + report.txt histogram, deepest paths op by op + mapped.mlirbc kept for re-screening other tops + + + + Who reads it + + + Pull requests touching RTL + timing_gate.yml, ~4 min for the 11 DUTs, + no Vivado, no licence: names the unit that + got deeper on the change that did it + + + RTL timing work (native-timing loop) + screen a whole GPU in ~1 min before a + 20-40 min Vivado run; fix list = units at + ≥ 12 levels, watch list ≥ 8 (300 MHz) + + + utils/timing (foldsim) + screen.sh summarize.py validate.py + retime_split.py + cone_dump.tcl cone_probe.tcl wire_probe.tcl + screen ↔ Vivado agreement, and the Vivado + probes for what levels cannot show + + + fpga_gate (nightly, Vivado) + same DUTs and configurations: a unit the + screen flags is checked against its + critical paths; Fmax stays its job + + + + + sources, defines + + netlist.mlir + + mapped.mlirbc + + + endpoints, instances + + + + + solid: data; dashed: who consumes it. Engine in foldsim (ships in the prebuilt), integration in Vortex. + diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index 84fc0fbc70..d8828c97b6 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -367,9 +367,10 @@ so the flow reports the signed worst slack instead. #### What the timing gate asserts -The screen counts logic levels, not slack: it does not see wires, so it gates -depth, which it predicts (Spearman 0.49–0.93 against Vivado per register on the -V80 gates), and leaves Fmax to `fpga_gate`. It counts them for the family of +The screen ([timing_screen.md](timing_screen.md)) counts logic levels, not +slack: it does not see wires, so it gates depth, which it predicts (Spearman +0.50–0.93 against Vivado per register on the V80 gates), and leaves Fmax to +`fpga_gate`. It counts them for the family of the DUT flow's `DEVICE` (the U55C by default), which `gen_platform.py --emit=family` reads from `hw/syn/xilinx/platforms`; the device is part of the baseline fingerprint, as it is for `fpga_gate`. The diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 0b96ad504b..d6637b2c06 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -26,7 +26,9 @@ on it, and runs orders of magnitude faster than Verilator. The whole flow is one IR from SystemVerilog to Verilog: `circt-verilog` imports Vortex, Foldsim's passes cut it into models and compose the simulator, `firtool` writes it out, and the card's native flow (IP-XACT -packaging, `slashkit link`) builds the image. Foldsim is a project of its own +packaging, `slashkit link`) builds the image. (`foldsim-opt` also carries the +timing screen, which counts the logic levels of a Vortex netlist before +Vivado: [timing_screen.md](timing_screen.md).) Foldsim is a project of its own that builds on CIRCT, as POCL builds on LLVM: the `foldsim` driver and `foldsim-opt` come from the toolchain (`$(TOOLDIR)/foldsim`), CIRCT from `$(TOOLDIR)/circt`. Nothing else is in the flow: no Yosys, no sv2v. diff --git a/docs/designs/timing_screen.md b/docs/designs/timing_screen.md new file mode 100644 index 0000000000..ed14ff9eb4 --- /dev/null +++ b/docs/designs/timing_screen.md @@ -0,0 +1,331 @@ +# Timing screen — logic depth of every path, before Vivado + +The timing screen counts the logic levels of every register-to-register path +of a Vortex design the way Vivado counts them, in seconds and without +Vivado. It names the RTL units whose paths are too deep for the target clock, +gates their depth on every RTL change (`timing_gate`), and is the first step +of RTL timing work: screen, fix the deep units, then confirm with one Vivado +run. + +| § | contents | +|---|---| +| [1](#1-overview) | Overview and how to run it | +| [2](#2-system-architecture) | System architecture: where each piece lives | +| [3](#3-compiler-architecture) | Compiler architecture: the two passes | +| [4](#4-the-model) | The model: levels per element, devices | +| [5](#5-outputs) | Outputs | +| [6](#6-integration) | Integration: the DUT flow, `timing_gate`, timing work | +| [7](#7-validation) | Validation against Vivado | +| [8](#8-diagnostics) | Diagnostics beyond levels | +| [9](#9-invariants-and-limitations) | Invariants and limitations | + +--- + +## 1. Overview + +It answers one question per register: **how many logic levels is its worst +input path?** At 300 MHz on the V80, failing end points have a median of +7–8 levels, and between half and four fifths of the registers at 12 or more +levels fail, depending on the gate (§7). So the screen reports: + +- **fix list:** units with end points at ≥ 12 levels; +- **watch list:** ≥ 8 levels. + +It does not predict slack, because it does not see wires (§9). + +One DUT of the synthesis catalog, from a configured build tree: + +```bash +make -C hw/syn/xilinx/dut tcu-screen +# build_tcu/screen/report.txt deepest paths, op by op +# build_tcu/screen_units.json deepest end point per RTL unit +``` + +The whole catalog as a gate, and a baseline update (a human step): + +```bash +ci/timing_gate.py -j 2 +ci/timing_gate.py -b om --update-baseline +``` + +Any CIRCT netlist, with foldsim's scripts (`$FOLDSIM_PATH/utils/timing`): + +```bash +screen.sh design.mlir runs/full # map, then screen +screen.sh runs/full/mapped.mlirbc runs/socket \ + top=VX_socket__lift black-boxes=VX_core__lift # one unit of a mapped design +summarize.py runs/full --min-levels 12 # deep end points by RTL unit +``` + +Or the two passes directly: + +```bash +foldsim-opt design.mlir --emit-bytecode -o mapped.mlirbc \ + --pass-pipeline='builtin.module(foldsim-timing-map)' +foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( + foldsim-estimate-timing{top=VX_core_top family=virtexuplusHBM + output=endpoints.csv instances=instances.csv report=report.txt + top-paths=50 trace=lsu_unit})' +``` + +| `foldsim-estimate-timing` option | meaning | +|---|---| +| `top=` | the top module (default: the one nothing instantiates) | +| `family=` | `versal` (V80, default) or `virtexuplusHBM` (U55C) | +| `black-boxes=` | modules to treat as external, as a harness stubs them | +| `output=`, `instances=`, `report=` | the files of §5 | +| `min-levels=` | write only end points at or above this many levels | +| `top-paths=` | deepest paths to print op by op (default 50) | +| `trace=` | also print the deepest path of each end point whose name contains this | +| `retime-levels=` | also estimate the depth after Vivado's retiming to this period (§4.3) | + +## 2. System architecture + +![Timing screen system](../assets/img/timing_screen_system.svg) + +The engine is two compiler passes of **foldsim** (`foldsim-opt`), built on +CIRCT like the rest of foldsim, and shipped in the foldsim prebuilt. Vortex +never builds it. The integration is in Vortex: + +| Piece | Where | Role | +|---|---|---| +| `foldsim-timing-map`, `foldsim-estimate-timing` | foldsim `lib/Transforms/EstimateTiming.cpp` | map to LUTs; count levels | +| `utils/timing/*` | foldsim | screen any netlist; compare with Vivado; Vivado probes | +| target `screen` | `hw/syn/xilinx/dut/common.mk` | one DUT: sources → netlist → screen → summary | +| `circt_elaborate.sh -f mlir` | `hw/scripts` | the lowered HW/Comb/Seq netlist, no `firtool` | +| `gen_platform.py --emit=family` | `hw/scripts` | the family of `DEVICE`, from `hw/syn/xilinx/platforms` | +| `screen_summary.py` | `hw/scripts` | `screen_summary.csv` and per-unit `screen_units.json` | +| `Screen` backend, `timing_gate.py` | `ci/synth_gate.py`, `ci/` | the gate, beside `fpga_gate` and `asic_gate` | +| `timing_gate.yml` | `.github/workflows` | runs it on pull requests that touch RTL | + +The screen reads the same CIRCT import that foldsim and the Yosys flow +use (`circt-verilog --ir-hw`), elaborated from the sources and defines +`gen_sources.sh` writes for Vivado, with the Xilinx floating-point IPs as +black boxes (`xilinx_ip_stubs.sv`). + +## 3. Compiler architecture + +![Timing screen compiler](../assets/img/timing_screen_compiler.svg) + +### 3.1 `foldsim-timing-map` + +A pipeline of CIRCT's synthesis passes plus three of foldsim's, per module: + +1. **Inline small modules** (`hw-flatten-modules`, < 300 ops) so LUTs pack + across buffers, arbiters and switches, as Vivado's flattened synthesis + does. Larger modules stay instances; the estimator elaborates them. +2. **Canonicalize and balance** mux chains (`comb-balance-mux`). +3. **foldsim passes:** + - `foldsim-lower-equality`: an equality compare is XORs and an OR tree, + which pack into LUTs; + - `foldsim-lower-constant-division`: a division by a constant is an + extract (power of two) or a table (dividend of ≤ 10 bits), as + synthesis writes it; + - `foldsim-fold-zero-width`: zero-width values are constants, which + CIRCT's lowering otherwise rejects. +4. **Lower to an and-inverter graph** (`convert-comb-to-synth`). Adders, + subtracters, compares, multipliers, dividers and remainders stay + word-level: the estimator models them as the carry chains, DSPs and + tables Vivado builds, which a LUT mapping of their gates would not + reproduce. +5. **Depth-optimize** the graph: timing-aware variadic lowering, maximum + AND cover, SOP balancing, structural hashing. +6. **Map to 6-input LUTs** (`synth-generic-lut-mapper`, timing strategy). + +The full GPU maps in about a minute and 18–20 GB. + +### 3.2 `foldsim-estimate-timing` + +The estimator works on the elaborated design: one **context** per instance, +and one **node** per (context, value, bit). It never enumerates paths: + +1. **Collect end points** by walking the instance tree from `top`. +2. **Resolve** each end point bit through wires, extracts, concatenations, + replicates and instance ports to the op that drives it. +3. **Evaluate** with an iterative depth-first walk: a node's levels are the + maximum over its inputs of the input's levels plus the edge's (§4), and + it keeps its one worst predecessor. An input still being evaluated + closes a combinational loop; that edge is left out and counted. +4. **Report**: each end point's start point is found by walking the worst + predecessors back; the walk also yields the path's largest fanout and its + module crossings. + +Time and memory are linear in the design's bits: the full GPU screens in +16–21 s and 2.7 GB, a DUT of the catalog in seconds. + +## 4. The model + +### 4.1 Levels per element + +| Element | Levels | +|---|---| +| LUT (`comb.truth_table`) | 1 | +| Adder or subtracter, input bit j to output bit i | 2 + ⌊i/8⌋ − ⌊j/8⌋: a LUT, then the 8-bit carry blocks; 1 if the output bit reads ≤ 6 input bits (one LUT6) | +| Order compare | as an adder, up to the highest bit not constant in both operands | +| Equality compare | lowered to LUTs by the mapping | +| Multiply, ≤ 10 variable operand bits | a table of constants: 1 + ⌈(bits − 6)/2⌉ | +| Multiply, wider | a DSP cascade over the significant bits (§4.2) | +| Divide or remainder | a table over the bits its operand cones depend on, if ≤ 10; else a ripple of one subtracter per quotient bit | +| Distributed-RAM read (`seq.firmem`, latency 0) | 1, plus 1 per doubling past 64 words (MUXF7, MUXF8, LUT), from the address | +| Other ops | 1 per op, listed in the report | + +Significant bits exclude constant bits on top and copies of the bit below +(a sign extension). A table is a LUT6 for six inputs and a level of 4:1 +multiplexers per two more. + +### 4.2 Devices + +The V80 and the U55C both have LUT6s, 8-bit carry blocks (LOOKAHEAD8, +CARRY8) and 64-word distributed RAMs; they differ only in their DSPs. The +profile is selected by `family=`, with the names of +`hw/syn/xilinx/platforms`: + +| `family=` | Device | DSP multiplier | fabric → registered product | product → fabric | per cascaded DSP | +|---|---|---|---|---|---| +| `versal` | V80 | DSP58, 27 × 24 | 1 | 6 | 3 | +| `virtexuplusHBM` | U55C | DSP48E2, 27 × 18 | 1 | 5 | 2 | + +A product takes ⌈a/A⌉·⌈b/B⌉ DSPs (the better orientation). The cell counts +are Vivado's: on Versal, M_DATA, ALUMUX, ALUADD and OUTPUT out of a DSP, +and ALUMUX, ALUADD and OUTPUT through each cascaded one. A register fed by +the product is the DSP's own (MREG or PREG), and so is a LUT that reads that +register back, its clock enable or feedback; any other load pays the output +stage. The fabric-to-product value is fitted on the V80 data. Any other +family is an error. + +### 4.3 Retiming (optional) + +All of Vortex's Vivado runs retime. With `retime-levels=N`, the `retimed` +column estimates each end point's depth after one hop of balancing to a +period of N levels. The end register moves backward into the room its +output stage leaves, and the start register moves forward into the room its +input stage leaves. Neither crosses a distributed RAM or a DSP, and only a +register Vivado can retime (synchronous reset) moves. Top ports and +black-box pins count as registers with empty neighbour stages, as a gate +harness makes them. It ranks no better than plain levels, so `levels` stays +the ranking; `retimed ≥ 12` at `retime-levels=10` is a short list of fix +candidates retiming is unlikely to absorb. + +## 5. Outputs + +| File | Contents | +|---|---| +| `endpoints.csv` | one row per end point bit: `levels,retimed,kind,start_kind,fanout,crossings,start_inst,start,end_inst,end` | +| `instances.csv` | the instance tree: `id,parent,instance,module` (the `*_inst` columns' ids) | +| `report.txt` | counts, combinational loops, ops without a model, the histogram of end points by levels, the `top-paths` deepest paths op by op, and the deepest path of each `trace=` match | +| `screen_summary.csv` | the DUT flow's one row: max levels, end points ≥ 12, ≥ 8, all | +| `screen_units.json` | the deepest end point of every unit at ≥ 8 levels, and the 10 deepest paths | + +A path in `report.txt`, deepest first, one op per line: levels at the op, +its kind, the fanout of its bit, and its instance path and name: + +``` +#1 levels 20: .../mem_req_queue/pending_size/used_r.d[0] (VX_socket__lift) + 20 lut fo 0 .../comb.truth_table[0] + 19 carry fo 1 .../comb.add[0] + ... + 0 reg fo 59 .../req_pending/in_fire_r[13] +``` + +## 6. Integration + +### 6.1 The DUT flow + +`hw/syn/xilinx/dut` builds every DUT of `catalog.mk` for Vivado; its +`screen` target screens the same DUT with the same sources, defines and +`DEVICE` (the U55C by default): + +1. `gen_sources.sh -P` into `screen/src`; +2. `circt_elaborate.sh -f mlir` into `screen/netlist.mlir`, the FP IPs as + black boxes; +3. `foldsim-timing-map`, then `foldsim-estimate-timing` with + `family=$(gen_platform.py --device=$(DEVICE) --emit=family)`; +4. `screen_summary.py`. + +### 6.2 `timing_gate` + +The third synthesis gate, beside `fpga_gate` (Vivado, nightly) and +`asic_gate` (Yosys + OpenSTA, nightly); see +[continuous_integration.md §3.5](continuous_integration.md). It screens +`fpga_gate`'s builds (`ci/testcases/timing_gate.yaml`, all but `top`, whose +`VX_cp_core` keeps LLHD reference ports) in about four minutes, and checks +each against `ci/baselines/synthesis/screen/`: + +- a **unit** that gets deeper fails, by name; +- a **new unit** at or above the fix threshold (12) fails; +- a unit that gets shallower, or leaves the watch list, is an improvement + to lock in by re-recording; +- end point counts are recorded, not gated (they grow with lanes and warps). + +Depth is deterministic, so the tolerance is zero. The baseline fingerprint +is the CIRCT and foldsim versions, `DEVICE` and `XLEN`: a new foldsim reads +as STALE, not as a regression. Baselines are golden data, re-recorded only +by a human (`--update-baseline`), as for the other gates. + +### 6.3 RTL timing work + +Screen first, then Vivado: + +```bash +screen.sh gen/design.mlir runs/full +summarize.py runs/full --min-levels 12 # the fix list, by unit +``` + +The screen ranks the units; one Vivado run confirms the fix. Compare a fix +against a native gate (`hw/syn/xilinx/dut` or the OOC gates), not the +folded design (§9). + +## 7. Validation + +Against Vivado, per RTL register or memory: the screen's deepest bit +against Vivado's deepest failing path and worst slack, on seven V80 datasets +with no new Vivado runs (foldsim `utils/timing/validate.py`): + +| Dataset | Vivado run | Spearman vs levels | Spearman vs slack | ≥ 12: precision / recall | +|---|---|---|---|---| +| full GPU, folded | 4.0 ns, routed, 270 k end points | 0.75 | 0.45 | — | +| socket | native gate, 4.0 ns, routed | 0.55 | 0.53 | 0.64 / 0.31 | +| D-cache | native, 4.0 ns, routed | 0.71 | 0.55 | 0.67 / 0.38 | +| top | native gate, 3.333 ns, placed | 0.53 | 0.39 | 0.19 / 0.53 | +| socket | native gate, 3.333 ns, placed | 0.50 | 0.33 | 0.52 / 0.18 | +| core | native gate, 3.333 ns, placed | 0.93 | 0.62 | 0.81 / 0.13 | +| compute slice | native gate, 3.333 ns, placed | 0.87 | 0.75 | 0.58 / 0.57 | + +The screen's depths run about 1.25× Vivado's on the native gates. The U55C +profile was checked against `fpga_gate`'s critical paths: it agrees with +the V80 profile wherever no DSP is on the path, and the remaining +disagreement is the same on both devices. + +## 8. Diagnostics beyond levels + +foldsim's `utils/timing` has three Vivado Tcl probes for questions the +screen cannot answer, run on a checkpoint +(`vivado -mode batch -source -tclargs ...`): + +| Probe | Answers | +|---|---| +| `cone_dump.tcl` | the worst path into a register, cell by cell, following retimed start registers back | +| `cone_probe.tcl` | whether paths exist between two register sets, and how deep Vivado built them | +| `wire_probe.tcl` | per path: length in tiles, bounding box and fanout beside logic and net delay | + +## 9. Invariants and limitations + +- **Depth, not slack.** On the placed 3.333 ns socket gate, wires are 83 % + of the worst paths' delay; levels explain 1 % of net delay, and the real + placement two thirds. No pre-placement wire term ranked slack better than + levels on every held-out gate, so the screen has none. Route-dominated + failures with few levels are invisible to it: recall at ≥ 12 levels is + 0.13–0.57. +- **Retiming and packing.** Vivado retimes and packs across what the RTL + separates: a popcount, its adders and a compare become a LUT6 tree with a + retimed register inside. The screen's depths are RTL stage depths, about + 1.25× Vivado's. +- **The fold.** In foldsim's folded designs, the fold turns some registers + into LUTRAM and cuts some chains, so its paths differ from the RTL's. The + native gates are the reference for native RTL. +- **Black boxes.** Vendor IPs and `black-boxes=` modules end and start + paths at their pins: their insides are not timed. +- **Devices.** The V80 and the U55C only. +- **Toolchain.** The catalog maps only with three CIRCT Synth fixes + (zero-width mux, constant SOP cut, a failing longest-path query), which + reach CI through the foldsim prebuilt. From c49b26b4d23225dea62138afdb78a1e29aed5312 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 15:26:55 -0700 Subject: [PATCH 250/336] ci: timing_screen.py and the Vivado timing probes The timing screen's scripts read Vortex designs -- its fold classes, its gate harnesses, its Vivado runs -- so they belong with Vortex's other utilities rather than in foldsim, which keeps only the two passes: - ci/timing_screen.py, one CLI like perfetto.py and roofline.py: `screen` maps an elaborated netlist and screens it (foldsim-opt from the PATH or FOLDSIM_OPT), `summarize` groups the deep end points by RTL unit, `validate` scores them per register against a Vivado end point table, `retime-split` splits that by Vivado's retiming. Each reproduces the former foldsim script's output exactly. - hw/scripts/xilinx_{cone_dump,cone_probe,wire_probe}.tcl, beside the other Vivado scripts: the worst path into a register cell by cell, whether paths exist between two register sets and how deep, and the placement each path's net delay comes from. timing_screen.md and its system figure name the new locations. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a1dd8d3085136688865f585bfc978500a34f440a) --- ci/timing_screen.py | 500 +++++++++++++++++++++++ docs/assets/img/timing_screen_system.svg | 10 +- docs/designs/timing_screen.md | 30 +- hw/scripts/xilinx_cone_dump.tcl | 59 +++ hw/scripts/xilinx_cone_probe.tcl | 64 +++ hw/scripts/xilinx_wire_probe.tcl | 62 +++ 6 files changed, 707 insertions(+), 18 deletions(-) create mode 100755 ci/timing_screen.py create mode 100644 hw/scripts/xilinx_cone_dump.tcl create mode 100644 hw/scripts/xilinx_cone_probe.tcl create mode 100644 hw/scripts/xilinx_wire_probe.tcl diff --git a/ci/timing_screen.py b/ci/timing_screen.py new file mode 100755 index 0000000000..aaa45562fb --- /dev/null +++ b/ci/timing_screen.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""timing_screen -- run foldsim's timing screen on a Vortex netlist, and read +its results by RTL unit or against Vivado. + + screen map an elaborated CIRCT netlist and count the logic levels of + every end point (foldsim-timing-map, foldsim-estimate-timing) + summarize the deep end points grouped by RTL unit, optionally beside + Vivado's failing ones + validate agreement per register with a Vivado end point table + retime-split the same agreement, split by whether Vivado retimed the paths + + ci/timing_screen.py screen gen/design.mlir runs/full + ci/timing_screen.py screen runs/full/mapped.mlirbc runs/socket \\ + top=VX_socket__lift black-boxes=VX_core__lift + ci/timing_screen.py summarize runs/full --min-levels 12 + ci/timing_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut + +A Vivado end point table is a CSV of `slack,levels,logic,net,start,end`, one +row per failing end point (a gate's ep.csv). The DUT flow and the timing gate +screen with the Makefiles instead (make -C hw/syn/xilinx/dut -screen, +ci/timing_gate.py); see docs/designs/timing_screen.md. +""" +import argparse +import collections +import csv +import os +import re +import resource +import subprocess +import sys +import time + +# Fold classes of the static fold: module in the design -> cell prefix in +# the folded netlist. +FOLD_CLASSES = { + 'VX_socket__lift': 'VX_socket__lift_model_copy', + 'VX_core__lift': 'VX_core__lift_model_copy', + 'VX_compute_slice__lift': 'VX_compute_slice__lift_copy', + 'foldsim_target': 'foldsim_target_model_copy', +} + +VIVADO_CELL = re.compile( + r'_reg((\[\d+\])*(_replica(_\d+)?)?(_[fb]ret(_\d+)?)*(_rep(_\d+)?)?' + r'|_\d+_\d+_\d+_\d+|_bram_\d+)$') +FOLD_SUFFIX = re.compile( + r'(_ext(_\d+)?|_fold|_read(_\d+)?|_[fb]ret(_\d+)?|_replica(_\d+)?)+$') +FOLD_SHELL = re.compile(r'^fold\d+_') +DSP_SUBCELL = re.compile(r'^DSP\w*(<\d+>)?_INST$') + + +def norm(s): + return s.replace('.', '_').replace('[', '_').replace(']', '') + + +def ep_key(cell_path, native): + """(class, key) of a Vivado end point pin, or None for fold-shell cells. + + Folded: sim//[/]/. + Native: //[/]/, being the + harness prefix. + """ + parts = cell_path.split('/') + if native: + cls = parts[0] + hier = parts[1:-1] + # Distributed RAM primitives below the memory cell, and the sub-cells + # of a DSP macro (DSP58C<0>_INST/DSP_M_DATA58_INST, + # DSP_OUTPUT_INST) below the register it absorbed. + for i, p in enumerate(hier): + if (p.startswith('Memory_reg') or re.match(r'RAM[A-H]', p) + or DSP_SUBCELL.match(p)): + hier = hier[:i] + break + else: + if parts[0] != 'sim' or len(parts) < 3: + return None + cls = parts[1] + hier = parts[2].split('2F') + if not hier or FOLD_SHELL.match(hier[-1]): + return None + leaf = VIVADO_CELL.sub('', hier[-1]) + leaf = FOLD_SUFFIX.sub('', leaf) + return cls, norm('/'.join(hier[:-1] + [leaf])) + + +def numpy(): + # Only the comparisons with Vivado need numpy; screening does not. + import numpy as np + return np + + +def spearman(a, b): + np = numpy() + + def rank(x): + x = np.asarray(x, dtype=float) + order = np.argsort(x, kind='mergesort') + ranks = np.empty(len(x)) + ranks[order] = np.arange(len(x)) + # Average the ranks of ties. + _, inv, counts = np.unique(x, return_inverse=True, return_counts=True) + sums = np.bincount(inv, weights=ranks) + return (sums / counts)[inv] + ra, rb = rank(a), rank(b) + if ra.std() == 0 or rb.std() == 0: + return float('nan') + return float(np.corrcoef(ra, rb)[0, 1]) + + +def endpoints(screen): + """The rows of a screen's end point table, as dicts.""" + with open(f'{screen}/endpoints.csv') as f: + yield from csv.DictReader(f) + + +def load_instances(path): + inst = {} + with open(path) as f: + for row in csv.DictReader(f): + inst[int(row['id'])] = (int(row['parent']), row['instance'], + row['module']) + return inst + + +def class_paths(inst, classes): + """ctx -> (class cell prefix, path below the class root).""" + memo = {} + + def get(ctx): + if ctx in memo: + return memo[ctx] + parent, name, module = inst[ctx] + if module in classes: + res = (classes[module], '') + elif ctx == 0: + res = (None, '') + else: + cls, p = get(parent) + res = (cls, f'{p}/{name}' if p else name) + memo[ctx] = res + return res + + sys.setrecursionlimit(10000) + return {ctx: get(ctx) for ctx in inst} + + +INDEX = re.compile(r'_\d+(?=_|/|$)') + + +def unit(key): + d = key.rpartition('/')[0] + return INDEX.sub('_*', d) + + +def cmd_screen(args): + """Map (unless the input is already mapped bytecode), then screen.""" + opt = os.environ.get("FOLDSIM_OPT") or "foldsim-opt" + os.makedirs(args.out, exist_ok=True) + asic = any(o == "family=asap7" for o in args.options) + mapped = args.design + steps = [] + if not args.design.endswith(".mlirbc"): + mapped = os.path.join(args.out, "mapped.mlirbc") + pipeline = "foldsim-timing-map-asic" if asic else "foldsim-timing-map" + steps.append(("map", [opt, args.design, "--emit-bytecode", "-o", mapped, + "--pass-pipeline=builtin.module(%s)" % pipeline])) + opts = " ".join(["output=%s/endpoints.csv" % args.out, + "instances=%s/instances.csv" % args.out, + "report=%s/report.txt" % args.out] + args.options) + steps.append(("screen", [opt, mapped, "-o", os.devnull, + "--pass-pipeline=builtin.module(" + "foldsim-estimate-timing{%s})" % opts])) + for name, cmd in steps: + start = time.time() + rc = subprocess.call(cmd) + peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss / 1e6 + print("%-7s %6.1f s peak %.1f GB" % (name, time.time() - start, peak)) + if rc: + return rc + return 0 + + +def cmd_validate(args): + np = numpy() + + inst = load_instances(f'{args.screen}/instances.csv') + classes = FOLD_CLASSES + sub = None + if args.native: + module, prefix = args.native.split('=') + prefix, _, sub = prefix.partition(':') + classes = {module: prefix} + paths = class_paths(inst, classes) + + ours = collections.defaultdict(lambda: [0, 0]) # key -> [max levels, n] + # (class, instance path) -> {register base name: key}, for Vivado names + # that place a register in the generate scope that assigns it rather + # than the one that declares it. + by_dir = collections.defaultdict(dict) + for row in endpoints(args.screen): + cls, p = paths[int(row['end_inst'])] + if cls is None: + continue + # Registers of inlined modules are named /. + base = row['end'].rsplit('.', 1)[0] + full = f'{p}/{base}' if p else base + if sub: + if not full.startswith(sub + '/'): + continue + full = full[len(sub) + 1:] + key = (cls, norm(full)) + v = ours[key] + if not v[1]: + d, _, base = full.rpartition('/') + by_dir[(cls, norm(d))][norm(base.rsplit('.', 1)[-1])] = key + v[0] = max(v[0], int(row[args.metric])) + v[1] += 1 + + def fallback(key): + cls, k = key + d, _, leaf = k.rpartition('/') + best = None + for name, cand in by_dir.get((cls, d), {}).items(): + if (leaf == name or leaf.endswith('_' + name)) and ( + best is None or len(name) > len(best[0])): + best = (name, cand) + return best[1] if best else key + + viv = collections.defaultdict(lambda: [0, 0.0, 0]) # max lv, min slack, n + rows = unmatched_rows = shell_rows = 0 + unmatched = collections.Counter() + with open(args.ep) as f: + r = csv.reader(f) + next(r) + for slack, levels, logic, net, start, end in r: + rows += 1 + key = ep_key(end, args.native) + if key is None or key[0] not in classes.values(): + shell_rows += 1 + continue + if key not in ours and key[1].endswith('_rdata'): + # The read register of a synchronous RAM is the memory's + # read port in the design. + key = (key[0], key[1][:-len('_rdata')]) + if key not in ours: + key = fallback(key) + if key not in ours: + unmatched_rows += 1 + unmatched[key] += 1 + continue + v = viv[key] + v[0] = max(v[0], int(levels)) + v[1] = min(v[1], float(slack)) + v[2] += 1 + + keys = sorted(viv) + if not keys: + print(f'ep rows {rows}: fold shell / channels {shell_rows}, ' + f'unmatched {unmatched_rows}, matched 0') + for (cls, k), n in unmatched.most_common(args.top): + print(f' {n:6d} {cls}: {k}') + return + a = np.array([ours[k][0] for k in keys]) + lv = np.array([viv[k][0] for k in keys]) + sl = np.array([viv[k][1] for k in keys]) + print(f'ep rows {rows}: fold shell / channels {shell_rows}, ' + f'unmatched {unmatched_rows}, matched {rows - shell_rows - unmatched_rows} ' + f'in {len(keys)} registers/memories') + print(f'screen keys in the classes: {len(ours)}') + print(f'Spearman(screen levels, Vivado levels) = {spearman(a, lv):.3f}') + print(f'Spearman(screen levels, -slack) = {spearman(a, -sl):.3f}') + ratio = a[lv > 0] / lv[lv > 0] + print(f'screen/Vivado levels: median {np.median(ratio):.2f}, ' + f'p10 {np.percentile(ratio, 10):.2f}, p90 {np.percentile(ratio, 90):.2f}') + + print('\nVivado levels -> screen levels (registers/memories):') + for v in range(0, int(lv.max()) + 1): + sel = a[lv == v] + if len(sel): + print(f' {v:3d}: n={len(sel):6d} screen median {np.median(sel):5.1f} ' + f'p10 {np.percentile(sel, 10):5.1f} p90 {np.percentile(sel, 90):5.1f}') + + if args.native: + # The ep.csv of a gate lists every failing end point, so a screened + # register without one meets timing. + scr = np.array([v[0] for v in ours.values()]) + fails = np.array([k in viv for k in ours]) + print('\nflag registers at >= T levels (gate lists all failing end ' + 'points):') + print(' T flagged precision recall') + for t in range(4, 21, 2): + flagged = scr >= t + if not flagged.any(): + break + prec = fails[flagged].mean() + rec = flagged[fails].mean() + print(f' {t:4d} {flagged.sum():8d} {prec:10.2f} {rec:7.2f}') + + if args.json: + summary = { + 'rows': rows, 'shell': shell_rows, 'unmatched': unmatched_rows, + 'registers': len(keys), 'screened': len(ours), + 'rho_levels': round(spearman(a, lv), 3), + 'rho_slack': round(spearman(a, -sl), 3), + 'ratio': round(float(np.median(ratio)), 2), + 'pairs': [[int(x), int(y), n] for (x, y), n in + sorted(collections.Counter(zip(a, lv)).items())], + } + if args.native: + summary['pr'] = [ + [t, int((scr >= t).sum()), round(float(fails[scr >= t].mean()), 3), + round(float((scr >= t)[fails].mean()), 3)] + for t in range(2, 23) if (scr >= t).any()] + with open(args.json, 'w') as f: + json.dump(summary, f) + + print(f'\nunmatched Vivado cells (top {args.top} by rows):') + for (cls, k), n in unmatched.most_common(args.top): + print(f' {n:6d} {cls}: {k}') + + order = np.argsort(-a, kind='mergesort') + print(f'\ndeepest screened registers with failing Vivado end points:') + for i in order[:args.top]: + k = keys[i] + print(f' screen {a[i]:3d} vivado {lv[i]:3d} slack {sl[i]:7.3f} ' + f'n {viv[k][2]:5d} {k[0]}: {k[1]}') + + + +def cmd_summarize(args): + + inst = load_instances(f'{args.screen}/instances.csv') + classes = FOLD_CLASSES + if args.native: + module, prefix = args.native.split('=') + classes = {module: prefix} + paths = class_paths(inst, classes) + + groups = collections.defaultdict( + lambda: {'n': 0, 'max': -1, 'hist': collections.Counter(), + 'worst': None}) + for row in endpoints(args.screen): + levels = int(row[args.metric]) + if levels < args.min_levels: + continue + cls, p = paths[int(row['end_inst'])] + end, start = row['end'], row['start'] + full = f'{p}/{end}' if p else end + g = groups[(cls, unit(norm(full)))] + g['n'] += 1 + g['hist'][levels] += 1 + if levels > g['max']: + sp = paths[int(row['start_inst'])][1] + g['max'] = levels + g['worst'] = (f'{sp}/{start}' if sp else start, full, row['fanout']) + + viv = collections.defaultdict(lambda: [0, 0]) # n, max levels + if args.ep: + with open(args.ep) as f: + r = csv.reader(f) + next(r) + for slack, levels, logic, net, start, end in r: + if int(levels) < args.min_levels: + continue + key = ep_key(end, args.native) + if key is None: + continue + v = viv[(key[0], unit(key[1]))] + v[0] += 1 + v[1] = max(v[1], int(levels)) + + order = sorted(groups, key=lambda k: (-groups[k]['max'], -groups[k]['n'])) + print(f'end point bits of at least {args.min_levels} levels, by unit ' + f'({sum(g["n"] for g in groups.values())} in {len(groups)} units)') + hdr = f'{"max":>4} {"mode":>4} {"bits":>8}' + if args.ep: + hdr += f' {"viv n":>7} {"viv max":>7}' + print(hdr + ' unit') + for k in order[:args.top]: + g = groups[k] + mode = g['hist'].most_common(1)[0][0] + line = f'{g["max"]:4d} {mode:4d} {g["n"]:8d}' + if args.ep: + v = viv.get(k, [0, 0]) + line += f' {v[0]:7d} {v[1]:7d}' + print(f'{line} {k[0] or ""}: {k[1]}') + s, e, fo = g['worst'] + print(f'{"":>20}from {s}\n{"":>20}to {e}') + if args.ep: + missed = sorted((k for k in viv if k not in groups), + key=lambda k: -viv[k][0]) + print(f'\nVivado units at or above {args.min_levels} levels the ' + f'screen ranks below it ({len(missed)}):') + for k in missed[:args.top]: + print(f'{viv[k][0]:8d} {viv[k][1]:4d} {k[0]}: {k[1]}') + + + +RETIMED = re.compile(r'_[fb]ret') + + +def cmd_retime_split(args): + np = numpy() + screen, ep, native, metric = args.screen, args.ep, args.native, args.metric + module, prefix = native.split('=') + inst = load_instances(f'{screen}/instances.csv') + paths = class_paths(inst, {module: prefix}) + ours = collections.defaultdict(int) + for row in endpoints(screen): + cls, p = paths[int(row['end_inst'])] + base = row['end'].rsplit('.', 1)[0] + key = (cls, norm(f'{p}/{base}' if p else base)) + ours[key] = max(ours[key], int(row[metric])) + viv = collections.defaultdict(lambda: [0, False]) + with open(ep) as f: + r = csv.reader(f) + next(r) + for slack, levels, logic, net, start, end in r: + key = ep_key(end, True) + if key not in ours: + continue + v = viv[key] + v[0] = max(v[0], int(levels)) + v[1] |= bool(RETIMED.search(start) or RETIMED.search(end)) + for retimed in (False, True): + keys = [k for k, v in viv.items() if v[1] == retimed] + a = np.array([ours[k] for k in keys]) + b = np.array([viv[k][0] for k in keys]) + if len(keys) < 3: + continue + ratio = np.median(a[b > 0] / b[b > 0]) if (b > 0).any() else float('nan') + print(f'retimed={retimed!s:5}: {len(keys):5d} registers, Spearman ' + f'{spearman(a, b):.3f}, median screen/Vivado {ratio:.2f}, ' + f'screen-Vivado median {np.median(a - b):+.1f}') + + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("screen", help="map and screen a netlist") + p.add_argument("design", help="elaborated netlist (.mlir), or mapped bytecode (.mlirbc)") + p.add_argument("out", help="output directory") + p.add_argument("options", nargs="*", + help="foldsim-estimate-timing options (top=, family=, black-boxes=, " + "trace=, retime-levels=, ...); family=asap7 maps for the ASIC library") + p.set_defaults(fn=cmd_screen) + + p = sub.add_parser("summarize", help="deep end points by RTL unit") + p.add_argument('screen') + p.add_argument('--min-levels', type=int, default=10) + p.add_argument('--ep', help="a Vivado end point table to group beside") + p.add_argument('--native', metavar='MODULE=PREFIX') + p.add_argument('--top', type=int, default=40) + p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], + help="the screen's column to group by") + p.set_defaults(fn=cmd_summarize) + + p = sub.add_parser("validate", help="agreement per register with Vivado") + p.add_argument('screen') + p.add_argument('ep') + p.add_argument('--top', type=int, default=30) + p.add_argument('--native', metavar='MODULE=PREFIX[:SUBPATH]', + help='native gate: the screened top and the harness ' + 'prefix of its cells (e.g. VX_socket__lift=dut); with ' + 'SUBPATH, only the screened end points below it, which ' + 'the harness instantiates as PREFIX') + p.add_argument('--json', help='also write the agreement to this file') + p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], + help="the screen's column to compare") + p.set_defaults(fn=cmd_validate) + + p = sub.add_parser("retime-split", help="agreement split by Vivado's retiming") + p.add_argument('screen') + p.add_argument('ep') + p.add_argument('native', metavar='MODULE=PREFIX') + p.add_argument('metric', nargs='?', default='levels', choices=['levels', 'retimed']) + p.set_defaults(fn=cmd_retime_split) + + args = ap.parse_args() + return args.fn(args) or 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docs/assets/img/timing_screen_system.svg b/docs/assets/img/timing_screen_system.svg index a0b550836a..1c6a0cb222 100644 --- a/docs/assets/img/timing_screen_system.svg +++ b/docs/assets/img/timing_screen_system.svg @@ -102,10 +102,10 @@ ≥ 12 levels, watch list ≥ 8 (300 MHz) - utils/timing (foldsim) - screen.sh summarize.py validate.py - retime_split.py - cone_dump.tcl cone_probe.tcl wire_probe.tcl + ci/timing_screen.py, hw/scripts/xilinx_*.tcl + screen summarize validate retime-split + xilinx_cone_dump.tcl xilinx_cone_probe.tcl + xilinx_wire_probe.tcl screen ↔ Vivado agreement, and the Vivado probes for what levels cannot show @@ -130,5 +130,5 @@ - solid: data; dashed: who consumes it. Engine in foldsim (ships in the prebuilt), integration in Vortex. + solid: data; dashed: who consumes it. Engine in foldsim (ships in the prebuilt); scripts and integration in Vortex. diff --git a/docs/designs/timing_screen.md b/docs/designs/timing_screen.md index ed14ff9eb4..4f40daabce 100644 --- a/docs/designs/timing_screen.md +++ b/docs/designs/timing_screen.md @@ -48,13 +48,16 @@ ci/timing_gate.py -j 2 ci/timing_gate.py -b om --update-baseline ``` -Any CIRCT netlist, with foldsim's scripts (`$FOLDSIM_PATH/utils/timing`): +Any CIRCT netlist, with `ci/timing_screen.py` (subcommands `screen`, +`summarize`, `validate`, `retime-split`; `foldsim-opt` from the `PATH` or +`FOLDSIM_OPT`): ```bash -screen.sh design.mlir runs/full # map, then screen -screen.sh runs/full/mapped.mlirbc runs/socket \ - top=VX_socket__lift black-boxes=VX_core__lift # one unit of a mapped design -summarize.py runs/full --min-levels 12 # deep end points by RTL unit +ci/timing_screen.py screen design.mlir runs/full # map, then screen +ci/timing_screen.py screen runs/full/mapped.mlirbc runs/socket \ + top=VX_socket__lift black-boxes=VX_core__lift # one unit of a mapped design +ci/timing_screen.py summarize runs/full --min-levels 12 # deep end points by RTL unit +ci/timing_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut ``` Or the two passes directly: @@ -90,7 +93,8 @@ never builds it. The integration is in Vortex: | Piece | Where | Role | |---|---|---| | `foldsim-timing-map`, `foldsim-estimate-timing` | foldsim `lib/Transforms/EstimateTiming.cpp` | map to LUTs; count levels | -| `utils/timing/*` | foldsim | screen any netlist; compare with Vivado; Vivado probes | +| `ci/timing_screen.py` | `ci/` | screen any netlist; read it by unit; compare with Vivado | +| `xilinx_cone_dump.tcl`, `xilinx_cone_probe.tcl`, `xilinx_wire_probe.tcl` | `hw/scripts` | Vivado probes for what levels cannot show (§8) | | target `screen` | `hw/syn/xilinx/dut/common.mk` | one DUT: sources → netlist → screen → summary | | `circt_elaborate.sh -f mlir` | `hw/scripts` | the lowered HW/Comb/Seq netlist, no `firtool` | | `gen_platform.py --emit=family` | `hw/scripts` | the family of `DEVICE`, from `hw/syn/xilinx/platforms` | @@ -267,8 +271,8 @@ by a human (`--update-baseline`), as for the other gates. Screen first, then Vivado: ```bash -screen.sh gen/design.mlir runs/full -summarize.py runs/full --min-levels 12 # the fix list, by unit +ci/timing_screen.py screen gen/design.mlir runs/full +ci/timing_screen.py summarize runs/full --min-levels 12 # the fix list, by unit ``` The screen ranks the units; one Vivado run confirms the fix. Compare a fix @@ -279,7 +283,7 @@ folded design (§9). Against Vivado, per RTL register or memory: the screen's deepest bit against Vivado's deepest failing path and worst slack, on seven V80 datasets -with no new Vivado runs (foldsim `utils/timing/validate.py`): +with no new Vivado runs (`ci/timing_screen.py validate`): | Dataset | Vivado run | Spearman vs levels | Spearman vs slack | ≥ 12: precision / recall | |---|---|---|---|---| @@ -298,15 +302,15 @@ disagreement is the same on both devices. ## 8. Diagnostics beyond levels -foldsim's `utils/timing` has three Vivado Tcl probes for questions the +`hw/scripts` has three Vivado Tcl probes for questions the screen cannot answer, run on a checkpoint (`vivado -mode batch -source -tclargs ...`): | Probe | Answers | |---|---| -| `cone_dump.tcl` | the worst path into a register, cell by cell, following retimed start registers back | -| `cone_probe.tcl` | whether paths exist between two register sets, and how deep Vivado built them | -| `wire_probe.tcl` | per path: length in tiles, bounding box and fanout beside logic and net delay | +| `xilinx_cone_dump.tcl` | the worst path into a register, cell by cell, following retimed start registers back | +| `xilinx_cone_probe.tcl` | whether paths exist between two register sets, and how deep Vivado built them | +| `xilinx_wire_probe.tcl` | per path: length in tiles, bounding box and fanout beside logic and net delay | ## 9. Invariants and limitations diff --git a/hw/scripts/xilinx_cone_dump.tcl b/hw/scripts/xilinx_cone_dump.tcl new file mode 100644 index 0000000000..7fc60dce0b --- /dev/null +++ b/hw/scripts/xilinx_cone_dump.tcl @@ -0,0 +1,59 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# xilinx_cone_dump.tcl ...: for each pattern, the worst +# setup path into the matching cells, cell by cell, whatever its slack. When the +# path starts at a retimed register (_fret/_bret), the worst path into that +# register follows, up to four hops, and the levels are summed: the depth of +# the RTL cone as Vivado built it. +set dcp [lindex $argv 0] +set out [lindex $argv 1] +open_checkpoint $dcp +set f [open $out w] + +proc dump_path {f p indent} { + puts $f "${indent}slack [get_property SLACK $p] levels [get_property LOGIC_LEVELS $p] logic [get_property DATAPATH_LOGIC_DELAY $p] net [get_property DATAPATH_NET_DELAY $p]" + puts $f "${indent} from [get_property STARTPOINT_PIN $p]" + puts $f "${indent} to [get_property ENDPOINT_PIN $p]" + foreach c [get_cells -of_objects $p] { + puts $f "${indent} [get_property REF_NAME $c] [get_property NAME $c]" + } +} + +foreach pat [lrange $argv 2 end] { + puts $f "=== $pat" + set cells [get_cells -quiet -hier -filter "IS_SEQUENTIAL && NAME =~ $pat"] + if {[llength $cells] == 0} { + set cells [get_cells -quiet -hier -filter "PRIMITIVE_LEVEL != MACRO && NAME =~ $pat"] + } + puts $f "cells [llength $cells]" + if {[llength $cells] == 0} continue + set pins [get_pins -quiet -of_objects $cells -filter "DIRECTION == IN && IS_CLOCK == 0"] + set p [get_timing_paths -quiet -setup -max_paths 1 -nworst 1 -to $pins] + if {[llength $p] == 0} { puts $f "no timed path"; continue } + set total 0 + set indent "" + for {set hop 0} {$hop < 5} {incr hop} { + dump_path $f $p $indent + set total [expr {$total + [get_property LOGIC_LEVELS $p]}] + set start [get_cells -quiet -of_objects [get_property STARTPOINT_PIN $p]] + if {$start eq "" || ![regexp {_[fb]ret} [get_property NAME $start]]} break + set spins [get_pins -quiet -of_objects $start -filter "DIRECTION == IN && IS_CLOCK == 0"] + set p [get_timing_paths -quiet -setup -max_paths 1 -nworst 1 -to $spins] + if {[llength $p] == 0} break + set indent "$indent " + puts $f "${indent}<- into retimed start [get_property NAME $start]" + } + puts $f "summed levels $total" +} +close $f diff --git a/hw/scripts/xilinx_cone_probe.tcl b/hw/scripts/xilinx_cone_probe.tcl new file mode 100644 index 0000000000..1a9e6c7c15 --- /dev/null +++ b/hw/scripts/xilinx_cone_probe.tcl @@ -0,0 +1,64 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# xilinx_cone_probe.tcl ...: for each from|to pair of cell +# patterns ("-" for any), up to 500 setup paths between them; prints the +# level histogram, the endpoints of the deepest and the deepest path cell by +# cell. Answers whether a cone the screen reports exists in Vivado's netlist, +# and how deep Vivado built it. +set dcp [lindex $argv 0] +set out [lindex $argv 1] +open_checkpoint $dcp +set f [open $out w] + +proc cells_of {pat} { + return [get_cells -quiet -hier -filter "IS_SEQUENTIAL && NAME =~ $pat"] +} + +foreach {from to} [lrange $argv 2 end] { + puts $f "=== from $from to $to" + set args [list -quiet -setup -max_paths 500 -nworst 1] + if {$from ne "-"} { + set fc [cells_of $from] + puts $f "from cells [llength $fc]" + if {[llength $fc] == 0} continue + lappend args -from $fc + } + if {$to ne "-"} { + set tc [cells_of $to] + puts $f "to cells [llength $tc]" + if {[llength $tc] == 0} continue + lappend args -to [get_pins -quiet -of_objects $tc -filter "DIRECTION == IN && IS_CLOCK == 0"] + } + set paths [get_timing_paths {*}$args] + puts $f "paths [llength $paths]" + if {[llength $paths] == 0} continue + array unset hist + set deepest "" + set dl -1 + foreach p $paths { + set l [get_property LOGIC_LEVELS $p] + incr hist($l) + if {$l > $dl} { set dl $l; set deepest $p } + } + set h "" + foreach l [lsort -integer [array names hist]] { append h " $l:$hist($l)" } + puts $f "levels$h" + puts $f "deepest slack [get_property SLACK $deepest] levels $dl" + puts $f " from [get_property STARTPOINT_PIN $deepest]" + puts $f " to [get_property ENDPOINT_PIN $deepest]" + foreach c [get_cells -of_objects $deepest] { + puts $f " [get_property REF_NAME $c] [get_property NAME $c]" + } +} +close $f diff --git a/hw/scripts/xilinx_wire_probe.tcl b/hw/scripts/xilinx_wire_probe.tcl new file mode 100644 index 0000000000..3fec054a0f --- /dev/null +++ b/hw/scripts/xilinx_wire_probe.tcl @@ -0,0 +1,62 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# xilinx_wire_probe.tcl [max paths]: for the worst setup path of up +# to N end points, the placement the wire delay comes from: the Manhattan +# length of the path in tiles (sum over consecutive cells), the bounding box +# of its cells, and the largest fanout of a net on it. Feature 5's upper +# bound: how much of the net delay a perfect placement estimate explains. +set dcp [lindex $argv 0] +set out [lindex $argv 1] +set n [expr {[llength $argv] > 2 ? [lindex $argv 2] : 5000}] +open_checkpoint $dcp +set f [open $out w] +puts $f "slack,levels,logic,net,length,bbox,fanout,start,end" + +proc tile_xy {c} { + set site [get_sites -quiet -of_objects $c] + if {$site eq ""} { return "" } + set t [get_tiles -quiet -of_objects $site] + if {$t eq ""} { return "" } + return [list [get_property COLUMN $t] [get_property ROW $t]] +} + +foreach p [get_timing_paths -setup -max_paths $n -nworst 1 -unique_pins] { + set len 0 + set prev "" + set xs {} + set ys {} + foreach c [get_cells -quiet -of_objects $p] { + set xy [tile_xy $c] + if {$xy eq ""} continue + lappend xs [lindex $xy 0] + lappend ys [lindex $xy 1] + if {$prev ne ""} { + set len [expr {$len + abs([lindex $xy 0] - [lindex $prev 0]) + abs([lindex $xy 1] - [lindex $prev 1])}] + } + set prev $xy + } + set bbox 0 + if {[llength $xs]} { + set xs [lsort -integer $xs] + set ys [lsort -integer $ys] + set bbox [expr {[lindex $xs end] - [lindex $xs 0] + [lindex $ys end] - [lindex $ys 0]}] + } + set fo 0 + foreach net [get_nets -quiet -of_objects $p] { + set k [get_property FLAT_PIN_COUNT $net] + if {$k > $fo} { set fo $k } + } + puts $f "[get_property SLACK $p],[get_property LOGIC_LEVELS $p],[get_property DATAPATH_LOGIC_DELAY $p],[get_property DATAPATH_NET_DELAY $p],$len,$bbox,[expr {$fo - 1}],[get_property STARTPOINT_PIN $p],[get_property ENDPOINT_PIN $p]" +} +close $f From 1e4ee9c46be84b24719da368b2513982dca9d684 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 15:39:22 -0700 Subject: [PATCH 251/336] hw/syn/yosys: the timing screen for ASAP7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `make -C hw/syn/yosys/dut TARGET=screen` screens a DUT of the ASIC flow with foldsim's ASAP7 profile (family=$(PDK)): the same sources, defines and SRAM black boxes as its synthesis, elaborated by circt_elaborate.sh -f mlir, mapped to 3-input cells by foldsim-timing-map-asic. report.txt has the deepest paths and the period they come to; a DUT takes 2-52 s where Yosys and OpenSTA take 2.5-37 min. - liberty_stage.py reads the cell, clock-to-output and setup delays the profile is built with from a Liberty library (ASAP7 RVT TT: 24, 58 and 6 ps). - sta_endpoints.tcl writes OpenSTA's slack at every flip-flop, the reference the profile was validated against per register. timing_screen.md §4.3, §6.1 and §7 document the model, the target and the validation (Spearman 0.56-0.93 per register on five DUTs, worst path within a mean 8 %). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a9e39533a70dc2e31cf08fcd5853243c48319b48) --- docs/assets/img/timing_screen_compiler.svg | 5 +- docs/designs/timing_screen.md | 69 ++++++++- hw/syn/yosys/Makefile | 16 +- hw/syn/yosys/liberty_stage.py | 171 +++++++++++++++++++++ hw/syn/yosys/sta_endpoints.tcl | 36 +++++ 5 files changed, 290 insertions(+), 7 deletions(-) create mode 100755 hw/syn/yosys/liberty_stage.py create mode 100644 hw/syn/yosys/sta_endpoints.tcl diff --git a/docs/assets/img/timing_screen_compiler.svg b/docs/assets/img/timing_screen_compiler.svg index 89e7e609f7..9d57cba9d7 100644 --- a/docs/assets/img/timing_screen_compiler.svg +++ b/docs/assets/img/timing_screen_compiler.svg @@ -117,10 +117,11 @@ histogram; top-paths and trace= op by op endpoints.csv, instances.csv, report.txt - - Device profiles (family=): only the DSPs differ + + Device profiles (family=): the FPGAs differ only in their DSPs versal DSP58 27×24 in 1 out 6 cascade 3 virtexuplusHBM DSP48E2 27×18 in 1 out 5 cascade 2 + asap7 cell stages (3-input cuts), prefix adders, Dadda products diff --git a/docs/designs/timing_screen.md b/docs/designs/timing_screen.md index 4f40daabce..d0fbd44451 100644 --- a/docs/designs/timing_screen.md +++ b/docs/designs/timing_screen.md @@ -74,13 +74,13 @@ foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( | `foldsim-estimate-timing` option | meaning | |---|---| | `top=` | the top module (default: the one nothing instantiates) | -| `family=` | `versal` (V80, default) or `virtexuplusHBM` (U55C) | +| `family=` | `versal` (V80, default), `virtexuplusHBM` (U55C), or `asap7` (ASIC, §4.3) | | `black-boxes=` | modules to treat as external, as a harness stubs them | | `output=`, `instances=`, `report=` | the files of §5 | | `min-levels=` | write only end points at or above this many levels | | `top-paths=` | deepest paths to print op by op (default 50) | | `trace=` | also print the deepest path of each end point whose name contains this | -| `retime-levels=` | also estimate the depth after Vivado's retiming to this period (§4.3) | +| `retime-levels=` | also estimate the depth after Vivado's retiming to this period (§4.4) | ## 2. System architecture @@ -197,7 +197,39 @@ register back, its clock enable or feedback; any other load pays the output stage. The fabric-to-product value is fitted on the V80 data. Any other family is an error. -### 4.3 Retiming (optional) +### 4.3 ASAP7 + +For the ASIC flow (`hw/syn/yosys`, ASAP7 7.5-track RVT, TT corner) the +screen counts **cell stages**, not LUT levels. `foldsim-timing-map-asic` +maps the same balanced and-inverter graph to 3-input cuts, each one cell +(NAND, AOI, MAJ, XOR), and `family=asap7` models the word-level operators +as an ASIC builds them: + +| Element | Stages | +|---|---| +| cell (`comb.truth_table`, 3 inputs) or unmapped and-inverter gate | 1 | +| adder, input bit j to output bit i | 2 + ⌈log₂(i − j + 1)⌉: a prefix adder | +| order compare | 1 + ⌈log₂ span⌉ | +| multiply | partial products, a Dadda tree of full adders (2 stages each), a prefix adder | +| table of k inputs (small products and dividers) | k (a multiplexer tree) | +| memory of flops (no macro) | 1 + ⌈log₂ depth⌉: a decoder and a multiplexer tree | + +SRAM macros (`VX_dp_ram_asic`, `VX_sp_ram_asic`) are black boxes, as in the +flow's synthesis: their pins end and start paths. The report converts the +deepest path to a period: + +``` +deepest path 45 stages: 58 + 167 + 45 x 24 + 6 = 1311 ps (763 MHz) + clk→Q path stages × cell setup +``` + +The cell (24 ps at fanout 3 and its own slew), clock-to-output (58 ps) and +setup (6 ps) delays come from the Liberty, by `hw/syn/yosys/liberty_stage.py`; +the path term (167 ps: the inverter after a DFFHQN's QN, buffering, slew) is +fitted on OpenSTA (§7). Re-derive them for another corner or threshold +voltage. + +### 4.4 Retiming (optional) All of Vortex's Vivado runs retime. With `retime-levels=N`, the `retimed` column estimates each end point's depth after one hop of balancing to a @@ -246,6 +278,14 @@ its kind, the fanout of its bit, and its instance path and name: `family=$(gen_platform.py --device=$(DEVICE) --emit=family)`; 4. `screen_summary.py`. +The ASIC flow has the same target, with the same sources, defines and SRAM +black boxes as its synthesis, and `family=$(PDK)`: + +```bash +make -C hw/syn/yosys/dut om TARGET=screen +# om_VX_om_core_top/screen/report.txt: the deepest paths and the period +``` + ### 6.2 `timing_gate` The third synthesis gate, beside `fpga_gate` (Vivado, nightly) and @@ -300,6 +340,25 @@ profile was checked against `fpga_gate`'s critical paths: it agrees with the V80 profile wherever no DSP is on the path, and the remaining disagreement is the same on both devices. +**ASAP7**, against OpenSTA on fresh syntheses of five DUTs (Yosys + ABC as +`hw/syn/yosys` builds them, flip-flops named after their registers, +`hw/syn/yosys/sta_endpoints.tcl` for the slack of each): + +| DUT | Registers | Spearman, stages vs arrival | Worst path: OpenSTA → screen | +|---|---|---|---| +| om | 93 | 0.91 | 827 → 879 ps | +| cache | 526 | 0.82 | 1450 → 1311 ps | +| dxa | 169 | 0.56 | 1018 → 1215 ps | +| raster | 134 | 0.58 | 879 → 879 ps | +| tex | 1002 | 0.93 | 840 → 807 ps | + +Per register, the fitted delay per stage is 17–25 ps, the library's 24. +The worst path is within a mean 8 % on these five, from which the path term +was fitted; on the six larger DUTs, the predicted Fmax is within a mean +13 % (−32 % to +3 %) of the asic_gate baselines, some of which predate the +current RTL. A DUT screens in 2–52 s, where Yosys and OpenSTA take +2.5–37 min. + ## 8. Diagnostics beyond levels `hw/scripts` has three Vivado Tcl probes for questions the @@ -329,7 +388,9 @@ screen cannot answer, run on a checkpoint native gates are the reference for native RTL. - **Black boxes.** Vendor IPs and `black-boxes=` modules end and start paths at their pins: their insides are not timed. -- **Devices.** The V80 and the U55C only. +- **Devices.** The V80 and the U55C (Vivado), and ASAP7 RVT TT (the Yosys + flow). The ASAP7 numbers are the screen's period estimate, a sizing + guide, not a sign-off: they come within about 10–30 % of OpenSTA. - **Toolchain.** The catalog maps only with three CIRCT Synth fixes (zero-width mux, constant SOP cut, a failing longest-path query), which reach CI through the foldsim prebuilt. diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index 231370f27b..bbc87445c3 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -189,7 +189,7 @@ CFLAGS += $(RTL_INCLUDE) # Build targets all: timing -.PHONY: all synthesis techmap timing gate-sim gate-saif normalize-saif asap7 asap7-models clean +.PHONY: all synthesis techmap timing screen gate-sim gate-saif normalize-saif asap7 asap7-models clean asap7: $(ASAP7_INSTALL) --install-dir=$(ASAP7_DIR) --vt=$(ASAP7_VT) @@ -285,6 +285,20 @@ timing: $(SYN_SRC_DEPS) $(SYN_LIB_DEPS) | $(BUILD_DIR) @$(SRC_DIR)/synth_summary.py --reports $(BUILD_DIR)/reports \ --top $(TOP_LEVEL_ENTITY) --clock-mhz $(CLOCK_FREQ) +# The timing screen (foldsim-estimate-timing, family=$(PDK)): the gate stages +# of every register-to-register path, with the same sources and SRAM black +# boxes as synthesis, in seconds and without Yosys; report.txt converts the +# deepest path to a period with the library's delays. +# docs/designs/timing_screen.md. +screen: $(BUILD_DIR)/src + mkdir -p $(BUILD_DIR)/screen + $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -b'$(BB_MODULES)' \ + -o$(BUILD_DIR)/screen/netlist.mlir + $(FOLDSIM_OPT) $(BUILD_DIR)/screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-timing-map-asic)' \ + --emit-bytecode -o $(BUILD_DIR)/screen/mapped.mlirbc + $(FOLDSIM_OPT) $(BUILD_DIR)/screen/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$(PDK) output=$(BUILD_DIR)/screen/endpoints.csv instances=$(BUILD_DIR)/screen/instances.csv report=$(BUILD_DIR)/screen/report.txt})' + @grep "^deepest path" $(BUILD_DIR)/screen/report.txt + gate-sim: techmap asap7-models @if [ "$(PDK)" != "asap7" ]; then echo "ERROR: gate-sim requires PDK=asap7 functional models" >&2; exit 1; fi @if [ -z "$(TB_FILELIST)" ]; then echo "ERROR: TB_FILELIST is required for gate-sim" >&2; exit 1; fi diff --git a/hw/syn/yosys/liberty_stage.py b/hw/syn/yosys/liberty_stage.py new file mode 100755 index 0000000000..5eb2635bca --- /dev/null +++ b/hw/syn/yosys/liberty_stage.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The delays the timing screen's ASIC profile (foldsim-estimate-timing +family=asap7) converts stages with, read from a Liberty library. + +usage: liberty_stage.py [--fanout N] [--suffix _ASAP7_75t_R] + +For a representative set of gates, the delay of one stage driving `fanout` +copies of a NAND2's input, at the slew such a stage produces (iterated to a +fixed point), and the flip-flop's clock-to-output delay and setup time at +that slew. Prints one line per cell and the three numbers the asap7 profile +is built with; re-derive them when the library or its corner changes. +""" +import argparse +import bisect +import re +import sys + +GATES = ['INVx1', 'NAND2xp5', 'NOR2xp33', 'AOI22xp5', 'OAI21xp5', 'XOR2xp5', + 'MAJIxp5', 'AO21x1'] +NAND = 'NAND2xp5' +FLOP = 'DFFHQNx1' + + +def cell_block(text, name): + """The text of `cell (name) { ... }`, by brace matching.""" + m = re.search(r'\bcell\s*\(\s*"?%s"?\s*\)\s*\{' % re.escape(name), text) + if not m: + sys.exit(f'no cell {name}') + depth, i = 1, m.end() + while depth: + c = text[i] + depth += c == '{' + depth -= c == '}' + i += 1 + return text[m.end():i - 1] + + +def groups(text, kind): + """(argument, body) of each `kind (arg) { ... }` at any depth.""" + for m in re.finditer(r'\b%s\s*\(\s*"?([^")]*)"?\s*\)\s*\{' % kind, text): + depth, i = 1, m.end() + while depth: + c = text[i] + depth += c == '{' + depth -= c == '}' + i += 1 + yield m.group(1), text[m.end():i - 1] + + +def numbers(s): + return [float(x) for x in re.findall(r'-?[0-9.]+(?:e-?[0-9]+)?', s)] + + +def table(body, kind): + """(index_1, index_2, rows) of the first `kind` table in `body`.""" + for _, t in groups(body, kind): + i1 = numbers(re.search(r'index_1\s*\(\s*"([^"]*)"', t).group(1)) + m2 = re.search(r'index_2\s*\(\s*"([^"]*)"', t) + i2 = numbers(m2.group(1)) if m2 else [0.0] + vals = numbers(re.search(r'values\s*\((.*?)\)\s*;', t, re.S).group(1)) + return i1, i2, [vals[k * len(i2):(k + 1) * len(i2)] + for k in range(len(i1))] + return None + + +def interp(tab, x, y): + i1, i2, v = tab + + def bracket(idx, q): + k = min(max(bisect.bisect_left(idx, q), 1), len(idx) - 1) + lo, hi = idx[k - 1], idx[k] + return k - 1, k, (q - lo) / (hi - lo) if hi != lo else 0.0 + + if len(i2) == 1: + a, b, t = bracket(i1, x) + return v[a][0] + t * (v[b][0] - v[a][0]) + a, b, t = bracket(i1, x) + c, d, u = bracket(i2, y) + top = v[a][c] + u * (v[a][d] - v[a][c]) + bot = v[b][c] + u * (v[b][d] - v[b][c]) + return top + t * (bot - top) + + +def pin_cap(cell, pin): + for name, body in groups(cell, 'pin'): + if name == pin: + return float(re.search(r'\bcapacitance\s*:\s*([0-9.]+)', body).group(1)) + sys.exit(f'no pin {pin}') + + +def arcs(cell, out, related=None): + """Timing groups of output pin `out`, optionally for one related pin.""" + for name, body in groups(cell, 'pin'): + if name != out: + continue + for _, t in groups(body, 'timing'): + rel = re.search(r'related_pin\s*:\s*"?(\w+)"?', t).group(1) + if related is None or rel == related: + yield rel, t + + +def gate_delay(cell, out, slew, load): + """Mean of rise and fall delay and transition over every input arc.""" + d, s, n = 0.0, 0.0, 0 + for _, t in arcs(cell, out): + if 'timing_type' in t and 'combinational' not in t: + continue + for kind, tkind in (('cell_rise', 'rise_transition'), + ('cell_fall', 'fall_transition')): + dt, st = table(t, kind), table(t, tkind) + if dt and st: + d += interp(dt, slew, load) + s += interp(st, slew, load) + n += 1 + return d / n, s / n + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('lib') + ap.add_argument('--fanout', type=float, default=3) + ap.add_argument('--suffix', default='_ASAP7_75t_R') + args = ap.parse_args() + text = open(args.lib).read() + nand = cell_block(text, NAND + args.suffix) + load = args.fanout * pin_cap(nand, 'A') + + slew = 20.0 + for _ in range(20): + delays = {} + out_slews = [] + for g in GATES: + cell = cell_block(text, g + args.suffix) + d, s = gate_delay(cell, 'Y', slew, load) + delays[g] = d + out_slews.append(s) + new = sum(out_slews) / len(out_slews) + if abs(new - slew) < 0.01: + break + slew = new + + flop = cell_block(text, FLOP + args.suffix) + clk2q = sum(interp(table(t, k), slew, load) + for _, t in arcs(flop, 'QN', 'CLK') + for k in ('cell_rise', 'cell_fall')) / 2 + setup = 0.0 + for _, t in arcs(flop, 'D', 'CLK'): + if 'setup_rising' in t: + setup = (interp(table(t, 'rise_constraint'), slew, slew) + + interp(table(t, 'fall_constraint'), slew, slew)) / 2 + for g in GATES: + print(f'{g:10s} {delays[g]:6.1f} ps') + stage = sum(delays.values()) / len(delays) + print(f'load {load:.2f} fF (fanout {args.fanout:g}), slew {slew:.1f} ps') + print(f'stage {stage:.1f} ps, clk->q {clk2q:.1f} ps, setup {setup:.1f} ps') + + +if __name__ == '__main__': + main() diff --git a/hw/syn/yosys/sta_endpoints.tcl b/hw/syn/yosys/sta_endpoints.tcl new file mode 100644 index 0000000000..463dab0d58 --- /dev/null +++ b/hw/syn/yosys/sta_endpoints.tcl @@ -0,0 +1,36 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# sta_endpoints.tcl: OpenSTA, the worst setup slack at the data input of +# every flip-flop, for comparing the timing screen per register with an +# ASIC flow (a CSV in the end point table format of ci/timing_screen.py). +# env: LIB NETLIST TOP SDC OUT +proc env {name} { return $::env($name) } +read_liberty [env LIB] +read_verilog [env NETLIST] +link_design [env TOP] +# The flow's SDC is in ns, whatever the library's time unit. +set_cmd_units -time ns -capacitance fF +read_sdc [env SDC] +set f [open [env OUT] w] +puts $f "slack,levels,logic,net,start,end" +# Flip-flops by library cell name (ASAP7: DFF*, SDF*); other cells have D +# pins too (OR4). +foreach pin [get_pins -hierarchical */D] { + set cell [get_cells -of_objects $pin] + if {![string match {[DS][DF]F*} [get_property $cell ref_name]]} continue + set slack [get_property $pin slack_max] + if {$slack eq "" || $slack eq "INF"} continue + puts $f "$slack,0,0,0,-,[get_full_name $cell]/D" +} +close $f From be3d77cce85c5f275b5111a209e072981828e2e3 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 15:45:40 -0700 Subject: [PATCH 252/336] cache_tags: keep the tag compare off the side stores' array enables The valid, dirty and partial LUTRAMs took their array write enable as the OR of their per-bit enables. Those carry the tag compare (an AMO's invalidate, a write hit, a sector refill), so the compare fanned into every LUTRAM write-enable pin of the bank. The array enable is now the set of operations that may write each store; a bit still changes only where its own enable is set, which the RAM already applies. The bank also passed the non-LLC AMO's invalidate qualified by the full hit, which ORs every way's match (plus the order-hazard and partial-miss terms) and routes it back into the array. The array ANDs each way's own match into its clear, so the bank now passes only the hit's other qualifiers; a forwarded AMO never has an order hazard, so that term drops. Both changes are cycle-exact. D-cache harness, place-only at 3.333 ns: the valid-store family -0.371 ns -> -0.090 ns; the whole harness -0.175 ns / 7434 failing endpoints -> -0.090 ns / 1233 (with the flush hold). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e7666ad59a3794f021147d04b5770e30ccbbe63f) --- hw/rtl/cache/VX_cache_bank.sv | 7 +++++-- hw/rtl/cache/VX_cache_tags.sv | 20 ++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index a823225984..0ed5359add 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -678,8 +678,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .read (do_read_st0 && ~pipe_stall), .write (do_write_st0 && ~pipe_stall), // non-LLC AMO forwards downstream and invalidates its own copy so the - // issuer's later plain load refetches the new value. - .invalidate (is_amo_fwd_st0 && lk_st0.is_hit && ~pipe_stall), + // issuer's later plain load refetches the new value. Only the hit's + // qualifiers are passed (a forwarded AMO has no order hazard): the + // array ANDs in each way's own tag match, which keeps the OR over the + // ways off the valid store's write enables. + .invalidate (is_amo_fwd_st0 && ~partial_miss_st0 && ~pipe_stall), .fill_nodata (lazy_st0), .line_idx (line_idx_st0), .line_idx_n (sel_req.req.addr[`CS_LINE_SEL_BITS-1:0]), diff --git a/hw/rtl/cache/VX_cache_tags.sv b/hw/rtl/cache/VX_cache_tags.sv index 62433812f8..e70eb402ef 100644 --- a/hw/rtl/cache/VX_cache_tags.sv +++ b/hw/rtl/cache/VX_cache_tags.sv @@ -48,7 +48,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( input wire fill, input wire read, input wire write, - input wire invalidate, // clear valid on the hit way + input wire invalidate, // clear valid on the hit way (qualifies the way's own match) // the fill installs no data: the sector becomes partial (cleared otherwise) input wire fill_nodata, // a refetch completes its resident partial sector: clear its partial bit @@ -296,6 +296,14 @@ module VX_cache_tags import VX_gpu_pkg::*; #( .rdata (tag_rdata) ); + // The side stores' array enables are the operations that may write them, + // never the OR of their per-bit enables: those carry the tag compare, which + // would otherwise fan into every LUTRAM write enable. A bit still changes + // only where its own enable is set. + wire valid_write = init || flush || fill || ((AMO_ENABLE != 0) && invalidate); + wire dirty_write = init || flush || fill || write || ((AMO_ENABLE != 0) && invalidate); + wire partial_write = fill || refetch; + // Decoupled per-sector valid store: mirrors the tag BRAM's access pattern // (look-ahead read, read-first) so pipeline alignment is identical, with // per-bit write-enable so a sector refill sets its bit without reading back. @@ -310,7 +318,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .read (~stall), - .write (| valid_wren), + .write (valid_write), .wren (valid_wren), .waddr (line_idx), .raddr (line_idx_n), @@ -333,7 +341,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .read (~stall), - .write (| dirty_wren), + .write (dirty_write), .wren (dirty_wren), .waddr (line_idx), .raddr (line_idx_n), @@ -342,7 +350,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( ); end else begin : g_no_dirty_store assign dirty_rdata = '0; - `UNUSED_VAR ({dirty_wren, dirty_wdata, dirty_rdata}) + `UNUSED_VAR ({dirty_wren, dirty_wdata, dirty_rdata, dirty_write}) end if (LAZY_FETCH != 0) begin : g_partial_store @@ -357,7 +365,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .read (~stall), - .write (| partial_wren), + .write (partial_write), .wren (partial_wren), .waddr (line_idx), .raddr (line_idx_n), @@ -368,7 +376,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( end else begin : g_no_partial_store assign partial_rdata = '0; assign partial_match = 1'b0; - `UNUSED_VAR ({partial_wren, partial_wdata, partial_rdata, partial_hit, read_partial, fill_nodata, refetch}) + `UNUSED_VAR ({partial_wren, partial_wdata, partial_rdata, partial_hit, read_partial, fill_nodata, refetch, partial_write}) end endmodule From 70de632203ec0f5437f776657a1e248068419c56 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 16:00:09 -0700 Subject: [PATCH 253/336] ci/timing_screen.py: map on one thread with --no-threads MLIR maps a design's modules in parallel, and the full GPU's largest modules together peak at 18 GB (53 s); on one thread the mapping peaks at 9 GB (107 s), the largest module's bit-level netlist. --no-threads passes --mlir-disable-threading to the mapping for hosts where memory, not time, is short. A DUT of the CI catalog needs neither. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b15df7927ccc8c9be504a2b849823984323fad63) --- ci/timing_screen.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/timing_screen.py b/ci/timing_screen.py index aaa45562fb..7e5dfbe1bb 100755 --- a/ci/timing_screen.py +++ b/ci/timing_screen.py @@ -172,11 +172,15 @@ def cmd_screen(args): asic = any(o == "family=asap7" for o in args.options) mapped = args.design steps = [] + # MLIR maps modules in parallel: the full GPU maps in 53 s and 18 GB, or + # 107 s and 9 GB on one thread. + threads = ["--mlir-disable-threading"] if args.no_threads else [] if not args.design.endswith(".mlirbc"): mapped = os.path.join(args.out, "mapped.mlirbc") pipeline = "foldsim-timing-map-asic" if asic else "foldsim-timing-map" steps.append(("map", [opt, args.design, "--emit-bytecode", "-o", mapped, - "--pass-pipeline=builtin.module(%s)" % pipeline])) + "--pass-pipeline=builtin.module(%s)" % pipeline] + + threads)) opts = " ".join(["output=%s/endpoints.csv" % args.out, "instances=%s/instances.csv" % args.out, "report=%s/report.txt" % args.out] + args.options) @@ -459,6 +463,8 @@ def main(): p.add_argument("options", nargs="*", help="foldsim-estimate-timing options (top=, family=, black-boxes=, " "trace=, retime-levels=, ...); family=asap7 maps for the ASIC library") + p.add_argument("--no-threads", action="store_true", + help="map on one thread: half the peak memory, twice the time") p.set_defaults(fn=cmd_screen) p = sub.add_parser("summarize", help="deep end points by RTL unit") From 470f68947defa71ff2157a07c69ba299b4be00bf Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 16:03:37 -0700 Subject: [PATCH 254/336] ci/timing_screen.py: port depths and the elastic-buffer advisor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `screen --keep MODULE` keeps MODULE's instances through the mapping's inlining (foldsim-keep-modules), and port-modules= writes ports.csv. - `ports` lists the depth at each port of the kept modules, deepest instance per unit. - `buffers` is the OUT_BUF advisor: the VX_elastic_buffers that pass a ready chain through combinationally at or above the watch threshold, where a skid buffer or OUT_BUF would cut it, and those whose output is computed, where OUT_REG would register it. On the CI catalog's core DUT it names three: two passthroughs carrying 9 levels and a flush-arbiter response buffer passing an 8-level ready chain. timing_screen.md §8 documents both, and the class-boundary use for foldsim. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 96144d767e058725f41997b3e04d9dae1661fe2c) --- ci/timing_screen.py | 103 +++++++++++++++++++++++++++++++++- docs/designs/timing_screen.md | 27 +++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) diff --git a/ci/timing_screen.py b/ci/timing_screen.py index 7e5dfbe1bb..9a6c77d9b9 100755 --- a/ci/timing_screen.py +++ b/ci/timing_screen.py @@ -21,12 +21,18 @@ Vivado's failing ones validate agreement per register with a Vivado end point table retime-split the same agreement, split by whether Vivado retimed the paths + ports the depth at the ports of chosen modules' instances, by unit + buffers which elastic buffers pass a deep ready chain or a computed + output through: evidence for an OUT_BUF (OUT_REG) change ci/timing_screen.py screen gen/design.mlir runs/full ci/timing_screen.py screen runs/full/mapped.mlirbc runs/socket \\ top=VX_socket__lift black-boxes=VX_core__lift ci/timing_screen.py summarize runs/full --min-levels 12 ci/timing_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut + ci/timing_screen.py screen --keep VX_elastic_buffer design.mlir runs/eb \\ + port-modules=VX_elastic_buffer + ci/timing_screen.py buffers runs/eb A Vivado end point table is a CSV of `slack,levels,logic,net,start,end`, one row per failing end point (a gate's ep.csv). The DUT flow and the timing gate @@ -178,12 +184,20 @@ def cmd_screen(args): if not args.design.endswith(".mlirbc"): mapped = os.path.join(args.out, "mapped.mlirbc") pipeline = "foldsim-timing-map-asic" if asic else "foldsim-timing-map" + if args.keep: + # Keep these modules' instances through inlining, so their ports + # can be reported (ports=, port-modules=). + pipeline = "foldsim-keep-modules{modules=%s},%s" % ( + ",".join(args.keep), pipeline) steps.append(("map", [opt, args.design, "--emit-bytecode", "-o", mapped, "--pass-pipeline=builtin.module(%s)" % pipeline] + threads)) - opts = " ".join(["output=%s/endpoints.csv" % args.out, - "instances=%s/instances.csv" % args.out, - "report=%s/report.txt" % args.out] + args.options) + files = ["output=%s/endpoints.csv" % args.out, + "instances=%s/instances.csv" % args.out, + "report=%s/report.txt" % args.out] + if any(o.startswith("port-modules=") for o in args.options): + files.append("ports=%s/ports.csv" % args.out) + opts = " ".join(files + args.options) steps.append(("screen", [opt, mapped, "-o", os.devnull, "--pass-pipeline=builtin.module(" "foldsim-estimate-timing{%s})" % opts])) @@ -413,6 +427,75 @@ def cmd_summarize(args): +def load_ports(screen): + """instance -> {port: (direction, levels)} from a screen's ports.csv.""" + ports = collections.defaultdict(dict) + with open(f'{screen}/ports.csv') as f: + for row in csv.DictReader(f): + ports[(row['instance'], row['module'])][row['port']] = ( + row['direction'], int(row['levels'])) + return ports + + +def cmd_ports(args): + """The deepest instance of each unit, port by port.""" + units = {} + for (inst, module), ports in load_ports(args.screen).items(): + deepest = max(levels for _, levels in ports.values()) + key = (INDEX.sub('_*', norm(inst)), module.rstrip('0123456789').rstrip('_')) + if key not in units or deepest > units[key][0]: + units[key] = (deepest, inst, ports) + for (unit, module), (deepest, inst, ports) in sorted( + units.items(), key=lambda kv: -kv[1][0])[:args.top]: + if deepest < args.min_levels: + break + print(f'{deepest:3d} {module} {unit}') + for port, (direction, levels) in ports.items(): + print(f' {direction:>3} {port:24s} {levels:3d}') + + +def cmd_buffers(args): + """VX_elastic_buffer instances whose ready chain or output is deep. + + ready_out is the input from downstream and ready_in the output upstream: + a buffer that passes ready combinationally drives ready_in at least as + deep as ready_out arrives, and a skid buffer (SIZE 2) or OUT_BUF cuts the + chain. An output (data_out, valid_out) above one level is computed, not + registered: OUT_REG would register it. + """ + rows = [] + for (inst, module), ports in load_ports(args.screen).items(): + if not module.startswith('VX_elastic_buffer'): + continue + get = lambda p: ports.get(p, ('', -1))[1] + chain = get('ready_in') + through = chain >= 1 and chain >= get('ready_out') + out = max(get('data_out'), get('valid_out')) + advice = [] + if through and chain >= args.min_levels: + advice.append(f'ready chain of {chain} levels passes through') + if out >= args.min_levels: + advice.append(f'output computed at {out} levels') + if advice: + rows.append((max(chain if through else 0, out), inst, module, + get('valid_in'), get('ready_out'), chain, out, + '; '.join(advice))) + units = {} + for r in rows: + key = INDEX.sub('_*', norm(r[1])) + if key not in units or r[0] > units[key][0][0]: + units[key] = (r, 1 + (units[key][1] if key in units else 0)) + else: + units[key] = (units[key][0], units[key][1] + 1) + print(f'elastic buffers at or above {args.min_levels} levels: {len(rows)} ' + f'in {len(units)} units') + print(f'{"lv":>3} {"n":>4} {"valid_in":>8} {"ready_out":>9} ' + f'{"ready_in":>8} {"out":>4} unit / advice') + for key, (r, n) in sorted(units.items(), key=lambda kv: -kv[1][0][0]): + print(f'{r[0]:3d} {n:4d} {r[3]:8d} {r[4]:9d} {r[5]:8d} {r[6]:4d} ' + f'{key}\n{"":32}{r[7]}') + + RETIMED = re.compile(r'_[fb]ret') @@ -465,6 +548,9 @@ def main(): "trace=, retime-levels=, ...); family=asap7 maps for the ASIC library") p.add_argument("--no-threads", action="store_true", help="map on one thread: half the peak memory, twice the time") + p.add_argument("--keep", action="append", default=[], metavar="MODULE", + help="keep MODULE's instances through inlining, to report " + "their ports with port-modules=MODULE (repeatable)") p.set_defaults(fn=cmd_screen) p = sub.add_parser("summarize", help="deep end points by RTL unit") @@ -491,6 +577,17 @@ def main(): help="the screen's column to compare") p.set_defaults(fn=cmd_validate) + p = sub.add_parser("ports", help="port depths of chosen modules, by unit") + p.add_argument('screen') + p.add_argument('--min-levels', type=int, default=8) + p.add_argument('--top', type=int, default=40) + p.set_defaults(fn=cmd_ports) + + p = sub.add_parser("buffers", help="elastic buffers that pass deep paths") + p.add_argument('screen') + p.add_argument('--min-levels', type=int, default=8) + p.set_defaults(fn=cmd_buffers) + p = sub.add_parser("retime-split", help="agreement split by Vivado's retiming") p.add_argument('screen') p.add_argument('ep') diff --git a/docs/designs/timing_screen.md b/docs/designs/timing_screen.md index d0fbd44451..147e2038aa 100644 --- a/docs/designs/timing_screen.md +++ b/docs/designs/timing_screen.md @@ -81,6 +81,7 @@ foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( | `top-paths=` | deepest paths to print op by op (default 50) | | `trace=` | also print the deepest path of each end point whose name contains this | | `retime-levels=` | also estimate the depth after Vivado's retiming to this period (§4.4) | +| `ports=`, `port-modules=` | the depth at each port of the named modules' instances (§8) | ## 2. System architecture @@ -361,6 +362,32 @@ current RTL. A DUT screens in 2–52 s, where Yosys and OpenSTA take ## 8. Diagnostics beyond levels +**Port depths.** `port-modules=` names modules whose instances' ports the +screen reports: the levels at which each input arrives and the levels behind +each output. Small modules are inlined by the mapping, so keep them with +`screen --keep`: + +```bash +ci/timing_screen.py screen --keep VX_elastic_buffer design.mlir runs/eb \ + port-modules=VX_elastic_buffer +ci/timing_screen.py buffers runs/eb # the OUT_BUF advisor +ci/timing_screen.py ports runs/eb # every kept module, by unit +``` + +- The **buffer advisor** lists the `VX_elastic_buffer`s that pass a deep + ready chain through combinationally (`ready_in`, upstream, driven at + least as deep as `ready_out` arrives from downstream), where a skid + buffer (SIZE 2) or `OUT_BUF` would cut it, and those whose output is + computed rather than registered, where `OUT_REG` would register it. It + is evidence for an RTL review: the change also moves cycles, so it comes + with the matching SimX timing update. +- For foldsim, the ports of the class modules (`VX_socket__lift`, + `VX_core__lift`, ...) show the outputs computed from logic, and how deep, + rather than registered: in the fold they cost host cycles on the class's + links. + +**Vivado probes.** + `hw/scripts` has three Vivado Tcl probes for questions the screen cannot answer, run on a checkpoint (`vivado -mode batch -source -tclargs ...`): From 3288ba5196a597b01bfd18bcce6a8258cecfe680 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 18:08:09 -0700 Subject: [PATCH 255/336] cache_mshr: look up every dequeue candidate's address in parallel The registered dequeue address was read from the entry table at the next dequeue pointer, which the dequeue handshake chooses late in the cycle (fill, chained successor, or a late joiner spliced this cycle). The late pointer therefore indexed a full-width table mux. The address and write flag of each candidate are now read in parallel from early indices (current pointer, its successor, the finalizing entry, the fill) and the same priority selects among them. Cycle-exact; a simulation assert checks the selected address against the table at the chosen pointer. D-cache harness, place-only at 3.333 ns: -0.090 ns / 1233 failing endpoints (the dequeue-address family at -0.069 ns) -> 0 failing. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 08271d019b602a5652d8dead896bcf3951670e72) Integration (phase 2): phase 2 erased the MSHR line table, so the change is ported to the per-entry address table alone: the candidates are read from addr_table/write_table at the current pointer (or the fill), its chained successor and the finalizing entry, and the assert checks the selection against the table at dequeue_id_n. --- hw/rtl/cache/VX_cache_mshr.sv | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index 64ffe48898..bac7c029f4 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -571,9 +571,28 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // stalls and the replay never drains. reg [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_r; reg dequeue_rw_r; + + // The pointer's next entry, as the update above chooses it: a fill opens a + // chain, a dequeue walks to the chained successor or to a late joiner + // spliced this cycle. The dequeue handshake arrives late, so the address + // and the write flag of every candidate are looked up in parallel and the + // same choice selects among them, instead of the late pointer indexing the + // tables. + wire deq_to_next = ~dequeue_park_fire && dequeue_fire && next_table[dequeue_id]; + wire deq_to_fin = ~dequeue_park_fire && dequeue_fire && ~next_table[dequeue_id] + && finalize_valid && finalize_is_pending && (finalize_previd == dequeue_id); + wire [MSHR_ADDR_WIDTH-1:0] deq_next_id = next_index[dequeue_id]; + wire [MSHR_ADDR_WIDTH-1:0] deq_base_id = fill_valid ? fill_id : dequeue_id; + wire [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_n = deq_to_next ? addr_table[deq_next_id] + : (deq_to_fin ? addr_table[finalize_id] : addr_table[deq_base_id]); + wire dequeue_rw_n = deq_to_next ? write_table[deq_next_id] + : (deq_to_fin ? write_table[finalize_id] : write_table[deq_base_id]); + `RUNTIME_ASSERT((dequeue_addr_n == addr_table[dequeue_id_n]) && (dequeue_rw_n == write_table[dequeue_id_n]), + ("*** %s dequeue candidate select out of sync: id=%0d", INSTANCE_ID, dequeue_id_n)) + always @(posedge clk) begin - dequeue_addr_r <= addr_table[dequeue_id_n]; - dequeue_rw_r <= write_table[dequeue_id_n]; + dequeue_addr_r <= dequeue_addr_n; + dequeue_rw_r <= dequeue_rw_n; end assign dequeue_addr = dequeue_addr_r; From 506a9dc9d33ec2b5ea46403d17c6c1552949f4f2 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 17:08:06 -0700 Subject: [PATCH 256/336] ci: rename the timing screen and its gate to logic_screen and logic_gate The screen counts logic only (levels now, cells next), with no placement and no wires; that is what sets it apart from fpga_gate and asic_gate, and the gate is named by its flow as they are. ci/timing_gate.py, ci/timing_screen.py, ci/testcases/timing_gate.yaml, timing_gate.yml and docs/designs/timing_screen.md become logic_gate / logic_screen; the mapping pipeline is foldsim-screen-map. The screen target, the Screen backend and ci/baselines/synthesis/screen/ keep their names. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a36fe64d5d91dd915d35c7c8a30a69a347fae2d0) --- .../{timing_gate.yml => logic_gate.yml} | 44 +++++++------- AGENTS.md | 4 +- ci/{timing_gate.py => logic_gate.py} | 14 ++--- ci/{timing_screen.py => logic_screen.py} | 22 +++---- ci/synth_gate.py | 8 +-- ci/synth_report.py | 2 +- .../{timing_gate.yaml => logic_gate.yaml} | 18 +++--- ...compiler.svg => logic_screen_compiler.svg} | 4 +- ...een_system.svg => logic_screen_system.svg} | 14 ++--- docs/designs/continuous_integration.md | 16 ++--- docs/designs/foldsim.md | 4 +- .../{timing_screen.md => logic_screen.md} | 58 +++++++++---------- hw/scripts/circt_elaborate.sh | 2 +- hw/scripts/screen_summary.py | 7 ++- hw/syn/xilinx/dut/Makefile | 2 +- hw/syn/xilinx/dut/common.mk | 6 +- hw/syn/yosys/Makefile | 6 +- hw/syn/yosys/liberty_stage.py | 2 +- hw/syn/yosys/sta_endpoints.tcl | 4 +- 19 files changed, 119 insertions(+), 118 deletions(-) rename .github/workflows/{timing_gate.yml => logic_gate.yml} (70%) rename ci/{timing_gate.py => logic_gate.py} (73%) rename ci/{timing_screen.py => logic_screen.py} (96%) rename ci/testcases/{timing_gate.yaml => logic_gate.yaml} (91%) rename docs/assets/img/{timing_screen_compiler.svg => logic_screen_compiler.svg} (98%) rename docs/assets/img/{timing_screen_system.svg => logic_screen_system.svg} (92%) rename docs/designs/{timing_screen.md => logic_screen.md} (90%) diff --git a/.github/workflows/timing_gate.yml b/.github/workflows/logic_gate.yml similarity index 70% rename from .github/workflows/timing_gate.yml rename to .github/workflows/logic_gate.yml index 2bfefa934b..349b631f4c 100644 --- a/.github/workflows/timing_gate.yml +++ b/.github/workflows/logic_gate.yml @@ -2,8 +2,8 @@ # # Licensed under the Apache License, Version 2.0 (the "License"). # -# timing_gate — logic-level regression gate on hosted runners. -# Screens the DUT catalog (ci/testcases/timing_gate.yaml) with +# logic_gate — logic-level regression gate on hosted runners. +# Screens the DUT catalog (ci/testcases/logic_gate.yaml) with # foldsim-estimate-timing and asserts the depth of every RTL unit against the # golden results in ci/baselines/synthesis/screen/, so an RTL change that makes # a unit deeper is named on the change that does it, not the next night. @@ -26,8 +26,8 @@ on: - 'hw/scripts/**' - 'hw/syn/xilinx/dut/**' - 'ci/synth_gate.py' - - 'ci/timing_gate.py' - - 'ci/testcases/timing_gate.yaml' + - 'ci/logic_gate.py' + - 'ci/testcases/logic_gate.yaml' - 'ci/baselines/synthesis/screen/**' push: branches: [master] @@ -37,8 +37,8 @@ on: - 'hw/scripts/**' - 'hw/syn/xilinx/dut/**' - 'ci/synth_gate.py' - - 'ci/timing_gate.py' - - 'ci/testcases/timing_gate.yaml' + - 'ci/logic_gate.py' + - 'ci/testcases/logic_gate.yaml' - 'ci/baselines/synthesis/screen/**' workflow_dispatch: inputs: @@ -47,7 +47,7 @@ on: default: "" concurrency: - group: timing-gate-${{ github.ref }} + group: logic-gate-${{ github.ref }} cancel-in-progress: true env: @@ -78,7 +78,7 @@ jobs: ci/toolchain_install.sh --foldsim # The command is the one the catalog declares (the single `run:` case of - # ci/testcases/timing_gate.yaml), so what CI runs and what the catalog + # ci/testcases/logic_gate.yaml), so what CI runs and what the catalog # says cannot drift apart. - name: Gate id: run @@ -86,13 +86,13 @@ jobs: IN_BUILDS: ${{ github.event.inputs.builds }} run: | set -uo pipefail - CMD=$(python3 -c "import yaml; print(yaml.safe_load(open('ci/testcases/timing_gate.yaml'))['tests'][0]['run'])") + CMD=$(python3 -c "import yaml; print(yaml.safe_load(open('ci/testcases/logic_gate.yaml'))['tests'][0]['run'])") # Dispatch inputs reach the shell through an env var, never # interpolated into the script. ARGS=() for b in ${IN_BUILDS:-}; do ARGS+=(-b "$b"); done rc=0 - $CMD "${ARGS[@]}" --timeout 3600 --report timing_gate.json || rc=$? + $CMD "${ARGS[@]}" --timeout 3600 --report logic_gate.json || rc=$? echo "rc=$rc" >> "$GITHUB_OUTPUT" exit 0 @@ -100,26 +100,26 @@ jobs: if: always() run: | set -uo pipefail - python3 ci/synth_report.py --annotate error timing_gate.json \ - > timing_gate_report.md || true - cat timing_gate_report.md >> "$GITHUB_STEP_SUMMARY" + python3 ci/synth_report.py --annotate error logic_gate.json \ + > logic_gate_report.md || true + cat logic_gate_report.md >> "$GITHUB_STEP_SUMMARY" - uses: actions/upload-artifact@v4 if: always() with: - name: timing-gate + name: logic-gate path: | - timing_gate.json - timing_gate_report.md - build32_timing_gate/hw/syn/xilinx/dut/timing_gate_*/build.log - build32_timing_gate/hw/syn/xilinx/dut/timing_gate_*/screen_* - build32_timing_gate/hw/syn/xilinx/dut/timing_gate_*/screen/report.txt + logic_gate.json + logic_gate_report.md + build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/build.log + build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/screen_* + build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/screen/report.txt if-no-files-found: warn - name: Verdict run: | case "${{ steps.run.outputs.rc }}" in - 0) echo "timing_gate passed" ;; - 1) echo "::error::timing_gate FAILED — a unit got deeper than its baseline"; exit 1 ;; - *) echo "::error::timing_gate build error (see logs)"; exit 1 ;; + 0) echo "logic_gate passed" ;; + 1) echo "::error::logic_gate FAILED — a unit got deeper than its baseline"; exit 1 ;; + *) echo "::error::logic_gate build error (see logs)"; exit 1 ;; esac diff --git a/AGENTS.md b/AGENTS.md index fb147f7a11..3b8ed75022 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,8 +86,8 @@ See [docs/testing.md](docs/testing.md) and [docs/debugging.md](docs/debugging.md - **RTL coverage path is `xrt`, not `rtlsim`.** When discussing or planning RTL verification, `xrt` is the canonical path — `rtlsim` bypasses the AFU surface. `rtlsim` remains useful for fast iteration on processor RTL; `xrt` is what proves the full integration. - **`ci/regression.sh` is the canonical source of tested configurations.** Use it to discover supported parameter combinations before inventing ad hoc ones. - **Perf-regression baselines (`ci/baselines/perf/*.json`) are golden data — never hand-edit them, and never "fix" a red perf gate by bumping the number.** They are regenerated only by `pytest ci -m perf_gate --update-baselines` (a human-run, reviewed step), and CI must never pass that flag. A `perf_gate` failure means real cycles moved: root-cause it, or — if the change is intended — regenerate the baseline so the diff shows the perf delta for review. Same rule as image goldens and `known_issue:`. -- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,screen}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/timing_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `timing_gate` means a unit got deeper, and its reasons name the unit. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). -- **Screen the logic depth of an RTL change before committing it.** A change to a pipeline, arbiter, queue or datapath can add logic levels that only the nightly `fpga_gate` would find. Run `make -C hw/syn/xilinx/dut -screen`, or `ci/timing_gate.py -b `, for the DUTs it touches. A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. See [docs/designs/timing_screen.md](docs/designs/timing_screen.md). +- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,screen}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/logic_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `logic_gate` means a unit got deeper, and its reasons name the unit. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). +- **Screen the logic depth of an RTL change before committing it.** A change to a pipeline, arbiter, queue or datapath can add logic levels that only the nightly `fpga_gate` would find. Run `make -C hw/syn/xilinx/dut -screen`, or `ci/logic_gate.py -b `, for the DUTs it touches. A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. See [docs/designs/logic_screen.md](docs/designs/logic_screen.md). - **SimX is the RTL's timing model — keep them in lockstep.** Any change that moves RTL cycles (pipeline structure, arbitration, queue depths, cache/memory behavior) must land together with the matching SimX timing-model update, and vice-versa. The `model_parity` CI gate enforces this: a `check: model_parity` case (`ci/testcases/core.yaml` + per-extension parity categories) runs the same app/args/configs on simx and rtlsim and asserts exact retired-instruction match plus cycle agreement within the case tolerance (default 5%). Never widen a tolerance to absorb a divergence — model the behavior. When adding a hardware feature, add or extend a parity case that exercises it. - **When RTL debugging stalls, switch to the SimX-as-oracle pattern.** For numerical bugs, deep pipeline races, or any failure mode where rtlsim is "close but wrong": (1) build/extend the SimX C++ model so it mirrors the *new* RTL architecture and gets to PASS; (2) add matching trace dumps to both SimX and RTL (cycle, FU events, SRAM addresses+data, hazards) — same CSV format on both sides; (3) diff trace files — the first divergence is the bug. Don't keep guessing from output values; localize via trace diff. See [docs/debugging.md](docs/debugging.md#simx-as-oracle-for-rtl-debug). diff --git a/ci/timing_gate.py b/ci/logic_gate.py similarity index 73% rename from ci/timing_gate.py rename to ci/logic_gate.py index f6d6fa2321..d220e423a6 100755 --- a/ci/timing_gate.py +++ b/ci/logic_gate.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""timing_gate — logic-level regression gate (the timing screen). +"""logic_gate — logic-level regression gate (the logic screen). Gates the depth of every RTL unit of the DUT catalog: foldsim-estimate-timing counts the logic levels of every register-to-register path of the CIRCT netlist @@ -9,9 +9,9 @@ gating machinery is shared with the synthesis gates and lives in ci/synth_gate.py; this entry point only pins the tool. - Spec : ci/testcases/timing_gate.yaml (what to screen) + Spec : ci/testcases/logic_gate.yaml (what to screen) Baselines : ci/baselines/synthesis/screen/.json (last accepted levels) - Builds : hw/syn/xilinx/dut/timing_gate_ (no Vivado) + Builds : hw/syn/xilinx/dut/logic_gate_ (no Vivado) Metrics : /screen_summary.csv, screen_units.json (written by hw/syn/xilinx/dut's `screen` target) @@ -19,10 +19,10 @@ Xilinx environment is not used. Usage (from a source checkout): - ci/timing_gate.py --list - ci/timing_gate.py # all builds, gate against baselines - ci/timing_gate.py -b cache -b tcu # explicit builds - ci/timing_gate.py --update-baseline # re-record (human-reviewed, never in CI) + ci/logic_gate.py --list + ci/logic_gate.py # all builds, gate against baselines + ci/logic_gate.py -b cache -b tcu # explicit builds + ci/logic_gate.py --update-baseline # re-record (human-reviewed, never in CI) The same screen runs by hand through the flow's own entry point -- `make -C hw/syn/xilinx/dut cache-screen` -- so the gate adds no second way to diff --git a/ci/timing_screen.py b/ci/logic_screen.py similarity index 96% rename from ci/timing_screen.py rename to ci/logic_screen.py index 9a6c77d9b9..976f441de3 100755 --- a/ci/timing_screen.py +++ b/ci/logic_screen.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""timing_screen -- run foldsim's timing screen on a Vortex netlist, and read +"""logic_screen -- run foldsim's logic screen on a Vortex netlist, and read its results by RTL unit or against Vivado. screen map an elaborated CIRCT netlist and count the logic levels of - every end point (foldsim-timing-map, foldsim-estimate-timing) + every end point (foldsim-screen-map, foldsim-estimate-timing) summarize the deep end points grouped by RTL unit, optionally beside Vivado's failing ones validate agreement per register with a Vivado end point table @@ -25,19 +25,19 @@ buffers which elastic buffers pass a deep ready chain or a computed output through: evidence for an OUT_BUF (OUT_REG) change - ci/timing_screen.py screen gen/design.mlir runs/full - ci/timing_screen.py screen runs/full/mapped.mlirbc runs/socket \\ + ci/logic_screen.py screen gen/design.mlir runs/full + ci/logic_screen.py screen runs/full/mapped.mlirbc runs/socket \\ top=VX_socket__lift black-boxes=VX_core__lift - ci/timing_screen.py summarize runs/full --min-levels 12 - ci/timing_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut - ci/timing_screen.py screen --keep VX_elastic_buffer design.mlir runs/eb \\ + ci/logic_screen.py summarize runs/full --min-levels 12 + ci/logic_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut + ci/logic_screen.py screen --keep VX_elastic_buffer design.mlir runs/eb \\ port-modules=VX_elastic_buffer - ci/timing_screen.py buffers runs/eb + ci/logic_screen.py buffers runs/eb A Vivado end point table is a CSV of `slack,levels,logic,net,start,end`, one -row per failing end point (a gate's ep.csv). The DUT flow and the timing gate +row per failing end point (a gate's ep.csv). The DUT flow and logic_gate screen with the Makefiles instead (make -C hw/syn/xilinx/dut -screen, -ci/timing_gate.py); see docs/designs/timing_screen.md. +ci/logic_gate.py); see docs/designs/logic_screen.md. """ import argparse import collections @@ -183,7 +183,7 @@ def cmd_screen(args): threads = ["--mlir-disable-threading"] if args.no_threads else [] if not args.design.endswith(".mlirbc"): mapped = os.path.join(args.out, "mapped.mlirbc") - pipeline = "foldsim-timing-map-asic" if asic else "foldsim-timing-map" + pipeline = "foldsim-screen-map-asic" if asic else "foldsim-screen-map" if args.keep: # Keep these modules' instances through inlining, so their ports # can be reported (ports=, port-modules=). diff --git a/ci/synth_gate.py b/ci/synth_gate.py index 1604039761..b7fc5468b7 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -12,7 +12,7 @@ --tool=xilinx Vivado synthesis + place-and-route -> ci/fpga_gate.py --tool=yosys Yosys + OpenSTA on an ASIC PDK -> ci/asic_gate.py - --tool=screen logic levels per RTL unit, no synthesis -> ci/timing_gate.py + --tool=screen logic levels per RTL unit, no synthesis -> ci/logic_gate.py Spec : ci/testcases/.yaml (what to build) Baselines : ci/baselines/synthesis//.json (last accepted metrics) @@ -454,7 +454,7 @@ def describe_baseline(self, base): class Screen(Tool): - """The timing screen: logic levels per RTL unit (hw/syn/xilinx/dut, target + """The logic screen: logic levels per RTL unit (hw/syn/xilinx/dut, target `screen`). foldsim-estimate-timing ranks every register-to-register path of the CIRCT @@ -470,7 +470,7 @@ class Screen(Tool): """ name = "screen" - gate = "timing_gate" + gate = "logic_gate" flow = "hw/syn/xilinx/dut" metrics = ("max_levels", "deep_endpoints", "watch_endpoints", "endpoints", @@ -493,7 +493,7 @@ class Screen(Tool): # The netlist exists once foldsim-opt starts mapping it; a typo in a define # or an include dies in gen_sources.sh or circt_elaborate.sh before that. - elab_done = "foldsim-timing-map" + elab_done = "foldsim-screen-map" elab_label = "elaboration" phases = ( ("gen_sources.sh", "sources"), diff --git a/ci/synth_report.py b/ci/synth_report.py index 84b3513307..dbfe8fa35b 100755 --- a/ci/synth_report.py +++ b/ci/synth_report.py @@ -2,7 +2,7 @@ """synth_report — render synthesis-gate reports as one Markdown summary. Every gate ends a run with `--report` JSON: asic_gate.yml one per DUT job, -fpga_gate.yml and timing_gate.yml one for the whole sweep. This joins them +fpga_gate.yml and logic_gate.yml one for the whole sweep. This joins them into a single Markdown document for the workflow summary -- every build's metrics against its baseline, why each build that did not pass did not, and where each design is tight -- so a run is diagnosed from its GitHub page diff --git a/ci/testcases/timing_gate.yaml b/ci/testcases/logic_gate.yaml similarity index 91% rename from ci/testcases/timing_gate.yaml rename to ci/testcases/logic_gate.yaml index 660b7a2a0f..0c0e3b1f68 100644 --- a/ci/testcases/timing_gate.yaml +++ b/ci/testcases/logic_gate.yaml @@ -1,6 +1,6 @@ -# Logic-level regression gate (the timing screen): the deepest end point of +# Logic-level regression gate (the logic screen): the deepest end point of # every RTL unit must not get deeper than a checked-in golden result -# (ci/baselines/synthesis/screen/.json). See ci/timing_gate.py. +# (ci/baselines/synthesis/screen/.json). See ci/logic_gate.py. # # Sibling of fpga_gate (fpga_gate.yaml) and asic_gate (asic_gate.yaml): same # discipline -- golden baseline, results only ever re-recorded by a human -- @@ -23,7 +23,7 @@ # # tests: what pytest/CI sees -- ONE case, whose `run:` is literally the # command CI executes. -# builds: what ci/timing_gate.py reads -- the build list. The pytest loader +# builds: what ci/logic_gate.py reads -- the build list. The pytest loader # ignores this key. # # The builds are fpga_gate.yaml's, DUT for DUT and config for config (all but @@ -33,14 +33,14 @@ # 300 MHz ones. # # Tier `timing` is OPT-IN (OPT_IN_TIERS in testcase.py): it needs foldsim, -# which the base toolchain does not install. .github/workflows/timing_gate.yml +# which the base toolchain does not install. .github/workflows/logic_gate.yml # installs it and runs the gate. # -# Run one by name: ci/timing_gate.py -b tcu +# Run one by name: ci/logic_gate.py -b tcu # By hand, no gate: make -C hw/syn/xilinx/dut tcu-screen -# Re-record: ci/timing_gate.py --update-baseline (human-reviewed, never CI) +# Re-record: ci/logic_gate.py --update-baseline (human-reviewed, never CI) # --------------------------------------------------------------------------- -category: timing_gate +category: logic_gate defaults: # 32-bit only, as the synthesis gates. xlen: @@ -53,10 +53,10 @@ defaults: tests: # Seconds to minutes per DUT: the whole catalog runs in one job. - id: gate - run: ci/timing_gate.py -j 2 + run: ci/logic_gate.py -j 2 # --------------------------------------------------------------------------- -# builds — read by ci/timing_gate.py, ignored by the pytest catalog loader. +# builds — read by ci/logic_gate.py, ignored by the pytest catalog loader. # # dut hw/syn/xilinx/dut/catalog.mk target to screen # clock_mhz the Fmax gate's target for this DUT (recorded, not asserted) diff --git a/docs/assets/img/timing_screen_compiler.svg b/docs/assets/img/logic_screen_compiler.svg similarity index 98% rename from docs/assets/img/timing_screen_compiler.svg rename to docs/assets/img/logic_screen_compiler.svg index 9d57cba9d7..84fb81948a 100644 --- a/docs/assets/img/timing_screen_compiler.svg +++ b/docs/assets/img/logic_screen_compiler.svg @@ -22,11 +22,11 @@ - Timing screen — the two passes + Logic screen — the two passes - foldsim-timing-map (per module) + foldsim-screen-map (per module) 1. hw-flatten-modules: inline modules < 300 ops diff --git a/docs/assets/img/timing_screen_system.svg b/docs/assets/img/logic_screen_system.svg similarity index 92% rename from docs/assets/img/timing_screen_system.svg rename to docs/assets/img/logic_screen_system.svg index 1c6a0cb222..72176f83b9 100644 --- a/docs/assets/img/timing_screen_system.svg +++ b/docs/assets/img/logic_screen_system.svg @@ -26,7 +26,7 @@ - Timing screen — where each piece lives, and what flows between them + Logic screen — where each piece lives, and what flows between them @@ -49,10 +49,10 @@ units = instance paths, generate indices folded - ci/timing_gate.py (synth_gate.py Screen backend) - ci/testcases/timing_gate.yaml + ci/logic_gate.py (synth_gate.py Screen backend) + ci/testcases/logic_gate.yaml ci/baselines/synthesis/screen/*.json - .github/workflows/timing_gate.yml + .github/workflows/logic_gate.yml PASS / FAIL per unit, zero tolerance on depth @@ -67,7 +67,7 @@ foldsim-opt (foldsim, on CIRCT) - foldsim-timing-map + foldsim-screen-map inline small modules, balance, lower to AIG, map to 6-input LUTs; adders, compares, multipliers and dividers stay word-level @@ -91,7 +91,7 @@ Pull requests touching RTL - timing_gate.yml, ~4 min for the 11 DUTs, + logic_gate.yml, ~4 min for the 11 DUTs, no Vivado, no licence: names the unit that got deeper on the change that did it @@ -102,7 +102,7 @@ ≥ 12 levels, watch list ≥ 8 (300 MHz) - ci/timing_screen.py, hw/scripts/xilinx_*.tcl + ci/logic_screen.py, hw/scripts/xilinx_*.tcl screen summarize validate retime-split xilinx_cone_dump.tcl xilinx_cone_probe.tcl xilinx_wire_probe.tcl diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index d8828c97b6..1f862f44ff 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -277,10 +277,10 @@ so the parity and perf views of one workload are two cases. `perf_gate` catches a change that costs cycles. The synthesis gates catch one that costs **timing closure or area**: they synthesize a catalog of designs and assert the results against the target clock and area budget each build -declares, recording every run's metrics in golden baselines. The timing gate +declares, recording every run's metrics in golden baselines. `logic_gate` catches one that makes a unit **deeper**, in seconds and without synthesis. -| | `fpga_gate` | `asic_gate` | `timing_gate` | +| | `fpga_gate` | `asic_gate` | `logic_gate` | |---|---|---|---| | flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | CIRCT → `foldsim-estimate-timing` (logic levels as Vivado counts them on `DEVICE`'s family) | | design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | `hw/syn/xilinx/dut`, target `screen` | @@ -304,8 +304,8 @@ tool-specific — catalog loading, the config hash, threshold resolution, `known_issue`, the resumable session, the scheduler, the gate and the report. A tool adapter supplies only what differs: metric names, how the flow is invoked, what its log looks like, which environment fields make two runs -comparable, and any assertion beyond the per-metric check (the timing gate's -per-unit depths). `fpga_gate.py`, `asic_gate.py` and `timing_gate.py` are +comparable, and any assertion beyond the per-metric check (`logic_gate`'s +per-unit depths). `fpga_gate.py`, `asic_gate.py` and `logic_gate.py` are entry points that pin the tool. #### Spec and baseline @@ -318,7 +318,7 @@ entry points that pin the tool. Groups are `core`, `tensor`, `graphics`, `dxa` and `vm`. `asic_gate` uses the same group names and gates a subset of the builds (§4.5), so where the two gates share a design, a divergence between the tools on the same module is a -finding, not noise. `timing_gate` screens exactly `fpga_gate`'s builds, so a +finding, not noise. `logic_gate` screens exactly `fpga_gate`'s builds, so a unit it flags can be read against the Fmax gate's critical paths. The config hash ties the two files together: edit the spec, and the gate @@ -365,9 +365,9 @@ so the flow reports the signed worst slack instead. | `XPASS` | a known issue stopped reproducing; clear the flag | no | | `RECORDED` | a baseline was written | no | -#### What the timing gate asserts +#### What `logic_gate` asserts -The screen ([timing_screen.md](timing_screen.md)) counts logic levels, not +The screen ([logic_screen.md](logic_screen.md)) counts logic levels, not slack: it does not see wires, so it gates depth, which it predicts (Spearman 0.50–0.93 against Vivado per register on the V80 gates), and leaves Fmax to `fpga_gate`. It counts them for the family of @@ -538,7 +538,7 @@ The largest designs — `top`, `gfx`, `tensor`, `vm` — are therefore gated on `fpga_gate` only, and `core` and `tex` are gated at a reduced width. Measure a design's peak memory before adding it: an over-budget design costs a silent job kill, not a red gate. -### 4.6 `timing_gate.yml` +### 4.6 `logic_gate.yml` On every pull request and push to master that touches the RTL, the DUT wrappers, the screen's scripts, the gate or its baselines; and by manual diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index d6637b2c06..adeba8e48e 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -27,8 +27,8 @@ The whole flow is one IR from SystemVerilog to Verilog: `circt-verilog` imports Vortex, Foldsim's passes cut it into models and compose the simulator, `firtool` writes it out, and the card's native flow (IP-XACT packaging, `slashkit link`) builds the image. (`foldsim-opt` also carries the -timing screen, which counts the logic levels of a Vortex netlist before -Vivado: [timing_screen.md](timing_screen.md).) Foldsim is a project of its own +logic screen, which counts the logic levels of a Vortex netlist before +Vivado: [logic_screen.md](logic_screen.md).) Foldsim is a project of its own that builds on CIRCT, as POCL builds on LLVM: the `foldsim` driver and `foldsim-opt` come from the toolchain (`$(TOOLDIR)/foldsim`), CIRCT from `$(TOOLDIR)/circt`. Nothing else is in the flow: no Yosys, no sv2v. diff --git a/docs/designs/timing_screen.md b/docs/designs/logic_screen.md similarity index 90% rename from docs/designs/timing_screen.md rename to docs/designs/logic_screen.md index 147e2038aa..cb6d7ec9d4 100644 --- a/docs/designs/timing_screen.md +++ b/docs/designs/logic_screen.md @@ -1,9 +1,9 @@ -# Timing screen — logic depth of every path, before Vivado +# Logic screen — logic depth of every path, before Vivado -The timing screen counts the logic levels of every register-to-register path +The logic screen counts the logic levels of every register-to-register path of a Vortex design the way Vivado counts them, in seconds and without Vivado. It names the RTL units whose paths are too deep for the target clock, -gates their depth on every RTL change (`timing_gate`), and is the first step +gates their depth on every RTL change (`logic_gate`), and is the first step of RTL timing work: screen, fix the deep units, then confirm with one Vivado run. @@ -14,7 +14,7 @@ run. | [3](#3-compiler-architecture) | Compiler architecture: the two passes | | [4](#4-the-model) | The model: levels per element, devices | | [5](#5-outputs) | Outputs | -| [6](#6-integration) | Integration: the DUT flow, `timing_gate`, timing work | +| [6](#6-integration) | Integration: the DUT flow, `logic_gate`, timing work | | [7](#7-validation) | Validation against Vivado | | [8](#8-diagnostics) | Diagnostics beyond levels | | [9](#9-invariants-and-limitations) | Invariants and limitations | @@ -44,27 +44,27 @@ make -C hw/syn/xilinx/dut tcu-screen The whole catalog as a gate, and a baseline update (a human step): ```bash -ci/timing_gate.py -j 2 -ci/timing_gate.py -b om --update-baseline +ci/logic_gate.py -j 2 +ci/logic_gate.py -b om --update-baseline ``` -Any CIRCT netlist, with `ci/timing_screen.py` (subcommands `screen`, +Any CIRCT netlist, with `ci/logic_screen.py` (subcommands `screen`, `summarize`, `validate`, `retime-split`; `foldsim-opt` from the `PATH` or `FOLDSIM_OPT`): ```bash -ci/timing_screen.py screen design.mlir runs/full # map, then screen -ci/timing_screen.py screen runs/full/mapped.mlirbc runs/socket \ +ci/logic_screen.py screen design.mlir runs/full # map, then screen +ci/logic_screen.py screen runs/full/mapped.mlirbc runs/socket \ top=VX_socket__lift black-boxes=VX_core__lift # one unit of a mapped design -ci/timing_screen.py summarize runs/full --min-levels 12 # deep end points by RTL unit -ci/timing_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut +ci/logic_screen.py summarize runs/full --min-levels 12 # deep end points by RTL unit +ci/logic_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut ``` Or the two passes directly: ```bash foldsim-opt design.mlir --emit-bytecode -o mapped.mlirbc \ - --pass-pipeline='builtin.module(foldsim-timing-map)' + --pass-pipeline='builtin.module(foldsim-screen-map)' foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( foldsim-estimate-timing{top=VX_core_top family=virtexuplusHBM output=endpoints.csv instances=instances.csv report=report.txt @@ -85,7 +85,7 @@ foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( ## 2. System architecture -![Timing screen system](../assets/img/timing_screen_system.svg) +![Logic screen system](../assets/img/logic_screen_system.svg) The engine is two compiler passes of **foldsim** (`foldsim-opt`), built on CIRCT like the rest of foldsim, and shipped in the foldsim prebuilt. Vortex @@ -93,15 +93,15 @@ never builds it. The integration is in Vortex: | Piece | Where | Role | |---|---|---| -| `foldsim-timing-map`, `foldsim-estimate-timing` | foldsim `lib/Transforms/EstimateTiming.cpp` | map to LUTs; count levels | -| `ci/timing_screen.py` | `ci/` | screen any netlist; read it by unit; compare with Vivado | +| `foldsim-screen-map`, `foldsim-estimate-timing` | foldsim `lib/Transforms/EstimateTiming.cpp` | map to LUTs; count levels | +| `ci/logic_screen.py` | `ci/` | screen any netlist; read it by unit; compare with Vivado | | `xilinx_cone_dump.tcl`, `xilinx_cone_probe.tcl`, `xilinx_wire_probe.tcl` | `hw/scripts` | Vivado probes for what levels cannot show (§8) | | target `screen` | `hw/syn/xilinx/dut/common.mk` | one DUT: sources → netlist → screen → summary | | `circt_elaborate.sh -f mlir` | `hw/scripts` | the lowered HW/Comb/Seq netlist, no `firtool` | | `gen_platform.py --emit=family` | `hw/scripts` | the family of `DEVICE`, from `hw/syn/xilinx/platforms` | | `screen_summary.py` | `hw/scripts` | `screen_summary.csv` and per-unit `screen_units.json` | -| `Screen` backend, `timing_gate.py` | `ci/synth_gate.py`, `ci/` | the gate, beside `fpga_gate` and `asic_gate` | -| `timing_gate.yml` | `.github/workflows` | runs it on pull requests that touch RTL | +| `Screen` backend, `logic_gate.py` | `ci/synth_gate.py`, `ci/` | the gate, beside `fpga_gate` and `asic_gate` | +| `logic_gate.yml` | `.github/workflows` | runs it on pull requests that touch RTL | The screen reads the same CIRCT import that foldsim and the Yosys flow use (`circt-verilog --ir-hw`), elaborated from the sources and defines @@ -110,9 +110,9 @@ black boxes (`xilinx_ip_stubs.sv`). ## 3. Compiler architecture -![Timing screen compiler](../assets/img/timing_screen_compiler.svg) +![Logic screen compiler](../assets/img/logic_screen_compiler.svg) -### 3.1 `foldsim-timing-map` +### 3.1 `foldsim-screen-map` A pipeline of CIRCT's synthesis passes plus three of foldsim's, per module: @@ -201,7 +201,7 @@ family is an error. ### 4.3 ASAP7 For the ASIC flow (`hw/syn/yosys`, ASAP7 7.5-track RVT, TT corner) the -screen counts **cell stages**, not LUT levels. `foldsim-timing-map-asic` +screen counts **cell stages**, not LUT levels. `foldsim-screen-map-asic` maps the same balanced and-inverter graph to 3-input cuts, each one cell (NAND, AOI, MAJ, XOR), and `family=asap7` models the word-level operators as an ASIC builds them: @@ -275,7 +275,7 @@ its kind, the fanout of its bit, and its instance path and name: 1. `gen_sources.sh -P` into `screen/src`; 2. `circt_elaborate.sh -f mlir` into `screen/netlist.mlir`, the FP IPs as black boxes; -3. `foldsim-timing-map`, then `foldsim-estimate-timing` with +3. `foldsim-screen-map`, then `foldsim-estimate-timing` with `family=$(gen_platform.py --device=$(DEVICE) --emit=family)`; 4. `screen_summary.py`. @@ -287,12 +287,12 @@ make -C hw/syn/yosys/dut om TARGET=screen # om_VX_om_core_top/screen/report.txt: the deepest paths and the period ``` -### 6.2 `timing_gate` +### 6.2 `logic_gate` The third synthesis gate, beside `fpga_gate` (Vivado, nightly) and `asic_gate` (Yosys + OpenSTA, nightly); see [continuous_integration.md §3.5](continuous_integration.md). It screens -`fpga_gate`'s builds (`ci/testcases/timing_gate.yaml`, all but `top`, whose +`fpga_gate`'s builds (`ci/testcases/logic_gate.yaml`, all but `top`, whose `VX_cp_core` keeps LLHD reference ports) in about four minutes, and checks each against `ci/baselines/synthesis/screen/`: @@ -312,8 +312,8 @@ by a human (`--update-baseline`), as for the other gates. Screen first, then Vivado: ```bash -ci/timing_screen.py screen gen/design.mlir runs/full -ci/timing_screen.py summarize runs/full --min-levels 12 # the fix list, by unit +ci/logic_screen.py screen gen/design.mlir runs/full +ci/logic_screen.py summarize runs/full --min-levels 12 # the fix list, by unit ``` The screen ranks the units; one Vivado run confirms the fix. Compare a fix @@ -324,7 +324,7 @@ folded design (§9). Against Vivado, per RTL register or memory: the screen's deepest bit against Vivado's deepest failing path and worst slack, on seven V80 datasets -with no new Vivado runs (`ci/timing_screen.py validate`): +with no new Vivado runs (`ci/logic_screen.py validate`): | Dataset | Vivado run | Spearman vs levels | Spearman vs slack | ≥ 12: precision / recall | |---|---|---|---|---| @@ -368,10 +368,10 @@ each output. Small modules are inlined by the mapping, so keep them with `screen --keep`: ```bash -ci/timing_screen.py screen --keep VX_elastic_buffer design.mlir runs/eb \ +ci/logic_screen.py screen --keep VX_elastic_buffer design.mlir runs/eb \ port-modules=VX_elastic_buffer -ci/timing_screen.py buffers runs/eb # the OUT_BUF advisor -ci/timing_screen.py ports runs/eb # every kept module, by unit +ci/logic_screen.py buffers runs/eb # the OUT_BUF advisor +ci/logic_screen.py ports runs/eb # every kept module, by unit ``` - The **buffer advisor** lists the `VX_elastic_buffer`s that pass a deep diff --git a/hw/scripts/circt_elaborate.sh b/hw/scripts/circt_elaborate.sh index 7049613065..024f1e4c03 100755 --- a/hw/scripts/circt_elaborate.sh +++ b/hw/scripts/circt_elaborate.sh @@ -35,7 +35,7 @@ # v2005 writes Verilog-2005 for tools with a Verilog frontend (yosys): no # packed arrays or structs, no local variables, no size casts; -f mlir # writes the HW/Comb/Seq netlist itself, for tools built on CIRCT (the -# timing screen), and exports nothing. +# logic screen), and exports nothing. # # The sources are a filelist (-l, as gen_sources.sh writes it: +define+, # +incdir+ and files in compile order), or include directories whose sources diff --git a/hw/scripts/screen_summary.py b/hw/scripts/screen_summary.py index b90b4a667e..664f87bcd8 100644 --- a/hw/scripts/screen_summary.py +++ b/hw/scripts/screen_summary.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -"""Summarize a timing screen (foldsim-estimate-timing) for the timing gate. +"""Summarize a logic screen (foldsim-estimate-timing) for logic_gate. The screen writes one row per end point bit: the logic levels of its longest -path, its start and its end. ci/timing_gate.py gates depth per RTL unit, so +path, its start and its end. ci/logic_gate.py gates depth per RTL unit, so this collapses the table into what it compares: screen_summary.csv one row: the deepest end point, the end point bits at or @@ -24,7 +24,8 @@ import re # 300 MHz on the V80: failing end points have a median of 7-8 levels, and -# about 55-60% of the registers at 12 or more fail (timing_screen_proposal.md). +# about 55-60% of the registers at 12 or more fail +# (docs/designs/logic_screen.md §7). FIX_LEVELS = 12 WATCH_LEVELS = 8 NUM_PATHS = 10 diff --git a/hw/syn/xilinx/dut/Makefile b/hw/syn/xilinx/dut/Makefile index e72a36bb5f..ba42d37ff4 100644 --- a/hw/syn/xilinx/dut/Makefile +++ b/hw/syn/xilinx/dut/Makefile @@ -23,7 +23,7 @@ $(DUTS): cp build.mk $(PREFIX)_$@/Makefile $(MAKE) -C $(PREFIX)_$@ DUT=$@ clean && $(MAKE) -C $(PREFIX)_$@ DUT=$@ > $(PREFIX)_$@/build.log 2>&1 & -# The timing screen of one DUT, in the foreground: make tcu-screen +# The logic screen of one DUT, in the foreground: make tcu-screen %-screen: mkdir -p $(PREFIX)_$* cp build.mk $(PREFIX)_$*/Makefile diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index 9082178e1e..242f2dbda2 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -84,12 +84,12 @@ power: TOOL_DIR=$(SCRIPT_DIR) SAIF_FILE=$(SAIF_FILE) SAIF_INST=$(SAIF_INST) \ $(VIVADO) -mode batch -source $(SCRIPT_DIR)/xilinx_power_analysis.tcl -# The timing screen (foldsim-estimate-timing): the logic levels of every +# The logic screen (foldsim-estimate-timing): the logic levels of every # register-to-register path of the design, counted as Vivado counts them on # DEVICE's family (hw/syn/xilinx/platforms), by RTL unit, in seconds and # without Vivado. It elaborates the same sources and # defines as gen-sources into a CIRCT netlist of its own, with the Xilinx -# floating-point cores as black boxes. ci/timing_gate.py gates +# floating-point cores as black boxes. ci/logic_gate.py gates # screen_summary.csv and screen_units.json; screen/report.txt has the deepest # paths op by op. screen: @@ -97,7 +97,7 @@ screen: $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -Cscreen/src -Oscreen/sources.f $(RTL_PKGS) $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t $(TOP_LEVEL_ENTITY) -l screen/sources.f \ -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r screen/memories.rpt -o screen/netlist.mlir - $(FOLDSIM_OPT) screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-timing-map)' \ + $(FOLDSIM_OPT) screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-screen-map)' \ --emit-bytecode -o screen/mapped.mlirbc family=$$(python3 $(SCRIPT_DIR)/gen_platform.py --device=$(DEVICE) --emit=family) && \ $(FOLDSIM_OPT) screen/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$$family output=screen/endpoints.csv instances=screen/instances.csv report=screen/report.txt})" diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index bbc87445c3..fa781974ea 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -285,16 +285,16 @@ timing: $(SYN_SRC_DEPS) $(SYN_LIB_DEPS) | $(BUILD_DIR) @$(SRC_DIR)/synth_summary.py --reports $(BUILD_DIR)/reports \ --top $(TOP_LEVEL_ENTITY) --clock-mhz $(CLOCK_FREQ) -# The timing screen (foldsim-estimate-timing, family=$(PDK)): the gate stages +# The logic screen (foldsim-estimate-timing, family=$(PDK)): the gate stages # of every register-to-register path, with the same sources and SRAM black # boxes as synthesis, in seconds and without Yosys; report.txt converts the # deepest path to a period with the library's delays. -# docs/designs/timing_screen.md. +# docs/designs/logic_screen.md. screen: $(BUILD_DIR)/src mkdir -p $(BUILD_DIR)/screen $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -b'$(BB_MODULES)' \ -o$(BUILD_DIR)/screen/netlist.mlir - $(FOLDSIM_OPT) $(BUILD_DIR)/screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-timing-map-asic)' \ + $(FOLDSIM_OPT) $(BUILD_DIR)/screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-screen-map-asic)' \ --emit-bytecode -o $(BUILD_DIR)/screen/mapped.mlirbc $(FOLDSIM_OPT) $(BUILD_DIR)/screen/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$(PDK) output=$(BUILD_DIR)/screen/endpoints.csv instances=$(BUILD_DIR)/screen/instances.csv report=$(BUILD_DIR)/screen/report.txt})' @grep "^deepest path" $(BUILD_DIR)/screen/report.txt diff --git a/hw/syn/yosys/liberty_stage.py b/hw/syn/yosys/liberty_stage.py index 5eb2635bca..d275695047 100755 --- a/hw/syn/yosys/liberty_stage.py +++ b/hw/syn/yosys/liberty_stage.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The delays the timing screen's ASIC profile (foldsim-estimate-timing +"""The delays the logic screen's ASIC profile (foldsim-estimate-timing family=asap7) converts stages with, read from a Liberty library. usage: liberty_stage.py [--fanout N] [--suffix _ASAP7_75t_R] diff --git a/hw/syn/yosys/sta_endpoints.tcl b/hw/syn/yosys/sta_endpoints.tcl index 463dab0d58..870da049e6 100644 --- a/hw/syn/yosys/sta_endpoints.tcl +++ b/hw/syn/yosys/sta_endpoints.tcl @@ -12,8 +12,8 @@ # limitations under the License. # sta_endpoints.tcl: OpenSTA, the worst setup slack at the data input of -# every flip-flop, for comparing the timing screen per register with an -# ASIC flow (a CSV in the end point table format of ci/timing_screen.py). +# every flip-flop, for comparing the logic screen per register with an +# ASIC flow (a CSV in the end point table format of ci/logic_screen.py). # env: LIB NETLIST TOP SDC OUT proc env {name} { return $::env($name) } read_liberty [env LIB] From ba381b07d1aaa3f0e4b5c8fa1b0025db970d7261 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 18:09:06 -0700 Subject: [PATCH 257/336] stream_buffer: load the skid register whenever the input side is open With an output register, the skid register loaded on fire_in, so its wide enable waited on valid_in, which is often a deep producer signal (the warp-control unit's exit decode, for one). The skid register is only read once the input side closes, and that happens only on a cycle that accepts an input and loads it; while the side is open its contents are unused. It now loads whenever the input side is open, so the enable is the registered ready alone. Cycle-exact at the buffer's outputs, for every OUT_REG stream buffer. Compute-slice gate, place-only at 3.333 ns: the warp-control request buffer family -0.332 ns -> -0.209 ns. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a73f18792b9029fa045bb8b7edc9e8301ea17d8b) Folded: the stream_buffer half of 8ee34bed3 (fire_in is used only without the output register, so it is declared in that branch; the OUT_REG variant warned it unused). --- hw/rtl/libs/VX_stream_buffer.sv | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hw/rtl/libs/VX_stream_buffer.sv b/hw/rtl/libs/VX_stream_buffer.sv index 9f0f08df4c..eb561140b6 100644 --- a/hw/rtl/libs/VX_stream_buffer.sv +++ b/hw/rtl/libs/VX_stream_buffer.sv @@ -46,7 +46,6 @@ module VX_stream_buffer #( reg [DATAW-1:0] data_out_r, buffer_r; reg valid_out_r, valid_in_r; - wire fire_in = valid_in && ready_in; wire flow_out = ready_out || ~valid_out; always @(posedge clk) begin @@ -67,8 +66,11 @@ module VX_stream_buffer #( if (OUT_REG != 0) begin : g_out_reg + // The skid register is read only once the input side closes, which + // takes an accepted input; while open it may follow the input bus, + // so its wide enable does not wait on valid_in. always @(posedge clk) begin - if (fire_in) begin + if (valid_in_r) begin buffer_r <= data_in; end end @@ -83,6 +85,8 @@ module VX_stream_buffer #( end else begin : g_no_out_reg + wire fire_in = valid_in && ready_in; + always @(posedge clk) begin if (fire_in) begin data_out_r <= data_in; From 5e890cff3607a28fbbe0d10f8338aadf98a64c9c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 17:21:19 -0700 Subject: [PATCH 258/336] logic screen: every deep path, op by op, by unit, with source lines The DUT flow screens with foldsim-estimate-timing's paths= (one path per pair of registers) at the watch threshold, and screen_summary.py writes screen_paths.txt: units deepest first, and in each its paths, with generate copies and bits folded as units fold them, so a unit logic_gate names is a heading there. A failing build names the file, and CI already uploads it with the build's screen_* files. ci/logic_screen.py paths prints the same list for any screen (--unit to filter). Mapped LUTs have no names, so each op carries its source line: circt_elaborate.sh keeps every op's location in the MLIR netlist, and the screen now elaborates hw/rtl through a filelist instead of preprocessed copies, whose lines differ from the repository's. Elaborating hw/rtl directly leaves one dead register that the preprocessed copies did not (VX_tcu_abuf's slot_is_sparse_r, whose only load is its own next state): tcu +1 and tensor +2 end points. Every unit's depth is unchanged on the 11 DUTs. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7e3523263b6e75af0ed9b316c200135ed8ec8b67) --- ci/logic_screen.py | 24 +++++++- ci/synth_gate.py | 42 ++++++++++--- docs/assets/img/logic_screen_compiler.svg | 7 ++- docs/assets/img/logic_screen_system.svg | 18 +++--- docs/designs/continuous_integration.md | 4 +- docs/designs/logic_screen.md | 53 ++++++++++++++-- hw/scripts/circt_elaborate.sh | 14 +++-- hw/scripts/screen_summary.py | 73 +++++++++++++++++++++++ hw/syn/xilinx/dut/common.mk | 12 ++-- 9 files changed, 210 insertions(+), 37 deletions(-) diff --git a/ci/logic_screen.py b/ci/logic_screen.py index 976f441de3..b9efe48dd9 100755 --- a/ci/logic_screen.py +++ b/ci/logic_screen.py @@ -19,6 +19,8 @@ every end point (foldsim-screen-map, foldsim-estimate-timing) summarize the deep end points grouped by RTL unit, optionally beside Vivado's failing ones + paths every deep path op by op, by RTL unit (one per pair of + registers, generate copies folded) validate agreement per register with a Vivado end point table retime-split the same agreement, split by whether Vivado retimed the paths ports the depth at the ports of chosen modules' instances, by unit @@ -29,6 +31,7 @@ ci/logic_screen.py screen runs/full/mapped.mlirbc runs/socket \\ top=VX_socket__lift black-boxes=VX_core__lift ci/logic_screen.py summarize runs/full --min-levels 12 + ci/logic_screen.py paths runs/full --unit lsu_unit ci/logic_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut ci/logic_screen.py screen --keep VX_elastic_buffer design.mlir runs/eb \\ port-modules=VX_elastic_buffer @@ -194,7 +197,8 @@ def cmd_screen(args): + threads)) files = ["output=%s/endpoints.csv" % args.out, "instances=%s/instances.csv" % args.out, - "report=%s/report.txt" % args.out] + "report=%s/report.txt" % args.out, + "paths=%s/paths.jsonl" % args.out] if any(o.startswith("port-modules=") for o in args.options): files.append("ports=%s/ports.csv" % args.out) opts = " ".join(files + args.options) @@ -427,6 +431,18 @@ def cmd_summarize(args): +def cmd_paths(args): + """Every path of at least --min-levels, by unit, op by op.""" + sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', 'hw', 'scripts')) + import screen_summary + count, _ = screen_summary.write_paths(sys.stdout, args.screen, + args.min_levels, args.unit) + if not count: + print('(the screen lists paths from its path-levels= on, 12 by ' + 'default)') + + def load_ports(screen): """instance -> {port: (direction, levels)} from a screen's ports.csv.""" ports = collections.defaultdict(dict) @@ -563,6 +579,12 @@ def main(): help="the screen's column to group by") p.set_defaults(fn=cmd_summarize) + p = sub.add_parser("paths", help="deep paths op by op, by RTL unit") + p.add_argument('screen') + p.add_argument('--min-levels', type=int, default=12) + p.add_argument('--unit', help="only the units whose name contains this") + p.set_defaults(fn=cmd_paths) + p = sub.add_parser("validate", help="agreement per register with Vivado") p.add_argument('screen') p.add_argument('ep') diff --git a/ci/synth_gate.py b/ci/synth_gate.py index b7fc5468b7..b042d863cf 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -162,6 +162,10 @@ def compare(self, metrics, base): fails.""" return [] + def evidence(self, work): + """Files of a finished build that explain its reasons.""" + return [] + def incomplete(self, metrics): """Whether a cached result predates something this tool now records. @@ -505,6 +509,7 @@ class Screen(Tool): error_re = re.compile(r"^(?:ERROR|FATAL|Error)\b|: error:|^\S+: \*\*\* ") UNITS = "screen_units.json" + PATHS = "screen_paths.txt" def env(self, args): return {"circt": tool_version("circt-verilog", "--version", @@ -585,6 +590,12 @@ def compare(self, metrics, base): "it in")) return reasons + def evidence(self, work): + # Every unit at or above the watch threshold, its paths op by op: a + # failing unit is a heading there. + path = os.path.join(work, self.PATHS) + return [path] if os.path.exists(path) else [] + def describe(self, m): return "max %d levels, %d end points >= %s" % ( m["max_levels"], m["deep_endpoints"], m.get("fix_levels")) @@ -1004,6 +1015,18 @@ def save_stamp(work, chash, status, stamp_file, metrics=None): fh.write("\n") +def build_work(build, build_dir, tool): + """A build's work directory. + + Unique PREFIX per build id: its own build tree and its own log, so parallel + builds (and other synthesis runs on this machine) cannot collide. It is + also what keeps the Yosys flow's cached $(BUILD_DIR)/src from serving one + DUT's sources to the next. + """ + return tool.work_dir(tool.dut_root(build_dir), tool.gate + "_" + build["id"], + build) + + def run_build(build, build_dir, env, args, tool): """Synthesize one DUT; return its metrics. Raises BuildError on failure.""" dut_root = tool.dut_root(build_dir) @@ -1011,12 +1034,7 @@ def run_build(build, build_dir, env, args, tool): raise BuildError("no DUT target '%s' in %s/catalog.mk" % (build["dut"], dut_root)) - # Unique PREFIX per build id: its own build tree and its own log, so parallel - # builds (and other synthesis runs on this machine) cannot collide. It is - # also what keeps the Yosys flow's cached $(BUILD_DIR)/src from serving one - # DUT's sources to the next. - prefix = tool.gate + "_" + build["id"] - work = tool.work_dir(dut_root, prefix, build) + work = build_work(build, build_dir, tool) stamp_file = tool.gate + ".json" chash = config_hash(build, env, tool) @@ -1120,14 +1138,17 @@ def run_all(builds, build_dir, env, args, tool): for b in order} for fut in concurrent.futures.as_completed(futures): b = futures[fut] + work = build_work(b, build_dir, tool) try: metrics = fut.result() say(b, "done", "%s (%dm)" % (tool.describe(metrics), metrics["build_time_s"] // 60)) - results.append({"build": b, "metrics": metrics, "error": None}) + results.append({"build": b, "metrics": metrics, "error": None, + "work": work}) except (BuildError, subprocess.CalledProcessError) as e: say(b, "FAIL", str(e)) - results.append({"build": b, "metrics": None, "error": str(e)}) + results.append({"build": b, "metrics": None, "error": str(e), + "work": work}) return sorted(results, key=lambda r: r["build"]["id"]) @@ -1296,10 +1317,15 @@ def cell(key, fmt): + tuple(cell(c[1], c[2]) for c in tool.columns)) + " %5dm %s" % (m["build_time_s"] // 60, verdict)) + works = {r["build"]["id"]: r.get("work") for r in results} for bid, verdict, reasons in failures: print("\n%s: %s" % (bid, verdict)) for reason in reasons: print(" - %s" % reason) + if works.get(bid) and verdict not in ("BUILD-FAIL", "STALE", + "NO-BASELINE"): + for path in tool.evidence(works[bid]): + print(" see %s" % path) return failures diff --git a/docs/assets/img/logic_screen_compiler.svg b/docs/assets/img/logic_screen_compiler.svg index 84fb81948a..14f9df9cde 100644 --- a/docs/assets/img/logic_screen_compiler.svg +++ b/docs/assets/img/logic_screen_compiler.svg @@ -110,12 +110,13 @@ DSP absorbs (with its enable logic) optional retime-levels=: one-hop balancing - + Report walk worst predecessors back to the start: start, largest fanout, module crossings; - histogram; top-paths and trace= op by op - endpoints.csv, instances.csv, report.txt + histogram; top-paths and trace= op by op; + paths=: each deep register pair, op by op + endpoints.csv, instances.csv, report.txt Device profiles (family=): the FPGAs differ only in their DSPs diff --git a/docs/assets/img/logic_screen_system.svg b/docs/assets/img/logic_screen_system.svg index 72176f83b9..bd8cbbe674 100644 --- a/docs/assets/img/logic_screen_system.svg +++ b/docs/assets/img/logic_screen_system.svg @@ -39,14 +39,15 @@ hw/syn/xilinx/dut — target screen - gen_sources.sh -P (defines from the toml) + gen_sources.sh (hw/rtl, toml defines) circt_elaborate.sh -f mlir -x xilinx_ip_stubs.sv gen_platform.py --device=$(DEVICE) --emit=family - - hw/scripts/screen_summary.py - screen_summary.csv, screen_units.json - units = instance paths, generate indices folded + + hw/scripts/screen_summary.py + screen_summary.csv, screen_units.json + screen_paths.txt (deep paths by unit) + units = instance paths, generate indices folded ci/logic_gate.py (synth_gate.py Screen backend) @@ -78,12 +79,13 @@ versal (V80) | virtexuplusHBM (U55C) optional: retime-levels=, trace=, black-boxes= - + Outputs (per DUT, screen/) endpoints.csv one row per end point bit instances.csv the elaborated instance tree report.txt histogram, deepest paths op by op - mapped.mlirbc kept for re-screening other tops + paths.jsonl every deep path, source lines + mapped.mlirbc kept for re-screening other tops @@ -126,7 +128,7 @@ endpoints, instances - + diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index 1f862f44ff..10428f8ebc 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -388,8 +388,8 @@ screen has profiles for the V80 and the U55C only, and fails on any other There is no tolerance: the same RTL and tools give the same levels. A new CIRCT or foldsim can map the same RTL differently, so both versions are in the config hash and a tool change reads `STALE`. `make -C hw/syn/xilinx/dut -screen` -runs one screen by hand; its `screen/report.txt` prints the deepest paths op by -op. +runs one screen by hand; its `screen_paths.txt` lists every deep path op by op, +by unit, with source lines, and a failing build names it. #### Running a multi-hour sweep diff --git a/docs/designs/logic_screen.md b/docs/designs/logic_screen.md index cb6d7ec9d4..1b4bb1071d 100644 --- a/docs/designs/logic_screen.md +++ b/docs/designs/logic_screen.md @@ -37,7 +37,7 @@ One DUT of the synthesis catalog, from a configured build tree: ```bash make -C hw/syn/xilinx/dut tcu-screen -# build_tcu/screen/report.txt deepest paths, op by op +# build_tcu/screen_paths.txt every path at >= 8 levels, op by op, by unit # build_tcu/screen_units.json deepest end point per RTL unit ``` @@ -49,14 +49,15 @@ ci/logic_gate.py -b om --update-baseline ``` Any CIRCT netlist, with `ci/logic_screen.py` (subcommands `screen`, -`summarize`, `validate`, `retime-split`; `foldsim-opt` from the `PATH` or -`FOLDSIM_OPT`): +`summarize`, `paths`, `validate`, `retime-split`, `ports`, `buffers`; +`foldsim-opt` from the `PATH` or `FOLDSIM_OPT`): ```bash ci/logic_screen.py screen design.mlir runs/full # map, then screen ci/logic_screen.py screen runs/full/mapped.mlirbc runs/socket \ top=VX_socket__lift black-boxes=VX_core__lift # one unit of a mapped design ci/logic_screen.py summarize runs/full --min-levels 12 # deep end points by RTL unit +ci/logic_screen.py paths runs/full --unit lsu_unit # its paths, op by op (§5) ci/logic_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut ``` @@ -80,6 +81,7 @@ foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( | `min-levels=` | write only end points at or above this many levels | | `top-paths=` | deepest paths to print op by op (default 50) | | `trace=` | also print the deepest path of each end point whose name contains this | +| `paths=`, `path-levels=` | every path of at least `path-levels` (12) levels, one per pair of registers, op by op (§5) | | `retime-levels=` | also estimate the depth after Vivado's retiming to this period (§4.4) | | `ports=`, `port-modules=` | the depth at each port of the named modules' instances (§8) | @@ -251,7 +253,9 @@ candidates retiming is unlikely to absorb. | `instances.csv` | the instance tree: `id,parent,instance,module` (the `*_inst` columns' ids) | | `report.txt` | counts, combinational loops, ops without a model, the histogram of end points by levels, the `top-paths` deepest paths op by op, and the deepest path of each `trace=` match | | `screen_summary.csv` | the DUT flow's one row: max levels, end points ≥ 12, ≥ 8, all | +| `paths.jsonl` | `paths=`: one JSON line per pair of start and end registers whose deepest bit has at least `path-levels` levels: that bit's levels, the pair's bit count, start, end, and `ops`, the path from the end back to the start as `[levels, kind, fanout, instance id, name, source]` | | `screen_units.json` | the deepest end point of every unit at ≥ 8 levels, and the 10 deepest paths | +| `screen_paths.txt` | the DUT flow's `paths.jsonl` at ≥ 8 levels, by unit (`ci/logic_screen.py paths` for any screen) | A path in `report.txt`, deepest first, one op per line: levels at the op, its kind, the fanout of its bit, and its instance path and name: @@ -264,6 +268,37 @@ its kind, the fanout of its bit, and its instance path and name: 0 reg fo 59 .../req_pending/in_fire_r[13] ``` +`report.txt`'s deepest paths are often one pair of registers, bit after bit. +The path list has **every** deep path once: + +- one per pair of start and end registers, its deepest bit, with the + number of bits it stands for; +- pairs whose names differ only in generate indices (`g_banks_0`, + `g_banks_1`) and bits fold into one, as units fold them, so a unit the + gate names is a heading of the list; +- units deepest first, and in each unit its paths deepest first; +- each op with the **source line** it came from. Mapped LUTs have no name + of their own, so the file and line are what locate them in the RTL. The + DUT flow elaborates `hw/rtl` itself (not preprocessed copies), so the + lines are the repository's. + +``` +== cache/g_cache_cache/g_banks_*_bank/g_amo_amo: 24 levels, 23 paths + + 24 levels, 1024 d bits in 4 copies + from cache/g_cache.cache/g_banks_0.bank/g_amo.amo/g_commit.chain_match_r[0] + to cache/g_cache.cache/g_banks_0.bank/g_amo.amo/g_commit.wbq_data.d[6] + 24 lut fo 0 VX_cache_amo.sv:211 .../g_amo.amo/comb.truth_table[0] + 22 lut fo 1 VX_cache_amo.sv:700 .../g_amo.amo/comb.truth_table[0] + ... + 10 carry fo 839 VX_cache_amo.sv:689 .../g_amo.amo/comb.sub[1] + ... + 0 reg fo 2 VX_cache_amo.sv:773 .../g_commit.chain_match_r[0] +``` + +On the full GPU, the 3,283 pairs at ≥ 12 levels fold into 169 paths in +69 units. + ## 6. Integration ### 6.1 The DUT flow @@ -272,12 +307,15 @@ its kind, the fanout of its bit, and its instance path and name: `screen` target screens the same DUT with the same sources, defines and `DEVICE` (the U55C by default): -1. `gen_sources.sh -P` into `screen/src`; +1. `gen_sources.sh` into `screen/sources.f`, a filelist of `hw/rtl` with + the defines (not preprocessed copies, so source lines are the + repository's); 2. `circt_elaborate.sh -f mlir` into `screen/netlist.mlir`, the FP IPs as - black boxes; + black boxes, with every op's source location; 3. `foldsim-screen-map`, then `foldsim-estimate-timing` with `family=$(gen_platform.py --device=$(DEVICE) --emit=family)`; -4. `screen_summary.py`. +4. `screen_summary.py`: `screen_summary.csv`, `screen_units.json` and + `screen_paths.txt` (§5). The ASIC flow has the same target, with the same sources, defines and SRAM black boxes as its synthesis, and `family=$(PDK)`: @@ -302,6 +340,9 @@ each against `ci/baselines/synthesis/screen/`: to lock in by re-recording; - end point counts are recorded, not gated (they grow with lanes and warps). +A failing build names its `screen_paths.txt`, where the failing unit is a +heading with its paths op by op; CI uploads it with the build's logs. + Depth is deterministic, so the tolerance is zero. The baseline fingerprint is the CIRCT and foldsim versions, `DEVICE` and `XLEN`: a new foldsim reads as STALE, not as a regression. Baselines are golden data, re-recorded only diff --git a/hw/scripts/circt_elaborate.sh b/hw/scripts/circt_elaborate.sh index 024f1e4c03..dcae42f694 100755 --- a/hw/scripts/circt_elaborate.sh +++ b/hw/scripts/circt_elaborate.sh @@ -140,11 +140,17 @@ for stub in "${stubs[@]}"; do done stem="${output_file%.*}" -# The netlist is the output itself in mlir format; keep the import beside it. -[ "$format" = mlir ] && stem="$stem.import" +# The netlist is the output itself in mlir format; keep the import beside it, +# and every op's source location: the tools built on the netlist name logic +# that has no name of its own by its file and line. +debuginfo=() +if [ "$format" = mlir ]; then + stem="$stem.import" + debuginfo=(--mlir-print-debuginfo) +fi "$CIRCT_VERILOG" --ir-hw --top="$top" --detect-memories --allow-use-before-declare \ --allow-hierarchical-const --ignore-uninstantiated-modules "${blackboxes[@]}" \ - "${defines[@]}" "${args[@]}" "${files[@]}" -o "$stem.mlir" + "${defines[@]}" "${args[@]}" "${files[@]}" "${debuginfo[@]}" -o "$stem.mlir" # Memories are detected on import; what remains of a conditional write at a # dynamic index (`if (c) x[i] = v`) is expanded into per-element selects. @@ -160,7 +166,7 @@ fi # users come first; sorting the ops first lets it merge every duplicate. if [ "$format" = mlir ]; then "$CIRCT_OPT" "$stem.mlir" "${passes[@]}" --canonicalize --topological-sort --cse \ - -o "$output_file" + "${debuginfo[@]}" -o "$output_file" exit 0 fi "$CIRCT_OPT" "$stem.mlir" "${passes[@]}" --canonicalize --topological-sort --cse \ diff --git a/hw/scripts/screen_summary.py b/hw/scripts/screen_summary.py index 664f87bcd8..ba16ec4d74 100644 --- a/hw/scripts/screen_summary.py +++ b/hw/scripts/screen_summary.py @@ -9,6 +9,8 @@ above the fix and watch thresholds, and all end points screen_units.json the deepest end point of every unit at or above the watch threshold, and the deepest paths (diagnostics) + screen_paths.txt if the screen wrote paths.jsonl: every path at or above + the watch threshold, op by op, by unit (write_paths) A unit is an end point's instance path with generate indices folded (g_banks_3_bank -> g_banks_*_bank), so one deep bank reads as one unit. @@ -121,6 +123,77 @@ def main(): print("screen: %s max %d levels, %d end point bits >= %d, %d units >= %d" % (args.top, summary["max_levels"], summary["deep_endpoints"], FIX_LEVELS, len(units), WATCH_LEVELS)) + if os.path.exists(os.path.join(args.screen, "paths.jsonl")): + with open(os.path.join(args.output_dir, "screen_paths.txt"), "w") as fh: + n, u = write_paths(fh, args.screen, WATCH_LEVELS, paths=paths) + print("screen: %d paths >= %d levels in %d units -> screen_paths.txt" + % (n, WATCH_LEVELS, u)) + + +def fold(name): + return INDEX.sub("_*", norm(name)) + + +def write_paths(out, screen, min_levels, unit_filter=None, paths=None): + """Write the screen's deep paths (paths.jsonl) by unit, op by op. + + The screen lists one path per pair of start and end registers; this folds + the pairs whose names differ only in generate indices and bits, as units + fold them, keeping the deepest. Returns (paths, units) written. + """ + if paths is None: + paths = instance_paths(os.path.join(screen, "instances.csv")) + + def full(ctx, name): + return "%s/%s" % (paths[ctx], name) if paths[ctx] else name + + units = collections.defaultdict(dict) + with open(os.path.join(screen, "paths.jsonl")) as fh: + for line in fh: + p = json.loads(line) + if p["levels"] < min_levels: + continue + start = full(p["start_inst"], p["start"]) + end = full(p["end_inst"], p["end"]) + u = unit(end) + if unit_filter and unit_filter not in u: + continue + group = units[u] + key = (fold(start), fold(end)) + if key in group: + group[key]["copies"] += 1 + group[key]["bits"] += p["bits"] + else: + group[key] = {"path": p, "copies": 1, "bits": p["bits"], + "start": start, "end": end} + + def deepest(u): + return max(g["path"]["levels"] for g in units[u].values()) + + order = sorted(units, key=lambda u: (-deepest(u), u)) + count = sum(len(g) for g in units.values()) + out.write("%d paths at or above %d levels in %d units, deepest unit " + "first.\n" % (count, min_levels, len(units))) + out.write("A path is its deepest end point bit, op by op from the end " + "point back to its\nstart: levels, kind, fanout, source line, " + "instance/name. Bits of the same pair\nof registers, and " + "generate copies (_0, _1, ...), are folded into it.\n") + for u in order: + group = sorted(units[u].values(), key=lambda g: -g["path"]["levels"]) + out.write("\n== %s: %d levels, %d paths\n" % (u, deepest(u), + len(group))) + for g in group: + p = g["path"] + retimed = (", %d after retiming" % p["retimed"] + if p["retimed"] != p["levels"] else "") + copies = " in %d copies" % g["copies"] if g["copies"] > 1 else "" + out.write("\n %d levels%s, %d %s bits%s\n from %s\n to %s\n" + % (p["levels"], retimed, g["bits"], p["kind"], copies, + g["start"], g["end"])) + for levels, kind, fanout, ctx, name, source in p["ops"]: + out.write(" %4d %-6s fo %-5d %-24s %s\n" + % (levels, kind, fanout, source, full(ctx, name))) + return count, len(units) if __name__ == "__main__": diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index 242f2dbda2..723c1495a9 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -90,22 +90,24 @@ power: # without Vivado. It elaborates the same sources and # defines as gen-sources into a CIRCT netlist of its own, with the Xilinx # floating-point cores as black boxes. ci/logic_gate.py gates -# screen_summary.csv and screen_units.json; screen/report.txt has the deepest -# paths op by op. +# screen_summary.csv and screen_units.json; screen_paths.txt has every path +# of at least SCREEN_PATH_LEVELS (screen_summary.py's watch threshold), op by +# op, by unit. +SCREEN_PATH_LEVELS ?= 8 screen: mkdir -p screen - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -Cscreen/src -Oscreen/sources.f $(RTL_PKGS) + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -Oscreen/sources.f $(RTL_PKGS) $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t $(TOP_LEVEL_ENTITY) -l screen/sources.f \ -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r screen/memories.rpt -o screen/netlist.mlir $(FOLDSIM_OPT) screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-screen-map)' \ --emit-bytecode -o screen/mapped.mlirbc family=$$(python3 $(SCRIPT_DIR)/gen_platform.py --device=$(DEVICE) --emit=family) && \ - $(FOLDSIM_OPT) screen/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$$family output=screen/endpoints.csv instances=screen/instances.csv report=screen/report.txt})" + $(FOLDSIM_OPT) screen/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$$family output=screen/endpoints.csv instances=screen/instances.csv report=screen/report.txt paths=screen/paths.jsonl path-levels=$(SCREEN_PATH_LEVELS)})" python3 $(SCRIPT_DIR)/screen_summary.py screen --top $(TOP_LEVEL_ENTITY) clean: ifndef RESUME - rm -rf project_1 screen screen_summary.csv screen_units.json + rm -rf project_1 screen screen_summary.csv screen_units.json screen_paths.txt rm -rf .Xil rm -f *.rpt rm -f *.log From 4fe8893473d41dfc0cd4664cc7f4972463f478d0 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 18:09:26 -0700 Subject: [PATCH 259/336] cache_amo: shorten the LLC commit stage's operand and queue paths Two equivalent rewrites in the LLC's AMO commit engine: - An AMO is a naturally aligned 4- or 8-byte access (the width already comes from its byte-enable count, and the byteen may not cross its lane), so its operand starts on a 32-bit boundary within the lane. The lane bit offset now keeps only that word bit, turning the operand and result barrel shifts into a select. A simulation assert checks the alignment. - The write-back queue's slots were updated through a slot index chosen by the drain (wb_fire) and written by the push (wb_push), both following the bank's stall, with the RMW ALU result behind the push/coalesce decode. Each slot's next entry is now formed for either drain outcome from early terms (target slot, coalesce, merged bytes), and the drain, then the push, select last; the ALU result enters only at the byte select. The new form is written with continuous assigns per slot (a nested procedural form crashed circt-verilog). Full-GPU top gate (L2), place-only at 3.333 ns: write-back queue families -0.239 ns / 881 failing endpoints -> -0.219 ns / 174; the queue lane-hit family -0.244 ns -> -0.051 ns. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0d8f843eda4759095db5b04b150b25eaf60369d4) Folded: the cache_amo half of 8ee34bed3 (the lane bit offset takes byte_off_n modulo LANE_SIZE in 32 bits; the narrow operand warned WIDTHEXPAND). --- hw/rtl/cache/VX_cache_amo.sv | 110 ++++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 41 deletions(-) diff --git a/hw/rtl/cache/VX_cache_amo.sv b/hw/rtl/cache/VX_cache_amo.sv index a1d3338c8f..34c119f42f 100644 --- a/hw/rtl/cache/VX_cache_amo.sv +++ b/hw/rtl/cache/VX_cache_amo.sv @@ -282,13 +282,15 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // Lane select and intra-lane bit offset, split from the encoded byte // offset. The AMO's byteen always sits inside one lane (natural // alignment), so the lane of its lowest set byte is the lane of all of - // them (asserted below). + // them (asserted below). A naturally aligned 4- or 8-byte operand starts + // on a 32-bit boundary, so the offset keeps only its word bit: the + // operand and result shifts are a select, not a funnel. reg [LANE_IDXW-1:0] lane_st1; reg [LANE_OFF_BITS-1:0] bit_off_st1; always @(posedge clk) begin if (~pipe_stall) begin lane_st1 <= (NUM_LANES > 1) ? LANE_IDXW'(byte_off_n >> `CLOG2(LANE_SIZE)) : '0; - bit_off_st1 <= LANE_OFF_BITS'({byte_off_n[`UP(`CLOG2(LANE_SIZE))-1:0], 3'b0}); + bit_off_st1 <= LANE_OFF_BITS'(((32'(byte_off_n) % LANE_SIZE) >> 2) << 5); end end // This AMO's byteen slice within its lane. @@ -642,6 +644,66 @@ module VX_cache_amo import VX_gpu_pkg::*; #( wire post_push_post = wb_push && post_coal_key_r; wire post_push_head = wb_push && coal_key_r[0]; + // Writeback queue: a drain (wb_fire) shifts every entry toward the head; + // a completed compute (wb_push) enqueues at the tail or byte-merges into + // its word's entry, written after the shift so it wins when both target + // the same slot. Both handshakes follow the bank's stall, the latest + // signals here, so each slot's next entry is formed for either drain + // outcome from early terms (target slot, coalesce, merged bytes) and + // the drain, then the push, select last. + wire [WBQ_SIZE-1:0][LINE_ADDR_BITS-1:0] wbq_addr_n; + wire [WBQ_SIZE-1:0][WORD_SEL_WIDTH-1:0] wbq_wsel_n; + wire [WBQ_SIZE-1:0][LANE_IDXW-1:0] wbq_lane_n; + wire [WBQ_SIZE-1:0][LANE_SIZE-1:0] wbq_byteen_n; + wire [WBQ_SIZE-1:0][LANE_WIDTH-1:0] wbq_data_n; + wire [WBQ_SIZE-1:0][TAG_WIDTH-1:0] wbq_tag_n; + wire [WBQ_SIZE-1:0][REQ_SEL_WIDTH-1:0] wbq_idx_n; + wire [WBQ_SIZE-1:0][ATTR_WIDTH-1:0] wbq_attr_n; + wire [LANE_SIZE-1:0] pbyteen_d = cmp_byteen | (wb_coalesce_drain ? coal_src_byteen : {LANE_SIZE{1'b0}}); + wire [LANE_SIZE-1:0] pbyteen_h = cmp_byteen | (wb_coalesce_hold ? coal_src_byteen : {LANE_SIZE{1'b0}}); + for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_wbq_next + // a drain moves slot i+1 into slot i; the last slot keeps its entry + localparam J = (i < WBQ_SIZE-1) ? (i + 1) : i; + wire [LANE_SIZE-1:0] byteen_d = (i < WBQ_SIZE-1) + ? (wbq_byteen[J] & ~(wbq_clr_hit[J] ? clr_byteen[J] : {LANE_SIZE{1'b0}})) + : wbq_byteen[i]; + wire [LANE_SIZE-1:0] byteen_h = wbq_byteen[i] & ~(wbq_clr_hit[i] ? clr_byteen[i] : {LANE_SIZE{1'b0}}); + wire [LANE_WIDTH-1:0] data_d = wbq_data[J]; + wire [LANE_WIDTH-1:0] data_h = wbq_data[i]; + wire [LANE_WIDTH-1:0] pdata_d, pdata_h; + for (genvar b = 0; b < LANE_SIZE; ++b) begin : g_bytes + assign pdata_d[b*8 +: 8] = cmp_byteen[b] ? wb_data_w[b*8 +: 8] + : (wb_coalesce_drain ? coal_src_data[b*8 +: 8] : data_d[b*8 +: 8]); + assign pdata_h[b*8 +: 8] = cmp_byteen[b] ? wb_data_w[b*8 +: 8] + : (wb_coalesce_hold ? coal_src_data[b*8 +: 8] : data_h[b*8 +: 8]); + end + wire push_d = wb_push && (wb_slot_drain == WBQ_IDXW'(i)); + wire push_h = wb_push && (wb_slot_hold == WBQ_IDXW'(i)); + assign wbq_addr_n[i] = wb_fire ? (push_d ? cmp_addr : wbq_addr[J]) : (push_h ? cmp_addr : wbq_addr[i]); + assign wbq_wsel_n[i] = wb_fire ? (push_d ? cmp_wsel : wbq_wsel[J]) : (push_h ? cmp_wsel : wbq_wsel[i]); + assign wbq_lane_n[i] = wb_fire ? (push_d ? cmp_lane : wbq_lane[J]) : (push_h ? cmp_lane : wbq_lane[i]); + assign wbq_byteen_n[i] = wb_fire ? (push_d ? pbyteen_d : byteen_d) : (push_h ? pbyteen_h : byteen_h); + assign wbq_data_n[i] = wb_fire ? (push_d ? pdata_d : data_d) : (push_h ? pdata_h : data_h); + assign wbq_tag_n[i] = wb_fire ? (push_d ? cmp_tag : wbq_tag[J]) : (push_h ? cmp_tag : wbq_tag[i]); + assign wbq_idx_n[i] = wb_fire ? (push_d ? cmp_idx : wbq_idx[J]) : (push_h ? cmp_idx : wbq_idx[i]); + assign wbq_attr_n[i] = wb_fire ? (push_d ? cmp_attr : wbq_attr[J]) : (push_h ? cmp_attr : wbq_attr[i]); + end + + always @(posedge clk) begin + if (~reset) begin + for (integer i = 0; i < WBQ_SIZE; ++i) begin + wbq_addr[i] <= wbq_addr_n[i]; + wbq_wsel[i] <= wbq_wsel_n[i]; + wbq_lane[i] <= wbq_lane_n[i]; + wbq_byteen[i] <= wbq_byteen_n[i]; + wbq_data[i] <= wbq_data_n[i]; + wbq_tag[i] <= wbq_tag_n[i]; + wbq_idx[i] <= wbq_idx_n[i]; + wbq_attr[i] <= wbq_attr_n[i]; + end + end + end + always @(posedge clk) begin if (reset) begin cmp_valid <= 1'b0; @@ -687,45 +749,6 @@ module VX_cache_amo import VX_gpu_pkg::*; #( cmp_valid <= 1'b0; end - // Writeback queue: a drain (wb_fire) shifts every entry toward - // the head; a completed compute (wb_push) enqueues at the tail or - // byte-merges into its word's entry. The push is written after - // the shift so it wins when both target the same slot. - if (wb_fire) begin - for (integer i = 0; i < WBQ_SIZE-1; ++i) begin - wbq_addr[i] <= wbq_addr[i+1]; - wbq_wsel[i] <= wbq_wsel[i+1]; - wbq_lane[i] <= wbq_lane[i+1]; - wbq_byteen[i] <= wbq_byteen[i+1] & ~(wbq_clr_hit[i+1] ? clr_byteen[i+1] : {LANE_SIZE{1'b0}}); - wbq_data[i] <= wbq_data[i+1]; - wbq_tag[i] <= wbq_tag[i+1]; - wbq_idx[i] <= wbq_idx[i+1]; - wbq_attr[i] <= wbq_attr[i+1]; - end - end else begin - for (integer i = 0; i < WBQ_SIZE; ++i) begin - if (wbq_clr_hit[i]) begin - wbq_byteen[i] <= wbq_byteen[i] & ~clr_byteen[i]; - end - end - end - if (wb_push) begin - wbq_addr[wb_slot] <= cmp_addr; - wbq_wsel[wb_slot] <= cmp_wsel; - wbq_lane[wb_slot] <= cmp_lane; - wbq_tag[wb_slot] <= cmp_tag; - wbq_idx[wb_slot] <= cmp_idx; - wbq_attr[wb_slot] <= cmp_attr; - // byte-merge: this AMO's bytes take the new result; a coalesce - // keeps the target entry's other bytes; a fresh entry zero-fills. - wbq_byteen[wb_slot] <= cmp_byteen | (wb_coalesce ? coal_src_byteen : {LANE_SIZE{1'b0}}); - for (integer b = 0; b < LANE_SIZE; ++b) begin - if (cmp_byteen[b]) - wbq_data[wb_slot][b*8 +: 8] <= wb_data_w[b*8 +: 8]; - else if (wb_coalesce) - wbq_data[wb_slot][b*8 +: 8] <= coal_src_data[b*8 +: 8]; - end - end // Count grows only on a new (non-coalescing) enqueue; a coalesce // updates in place. Pop removes the head. wbq_count <= wbq_count_n; @@ -844,6 +867,11 @@ module VX_cache_amo import VX_gpu_pkg::*; #( `RUNTIME_ASSERT (~(amo_st1.amo_valid && valid_st1 && is_creq_st1) || ((byteen_st1 & ~(WORD_SIZE'({LANE_SIZE{1'b1}}) << (lane_st1 * LANE_SIZE))) == '0), ("%t: AMO byteen crosses its lane (addr=0x%0h)", $time, addr_st1)) + // The word-granular offset rests on natural alignment as well. + `RUNTIME_ASSERT (~(amo_st1.amo_valid && valid_st1 && is_creq_st1) + || (&lane_byteen_st1) || (lane_byteen_st1 == LANE_SIZE'(4'hF)) + || ((LANE_SIZE == 8) && (lane_byteen_st1 == LANE_SIZE'(8'hF0))), + ("%t: AMO operand is not a naturally aligned word or doubleword (addr=0x%0h)", $time, addr_st1)) // The narrowed arithmetic width rests on the ISA: on a 32-bit hart // only the Zacas pair carries a 64-bit operand. if (`MIN(`VX_CFG_XLEN, AMO_OLD_BITS) < AMO_OLD_BITS) begin : g_arith_width_guard From 7d435429defd1e82ba91d43af0f3afcc3d077abf Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 17:42:50 -0700 Subject: [PATCH 260/336] lsu_scheduler: a client's ready no longer waits on its own request With the TCU metadata AGU as a second client, the LSU's ready went from its own valid through the round-robin grant, the request mux (the granted client's rw selects the scheduler's ready: a read also needs a pending- table slot) and back into the LSU, which gates the 1.5 kbit output register of the LSU dispatch buffer. A ready is only consumed while its client requests, so each client now takes the scheduler's ready for its own rw (VX_mem_scheduler exports both flavors) and is blocked only by a client ahead of it in the rotation that is requesting. That equals the granted ready whenever the client is valid; the arbiter state and every fire are unchanged. A simulation assert checks it against the granted ready. Cycle-exact; no buffer added. Other mem_scheduler users leave the new outputs unconnected. Compute-slice gate, place-only at 3.333 ns: closed with +0.002 ns of slack (with the integer ALU's branch-equal compare below). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a185ae83827b8c74beac64d14f386df09fa7b3a7) --- hw/rtl/core/VX_lsu_scheduler.sv | 24 ++++++++++++++++++++++-- hw/rtl/libs/VX_mem_scheduler.sv | 6 ++++++ hw/rtl/om/VX_om_mem.sv | 2 ++ hw/rtl/raster/VX_raster_earlyz.sv | 2 ++ hw/rtl/raster/VX_raster_mem.sv | 2 ++ hw/rtl/tex/VX_tex_mem.sv | 2 ++ 6 files changed, 36 insertions(+), 2 deletions(-) diff --git a/hw/rtl/core/VX_lsu_scheduler.sv b/hw/rtl/core/VX_lsu_scheduler.sv index 626e43400e..f093e8cd7e 100644 --- a/hw/rtl/core/VX_lsu_scheduler.sv +++ b/hw/rtl/core/VX_lsu_scheduler.sv @@ -60,6 +60,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] sched_req_data; wire [SCHED_TAG_WIDTH-1:0] sched_req_tag; wire sched_req_ready; + wire sched_req_ready_rd, sched_req_ready_wr; wire sched_rsp_valid; wire [NUM_LANES-1:0] sched_rsp_mask; @@ -100,6 +101,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign sched_req_data = cli_req_data[0].data; assign sched_req_tag = cli_req_data[0].tag; assign cli_req_ready[0] = sched_req_ready; + `UNUSED_VAR ({sched_req_ready_rd, sched_req_ready_wr}) assign cli_rsp_valid[0] = sched_rsp_valid; assign cli_rsp_data[0].mask = sched_rsp_mask; @@ -154,10 +156,26 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign sched_req_data = cli_req_data[grant_idx].data; assign sched_req_tag = {grant_idx, cli_req_data[grant_idx].tag}; - // Drive per-client ready: only the granted client sees ready high. + // Per-client ready: a client is granted unless a client ahead of it in + // the rotation is requesting, and it takes the scheduler's ready for its + // own rw. This equals the granted ready whenever the client requests -- + // the only time a ready is consumed -- without the client's own valid, + // the grant and the request mux in front of the requester's enables. for (genvar i = 0; i < NUM_CLIENTS; ++i) begin : g_creq_rdy - assign cli_req_ready[i] = sched_req_ready && (grant_idx == CLIENT_ID_BITS'(i)); + reg blocked; + always @(*) begin + blocked = 1'b0; + for (integer k = 0; k < NUM_CLIENTS; ++k) begin + if ((k != i) && cli_req_valid[k] + && (((k - int'(rr_ptr_r) + NUM_CLIENTS) % NUM_CLIENTS) < ((i - int'(rr_ptr_r) + NUM_CLIENTS) % NUM_CLIENTS))) begin + blocked = 1'b1; + end + end + end + assign cli_req_ready[i] = ~blocked && (cli_req_data[i].rw ? sched_req_ready_wr : sched_req_ready_rd); end + `RUNTIME_ASSERT (~any_valid || (cli_req_ready[grant_idx] == sched_req_ready), + ("%t: %s client ready out of sync with the granted ready", $time, INSTANCE_ID)) // Demux response: extract client_id from extended tag. wire [CLIENT_ID_BITS-1:0] rsp_cid = sched_rsp_tag[SCHED_TAG_WIDTH-1 -: CLIENT_ID_BITS]; @@ -231,6 +249,8 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( .core_req_data (sched_req_data), .core_req_tag (sched_req_tag), .core_req_ready (sched_req_ready), + .core_req_ready_rd (sched_req_ready_rd), + .core_req_ready_wr (sched_req_ready_wr), .req_queue_empty(sched_req_queue_empty), `UNUSED_PIN (req_queue_rw_notify), diff --git a/hw/rtl/libs/VX_mem_scheduler.sv b/hw/rtl/libs/VX_mem_scheduler.sv index 96f80e5ae4..5ac1bca106 100644 --- a/hw/rtl/libs/VX_mem_scheduler.sv +++ b/hw/rtl/libs/VX_mem_scheduler.sv @@ -69,6 +69,10 @@ module VX_mem_scheduler #( input wire [CORE_REQS-1:0][WORD_WIDTH-1:0] core_req_data, input wire [TAG_WIDTH-1:0] core_req_tag, output wire core_req_ready, + // core_req_ready for a read and for a write request: a requester whose + // rw is known ahead of a shared request mux can take its own + output wire core_req_ready_rd, + output wire core_req_ready_wr, // Core request queue output wire req_queue_empty, @@ -223,6 +227,8 @@ module VX_mem_scheduler #( // can accept another request? assign core_req_ready = reqq_ready_in && ibuf_ready; + assign core_req_ready_rd = reqq_ready_in && ~ibuf_full; + assign core_req_ready_wr = reqq_ready_in; // request queue status. `coalescer_empty` (1 when no coalescer) keeps a // store still buffered in the coalescer from prematurely signalling empty. diff --git a/hw/rtl/om/VX_om_mem.sv b/hw/rtl/om/VX_om_mem.sv index 45841002fb..f2c5e90097 100644 --- a/hw/rtl/om/VX_om_mem.sv +++ b/hw/rtl/om/VX_om_mem.sv @@ -271,6 +271,8 @@ module VX_om_mem import VX_gpu_pkg::*; import VX_om_pkg::*; #( .core_req_data (mreq_data_r), .core_req_tag (mreq_tag_r), .core_req_ready (mreq_ready_r), + `UNUSED_PIN (core_req_ready_rd), + `UNUSED_PIN (core_req_ready_wr), .req_queue_rw_notify (write_notify), .req_queue_empty (req_queue_empty_w), diff --git a/hw/rtl/raster/VX_raster_earlyz.sv b/hw/rtl/raster/VX_raster_earlyz.sv index 7759ba7232..c95061aa05 100644 --- a/hw/rtl/raster/VX_raster_earlyz.sv +++ b/hw/rtl/raster/VX_raster_earlyz.sv @@ -324,6 +324,8 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX .core_req_data ('0), .core_req_tag (SCHED_TAG_WIDTH'(0)), .core_req_ready (mreq_ready), + `UNUSED_PIN (core_req_ready_rd), + `UNUSED_PIN (core_req_ready_wr), `UNUSED_PIN (req_queue_empty), `UNUSED_PIN (req_queue_rw_notify), diff --git a/hw/rtl/raster/VX_raster_mem.sv b/hw/rtl/raster/VX_raster_mem.sv index 002a9bc5ce..b677ce96d5 100644 --- a/hw/rtl/raster/VX_raster_mem.sv +++ b/hw/rtl/raster/VX_raster_mem.sv @@ -373,6 +373,8 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( .core_req_data ('0), .core_req_tag (sched_req_tag), .core_req_ready (mem_req_ready), + `UNUSED_PIN (core_req_ready_rd), + `UNUSED_PIN (core_req_ready_wr), `UNUSED_PIN (req_queue_empty), `UNUSED_PIN (req_queue_rw_notify), diff --git a/hw/rtl/tex/VX_tex_mem.sv b/hw/rtl/tex/VX_tex_mem.sv index 8907ff267d..597c45cd95 100644 --- a/hw/rtl/tex/VX_tex_mem.sv +++ b/hw/rtl/tex/VX_tex_mem.sv @@ -159,6 +159,8 @@ module VX_tex_mem import VX_gpu_pkg::*; import VX_tex_pkg::*; #( .core_req_data ('0), .core_req_tag (mem_req_tag), .core_req_ready (mem_req_ready), + `UNUSED_PIN (core_req_ready_rd), + `UNUSED_PIN (core_req_ready_wr), `UNUSED_PIN (req_queue_empty), `UNUSED_PIN (req_queue_rw_notify), From 1424e368a730bff01eaeffbc63a36947aa4b622f Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 19:22:50 -0700 Subject: [PATCH 261/336] cache_tags: fills, writes and flushes rewrite whole sets In a set-associative bank the way an operation updates follows the tag compare: a write hit sets the hit way's dirty bit, and a fill targets the resident way on a refill or the victim otherwise. Each of those per-way choices sat on the tag, valid and dirty stores' write enables, so the tag BRAM's read, the compare and the way decode fanned into every array's write-enable pins (the L2's worst families at 300 MHz). The enables are now the operation alone, and the choice moves into the data: a fill rewrites the whole set's tags (the filled way with the new tag, the others with their current tag), valid bits and dirty bits; a write rewrites the requested sector's dirty bit in every way. The current values are the stage's own lookup results: the tag and valid readouts already carry last cycle's fill, the dirty readout forwards every update, and the valid rewrite masks last cycle's same-set clears (init, flush, invalidate) on the data path only, off the hit path. A flush decodes its way from the request's registered way rather than the shared evict-way mux, whose fill leg follows the compare. All changes are cycle-exact. Full-GPU top gate (L2), place-only at 3.333 ns: dirty store -0.312 ns -> -0.150 ns, valid store -0.325 ns -> -0.112 ns, top -0.325 ns / 2768 failing endpoints -> -0.196 ns / 1610. The D-cache harness stays closed (+0.002 ns). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5d71fdf74c9037c95c311fbe303224837b92d216) --- hw/rtl/cache/VX_cache_bank.sv | 1 + hw/rtl/cache/VX_cache_tags.sv | 65 ++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 0ed5359add..ed02e49dd0 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -689,6 +689,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .line_tag (line_tag_st0), .sector_idx (sector_idx_st0), .evict_way (evict_way_st0), + .flush_way (st0.req.way_idx), .fill_way_oh (fill_way_oh_st0), .tag_matches (tag_matches_st0), .line_present (line_present_st0), diff --git a/hw/rtl/cache/VX_cache_tags.sv b/hw/rtl/cache/VX_cache_tags.sv index e70eb402ef..0a33b2df5d 100644 --- a/hw/rtl/cache/VX_cache_tags.sv +++ b/hw/rtl/cache/VX_cache_tags.sv @@ -58,6 +58,9 @@ module VX_cache_tags import VX_gpu_pkg::*; #( input wire [`CS_TAG_SEL_BITS-1:0] line_tag, input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx, // requested sector within the line input wire [`CS_WAY_SEL_WIDTH-1:0] evict_way, + // the way a flush walks: the request's registered way, never a fill's + // compare-derived target, so the flush's clears stay off the tag read + input wire [`CS_WAY_SEL_WIDTH-1:0] flush_way, // fill target way as one-hot, taken from line_present/victim directly: // the binary evict_way round trip (encode, mux, per-way compare) put the // tag-read cone on this array's own write-enable pins. @@ -110,9 +113,8 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // Per-way decoded write strobes and payloads. At most one operation type // fires per cycle (input arbitration in the bank). The tag store holds the - // tag only and is written (with the registered line_tag) only on a fill, so - // its write never depends on the tag compare. - wire [NUM_WAYS-1:0] line_write; + // tag only and is written only on a fill, so its write never depends on + // the tag compare. wire [NUM_WAYS-1:0][`CS_TAG_SEL_BITS-1:0] line_wdata; wire [NUM_WAYS-1:0][`CS_TAG_SEL_BITS-1:0] tag_rdata; @@ -138,7 +140,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][SEC-1:0] partial_rdata; for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_way_decode - wire way_en = (NUM_WAYS == 1) || (evict_way == i); + wire way_en = (NUM_WAYS == 1) || (flush_way == i); wire do_init = init; // init all ways // The fill way select comes in one-hot (line_present or the registered // victim decode); the binary way_en compare stays only for flush, whose @@ -148,11 +150,12 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire do_write = WRITEBACK && write && tag_matches[i]; // only write on tag hit // Tag store: the tag changes only on a fill (a refill rewrites the - // identical tag, harmless), so its write-enable does not depend on the - // compare and its write-data is the registered line_tag. init/flush/ - // invalidate only touch the valid vector (below), never the tag. - assign line_write[i] = do_fill; - assign line_wdata[i] = line_tag; + // identical tag, harmless). init/flush/invalidate only touch the valid + // vector (below), never the tag. The fill way follows the tag compare + // (a refill targets the resident way), so a fill rewrites every way -- + // the filled one with line_tag, the others with their current tag -- + // and the compare reaches the array's data, not its write enables. + assign line_wdata[i] = do_fill ? line_tag : read_tag[i]; // Read-First BRAM: a fill committed on the previous cycle isn't yet in // the readout. The bypass is keyed on the SET and the FILLED WAY: that @@ -203,12 +206,21 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // read-modify-write): set the filled sector; clear the whole line on // init/flush, the other sectors on a fresh victim install, the requested // sector on invalidate. - wire [SEC-1:0] vset_oh = do_fill ? sec_oh : {SEC{1'b0}}; - wire [SEC-1:0] vclr_oh = ((do_init || do_flush) ? {SEC{1'b1}} : {SEC{1'b0}}) - | ((do_fill && ~fill_refill) ? ~sec_oh : {SEC{1'b0}}) - | (do_inval ? sec_oh : {SEC{1'b0}}); - assign valid_wren[i] = vset_oh | vclr_oh; - assign valid_wdata[i] = vset_oh; + // The fill way follows the tag compare (a refill targets the resident + // way), so a fill rewrites every way's valid bits -- the filled way + // with its update, the others with their current bits -- and the + // compare reaches the store's data, not its write enables. The current + // bits are read_valid, which already carries last cycle's fill, with + // last cycle's other same-set clears applied here, off the hit path. + wire [SEC-1:0] vclr_op = ((do_init || do_flush) ? {SEC{1'b1}} : {SEC{1'b0}}) + | (do_inval ? sec_oh : {SEC{1'b0}}); + wire same_set_v = (line_idx == line_idx_n); + wire [SEC-1:0] rdw_vclr; + `BUFFER_EX(rdw_vclr, (same_set_v ? vclr_op : {SEC{1'b0}}), ~stall, SEC, 1); + wire [SEC-1:0] cur_valid = read_valid[i] & ~rdw_vclr; + wire [SEC-1:0] fill_valid = do_fill ? (fill_refill ? (cur_valid | sec_oh) : sec_oh) : cur_valid; + assign valid_wren[i] = (fill ? {SEC{1'b1}} : {SEC{1'b0}}) | vclr_op; + assign valid_wdata[i] = fill ? fill_valid : {SEC{1'b0}}; // Tag from the tag BRAM, valid from the decoupled LUTRAM. Both share the // same look-ahead read / read-first access pattern, so the way_filled @@ -223,12 +235,23 @@ module VX_cache_tags import VX_gpu_pkg::*; #( if (WRITEBACK) begin : g_dirty // set the written sector on a write hit; clear sectors on // fill/inval; clear all on init/flush or a fresh (evicting) fill. + // The hit way of a write and the target way of a fill follow the + // tag compare, so those operations rewrite every way -- the target + // with its update, the others with their current bits (read_dirty + // is exact: it forwards last cycle's update) -- and the compare + // reaches the store's data, not its write enables. + wire write_op = WRITEBACK && write; wire [SEC-1:0] dset_oh = do_write ? sec_oh : {SEC{1'b0}}; wire dclr_all = do_init || do_flush || (do_fill && ~fill_refill); wire [SEC-1:0] dclr_oh = (do_inval ? sec_oh : {SEC{1'b0}}) | ((do_fill && fill_refill) ? sec_oh : {SEC{1'b0}}); - assign dirty_wren[i] = dset_oh | dclr_oh | {SEC{dclr_all}}; - assign dirty_wdata[i] = dset_oh; // 1 only where setting; 0 on any clear + wire [SEC-1:0] fill_keep = do_fill ? (fill_refill ? (read_dirty[i] & ~sec_oh) : {SEC{1'b0}}) + : read_dirty[i]; + assign dirty_wren[i] = (write_op ? sec_oh : {SEC{1'b0}}) + | {SEC{fill || do_init || do_flush}} + | (do_inval ? sec_oh : {SEC{1'b0}}); + assign dirty_wdata[i] = fill ? fill_keep + : ((write_op ? read_dirty[i] : {SEC{1'b0}}) | dset_oh); // 0 on any clear // Read-first forwarding for the dirty LUTRAM (same-set, 1 cycle, held // across stall): a dirty update to this set last cycle isn't in the @@ -276,7 +299,6 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // written at line_idx, read-first to match the fill/replay ordering. VX_dp_ram #( .DATAW (NUM_WAYS * `CS_TAG_SEL_BITS), - .WRENW (NUM_WAYS), .SIZE (`CS_LINES_PER_BANK), .OUT_REG (1), .RDW_MODE ("R") @@ -284,12 +306,9 @@ module VX_cache_tags import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .read (~stall), - // fill alone: only fills write the tag store, and the fill way select - // (one-hot) always names exactly one way, so |line_write == fill -- - // an identity synthesis cannot prove, leaving the way-select cone on - // the array's global enable. + // a fill writes the whole set's tags (see line_wdata) .write (fill), - .wren (line_write), + .wren (1'b1), .waddr (line_idx), .raddr (line_idx_n), .wdata (line_wdata), From c0a112fa1c716341c965f7db933ae4d02e497eb8 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Fri, 23 Oct 2026 20:15:27 -0700 Subject: [PATCH 262/336] cache: read the evicted tag at the writeback way The evicted tag registered into the commit stage was read at the fill way, which on a fill follows the tag compare (a refill targets the resident way), so the tag BRAM, the compare, the way select and a tag mux ran into the commit register in one cycle. The evicted tag only forms the writeback address, and a writeback leaves only from a fresh fill (whose way is the replacement victim, decoded from registered state) or a flushed way (the request's registered way); a refill evicts nothing. The tags now take that writeback way for evict_tag. The dirty flag and mask keep the fill way, so the eviction-bytes checks are unchanged. Cycle-exact. Full-GPU top gate (L2), place-only at 3.333 ns: commit-register family -0.196 ns -> -0.059 ns; top -0.196 ns / 1610 failing endpoints -> -0.175 ns / 885. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 78f07f952c3975f00b5d850411e8d35cf1248121) --- hw/rtl/cache/VX_cache_bank.sv | 2 ++ hw/rtl/cache/VX_cache_tags.sv | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index ed02e49dd0..660f4afcbd 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -690,6 +690,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .sector_idx (sector_idx_st0), .evict_way (evict_way_st0), .flush_way (st0.req.way_idx), + // a writeback leaves only from a fresh fill's victim or a flushed way + .wb_way (st0.req.is_fill ? victim_way : st0.req.way_idx), .fill_way_oh (fill_way_oh_st0), .tag_matches (tag_matches_st0), .line_present (line_present_st0), diff --git a/hw/rtl/cache/VX_cache_tags.sv b/hw/rtl/cache/VX_cache_tags.sv index 0a33b2df5d..c821a024ed 100644 --- a/hw/rtl/cache/VX_cache_tags.sv +++ b/hw/rtl/cache/VX_cache_tags.sv @@ -61,6 +61,9 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // the way a flush walks: the request's registered way, never a fill's // compare-derived target, so the flush's clears stay off the tag read input wire [`CS_WAY_SEL_WIDTH-1:0] flush_way, + // the way a writeback evicts, for evict_tag: a fill's replacement victim + // (a refill evicts nothing) or a flush's way -- never the tag compare + input wire [`CS_WAY_SEL_WIDTH-1:0] wb_way, // fill target way as one-hot, taken from line_present/victim directly: // the binary evict_way round trip (encode, mux, per-way compare) put the // tag-read cone on this array's own write-enable pins. @@ -102,10 +105,11 @@ module VX_cache_tags import VX_gpu_pkg::*; #( if (WRITEBACK) begin : g_evict_tag_wb assign evict_dirty = (| read_dirty[evict_way]); // dirty if any sector dirty assign evict_dirty_mask = read_dirty[evict_way]; - assign evict_tag = read_tag[evict_way]; + assign evict_tag = read_tag[wb_way]; end else begin : g_evict_tag_wt `UNUSED_VAR (read_dirty) `UNUSED_VAR (read_tag) + `UNUSED_VAR (wb_way) assign evict_dirty = 1'b0; assign evict_dirty_mask = '0; assign evict_tag = '0; From de95c7b963f8e9bcee187026146deccf6df9b7a8 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sat, 24 Oct 2026 03:19:36 -0700 Subject: [PATCH 263/336] cache_tags: write the tag store by way, the victim's tag on a line fill The tag store takes the replacement victim as a per-way write enable: a line fill writes the victim way's new tag, and a sector refill rewrites the line's own tags. This is the form the native timing runs measured (c17); the whole-set rewrite it replaces was never measured routed. Routed at 3.333 ns with the reverted distributed-RAM store: top -0.159 ns / 6603 (block-RAM tags) against -0.197 / 7374 with the distributed RAM; socket -0.046 / 2979; D-cache harness -0.066 / 4058. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4f517be7ad6dcb13859ac10866349c9683f4d045) --- hw/rtl/cache/VX_cache_bank.sv | 1 + hw/rtl/cache/VX_cache_tags.sv | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 660f4afcbd..5bef208111 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -693,6 +693,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // a writeback leaves only from a fresh fill's victim or a flushed way .wb_way (st0.req.is_fill ? victim_way : st0.req.way_idx), .fill_way_oh (fill_way_oh_st0), + .victim_way_oh (victim_way_oh), .tag_matches (tag_matches_st0), .line_present (line_present_st0), .evict_dirty (evict_dirty_st0), diff --git a/hw/rtl/cache/VX_cache_tags.sv b/hw/rtl/cache/VX_cache_tags.sv index c821a024ed..27a7e2f6ab 100644 --- a/hw/rtl/cache/VX_cache_tags.sv +++ b/hw/rtl/cache/VX_cache_tags.sv @@ -68,6 +68,8 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // the binary evict_way round trip (encode, mux, per-way compare) put the // tag-read cone on this array's own write-enable pins. input wire [NUM_WAYS-1:0] fill_way_oh, + // the replacement victim as one-hot: the tag store's write enable + input wire [NUM_WAYS-1:0] victim_way_oh, // outputs output wire [NUM_WAYS-1:0] tag_matches, @@ -143,6 +145,9 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][SEC-1:0] partial_wdata; wire [NUM_WAYS-1:0][SEC-1:0] partial_rdata; + // a sector refill rewrites the line's own tags; a line fill writes the + // victim way's new tag + wire fill_refill_any; for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_way_decode wire way_en = (NUM_WAYS == 1) || (flush_way == i); wire do_init = init; // init all ways @@ -159,7 +164,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // (a refill targets the resident way), so a fill rewrites every way -- // the filled one with line_tag, the others with their current tag -- // and the compare reaches the array's data, not its write enables. - assign line_wdata[i] = do_fill ? line_tag : read_tag[i]; + assign line_wdata[i] = fill_refill_any ? read_tag[i] : line_tag; // Read-First BRAM: a fill committed on the previous cycle isn't yet in // the readout. The bypass is keyed on the SET and the FILLED WAY: that @@ -297,12 +302,14 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // (a same-cycle-prior fill is folded into tag_eq/read_valid). assign line_present[i] = tag_eq && (| read_valid[i]); end + assign fill_refill_any = (SEC > 1) && (| line_present); // Single tag array: one BRAM word holds all ways' tags; per-way write-enable // updates a single way on a fill. Read at line_idx_n (one cycle ahead), // written at line_idx, read-first to match the fill/replay ordering. VX_dp_ram #( .DATAW (NUM_WAYS * `CS_TAG_SEL_BITS), + .WRENW (NUM_WAYS), .SIZE (`CS_LINES_PER_BANK), .OUT_REG (1), .RDW_MODE ("R") @@ -310,9 +317,9 @@ module VX_cache_tags import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .read (~stall), - // a fill writes the whole set's tags (see line_wdata) + // a fill writes the victim way's tag (see line_wdata) .write (fill), - .wren (1'b1), + .wren (victim_way_oh), .waddr (line_idx), .raddr (line_idx_n), .wdata (line_wdata), From 7c78dec7ae3ee4f60155e2f1698794acd3708403 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 25 Oct 2026 04:55:46 -0700 Subject: [PATCH 264/336] runtime/foldsim: lock the kernel clock below the image's timed rate VORTEX_FOLDSIM_CLOCK_HZ sets the kernel clock to a rate in (0, timed]: an image can be run slower than it was timed for, as a GPU's clocks can be locked, to tell a timing failure from a functional one on the card. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a1ede060ae12ecec7c1c00e54afd6cace284284e) --- sw/runtime/foldsim/vortex.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp index 33eb72e087..d5b163b8dc 100644 --- a/sw/runtime/foldsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -122,9 +122,20 @@ class transport_t { } VRT_TRY() device_ = std::make_unique(bdf, vbin, /*program=*/true); - // Clock the kernel at the rate the image was timed for. + // Clock the kernel at the rate the image was timed for, or lower if + // VORTEX_FOLDSIM_CLOCK_HZ locks it there. const uint64_t timed_hz = device_->getMaxFrequency(); - device_->setFrequency(timed_hz); + uint64_t want_hz = timed_hz; + if (const char* s = getenv("VORTEX_FOLDSIM_CLOCK_HZ")) { + const uint64_t hz = strtoull(s, nullptr, 0); + if (hz == 0 || hz > timed_hz) { + fprintf(stderr, "[VXDRV] VORTEX_FOLDSIM_CLOCK_HZ=%s: must be in (0, %lu]\n", + s, (unsigned long)timed_hz); + return -1; + } + want_hz = hz; + } + device_->setFrequency(want_hz); const uint64_t clk_hz = device_->getFrequency(); if (clk_hz == 0 || clk_hz > timed_hz + timed_hz / 200) { fprintf(stderr, "[VXDRV] user clock reads %lu Hz, image is timed for %lu Hz\n", From 63c3e46c6a59acb4d120f677fbea6e2854897d2a Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 25 Oct 2026 11:57:24 -0700 Subject: [PATCH 265/336] foldsim runtime: never run the kernel clock above the image's timed rate The clock wizard settles on the nearest rate its dividers reach, which can be a step above the request (103.59 MHz requested, 107.16 MHz set). Lower the request below that step until the rate read back is at most the timed rate or the VORTEX_FOLDSIM_CLOCK_HZ lock, and require exactly that. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b8f714a27c5790b991d77bb51bff2e8b3e19c303) --- sw/runtime/foldsim/vortex.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp index d5b163b8dc..428a94bdb1 100644 --- a/sw/runtime/foldsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -135,9 +135,18 @@ class transport_t { } want_hz = hz; } - device_->setFrequency(want_hz); - const uint64_t clk_hz = device_->getFrequency(); - if (clk_hz == 0 || clk_hz > timed_hz + timed_hz / 200) { + // The wizard settles on the nearest rate its dividers reach, which can + // be a step above the request: lower the request below that step. + const uint64_t limit_hz = want_hz; + uint64_t clk_hz = 0; + for (int i = 0; i < 8; ++i) { + device_->setFrequency(want_hz); + clk_hz = device_->getFrequency(); + if (clk_hz == 0 || clk_hz <= limit_hz) + break; + want_hz -= (clk_hz - want_hz) + limit_hz / 100; + } + if (clk_hz == 0 || clk_hz > limit_hz) { fprintf(stderr, "[VXDRV] user clock reads %lu Hz, image is timed for %lu Hz\n", (unsigned long)clk_hz, (unsigned long)timed_hz); return -1; From 87dc4fb5ddd7e8bef13a4c7e5d4de329a46fa20b Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 25 Oct 2026 20:24:46 -0700 Subject: [PATCH 266/336] foldsim runtime: park the kernel clock when a loaded design fails to come up A design that failed its clock check or its VRT setup stayed loaded and running at its last rate; on 2026-10-25 the next design write then hung the V80's management controller. On every failure after programming, set the kernel clock to 50 MHz, below every image's timing, before returning. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 08b24da5b9daf9967b30bd22f592d17214fb911c) --- sw/runtime/foldsim/vortex.cpp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp index 428a94bdb1..ea099c3f60 100644 --- a/sw/runtime/foldsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -131,6 +131,7 @@ class transport_t { if (hz == 0 || hz > timed_hz) { fprintf(stderr, "[VXDRV] VORTEX_FOLDSIM_CLOCK_HZ=%s: must be in (0, %lu]\n", s, (unsigned long)timed_hz); + park_clock(); return -1; } want_hz = hz; @@ -149,6 +150,7 @@ class transport_t { if (clk_hz == 0 || clk_hz > limit_hz) { fprintf(stderr, "[VXDRV] user clock reads %lu Hz, image is timed for %lu Hz\n", (unsigned long)clk_hz, (unsigned long)timed_hz); + park_clock(); return -1; } printf("[VXDRV] foldsim kernel clock %lu Hz\n", (unsigned long)clk_hz); @@ -160,7 +162,11 @@ class transport_t { auto vdev = session_->getDeviceByBdf(bdf); dma_ = std::make_unique(vdev.openHbmBuffer(/*region=*/8, kDmaChunk)); fd_ = dma_->getFd(); - VRT_CATCH(-1) + } catch (const std::exception& e) { + fprintf(stderr, "[VXDRV] VRT exception: %s\n", e.what()); + park_clock(); + return -1; + } // The host side of the transfers: a kernel DMA buffer, mapped. struct slash_qdma_buf_create create = {}; create.size = sizeof(create); @@ -180,6 +186,20 @@ class transport_t { return 0; } + // A design that fails to come up stays loaded: leave it clocked below any + // image's timing so it cannot disturb the shell before the next load. + void park_clock() noexcept { + constexpr uint64_t kSafeHz = 50000000; + if (!device_) + return; + try { + device_->setFrequency(kSafeHz); + fprintf(stderr, "[VXDRV] kernel clock parked at %lu Hz\n", + (unsigned long)device_->getFrequency()); + } catch (...) { + } + } + ~transport_t() { if (staging_ != nullptr) munmap(staging_, kStaging); From aaf71c424bd1a43edb1d788ffb3c2102324cac38 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Sun, 25 Oct 2026 20:25:18 -0700 Subject: [PATCH 267/336] aved runtime: never above the timed clock; park it when the design fails to come up The clock check accepted a read-back up to 0.5 % above the image's timed rate and returned with the design still loaded and running. Step the request down until the read-back is at most the timed rate, and on any failure after programming set the kernel clock to 50 MHz, below every image's timing. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7ed41cf9bbd3d571adda1ab8af53bcc395de022d) --- sw/runtime/aved/vortex.cpp | 39 ++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/sw/runtime/aved/vortex.cpp b/sw/runtime/aved/vortex.cpp index 2be4583213..977d48e8bd 100644 --- a/sw/runtime/aved/vortex.cpp +++ b/sw/runtime/aved/vortex.cpp @@ -240,18 +240,35 @@ class vx_device { // previous image set, which can be far above or below this one's. if (!sim_mode_) { const uint64_t timed_hz = vrtDevice_.getMaxFrequency(); - vrtDevice_.setFrequency(timed_hz); - const uint64_t clk_hz = vrtDevice_.getFrequency(); - if (clk_hz == 0 || clk_hz > timed_hz + timed_hz / 200) { + // The wizard settles on the nearest rate its dividers reach, which can + // be a step above the request: lower the request below that step. + uint64_t want_hz = timed_hz, clk_hz = 0; + for (int i = 0; i < 8; ++i) { + vrtDevice_.setFrequency(want_hz); + clk_hz = vrtDevice_.getFrequency(); + if (clk_hz == 0 || clk_hz <= timed_hz) + break; + want_hz -= (clk_hz - want_hz) + timed_hz / 100; + } + if (clk_hz == 0 || clk_hz > timed_hz) { fprintf(stderr, "[VXDRV] Error: user clock reads %lu Hz, image is timed for %lu Hz\n", (unsigned long)clk_hz, (unsigned long)timed_hz); + park_clock(); return -1; } printf("[VXDRV] kernel clock %lu Hz (image timed for %lu Hz)\n", (unsigned long)clk_hz, (unsigned long)timed_hz); } vrtKernel_ = vrt::Kernel(vrtDevice_, KERNEL_NAME); - VRT_CATCH(-1) + } catch (const std::exception& e) { + fprintf(stderr, "[VXDRV] VRT exception: %s\n", e.what()); + park_clock(); + return -1; + } catch (...) { + fprintf(stderr, "[VXDRV] VRT exception (unknown)\n"); + park_clock(); + return -1; + } // Must follow vrtKernel_: the connection map comes from the vbin's // system_map.xml, which the kernel handle parses. @@ -818,6 +835,20 @@ class vx_device { private: #ifdef CPP_API + // A design that fails to come up stays loaded: leave it clocked below any + // image's timing so it cannot disturb the shell before the next load. + void park_clock() noexcept { + constexpr uint64_t kSafeHz = 50000000; + if (sim_mode_) + return; + try { + vrtDevice_.setFrequency(kSafeHz); + fprintf(stderr, "[VXDRV] kernel clock parked at %lu Hz\n", + (unsigned long)vrtDevice_.getFrequency()); + } catch (...) { + } + } + // ----- Staged CP memory (m_axi_host wired to device memory) ----- // // WHY THIS EXISTS From 14898fc3d9967aa948e407d722a721260d9a3c5b Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 18:10:39 -0700 Subject: [PATCH 268/336] foldsim kernel: control endpoint decoupled from the target's outputs; FOLDSIM_MEM_SPEC The control endpoint's tokens depend only on its command queue, so its cycle no longer waits for the target's DCR response and busy: those are taken in cycle order as they come and only tell the host. With FOLDSIM_MEM_SPEC (foldsim --axi-spec in FOLDSIM_FLAGS), the kernel's memory endpoints run SPEC and R and AR's ready channels carry both candidates. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 416b2aa7e830d3e90c1765e4c7610e5885c12c93) Integration (phase 2): only the hw/syn/foldsim/Makefile part is taken; the foldsim shell RTL it also changes (hw/rtl/afu/foldsim) moves to the Foldsim repository in 644b38ffe, so those intermediate edits are not replayed. --- hw/syn/foldsim/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index f7a0f2736b..a58ac45d0f 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -186,7 +186,8 @@ $(KERNEL_VH): FORCE @mkdir -p $(@D) @printf '%s\n' '// Generated by hw/syn/foldsim. Do not edit.' \ '`define FOLDSIM_MEM_PORTS $(MEM_PORTS)' '`define FOLDSIM_MEM_CH_SHIFT $(strip $(MEM_CH_SHIFT))' \ - '`define FOLDSIM_MEM_ID_W $(MEM_ID_W)' '`define FOLDSIM_MEM_REPEAT(f,s) $(MEM_REPEAT)' > $@.tmp + '`define FOLDSIM_MEM_ID_W $(MEM_ID_W)' '`define FOLDSIM_MEM_REPEAT(f,s) $(MEM_REPEAT)' \ + $(if $(findstring --axi-spec,$(FOLDSIM_FLAGS)),'`define FOLDSIM_MEM_SPEC 1') > $@.tmp @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ $(KERNEL_SRCS): $(GEN_SV) $(KERNEL_VH) From 938b1e5379adeabaa9696816bbdc7cfc2d9a3867 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 28 Oct 2026 18:10:39 -0700 Subject: [PATCH 269/336] foldsim/afu: FOLDSIM_MEM_SPEC carries the speculation depth At depth 2 the endpoints' R, AR, AW and W ready channels carry four values each; --axi-spec=2 writes the depth into the kernel's define. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5485376017e169b876bbeea625636026c29a5a19) Integration (phase 2): only the hw/syn/foldsim/Makefile part is taken; the foldsim shell RTL it also changes (hw/rtl/afu/foldsim) moves to the Foldsim repository in 644b38ffe, so those intermediate edits are not replayed. --- hw/syn/foldsim/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index a58ac45d0f..172a3f730a 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -187,7 +187,7 @@ $(KERNEL_VH): FORCE @printf '%s\n' '// Generated by hw/syn/foldsim. Do not edit.' \ '`define FOLDSIM_MEM_PORTS $(MEM_PORTS)' '`define FOLDSIM_MEM_CH_SHIFT $(strip $(MEM_CH_SHIFT))' \ '`define FOLDSIM_MEM_ID_W $(MEM_ID_W)' '`define FOLDSIM_MEM_REPEAT(f,s) $(MEM_REPEAT)' \ - $(if $(findstring --axi-spec,$(FOLDSIM_FLAGS)),'`define FOLDSIM_MEM_SPEC 1') > $@.tmp + $(if $(findstring --axi-spec,$(FOLDSIM_FLAGS)),'`define FOLDSIM_MEM_SPEC $(if $(findstring --axi-spec=2,$(FOLDSIM_FLAGS)),2,1)') > $@.tmp @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ $(KERNEL_SRCS): $(GEN_SV) $(KERNEL_VH) From 31100c3f19097c5b8626a764c04627d105c9213c Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 27 Oct 2026 10:41:58 -0700 Subject: [PATCH 270/336] AGENTS.md: Vivado floating-point IP latency is an Fmax knob, already at its maximum Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 01d96a2a5d7cda4af9e0c0df8eb111f92860319e) --- AGENTS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 3b8ed75022..83cc984da4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -135,6 +135,12 @@ python3 ci/roofline.py --app=sgemm --driver=simx --args="-n128" --flops=4194304 - **Align with mainstream CUDA, HIP, OpenCL, and Vulkan API and driver stacks.** For any design question — driver surface, command-processor model, memory model, scheduling — pick the solution those stacks would use. This keeps Vortex's externals familiar to mainstream software and avoids one-off abstractions. +- **Vivado floating-point IP latency is an Fmax knob, and it is already at its maximum.** + - `hw/scripts/xilinx_ip_gen.tcl` sets each `floating_point` IP's `C_Latency`. More cycles means more internal pipelining, up to a per-operation maximum; Vivado rejects any value above it. + - The single-precision maxima on V80 and U55C are: FMA 16 (`Medium_Usage`; 19 with `Full_Usage`), divide 28, square root 28, multiply 8, add 11. `xil_fma`, `xil_fdiv`, `xil_fsqrt` and the TCU's `xil_fmul`/`xil_fadd` already use them, so FMA with `Full_Usage` (more DSPs) is the only headroom left. + - A change must move together: `C_Latency`, the matching `VX_CFG_*_LATENCY` in `VX_config.toml` (checked by the `STATIC_ASSERT`s in `hw/rtl/fpu/VX_f*_unit.sv`), and the SimX timing that reads it. + - This applies only to `FPU_TYPE_DSP`. The STD FPU's div/sqrt latency is fixed by its SRT stage count, and only its FMA latency is configurable. + - **SimX Cardinal Rule — modules communicate *only* through channels.** A `SimObject` may observe or mutate another module's state *only* via its bound `SimChannel` ports (`MemReq`/`MemRsp`, `result_if`, …). **Never reach across the ownership hierarchy** to touch another object directly (`core_->processor()->memsim()`, `parent()->child()->field`, a leaf unit grabbing the global `Memory`). See [docs/designs/simobject.md](docs/designs/simobject.md#the-cardinal-rule). --- From 1e6c467864c9a231170da38c21da4cdf88f0306b Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 27 Oct 2026 10:47:22 -0700 Subject: [PATCH 271/336] VERSION: pin FIRESIM_REV to the fork's vortex_3.x head 57bb93f1 On top of 1ba01ca0, the aved platform: - repackages the kernel when the Vortex source list changes (1ed0393a); - never runs the kernel clock above the image's timed rate (40058778), the clock lock our runtime relies on; - parks the kernel clock when a loaded target fails its checks (57bb93f1). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 43e822df6be0131e05697376d43623e0171aeb7a) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index db7a93ab72..be388f21fc 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 -FIRESIM_REV=1ba01ca07ae78f7e2fdac50f70544602c80995d0 +FIRESIM_REV=57bb93f1e23daf46198659ddd93608f70121fee0 From 314af1d8ae56c8fcf92548a7b408f8124af099b7 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 27 Oct 2026 10:58:44 -0700 Subject: [PATCH 272/336] sim/foldsim: define VX_CFG_XLEN for the DPI floating point Master's Zfhmin change makes hw/dpi/dpi_float.cpp size its NaN-boxing by VX_CFG_XLEN (kRegBits). The other simulators that compile it (rtlsim, avedsim, opaesim, xrtsim) define VX_CFG_XLEN in their CXXFLAGS; the foldsim in-process simulator did not, so libfoldsim_emul.so stopped compiling after the merge: hw/dpi/dpi_float.cpp:55:31: error: 'VX_CFG_XLEN' was not declared ci/testcases/foldsim.yaml parity-demo and parity-sgemm now pass, exact against rtlsim (demo 1222 instrs / 16375 cycles, sgemm -n16 15240 / 195152). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 67652650d805561eae2631ef3b6fbf85d676bb78) --- sim/foldsim/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/sim/foldsim/Makefile b/sim/foldsim/Makefile index c56e81be69..3020a733e6 100644 --- a/sim/foldsim/Makefile +++ b/sim/foldsim/Makefile @@ -20,6 +20,7 @@ LDFLAGS += -shared -pthread # The Xilinx floating-point cores' simulation models compute with Vortex's DPI # floating point. CXXFLAGS += -I$(SW_COMMON_DIR) -I$(THIRD_PARTY_DIR)/softfloat/source/include +CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a VL_FLAGS += -O3 -Wno-fatal -Wno-lint -Wno-style -Wno-MULTIDRIVEN --x-initial 0 --x-assign 0 From f9548355b57c4a5878a73e2cfa5f06bcc4c7b991 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 27 Oct 2026 10:59:53 -0700 Subject: [PATCH 273/336] hw/syn/foldsim: lift the socket without a SOCKET_ID port; docs take the gated class set VX_socket takes socket_id as a port, so the default lift's port=SOCKET_ID made the lift step fail ("promoted parameter SOCKET_ID is not overridden") and the Foldsim parity CI case with it. The docs' example recipe named parameters the units no longer have; it now uses the socket/core/compute-slice classes of the gated runs. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 02beec3b97fc62fe477b1e4ab41c49ac91a6b606) --- docs/designs/foldsim.md | 13 +++++++------ docs/simulation.md | 4 ++-- hw/syn/foldsim/Makefile | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index adeba8e48e..0dcfdc67aa 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -34,8 +34,8 @@ that builds on CIRCT, as POCL builds on LLVM: the `foldsim` driver and `$(TOOLDIR)/circt`. Nothing else is in the flow: no Yosys, no sv2v. ```bash -export FOLDSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift+VX_lsu_scheduler__lift VX_fpu_std" -export FOLDSIM_LIFT="VX_socket:drop=INSTANCE_ID;port=SOCKET_ID VX_issue_slice:drop=ISSUE_ID VX_alu_int:port=BLOCK_IDX VX_alu_muldiv VX_lsu_slice:port=CORE_ID VX_lsu_scheduler" +export FOLDSIM_CLASSES="VX_socket__lift VX_core__lift VX_compute_slice__lift" +export FOLDSIM_LIFT="VX_socket:drop=INSTANCE_ID VX_core:drop=INSTANCE_ID VX_compute_slice:drop=INSTANCE_ID,SLICE_STR" CONFIGS="-DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_ISSUE_WIDTH=2" CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=foldsim --app=sgemm # Verilator make -C hw/syn/foldsim CONFIGS="$CONFIGS" # the image @@ -47,10 +47,11 @@ FPGA_BIN_DIR=$PWD/hw/syn/foldsim/build32_foldsim_hw/bin CONFIGS="$CONFIGS" \ others (the socket contains the issue slices). `FOLDSIM_LIFT` gives interface-port units a plain-port boundary first, so they can be classes (`hw/syn/foldsim/lift_boundaries.py`, a source-to-source pass on pyslang; the -RTL is not changed). `port=P` turns an instance-ID parameter into an input -port, and every parameter derived from it below, so that all instances of the -class elaborate to one module: the socket's `SOCKET_ID` carries every core's -`CORE_ID` with it. The tools are found through `FOLDSIM` (the driver) and +RTL is not changed). `drop=P` removes a parameter that only names an instance (`INSTANCE_ID`), so +all instances of the class elaborate to one module; instance IDs that reach +logic are already input ports in the RTL (the socket's `socket_id`). +`port=P` turns such a parameter into an input port where a unit still has one. +The tools are found through `FOLDSIM` (the driver) and `CIRCT_BIN`, by default `hw/syn/common.mk`'s `FOLDSIM_PATH` and `CIRCT_PATH`. A class `A+B` fuses two modules of the same parent (`foldsim-fuse-models`): diff --git a/docs/simulation.md b/docs/simulation.md index 5b9426ccb9..b0ffa3eea2 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -99,8 +99,8 @@ it like the native V80 driver does (AXI4-Lite registers, QDMA for memory). ```bash # the classes to thread, and the units to give plain-port boundaries first -export FOLDSIM_CLASSES="VX_socket__lift VX_issue_slice__lift VX_alu_int__lift VX_alu_muldiv__lift VX_lsu_slice__lift VX_lsu_scheduler__lift VX_fpu_std" -export FOLDSIM_LIFT="VX_socket:drop=INSTANCE_ID;port=SOCKET_ID VX_issue_slice:drop=ISSUE_ID VX_alu_int:port=BLOCK_IDX VX_alu_muldiv VX_lsu_slice:port=CORE_ID VX_lsu_scheduler" +export FOLDSIM_CLASSES="VX_socket__lift VX_core__lift VX_compute_slice__lift" +export FOLDSIM_LIFT="VX_socket:drop=INSTANCE_ID VX_core:drop=INSTANCE_ID VX_compute_slice:drop=INSTANCE_ID,SLICE_STR" CONFIGS="-DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_ISSUE_WIDTH=2" # in Verilator, no image needed diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index 172a3f730a..0b129dfea0 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -46,7 +46,7 @@ GEN_DIR := $(BUILD_DIR)/gen # The classes threaded by FAME-5, and the units given plain-port boundaries # first so they can be classes (lift_boundaries.py; [:drop=P,...][;port=P,...]). FOLDSIM_CLASSES ?= VX_socket__lift -FOLDSIM_LIFT ?= VX_socket:drop=INSTANCE_ID;port=SOCKET_ID +FOLDSIM_LIFT ?= VX_socket:drop=INSTANCE_ID # Where circt-verilog, circt-opt and firtool are (hw/syn/common.mk's CIRCT); # the foldsim driver is hw/syn/common.mk's FOLDSIM. CIRCT_BIN ?= $(CIRCT_PATH)/bin From 102c2ea2700024b4d2de0151c48731942e72ed68 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Tue, 27 Oct 2026 21:40:08 -0700 Subject: [PATCH 274/336] foldsim image: apply the --slr-groups floorplan before opt_design foldsim --slr-groups writes foldsim_floorplan.tcl (each threaded copy kept whole in one SLR, the SLR left to the placer) for an implementation flow to source after synthesis, but neither the image nor the ooc target sourced it. With FOLDSIM_FLAGS containing --slr-groups, the Makefile now passes FOLDSIM_FLOORPLAN to slashkit and to ooc.tcl; pre_synth_hook.tcl attaches a foldsim pre-opt hook to impl_1 that runs the native aved pre-opt hook and then floorplan.tcl. floorplan.tcl records the assignments in floorplan.txt and fails when the floorplan matches no cell, so an image never silently places without it. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 94bfc63ecf230ab97315cec7cc4074cd294b9fcd) --- hw/syn/foldsim/Makefile | 8 ++++++-- hw/syn/foldsim/floorplan.tcl | 30 ++++++++++++++++++++++++++++++ hw/syn/foldsim/ooc.tcl | 1 + hw/syn/foldsim/pre_opt_hook.tcl | 12 ++++++++++++ hw/syn/foldsim/pre_synth_hook.tcl | 10 +++++++++- 5 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 hw/syn/foldsim/floorplan.tcl create mode 100644 hw/syn/foldsim/pre_opt_hook.tcl diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index 0b129dfea0..79194b7323 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -214,6 +214,10 @@ $(IP_DIR)/component.xml: $(KERNEL_SRCS) $(KERNEL_VH) $(OOC_XDC) $(wildcard $(FOL -tclargs foldsim_afu $(KERNEL_SRCS) . $(DEVICE) ooc-xdc=$(OOC_XDC) \ $$(grep -qE '^ *xil_f(ma|div|sqrt|mul|add) ' $(GEN_SV) || echo no-vendor-ip) +# foldsim --slr-groups writes a floorplan that keeps each threaded copy whole +# in one SLR; the implementation applies it before opt_design (floorplan.tcl). +FLOORPLAN_ENV := $(if $(findstring --slr-groups,$(FOLDSIM_FLAGS)),FOLDSIM_FLOORPLAN=$(GEN_DIR)/foldsim_floorplan.tcl) + # The kernel alone, out of context at FOLDSIM_FREQ: synthesis with global # retiming, placement and routing, without the shell, its reports in # $(BUILD_DIR)/ooc. It answers what a change does to the kernel's Fmax in @@ -221,7 +225,7 @@ $(IP_DIR)/component.xml: $(KERNEL_SRCS) $(KERNEL_VH) $(OOC_XDC) $(wildcard $(FOL OOC_DIR := $(BUILD_DIR)/ooc ooc: $(KERNEL_SRCS) rm -rf $(OOC_DIR) && mkdir -p $(OOC_DIR) - cd $(OOC_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -nojournal -log vivado.log \ + cd $(OOC_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(FLOORPLAN_ENV) $(VIVADO) -mode batch -nojournal -log vivado.log \ -source $(SRC_DIR)/ooc.tcl -tclargs $(KERNEL_SRCS) $(DEVICE) \ $$(python3 -c 'print(round(1000 / $(FOLDSIM_FREQ), 3))') \ $$(grep -qE '^ *xil_f(ma|div|sqrt|mul|add) ' $(GEN_SV) || echo no-vendor-ip) @@ -236,7 +240,7 @@ FORCE: $(VBIN): $(IP_DIR)/component.xml $(CFG) mkdir -p $(BIN_DIR) - TOOL_DIR=$(SCRIPT_DIR) $(SLASHKIT) link -c $(CFG) -p hw -o $@ -k $(IP_DIR)/component.xml \ + TOOL_DIR=$(SCRIPT_DIR) $(FLOORPLAN_ENV) $(SLASHKIT) link -c $(CFG) -p hw -o $@ -k $(IP_DIR)/component.xml \ --vivado $(VIVADO) $(if $(SLASHKIT_JOBS),--jobs $(SLASHKIT_JOBS)) \ --pre-synth-tcls $(SRC_DIR)/pre_synth_hook.tcl @echo "image: $@" diff --git a/hw/syn/foldsim/floorplan.tcl b/hw/syn/foldsim/floorplan.tcl new file mode 100644 index 0000000000..39a4ec11c4 --- /dev/null +++ b/hw/syn/foldsim/floorplan.tcl @@ -0,0 +1,30 @@ +# Applies foldsim's --slr-groups floorplan, $::env(FOLDSIM_FLOORPLAN), to the +# open synthesized design: each threaded copy kept whole in one SLR, the SLR +# left to the placer. A copy split over an SLR boundary pays a crossing on +# paths wholly inside it. Sourced before opt_design by ooc.tcl and by the +# image's implementation (pre_opt_hook.tcl); the Makefile sets the variable +# when FOLDSIM_FLAGS has --slr-groups. The assignments land in floorplan.txt. + +if {[info exists ::env(FOLDSIM_FLOORPLAN)] && $::env(FOLDSIM_FLOORPLAN) ne ""} { + set fs_floorplan $::env(FOLDSIM_FLOORPLAN) + if {![file exists $fs_floorplan]} { + error "foldsim floorplan not found: $fs_floorplan" + } + set fp [open $fs_floorplan r] + set fs_groups [regexp -all {USER_SLR_ASSIGNMENT} [read $fp]] + close $fp + source $fs_floorplan + set fs_cells [get_cells -quiet -hierarchical -filter {USER_SLR_ASSIGNMENT =~ "fs_*"}] + set fp [open floorplan.txt w] + foreach c $fs_cells { puts $fp "$c [get_property USER_SLR_ASSIGNMENT $c]" } + close $fp + # A floorplan that matches nothing would leave the copies to area-only + # placement without a trace in the reports. + if {[llength $fs_cells] == 0} { + error "foldsim floorplan $fs_floorplan matched no cell" + } + if {[llength $fs_cells] < $fs_groups} { + puts "CRITICAL WARNING: foldsim floorplan: [llength $fs_cells] of $fs_groups copies found" + } + puts "INFO: foldsim floorplan: [llength $fs_cells] threaded copies in SLR groups ($fs_floorplan)" +} diff --git a/hw/syn/foldsim/ooc.tcl b/hw/syn/foldsim/ooc.tcl index e6436114e3..2901cd62b7 100644 --- a/hw/syn/foldsim/ooc.tcl +++ b/hw/syn/foldsim/ooc.tcl @@ -62,6 +62,7 @@ synth_design -top foldsim_afu -part $part -mode out_of_context -include_dirs $in set t1 [clock seconds] report_utilization -file synth_util.rpt report_timing_summary -max_paths 20 -file synth_timing_summary.rpt +source [file join [file dirname [file normalize [info script]]] floorplan.tcl] opt_design place_design -directive Explore phys_opt_design -directive AggressiveExplore diff --git a/hw/syn/foldsim/pre_opt_hook.tcl b/hw/syn/foldsim/pre_opt_hook.tcl new file mode 100644 index 0000000000..8be17f2aa3 --- /dev/null +++ b/hw/syn/foldsim/pre_opt_hook.tcl @@ -0,0 +1,12 @@ +# Runs on impl_1 immediately before opt_design, attached by pre_synth_hook.tcl +# when the image has a floorplan: the native flow's pre-opt hook, then the +# floorplan. + +set fs_hook_dir [file dirname [file normalize [info script]]] +source [file join $fs_hook_dir .. xilinx aved pre_opt_hook.tcl] +# The variable reaches the run from the linker's environment; a run without +# it would silently place without the floorplan the image was built for. +if {![info exists ::env(FOLDSIM_FLOORPLAN)] || $::env(FOLDSIM_FLOORPLAN) eq ""} { + error "FOLDSIM_FLOORPLAN is not set in the implementation run" +} +source [file join $fs_hook_dir floorplan.tcl] diff --git a/hw/syn/foldsim/pre_synth_hook.tcl b/hw/syn/foldsim/pre_synth_hook.tcl index 957a75ba28..e63b565ae6 100644 --- a/hw/syn/foldsim/pre_synth_hook.tcl +++ b/hw/syn/foldsim/pre_synth_hook.tcl @@ -3,7 +3,15 @@ # The native flow's hook: the pre-opt utilization report and post-route # phys_opt_design -directive AggressiveExplore. -source [file join [file dirname [file normalize [info script]]] .. xilinx aved pre_synth_hook.tcl] +set fs_hook_dir [file dirname [file normalize [info script]]] +source [file join $fs_hook_dir .. xilinx aved pre_synth_hook.tcl] + +# An image with a floorplan (foldsim --slr-groups) applies it before +# opt_design, after the native pre-opt hook. +if {[info exists ::env(FOLDSIM_FLOORPLAN)] && $::env(FOLDSIM_FLOORPLAN) ne ""} { + set_property STEPS.OPT_DESIGN.TCL.PRE [file join $fs_hook_dir pre_opt_hook.tcl] [get_runs impl_1] + puts "INFO: foldsim floorplan $::env(FOLDSIM_FLOORPLAN) attached to impl_1" +} # The kernel's out-of-context synthesis sees its clock (the IP's OOC XDC) and # retimes against it: a threaded model's registered state is retimable. From 4d6d13e9a81ffe226bc13d57e9944f7c12e05a3f Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Wed, 28 Oct 2026 16:01:59 -0700 Subject: [PATCH 275/336] foldsim: the shell comes from Foldsim; Vortex keeps only its glue Foldsim is a general-purpose RTL simulator: the FPGA shell around the generated simulator has nothing specific to Vortex, and now lives in the Foldsim repository (shell/, installed as share/foldsim/shell; foldsim 46f4d0a). Removed here and taken from FOLDSIM_SHELL instead: hw/rtl/afu/foldsim/*.sv, foldsim_regs.vh -> shell/rtl sw/common/foldsim_regs.h -> shell/include sim/foldsim/foldsim_emul.{h,cpp} -> shell/include, shell/emul hw/unittest/foldsim_mem* (8 variants) -> shell/test hw/syn/foldsim/lift_boundaries.py -> bin/foldsim-lift config.mk sets FOLDSIM_PATH and FOLDSIM_SHELL (FOLDSIM_PATH moves there from hw/syn/common.mk, so sim/ and sw/ see it); hw/syn/common.mk adds FOLDSIM_LIFT_TOOL. Vortex keeps the target wrapper (hw/rtl/afu/foldsim/target), the folding choices and V80 image flow (hw/syn/foldsim), the in-process build with the Xilinx FP models (sim/foldsim), the runtime driver (sw/runtime/foldsim) and the parity case. sim/common/dram_bank_model.h stays for rtlsim's AXI_MEM_MODEL=bank; the shell's copy is normative. Tested: ci foldsim parity (demo, sgemm: Foldsim in Verilator vs rtlsim, cycle-exact) passes against the restructured Foldsim build. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 644b38ffe6bb66162fa3f36e5ee7d6d94581f284) Integration (phase 2): hw/unittest/Makefile keeps phase 2's mm_axi_arb, cp_llc_port and stream_arb tests; the foldsim_mem shell files this removes were at earlier states here (their intermediate edits were not replayed). --- CHANGELOG.md | 6 + config.mk.in | 6 + docs/assets/img/foldsim_architecture.svg | 2 +- docs/building_toolchain.md | 7 +- docs/designs/foldsim.md | 32 +- hw/rtl/afu/foldsim/foldsim_afu.sv | 368 -------- hw/rtl/afu/foldsim/foldsim_ctrl.sv | 367 -------- hw/rtl/afu/foldsim/foldsim_mem.sv | 470 --------- hw/rtl/afu/foldsim/foldsim_mem_bank.sv | 1098 ---------------------- hw/rtl/afu/foldsim/foldsim_mem_bmap.sv | 79 -- hw/rtl/afu/foldsim/foldsim_regs.vh | 67 -- hw/rtl/afu/foldsim/foldsim_token.sv | 81 -- hw/syn/common.mk | 5 +- hw/syn/foldsim/Makefile | 30 +- hw/syn/foldsim/lift_boundaries.py | 574 ----------- hw/unittest/Makefile | 9 - hw/unittest/foldsim_mem/Makefile | 24 - hw/unittest/foldsim_mem/main.cpp | 848 ----------------- hw/unittest/foldsim_mem_ch2/Makefile | 28 - hw/unittest/foldsim_mem_staged/Makefile | 27 - sim/common/dram_bank_model.h | 9 +- sim/foldsim/Makefile | 15 +- sim/foldsim/foldsim_emul.cpp | 416 -------- sim/foldsim/foldsim_emul.h | 45 - sw/common/foldsim_regs.h | 92 -- sw/runtime/foldsim/Makefile | 4 +- sw/runtime/foldsim/vortex.cpp | 4 +- 27 files changed, 80 insertions(+), 4633 deletions(-) delete mode 100644 hw/rtl/afu/foldsim/foldsim_afu.sv delete mode 100644 hw/rtl/afu/foldsim/foldsim_ctrl.sv delete mode 100644 hw/rtl/afu/foldsim/foldsim_mem.sv delete mode 100644 hw/rtl/afu/foldsim/foldsim_mem_bank.sv delete mode 100644 hw/rtl/afu/foldsim/foldsim_mem_bmap.sv delete mode 100644 hw/rtl/afu/foldsim/foldsim_regs.vh delete mode 100644 hw/rtl/afu/foldsim/foldsim_token.sv delete mode 100644 hw/syn/foldsim/lift_boundaries.py delete mode 100644 hw/unittest/foldsim_mem/Makefile delete mode 100644 hw/unittest/foldsim_mem/main.cpp delete mode 100644 hw/unittest/foldsim_mem_ch2/Makefile delete mode 100644 hw/unittest/foldsim_mem_staged/Makefile delete mode 100644 sim/foldsim/foldsim_emul.cpp delete mode 100644 sim/foldsim/foldsim_emul.h delete mode 100644 sw/common/foldsim_regs.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ee45652e5..59407123e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,12 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, ### Changed +- **Foldsim's shell lives in the Foldsim repository.** The control and memory endpoints, the DRAM bank timing model, the register + map, the in-process host, the endpoint's unit tests and the boundary lifting (`foldsim-lift`) moved out of `hw/rtl/afu/foldsim`, + `sim/foldsim`, `sw/common`, `hw/unittest/foldsim_mem*` and `hw/syn/foldsim`; they come with the Foldsim installation + (`FOLDSIM_SHELL`, `$(FOLDSIM_PATH)/share/foldsim/shell`). Vortex keeps the target wrapper, the image flow, the runtime driver and + the parity case. *Why:* Foldsim is a general-purpose RTL simulator; nothing in the shell is specific to Vortex. + - **RV64 portability.** Device index math, DXA/OM addresses, pack-load strides, RTU scene/payload pointers and the ray log carry full width on RV64 (`size_t`/`uintptr_t`, identical RV32 code); RTU slot reads into FP registers are NaN-boxed. - **Zero warnings restored.** Every C/C++ build (simulators, runtime, kernel library, tests, OpenCL/HIP/Vulkan apps) is `-Werror` and diff --git a/config.mk.in b/config.mk.in index dad8cccf06..d2987da576 100644 --- a/config.mk.in +++ b/config.mk.in @@ -44,6 +44,12 @@ RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv$(XLEN)-gnu-toolchain FIRESIM_PATH ?= $(TOOLDIR)/firesim # vortexgpgpu/firesim commit this tree is built and tested against (VERSION). FIRESIM_REV := @FIRESIM_REV@ +# A Foldsim installation (ci/foldsim_install.sh): the compiler that builds a +# FAME-5 simulator of Vortex (hw/syn/foldsim), and the FPGA shell the simulator +# runs in, which the image, the in-process simulator (sim/foldsim) and the +# runtime (sw/runtime/foldsim) all take from it. +FOLDSIM_PATH ?= $(TOOLDIR)/foldsim +FOLDSIM_SHELL ?= $(FOLDSIM_PATH)/share/foldsim/shell RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf RISCV_SYSROOT ?= $(RISCV_TOOLCHAIN_PATH)/$(RISCV_PREFIX) diff --git a/docs/assets/img/foldsim_architecture.svg b/docs/assets/img/foldsim_architecture.svg index 649c1349c2..43226ed218 100644 --- a/docs/assets/img/foldsim_architecture.svg +++ b/docs/assets/img/foldsim_architecture.svg @@ -45,7 +45,7 @@ - Run: the kernel (hw/rtl/afu/foldsim) and its host + Run: Foldsim's shell (share/foldsim/shell) and its host foldsim_afu diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index 63b5069b75..2302218fa6 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -1215,8 +1215,11 @@ cmake -G Ninja -S $FOLDSIM_SRC -B $FOLDSIM_SRC/build \ ### Notes - `foldsim-opt` links CIRCT statically; the driver finds it next to itself - and the CIRCT tools through `--circt-bin`. `hw/syn/common.mk` sets - `FOLDSIM_PATH ?= $(TOOLDIR)/foldsim` and `FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim`. + and the CIRCT tools through `--circt-bin`. `config.mk` sets + `FOLDSIM_PATH ?= $(TOOLDIR)/foldsim` and `FOLDSIM_SHELL ?= + $(FOLDSIM_PATH)/share/foldsim/shell` (the FPGA shell the simulator runs in); + `hw/syn/common.mk` sets `FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim` and + `FOLDSIM_LIFT_TOOL ?= $(FOLDSIM_PATH)/bin/foldsim-lift`. - A CIRCT change Foldsim's passes rely on lands in CIRCT first; both prebuilts are refreshed together. - The prebuilt is opt-in: `./ci/toolchain_install.sh --foldsim` (or `--all`) diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 0dcfdc67aa..09ee21d3da 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -33,6 +33,22 @@ that builds on CIRCT, as POCL builds on LLVM: the `foldsim` driver and `foldsim-opt` come from the toolchain (`$(TOOLDIR)/foldsim`), CIRCT from `$(TOOLDIR)/circt`. Nothing else is in the flow: no Yosys, no sv2v. +Foldsim is a general-purpose RTL simulator: everything that is not specific +to Vortex is in the Foldsim repository. That is the compiler, the boundary +lifting (`foldsim-lift`), and the FPGA shell the simulator runs in (control +and memory endpoints, the DRAM bank timing model, the register map, the +in-process host and the endpoint's unit tests), installed as +`$(FOLDSIM_PATH)/share/foldsim/shell` (`FOLDSIM_SHELL`). Vortex keeps only its +glue: + +| Vortex path | what | +|---|---| +| `hw/rtl/afu/foldsim/target/foldsim_target.sv` | Vortex with its AXI4 ports, DCR, start and busy as the shell's plain target ports | +| `hw/syn/foldsim` | which units to lift and fold, and the V80 image flow on the native aved flow | +| `sim/foldsim` | the shell's in-process host built for a Vortex configuration, with the Xilinx FP cores' models | +| `sw/runtime/foldsim` | the Vortex runtime driver over the shell's registers | +| `ci/testcases/foldsim.yaml` | the cycle-exact parity check against rtlsim | + ```bash export FOLDSIM_CLASSES="VX_socket__lift VX_core__lift VX_compute_slice__lift" export FOLDSIM_LIFT="VX_socket:drop=INSTANCE_ID VX_core:drop=INSTANCE_ID VX_compute_slice:drop=INSTANCE_ID,SLICE_STR" @@ -46,13 +62,14 @@ FPGA_BIN_DIR=$PWD/hw/syn/foldsim/build32_foldsim_hw/bin CONFIGS="$CONFIGS" \ `FOLDSIM_CLASSES` names the modules threaded as classes; a class may contain others (the socket contains the issue slices). `FOLDSIM_LIFT` gives interface-port units a plain-port boundary first, so they can be classes -(`hw/syn/foldsim/lift_boundaries.py`, a source-to-source pass on pyslang; the +(Foldsim's `foldsim-lift`, a source-to-source pass on pyslang; the RTL is not changed). `drop=P` removes a parameter that only names an instance (`INSTANCE_ID`), so all instances of the class elaborate to one module; instance IDs that reach logic are already input ports in the RTL (the socket's `socket_id`). `port=P` turns such a parameter into an input port where a unit still has one. -The tools are found through `FOLDSIM` (the driver) and -`CIRCT_BIN`, by default `hw/syn/common.mk`'s `FOLDSIM_PATH` and `CIRCT_PATH`. +The tools are found through `FOLDSIM` (the driver), `FOLDSIM_SHELL` and +`CIRCT_BIN`, by default under `config.mk`'s `FOLDSIM_PATH` and +`hw/syn/common.mk`'s `CIRCT_PATH`. A class `A+B` fuses two modules of the same parent (`foldsim-fuse-models`): each `A` is paired with the one `B` it exchanges bits with, and each pair is @@ -209,7 +226,7 @@ chosen per image from a Verilator run. ## 4. The kernel -`hw/rtl/afu/foldsim/foldsim_afu.sv` has the native V80 kernel's ports but for +The shell's `rtl/foldsim_afu.sv` has the native V80 kernel's ports but for the host-memory master: `s_axi_control` and one device-memory master `m_axi_mem_

` per memory port of the target, each linked to the `MEM` HBM aperture (the linker gives each its own way in). The target has as many @@ -345,7 +362,7 @@ addresses. `TARGET=hw` uses VRT — the kernel's registers through the same kernel in Verilator (`sim/foldsim`), with a sparse device memory and each register access costing the kernel cycles a PCIe access would. -The register map is `sw/common/foldsim_regs.h` (`hw/rtl/afu/foldsim/foldsim_regs.vh` +The register map is the shell's `include/foldsim_regs.h` (`rtl/foldsim_regs.vh` for the RTL): | offset | register | @@ -399,7 +416,8 @@ advancing for that time (60 s when it is unset). in Verilator): random target inputs, every channel stalled at random, every output token compared. s2w4i2: 3,000 target cycles, 51,000 tokens, 0 differences. -- **The memory endpoint** (`hw/unittest/foldsim_mem`, in Verilator): an +- **The memory endpoint** (the shell's `test`, `ninja check-foldsim-shell` in + the Foldsim build, in Verilator): an AXI4 master as the target, HBM with random ready and latency checking the device-side protocol, resets at random points with HBM busy, and each kind of burst; every target cycle's outputs compared with the C++ model's @@ -408,7 +426,7 @@ advancing for that time (60 s when it is unset). h100 rates; small queues; row hits, random rows, one bank to the horizon, streams, read- and write-heavy) and five with refresh (per bank and all banks, at the standard intervals and faster), on the endpoint alone, with - staged token ends, and as one of two ports (`foldsim_mem_ch2`): 0 + staged token ends, with speculative reads, and as one of two ports (`ch2`): 0 differences. - **Refresh against its interval** (`tests/unittest/dram_bank_model refresh`): one refresh per interval, to the operation, at every memory and diff --git a/hw/rtl/afu/foldsim/foldsim_afu.sv b/hw/rtl/afu/foldsim/foldsim_afu.sv deleted file mode 100644 index db92addcd5..0000000000 --- a/hw/rtl/afu/foldsim/foldsim_afu.sv +++ /dev/null @@ -1,368 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "foldsim_sim.vh" -`include "foldsim_kernel.vh" - -`define FOLDSIM_COMMA , -`define FOLDSIM_SEMI ; - -// The foldsim kernel: the simulator CIRCT composes of Vortex (foldsim_sim, of -// the target foldsim_target), a memory endpoint on each of its AXI4 ports and -// the control endpoint on its DCR, start and busy signals. Its ports are the -// native V80 kernel's, but for the host-memory master it does not need: one -// device-memory master per memory port of the target (FOLDSIM_MEM_PORTS, from -// foldsim_kernel.vh, with the ports' id width), each its own way into device -// memory. -module foldsim_afu #( - parameter C_S_AXI_CONTROL_ADDR_WIDTH = 16, - parameter C_S_AXI_CONTROL_DATA_WIDTH = 32, - parameter C_M_AXI_MEM_ID_WIDTH = 1, - parameter C_M_AXI_MEM_DATA_WIDTH = 512, - parameter C_M_AXI_MEM_ADDR_WIDTH = 64, - // The target's AXI4 ports. - parameter TARGET_ID_W = `FOLDSIM_MEM_ID_W, - parameter TARGET_ADDR_W = 64, - parameter DCR_ADDR_W = 12, - parameter DCR_DATA_W = 32 -) ( - input wire ap_clk, - input wire ap_rst_n, - -`define FOLDSIM_AXI_MEM(i) \ - output wire m_axi_mem_``i``_awvalid, \ - input wire m_axi_mem_``i``_awready, \ - output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_``i``_awaddr, \ - output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_``i``_awid, \ - output wire [7:0] m_axi_mem_``i``_awlen, \ - output wire m_axi_mem_``i``_wvalid, \ - input wire m_axi_mem_``i``_wready, \ - output wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_``i``_wdata, \ - output wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_``i``_wstrb, \ - output wire m_axi_mem_``i``_wlast, \ - output wire m_axi_mem_``i``_arvalid, \ - input wire m_axi_mem_``i``_arready, \ - output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_``i``_araddr, \ - output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_``i``_arid, \ - output wire [7:0] m_axi_mem_``i``_arlen, \ - input wire m_axi_mem_``i``_rvalid, \ - output wire m_axi_mem_``i``_rready, \ - input wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_``i``_rdata, \ - input wire m_axi_mem_``i``_rlast, \ - input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_``i``_rid, \ - input wire [1:0] m_axi_mem_``i``_rresp, \ - input wire m_axi_mem_``i``_bvalid, \ - output wire m_axi_mem_``i``_bready, \ - input wire [1:0] m_axi_mem_``i``_bresp, \ - input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_``i``_bid - - `FOLDSIM_MEM_REPEAT(FOLDSIM_AXI_MEM, FOLDSIM_COMMA), - - input wire s_axi_control_awvalid, - output wire s_axi_control_awready, - input wire [C_S_AXI_CONTROL_ADDR_WIDTH-1:0] s_axi_control_awaddr, - input wire s_axi_control_wvalid, - output wire s_axi_control_wready, - input wire [C_S_AXI_CONTROL_DATA_WIDTH-1:0] s_axi_control_wdata, - input wire [C_S_AXI_CONTROL_DATA_WIDTH/8-1:0] s_axi_control_wstrb, - input wire s_axi_control_arvalid, - output wire s_axi_control_arready, - input wire [C_S_AXI_CONTROL_ADDR_WIDTH-1:0] s_axi_control_araddr, - output wire s_axi_control_rvalid, - input wire s_axi_control_rready, - output wire [C_S_AXI_CONTROL_DATA_WIDTH-1:0] s_axi_control_rdata, - output wire [1:0] s_axi_control_rresp, - output wire s_axi_control_bvalid, - input wire s_axi_control_bready, - output wire [1:0] s_axi_control_bresp -); -`undef FOLDSIM_AXI_MEM - localparam DATA_W = C_M_AXI_MEM_DATA_WIDTH; - localparam PORTS = `FOLDSIM_MEM_PORTS; - - // A registered reset, so it reaches the whole kernel in one cycle's time. - reg reset; - always @(posedge ap_clk) - reset <= ~ap_rst_n; - - // ---- The simulator's channels, named after the target's ports ---- -`define FOLDSIM_CH(name, w) \ - wire name``_valid, name``_ready; \ - wire [w-1:0] name``_bits; - - `FOLDSIM_CH(reset, 1) - `FOLDSIM_CH(dcr_req_valid, 1) - `FOLDSIM_CH(dcr_req_rw, 1) - `FOLDSIM_CH(dcr_req_addr, DCR_ADDR_W) - `FOLDSIM_CH(dcr_req_data, DCR_DATA_W) - `FOLDSIM_CH(start, 1) - `FOLDSIM_CH(dcr_rsp_valid, 1) - `FOLDSIM_CH(dcr_rsp_data, DCR_DATA_W) - `FOLDSIM_CH(busy, 1) -`undef FOLDSIM_CH - - // A memory port's channels, by port. -`define FOLDSIM_MCH(name, w) \ - wire mem_``name``_valid [PORTS]; \ - wire mem_``name``_ready [PORTS]; \ - wire [w-1:0] mem_``name``_bits [PORTS]; - - `FOLDSIM_MCH(awready, 1) - `FOLDSIM_MCH(wready, 1) - `FOLDSIM_MCH(arready, 1) - `FOLDSIM_MCH(rvalid, 1) - `FOLDSIM_MCH(rdata, DATA_W) - `FOLDSIM_MCH(rlast, 1) - `FOLDSIM_MCH(rid, TARGET_ID_W) - `FOLDSIM_MCH(rresp, 2) - `FOLDSIM_MCH(bvalid, 1) - `FOLDSIM_MCH(bresp, 2) - `FOLDSIM_MCH(bid, TARGET_ID_W) - `FOLDSIM_MCH(awvalid, 1) - `FOLDSIM_MCH(awaddr, TARGET_ADDR_W) - `FOLDSIM_MCH(awid, TARGET_ID_W) - `FOLDSIM_MCH(awlen, 8) - `FOLDSIM_MCH(wvalid, 1) - `FOLDSIM_MCH(wdata, DATA_W) - `FOLDSIM_MCH(wstrb, DATA_W/8) - `FOLDSIM_MCH(wlast, 1) - `FOLDSIM_MCH(arvalid, 1) - `FOLDSIM_MCH(araddr, TARGET_ADDR_W) - `FOLDSIM_MCH(arid, TARGET_ID_W) - `FOLDSIM_MCH(arlen, 8) - `FOLDSIM_MCH(rready, 1) - `FOLDSIM_MCH(bready, 1) -`undef FOLDSIM_MCH - - // Device memory, by port. - wire hbm_awvalid [PORTS], hbm_awready [PORTS]; - wire [63:0] hbm_awaddr [PORTS]; - wire [7:0] hbm_awlen [PORTS]; - wire hbm_wvalid [PORTS], hbm_wready [PORTS], hbm_wlast [PORTS]; - wire [DATA_W-1:0] hbm_wdata [PORTS]; - wire [DATA_W/8-1:0] hbm_wstrb [PORTS]; - wire hbm_bvalid [PORTS], hbm_bready [PORTS]; - wire hbm_arvalid [PORTS], hbm_arready [PORTS]; - wire [63:0] hbm_araddr [PORTS]; - wire [7:0] hbm_arlen [PORTS]; - wire hbm_rvalid [PORTS], hbm_rready [PORTS]; - wire [DATA_W-1:0] hbm_rdata [PORTS]; - -`define FOLDSIM_HBM(i) \ - assign m_axi_mem_``i``_awvalid = hbm_awvalid[i]; \ - assign hbm_awready[i] = m_axi_mem_``i``_awready; \ - assign m_axi_mem_``i``_awaddr = hbm_awaddr[i]; \ - assign m_axi_mem_``i``_awid = '0; \ - assign m_axi_mem_``i``_awlen = hbm_awlen[i]; \ - assign m_axi_mem_``i``_wvalid = hbm_wvalid[i]; \ - assign hbm_wready[i] = m_axi_mem_``i``_wready; \ - assign m_axi_mem_``i``_wdata = hbm_wdata[i]; \ - assign m_axi_mem_``i``_wstrb = hbm_wstrb[i]; \ - assign m_axi_mem_``i``_wlast = hbm_wlast[i]; \ - assign hbm_bvalid[i] = m_axi_mem_``i``_bvalid; \ - assign m_axi_mem_``i``_bready = hbm_bready[i]; \ - assign m_axi_mem_``i``_arvalid = hbm_arvalid[i]; \ - assign hbm_arready[i] = m_axi_mem_``i``_arready; \ - assign m_axi_mem_``i``_araddr = hbm_araddr[i]; \ - assign m_axi_mem_``i``_arid = '0; \ - assign m_axi_mem_``i``_arlen = hbm_arlen[i]; \ - assign hbm_rvalid[i] = m_axi_mem_``i``_rvalid; \ - assign m_axi_mem_``i``_rready = hbm_rready[i]; \ - assign hbm_rdata[i] = m_axi_mem_``i``_rdata - - `FOLDSIM_MEM_REPEAT(FOLDSIM_HBM, FOLDSIM_SEMI); -`undef FOLDSIM_HBM - -`define FOLDSIM_IN(name) \ - .name``_valid (name``_valid), .name``_ready (name``_ready), .name``_bits (name``_bits) -`define FOLDSIM_MIN(i, name) \ - .m_axi_mem_``i``_``name``_valid (mem_``name``_valid[i]), \ - .m_axi_mem_``i``_``name``_ready (mem_``name``_ready[i]), \ - .m_axi_mem_``i``_``name``_bits (mem_``name``_bits[i]) -`define FOLDSIM_SIM_MEM(i) \ - `FOLDSIM_MIN(i, awready), `FOLDSIM_MIN(i, wready), `FOLDSIM_MIN(i, arready), \ - `FOLDSIM_MIN(i, rvalid), `FOLDSIM_MIN(i, rdata), `FOLDSIM_MIN(i, rlast), `FOLDSIM_MIN(i, rid), \ - `FOLDSIM_MIN(i, rresp), `FOLDSIM_MIN(i, bvalid), `FOLDSIM_MIN(i, bresp), `FOLDSIM_MIN(i, bid), \ - `FOLDSIM_MIN(i, awvalid), `FOLDSIM_MIN(i, awaddr), `FOLDSIM_MIN(i, awid), `FOLDSIM_MIN(i, awlen), \ - `FOLDSIM_MIN(i, wvalid), `FOLDSIM_MIN(i, wdata), `FOLDSIM_MIN(i, wstrb), `FOLDSIM_MIN(i, wlast), \ - `FOLDSIM_MIN(i, arvalid), `FOLDSIM_MIN(i, araddr), `FOLDSIM_MIN(i, arid), `FOLDSIM_MIN(i, arlen), \ - `FOLDSIM_MIN(i, rready), `FOLDSIM_MIN(i, bready) - - wire [`FOLDSIM_NUM_CLASSES-1:0] profile_finish /*verilator public_flat_rd*/; - - foldsim_sim sim ( - .hostClock (ap_clk), - .hostReset (reset), - .profile_finish (profile_finish), - `FOLDSIM_MEM_REPEAT(FOLDSIM_SIM_MEM, FOLDSIM_COMMA), - `FOLDSIM_IN(reset), - `FOLDSIM_IN(dcr_req_valid), - `FOLDSIM_IN(dcr_req_rw), - `FOLDSIM_IN(dcr_req_addr), - `FOLDSIM_IN(dcr_req_data), - `FOLDSIM_IN(start), - `FOLDSIM_IN(dcr_rsp_valid), - `FOLDSIM_IN(dcr_rsp_data), - `FOLDSIM_IN(busy) - ); -`undef FOLDSIM_SIM_MEM -`undef FOLDSIM_MIN - - // Each memory endpoint's bank model: read slots, and a ring of 2^9 DRAM - // clocks (MEM_CAPS reports both). - localparam MEM_SLOTS = 32; - localparam MEM_HLOG2 = 9; - // The memory endpoints' token ends register have and done, so their finish - // starts at their own state, not at the simulator's handshake. A token then - // counts a host cycle later; s2w4i2's host cycles do not move. - localparam MEM_STAGED = 1; - // Ports, engines and device-memory masters: one of each per port. - localparam [31:0] MEM_CAPS = {4'd0, 4'(PORTS), 4'(PORTS), 8'(MEM_SLOTS), 4'(MEM_HLOG2 - 4), 8'(PORTS)}; - - wire [63:0] mem_base; - wire [31:0] mem_clock, mem_queue; - wire [15:0] mem_latency, mem_wmark; - wire [12:0] mem_geom; - wire [18*12-1:0] mem_timing; - wire [PORTS*8*64-1:0] mem_stats; - wire [1:0] mem_refresh; - wire [15:0] mem_refi, mem_rfc; - wire [PORTS-1:0] mem_idle, mem_error, mem_stall; - wire [PORTS-1:0] mem_reset_valid, mem_reset_ready, mem_reset_bits; - -`define FOLDSIM_MEM(name) \ - .name``_valid (mem_``name``_valid[p]), .name``_ready (mem_``name``_ready[p]), \ - .name``_bits (mem_``name``_bits[p]) - - for (genvar p = 0; p < PORTS; ++p) begin : g_mem - foldsim_mem #( - .ID_W (TARGET_ID_W), - .ADDR_W (TARGET_ADDR_W), - .DATA_W (DATA_W), - .SLOTS (MEM_SLOTS), - .HLOG2 (MEM_HLOG2), - .STAGED (MEM_STAGED), - .CH_SHIFT (`FOLDSIM_MEM_CH_SHIFT) - ) mem ( - .clk (ap_clk), - .reset (reset), - .base (mem_base), - .cfg_data_rate (mem_clock[15:0]), - .cfg_core_clock(mem_clock[31:16]), - .cfg_latency (mem_latency), - .cfg_queue (mem_queue[15:0]), - .cfg_rq (mem_queue[23:16]), - .cfg_wq (mem_queue[31:24]), - .cfg_wr_high (mem_wmark[7:0]), - .cfg_wr_low (mem_wmark[15:8]), - .cfg_geom (mem_geom), - .cfg_t (mem_timing), - .cfg_refresh (mem_refresh), - .cfg_refi (mem_refi), - .cfg_rfc (mem_rfc), - .stats (mem_stats[p*8*64 +: 8*64]), - .idle (mem_idle[p]), - .error (mem_error[p]), - .stall (mem_stall[p]), - .treset_valid (mem_reset_valid[p]), - .treset_ready (mem_reset_ready[p]), - .treset_bits (mem_reset_bits[p]), - `FOLDSIM_MEM(awvalid), `FOLDSIM_MEM(awaddr), `FOLDSIM_MEM(awid), `FOLDSIM_MEM(awlen), - `FOLDSIM_MEM(wvalid), `FOLDSIM_MEM(wdata), `FOLDSIM_MEM(wstrb), `FOLDSIM_MEM(wlast), - `FOLDSIM_MEM(arvalid), `FOLDSIM_MEM(araddr), `FOLDSIM_MEM(arid), `FOLDSIM_MEM(arlen), - `FOLDSIM_MEM(rready), `FOLDSIM_MEM(bready), - `FOLDSIM_MEM(awready), `FOLDSIM_MEM(wready), `FOLDSIM_MEM(arready), - `FOLDSIM_MEM(rvalid), `FOLDSIM_MEM(rdata), `FOLDSIM_MEM(rlast), `FOLDSIM_MEM(rid), `FOLDSIM_MEM(rresp), - `FOLDSIM_MEM(bvalid), `FOLDSIM_MEM(bresp), `FOLDSIM_MEM(bid), - .m_awvalid (hbm_awvalid[p]), - .m_awready (hbm_awready[p]), - .m_awaddr (hbm_awaddr[p]), - .m_awlen (hbm_awlen[p]), - .m_wvalid (hbm_wvalid[p]), - .m_wready (hbm_wready[p]), - .m_wdata (hbm_wdata[p]), - .m_wstrb (hbm_wstrb[p]), - .m_wlast (hbm_wlast[p]), - .m_bvalid (hbm_bvalid[p]), - .m_bready (hbm_bready[p]), - .m_arvalid (hbm_arvalid[p]), - .m_arready (hbm_arready[p]), - .m_araddr (hbm_araddr[p]), - .m_arlen (hbm_arlen[p]), - .m_rvalid (hbm_rvalid[p]), - .m_rready (hbm_rready[p]), - .m_rdata (hbm_rdata[p]) - ); - end -`undef FOLDSIM_MEM - - foldsim_ctrl #( - .DCR_ADDR_W (DCR_ADDR_W), - .DCR_DATA_W (DCR_DATA_W), - .NUM_CLASSES (`FOLDSIM_NUM_CLASSES), - .MEM_PORTS (PORTS), - .MEM_CAPS (MEM_CAPS) - ) ctrl ( - .clk (ap_clk), - .reset (reset), - .s_awvalid (s_axi_control_awvalid), - .s_awready (s_axi_control_awready), - .s_awaddr (s_axi_control_awaddr[11:0]), - .s_wvalid (s_axi_control_wvalid), - .s_wready (s_axi_control_wready), - .s_wdata (s_axi_control_wdata), - .s_bvalid (s_axi_control_bvalid), - .s_bready (s_axi_control_bready), - .s_arvalid (s_axi_control_arvalid), - .s_arready (s_axi_control_arready), - .s_araddr (s_axi_control_araddr[11:0]), - .s_rvalid (s_axi_control_rvalid), - .s_rready (s_axi_control_rready), - .s_rdata (s_axi_control_rdata), - .mem_base (mem_base), - .mem_clock (mem_clock), - .mem_latency (mem_latency), - .mem_queue (mem_queue), - .mem_geom (mem_geom), - .mem_wmark (mem_wmark), - .mem_timing (mem_timing), - .mem_refresh (mem_refresh), - .mem_refi (mem_refi), - .mem_rfc (mem_rfc), - .mem_stats (mem_stats), - .mem_idle (&mem_idle), - .mem_error (|mem_error), - .mem_stall (|mem_stall), - .profile_finish (profile_finish), - .mem_reset_valid (mem_reset_valid), - .mem_reset_ready (mem_reset_ready), - .mem_reset_bits (mem_reset_bits), - `FOLDSIM_IN(reset), - `FOLDSIM_IN(dcr_req_valid), - `FOLDSIM_IN(dcr_req_rw), - `FOLDSIM_IN(dcr_req_addr), - `FOLDSIM_IN(dcr_req_data), - `FOLDSIM_IN(start), - `FOLDSIM_IN(dcr_rsp_valid), - `FOLDSIM_IN(dcr_rsp_data), - `FOLDSIM_IN(busy) - ); - assign s_axi_control_rresp = 2'b00; - assign s_axi_control_bresp = 2'b00; - -`undef FOLDSIM_IN - -endmodule - -`undef FOLDSIM_COMMA -`undef FOLDSIM_SEMI diff --git a/hw/rtl/afu/foldsim/foldsim_ctrl.sv b/hw/rtl/afu/foldsim/foldsim_ctrl.sv deleted file mode 100644 index ee865f9251..0000000000 --- a/hw/rtl/afu/foldsim/foldsim_ctrl.sv +++ /dev/null @@ -1,367 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "foldsim_regs.vh" - -// The host's view of a foldsim simulator: an AXI4-Lite register block, and the -// endpoint that drives the target's control inputs from it. -// -// The target runs free: the endpoint gives it a token every cycle. The host -// queues commands -- a DCR write or read, a start, a reset -- and each is -// applied to the target for one cycle (a reset for its count of cycles), in -// order, from the cycle after the queue reaches it. The memory endpoint gets -// the target's reset for each target cycle too, and resets its side of the -// target's memory port with it (foldsim_mem). A DCR read's response is -// kept for the host; `busy` covers everything queued or in flight, so a host -// that sees the target idle has every effect of what it asked for. -// -// See foldsim_regs.vh for the register map. -module foldsim_ctrl #( - parameter DCR_ADDR_W = 12, - parameter DCR_DATA_W = 32, - parameter QUEUE = 16, - parameter RESET_CYCLES = 64, - parameter NUM_CLASSES = 1, - // The memory endpoints, one per memory port of the target. - parameter MEM_PORTS = 1, - // MEM_CAPS: what the memory endpoints' bank models were built with. - parameter MEM_CAPS = 32'd0 -) ( - input wire clk, - input wire reset, - - input wire s_awvalid, - output wire s_awready, - input wire [11:0] s_awaddr, - input wire s_wvalid, - output wire s_wready, - input wire [31:0] s_wdata, - output reg s_bvalid, - input wire s_bready, - input wire s_arvalid, - output wire s_arready, - input wire [11:0] s_araddr, - output reg s_rvalid, - input wire s_rready, - output reg [31:0] s_rdata, - - // The memory endpoints' settings and state: where the target's memory - // is, and the bank model's settings and counters (foldsim_mem_bank). - output reg [63:0] mem_base, - output reg [31:0] mem_clock, - output reg [15:0] mem_latency, - output reg [31:0] mem_queue, - output reg [12:0] mem_geom, - output reg [15:0] mem_wmark, - output wire [18*12-1:0] mem_timing, - // Refresh: [0] on, [1] all banks; its interval and busy time (clocks). - output wire [1:0] mem_refresh, - output reg [15:0] mem_refi, - output reg [15:0] mem_rfc, - // Per memory port, its counters. - input wire [MEM_PORTS*8*64-1:0] mem_stats, - input wire mem_idle, - input wire mem_error, - input wire mem_stall, - // Which classes of the simulator's models finish a thread this cycle. - input wire [NUM_CLASSES-1:0] profile_finish, - // The target's reset, a token per target cycle for each memory endpoint. - output wire [MEM_PORTS-1:0] mem_reset_valid, input wire [MEM_PORTS-1:0] mem_reset_ready, - output wire [MEM_PORTS-1:0] mem_reset_bits, - - // Target inputs, as channels of the simulator: the endpoint's outputs. - output wire reset_valid, input wire reset_ready, output wire reset_bits, - output wire dcr_req_valid_valid, input wire dcr_req_valid_ready, output wire dcr_req_valid_bits, - output wire dcr_req_rw_valid, input wire dcr_req_rw_ready, output wire dcr_req_rw_bits, - output wire dcr_req_addr_valid, input wire dcr_req_addr_ready, output wire [DCR_ADDR_W-1:0] dcr_req_addr_bits, - output wire dcr_req_data_valid, input wire dcr_req_data_ready, output wire [DCR_DATA_W-1:0] dcr_req_data_bits, - output wire start_valid, input wire start_ready, output wire start_bits, - - // Target outputs: the endpoint's inputs. - input wire dcr_rsp_valid_valid, output wire dcr_rsp_valid_ready, input wire dcr_rsp_valid_bits, - input wire dcr_rsp_data_valid, output wire dcr_rsp_data_ready, input wire [DCR_DATA_W-1:0] dcr_rsp_data_bits, - input wire busy_valid, output wire busy_ready, input wire busy_bits -); - localparam QW = $clog2(QUEUE); - localparam QC = $clog2(QUEUE + 1); - - // ---- Commands ---- - reg [2:0] q_op [QUEUE]; - reg [DCR_ADDR_W-1:0] q_addr [QUEUE]; - reg [DCR_DATA_W-1:0] q_data [QUEUE]; - reg [QW-1:0] q_head, q_tail; - reg [QC-1:0] q_count; - reg [DCR_ADDR_W-1:0] cmd_addr; - reg [DCR_DATA_W-1:0] cmd_data; - // The command applied this target cycle, taken from the queue when the - // previous cycle finished: the target's control tokens are functions of it - // alone, so they do not change between the target taking one and the - // endpoint finishing the cycle. - reg [2:0] cur_op; - reg [DCR_ADDR_W-1:0] cur_addr; - reg [DCR_DATA_W-1:0] cur_data; - - reg run; - reg [31:0] reset_count; - reg start_pending; - reg [15:0] start_wait; - reg target_busy; - reg rsp_valid; - reg [DCR_DATA_W-1:0] rsp_data; - reg [63:0] target_cycles, host_cycles, mem_stalls; - reg [47:0] finishes [NUM_CLASSES]; - reg [1:0] mem_refresh_r; - reg [11:0] mem_t [18]; - - // The refresh in effect from a reset on: the memory endpoint takes it at - // the end of every reset cycle, so a reset command's own cycle has it - // already. - assign mem_refresh = (cur_op == `FOLDSIM_CMD_RESET) ? cur_addr[1:0] : mem_refresh_r; - for (genvar i = 0; i < 18; ++i) begin : g_mem_t - assign mem_timing[i*12 +: 12] = mem_t[i]; - end - // The default machine's DRAM (HBM2 at 800 MT/s behind a 400 MHz core), - // until the runtime sets its own: tBL, tCCD_S, tCCD_L, tCL, tCWL, - // tRCD_RD, tRCD_WR, tRP, tRAS, tRC, tRTP, tWR, tRRD_S, tRRD_L, tFAW, - // tWTR_S, tWTR_L, read-to-write extra. - localparam [18*12-1:0] MEM_T_RESET = {12'd2, 12'd2, 12'd2, 12'd6, 12'd2, 12'd2, 12'd5, 12'd2, - 12'd19, 12'd14, 12'd6, 12'd6, 12'd6, 12'd1, 12'd6, 12'd4, - 12'd2, 12'd2}; - - wire q_empty = (q_count == 0); - - // ---- The target's control inputs this cycle ---- - wire in_reset = (reset_count != 0) | (cur_op == `FOLDSIM_CMD_RESET); - wire dcr_valid = ~in_reset & ((cur_op == `FOLDSIM_CMD_DCR_WRITE) | (cur_op == `FOLDSIM_CMD_DCR_READ)); - wire dcr_rw = (cur_op == `FOLDSIM_CMD_DCR_WRITE); - wire do_start = ~in_reset & (cur_op == `FOLDSIM_CMD_START); - // Reset cycles left after this one; a reset command lasts its count of - // cycles, at least one. - wire [31:0] reset_next = (reset_count != 0) ? reset_count - 32'd1 - : ((cur_op == `FOLDSIM_CMD_RESET) && (cur_data > 1)) ? 32'(cur_data) - 32'd1 - : 32'd0; - - wire finish; - wire [6+MEM_PORTS-1:0] done; - wire [2:0] have; - foldsim_token_out #(1) o_reset (clk, reset, reset_valid, reset_ready, run, finish, done[0]); - foldsim_token_out #(1) o_dvalid (clk, reset, dcr_req_valid_valid, dcr_req_valid_ready, run, finish, done[1]); - foldsim_token_out #(1) o_drw (clk, reset, dcr_req_rw_valid, dcr_req_rw_ready, run, finish, done[2]); - foldsim_token_out #(1) o_daddr (clk, reset, dcr_req_addr_valid, dcr_req_addr_ready, run, finish, done[3]); - foldsim_token_out #(1) o_ddata (clk, reset, dcr_req_data_valid, dcr_req_data_ready, run, finish, done[4]); - foldsim_token_out #(1) o_start (clk, reset, start_valid, start_ready, run, finish, done[5]); - for (genvar p = 0; p < MEM_PORTS; ++p) begin : g_mreset - foldsim_token_out #(1) o_mreset (clk, reset, mem_reset_valid[p], mem_reset_ready[p], run, finish, done[6+p]); - assign mem_reset_bits[p] = in_reset; - end - assign reset_bits = in_reset; - assign dcr_req_valid_bits = dcr_valid; - assign dcr_req_rw_bits = dcr_rw; - assign dcr_req_addr_bits = cur_addr; - assign dcr_req_data_bits = cur_data; - assign start_bits = do_start; - - wire rsp_v, busy; - wire [DCR_DATA_W-1:0] rsp_d; - foldsim_token_in #(1, 1) t_rspv (clk, reset, dcr_rsp_valid_valid, dcr_rsp_valid_ready, dcr_rsp_valid_bits, finish, have[0], rsp_v); - foldsim_token_in #(DCR_DATA_W, 1) t_rspd (clk, reset, dcr_rsp_data_valid, dcr_rsp_data_ready, dcr_rsp_data_bits, finish, have[1], rsp_d); - foldsim_token_in #(1, 1) t_busy (clk, reset, busy_valid, busy_ready, busy_bits, finish, have[2], busy); - - // Staged token ends: the cycle finishes from the endpoint's registers, a - // host cycle after its last token. - assign finish = run & (&have) & (&done); - - // ---- The host's registers ---- - reg aw_got, w_got, ar_got; - reg [11:0] aw_addr, ar_addr; - reg [31:0] w_data; - assign s_awready = ~aw_got; - assign s_wready = ~w_got; - assign s_arready = ~ar_got & ~s_rvalid; - - wire wr = aw_got & w_got & ~s_bvalid; - wire [11:0] wa = {aw_addr[11:2], 2'b00}; - wire push = wr & (wa == `FOLDSIM_REG_CMD) & (q_count != QC'(QUEUE)); - wire pop = finish & ~q_empty & (reset_next == 0); - - wire status_busy = ~q_empty | (cur_op != `FOLDSIM_CMD_NONE) | start_pending | target_busy - | ~mem_idle | (reset_count != 0); - - always @(posedge clk) begin - if (reset) begin - q_head <= 0; - q_tail <= 0; - q_count <= 0; - run <= 1'b1; - reset_count <= RESET_CYCLES; - cur_op <= `FOLDSIM_CMD_NONE; - start_pending <= 1'b0; - target_busy <= 1'b0; - rsp_valid <= 1'b0; - target_cycles <= 0; - host_cycles <= 0; - mem_stalls <= 0; - for (integer c = 0; c < NUM_CLASSES; ++c) - finishes[c] <= 0; - mem_base <= 64'h40_0000_0000; - mem_refresh_r <= 2'd0; - mem_refi <= 16'd0; - mem_rfc <= 16'd0; - mem_clock <= {16'd400, 16'd800}; - mem_latency <= 16'd0; - mem_queue <= {8'd32, 8'd32, 16'd0}; - mem_geom <= {1'b1, 2'd2, 4'd5, 2'd2, 2'd2, 2'd1}; - mem_wmark <= {8'd7, 8'd26}; - for (integer i = 0; i < 18; ++i) - mem_t[i] <= MEM_T_RESET[i*12 +: 12]; - aw_got <= 1'b0; - w_got <= 1'b0; - ar_got <= 1'b0; - s_bvalid <= 1'b0; - s_rvalid <= 1'b0; - end else begin - host_cycles <= host_cycles + 64'd1; - if (mem_stall) - mem_stalls <= mem_stalls + 64'd1; - for (integer c = 0; c < NUM_CLASSES; ++c) - if (profile_finish[c]) - finishes[c] <= finishes[c] + 48'd1; - - // One target cycle of the endpoint. - if (finish) begin - target_cycles <= target_cycles + 64'd1; - reset_count <= reset_next; - cur_op <= pop ? q_op[q_head] : `FOLDSIM_CMD_NONE; - cur_addr <= q_addr[q_head]; - cur_data <= q_data[q_head]; - if (rsp_v) begin - rsp_valid <= 1'b1; - rsp_data <= rsp_d; - end - target_busy <= busy; - // A reset command's address chooses the refresh the target - // has from the reset on. - if (cur_op == `FOLDSIM_CMD_RESET) - mem_refresh_r <= cur_addr[1:0]; - // A start counts as busy until the target says it is, or plainly - // never will. - if (do_start) begin - start_pending <= 1'b1; - start_wait <= 16'hffff; - end else if (start_pending) begin - if (busy || start_wait == 0) - start_pending <= 1'b0; - start_wait <= start_wait - 16'd1; - end - end - if (pop) - q_head <= q_head + 1'b1; - if (push) begin - q_op[q_tail] <= w_data[2:0]; - q_addr[q_tail] <= cmd_addr; - q_data[q_tail] <= cmd_data; - q_tail <= q_tail + 1'b1; - // A read's response is the next one. - if (w_data[2:0] == `FOLDSIM_CMD_DCR_READ) - rsp_valid <= 1'b0; - end - q_count <= q_count + QC'(push) - QC'(pop); - - // AXI4-Lite: a write once both its address and data are in. - if (s_awvalid & s_awready) begin - aw_got <= 1'b1; - aw_addr <= s_awaddr; - end - if (s_wvalid & s_wready) begin - w_got <= 1'b1; - w_data <= s_wdata; - end - if (wr) begin - aw_got <= 1'b0; - w_got <= 1'b0; - s_bvalid <= 1'b1; - case (wa) - `FOLDSIM_REG_CTRL: run <= w_data[0]; - `FOLDSIM_REG_CMD_ADDR: cmd_addr <= DCR_ADDR_W'(w_data); - `FOLDSIM_REG_CMD_DATA: cmd_data <= DCR_DATA_W'(w_data); - `FOLDSIM_REG_MEM_BASE_LO: mem_base[31:0] <= w_data; - `FOLDSIM_REG_MEM_BASE_HI: mem_base[63:32] <= w_data; - `FOLDSIM_REG_MEM_CLOCK: mem_clock <= w_data; - `FOLDSIM_REG_MEM_LATENCY: mem_latency <= w_data[15:0]; - `FOLDSIM_REG_MEM_QUEUE: mem_queue <= w_data; - `FOLDSIM_REG_MEM_GEOM: mem_geom <= w_data[12:0]; - `FOLDSIM_REG_MEM_WMARK: mem_wmark <= w_data[15:0]; - `FOLDSIM_REG_MEM_REFI: mem_refi <= w_data[15:0]; - `FOLDSIM_REG_MEM_RFC: mem_rfc <= w_data[15:0]; - default:; - endcase - if (wa >= `FOLDSIM_REG_MEM_TIMING && wa < `FOLDSIM_REG_MEM_TIMING + 12'(4 * 18)) - mem_t[5'((wa - `FOLDSIM_REG_MEM_TIMING) >> 2)] <= w_data[11:0]; - end - if (s_bvalid & s_bready) - s_bvalid <= 1'b0; - - if (s_arvalid & s_arready) begin - ar_got <= 1'b1; - ar_addr <= s_araddr; - end - if (ar_got) begin - ar_got <= 1'b0; - s_rvalid <= 1'b1; - case ({ar_addr[11:2], 2'b00}) - `FOLDSIM_REG_ID: s_rdata <= `FOLDSIM_ID; - `FOLDSIM_REG_VERSION: s_rdata <= `FOLDSIM_VERSION; - `FOLDSIM_REG_CTRL: s_rdata <= {31'b0, run}; - `FOLDSIM_REG_STATUS: s_rdata <= {27'b0, mem_error, rsp_valid, - (q_count == QC'(QUEUE)), status_busy}; - `FOLDSIM_REG_CMD_ADDR: s_rdata <= 32'(cmd_addr); - `FOLDSIM_REG_CMD_DATA: s_rdata <= 32'(cmd_data); - `FOLDSIM_REG_DCR_RSP: s_rdata <= 32'(rsp_data); - `FOLDSIM_REG_TCYCLES_LO: s_rdata <= target_cycles[31:0]; - `FOLDSIM_REG_TCYCLES_HI: s_rdata <= target_cycles[63:32]; - `FOLDSIM_REG_HCYCLES_LO: s_rdata <= host_cycles[31:0]; - `FOLDSIM_REG_HCYCLES_HI: s_rdata <= host_cycles[63:32]; - `FOLDSIM_REG_MEM_BASE_LO: s_rdata <= mem_base[31:0]; - `FOLDSIM_REG_MEM_BASE_HI: s_rdata <= mem_base[63:32]; - `FOLDSIM_REG_MSTALL_LO: s_rdata <= mem_stalls[31:0]; - `FOLDSIM_REG_MSTALL_HI: s_rdata <= mem_stalls[63:32]; - `FOLDSIM_REG_NUM_CLASSES: s_rdata <= 32'(NUM_CLASSES); - `FOLDSIM_REG_MEM_REFRESH: s_rdata <= 32'(mem_refresh_r); - `FOLDSIM_REG_MEM_CAPS: s_rdata <= MEM_CAPS; - `FOLDSIM_REG_MEM_CLOCK: s_rdata <= mem_clock; - `FOLDSIM_REG_MEM_LATENCY: s_rdata <= 32'(mem_latency); - `FOLDSIM_REG_MEM_QUEUE: s_rdata <= mem_queue; - `FOLDSIM_REG_MEM_GEOM: s_rdata <= 32'(mem_geom); - `FOLDSIM_REG_MEM_WMARK: s_rdata <= 32'(mem_wmark); - `FOLDSIM_REG_MEM_REFI: s_rdata <= 32'(mem_refi); - `FOLDSIM_REG_MEM_RFC: s_rdata <= 32'(mem_rfc); - default: s_rdata <= 32'd0; - endcase - if (ar_addr >= `FOLDSIM_REG_MEM_TIMING && ar_addr < `FOLDSIM_REG_MEM_TIMING + 12'(4 * 18)) - s_rdata <= 32'(mem_t[5'((ar_addr - `FOLDSIM_REG_MEM_TIMING) >> 2)]); - // A port's counters at FOLDSIM_REG_MEM_STATS + 0x40 per port. - if (ar_addr >= `FOLDSIM_REG_MEM_STATS && ar_addr < `FOLDSIM_REG_MEM_STATS + 12'(64 * MEM_PORTS)) - s_rdata <= mem_stats[32'((ar_addr - `FOLDSIM_REG_MEM_STATS) >> 6) * (8 * 64) - + 32'(ar_addr[5:0]) * 8 +: 32]; - if (ar_addr >= `FOLDSIM_REG_FINISH_BASE - && ar_addr < `FOLDSIM_REG_FINISH_BASE + 12'(8 * NUM_CLASSES)) begin - s_rdata <= ar_addr[2] ? 32'(finishes[(ar_addr - `FOLDSIM_REG_FINISH_BASE) >> 3] >> 32) - : finishes[(ar_addr - `FOLDSIM_REG_FINISH_BASE) >> 3][31:0]; - end - end - if (s_rvalid & s_rready) - s_rvalid <= 1'b0; - end - end - -endmodule diff --git a/hw/rtl/afu/foldsim/foldsim_mem.sv b/hw/rtl/afu/foldsim/foldsim_mem.sv deleted file mode 100644 index 5928849aca..0000000000 --- a/hw/rtl/afu/foldsim/foldsim_mem.sv +++ /dev/null @@ -1,470 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The target's AXI4 memory port, as a foldsim endpoint. -// -// Timing is a DRAM channel's banks (foldsim_mem_bank, cycle for cycle -// sim/common/dram_bank_model.h's DramBankAxi): each read holds a slot, and -// responses leave in the order their data departs. Every output is a function -// of the model's state, so all of them are offered at the start of a target -// cycle. -// -// Data lives in device memory at `base` plus the target's address. A read -// goes to memory the cycle the target makes it, a write the cycle it has both -// its AW and its W; the response is offered to -// the target when the model releases it, and a target cycle that would offer -// one memory has not returned yet simply does not finish until it has. A -// write is acknowledged to the target only once memory has acknowledged it, -// so any later request of the target sees it. -// -// Single-beat transactions only, as Vortex issues them: a burst sets `error`, -// goes nowhere, and from the next cycle on the endpoint is ready for nothing. -// -// The target's reset is a token of the control endpoint's. A target cycle in -// reset takes no request; it finishes once every transaction memory has from -// before it has completed, and drops the model's state -- requests not -// answered, data not delivered, a write's lone half -- and `error`. A response from before a -// reset never reaches the target after it. -module foldsim_mem #( - parameter ID_W = 4, - parameter ADDR_W = 64, - parameter DATA_W = 512, - // Writes in flight. - parameter DEPTH = 16, - // The bank model: read slots, and a ring of 2^HLOG2 DRAM clocks. - parameter SLOTS = 32, - parameter HLOG2 = 9, - // Token ends with registered have and done (foldsim_token): the cycle's - // finish starts at the endpoint's own registers, a host cycle later. - parameter STAGED = 0, - // The target's ports interleave its address space by block, 2^CH_SHIFT of - // them: the bank model sees the block within the port's channel, as - // dram_sim's banks do. Device memory still takes the target's address. - parameter CH_SHIFT = 0 -) ( - input wire clk, - input wire reset, - - input wire [63:0] base, - // The bank model's settings (foldsim_mem_bank). - input wire [15:0] cfg_data_rate, - input wire [15:0] cfg_core_clock, - input wire [15:0] cfg_latency, - input wire [15:0] cfg_queue, - input wire [7:0] cfg_rq, - input wire [7:0] cfg_wq, - input wire [7:0] cfg_wr_high, - input wire [7:0] cfg_wr_low, - input wire [12:0] cfg_geom, - input wire [18*12-1:0] cfg_t, - input wire [1:0] cfg_refresh, - input wire [15:0] cfg_refi, - input wire [15:0] cfg_rfc, - // Reads, writes, row hits, misses, conflicts, read latency sum, - // refreshes, target cycles the port could take no read. - output wire [8*64-1:0] stats, - // No transaction in flight, at the target or in memory. - output wire idle, - output reg error, - // A host cycle a target cycle waits for memory. - output wire stall, - - // The target's reset, from the control endpoint. - input wire treset_valid, output wire treset_ready, input wire treset_bits, - - // Target outputs, as channels of the simulator: the endpoint's inputs. - input wire awvalid_valid, output wire awvalid_ready, input wire awvalid_bits, - input wire awaddr_valid, output wire awaddr_ready, input wire [ADDR_W-1:0] awaddr_bits, - input wire awid_valid, output wire awid_ready, input wire [ID_W-1:0] awid_bits, - input wire awlen_valid, output wire awlen_ready, input wire [7:0] awlen_bits, - input wire wvalid_valid, output wire wvalid_ready, input wire wvalid_bits, - input wire wdata_valid, output wire wdata_ready, input wire [DATA_W-1:0] wdata_bits, - input wire wstrb_valid, output wire wstrb_ready, input wire [DATA_W/8-1:0] wstrb_bits, - input wire wlast_valid, output wire wlast_ready, input wire wlast_bits, - input wire arvalid_valid, output wire arvalid_ready, input wire arvalid_bits, - input wire araddr_valid, output wire araddr_ready, input wire [ADDR_W-1:0] araddr_bits, - input wire arid_valid, output wire arid_ready, input wire [ID_W-1:0] arid_bits, - input wire arlen_valid, output wire arlen_ready, input wire [7:0] arlen_bits, - input wire rready_valid, output wire rready_ready, input wire rready_bits, - input wire bready_valid, output wire bready_ready, input wire bready_bits, - - // Target inputs: the endpoint's outputs. - output wire awready_valid, input wire awready_ready, output wire awready_bits, - output wire wready_valid, input wire wready_ready, output wire wready_bits, - output wire arready_valid, input wire arready_ready, output wire arready_bits, - output wire rvalid_valid, input wire rvalid_ready, output wire rvalid_bits, - output wire rdata_valid, input wire rdata_ready, output wire [DATA_W-1:0] rdata_bits, - output wire rlast_valid, input wire rlast_ready, output wire rlast_bits, - output wire rid_valid, input wire rid_ready, output wire [ID_W-1:0] rid_bits, - output wire rresp_valid, input wire rresp_ready, output wire [1:0] rresp_bits, - output wire bvalid_valid, input wire bvalid_ready, output wire bvalid_bits, - output wire bresp_valid, input wire bresp_ready, output wire [1:0] bresp_bits, - output wire bid_valid, input wire bid_ready, output wire [ID_W-1:0] bid_bits, - - // Device memory. - output wire m_awvalid, - input wire m_awready, - output wire [63:0] m_awaddr, - output wire [7:0] m_awlen, - output wire m_wvalid, - input wire m_wready, - output wire [DATA_W-1:0] m_wdata, - output wire [DATA_W/8-1:0] m_wstrb, - output wire m_wlast, - input wire m_bvalid, - output wire m_bready, - output wire m_arvalid, - input wire m_arready, - output wire [63:0] m_araddr, - output wire [7:0] m_arlen, - input wire m_rvalid, - output wire m_rready, - input wire [DATA_W-1:0] m_rdata -); - localparam CW = $clog2(DEPTH + 1); - localparam PW = $clog2(DEPTH); - // Memory's side holds every read and write the model may have in flight. - localparam HD = (SLOTS > DEPTH) ? SLOTS : DEPTH; - localparam HCW = $clog2(HD + 1); - localparam HPW = $clog2(HD); - localparam SW = $clog2(SLOTS); - localparam LB = $clog2(DATA_W / 8); - - wire finish; - - // ---- The target's request side, one token per channel per cycle ---- - wire awvalid, wvalid, wlast, arvalid, rready, bready, treset; - wire [ADDR_W-1:0] awaddr, araddr; - wire [ID_W-1:0] awid, arid; - wire [7:0] awlen, arlen; - wire [DATA_W-1:0] wdata; - wire [DATA_W/8-1:0] wstrb; - wire [14:0] have; - - foldsim_token_in #(1, STAGED) t_awvalid (clk, reset, awvalid_valid, awvalid_ready, awvalid_bits, finish, have[0], awvalid); - foldsim_token_in #(ADDR_W, STAGED) t_awaddr (clk, reset, awaddr_valid, awaddr_ready, awaddr_bits, finish, have[1], awaddr); - foldsim_token_in #(ID_W, STAGED) t_awid (clk, reset, awid_valid, awid_ready, awid_bits, finish, have[2], awid); - foldsim_token_in #(8, STAGED) t_awlen (clk, reset, awlen_valid, awlen_ready, awlen_bits, finish, have[3], awlen); - foldsim_token_in #(1, STAGED) t_wvalid (clk, reset, wvalid_valid, wvalid_ready, wvalid_bits, finish, have[4], wvalid); - foldsim_token_in #(DATA_W, STAGED) t_wdata (clk, reset, wdata_valid, wdata_ready, wdata_bits, finish, have[5], wdata); - foldsim_token_in #(DATA_W/8, STAGED) t_wstrb (clk, reset, wstrb_valid, wstrb_ready, wstrb_bits, finish, have[6], wstrb); - foldsim_token_in #(1, STAGED) t_wlast (clk, reset, wlast_valid, wlast_ready, wlast_bits, finish, have[7], wlast); - foldsim_token_in #(1, STAGED) t_arvalid (clk, reset, arvalid_valid, arvalid_ready, arvalid_bits, finish, have[8], arvalid); - foldsim_token_in #(ADDR_W, STAGED) t_araddr (clk, reset, araddr_valid, araddr_ready, araddr_bits, finish, have[9], araddr); - foldsim_token_in #(ID_W, STAGED) t_arid (clk, reset, arid_valid, arid_ready, arid_bits, finish, have[10], arid); - foldsim_token_in #(8, STAGED) t_arlen (clk, reset, arlen_valid, arlen_ready, arlen_bits, finish, have[11], arlen); - foldsim_token_in #(1, STAGED) t_rready (clk, reset, rready_valid, rready_ready, rready_bits, finish, have[12], rready); - foldsim_token_in #(1, STAGED) t_bready (clk, reset, bready_valid, bready_ready, bready_bits, finish, have[13], bready); - foldsim_token_in #(1, STAGED) t_treset (clk, reset, treset_valid, treset_ready, treset_bits, finish, have[14], treset); - - // The requests' addresses within the port's channel, the bits the bank - // model takes: the block index drops the port's. - wire [29:5] ar_ch = {araddr[29 + CH_SHIFT -: 30 - LB], araddr[LB-1:5]}; - wire [29:5] aw_ch = {awaddr[29 + CH_SHIFT -: 30 - LB], awaddr[LB-1:5]}; - - // ---- The endpoint's state ---- - // Requests in flight: writes' halves taken and not yet acknowledged. - reg [CW-1:0] pending_aw, pending_w; - - // AW ids and channel addresses waiting for their write data. - reg [ID_W-1:0] awq_id [DEPTH]; - reg [29:5] awq_addr [DEPTH]; - reg [PW-1:0] awq_head, awq_tail; - reg [CW-1:0] awq_count; - - // Writes memory has acknowledged, and the target has been given. - reg [7:0] b_acked, b_given; - // Reads memory has not answered, and writes it has not acknowledged. - reg [HCW-1:0] rd_out, wr_out; - wire drained = (rd_out == 0) & (wr_out == 0); - - // The bank model's state: the port's readiness, R and B. - wire bk_room, bk_slot_free, bk_r_valid, bk_b_valid, bk_no_reads, bk_done; - wire [SW-1:0] bk_alloc_slot, bk_r_slot; - wire [ID_W-1:0] bk_r_id, bk_b_id; - // Read data by slot, and which slots memory has answered. - reg [DATA_W-1:0] slot_data [SLOTS]; - reg [SLOTS-1:0] arrived; - - // ---- The model's outputs: functions of its state ---- - wire ar_ready = ~error & bk_room & bk_slot_free; - wire aw_ready = ~error & bk_room & (32'(pending_aw) < DEPTH); - wire w_ready = ~error & (32'(pending_w) < DEPTH); - wire r_valid = bk_r_valid; - wire b_valid = bk_b_valid; - // R needs its data back from memory, and B its write acknowledged. - wire ok = (~bk_r_valid | arrived[bk_r_slot]) & (~bk_b_valid | (b_acked != b_given)); - wire [10:0] done; - - foldsim_token_out #(STAGED) o_awready (clk, reset, awready_valid, awready_ready, ok, finish, done[0]); - foldsim_token_out #(STAGED) o_wready (clk, reset, wready_valid, wready_ready, ok, finish, done[1]); - foldsim_token_out #(STAGED) o_arready (clk, reset, arready_valid, arready_ready, ok, finish, done[2]); - foldsim_token_out #(STAGED) o_rvalid (clk, reset, rvalid_valid, rvalid_ready, ok, finish, done[3]); - foldsim_token_out #(STAGED) o_rdata (clk, reset, rdata_valid, rdata_ready, ok, finish, done[4]); - foldsim_token_out #(STAGED) o_rlast (clk, reset, rlast_valid, rlast_ready, ok, finish, done[5]); - foldsim_token_out #(STAGED) o_rid (clk, reset, rid_valid, rid_ready, ok, finish, done[6]); - foldsim_token_out #(STAGED) o_rresp (clk, reset, rresp_valid, rresp_ready, ok, finish, done[7]); - foldsim_token_out #(STAGED) o_bvalid (clk, reset, bvalid_valid, bvalid_ready, ok, finish, done[8]); - foldsim_token_out #(STAGED) o_bresp (clk, reset, bresp_valid, bresp_ready, ok, finish, done[9]); - foldsim_token_out #(STAGED) o_bid (clk, reset, bid_valid, bid_ready, ok, finish, done[10]); - - assign awready_bits = aw_ready; - assign wready_bits = w_ready; - assign arready_bits = ar_ready; - assign rvalid_bits = r_valid; - assign rdata_bits = slot_data[bk_r_slot]; - assign rlast_bits = 1'b1; - assign rid_bits = bk_r_id; - assign rresp_bits = 2'b00; - assign bvalid_bits = b_valid; - assign bresp_bits = 2'b00; - assign bid_bits = bk_b_id; - - // A reset cycle waits for memory to answer everything it has, and every - // cycle for the bank model's DRAM clocks of it. - wire drain_wait = have[14] & treset & ~drained; - wire tokens_in = (&have) & (&done); - assign finish = tokens_in & ~drain_wait & bk_done; - assign stall = ~ok | drain_wait | (tokens_in & ~bk_done); - assign idle = bk_no_reads & (pending_aw == 0) & (pending_w == 0) & drained; - - // ---- One target cycle, on the request side's tokens ---- - wire ar_hs = arvalid & ar_ready; - wire aw_hs = awvalid & aw_ready; - wire w_hs = wvalid & w_ready; - wire burst = (ar_hs & (arlen != 0)) | (aw_hs & (awlen != 0)) | (w_hs & ~wlast); - wire ar_fire = ar_hs & (arlen == 0); - wire aw_fire = aw_hs & (awlen == 0); - wire w_fire = w_hs & wlast; - wire r_fire = r_valid & rready; - wire b_fire = b_valid & bready; - - // Writes: an AW and its W, in either order, form one write. - wire new_write = ((pending_w > pending_aw) & aw_fire) - | ((pending_w < pending_aw) & w_fire) - | (aw_fire & w_fire); - wire awq_empty = (awq_count == 0); - wire [ID_W-1:0] w_form_id = awq_empty ? awid : awq_id[awq_head]; - wire [29:5] w_form_addr = awq_empty ? aw_ch : awq_addr[awq_head]; - wire awq_pop = new_write & ~awq_empty; - wire awq_push = aw_fire & ~(new_write & awq_empty); - - function automatic [CW-1:0] count(input [CW-1:0] value, input inc, input dec); - count = (inc & ~dec) ? value + 1'b1 : ((dec & ~inc & (value != 0)) ? value - 1'b1 : value); - endfunction - - always @(posedge clk) begin - if (reset) begin - pending_aw <= 0; - pending_w <= 0; - awq_head <= 0; - awq_tail <= 0; - awq_count <= 0; - error <= 1'b0; - end else if (finish) begin - if (treset) begin - pending_aw <= 0; - pending_w <= 0; - awq_head <= 0; - awq_tail <= 0; - awq_count <= 0; - error <= 1'b0; - end else begin - if (awq_push) begin - awq_id[awq_tail] <= awid; - awq_addr[awq_tail] <= aw_ch; - awq_tail <= awq_tail + 1'b1; - end - if (awq_pop) - awq_head <= awq_head + 1'b1; - awq_count <= awq_count + CW'(awq_push) - CW'(awq_pop); - - pending_aw <= count(pending_aw, aw_fire, b_fire); - pending_w <= count(pending_w, w_fire, b_fire); - if (burst) - error <= 1'b1; - end - end - end - - // ---- Memory: requests as the target makes them, in order ---- - // A write goes to memory once both its halves are in: only whole writes - // are offered, so a lone half can be dropped without breaking AXI. - reg [63:0] arq_addr [HD]; - reg [SW-1:0] arq_slot [HD]; - reg [HPW-1:0] arq_head, arq_tail, rs_head; - reg [HCW-1:0] arq_count; - reg [63:0] awm_addr [DEPTH]; - reg [PW-1:0] awm_head, awm_tail; - reg [DATA_W-1:0] wm_data [DEPTH]; - reg [DATA_W/8-1:0] wm_strb [DEPTH]; - reg [PW-1:0] wm_head, wm_tail; - // Whole writes whose AW, and whose W, memory has not taken yet. - reg [CW-1:0] aw_formed, w_formed; - - wire take = finish & ~treset; - wire flush = finish & treset; - wire arq_push = take & ar_fire; - wire arq_pop = m_arvalid & m_arready; - wire awm_push = take & aw_fire; - wire awm_pop = m_awvalid & m_awready; - wire wm_push = take & w_fire; - wire wm_pop = m_wvalid & m_wready; - wire form = take & new_write; - wire rd_push = m_rvalid & m_rready; - wire b_push = m_bvalid & m_bready; - - assign m_arvalid = (arq_count != 0); - assign m_araddr = arq_addr[arq_head]; - assign m_arlen = 8'd0; - assign m_rready = 1'b1; - assign m_awvalid = (aw_formed != 0); - assign m_awaddr = awm_addr[awm_head]; - assign m_awlen = 8'd0; - assign m_wvalid = (w_formed != 0); - assign m_wdata = wm_data[wm_head]; - assign m_wstrb = wm_strb[wm_head]; - assign m_wlast = 1'b1; - assign m_bready = 1'b1; - - always @(posedge clk) begin - if (arq_push) begin - arq_addr[arq_tail] <= base + 64'(araddr); - arq_slot[arq_tail] <= bk_alloc_slot; - end - if (awm_push) - awm_addr[awm_tail] <= base + 64'(awaddr); - if (wm_push) begin - wm_data[wm_tail] <= wdata; - wm_strb[wm_tail] <= wstrb; - end - // Memory answers reads in the order they were made: the data goes - // to the read's slot. - if (rd_push) - slot_data[arq_slot[rs_head]] <= m_rdata; - end - - // A flush comes once memory has answered everything: nothing is offered - // to it or expected from it, and what is left are lone halves of writes - // and data the target will no longer take. - always @(posedge clk) begin - if (reset || flush) begin - arq_head <= 0; - arq_tail <= 0; - arq_count <= 0; - rs_head <= 0; - arrived <= '0; - awm_head <= 0; - awm_tail <= 0; - wm_head <= 0; - wm_tail <= 0; - aw_formed <= 0; - w_formed <= 0; - rd_out <= 0; - wr_out <= 0; - b_acked <= 0; - b_given <= 0; - end else begin - if (arq_push) - arq_tail <= arq_tail + 1'b1; - if (arq_pop) - arq_head <= arq_head + 1'b1; - arq_count <= arq_count + HCW'(arq_push) - HCW'(arq_pop); - if (rd_push) - rs_head <= rs_head + 1'b1; - // A slot's data has not arrived from the cycle its read is taken. - begin : arrived_ - reg [SLOTS-1:0] a; - a = arrived; - if (arq_push) - a[bk_alloc_slot] = 1'b0; - if (rd_push) - a[arq_slot[rs_head]] = 1'b1; - arrived <= a; - end - - if (awm_push) - awm_tail <= awm_tail + 1'b1; - if (awm_pop) - awm_head <= awm_head + 1'b1; - if (wm_push) - wm_tail <= wm_tail + 1'b1; - if (wm_pop) - wm_head <= wm_head + 1'b1; - aw_formed <= aw_formed + CW'(form) - CW'(awm_pop); - w_formed <= w_formed + CW'(form) - CW'(wm_pop); - - rd_out <= rd_out + HCW'(arq_push) - HCW'(rd_push); - wr_out <= wr_out + HCW'(form) - HCW'(b_push); - if (b_push) - b_acked <= b_acked + 8'd1; - if (take & b_fire) - b_given <= b_given + 8'd1; - end - end - - // ---- The bank model ---- - foldsim_mem_bank #( - .ID_W (ID_W), - .SLOTS (SLOTS), - .WSLOTS (DEPTH), - .HLOG2 (HLOG2), - .BURSTS (DATA_W / 256) - ) bank_model ( - .clk (clk), - .reset (reset), - .cfg_data_rate (cfg_data_rate), - .cfg_core_clock (cfg_core_clock), - .cfg_latency (cfg_latency), - .cfg_queue (cfg_queue), - .cfg_rq (cfg_rq), - .cfg_wq (cfg_wq), - .cfg_wr_high (cfg_wr_high), - .cfg_wr_low (cfg_wr_low), - .cfg_geom (cfg_geom), - .cfg_t (cfg_t), - .cfg_refresh (cfg_refresh), - .cfg_refi (cfg_refi), - .cfg_rfc (cfg_rfc), - .finish (finish), - .treset (treset), - .ar_fire (ar_fire), - .ar_id (arid), - .ar_addr (ar_ch), - .aw_fire (aw_fire), - .new_write (new_write), - .nw_id (w_form_id), - .nw_addr (w_form_addr), - .r_fire (r_fire), - .b_fire (b_fire), - .room (bk_room), - .slot_free (bk_slot_free), - .alloc_slot (bk_alloc_slot), - .r_valid (bk_r_valid), - .r_id (bk_r_id), - .r_slot (bk_r_slot), - .b_valid (bk_b_valid), - .b_id (bk_b_id), - .no_reads (bk_no_reads), - .done (bk_done), - .stat_reads (stats[0*64 +: 64]), - .stat_writes (stats[1*64 +: 64]), - .stat_hits (stats[2*64 +: 64]), - .stat_misses (stats[3*64 +: 64]), - .stat_conflicts (stats[4*64 +: 64]), - .stat_latency (stats[5*64 +: 64]), - .stat_refreshes (stats[6*64 +: 64]), - .stat_full (stats[7*64 +: 64]) - ); - -endmodule diff --git a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv b/hw/rtl/afu/foldsim/foldsim_mem_bank.sv deleted file mode 100644 index a530f2f8e2..0000000000 --- a/hw/rtl/afu/foldsim/foldsim_mem_bank.sv +++ /dev/null @@ -1,1098 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The timing of foldsim's memory endpoint as a DRAM bank model: the model of -// sim/common/dram_bank_model.h (DramBankAxi over DramBankModel), decision for -// decision, for one AXI4 port and one DRAM channel. -// -// foldsim_mem gives it each target cycle's requests when the cycle finishes; -// it answers with the port's state for the next one: whether a read or write -// may be taken, and the read and write acknowledgement R and B present. The -// DRAM clocks of a target cycle run in host cycles while the target evaluates -// it, and the cycle does not finish before they have (`done`): requests of a -// cycle reach the channel in the next one, and a read departs at least -// tCL + tBL clocks after it was reserved, so a cycle's clocks depend only on -// what earlier cycles gave. -// -// The calendars are bitmaps over a ring of 2^HLOG2 DRAM clocks in distributed -// RAM (foldsim_mem_bmap): which clocks hold a command, the row-command bus, -// and per pseudo-channel and bank group the clocks a read, a write or an ACT -// may not take -- each command, once reserved, marks the clocks its -// constraints forbid to later ones, so placing a command is a first-free -// search over 64 clocks a host cycle. A reservation reaches at most -// 2^HLOG2 - 64 clocks ahead: the ring's last word is the one being retired. -// Times of the bank table and the controller are 16-bit, modulo; a time too -// old to constrain anything is moved up before it could wrap. -// -// Refresh, when on, takes a DRAM clock's turn to reserve every `cfg_refi` -// clocks: a precharge and a REF, placed as a precharge is, on the next bank in -// turn or (all-bank) on every bank. For all-bank refresh the channel keeps the -// latest time a precharge and an activate allow it, over the banks opened -// since the last one, instead of reading the bank table. -module foldsim_mem_bank #( - parameter ID_W = 4, - parameter SLOTS = 32, - parameter WSLOTS = 16, - parameter HLOG2 = 9, - parameter BURSTS = 2, - parameter ROW_W = 20, - parameter INQ = 64, - parameter SW = $clog2(SLOTS) -) ( - input wire clk, - input wire reset, - - input wire [15:0] cfg_data_rate, - input wire [15:0] cfg_core_clock, - input wire [15:0] cfg_latency, - input wire [15:0] cfg_queue, - input wire [7:0] cfg_rq, - input wire [7:0] cfg_wq, - input wire [7:0] cfg_wr_high, - input wire [7:0] cfg_wr_low, - // [1:0] log2 pseudo-channels, [3:2] log2 bank groups, [5:4] log2 banks per - // group, [9:6] log2 bursts per row, [11:10] log2 bursts per row visit, - // [12] an ACT holds the row bus for two clocks. - input wire [12:0] cfg_geom, - // tBL, tCCD_S, tCCD_L, tCL, tCWL, tRCD_RD, tRCD_WR, tRP, tRAS, tRC, tRTP, - // tWR, tRRD_S, tRRD_L, tFAW, tWTR_S, tWTR_L, read-to-write extra: DRAM - // clocks, 12 bits each. - input wire [18*12-1:0] cfg_t, - // Refresh: [0] on, [1] all banks at once; clocks between refreshes, and - // clocks a refreshed bank may not activate. - input wire [1:0] cfg_refresh, - input wire [15:0] cfg_refi, - input wire [15:0] cfg_rfc, - - // The target cycle finishes; a reset cycle drops everything. - input wire finish, - input wire treset, - // What the cycle takes: a read, a write formed of an AW and its W, and the - // responses R and B give. - input wire ar_fire, - input wire [ID_W-1:0] ar_id, - input wire [29:5] ar_addr, - input wire aw_fire, - input wire new_write, - input wire [ID_W-1:0] nw_id, - input wire [29:5] nw_addr, - input wire r_fire, - input wire b_fire, - - // The port's state for the target cycle. - output wire room, - output wire slot_free, - output wire [SW-1:0] alloc_slot, - output wire r_valid, - output wire [ID_W-1:0] r_id, - output wire [SW-1:0] r_slot, - output wire b_valid, - output wire [ID_W-1:0] b_id, - output wire no_reads, - // The target cycle's DRAM clocks are done. - output wire done, - - // Reads, writes, row hits, misses, conflicts, the sum of read latencies - // (target cycles from AR to R), refreshes, and target cycles the port - // could take no read. - output reg [63:0] stat_reads, - output reg [63:0] stat_writes, - output reg [63:0] stat_hits, - output reg [63:0] stat_misses, - output reg [63:0] stat_conflicts, - output reg [63:0] stat_latency, - output reg [63:0] stat_refreshes, - output reg [63:0] stat_full -); - localparam H = 1 << HLOG2; - localparam HE = H - 64; - localparam WORDS = H / 64; - localparam WW = HLOG2 - 6; - localparam NB = BURSTS; - localparam TAG_W = (SW > ID_W) ? SW : ID_W; - localparam BKW = 5; - localparam IQW = $clog2(INQ); - localparam BQ = 32; - localparam BQW = $clog2(BQ); - - // The model has its settings: it starts at the first target reset, after - // the runtime has written them. - reg active; - - // ---- Configuration and its derived constants ---- - wire [11:0] T [18]; - for (genvar i = 0; i < 18; ++i) begin : g_t - assign T[i] = cfg_t[i*12 +: 12]; - end - localparam I_BL = 0, I_CCDS = 1, I_CCDL = 2, I_CL = 3, I_CWL = 4, I_RCDRD = 5, I_RCDWR = 6, - I_RP = 7, I_RAS = 8, I_RC = 9, I_RTP = 10, I_WR = 11, I_RRDS = 12, I_RRDL = 13, - I_FAW = 14, I_WTRS = 15, I_WTRL = 16, I_RTWX = 17; - - wire [1:0] g_lpcs = cfg_geom[1:0]; - wire [1:0] g_lbgs = cfg_geom[3:2]; - wire [1:0] g_lbanks = cfg_geom[5:4]; - wire [3:0] g_lrow = cfg_geom[9:6]; - wire [1:0] g_lvisit = cfg_geom[11:10]; - wire g_act2 = cfg_geom[12]; - - function automatic signed [15:0] smax(input signed [15:0] a, input signed [15:0] b); - smax = (a > b) ? a : b; - endfunction - - reg signed [15:0] k_ccds, k_ccdl, k_rtw, k_wtrs, k_wtrl, k_reach_r, k_reach_w, k_reach_a; - reg signed [15:0] k_wrpre, k_dep, k_ccdl1, k_rcdrd, k_rcdwr, k_rp, k_ras, k_rc, k_rtp; - reg signed [15:0] k_rrds, k_rrdl, k_rfc; - reg [16:0] k_twoclk; - reg [2:0] k_bb; - reg [4:0] k_rowsh; - reg [24:0] k_bmask; - always @(posedge clk) begin - k_ccds <= 16'(T[I_CCDS]); - k_ccdl <= 16'(T[I_CCDL]); - k_rtw <= smax(16'(T[I_CCDS]), 16'(T[I_CL]) + 16'(T[I_BL]) + 16'(T[I_RTWX]) - 16'(T[I_CWL])); - k_wtrs <= 16'(T[I_CWL]) + 16'(T[I_BL]) + 16'(T[I_WTRS]); - k_wtrl <= 16'(T[I_CWL]) + 16'(T[I_BL]) + 16'(T[I_WTRL]); - k_reach_r <= smax(smax(k_ccds, k_ccdl), k_rtw) - 16'sd1; - k_reach_w <= smax(smax(k_ccds, k_ccdl), smax(k_wtrs, k_wtrl)) - 16'sd1; - k_reach_a <= smax(smax(smax(16'(T[I_RRDS]), 16'(T[I_RRDL])), 16'(T[I_FAW])) - 16'sd1, - g_act2 ? 16'sd1 : 16'sd0); - k_wrpre <= 16'(T[I_CWL]) + 16'(T[I_BL]) + 16'(T[I_WR]); - k_dep <= 16'(T[I_CL]) + 16'(T[I_BL]); - k_ccdl1 <= smax(16'(T[I_CCDL]), 16'sd1); - k_rcdrd <= 16'(T[I_RCDRD]); - k_rcdwr <= 16'(T[I_RCDWR]); - k_rp <= 16'(T[I_RP]); - k_ras <= 16'(T[I_RAS]); - k_rc <= 16'(T[I_RC]); - k_rtp <= 16'(T[I_RTP]); - k_rrds <= 16'(T[I_RRDS]); - k_rrdl <= 16'(T[I_RRDL]); - k_rfc <= $signed(cfg_rfc); - k_twoclk <= {cfg_core_clock, 1'b0}; - k_bb <= 3'(g_lpcs) + 3'(g_lbgs) + 3'(g_lbanks); - k_rowsh <= 5'(g_lpcs) + 5'(g_lbgs) + 5'(g_lbanks) + 5'(g_lrow); - k_bmask <= (25'd1 << (3'(g_lpcs) + 3'(g_lbgs) + 3'(g_lbanks))) - 25'd1; - end - - // The bank and row of a channel address (sim/common/dram_bank_model.h's - // decode): a row visit, then the bank bits, the row above the column, and - // the bank XORed with the row folded to its width. - // The row folded onto a bank field of each width a geometry may have, so - // the bank's width only selects among them. - function automatic [5:0] fold(input [ROW_W-1:0] r, input integer bb); - fold = '0; - for (integer k = 0; k < ROW_W; k += bb) - fold = fold ^ 6'((32'(r) >> k) & ((32'd1 << bb) - 32'd1)); - endfunction - - function automatic [BKW+ROW_W-1:0] decode(input [29:5] a); - reg [24:0] tx; - reg [24:0] b; - reg [ROW_W-1:0] r; - reg [5:0] f; - tx = a[29:5]; - b = (tx >> g_lvisit) & k_bmask; - r = ROW_W'(tx >> k_rowsh); - case (k_bb) - 3'd1: f = fold(r, 1); - 3'd2: f = fold(r, 2); - 3'd3: f = fold(r, 3); - 3'd4: f = fold(r, 4); - 3'd5: f = fold(r, 5); - default: f = fold(r, 6); - endcase - decode = {BKW'(b ^ 25'(f)), r}; - endfunction - - // ---- The port: slots, responses, requests in flight ---- - reg [SLOTS-1:0] slot_busy; - reg [ID_W-1:0] slot_id [SLOTS]; - reg [SW:0] slots_used; - reg [15:0] inflight; - reg cur_r_valid, cur_b_valid; - reg [SW-1:0] cur_r_slot; - reg [ID_W-1:0] cur_r_id, cur_b_id; - reg [31:0] tcyc, tcyc1; // the target cycle, and the next - - // Reads departed, in order, with the target cycle they are ready in; the - // first `rf_ready` are ready in the next cycle. A departure enters the - // host cycle after its DRAM clock ends. - // Ready cycles are kept modulo 2^24: an entry not yet counted ready is at - // most LATENCY + 1 cycles ahead. - reg [SW-1:0] rf_slot [SLOTS]; - reg [23:0] rf_cyc [SLOTS]; - reg [SW-1:0] rf_head, rf_tail, rf_scan; // rf_scan = rf_head + rf_ready - reg [SW:0] rf_count, rf_ready; - reg rfp_valid; - reg [SW-1:0] rfp_slot; - reg [23:0] rfp_cyc; - // Writes the controller has buffered, to acknowledge in order. - localparam AQW = $clog2(WSLOTS); - reg [ID_W-1:0] af_id [WSLOTS]; - reg [AQW-1:0] af_head, af_tail; - reg [AQW:0] af_count; - // What this target cycle's DRAM clocks made ready by the next one. - reg [SW:0] new_ready; - reg [AQW:0] new_acks; - // The last cycle's reads taken by AR and by R, for the latency sum. - reg lat_ar, lat_r; - reg [31:0] lat_t; - - assign room = (cfg_queue == 16'd0) | (inflight < cfg_queue); - assign slot_free = (32'(slots_used) < SLOTS); - assign r_valid = cur_r_valid; - assign r_id = cur_r_id; - assign r_slot = cur_r_slot; - assign b_valid = cur_b_valid; - assign b_id = cur_b_id; - assign no_reads = (slots_used == 0); - - // The lowest free slot once R has taken its read. - wire [SLOTS-1:0] free_slots = ~slot_busy | ((r_fire ? SLOTS'(1) : SLOTS'(0)) << cur_r_slot); - reg [SW-1:0] lowest_free; - always @(*) begin - lowest_free = '0; - for (integer s = SLOTS - 1; s >= 0; --s) - if (free_slots[s]) - lowest_free = SW'(s); - end - assign alloc_slot = lowest_free; - - // This cycle's requests, staged for the channel: a write, then a read. - reg st_w_valid, st_r_valid; - reg [ID_W-1:0] st_w_id; - reg [SW-1:0] st_r_slot; - reg [29:5] st_w_addr, st_r_addr; - - // ---- The channel ---- - // Requests in the order they formed, decoded; the controller's read and - // write buffers. - reg iq_w [INQ]; - reg [TAG_W-1:0] iq_tag [INQ]; - reg [BKW-1:0] iq_bank[INQ]; - reg [ROW_W-1:0] iq_row [INQ]; - reg [IQW-1:0] iq_head, iq_tail; - reg [IQW:0] iq_count; - reg [TAG_W-1:0] rb_tag [BQ], wb_tag [BQ]; - reg [BKW-1:0] rb_bank[BQ], wb_bank[BQ]; - reg [ROW_W-1:0] rb_row [BQ], wb_row [BQ]; - reg [BQW-1:0] rb_head, rb_tail, wb_head, wb_tail; - reg [BQW:0] rb_count, wb_count; - - reg [15:0] d; // the DRAM clock, modulo - reg [31:0] credit; - reg [15:0] ingress_ready; - reg [7:0] rq_cnt, wq_cnt; // reserved bursts whose column has not issued - reg wmode; - reg [15:0] frontier, batch_floor, batch_last, read_floor; - // Refresh: when the next is due, and the bank it is for. All-bank: a bank - // was opened since the last; the latest precharge and activate + tRC any - // bank allows; and no bank activates before ab_floor. - reg [15:0] ref_due; - reg [4:0] ref_bank; - reg ab_open; - reg [15:0] ab_prer, ab_actrc, ab_floor; - - // The bank table, written at one port. A fresh bank has no row open and - // no time that matters. - reg [31:0] bt_open, bt_fresh; - reg [ROW_W-1:0] bt_row [32]; - reg [15:0] bt_act [32]; - reg [15:0] bt_colr [32]; - reg [15:0] bt_prer [32]; - reg [4:0] scrub_b; - - wire [15:0] e = d + 16'd1; // the first clock a reservation may take - wire [HLOG2-1:0] base = HLOG2'(e); // its place on the ring - function automatic signed [15:0] rel(input [15:0] t); - rel = $signed(t - e); - endfunction - - // ---- Calendars ---- - // The command bus is two calendars, row and column commands, so a - // reservation marks both at once. - localparam B_CMDR = 0, B_ROW = 1, B_KR = 2, B_KW = 3, B_DEPV = 4, B_FRP = 5, B_FWP = 6, - B_FRG = 7, B_FWG = 8, B_FAP = 9, B_FAG = 10, B_CMDC = 11, NBM = 12; - - reg [WW-1:0] s_word; // the word the search reads - wire [WW-1:0] s_word1 = s_word + WW'(1); - reg op_w; - reg op_ref; // the operation is a refresh - reg [TAG_W-1:0] op_tag; - reg [BKW-1:0] op_bank; - reg [ROW_W-1:0] op_row; - reg op_pc; - reg [2:0] op_pg; - - wire [WW-1:0] end_word = WW'(base >> 6); - reg bm_clear; - reg [WW-1:0] bm_clear_word; - // A target reset (or the port leaving the bank model) clears the - // calendars in the host cycle after it; that cycle takes no DRAM clock. - reg bm_clear_all; - reg [NBM-1:0] wr_en; - reg [WW-1:0] wr_word [NBM]; - reg [63:0] wr_mask [NBM]; - wire [63:0] rd_cmdr, rd_cmdc, rd_row, rd_kr, rd_kw, rd_depv; - wire [63:0] rd_cmd = rd_cmdr | rd_cmdc; - /* verilator lint_off UNUSEDSIGNAL */ - wire [63:0] rd_row1; - /* verilator lint_on UNUSEDSIGNAL */ - wire [63:0] rd_frp, rd_fwp, rd_frg, rd_fwg, rd_fap, rd_fag; - - foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_cmdr ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(1'b0), .rd_word(s_word), .rd_data(rd_cmdr), - .we(wr_en[B_CMDR]), .wr_inst(1'b0), .wr_word(wr_word[B_CMDR]), .wr_mask(wr_mask[B_CMDR])); - foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_cmdc ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(1'b0), .rd_word(s_word), .rd_data(rd_cmdc), - .we(wr_en[B_CMDC]), .wr_inst(1'b0), .wr_word(wr_word[B_CMDC]), .wr_mask(wr_mask[B_CMDC])); - foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(2)) m_row ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(2'b00), .rd_word({s_word1, s_word}), .rd_data({rd_row1, rd_row}), - .we(wr_en[B_ROW]), .wr_inst(1'b0), .wr_word(wr_word[B_ROW]), .wr_mask(wr_mask[B_ROW])); - foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_kr ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(1'b0), .rd_word(end_word), .rd_data(rd_kr), - .we(wr_en[B_KR]), .wr_inst(1'b0), .wr_word(wr_word[B_KR]), .wr_mask(wr_mask[B_KR])); - foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_kw ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(1'b0), .rd_word(end_word), .rd_data(rd_kw), - .we(wr_en[B_KW]), .wr_inst(1'b0), .wr_word(wr_word[B_KW]), .wr_mask(wr_mask[B_KW])); - foldsim_mem_bmap #(.INST(1), .WORDS(WORDS), .NREAD(1)) m_depv ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(1'b0), .rd_word(end_word), .rd_data(rd_depv), - .we(wr_en[B_DEPV]), .wr_inst(1'b0), .wr_word(wr_word[B_DEPV]), .wr_mask(wr_mask[B_DEPV])); - foldsim_mem_bmap #(.INST(2), .WORDS(WORDS), .NREAD(1)) m_frp ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(op_pc), .rd_word(s_word), .rd_data(rd_frp), - .we(wr_en[B_FRP]), .wr_inst(op_pc), .wr_word(wr_word[B_FRP]), .wr_mask(wr_mask[B_FRP])); - foldsim_mem_bmap #(.INST(2), .WORDS(WORDS), .NREAD(1)) m_fwp ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(op_pc), .rd_word(s_word), .rd_data(rd_fwp), - .we(wr_en[B_FWP]), .wr_inst(op_pc), .wr_word(wr_word[B_FWP]), .wr_mask(wr_mask[B_FWP])); - foldsim_mem_bmap #(.INST(8), .WORDS(WORDS), .NREAD(1)) m_frg ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(op_pg), .rd_word(s_word), .rd_data(rd_frg), - .we(wr_en[B_FRG]), .wr_inst(op_pg), .wr_word(wr_word[B_FRG]), .wr_mask(wr_mask[B_FRG])); - foldsim_mem_bmap #(.INST(8), .WORDS(WORDS), .NREAD(1)) m_fwg ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(op_pg), .rd_word(s_word), .rd_data(rd_fwg), - .we(wr_en[B_FWG]), .wr_inst(op_pg), .wr_word(wr_word[B_FWG]), .wr_mask(wr_mask[B_FWG])); - foldsim_mem_bmap #(.INST(2), .WORDS(WORDS), .NREAD(1)) m_fap ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(op_pc), .rd_word(s_word), .rd_data(rd_fap), - .we(wr_en[B_FAP]), .wr_inst(op_pc), .wr_word(wr_word[B_FAP]), .wr_mask(wr_mask[B_FAP])); - foldsim_mem_bmap #(.INST(8), .WORDS(WORDS), .NREAD(1)) m_fag ( - .clk, .reset, .clear_all(bm_clear_all), .clear(bm_clear), .clear_word(bm_clear_word), - .rd_inst(op_pg), .rd_word(s_word), .rd_data(rd_fag), - .we(wr_en[B_FAG]), .wr_inst(op_pg), .wr_word(wr_word[B_FAG]), .wr_mask(wr_mask[B_FAG])); - - // The read each departure completes: a slot per clock of the ring. - reg [SW-1:0] dep_slot [H]; - - // ---- The engine ---- - localparam S_IDLE = 4'd0, S_PUSH = 4'd1, S_TICK = 4'd2, S_MODE = 4'd3, S_BANK = 4'd4, - S_BANK2 = 4'd5, S_SETUP = 4'd6, S_SETUP2 = 4'd7, S_SRCH = 4'd8, S_CHECK = 4'd9, - S_CHECK2 = 4'd10, S_COMMIT = 4'd11, S_END = 4'd12, S_WALK = 4'd13, - S_RBANK = 4'd14, S_RBANK2 = 4'd15; - localparam K_PRE = 3'd0, K_ACT = 3'd1, K_C0 = 3'd2, K_C1 = 3'd3, K_REF = 3'd4; - reg [3:0] state; - - assign done = (state == S_IDLE); - - // The operation: the request being reserved, and its commands, as clocks - // relative to e. - reg op_hit, op_conf; - reg signed [15:0] r_act, r_colr, r_prer, r_floor; - reg signed [15:0] c_pre, c_act, c_col0, c_col1, c_ref; - reg [2:0] kind; - - // The search: its lower bound; the next word to read (as the ring - // position and the clocks left to the horizon) and the one read last - // cycle, to decide on. - reg signed [15:0] s_lb; - reg [HLOG2-1:0] s_pos; - reg signed [15:0] s_left; - reg s_fetch; - reg sb_valid, b_last; - reg [63:0] b_cand; - reg [WW-1:0] b_word; - - function automatic [63:0] span_mask(input [5:0] off, input signed [15:0] left); - reg [6:0] top; - top = (left >= 16'sd64 - 16'(off)) ? 7'd64 : 7'(7'(off) + 7'(left)); - span_mask = (~64'd0 << off) & ((top == 7'd64) ? ~64'd0 : ~(~64'd0 << top)); - endfunction - - wire [5:0] s_off = s_pos[5:0]; - wire [63:0] s_mask = span_mask(s_off, s_left); - wire signed [15:0] s_take = (s_left >= 16'sd64 - 16'(s_off)) ? 16'sd64 - 16'(s_off) : s_left; - wire [63:0] f_col = rd_cmd | (op_w ? (rd_fwp | rd_fwg) : (rd_frp | rd_frg)); - wire [63:0] f_act = rd_cmd | rd_row | (g_act2 ? {rd_row1[0], rd_row[63:1]} : 64'd0) | rd_fap | rd_fag; - wire [63:0] f_pre = rd_cmd | rd_row; - wire [63:0] forb = ((kind == K_PRE) | (kind == K_REF)) ? f_pre : ((kind == K_ACT) ? f_act : f_col); - - reg [5:0] b_ffs; - always @(*) begin - b_ffs = '0; - for (integer j = 63; j >= 0; --j) - if (b_cand[j]) - b_ffs = 6'(j); - end - wire [HLOG2-1:0] b_pos = HLOG2'({b_word, b_ffs}); - wire signed [15:0] b_rel = 16'(HLOG2'(b_pos - base)); - - // ---- Commit: per calendar, up to two ranges of clocks to mark ---- - // (computed from the reservation, registered as the horizon is checked) - reg cx_v [NBM][2]; - reg signed [15:0] cx_lo [NBM][2]; - reg signed [15:0] cx_hi [NBM][2]; - reg cm_v [NBM][2]; - reg signed [15:0] cm_lo [NBM][2]; - reg signed [15:0] cm_hi [NBM][2]; - reg [HLOG2-1:0] cm_pos [NBM]; - reg signed [15:0] cm_left[NBM]; - reg cm_busy [NBM]; - reg cm_next [NBM]; // the second range is still to mark - reg cm_start; - - reg signed [15:0] ck_colr, ck_prer, ck_lastcol, ck_tcol, ck_tdep, ck_tact; - reg cm_any; - always @(*) begin - cm_any = 1'b0; - for (integer m = 0; m < NBM; ++m) - cm_any = cm_any | cm_busy[m]; - end - - // Clock d+1, read when its DRAM clock starts: the read departing in it and - // the columns issued in it -- and the columns the clock's own reservation - // places in it. - reg ed_depv, ed_kr, ed_kw; - reg [SW-1:0] ed_slot; - - // Ingress head, and whether it enters its buffer this tick. - wire ih_w = iq_w[iq_head]; - wire [8:0] wb_after = 9'(wq_cnt) + 9'(NB) * (9'(wb_count) + 9'd1); - wire [8:0] rb_after = 9'(rq_cnt) + 9'(NB) * (9'(rb_count) + 9'd1); - wire ih_time = ($signed(d - ingress_ready) >= 0); - wire ih_room = ih_w ? (wb_after <= 9'(cfg_wq)) : (rb_after <= 9'(cfg_rq)); - wire move = (iq_count != 0) & ih_time & ih_room; - // The buffers' bursts, and the mode they put the channel in. - wire [8:0] wn = 9'(wq_cnt) + 9'(NB) * 9'(wb_count); - wire [8:0] rn = 9'(rq_cnt) + 9'(NB) * 9'(rb_count); - wire to_write = ~wmode & ((wn >= 9'(cfg_wr_high)) | (rn == 0)); - wire to_read = wmode & (wn < 9'(cfg_wr_low)) & (rn != 0); - wire wmode_n = to_write | (wmode & ~to_read); - wire sel_any = wmode_n ? (wb_count != 0) : (rb_count != 0); - wire ref_now = cfg_refresh[0] & (cfg_refi != 16'd0) & ($signed(e - ref_due) >= 0); - - // The bank table's port: a reservation's bank, or the scrub's. - // The scrub reads a bank in one host cycle and writes it in the next. - reg sc_pend; - reg [4:0] sc_b; - reg [15:0] sc_act, sc_colr, sc_prer; - reg [ROW_W-1:0] sc_row; - reg [15:0] cw_act, cw_colr, cw_prer; // the reservation's, absolute - reg [ROW_W-1:0] cw_row; - wire bt_commit = (state == S_COMMIT) & cm_start & ~(op_ref & cfg_refresh[1]); - wire [4:0] bt_wa = bt_commit ? op_bank : sc_b; - function automatic [15:0] scrubbed(input [15:0] t); - scrubbed = ($signed(t - (e + 16'd1)) < -16'sd16384) ? e + 16'd1 - 16'd16384 : t; - endfunction - always @(posedge clk) begin - if (bt_commit | sc_pend) begin - bt_row[bt_wa] <= bt_commit ? cw_row : sc_row; - bt_act[bt_wa] <= bt_commit ? cw_act : sc_act; - bt_colr[bt_wa] <= bt_commit ? cw_colr : sc_colr; - bt_prer[bt_wa] <= bt_commit ? cw_prer : sc_prer; - end - end - - // The end of DRAM clock d: clock d+1's departing read and the columns it - // issues leave the controller; then the next clock. - task automatic end_clock(); - rfp_valid <= ed_depv; - rfp_slot <= ed_slot; - rfp_cyc <= 24'(tcyc1 + 32'(cfg_latency)); - rq_cnt <= rq_cnt - (ed_kr ? 8'd1 : 8'd0); - wq_cnt <= wq_cnt - (ed_kw ? 8'd1 : 8'd0); - // The ring's word ends with this clock: it is all past. - if (base[5:0] == 6'd63) begin - bm_clear <= 1'b1; - bm_clear_word <= end_word; - end - if ($signed(frontier - e) < 0) frontier <= e; - if ($signed(batch_last - e) < 0) batch_last <= e; - if ($signed(batch_floor - e) < 0) batch_floor <= e; - if ($signed(read_floor - e) < 0) read_floor <= e; - if ($signed(ingress_ready - e) < 0) ingress_ready <= e; - if ($signed(ab_prer - e) < 0) ab_prer <= e; - if ($signed(ab_actrc - e) < 0) ab_actrc <= e; - if ($signed(ab_floor - e) < 0) ab_floor <= e; - d <= d + 16'd1; - endtask - - always @(posedge clk) begin - bm_clear <= 1'b0; - bm_clear_all <= 1'b0; - cm_start <= 1'b0; - rfp_valid <= 1'b0; - sc_pend <= 1'b0; - - if (rfp_valid) begin - rf_slot[rf_tail] <= rfp_slot; - rf_cyc[rf_tail] <= rfp_cyc; - rf_tail <= rf_tail + 1'b1; - rf_count <= rf_count + 1'b1; - end - lat_ar <= 1'b0; - lat_r <= 1'b0; - if (lat_ar | lat_r) - stat_latency <= stat_latency + (lat_r ? 64'(lat_t) : 64'd0) - (lat_ar ? 64'(lat_t) : 64'd0); - - // The engine runs while the target evaluates its cycle; the cycle - // finishes only once the engine is idle. - case (state) - S_PUSH: begin - // A cycle's write reaches the channel before its read. - if (st_w_valid) begin - iq_w[iq_tail] <= 1'b1; - iq_tag[iq_tail] <= TAG_W'(st_w_id); - {iq_bank[iq_tail], iq_row[iq_tail]} <= decode(st_w_addr[29:5]); - iq_tail <= iq_tail + 1'b1; - iq_count <= iq_count + 1'b1; - st_w_valid <= 1'b0; - end else begin - if (st_r_valid) begin - iq_w[iq_tail] <= 1'b0; - iq_tag[iq_tail] <= TAG_W'(st_r_slot); - {iq_bank[iq_tail], iq_row[iq_tail]} <= decode(st_r_addr[29:5]); - iq_tail <= iq_tail + 1'b1; - iq_count <= iq_count + 1'b1; - st_r_valid <= 1'b0; - end - state <= S_TICK; - end - end - S_TICK: begin - if (credit >= 32'(k_twoclk)) begin - // One DRAM clock d: the ingress head enters its buffer if it - // may. - credit <= credit - 32'(k_twoclk); - if (move) begin - iq_head <= iq_head + 1'b1; - iq_count <= iq_count - 1'b1; - ingress_ready <= d + 16'(NB); - if (ih_w) begin - wb_tag[wb_tail] <= iq_tag[iq_head]; - wb_bank[wb_tail] <= iq_bank[iq_head]; - wb_row[wb_tail] <= iq_row[iq_head]; - wb_tail <= wb_tail + 1'b1; - wb_count <= wb_count + 1'b1; - af_id[af_tail] <= ID_W'(iq_tag[iq_head]); - af_tail <= af_tail + 1'b1; - af_count <= af_count + 1'b1; - new_acks <= new_acks + 1'b1; - end else begin - rb_tag[rb_tail] <= iq_tag[iq_head]; - rb_bank[rb_tail] <= iq_bank[iq_head]; - rb_row[rb_tail] <= iq_row[iq_head]; - rb_tail <= rb_tail + 1'b1; - rb_count <= rb_count + 1'b1; - end - end - ed_depv <= rd_depv[base[5:0]]; - ed_kr <= rd_kr[base[5:0]]; - ed_kw <= rd_kw[base[5:0]]; - ed_slot <= dep_slot[base]; - state <= S_MODE; - end else begin - // The cycle's clocks are done; a bank's times move up (the - // table's port is free), and the ready reads are counted. - sc_pend <= ~bt_fresh[scrub_b]; - sc_b <= scrub_b; - sc_row <= bt_row[scrub_b]; - sc_act <= scrubbed(bt_act[scrub_b]); - sc_colr <= scrubbed(bt_colr[scrub_b]); - sc_prer <= scrubbed(bt_prer[scrub_b]); - scrub_b <= scrub_b + 1'b1; - state <= S_WALK; - end - end - S_MODE: begin - // The channel picks reads or writes; the head of that buffer is - // reserved if it can be, and with nothing to reserve the clock - // ends now. - wmode <= wmode_n; - if (to_write) - batch_floor <= ($signed(frontier + 16'd1 - e) > 0) ? frontier + 16'd1 : e; - if (to_read) - read_floor <= batch_last + 16'd1; - if (ref_now) begin - state <= S_RBANK; - end else if (sel_any) begin - state <= S_BANK; - end else begin - end_clock(); - state <= S_TICK; - end - end - S_BANK: begin - // The request at the head of the buffer the channel serves. - op_ref <= 1'b0; - op_w <= wmode; - op_tag <= wmode ? wb_tag[wb_head] : rb_tag[rb_head]; - op_bank <= wmode ? wb_bank[wb_head] : rb_bank[rb_head]; - op_row <= wmode ? wb_row[wb_head] : rb_row[rb_head]; - r_floor <= rel(wmode ? batch_floor : read_floor); - state <= S_BANK2; - end - S_BANK2: begin - // Against its bank. - op_pc <= (g_lpcs != 0) ? op_bank[0] : 1'b0; - op_pg <= 3'((32'(op_bank) & ((32'd1 << (32'(g_lpcs) + 32'(g_lbgs))) - 32'd1))); - op_hit <= bt_open[op_bank] & (bt_row[op_bank] == op_row); - op_conf <= bt_open[op_bank] & (bt_row[op_bank] != op_row); - r_act <= bt_fresh[op_bank] ? -16'sd16384 : rel(bt_act[op_bank]); - r_colr <= bt_fresh[op_bank] ? -16'sd1 : rel(bt_colr[op_bank]); - r_prer <= bt_fresh[op_bank] ? -16'sd1 : rel(bt_prer[op_bank]); - kind <= (bt_open[op_bank] & (bt_row[op_bank] == op_row)) ? K_C0 - : (bt_open[op_bank] ? K_PRE : K_ACT); - state <= S_SETUP; - end - S_SETUP: begin - // The search's lower bound, the first clock it may place at. - case (kind) - K_PRE: s_lb <= smax(16'sd0, r_prer); - K_ACT: s_lb <= smax(smax(op_conf ? c_pre + k_rp : 16'sd0, r_act + k_rc), rel(ab_floor)); - K_REF: s_lb <= smax(op_conf ? c_pre + k_rp : 16'sd0, r_act + k_rc); - K_C0: s_lb <= smax(smax(16'sd0, r_floor), - op_hit ? r_colr : c_act + (op_w ? k_rcdwr : k_rcdrd)); - default: s_lb <= c_col0 + k_ccdl1; - endcase - state <= S_SETUP2; - end - S_SETUP2: begin - if (s_lb >= 16'(HE)) begin - state <= S_END; - end else begin - s_pos <= HLOG2'(base + HLOG2'(s_lb)); - s_left <= 16'(HE) - s_lb; - s_word <= WW'(HLOG2'(base + HLOG2'(s_lb)) >> 6); - s_fetch <= 1'b1; - sb_valid <= 1'b0; - state <= S_SRCH; - end - end - S_SRCH: begin - // A word a cycle: the word read now, masked to the clocks left; - // and last cycle's word, whose first free clock wins. - if (s_fetch) begin - b_cand <= ~forb & s_mask; - b_word <= s_word; - sb_valid <= 1'b1; - b_last <= (s_left - s_take == 0); - s_pos <= HLOG2'(s_pos + HLOG2'(s_take)); - s_word <= WW'(HLOG2'(s_pos + HLOG2'(s_take)) >> 6); - s_left <= s_left - s_take; - s_fetch <= (s_left - s_take != 0); - end - if (sb_valid) begin - if (b_cand != 0) begin - case (kind) - K_PRE: begin c_pre <= b_rel; kind <= op_ref ? K_REF : K_ACT; state <= S_SETUP; end - K_REF: begin c_ref <= b_rel; state <= S_CHECK; end - K_ACT: begin c_act <= b_rel; kind <= K_C0; state <= S_SETUP; end - K_C0: begin - c_col0 <= b_rel; - if (NB == 2) begin - kind <= K_C1; - state <= S_SETUP; - end else begin - state <= S_CHECK; - end - end - default: begin c_col1 <= b_rel; state <= S_CHECK; end - endcase - end else if (b_last) begin - state <= S_END; - end - end - end - S_CHECK: begin - // How far the reservation reaches, and the bank's new times. - begin : check_ - reg signed [15:0] lastcol, colr, prer; - lastcol = (NB == 2) ? c_col1 : c_col0; - colr = op_hit ? r_colr : c_act + (op_w ? k_rcdwr : k_rcdrd); - prer = op_hit ? r_prer : c_act + k_ras; - ck_lastcol <= lastcol; - ck_colr <= smax(colr, lastcol + 16'sd1); - ck_prer <= smax(prer, lastcol + (op_w ? k_wrpre : k_rtp)); - ck_tcol <= lastcol + (op_w ? k_reach_w : k_reach_r); - ck_tdep <= op_w ? -16'sd16384 : lastcol + k_dep; - ck_tact <= op_hit ? -16'sd16384 : c_act + k_reach_a; - // A refresh's commands are within the horizon by its search. - if (op_ref) begin - ck_tcol <= -16'sd16384; - ck_tdep <= -16'sd16384; - ck_tact <= -16'sd16384; - end - end - state <= S_CHECK2; - end - S_CHECK2: begin - // The reservation must reach no further than the horizon. - for (integer m = 0; m < NBM; ++m) begin - for (integer k = 0; k < 2; ++k) begin - cm_v[m][k] <= cx_v[m][k]; - cm_lo[m][k] <= cx_lo[m][k]; - cm_hi[m][k] <= cx_hi[m][k]; - end - end - cw_row <= (op_hit | op_ref) ? bt_row[op_bank] : op_row; - cw_act <= op_ref ? e + c_ref + k_rfc - k_rc : (op_hit ? bt_act[op_bank] : e + c_act); - cw_colr <= op_ref ? bt_colr[op_bank] : e + ck_colr; - cw_prer <= op_ref ? bt_prer[op_bank] : e + ck_prer; - begin : check2_ - reg ok; - ok = (ck_tcol < 16'(HE)) & (ck_tdep < 16'(HE)) & (ck_tact < 16'(HE)); - state <= ok ? S_COMMIT : S_END; - cm_start <= ok; - end - end - S_COMMIT: begin - if (cm_start & op_ref) begin - // A refresh closes its bank (every bank), and the next is due. - if (cfg_refresh[1]) begin - bt_open <= '0; - ab_open <= 1'b0; - ab_floor <= e + c_ref + k_rfc; - ab_actrc <= e + c_ref + k_rfc; - end else begin - bt_open[op_bank] <= 1'b0; - bt_fresh[op_bank] <= 1'b0; - ref_bank <= (ref_bank + 5'd1) & k_bmask[4:0]; - end - ref_due <= ref_due + cfg_refi; - stat_refreshes <= stat_refreshes + 64'd1; - end else if (cm_start) begin - // The bank (its times through the table's port), the - // controller, the read's departure, and a column in clock d+1 - // for the clock's end. - if (~op_hit) - bt_open[op_bank] <= 1'b1; - bt_fresh[op_bank] <= 1'b0; - ab_open <= 1'b1; - if ($signed(cw_prer - ab_prer) > 0) - ab_prer <= cw_prer; - if (~op_hit && $signed(e + c_act + k_rc - ab_actrc) > 0) - ab_actrc <= e + c_act + k_rc; - if (op_w) begin - wq_cnt <= wq_cnt + 8'(NB); - wb_head <= wb_head + 1'b1; - wb_count <= wb_count - 1'b1; - batch_last <= ($signed(e + ck_lastcol - batch_last) > 0) ? e + ck_lastcol : batch_last; - stat_writes <= stat_writes + 64'd1; - if (c_col0 == 0) ed_kw <= 1'b1; - end else begin - rq_cnt <= rq_cnt + 8'(NB); - rb_head <= rb_head + 1'b1; - rb_count <= rb_count - 1'b1; - dep_slot[HLOG2'(base + HLOG2'(ck_lastcol + k_dep))] <= SW'(op_tag); - stat_reads <= stat_reads + 64'd1; - if (c_col0 == 0) ed_kr <= 1'b1; - end - frontier <= ($signed(e + ck_lastcol - frontier) > 0) ? e + ck_lastcol : frontier; - if (op_hit) - stat_hits <= stat_hits + 64'd1; - else if (op_conf) - stat_conflicts <= stat_conflicts + 64'd1; - else - stat_misses <= stat_misses + 64'd1; - end else if (!cm_any) begin - state <= S_END; - end - end - S_END: begin - end_clock(); - state <= S_TICK; - end - S_RBANK: begin - // A refresh: the bank in turn. - op_ref <= 1'b1; - op_w <= 1'b0; - op_bank <= ref_bank; - state <= S_RBANK2; - end - S_RBANK2: begin - // Precharged first if open, then REF: against the bank, or the - // channel's latest times for all banks. - op_hit <= 1'b0; - if (cfg_refresh[1]) begin - op_conf <= ab_open; - r_prer <= rel(ab_prer); - r_act <= rel(ab_actrc) - k_rc; - kind <= ab_open ? K_PRE : K_REF; - end else begin - op_conf <= bt_open[op_bank]; - r_prer <= bt_fresh[op_bank] ? -16'sd1 : rel(bt_prer[op_bank]); - r_act <= bt_fresh[op_bank] ? -16'sd16384 : rel(bt_act[op_bank]); - kind <= bt_open[op_bank] ? K_PRE : K_REF; - end - state <= S_SETUP; - end - S_WALK: begin - // Reads that are ready in the next target cycle. - if (rf_ready != rf_count && $signed(rf_cyc[rf_scan] - tcyc1[23:0]) <= 0) begin - rf_ready <= rf_ready + 1'b1; - rf_scan <= rf_scan + 1'b1; - new_ready <= new_ready + 1'b1; - end else begin - state <= S_IDLE; - end - end - default:; - endcase - - if (reset || finish) begin - // ---- The target cycle ends: the port's state for the next ---- - if (finish && treset) - active <= 1'b1; - if (reset || treset || ~active) begin - slot_busy <= '0; - slots_used <= '0; - inflight <= '0; - cur_r_valid <= 1'b0; - cur_b_valid <= 1'b0; - tcyc <= '0; - tcyc1 <= 32'd1; - rf_head <= '0; - rf_tail <= '0; - rf_count <= '0; - rf_ready <= '0; - rf_scan <= '0; - af_head <= '0; - af_tail <= '0; - af_count <= '0; - st_w_valid <= 1'b0; - st_r_valid <= 1'b0; - iq_head <= '0; - iq_tail <= '0; - iq_count <= '0; - rb_head <= '0; - rb_tail <= '0; - rb_count <= '0; - wb_head <= '0; - wb_tail <= '0; - wb_count <= '0; - d <= '0; - credit <= 32'(cfg_data_rate); - ingress_ready <= '0; - rq_cnt <= '0; - wq_cnt <= '0; - wmode <= 1'b0; - frontier <= '0; - batch_floor <= '0; - batch_last <= '0; - read_floor <= '0; - bt_open <= '0; - bt_fresh <= '1; - scrub_b <= '0; - bm_clear_all <= 1'b1; - ref_due <= cfg_refi; - ref_bank <= '0; - ab_open <= 1'b0; - ab_prer <= '0; - ab_actrc <= '0; - ab_floor <= '0; - end else begin - tcyc <= tcyc1; - tcyc1 <= tcyc1 + 32'd1; - if (~(room & slot_free)) - stat_full <= stat_full + 64'd1; - // R and B, then what is ready next cycle. - begin : release_ - reg r_load, b_load; - r_load = (~cur_r_valid | r_fire) & (rf_ready != 0); - b_load = (~cur_b_valid | b_fire) & (af_count != 0); - if (r_load) begin - cur_r_valid <= 1'b1; - cur_r_slot <= rf_slot[rf_head]; - cur_r_id <= slot_id[rf_slot[rf_head]]; - rf_head <= rf_head + 1'b1; - rf_count <= rf_count - 1'b1; - rf_ready <= rf_ready - 1'b1; - end else if (r_fire) begin - cur_r_valid <= 1'b0; - end - if (b_load) begin - cur_b_valid <= 1'b1; - cur_b_id <= af_id[af_head]; - af_head <= af_head + 1'b1; - af_count <= af_count - 1'b1; - end else if (b_fire) begin - cur_b_valid <= 1'b0; - end - if (r_fire) - slot_busy[cur_r_slot] <= 1'b0; - if (ar_fire) begin - slot_busy[alloc_slot] <= 1'b1; - slot_id[alloc_slot] <= ar_id; - end - slots_used <= slots_used + (SW+1)'(ar_fire) - (SW+1)'(r_fire); - inflight <= inflight + 16'(ar_fire) + 16'(aw_fire) - 16'(new_ready) - 16'(new_acks); - lat_ar <= ar_fire; - lat_r <= r_fire; - lat_t <= tcyc; - end - st_w_valid <= new_write; - st_w_id <= nw_id; - st_w_addr <= nw_addr[29:5]; - st_r_valid <= ar_fire; - st_r_slot <= alloc_slot; - st_r_addr <= ar_addr[29:5]; - credit <= credit + 32'(cfg_data_rate); - end - // A clearing cycle's next host cycle clears the calendars, so the - // first DRAM clock waits a host cycle (in S_PUSH). - state <= ~(treset | active) ? S_IDLE - : (((treset | ~active) | new_write | ar_fire) ? S_PUSH : S_TICK); - new_ready <= '0; - new_acks <= '0; - end - if (reset) begin - state <= S_IDLE; - active <= 1'b0; - stat_reads <= '0; - stat_writes <= '0; - stat_hits <= '0; - stat_misses <= '0; - stat_conflicts <= '0; - stat_latency <= '0; - stat_refreshes <= '0; - stat_full <= '0; - lat_ar <= 1'b0; - lat_r <= 1'b0; - rfp_valid <= 1'b0; - sc_pend <= 1'b0; - end - end - - // ---- Commit: ranges of clocks marked in each calendar, a word a cycle ---- - function automatic signed [15:0] clamp0(input signed [15:0] v); - clamp0 = (v < 0) ? 16'sd0 : v; - endfunction - - // The ranges each calendar marks, from the reservation; an empty one is - // dropped. - always @(*) begin - for (integer m = 0; m < NBM; ++m) begin - for (integer k = 0; k < 2; ++k) begin - cx_v[m][k] = 1'b0; - cx_lo[m][k] = 16'sd0; - cx_hi[m][k] = 16'sd0; - end - end - // Row commands: PRE, ACT; the row bus for PRE, and ACT's one or two - // clocks. Column commands, and their directions for the controller's - // queues. - cx_v[B_CMDR][0] = op_conf; cx_lo[B_CMDR][0] = c_pre; cx_hi[B_CMDR][0] = c_pre; - cx_v[B_CMDR][1] = ~op_hit; cx_lo[B_CMDR][1] = c_act; cx_hi[B_CMDR][1] = c_act; - cx_v[B_ROW][0] = op_conf; cx_lo[B_ROW][0] = c_pre; cx_hi[B_ROW][0] = c_pre; - cx_v[B_ROW][1] = ~op_hit; cx_lo[B_ROW][1] = c_act; - cx_hi[B_ROW][1] = c_act + (g_act2 ? 16'sd1 : 16'sd0); - cx_v[B_CMDC][0] = 1'b1; cx_lo[B_CMDC][0] = c_col0; cx_hi[B_CMDC][0] = c_col0; - cx_v[B_CMDC][1] = (NB == 2); cx_lo[B_CMDC][1] = c_col1; cx_hi[B_CMDC][1] = c_col1; - cx_v[op_w ? B_KW : B_KR][0] = 1'b1; - cx_lo[op_w ? B_KW : B_KR][0] = c_col0; cx_hi[op_w ? B_KW : B_KR][0] = c_col0; - cx_v[op_w ? B_KW : B_KR][1] = (NB == 2); - cx_lo[op_w ? B_KW : B_KR][1] = c_col1; cx_hi[op_w ? B_KW : B_KR][1] = c_col1; - // The read's departure. - cx_v[B_DEPV][0] = ~op_w; - cx_lo[B_DEPV][0] = ck_lastcol + k_dep; cx_hi[B_DEPV][0] = ck_lastcol + k_dep; - // What each column forbids to later columns of its pseudo-channel and - // bank group, by direction. - for (integer k = 0; k < NB; ++k) begin - reg signed [15:0] c; - c = (k == 0) ? c_col0 : c_col1; - if (~op_w) begin - cx_v[B_FRP][k] = 1'b1; cx_lo[B_FRP][k] = clamp0(c - k_ccds + 16'sd1); cx_hi[B_FRP][k] = c + k_ccds - 16'sd1; - cx_v[B_FWP][k] = 1'b1; cx_lo[B_FWP][k] = clamp0(c - k_wtrs + 16'sd1); cx_hi[B_FWP][k] = c + k_rtw - 16'sd1; - cx_v[B_FRG][k] = 1'b1; cx_lo[B_FRG][k] = clamp0(c - k_ccdl + 16'sd1); cx_hi[B_FRG][k] = c + k_ccdl - 16'sd1; - cx_v[B_FWG][k] = 1'b1; cx_lo[B_FWG][k] = clamp0(c - k_wtrl + 16'sd1); cx_hi[B_FWG][k] = c; - end else begin - cx_v[B_FWP][k] = 1'b1; cx_lo[B_FWP][k] = clamp0(c - k_ccds + 16'sd1); cx_hi[B_FWP][k] = c + k_ccds - 16'sd1; - cx_v[B_FRP][k] = 1'b1; cx_lo[B_FRP][k] = clamp0(c - k_rtw + 16'sd1); cx_hi[B_FRP][k] = c + k_wtrs - 16'sd1; - cx_v[B_FWG][k] = 1'b1; cx_lo[B_FWG][k] = clamp0(c - k_ccdl + 16'sd1); cx_hi[B_FWG][k] = c + k_ccdl - 16'sd1; - cx_v[B_FRG][k] = 1'b1; cx_lo[B_FRG][k] = c; cx_hi[B_FRG][k] = c + k_wtrl - 16'sd1; - end - end - // What the ACT forbids to later ACTs: tRRD within the pseudo-channel, - // tRRD_L within the bank group. - cx_v[B_FAP][0] = ~op_hit; - cx_lo[B_FAP][0] = clamp0(c_act - k_rrds + 16'sd1); cx_hi[B_FAP][0] = c_act + k_rrds - 16'sd1; - cx_v[B_FAG][0] = ~op_hit; - cx_lo[B_FAG][0] = clamp0(c_act - k_rrdl + 16'sd1); cx_hi[B_FAG][0] = c_act + k_rrdl - 16'sd1; - // A refresh: its precharge, if any, and REF, on the row-command bus. - if (op_ref) begin - for (integer m = 0; m < NBM; ++m) - for (integer k = 0; k < 2; ++k) - cx_v[m][k] = 1'b0; - cx_v[B_CMDR][0] = op_conf; cx_lo[B_CMDR][0] = c_pre; cx_hi[B_CMDR][0] = c_pre; - cx_v[B_CMDR][1] = 1'b1; cx_lo[B_CMDR][1] = c_ref; cx_hi[B_CMDR][1] = c_ref; - cx_v[B_ROW][0] = op_conf; cx_lo[B_ROW][0] = c_pre; cx_hi[B_ROW][0] = c_pre; - cx_v[B_ROW][1] = 1'b1; cx_lo[B_ROW][1] = c_ref; cx_hi[B_ROW][1] = c_ref; - end - for (integer m = 0; m < NBM; ++m) - for (integer k = 0; k < 2; ++k) - if (cx_hi[m][k] < cx_lo[m][k]) - cx_v[m][k] = 1'b0; - end - - // A word a cycle per calendar; a range's last word starts the next. - always @(posedge clk) begin - wr_en <= '0; - for (integer m = 0; m < NBM; ++m) begin - if (reset) begin - cm_busy[m] <= 1'b0; - end else if (cm_start) begin - cm_busy[m] <= cm_v[m][0] | cm_v[m][1]; - cm_next[m] <= cm_v[m][0] & cm_v[m][1]; - cm_pos[m] <= HLOG2'(base + HLOG2'(cm_v[m][0] ? cm_lo[m][0] : cm_lo[m][1])); - cm_left[m] <= cm_v[m][0] ? cm_hi[m][0] - cm_lo[m][0] + 16'sd1 - : cm_hi[m][1] - cm_lo[m][1] + 16'sd1; - end else if (cm_busy[m]) begin - reg signed [15:0] take; - reg [5:0] off; - off = cm_pos[m][5:0]; - take = (cm_left[m] >= 16'sd64 - 16'(off)) ? 16'sd64 - 16'(off) : cm_left[m]; - wr_en[m] <= 1'b1; - wr_word[m] <= WW'(cm_pos[m] >> 6); - wr_mask[m] <= span_mask(off, cm_left[m]); - if (cm_left[m] == take) begin - cm_busy[m] <= cm_next[m]; - cm_next[m] <= 1'b0; - cm_pos[m] <= HLOG2'(base + HLOG2'(cm_lo[m][1])); - cm_left[m] <= cm_hi[m][1] - cm_lo[m][1] + 16'sd1; - end else begin - cm_pos[m] <= HLOG2'(cm_pos[m] + HLOG2'(take)); - cm_left[m] <= cm_left[m] - take; - end - end - end - end - -endmodule diff --git a/hw/rtl/afu/foldsim/foldsim_mem_bmap.sv b/hw/rtl/afu/foldsim/foldsim_mem_bmap.sv deleted file mode 100644 index efe7aef633..0000000000 --- a/hw/rtl/afu/foldsim/foldsim_mem_bmap.sv +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// `INST` bitmaps over the bank model's ring of DRAM clocks (foldsim_mem_bank), -// each WORDS words of 64 clocks, in distributed RAM. A word with no write -// since the last `clear` of it reads zero: a valid bit per word stands for -// the clearing, so a word of every bitmap is cleared in one cycle. A write -// ORs a mask into a word. -module foldsim_mem_bmap #( - parameter INST = 1, - parameter WORDS = 8, - parameter NREAD = 1, - parameter IW = (INST > 1) ? $clog2(INST) : 1, - parameter WW = $clog2(WORDS) -) ( - input wire clk, - input wire reset, - // Every word of every bitmap reads zero from the next cycle on. - input wire clear_all, - // Word `clear_word` of every bitmap reads zero from the next cycle on. - input wire clear, - input wire [WW-1:0] clear_word, - - input wire [NREAD*IW-1:0] rd_inst, - input wire [NREAD*WW-1:0] rd_word, - output wire [NREAD*64-1:0] rd_data, - - input wire we, - input wire [IW-1:0] wr_inst, - input wire [WW-1:0] wr_word, - input wire [63:0] wr_mask -); - localparam N = INST * WORDS; - localparam AW = (N > 1) ? $clog2(N) : 1; - - reg [63:0] mem [N]; - reg [N-1:0] valid; - - function automatic [AW-1:0] addr(input [IW-1:0] inst, input [WW-1:0] word); - addr = AW'(32'(inst) * WORDS + 32'(word)); - endfunction - - for (genvar i = 0; i < NREAD; ++i) begin : g_rd - wire [AW-1:0] a = addr(rd_inst[i*IW +: IW], rd_word[i*WW +: WW]); - assign rd_data[i*64 +: 64] = valid[a] ? mem[a] : 64'd0; - end - - wire [AW-1:0] wa = addr(wr_inst, wr_word); - wire [63:0] old = valid[wa] ? mem[wa] : 64'd0; - - always @(posedge clk) begin - if (we) - mem[wa] <= old | wr_mask; - end - - always @(posedge clk) begin - if (reset || clear_all) begin - valid <= '0; - end else begin - if (clear) begin - for (integer k = 0; k < INST; ++k) - valid[k * WORDS + 32'(clear_word)] <= 1'b0; - end - if (we) - valid[wa] <= 1'b1; - end - end - -endmodule diff --git a/hw/rtl/afu/foldsim/foldsim_regs.vh b/hw/rtl/afu/foldsim/foldsim_regs.vh deleted file mode 100644 index dcf1ba7c3c..0000000000 --- a/hw/rtl/afu/foldsim/foldsim_regs.vh +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The foldsim simulator's register map, as sw/common/foldsim_regs.h gives it to -// the runtime. Byte offsets on s_axi_control; unlisted offsets read zero. -// -// 0x040-0x0BF is left empty: on the V80 the PCIe block's MSI-X table sits at -// that offset of the BAR the first kernel is mapped at, and answers those -// accesses itself. - -`ifndef FOLDSIM_REGS_VH -`define FOLDSIM_REGS_VH - -`define FOLDSIM_ID 32'h464F4C44 -`define FOLDSIM_VERSION 32'd5 - -`define FOLDSIM_REG_ID 12'h000 -`define FOLDSIM_REG_VERSION 12'h004 -`define FOLDSIM_REG_CTRL 12'h008 -`define FOLDSIM_REG_STATUS 12'h00C -`define FOLDSIM_REG_CMD_ADDR 12'h010 -`define FOLDSIM_REG_CMD_DATA 12'h014 -`define FOLDSIM_REG_CMD 12'h018 -`define FOLDSIM_REG_DCR_RSP 12'h01C -`define FOLDSIM_REG_TCYCLES_LO 12'h020 -`define FOLDSIM_REG_TCYCLES_HI 12'h024 -`define FOLDSIM_REG_HCYCLES_LO 12'h028 -`define FOLDSIM_REG_HCYCLES_HI 12'h02C -`define FOLDSIM_REG_MSTALL_LO 12'h030 -`define FOLDSIM_REG_MSTALL_HI 12'h034 -`define FOLDSIM_REG_NUM_CLASSES 12'h038 -`define FOLDSIM_REG_MEM_BASE_LO 12'h0C0 -`define FOLDSIM_REG_MEM_BASE_HI 12'h0C4 -// The memory model (sw/common/foldsim_regs.h has the fields). -`define FOLDSIM_REG_MEM_REFRESH 12'h0D8 -`define FOLDSIM_REG_MEM_CAPS 12'h0DC -`define FOLDSIM_REG_MEM_CLOCK 12'h0E0 -`define FOLDSIM_REG_MEM_LATENCY 12'h0E4 -`define FOLDSIM_REG_MEM_QUEUE 12'h0E8 -`define FOLDSIM_REG_MEM_GEOM 12'h0EC -`define FOLDSIM_REG_MEM_WMARK 12'h0F0 -`define FOLDSIM_REG_MEM_REFI 12'h0F4 -`define FOLDSIM_REG_MEM_RFC 12'h0F8 -// Per class c: the host cycles a thread of it finished its target cycle. -`define FOLDSIM_REG_FINISH_BASE 12'h100 -// The bank model's DRAM timings, a word each, and per memory port p its -// counters, 64-bit each, from FOLDSIM_REG_MEM_STATS + 0x40 * p. -`define FOLDSIM_REG_MEM_TIMING 12'h800 -`define FOLDSIM_REG_MEM_STATS 12'hC00 - -`define FOLDSIM_CMD_NONE 3'd0 -`define FOLDSIM_CMD_DCR_WRITE 3'd1 -`define FOLDSIM_CMD_DCR_READ 3'd2 -`define FOLDSIM_CMD_START 3'd3 -`define FOLDSIM_CMD_RESET 3'd4 - -`endif // FOLDSIM_REGS_VH diff --git a/hw/rtl/afu/foldsim/foldsim_token.sv b/hw/rtl/afu/foldsim/foldsim_token.sv deleted file mode 100644 index b9eecb7634..0000000000 --- a/hw/rtl/afu/foldsim/foldsim_token.sv +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The token ends of a foldsim endpoint: a model of the target's surroundings -// that takes one token on each input channel and gives one on each output -// channel per target cycle, finishing the cycle once it has them all. -// -// Valid and ready depend on the endpoint's state alone, never on the -// simulator's handshake, so the simulator may connect them to its models -// combinationally. With STAGED, `have` and `done` do too: a token counts from -// the host cycle after it is taken, so the endpoint's finish starts at its own -// registers rather than at the simulator's valid and ready. - -// An input channel: the token is kept until the endpoint finishes its cycle. -module foldsim_token_in #( - parameter W = 1, - parameter STAGED = 0 -) ( - input wire clk, - input wire reset, - input wire valid, - output wire ready, - input wire [W-1:0] bits, - input wire finish, - output wire have, - output wire [W-1:0] data -); - reg got; - reg [W-1:0] held; - - assign ready = ~got; - assign have = STAGED ? got : (got | valid); - assign data = (STAGED || got) ? held : bits; - - always @(posedge clk) begin - if (reset || finish) begin - got <= 1'b0; - end else if (valid) begin - got <= 1'b1; - end - if (~got) begin - held <= bits; - end - end -endmodule - -// An output channel: the token is offered once `ok` until it is taken. -module foldsim_token_out #( - parameter STAGED = 0 -) ( - input wire clk, - input wire reset, - output wire valid, - input wire ready, - input wire ok, - input wire finish, - output wire done -); - reg fired; - - assign valid = ~fired & ok; - assign done = STAGED ? fired : (fired | (ok & ready)); - - always @(posedge clk) begin - if (reset || finish) begin - fired <= 1'b0; - end else if (valid & ready) begin - fired <= 1'b1; - end - end -endmodule diff --git a/hw/syn/common.mk b/hw/syn/common.mk index d2726d79dd..f254765268 100644 --- a/hw/syn/common.mk +++ b/hw/syn/common.mk @@ -26,8 +26,6 @@ endif # and its elaboration into the one netlist a synthesis tool reads # (circt_elaborate.sh; gen_sources.sh -E). CIRCT_PATH ?= $(TOOLDIR)/circt -# foldsim builds FAME-5 simulators on CIRCT (hw/syn/foldsim). -FOLDSIM_PATH ?= $(TOOLDIR)/foldsim YOSYS_PATH ?= $(TOOLDIR)/yosys STA_PATH ?= $(TOOLDIR)/sta VERILATOR_PATH ?= $(TOOLDIR)/verilator @@ -39,8 +37,9 @@ CIRCT_OPT ?= $(CIRCT_PATH)/bin/circt-opt FIRTOOL ?= $(CIRCT_PATH)/bin/firtool FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim FOLDSIM_OPT ?= $(FOLDSIM_PATH)/bin/foldsim-opt +FOLDSIM_LIFT_TOOL ?= $(FOLDSIM_PATH)/bin/foldsim-lift YOSYS ?= $(YOSYS_PATH)/bin/yosys STA ?= $(STA_PATH)/bin/sta VERILATOR ?= $(VERILATOR_PATH)/bin/verilator -export CIRCT_PATH FOLDSIM_PATH YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL FOLDSIM FOLDSIM_OPT YOSYS STA VERILATOR +export CIRCT_PATH FOLDSIM_PATH FOLDSIM_SHELL FOLDSIM_LIFT_TOOL YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL FOLDSIM FOLDSIM_OPT YOSYS STA VERILATOR diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index 79194b7323..ae4feee805 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -6,9 +6,11 @@ # # foldsim compiles Vortex into a simulator, on CIRCT: each class threaded # through one physical copy (FAME-5), the rest of the target a model of its -# own, and every channel between them in one token fabric. The kernel -# (hw/rtl/afu/foldsim) adds the memory and control endpoints around it. The -# image lands in $(BUILD_DIR)/bin/foldsim.vbin; run it with +# own, and every channel between them in one token fabric. Foldsim's shell +# (FOLDSIM_SHELL, part of the Foldsim installation) adds the memory and +# control endpoints around it; Vortex supplies only the target wrapper +# (hw/rtl/afu/foldsim/target) and this flow. The image lands in +# $(BUILD_DIR)/bin/foldsim.vbin; run it with # FPGA_BIN_DIR= ./ci/blackbox.sh --driver=foldsim --target=hw ... # The runtime's avedsim target simulates the same generated design in-process. @@ -19,7 +21,9 @@ include $(VORTEX_HOME)/hw/syn/common.mk SRC_DIR := $(VORTEX_HOME)/hw/syn/foldsim RTL_DIR := $(VORTEX_HOME)/hw/rtl SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts -FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim +# The shell's RTL, and the wrapper that gives Vortex the shell's target ports. +FOLDSIM_RTL_DIR := $(FOLDSIM_SHELL)/rtl +FOLDSIM_TARGET_DIR := $(RTL_DIR)/afu/foldsim/target VIVADO := $(XILINX_VIVADO)/bin/vivado SLASHKIT ?= slashkit # Parallel link runs; each Vivado of a large simulator needs tens of GB. @@ -44,7 +48,7 @@ BIN_DIR := $(BUILD_DIR)/bin GEN_DIR := $(BUILD_DIR)/gen # The classes threaded by FAME-5, and the units given plain-port boundaries -# first so they can be classes (lift_boundaries.py; [:drop=P,...][;port=P,...]). +# first so they can be classes (Foldsim's foldsim-lift; [:drop=P,...][;port=P,...]). FOLDSIM_CLASSES ?= VX_socket__lift FOLDSIM_LIFT ?= VX_socket:drop=INSTANCE_ID # Where circt-verilog, circt-opt and firtool are (hw/syn/common.mk's CIRCT); @@ -69,7 +73,7 @@ TARGET_TOP := foldsim_target CONFIGS ?= RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs \ -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm \ - -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(FOLDSIM_RTL_DIR)/target + -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(FOLDSIM_TARGET_DIR) SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG ifdef PERF PLATFORM_DEFINES := -DPERF_ENABLE @@ -128,15 +132,11 @@ IP_DIR := $(BUILD_DIR)/foldsim_afu_ip CFG := $(BUILD_DIR)/config.cfg VBIN := $(BIN_DIR)/foldsim.vbin -# The kernel's own sources are not the target's: they are read only when the -# kernel is packaged or emulated. -KERNEL_RTL := $(wildcard $(FOLDSIM_RTL_DIR)/*.sv $(FOLDSIM_RTL_DIR)/*.vh) -RTL_SRCS := $(filter-out $(KERNEL_RTL),$(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh')) \ +RTL_SRCS := $(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh') \ $(wildcard $(ROOT_DIR)/hw/VX_config.vh) $(wildcard $(ROOT_DIR)/hw/VX_types.vh) \ $(firstword $(MAKEFILE_LIST)) $(CONFIG_STAMP) -# The AFU shells of the native flows are second tops, and the foldsim kernel is -# compiled apart from the target. -SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv) $(wildcard $(FOLDSIM_RTL_DIR)/*.sv)) +# The AFU shells of the native flows are second tops. +SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv)) .PHONY: all sources generate package-ip ooc clean FORCE @@ -163,9 +163,9 @@ $(SRCS_LIST): $(RTL_SRCS) $(IP_STUBS) $(SCRIPT_DIR)/xilinx_ip_gen.tcl fi mv $@.tmp $@ -$(LIFT_LIST): $(SRCS_LIST) $(SRC_DIR)/lift_boundaries.py $(CLASSES_STAMP) +$(LIFT_LIST): $(SRCS_LIST) $(FOLDSIM_LIFT_TOOL) $(CLASSES_STAMP) rm -rf $(BUILD_DIR)/lift - $(LIFT_PYTHON) $(SRC_DIR)/lift_boundaries.py --srcs $(SRCS_LIST) --top $(TARGET_TOP) \ + $(LIFT_PYTHON) $(FOLDSIM_LIFT_TOOL) --srcs $(SRCS_LIST) --top $(TARGET_TOP) \ --out $(BUILD_DIR)/lift $(foreach u,$(FOLDSIM_LIFT),--unit '$(u)') generate: $(GEN_SV) $(KERNEL_VH) diff --git a/hw/syn/foldsim/lift_boundaries.py b/hw/syn/foldsim/lift_boundaries.py deleted file mode 100644 index ec8afdf965..0000000000 --- a/hw/syn/foldsim/lift_boundaries.py +++ /dev/null @@ -1,574 +0,0 @@ -#!/usr/bin/env python3 -"""Interface boundary lifting: make interface-port units foldsim classes. - -A class is threaded and wired into the token fabric port by port, so it must be -a module whose ports are plain signals; units such as VX_issue_slice or -VX_cluster have SystemVerilog interface ports instead. This pass gives each -requested unit a boundary made of plain wires, without touching the RTL: it -rewrites a copy of the (preprocessed) sources. - -For each unit U: - * every elaborated instance must have the same parameter values (parameters - listed as droppable -- debug labels such as ISSUE_ID -- are left at their - defaults); - * a parameter-free wrapper U__lift is generated: each interface port becomes - one plain port per signal (per array element, packed), with the direction - taken from the port's modport or, for a bare interface port, from where - the signal is driven (inside the unit -> output); the wrapper instantiates - the interfaces with their elaborated parameters and U with its elaborated - parameter values; - * every instantiation site of U is replaced by glue wires/assigns between - the parent's interfaces and an instance of U__lift. -Behaviour is unchanged: the glue is pure wiring. - -Parameters that are functional instance IDs (e.g. VX_cluster's CLUSTER_ID) -are promoted to 32-bit input ports (port=P): the parameter becomes a port of U, -every child parameter override whose expression uses it is promoted the same -way down the hierarchy, and every affected instantiation passes the value as a -port connection instead (the old default where it was not overridden). Uses in -constant contexts (widths, generate conditions) make the re-elaboration fail, -so an invalid promotion cannot go through silently. - -usage: lift_boundaries.py --srcs --top --out

- --unit U[:drop=P1,P2][;port=P3] [--unit ...] -""" -import argparse -import collections -import os -import re -import shutil -import sys - -import pyslang -from pyslang import ast, analysis - -SIG_KINDS = (ast.SymbolKind.Variable, ast.SymbolKind.Net) -SK = pyslang.syntax.SyntaxKind - - -def read_srcs(path): - defines, incdirs, files = [], [], [] - for line in open(path): - line = line.strip() - if line.startswith("+define+"): - defines.append(line[len("+define+"):]) - elif line.startswith("+incdir+"): - incdirs.append(line[len("+incdir+"):]) - elif line: - files.append(line) - return defines, incdirs, files - - -def q(a): - return "'" + a.replace("'", "'\\''") + "'" if any(c in a for c in " '\"") else a - - -def elaborate(defines, incdirs, files, top): - d = pyslang.driver.Driver() - d.addStandardArgs() - # The Vortex RTL references some nets before their declaration (VX_tcu_core), - # which Vivado and Verilator accept; slang makes it an error unless allowed. - args = ["slang", "--top", top, "--allow-hierarchical-const", "--allow-use-before-declare", - "--ignore-unknown-modules"] - args += [f"-D{x}" for x in defines] + [f"-I{x}" for x in incdirs] + files - if not (d.parseCommandLine(" ".join(q(a) for a in args), pyslang.driver.CommandLineOptions()) - and d.processOptions() and d.parseAllSources()): - sys.exit("lift: slang could not parse the sources") - comp = d.createCompilation() - used = set() - - def visit(sym): - if sym.kind == ast.SymbolKind.Instance: - used.add(sym.definition.name) - return ast.VisitAction.Advance - for t in comp.getRoot().topInstances: - t.visit(visit) - sm = comp.sourceManager - - def unused_only(fname): - # A file that only defines modules outside the elaborated hierarchy: - # errors there cannot affect the design. - names = re.findall(r"^\s*(module|interface|package)\s+(\w+)", open(fname).read(), re.M) - return bool(names) and all(k == "module" and n not in used for k, n in names) - errs = [x for x in comp.getAllDiagnostics() if x.isError() - and not unused_only(str(sm.getFileName(x.location)))] - if errs: - x = errs[0] - sys.exit(f"lift: {len(errs)} elaboration errors (first: {x.code} at " - f"{sm.getFileName(x.location)}:{sm.getLineNumber(x.location)})") - am = analysis.AnalysisManager() - am.analyze(comp) - return d, comp, am - - -def vec(w): - return "" if w == 1 else f"[{w - 1}:0] " - - -def lit(v): - s = str(v) - return s - - -class Unit: - def __init__(self, spec): - name, _, opts = spec.partition(":") - self.name = name - self.drop = set() - self.ports = [] # parameters promoted to input ports (functional instance IDs) - for o in filter(None, opts.split(";")): - k, _, v = o.partition("=") - if k == "drop": - self.drop |= set(filter(None, v.split(","))) - elif k == "port": - self.ports += [x for x in v.split(",") if x] - else: - sys.exit(f"lift: unknown option '{o}' for {name} (U[:drop=P,...][;port=P,...])") - self.insts = [] - - -def params_of(inst, drop=()): - """Overridable value and type parameters of an elaborated instance, as SV text.""" - out = collections.OrderedDict() - for m in inst.body: - if m.name in drop: - continue - if m.kind == ast.SymbolKind.Parameter and not m.isLocalParam: - out[m.name] = lit(m.value) - elif m.kind == ast.SymbolKind.TypeParameter and not m.isLocalParam: - t = str(m.targetType.type) - if not re.match(r"^[A-Za-z_][\w$]*(::[A-Za-z_][\w$]*)*$", t) and not re.match(r"^(logic|bit)( ?\[\d+:\d+\])*$", t): - sys.exit(f"lift: type parameter {inst.hierarchicalPath}.{m.name} = '{t}' has no name to pass") - out[m.name] = t - return out - - -def iface_elements(conn): - target, modport = conn.ifaceConn - if target.kind == ast.SymbolKind.InstanceArray: - return list(target.elements), modport - return [target], modport - - -def signal_dirs(unit_inst, elem, modport_name, am): - """[(signal, 'input'|'output', width)] from the unit's point of view.""" - out = [] - mp = None - if modport_name: - mp = next((m for m in elem.body if m.kind == ast.SymbolKind.Modport and m.name == modport_name), None) - mp_dirs = {} - if mp is not None: - for p in mp: - mp_dirs[p.name] = "input" if str(p.direction).endswith("In") else "output" - upath = unit_inst.hierarchicalPath - for m in elem.body: - if m.kind not in SIG_KINDS: - continue - if m.type.isUnpackedArray: - sys.exit(f"lift: unpacked interface signal {elem.hierarchicalPath}.{m.name} not supported") - if mp is not None: - if m.name not in mp_dirs: - continue - d = mp_dirs[m.name] - else: - drv = am.getDrivers(m) - inside = any(dv.containingSymbol.hierarchicalPath == upath - or dv.containingSymbol.hierarchicalPath.startswith(upath + ".") - or dv.containingSymbol.hierarchicalPath.startswith(upath + "[") for dv in drv) - d = "output" if inside else "input" - out.append((m.name, d, m.type.bitWidth)) - return out - - -def port_range(port): - """Declared interface-array range as text ('[0:3]'), or '' for a single interface.""" - r = port.declaredRange - if not r: - return "", 1 - rr = r[0] - lo, hi = rr.left, rr.right - return f"[{lo}:{hi}]", abs(hi - lo) + 1 - - -def build_wrapper(u, rep, am): - ports, body, inner_conns = [], [], [] - shape = [] # per port: ('plain', name, dir, w) or ('iface', name, rng, n, [(sig, dir, w)]) - for c in rep.portConnections: - p = c.port - if p.kind == ast.SymbolKind.InterfacePort: - elems, mp = iface_elements(c) - rng, n = port_range(p) - sigs = signal_dirs(rep, elems[0], p.modport, am) - idef = p.interfaceDef.name - iparams = params_of(elems[0]) - pa = "" - if iparams: - pa = " #(" + ", ".join(f".{k}({v})" for k, v in iparams.items()) + ")" - body.append(f" {idef}{pa} {p.name} {rng}();".replace(" ();", "();")) - for s, d, w in sigs: - pn = f"{p.name}__{s}" - ports.append(f" {d} wire {('[' + str(n - 1) + ':0]') if rng else ''}{vec(w)}{pn}") - for k in range(n): - ref = f"{p.name}[{k}].{s}" if rng else f"{p.name}.{s}" - pk = f"{pn}[{k}]" if rng else pn - body.append(f" assign {ref} = {pk};" if d == "input" else f" assign {pk} = {ref};") - inner_conns.append(f" .{p.name} ({p.name})") - shape.append(("iface", p.name, rng, n, sigs)) - else: - d = str(p.direction).split(".")[-1].lower() - d = {"in": "input", "out": "output", "inout": "inout"}.get(d, d) - if p.type.isUnpackedArray: - sys.exit(f"lift: unpacked port {u.name}.{p.name} not supported") - w = p.type.bitWidth - ports.append(f" {d} wire {vec(w)}{p.name}") - inner_conns.append(f" .{p.name} ({p.name})") - shape.append(("plain", p.name, d, w)) - for pp in u.ports: - ports.insert(0, f" input wire [31:0] {pp}") - inner_conns.insert(0, f" .{pp} ({pp})") - shape.insert(0, ("promoted", pp)) - prm = params_of(rep, u.drop | set(u.ports)) - pa = (" #(\n" + ",\n".join(f" .{k} ({v})" for k, v in prm.items()) + "\n )") if prm else "" - text = (f"// Generated by lift_boundaries.py: {u.name} with plain ports (FAME-5 fold boundary).\n" - f"module {u.name}__lift (\n" + ",\n".join(ports) + "\n);\n" + "\n".join(body) + "\n\n" - f" {u.name}{pa} inner (\n" + ",\n".join(inner_conns) + "\n );\nendmodule\n") - return text, shape - - -SLICE = re.compile(r"^(.*)\[(.+)\+:(.+)\]$") - - -def elem_expr(e, k, is_array): - e = e.strip() - if not is_array: - return e - m = SLICE.match(e) - if m: - return f"{m.group(1)}[({m.group(2).strip()}) + {k}]" - if re.match(r"^[A-Za-z_]\w*$", e): - return f"{e}[{k}]" - if k == 0: - return e # a single element bound to a one-element array port - sys.exit(f"lift: cannot index connection expression '{e}'") - - -def site_connections(u, inst, shape): - """{port: expression text} of an instance's connections, by name. - - `.p` is `.p (p)`, and `.*` connects each port not named otherwise to the - signal of its name, as SystemVerilog defines them. Connections by - position are refused: the rewrite reorders the ports.""" - iname = str(inst.decl.name).strip() - conns, wildcard = {}, False - for c in inst.connections: - if isinstance(c, pyslang.parsing.Token): - continue # separator - k = c.kind - if k == SK.NamedPortConnection: - nm = str(c.name).strip() - if c.openParen.kind != pyslang.parsing.TokenKind.OpenParenthesis: - conns[nm] = nm - else: - conns[nm] = str(c.expr).strip() if c.expr is not None else "" - elif k == SK.WildcardPortConnection: - wildcard = True - elif k in (SK.OrderedPortConnection, SK.EmptyPortConnection): - sys.exit(f"lift: {u.name} instance {iname}: ports connected by position; " - f"connect them by name") - else: - sys.exit(f"lift: {u.name} instance {iname}: unsupported port connection {k}") - if wildcard: - for kind, *rest in shape: - if kind in ("plain", "iface") and rest[0] not in conns: - conns[rest[0]] = rest[0] - return conns - - -def rewrite_site(u, site_syntax, shape, uid): - """Replacement text for one instantiation statement.""" - inst = site_syntax.instances[0] - iname = str(inst.decl.name).strip() - conns = site_connections(u, inst, shape) - overrides = site_overrides(site_syntax) - pre = f"__lift{uid}" - decl, glue, pc = [], [], [] - for kind, *rest in shape: - if kind == "promoted": - pp = rest[0] - if pp not in overrides: - sys.exit(f"lift: {u.name} instance {iname}: promoted parameter {pp} is not overridden at the site") - pc.append(f" .{pp} ({overrides[pp]})") - continue - if kind == "plain": - name, d, w = rest - pc.append(f" .{name} ({conns.get(name, '')})") - continue - name, rng, n, sigs = rest - e = conns.get(name) - if e is None: - sys.exit(f"lift: {u.name} instance {iname}: interface port {name} not connected by name") - for s, d, w in sigs: - wn = f"{pre}_{name}__{s}" - decl.append(f"wire {('[' + str(n - 1) + ':0]') if rng else ''}{vec(w)}{wn};") - for k in range(n): - ek = elem_expr(e, k, bool(rng)) - wk = f"{wn}[{k}]" if rng else wn - glue.append(f"assign {wk} = {ek}.{s};" if d == "input" else f"assign {ek}.{s} = {wk};") - pc.append(f" .{name}__{s} ({wn})") - return ("// lift_boundaries: " + u.name + " -> " + u.name + "__lift\n " - + "\n ".join(decl + glue) + "\n " - + f"{u.name}__lift {iname} (\n" + ",\n".join(pc) + "\n );") - - -def site_overrides(site): - """{param: expr text} of a HierarchyInstantiationSyntax's #(...) named overrides.""" - out = {} - pva = site.parameters - if pva is None: - return out - for it in pva.parameters: - if hasattr(it, "name") and hasattr(it, "expr"): - out[str(it.name).strip()] = str(it.expr).strip() if it.expr is not None else "" - return out - - -def _remove_item(text_len_getter, node, sm): - """(start, end) covering a separated-list item and one adjacent comma.""" - return node.sourceRange.start.offset, node.sourceRange.end.offset - - -class Promoter: - """Parameter-to-port promotion across definitions (see module docstring).""" - - def __init__(self, comp, sm, lifted_sites): - self.comp, self.sm = comp, sm - self.lifted_sites = lifted_sites # site keys rewritten by lifting: skip - self.done = {} # (def, param) -> default text - self.edits = collections.defaultdict(list) - self.site_conv = collections.defaultdict(set) # site key -> {param} converted - self.insts = collections.defaultdict(list) - for t in comp.getRoot().topInstances: - t.visit(self._collect) - - def _collect(self, sym): - if sym.kind == ast.SymbolKind.Instance: - self.insts[sym.definition.name].append(sym) - return ast.VisitAction.Advance - - def key(self, node): - r = node.sourceRange - return (self.sm.getFileName(r.start), r.start.offset) - - def promote(self, defname, param): - if (defname, param) in self.done: - return - rep = self.insts[defname][0] - dsyn = rep.definition.syntax - hdr = dsyn.header - f = self.sm.getFileName(dsyn.sourceRange.start) - text = open(f).read() - default = None - target = None - for it in hdr.parameters.declarations: - if not hasattr(it, "declarators"): - continue - names = [str(dd.name).strip() for dd in it.declarators if hasattr(dd, "name")] - if param in names: - if len(names) != 1: - sys.exit(f"lift: {defname}.{param} shares a declaration; split it first") - dd = [dd for dd in it.declarators if hasattr(dd, "name")][0] - default = str(dd.initializer.expr).strip() if dd.initializer is not None else "0" - target = it - if target is None: - sys.exit(f"lift: no parameter {param} in {defname}") - self.done[(defname, param)] = default - a, b = target.sourceRange.start.offset, target.sourceRange.end.offset - self.edits[f].append(_with_comma(text, a, b)) - op = hdr.ports.openParen.location.offset + 1 - self.edits[f].append((op, op, f"\n input wire [31:0] {param},")) - # children whose overrides use the promoted value - for ch in rep.body: - pass - for cinst in self._children(rep): - site = cinst.syntax.parent - ov = site_overrides(site) - for q, ex in ov.items(): - if re.search(rf"\b{re.escape(param)}\b", ex): - self.promote(cinst.definition.name, q) - - def _children(self, inst): - out = [] - def v(sym): - if sym is inst: - return ast.VisitAction.Advance - if sym.kind == ast.SymbolKind.Instance: - out.append(sym) - return ast.VisitAction.Skip - return ast.VisitAction.Advance - inst.visit(v) - return out - - def convert_sites(self): - """Every instantiation of a promoted (def, param): override -> port connection.""" - for (defname, param), default in self.done.items(): - for inst in self.insts[defname]: - site = inst.syntax.parent - k = self.key(site) - if k in self.lifted_sites or param in self.site_conv[k]: - continue - self.site_conv[k].add(param) - f = k[0] - text = open(f).read() - ov_items = {} - if site.parameters is not None: - for it in site.parameters.parameters: - if hasattr(it, "name"): - ov_items[str(it.name).strip()] = it - if param in ov_items: - it = ov_items[param] - ex = str(it.expr).strip() - a, b = it.sourceRange.start.offset, it.sourceRange.end.offset - self.edits[f].append(_with_comma(text, a, b)) - else: - ex = default - inst_syn = site.instances[0] - if any(not isinstance(c, pyslang.parsing.Token) - and c.kind in (SK.OrderedPortConnection, SK.EmptyPortConnection) - for c in inst_syn.connections): - sys.exit(f"lift: {defname} instance {str(inst_syn.decl.name).strip()}: ports " - f"connected by position cannot take {param} as a port; " - f"connect them by name") - op = inst_syn.openParen.location.offset + 1 - self.edits[f].append((op, op, f"\n .{param} ({ex}),")) - - -def _with_comma(text, a, b): - """Deletion range for a list item at [a, b) plus its separating comma.""" - j = b - while j < len(text) and text[j] in " \t\r\n": - j += 1 - if j < len(text) and text[j] == ",": - return (a, j + 1, "") - i = a - 1 - while i >= 0 and text[i] in " \t\r\n": - i -= 1 - if i >= 0 and text[i] == ",": - return (i, b, "") - return (a, b, "") - - -def apply_edits(text, edits): - """Apply (start, end, repl) edits; overlapping pure deletions are merged.""" - edits = sorted(edits, key=lambda e: (e[0], e[1])) - merged = [] - for s, e, t in edits: - if merged and s < merged[-1][1]: - ps, pe, pt = merged[-1] - if t or pt: - sys.exit(f"lift: overlapping edits at {s}") - merged[-1] = (ps, max(pe, e), "") - else: - merged.append((s, e, t)) - for s, e, t in reversed(merged): - text = text[:s] + t + text[e:] - return text - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--srcs", required=True) - ap.add_argument("--top", required=True) - ap.add_argument("--out", required=True) - ap.add_argument("--unit", action="append", required=True) - a = ap.parse_args() - a.out = os.path.abspath(a.out) - defines, incdirs, files = read_srcs(a.srcs) - d, comp, am = elaborate(defines, incdirs, files, a.top) - sm = comp.sourceManager - units = {u.name: u for u in (Unit(s) for s in a.unit)} - - def visit(sym): - if sym.kind == ast.SymbolKind.Instance and sym.definition.name in units: - units[sym.definition.name].insts.append(sym) - return ast.VisitAction.Advance - # Only the elaborated hierarchy: slang also builds placeholder instances of - # uninstantiated definitions, which must not count. - for t in comp.getRoot().topInstances: - t.visit(visit) - - edits = collections.defaultdict(list) # file -> [(start, end, text)] - wrappers = {} - uid = 0 - lifted_sites = set() - for u in units.values(): - if not u.insts: - sys.exit(f"lift: no instance of {u.name}") - skip = u.drop | set(u.ports) - groups = collections.Counter(tuple(params_of(i, skip).items()) for i in u.insts) - if len(groups) > 1: - keys = list(params_of(u.insts[0], skip).keys()) - diff = [k for k in keys if len({dict(g)[k] for g in groups}) > 1] - sys.exit(f"lift: instances of {u.name} differ in {diff}; drop them if they are labels") - text, shape = build_wrapper(u, u.insts[0], am) - wrappers[u.name] = text - seen = set() - for i in u.insts: - site = i.syntax.parent # HierarchyInstantiationSyntax - rng = site.sourceRange - key = (sm.getFileName(rng.start), rng.start.offset) - if key in seen: - continue - seen.add(key) - lifted_sites.add(key) - if len(site.instances) != 1: - sys.exit(f"lift: {u.name}: multi-instance statement at {key}") - edits[key[0]].append((rng.start.offset, rng.end.offset, rewrite_site(u, site, shape, uid))) - uid += 1 - print(f"lift: {u.name}: {len(u.insts)} instances, {len(seen)} sites, " - f"{sum(1 for s in shape if s[0] == 'iface')} interface ports, " - f"{sum(len(s[4]) for s in shape if s[0] == 'iface')} lifted signals") - - pr = Promoter(comp, sm, lifted_sites) - for u in units.values(): - for pp in u.ports: - pr.promote(u.name, pp) - pr.convert_sites() - for f, es in pr.edits.items(): - edits[f].extend(es) - if pr.done: - print("lift: promoted " + ", ".join(f"{d}.{p}" for d, p in pr.done)) - - os.makedirs(os.path.join(a.out, "src"), exist_ok=True) - out_files = [] - by_real = {os.path.realpath(str(k)): k for k in edits} - applied, matched = 0, set() - for f in files: - dst = os.path.join(a.out, "src", os.path.basename(f)) - key = by_real.get(os.path.realpath(f)) - if key is not None: - open(dst, "w").write(apply_edits(open(f).read(), edits[key])) - applied += 1 - matched.add(key) - else: - shutil.copyfile(f, dst) - out_files.append(dst) - if applied != len(edits): - sys.exit(f"lift: edits for unmatched files: {sorted(set(edits) - matched)}") - for name, text in wrappers.items(): - p = os.path.join(a.out, "src", f"{name}__lift.sv") - open(p, "w").write(text) - out_files.append(p) - # Written last: the list is the build's mark of a finished lift. - with open(os.path.join(a.out, "srcs.f"), "w") as o: - for x in defines: - o.write(f"+define+{x}\n") - for x in incdirs: - o.write(f"+incdir+{x}\n") - o.write(f"+incdir+{os.path.join(a.out, 'src')}\n") - for x in out_files: - o.write(x + "\n") - print(f"lift: wrote {len(out_files)} files ({applied} of {len(edits)} rewritten) to {a.out}") - - -if __name__ == "__main__": - main() diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 29aed69f98..c6c1f5f4e1 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -20,9 +20,6 @@ all: $(MAKE) -C afu_axil_demux $(MAKE) -C afu_reset_seq $(MAKE) -C mm_axi_arb - $(MAKE) -C foldsim_mem - $(MAKE) -C foldsim_mem_staged - $(MAKE) -C foldsim_mem_ch2 $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma $(MAKE) -C cp_llc_port @@ -58,9 +55,6 @@ run: $(MAKE) -C afu_axil_demux run $(MAKE) -C afu_reset_seq run $(MAKE) -C mm_axi_arb run - $(MAKE) -C foldsim_mem run - $(MAKE) -C foldsim_mem_staged run - $(MAKE) -C foldsim_mem_ch2 run $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run $(MAKE) -C cp_llc_port run @@ -213,9 +207,6 @@ clean: $(MAKE) -C stream_arb clean $(MAKE) -C mm_axi_arb clean $(MAKE) -C cp_core clean - $(MAKE) -C foldsim_mem clean - $(MAKE) -C foldsim_mem_staged clean - $(MAKE) -C foldsim_mem_ch2 clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean $(MAKE) -C fsqrt_unit clean diff --git a/hw/unittest/foldsim_mem/Makefile b/hw/unittest/foldsim_mem/Makefile deleted file mode 100644 index d1c66fb359..0000000000 --- a/hw/unittest/foldsim_mem/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -ROOT_DIR := $(realpath ../../..) -include $(ROOT_DIR)/config.mk - -PROJECT := foldsim_mem - -RTL_DIR := $(VORTEX_HOME)/hw/rtl -FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim - -SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) - -CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) - -SRCS := $(SRC_DIR)/main.cpp - -DBG_TRACE_FLAGS := - -RTL_INCLUDE := -I$(FOLDSIM_RTL_DIR) - -# The token modules are not named after their file. -VL_FLAGS += $(FOLDSIM_RTL_DIR)/foldsim_token.sv - -TOP := foldsim_mem - -include ../common.mk diff --git a/hw/unittest/foldsim_mem/main.cpp b/hw/unittest/foldsim_mem/main.cpp deleted file mode 100644 index 392bbb3cc4..0000000000 --- a/hw/unittest/foldsim_mem/main.cpp +++ /dev/null @@ -1,848 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// foldsim's memory endpoint (hw/rtl/afu/foldsim/foldsim_mem.sv), driven as the -// simulator and the control endpoint drive it -- one token per channel per -// target cycle -- by a model of an AXI4 master, against a model of HBM with -// random ready and latency that checks the AXI4 protocol on the device side. -// Under a range of the bank model's settings: -// * every target cycle's outputs equal the C++ model's -// (sim/common/dram_bank_model.h's DramBankAxi), the model stepped on the -// same requests; -// * read data, write acknowledgements and ids are right; -// * a reset with HBM transactions in flight finishes only once HBM has -// answered them all, takes no request, and delivers nothing from before -// it to the target after it; -// * a burst sets `error` and reaches HBM in no part, and a reset clears it. -// -// ./foldsim_mem every setup -// ./foldsim_mem one setup - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "Vfoldsim_mem.h" -#include "verilated.h" - -using namespace vortex; - -namespace { - -constexpr int kWords = 16; // DATA_W = 512 -constexpr uint64_t kBase = 0x10000000ull; -constexpr uint64_t kMagic = 0x3ff7f000ull; // offered only in reset cycles -constexpr uint64_t kBurstAddr = 0x3ff7e000ull; // offered only as a burst - -using Line = std::array; - -// The endpoint's CH_SHIFT: the port is one of 2^shift, and the bank model -// sees the block within the port's channel. -#ifndef FOLDSIM_MEM_CH_SHIFT -#define FOLDSIM_MEM_CH_SHIFT 0 -#endif -uint64_t channel_addr(uint64_t a) { - return ((a >> (6 + FOLDSIM_MEM_CH_SHIFT)) << 6) | (a & 63); -} - -// The traffic a setup makes: the addresses its requests take. -enum class Traffic { - Small, // 64 lines, 4 KB: row hits - Random, // anywhere in 64 MB - Rows, // random rows 32 KB apart: row conflicts - Stream, // sequential, a few streams - OneBank, // random rows of one bank: conflicts back to back, to the horizon -}; - -struct Setup { - const char* name; - dram_timing::Standard type; - uint32_t core_clock, data_rate, latency, queue; - uint32_t rq, wq, wr_high, wr_low; - Traffic traffic; - double p_read, p_write, p_ready; - uint64_t cycles; - // Refresh, and its interval and busy time if not the standard's (0). - DramBankModel::Refresh refresh = DramBankModel::Refresh::Off; - uint32_t refi = 0, rfc = 0; -}; - -const Setup kSetups[] = { - {"bank", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, - Traffic::Small, 0.3, 0.25, 1.0, 3000}, - {"bank_random", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, - Traffic::Random, 0.5, 0.3, 0.8, 4000}, - {"bank_rows", dram_timing::Standard::HBM2, 400, 800, 3, 0, 32, 32, 26, 7, - Traffic::Rows, 0.6, 0.2, 0.9, 4000}, - {"bank_stream", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, - Traffic::Stream, 0.7, 0.4, 0.9, 4000}, - {"bank_a100", dram_timing::Standard::HBM2, 1410, 3038, 190, 257, 32, 32, 26, 7, - Traffic::Random, 0.5, 0.3, 0.8, 4000}, - {"bank_h100", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, - Traffic::Rows, 0.5, 0.3, 0.8, 4000}, - {"bank_h100_s", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, - Traffic::Stream, 0.8, 0.5, 0.95, 4000}, - {"bank_queues", dram_timing::Standard::HBM2, 700, 3200, 7, 5, 6, 4, 3, 2, - Traffic::Random, 0.6, 0.5, 0.7, 4000}, - {"bank_reads", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, - Traffic::Random, 0.9, 0.05, 0.9, 4000}, - {"bank_onebank", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, - Traffic::OneBank, 0.9, 0.1, 0.9, 4000}, - {"bank_onebank2", dram_timing::Standard::HBM2, 400, 800, 11, 3, 32, 32, 26, 7, - Traffic::OneBank, 0.7, 0.3, 0.8, 4000}, - {"bank_v100", dram_timing::Standard::HBM2, 1530, 1752, 0, 0, 32, 32, 26, 7, - Traffic::Rows, 0.8, 0.2, 0.9, 4000}, - {"ref_bank", dram_timing::Standard::HBM2, 400, 800, 0, 0, 32, 32, 26, 7, - Traffic::Random, 0.5, 0.3, 0.8, 4000, DramBankModel::Refresh::PerBank}, - {"ref_bank_s", dram_timing::Standard::HBM2, 1410, 3038, 190, 257, 32, 32, 26, 7, - Traffic::Stream, 0.8, 0.4, 0.9, 4000, DramBankModel::Refresh::PerBank}, - {"ref_all", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, - Traffic::Rows, 0.5, 0.3, 0.8, 4000, DramBankModel::Refresh::AllBank}, - {"ref_all_fast", dram_timing::Standard::HBM2, 400, 800, 3, 0, 32, 32, 26, 7, - Traffic::Random, 0.6, 0.3, 0.9, 4000, DramBankModel::Refresh::AllBank, 600, 0}, - {"ref_bank_fast", dram_timing::Standard::HBM3, 1980, 6545, 0, 0, 32, 32, 26, 7, - Traffic::Rows, 0.8, 0.2, 0.9, 4000, DramBankModel::Refresh::PerBank, 60, 0}, -}; - -const Setup* setup = nullptr; -Vfoldsim_mem* dut = nullptr; -std::mt19937 rng(7); -uint64_t host_cycle = 0; -uint64_t tcycle = 0; -int failures = 0; - -#define CHECK(cond, ...) \ - do { \ - if (!(cond)) { \ - if (failures++ < 20) { \ - printf("FAIL %s host %lu target %lu: ", setup->name, \ - (unsigned long)host_cycle, (unsigned long)tcycle); \ - printf(__VA_ARGS__); \ - printf("\n"); \ - } \ - } \ - } while (0) - -bool chance(double p) { return std::uniform_real_distribution<>(0, 1)(rng) < p; } -int rand_int(int lo, int hi) { return std::uniform_int_distribution<>(lo, hi)(rng); } - -Line make_line(uint64_t addr, uint32_t version) { - Line l; - for (int i = 0; i < kWords; ++i) - l[i] = uint32_t(addr * 2654435761u) ^ (version << 12) ^ (uint32_t(i) * 0x9e3779b9u); - return l; -} - -// ---- The C++ models, stepped on what the target offered ---- -std::unique_ptr golden_bank; - -DramBankAxi::Config bank_config(const Setup& s) { - DramBankAxi::Config c; - c.model = DramBankModel::Config::make(s.type, s.data_rate, 64, s.refresh); - if (s.refi) c.model.refi = s.refi; - if (s.rfc) c.model.rfc = s.rfc; - c.model.read_queue = s.rq; - c.model.write_queue = s.wq; - c.model.wr_high = s.wr_high; - c.model.wr_low = s.wr_low; - c.core_clock = s.core_clock; - c.data_rate = s.data_rate; - c.latency = s.latency; - c.queue = s.queue; - c.read_slots = 32; - c.write_slots = 16; - return c; -} - -// ---- HBM: an AXI4 slave with random ready and latency ---- -struct Hbm { - std::map mem; - double ready_p = 0.7; - int lat_lo = 1, lat_hi = 40; - struct Pending { uint64_t at; Line data; }; - std::deque r; - std::deque b; - std::deque aw; - std::deque> w; - std::set ar_seen; - uint64_t n_ar = 0, n_aw = 0, n_w = 0; - bool arready = false, awready = false, wready = false; - bool ar_held = false, aw_held = false, w_held = false; - uint64_t ar_a = 0, aw_a = 0; - Line w_d{}; - - size_t outstanding() const { return r.size() + b.size() + aw.size() + w.size(); } - - uint64_t when(uint64_t last) { - return std::max(last, host_cycle + uint64_t(rand_int(lat_lo, lat_hi))); - } - - void drive() { - arready = chance(ready_p); - awready = chance(ready_p); - wready = chance(ready_p); - dut->m_arready = arready; - dut->m_awready = awready; - dut->m_wready = wready; - dut->m_rvalid = !r.empty() && r.front().at <= host_cycle; - for (int i = 0; i < kWords; ++i) - dut->m_rdata[i] = r.empty() ? 0 : r.front().data[i]; - dut->m_bvalid = !b.empty() && b.front() <= host_cycle; - } - - Line wdata() const { - Line l; - for (int i = 0; i < kWords; ++i) - l[i] = dut->m_wdata[i]; - return l; - } - - void sample() { - if (ar_held) - CHECK(dut->m_arvalid && dut->m_araddr == ar_a, "HBM: AR withdrawn or changed"); - if (aw_held) - CHECK(dut->m_awvalid && dut->m_awaddr == aw_a, "HBM: AW withdrawn or changed"); - if (w_held) - CHECK(dut->m_wvalid && wdata() == w_d, "HBM: W withdrawn or changed"); - if (dut->m_arvalid) - CHECK(dut->m_arlen == 0, "HBM: AR burst"); - if (dut->m_awvalid) - CHECK(dut->m_awlen == 0, "HBM: AW burst"); - if (dut->m_wvalid) - CHECK(dut->m_wlast, "HBM: W without WLAST"); - CHECK(dut->m_rready && dut->m_bready, "HBM: R or B not ready"); - - const bool ar_hs = dut->m_arvalid && arready; - const bool aw_hs = dut->m_awvalid && awready; - const bool w_hs = dut->m_wvalid && wready; - ar_held = dut->m_arvalid && !arready; - aw_held = dut->m_awvalid && !awready; - w_held = dut->m_wvalid && !wready; - ar_a = dut->m_araddr; - aw_a = dut->m_awaddr; - w_d = wdata(); - - if (dut->m_rvalid) - r.pop_front(); - if (dut->m_bvalid) - b.pop_front(); - if (ar_hs) { - ++n_ar; - const uint64_t a = dut->m_araddr; - ar_seen.insert(a); - auto it = mem.find(a); - r.push_back({when(r.empty() ? 0 : r.back().at), it != mem.end() ? it->second : Line{}}); - } - if (aw_hs) { - ++n_aw; - aw.push_back(dut->m_awaddr); - } - if (w_hs) { - ++n_w; - w.push_back({wdata(), dut->m_wstrb}); - } - while (!aw.empty() && !w.empty()) { - Line& m = mem[aw.front()]; - const Line& d = w.front().first; - const uint64_t strb = w.front().second; - for (int i = 0; i < kWords; ++i) - for (int k = 0; k < 4; ++k) - if ((strb >> (4 * i + k)) & 1) - m[i] = (m[i] & ~(0xffu << (8 * k))) | (d[i] & (0xffu << (8 * k))); - aw.pop_front(); - w.pop_front(); - b.push_back(when(b.empty() ? 0 : b.back())); - } - } -}; - -Hbm hbm; - -// ---- The target: an AXI4 master issuing single-beat reads and writes ---- -// Like Vortex's AXI adapter, it has no two reads with one id outstanding. -struct Op { uint64_t addr; int id; Line data; }; - -struct Master { - bool generate = false; - bool reset = false; - bool offer_magic = false; - int burst = 0; // 0 none, 1 AR, 2 AW, 3 a W without WLAST - bool rready = true, bready = true; - - bool ar_v = false, aw_v = false, w_v = false; - Op ar{}, aw{}, w{}; - int ar_len = 0, aw_len = 0; - bool w_last = true; - - std::deque ar_q, aw_q, w_q; - struct ReadExp { Op op; bool known; uint64_t fired; }; - std::map reads; // fired, awaiting R, by id - std::deque aw_only, w_only; - struct WriteExp { Op op; uint64_t formed; }; - std::deque writes; - - std::map known; - std::set unknown; - std::map busy; - std::set pending_writes; - std::set free_rids; - uint32_t version = 1; - int next_wid = 0; - uint64_t stream = 0; - uint64_t n_r = 0, n_b = 0, stale = 0; - - Master() { this->clear_ids(); } - - void clear_ids() { - free_rids.clear(); - for (int i = 0; i < 16; ++i) - free_rids.insert(i); - } - - bool idle() const { - return ar_q.empty() && aw_q.empty() && w_q.empty() && reads.empty() && aw_only.empty() - && w_only.empty() && writes.empty() && !ar_v && !aw_v && !w_v; - } - - uint64_t pick_addr() { - switch (setup->traffic) { - case Traffic::Small: return uint64_t(rand_int(0, 63)) * 64; - case Traffic::Random: return uint64_t(rand_int(0, (1 << 20) - 1)) * 64; - case Traffic::Rows: return uint64_t(rand_int(0, 63)) * 32768 + uint64_t(rand_int(0, 3)) * 64; - case Traffic::Stream: { - uint64_t s = uint64_t(rand_int(0, 3)); - return (s << 22) + ((stream++ & 0xfff) * 64); - } - case Traffic::OneBank: { - // A random row, and the block of it the bank model maps to bank 5. - for (;;) { - uint64_t a = uint64_t(rand_int(0, 255)) * 32768 + uint64_t(rand_int(0, 511)) * 64; - uint32_t bank, row; - golden_bank->model().decode(a, &bank, &row); - if (bank == 5) - return a; - } - } - } - return 0; - } - - void make_requests() { - if (!generate || reset) - return; - if (ar_q.size() + reads.size() < 16 && !free_rids.empty() && chance(setup->p_read)) { - const uint64_t a = this->pick_addr(); - if (!pending_writes.count(a)) { - ++busy[a]; - const int id = *free_rids.begin(); - free_rids.erase(free_rids.begin()); - ar_q.push_back({a, id, {}}); - } - } - if (aw_q.size() + writes.size() < 24 && chance(setup->p_write)) { - const uint64_t a = this->pick_addr(); - if (busy[a] == 0) { - ++busy[a]; - Op op{a, next_wid++ & 15, make_line(a, version++)}; - aw_q.push_back(op); - w_q.push_back(op); - pending_writes.insert(a); - } - } - } - - void offer() { - rready = chance(setup->p_ready); - bready = chance(setup->p_ready); - if (reset) { - ar_v = offer_magic; - ar = {kMagic, 0, {}}; - ar_len = 0; - aw_v = w_v = false; - return; - } - if (burst == 1 && !ar_v) { - ar_v = true; - ar = {kBurstAddr, 1, {}}; - ar_len = 3; - } else if (burst == 2 && !aw_v) { - aw_v = true; - aw = {kBurstAddr, 2, {}}; - aw_len = 1; - } else if (burst == 3 && !w_v) { - w_v = true; - w = {kBurstAddr, 3, make_line(kBurstAddr, 0)}; - w_last = false; - } - if (!ar_v && !ar_q.empty() && chance(0.8)) { - ar_v = true; - ar = ar_q.front(); - ar_len = 0; - ar_q.pop_front(); - } - if (!aw_v && !aw_q.empty() && chance(0.6)) { - aw_v = true; - aw = aw_q.front(); - aw_len = 0; - aw_q.pop_front(); - } - if (!w_v && !w_q.empty() && chance(0.6)) { - w_v = true; - w = w_q.front(); - w_last = true; - w_q.pop_front(); - } - } - - void read_done(uint64_t a) { - if (--busy[a] == 0) - busy.erase(a); - } - - void cycle(bool arready, bool awready, bool wready, bool rvalid, const Line& rdata, int rid, - bool bvalid, int bid) { - if (reset) { - for (auto& kv : reads) - read_done(kv.second.op.addr); - for (auto* q : {&aw_only, &w_only}) - for (auto& op : *q) - unknown.insert(op.addr); - for (auto& wr : writes) - unknown.insert(wr.op.addr); - for (auto& op : ar_q) - read_done(op.addr); - for (auto& op : aw_q) - unknown.insert(op.addr); - for (auto& op : w_q) - unknown.insert(op.addr); - if (aw_v) unknown.insert(aw.addr); - if (w_v) unknown.insert(w.addr); - if (ar_v && ar.addr != kMagic) read_done(ar.addr); - reads.clear(); - aw_only.clear(); - w_only.clear(); - writes.clear(); - ar_q.clear(); - aw_q.clear(); - w_q.clear(); - ar_v = aw_v = w_v = false; - busy.clear(); - pending_writes.clear(); - this->clear_ids(); - return; - } - if (rvalid && rready) { - ++n_r; - auto it = reads.find(rid); - if (it == reads.end()) { - ++stale; - CHECK(false, "R id %d with no read outstanding", rid); - } else { - const ReadExp e = it->second; - reads.erase(it); - free_rids.insert(rid); - if (e.known) - CHECK(rdata == e.op.data, "R data wrong at 0x%lx", (unsigned long)e.op.addr); - read_done(e.op.addr); - } - } - if (bvalid && bready) { - ++n_b; - if (writes.empty()) { - ++stale; - CHECK(false, "B with no write outstanding"); - } else { - const WriteExp e = writes.front(); - writes.pop_front(); - CHECK(bid == e.op.id, "B id %d, expected %d", bid, e.op.id); - known[e.op.addr] = e.op.data; - unknown.erase(e.op.addr); - pending_writes.erase(e.op.addr); - read_done(e.op.addr); - } - } - if (ar_v && arready) { - ar_v = false; - if (ar_len == 0) { - const auto it = known.find(ar.addr); - const bool k = !unknown.count(ar.addr); - ReadExp e{ar, k, tcycle}; - e.op.data = (it != known.end()) ? it->second : Line{}; - reads[ar.id] = e; - } else { - burst = 0; - } - } - const bool aw_fire = aw_v && awready; - const bool w_fire = w_v && wready; - if (aw_fire) { - aw_v = false; - if (aw_len == 0) - aw_only.push_back(aw); - else - burst = 0; - } - if (w_fire) { - w_v = false; - if (w_last) - w_only.push_back(w); - else - burst = 0; - } - while (!aw_only.empty() && !w_only.empty()) { - writes.push_back({aw_only.front(), tcycle}); - aw_only.pop_front(); - w_only.pop_front(); - } - } -}; - -Master master; - -// ---- The token channels ---- -enum { I_AWVALID, I_AWADDR, I_AWID, I_AWLEN, I_WVALID, I_WDATA, I_WSTRB, I_WLAST, I_ARVALID, - I_ARADDR, I_ARID, I_ARLEN, I_RREADY, I_BREADY, I_TRESET, NUM_IN }; -enum { O_AWREADY, O_WREADY, O_ARREADY, O_RVALID, O_RDATA, O_RLAST, O_RID, O_RRESP, O_BVALID, - O_BRESP, O_BID, NUM_OUT }; - -bool taken[NUM_IN], got[NUM_OUT]; -bool o_awready, o_wready, o_arready, o_rvalid, o_bvalid; -int o_rid, o_bid; -Line o_rdata; -bool after_reset = false; -uint64_t reset_cycles_done = 0, drained_at_reset = 0, compared = 0; - -void drive_tokens() { - const Master& m = master; -#define IN(ch, name, value) \ - dut->name##_valid = !taken[ch]; \ - dut->name##_bits = (value) - IN(I_AWVALID, awvalid, m.aw_v); - IN(I_AWADDR, awaddr, m.aw.addr); - IN(I_AWID, awid, m.aw.id); - IN(I_AWLEN, awlen, m.aw_len); - IN(I_WVALID, wvalid, m.w_v); - dut->wdata_valid = !taken[I_WDATA]; - for (int i = 0; i < kWords; ++i) - dut->wdata_bits[i] = m.w.data[i]; - IN(I_WSTRB, wstrb, ~0ull); - IN(I_WLAST, wlast, m.w_last); - IN(I_ARVALID, arvalid, m.ar_v); - IN(I_ARADDR, araddr, m.ar.addr); - IN(I_ARID, arid, m.ar.id); - IN(I_ARLEN, arlen, m.ar_len); - IN(I_RREADY, rready, m.rready); - IN(I_BREADY, bready, m.bready); - IN(I_TRESET, treset, m.reset); -#undef IN -#define OUT(ch, name) dut->name##_ready = !got[ch] - OUT(O_AWREADY, awready); OUT(O_WREADY, wready); OUT(O_ARREADY, arready); - OUT(O_RVALID, rvalid); OUT(O_RDATA, rdata); OUT(O_RLAST, rlast); OUT(O_RID, rid); - OUT(O_RRESP, rresp); OUT(O_BVALID, bvalid); OUT(O_BRESP, bresp); OUT(O_BID, bid); -#undef OUT -} - -void sample_tokens(bool in_take[NUM_IN], bool out_take[NUM_OUT]) { -#define IN(ch, name) in_take[ch] = dut->name##_valid && dut->name##_ready - IN(I_AWVALID, awvalid); IN(I_AWADDR, awaddr); IN(I_AWID, awid); IN(I_AWLEN, awlen); - IN(I_WVALID, wvalid); IN(I_WDATA, wdata); IN(I_WSTRB, wstrb); IN(I_WLAST, wlast); - IN(I_ARVALID, arvalid); IN(I_ARADDR, araddr); IN(I_ARID, arid); IN(I_ARLEN, arlen); - IN(I_RREADY, rready); IN(I_BREADY, bready); IN(I_TRESET, treset); -#undef IN -#define OUT(ch, name) out_take[ch] = dut->name##_valid && dut->name##_ready - OUT(O_AWREADY, awready); OUT(O_WREADY, wready); OUT(O_ARREADY, arready); - OUT(O_RVALID, rvalid); OUT(O_RDATA, rdata); OUT(O_RLAST, rlast); OUT(O_RID, rid); - OUT(O_RRESP, rresp); OUT(O_BVALID, bvalid); OUT(O_BRESP, bresp); OUT(O_BID, bid); -#undef OUT - if (out_take[O_AWREADY]) o_awready = dut->awready_bits; - if (out_take[O_WREADY]) o_wready = dut->wready_bits; - if (out_take[O_ARREADY]) o_arready = dut->arready_bits; - if (out_take[O_RVALID]) o_rvalid = dut->rvalid_bits; - if (out_take[O_RDATA]) - for (int i = 0; i < kWords; ++i) - o_rdata[i] = dut->rdata_bits[i]; - if (out_take[O_RID]) o_rid = dut->rid_bits; - if (out_take[O_BVALID]) o_bvalid = dut->bvalid_bits; - if (out_take[O_BID]) o_bid = dut->bid_bits; - if (out_take[O_RLAST]) CHECK(dut->rlast_bits, "RLAST low"); - if (out_take[O_RRESP]) CHECK(dut->rresp_bits == 0, "RRESP not OKAY"); - if (out_take[O_BRESP]) CHECK(dut->bresp_bits == 0, "BRESP not OKAY"); -} - -void host_step() { - drive_tokens(); - hbm.drive(); - dut->clk = 0; - dut->eval(); - bool in_take[NUM_IN], out_take[NUM_OUT]; - sample_tokens(in_take, out_take); - bool any_out = false; - for (int j = 0; j < NUM_OUT; ++j) - any_out |= out_take[j]; - if (after_reset && any_out) { - after_reset = false; - ++reset_cycles_done; - CHECK(hbm.outstanding() == 0, "reset cycle finished with %zu HBM transactions open", - hbm.outstanding()); - CHECK(!dut->m_arvalid && !dut->m_awvalid && !dut->m_wvalid, - "reset cycle finished with requests offered to HBM"); - } - hbm.sample(); - dut->clk = 1; - dut->eval(); - ++host_cycle; - for (int i = 0; i < NUM_IN; ++i) - taken[i] |= in_take[i]; - for (int j = 0; j < NUM_OUT; ++j) - got[j] |= out_take[j]; -} - -// The cycle's outputs against the C++ model's; then the model takes the -// cycle's requests. -void golden_cycle() { - const Master& m = master; - auto& g = *golden_bank; - if (m.reset) { - g.reset(); - return; - } - ++compared; - CHECK(o_arready == g.ar_ready(), "ARREADY %d, model %d", o_arready, g.ar_ready()); - CHECK(o_awready == g.aw_ready(), "AWREADY %d, model %d", o_awready, g.aw_ready()); - CHECK(o_wready == g.w_ready(), "WREADY %d, model %d", o_wready, g.w_ready()); - CHECK(o_rvalid == g.r_valid(), "RVALID %d, model %d", o_rvalid, g.r_valid()); - if (o_rvalid && g.r_valid()) - CHECK(uint32_t(o_rid) == g.r_id(), "RID %d, model %u", o_rid, g.r_id()); - CHECK(o_bvalid == g.b_valid(), "BVALID %d, model %d", o_bvalid, g.b_valid()); - if (o_bvalid && g.b_valid()) - CHECK(uint32_t(o_bid) == g.b_id(), "BID %d, model %u", o_bid, g.b_id()); - DramBankAxi::Inputs in; - in.ar_valid = m.ar_v; - in.ar_id = m.ar.id; - in.ar_addr = channel_addr(m.ar.addr); - in.ar_len = m.ar_len; - in.aw_valid = m.aw_v; - in.aw_id = m.aw.id; - in.aw_addr = channel_addr(m.aw.addr); - in.aw_len = m.aw_len; - in.w_valid = m.w_v; - in.w_last = m.w_last; - in.r_ready = m.rready; - in.b_ready = m.bready; - g.step(in); -} - -void run(uint64_t n) { - for (uint64_t k = 0; k < n; ++k) { - master.make_requests(); - master.offer(); - if (master.reset && hbm.outstanding() != 0) - ++drained_at_reset; - for (int i = 0; i < NUM_IN; ++i) - taken[i] = false; - for (int j = 0; j < NUM_OUT; ++j) - got[j] = false; - uint64_t spins = 0; - for (;;) { - bool all = true; - for (int i = 0; i < NUM_IN; ++i) - all &= taken[i]; - for (int j = 0; j < NUM_OUT; ++j) - all &= got[j]; - if (all) - break; - host_step(); - if (++spins > 200000) { - printf("FAIL %s: target cycle %lu never completes\n", setup->name, (unsigned long)tcycle); - exit(1); - } - } - if (master.reset) - after_reset = true; - golden_cycle(); - master.cycle(o_arready, o_awready, o_wready, o_rvalid, o_rdata, o_rid, o_bvalid, o_bid); - ++tcycle; - } -} - -void reset_cycles(int n, bool magic) { - master.reset = true; - master.offer_magic = magic; - run(n); - master.reset = false; - master.offer_magic = false; -} - -void drain() { - master.generate = false; - for (int i = 0; i < 100000 && !master.idle(); ++i) - run(1); - CHECK(master.idle(), "the target's transactions never complete"); - run(4); -} - -void burst_case(int kind, const char* what) { - drain(); - const uint64_t ar = hbm.n_ar, aw = hbm.n_aw, w = hbm.n_w; - master.burst = kind; - for (int i = 0; i < 64 && master.burst != 0; ++i) - run(1); - CHECK(master.burst == 0, "%s: never taken", what); - run(2); - CHECK(dut->error, "%s: no error", what); - CHECK(!o_arready && !o_awready && !o_wready, "%s: still ready after the error", what); - master.generate = true; - run(50); - master.generate = false; - CHECK(hbm.n_ar == ar && hbm.n_aw == aw && hbm.n_w == w, "%s: reached HBM", what); - CHECK(dut->error, "%s: error cleared without a reset", what); - reset_cycles(3, false); - CHECK(!dut->error, "%s: reset left the error", what); - master.generate = true; - run(800); -} - -void configure() { - const Setup& s = *setup; - dut->base = kBase; - const auto cfg = bank_config(s); - const auto& t = cfg.model.t; - const auto& g = cfg.model.geom; - dut->cfg_data_rate = s.data_rate; - dut->cfg_core_clock = s.core_clock; - dut->cfg_latency = s.latency; - dut->cfg_queue = s.queue; - dut->cfg_rq = s.rq; - dut->cfg_wq = s.wq; - dut->cfg_wr_high = s.wr_high; - dut->cfg_wr_low = s.wr_low; - dut->cfg_geom = g.log2_pcs | (g.log2_bgs << 2) | (g.log2_banks << 4) | (g.log2_row_bursts << 6) - | (g.log2_visit << 10) | (uint32_t(g.two_clock_act) << 12); - dut->cfg_refresh = (cfg.model.refresh != DramBankModel::Refresh::Off) - | (uint32_t(cfg.model.refresh == DramBankModel::Refresh::AllBank) << 1); - dut->cfg_refi = cfg.model.refi; - dut->cfg_rfc = cfg.model.rfc; - const uint32_t v[18] = {t.BL, t.CCDS, t.CCDL, t.CL, t.CWL, t.RCDRD, t.RCDWR, t.RP, t.RAS, t.RC, - t.RTP, t.WR, t.RRDS, t.RRDL, t.FAW, t.WTRS, t.WTRL, g.rtw_extra}; - for (int w = 0; w < 7; ++w) - dut->cfg_t[w] = 0; - for (int i = 0; i < 18; ++i) - for (int b = 0; b < 12; ++b) - if ((v[i] >> b) & 1) - dut->cfg_t[(i * 12 + b) / 32] |= 1u << ((i * 12 + b) % 32); - golden_bank = std::make_unique(cfg); -} - -int run_setup(const Setup& s) { - setup = &s; - rng.seed(7); - host_cycle = tcycle = 0; - reset_cycles_done = drained_at_reset = compared = 0; - hbm = Hbm(); - master = Master(); - for (int i = 0; i < NUM_IN; ++i) taken[i] = false; - for (int j = 0; j < NUM_OUT; ++j) got[j] = false; - after_reset = false; - const int failures_before = failures; - - dut = new Vfoldsim_mem(); - configure(); - dut->reset = 1; - for (int i = 0; i < 8; ++i) { - hbm.drive(); - dut->clk = 0; - dut->eval(); - dut->clk = 1; - dut->eval(); - } - dut->reset = 0; - - reset_cycles(4, true); - master.generate = true; - run(s.cycles); - - // Resets with HBM slow and busy, at random points. - for (int k = 0; k < 8; ++k) { - hbm.lat_lo = 100; - hbm.lat_hi = 400; - hbm.ready_p = 0.3; - run(rand_int(50, 300)); - hbm.lat_lo = 1; - hbm.lat_hi = 40; - hbm.ready_p = 0.7; - reset_cycles(rand_int(1, 4), true); - run(s.cycles / 2); - } - - burst_case(1, "AR burst"); - burst_case(2, "AW burst"); - burst_case(3, "W without WLAST"); - - drain(); - CHECK(hbm.n_aw == hbm.n_w, "HBM took %lu AWs and %lu Ws", (unsigned long)hbm.n_aw, - (unsigned long)hbm.n_w); - CHECK(hbm.outstanding() == 0, "HBM left with %zu transactions", hbm.outstanding()); - CHECK(!hbm.ar_seen.count(kBase + kMagic), "a request made in reset reached HBM"); - CHECK(!hbm.ar_seen.count(kBase + kBurstAddr), "a burst reached HBM"); - CHECK(dut->idle, "endpoint not idle at the end"); - CHECK(master.stale == 0, "%lu stale responses", (unsigned long)master.stale); - // One bank's conflicts leave too little in flight for every reset to meet. - if (s.traffic != Traffic::OneBank) - CHECK(drained_at_reset >= 4, "only %lu resets found HBM busy", (unsigned long)drained_at_reset); - - const auto& st = golden_bank->model().stats(); - printf("%-12s target cycles %lu, host cycles %lu, %lu compared; %lu reads, %lu writes; " - "%lu reset cycles, %lu with HBM busy; model since last reset: %lu reads, %lu writes, " - "%lu hits, %lu misses, %lu conflicts, %lu refreshes, %lu deferred: %s\n", - s.name, (unsigned long)tcycle, (unsigned long)host_cycle, (unsigned long)compared, - (unsigned long)master.n_r, (unsigned long)master.n_b, (unsigned long)reset_cycles_done, - (unsigned long)drained_at_reset, (unsigned long)st.reads, (unsigned long)st.writes, - (unsigned long)st.hits, (unsigned long)st.misses, (unsigned long)st.conflicts, - (unsigned long)st.refreshes, (unsigned long)st.deferred, - failures == failures_before ? "ok" : "FAILED"); - delete dut; - dut = nullptr; - return failures - failures_before; -} - -} // namespace - -int main(int argc, char** argv) { - Verilated::commandArgs(argc, argv); - const char* only = nullptr; - for (int i = 1; i < argc; ++i) - if (argv[i][0] != '+') - only = argv[i]; - for (const auto& s : kSetups) { - if (only && std::strcmp(only, s.name)) - continue; - run_setup(s); - } - if (failures != 0) { - printf("FAILED: %d check(s)\n", failures); - return 1; - } - printf("PASSED!\n"); - return 0; -} diff --git a/hw/unittest/foldsim_mem_ch2/Makefile b/hw/unittest/foldsim_mem_ch2/Makefile deleted file mode 100644 index 3509bb338e..0000000000 --- a/hw/unittest/foldsim_mem_ch2/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -ROOT_DIR := $(realpath ../../..) -include $(ROOT_DIR)/config.mk - -# hw/unittest/foldsim_mem's test, on the kernel's endpoint (staged token ends) -# as one of two ports: the bank model sees the port's channel. -PROJECT := foldsim_mem - -RTL_DIR := $(VORTEX_HOME)/hw/rtl -FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim - -SRC_DIR := $(VORTEX_HOME)/hw/unittest/foldsim_mem - -CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) -CXXFLAGS += -DFOLDSIM_MEM_CH_SHIFT=1 - -SRCS := $(SRC_DIR)/main.cpp - -DBG_TRACE_FLAGS := - -RTL_INCLUDE := -I$(FOLDSIM_RTL_DIR) -PARAMS += -GSTAGED=1 -GCH_SHIFT=1 - -# The token modules are not named after their file. -VL_FLAGS += $(FOLDSIM_RTL_DIR)/foldsim_token.sv - -TOP := foldsim_mem - -include ../common.mk diff --git a/hw/unittest/foldsim_mem_staged/Makefile b/hw/unittest/foldsim_mem_staged/Makefile deleted file mode 100644 index ea68da7673..0000000000 --- a/hw/unittest/foldsim_mem_staged/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -ROOT_DIR := $(realpath ../../..) -include $(ROOT_DIR)/config.mk - -# hw/unittest/foldsim_mem's test, on the endpoint with staged token ends (the -# kernel's). -PROJECT := foldsim_mem - -RTL_DIR := $(VORTEX_HOME)/hw/rtl -FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim - -SRC_DIR := $(VORTEX_HOME)/hw/unittest/foldsim_mem - -CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) - -SRCS := $(SRC_DIR)/main.cpp - -DBG_TRACE_FLAGS := - -RTL_INCLUDE := -I$(FOLDSIM_RTL_DIR) -PARAMS += -GSTAGED=1 - -# The token modules are not named after their file. -VL_FLAGS += $(FOLDSIM_RTL_DIR)/foldsim_token.sv - -TOP := foldsim_mem - -include ../common.mk diff --git a/sim/common/dram_bank_model.h b/sim/common/dram_bank_model.h index e678152e85..0bc8b3c694 100644 --- a/sim/common/dram_bank_model.h +++ b/sim/common/dram_bank_model.h @@ -52,9 +52,12 @@ namespace vortex { // every bank with one precharge and one REF. A refresh that would reach past // the horizon waits, and the requests with it. // -// This is the normative model: foldsim's memory endpoint -// (hw/rtl/afu/foldsim/foldsim_mem_bank.sv) implements the same decisions, and -// the faces below put it behind DramSim's interface and behind an AXI4 port. +// Foldsim's memory endpoint (its shell's rtl/foldsim_mem_bank.sv) implements +// the same decisions; the normative copy of this model is the shell's +// model/dram_bank_model.h, and this one gives Vortex's simulators the same +// timing (rtlsim's AXI_MEM_MODEL=bank, against which ci/testcases/foldsim.yaml +// holds Foldsim cycle-exact). The faces below put it behind DramSim's +// interface and behind an AXI4 port. class DramBankModel { public: enum class Refresh { Off, PerBank, AllBank }; diff --git a/sim/foldsim/Makefile b/sim/foldsim/Makefile index 3020a733e6..3572f0ebe7 100644 --- a/sim/foldsim/Makefile +++ b/sim/foldsim/Makefile @@ -1,13 +1,15 @@ include ../common.mk # The foldsim kernel in Verilator, as a library the runtime's avedsim target -# links: the same simulator hw/syn/foldsim generates for the image, so a run -# here is the image's run, cycle for cycle, but for device memory's timing. +# links: the same simulator hw/syn/foldsim generates for the image, in +# Foldsim's shell and its in-process host (FOLDSIM_SHELL), so a run here is +# the image's run, cycle for cycle, but for device memory's timing. Vortex adds +# only the Xilinx floating-point cores' models its FPU may instantiate. DESTDIR ?= $(CURDIR) SRC_DIR := $(VORTEX_HOME)/sim/foldsim -FOLDSIM_RTL_DIR := $(RTL_DIR)/afu/foldsim +FOLDSIM_RTL_DIR := $(FOLDSIM_SHELL)/rtl WORK_DIR := $(DESTDIR)/foldsim_emul GEN_SV := $(WORK_DIR)/gen/foldsim_sim.sv # The kernel's memory ports, as the image's flow derived them. @@ -15,7 +17,7 @@ KERNEL_VH := $(WORK_DIR)/gen/foldsim_kernel.vh CONFIG_FILE := $(DESTDIR)/foldsim_emul_config.stamp -CXXFLAGS += -std=c++17 -fPIC -O2 -DNDEBUG -I$(SRC_DIR) +CXXFLAGS += -std=c++17 -fPIC -O2 -DNDEBUG -I$(FOLDSIM_SHELL)/include LDFLAGS += -shared -pthread # The Xilinx floating-point cores' simulation models compute with Vortex's DPI # floating point. @@ -37,7 +39,7 @@ TOP := foldsim_afu RTL_SRCS := $(SRC_DIR)/xilinx_fp_models.sv $(GEN_SV) $(FOLDSIM_RTL_DIR)/foldsim_token.sv \ $(FOLDSIM_RTL_DIR)/foldsim_mem_bmap.sv $(FOLDSIM_RTL_DIR)/foldsim_mem_bank.sv \ $(FOLDSIM_RTL_DIR)/foldsim_mem.sv $(FOLDSIM_RTL_DIR)/foldsim_ctrl.sv $(FOLDSIM_RTL_DIR)/foldsim_afu.sv -SRCS := $(SRC_DIR)/foldsim_emul.cpp $(DPI_DIR)/dpi_float.cpp $(DPI_DIR)/dpi_util.cpp \ +SRCS := $(FOLDSIM_SHELL)/emul/foldsim_emul.cpp $(DPI_DIR)/dpi_float.cpp $(DPI_DIR)/dpi_util.cpp \ $(SW_COMMON_DIR)/softfloat_ext.cpp $(SW_COMMON_DIR)/rvfloats.cpp PROJECT := libfoldsim_emul.so @@ -55,7 +57,8 @@ $(GEN_SV): force $(if $(FOLDSIM_FLAGS),FOLDSIM_FLAGS='$(FOLDSIM_FLAGS)') \ $(if $(LIFT_PYTHON),LIFT_PYTHON=$(LIFT_PYTHON)) $(if $(FOLDSIM),FOLDSIM=$(FOLDSIM)) -$(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(SRC_DIR)/xilinx_fp_models.sv $(wildcard $(FOLDSIM_RTL_DIR)/*) $(CONFIG_FILE) +$(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(SRC_DIR)/xilinx_fp_models.sv $(wildcard $(FOLDSIM_RTL_DIR)/*) \ + $(FOLDSIM_SHELL)/include/foldsim_emul.h $(CONFIG_FILE) $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(RTL_SRCS) --top-module $(TOP) \ $(SRCS) -CFLAGS '$(CXXFLAGS) -DFOLDSIM_MEM_PORTS='$$(sed -n 's/^`define FOLDSIM_MEM_PORTS //p' $(KERNEL_VH)) \ -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ diff --git a/sim/foldsim/foldsim_emul.cpp b/sim/foldsim/foldsim_emul.cpp deleted file mode 100644 index 35c3d63079..0000000000 --- a/sim/foldsim/foldsim_emul.cpp +++ /dev/null @@ -1,416 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "foldsim_emul.h" - -#include "Vfoldsim_afu.h" -#include "Vfoldsim_afu___024root.h" -#include "verilated.h" -#ifdef VM_TRACE -#include "verilated_vcd_c.h" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace vortex; - -double sc_time_stamp() { return 0; } - -namespace { - -uint64_t env_or(const char* name, uint64_t value) { - const char* s = getenv(name); - return (s != nullptr && s[0] != '\0') ? strtoull(s, nullptr, 0) : value; -} - -constexpr uint64_t kPage = 4096; -constexpr uint32_t kBeat = 64; - -#ifndef FOLDSIM_MEM_PORTS -#error "FOLDSIM_MEM_PORTS comes from the generated foldsim_kernel.vh" -#endif -static_assert(FOLDSIM_MEM_PORTS >= 1 && FOLDSIM_MEM_PORTS <= 8, "the kernel has 1 to 8 memory ports"); - -// Device memory: sparse, zero where never written. -class memory_t { -public: - void write(uint64_t addr, const uint8_t* data, uint64_t size, const uint8_t* strobe = nullptr) { - for (uint64_t i = 0; i < size; ++i) { - if (strobe == nullptr || strobe[i]) - page(addr + i)[(addr + i) % kPage] = data[i]; - } - } - void read(uint64_t addr, uint8_t* data, uint64_t size) { - for (uint64_t i = 0; i < size; ++i) { - auto it = pages_.find((addr + i) / kPage); - data[i] = (it == pages_.end()) ? 0 : it->second[(addr + i) % kPage]; - } - } - -private: - std::array& page(uint64_t addr) { - auto& p = pages_[addr / kPage]; - return p; - } - std::unordered_map> pages_; -}; - -} // namespace - -class foldsim_emul::Impl { -public: - Impl() - : mem_latency_(env_or("FOLDSIM_EMUL_MEM_LATENCY", 100)), - mmio_cycles_(env_or("FOLDSIM_EMUL_MMIO_CYCLES", 250)) {} - - ~Impl() { -#ifdef VM_TRACE - if (vcd_) - vcd_->close(); -#endif - if (top_) - top_->final(); - if (trace_ != nullptr) - fclose(trace_); - if (axi_trace_ != nullptr) - fclose(axi_trace_); - if (reg_trace_ != nullptr) - fclose(reg_trace_); - } - - int init() { - // FOLDSIM_EMUL_TRACE=: which classes finish a thread, each kernel - // cycle, one byte per cycle of up to 8 classes. - if (const char* path = getenv("FOLDSIM_EMUL_TRACE")) - trace_ = fopen(path, "wb"); - // FOLDSIM_EMUL_AXI_TRACE=: the kernel's memory port, one line per - // request and response. - if (const char* path = getenv("FOLDSIM_EMUL_AXI_TRACE")) - axi_trace_ = fopen(path, "w"); - // FOLDSIM_EMUL_REG_TRACE=: every register access of the host. - if (const char* path = getenv("FOLDSIM_EMUL_REG_TRACE")) - reg_trace_ = fopen(path, "w"); - context_ = std::make_unique(); - top_ = std::make_unique(context_.get()); - bind_ports(); -#ifdef VM_TRACE - // FOLDSIM_EMUL_VCD=: a waveform of the kernel, in a build with --trace, - // from kernel cycle FOLDSIM_EMUL_VCD_FROM on, FOLDSIM_EMUL_VCD_DEPTH levels - // of hierarchy deep (2: the kernel's channels). - vcd_from_ = env_or("FOLDSIM_EMUL_VCD_FROM", 0); - if (const char* path = getenv("FOLDSIM_EMUL_VCD")) { - context_->traceEverOn(true); - vcd_ = std::make_unique(); - top_->trace(vcd_.get(), int(env_or("FOLDSIM_EMUL_VCD_DEPTH", 99))); - vcd_->open(path); - } -#endif - top_->ap_rst_n = 0; - idle_master(); - for (int i = 0; i < 16; ++i) - tick(); - top_->ap_rst_n = 1; - tick(); - return 0; - } - - int reg_write(uint32_t offset, uint32_t value) { - std::lock_guard lock(mutex_); - run(mmio_cycles_); - top_->s_axi_control_awvalid = 1; - top_->s_axi_control_awaddr = offset; - top_->s_axi_control_wvalid = 1; - top_->s_axi_control_wdata = value; - top_->s_axi_control_wstrb = 0xf; - top_->s_axi_control_bready = 1; - bool aw = false, w = false, b = false; - for (int i = 0; i < 1000 && !b; ++i) { - top_->ap_clk = 0; - top_->eval(); - bool aw_fire = top_->s_axi_control_awvalid && top_->s_axi_control_awready; - bool w_fire = top_->s_axi_control_wvalid && top_->s_axi_control_wready; - b = top_->s_axi_control_bvalid; - edge(); - aw |= aw_fire; - w |= w_fire; - top_->s_axi_control_awvalid = !aw; - top_->s_axi_control_wvalid = !w; - } - top_->s_axi_control_bready = 0; - if (reg_trace_ != nullptr) - fprintf(reg_trace_, "%lu W %03x %08x\n", (unsigned long)cycle_, offset, value); - return b ? 0 : -1; - } - - int reg_read(uint32_t offset, uint32_t* value) { - std::lock_guard lock(mutex_); - run(mmio_cycles_); - top_->s_axi_control_arvalid = 1; - top_->s_axi_control_araddr = offset; - top_->s_axi_control_rready = 1; - bool r = false; - for (int i = 0; i < 1000 && !r; ++i) { - top_->ap_clk = 0; - top_->eval(); - bool ar_fire = top_->s_axi_control_arvalid && top_->s_axi_control_arready; - r = top_->s_axi_control_rvalid; - if (r) - *value = top_->s_axi_control_rdata; - edge(); - if (ar_fire) - top_->s_axi_control_arvalid = 0; - } - top_->s_axi_control_rready = 0; - if (reg_trace_ != nullptr) - fprintf(reg_trace_, "%lu R %03x %08x\n", (unsigned long)cycle_, offset, *value); - return r ? 0 : -1; - } - - int mem_write(uint64_t addr, const void* data, uint64_t size) { - std::lock_guard lock(mutex_); - memory_.write(addr, static_cast(data), size); - return 0; - } - - int mem_read(uint64_t addr, void* data, uint64_t size) { - std::lock_guard lock(mutex_); - memory_.read(addr, static_cast(data), size); - return 0; - } - -private: - // A read, with the data memory held when it took the request: a write - // that comes after it is not in its response. - struct request_t { - uint64_t at; - std::array data; - }; - - // One of the kernel's device-memory masters: its signals, and what memory - // has in flight on it. - struct port_t { - CData *awvalid, *awready, *wvalid, *wready, *arvalid, *arready; - CData *rvalid, *rready, *rlast, *rid, *rresp, *bvalid, *bready, *bid, *bresp; - QData *awaddr, *araddr, *wstrb; - VlWide<16> *wdata, *rdata; - std::deque reads; - std::deque aw; - std::deque> w; - std::deque writes; - }; - -#define FOLDSIM_PORT(i) \ - { \ - port_t& p = ports_[i]; \ - p.awvalid = &top_->m_axi_mem_##i##_awvalid; p.awready = &top_->m_axi_mem_##i##_awready; \ - p.wvalid = &top_->m_axi_mem_##i##_wvalid; p.wready = &top_->m_axi_mem_##i##_wready; \ - p.arvalid = &top_->m_axi_mem_##i##_arvalid; p.arready = &top_->m_axi_mem_##i##_arready; \ - p.rvalid = &top_->m_axi_mem_##i##_rvalid; p.rready = &top_->m_axi_mem_##i##_rready; \ - p.rlast = &top_->m_axi_mem_##i##_rlast; p.rid = &top_->m_axi_mem_##i##_rid; \ - p.rresp = &top_->m_axi_mem_##i##_rresp; p.bvalid = &top_->m_axi_mem_##i##_bvalid; \ - p.bready = &top_->m_axi_mem_##i##_bready; p.bid = &top_->m_axi_mem_##i##_bid; \ - p.bresp = &top_->m_axi_mem_##i##_bresp; p.awaddr = &top_->m_axi_mem_##i##_awaddr; \ - p.araddr = &top_->m_axi_mem_##i##_araddr; p.wstrb = &top_->m_axi_mem_##i##_wstrb; \ - p.wdata = &top_->m_axi_mem_##i##_wdata; p.rdata = &top_->m_axi_mem_##i##_rdata; \ - } - - void bind_ports() { - FOLDSIM_PORT(0) -#if FOLDSIM_MEM_PORTS > 1 - FOLDSIM_PORT(1) -#endif -#if FOLDSIM_MEM_PORTS > 2 - FOLDSIM_PORT(2) - FOLDSIM_PORT(3) -#endif -#if FOLDSIM_MEM_PORTS > 4 - FOLDSIM_PORT(4) - FOLDSIM_PORT(5) - FOLDSIM_PORT(6) - FOLDSIM_PORT(7) -#endif - } -#undef FOLDSIM_PORT - - void idle_master() { - top_->s_axi_control_awvalid = 0; - top_->s_axi_control_wvalid = 0; - top_->s_axi_control_bready = 0; - top_->s_axi_control_arvalid = 0; - top_->s_axi_control_rready = 0; - } - - void run(uint64_t cycles) { - for (uint64_t i = 0; i < cycles; ++i) - tick(); - } - - void tick() { - top_->ap_clk = 0; - top_->eval(); - edge(); - } - - // The memory's side of the kernel's AXI4 ports: every request accepted, and - // answered in order on its port once its latency has passed. - void drive_memory() { - for (auto& p : ports_) { - *p.awready = 1; - *p.wready = 1; - *p.arready = 1; - bool r = !p.reads.empty() && p.reads.front().at <= cycle_; - *p.rvalid = r; - if (r) - std::memcpy(p.rdata->data(), p.reads.front().data.data(), kBeat); - *p.rlast = 1; - *p.rid = 0; - *p.rresp = 0; - *p.bvalid = !p.writes.empty() && p.writes.front() <= cycle_; - *p.bid = 0; - *p.bresp = 0; - } - } - - // What a port did on the edge, as the kernel offered it before the edge. - struct sample_t { - bool ar, aw, w, r_fire, b_fire; - uint64_t araddr, awaddr; - uint8_t wdata[kBeat], strobe[kBeat]; - }; - - // The rising edge: the handshakes as the kernel saw them before it, then - // the edge itself. - void edge() { - drive_memory(); - top_->eval(); - std::array smp; - for (int k = 0; k < FOLDSIM_MEM_PORTS; ++k) { - auto& p = ports_[k]; - auto& x = smp[k]; - x.ar = *p.arvalid; - x.araddr = *p.araddr; - x.aw = *p.awvalid; - x.awaddr = *p.awaddr; - x.w = *p.wvalid; - if (x.w) { - std::memcpy(x.wdata, p.wdata->data(), kBeat); - uint64_t s = *p.wstrb; - for (uint32_t i = 0; i < kBeat; ++i) - x.strobe[i] = (s >> i) & 1; - } - x.r_fire = *p.rvalid && *p.rready; - x.b_fire = *p.bvalid && *p.bready; - } - - if (trace_ != nullptr) - fputc(top_->rootp->foldsim_afu__DOT__profile_finish & 0xff, trace_); - top_->ap_clk = 1; - top_->eval(); -#ifdef VM_TRACE - if (vcd_ && cycle_ >= vcd_from_) - vcd_->dump(cycle_); -#endif - ++cycle_; - - // Each block of the target's memory belongs to one port, so the order - // the ports are served in is not visible. - for (int k = 0; k < FOLDSIM_MEM_PORTS; ++k) { - auto& p = ports_[k]; - const auto& x = smp[k]; - if (axi_trace_ != nullptr) { - // Port 0's lines keep the single-port format. - const char* tag = (k == 0) ? "" : port_tag_[k].c_str(); - if (x.ar) - fprintf(axi_trace_, "%lu %sAR %lx\n", (unsigned long)cycle_, tag, (unsigned long)x.araddr); - if (x.aw) - fprintf(axi_trace_, "%lu %sAW %lx\n", (unsigned long)cycle_, tag, (unsigned long)x.awaddr); - if (x.w) - fprintf(axi_trace_, "%lu %sW\n", (unsigned long)cycle_, tag); - if (x.r_fire) - fprintf(axi_trace_, "%lu %sR\n", (unsigned long)cycle_, tag); - if (x.b_fire) - fprintf(axi_trace_, "%lu %sB\n", (unsigned long)cycle_, tag); - } - if (x.ar) { - request_t req; - req.at = cycle_ + mem_latency_; - memory_.read(x.araddr, req.data.data(), kBeat); - p.reads.push_back(req); - } - if (x.r_fire) - p.reads.pop_front(); - if (x.aw) - p.aw.push_back(x.awaddr); - if (x.w) { - std::array beat; - std::memcpy(beat.data(), x.wdata, kBeat); - std::memcpy(beat.data() + kBeat, x.strobe, kBeat); - p.w.push_back(beat); - } - while (!p.aw.empty() && !p.w.empty()) { - memory_.write(p.aw.front(), p.w.front().data(), kBeat, p.w.front().data() + kBeat); - p.aw.pop_front(); - p.w.pop_front(); - p.writes.push_back(cycle_ + mem_latency_); - } - if (x.b_fire) - p.writes.pop_front(); - } - } - - std::unique_ptr context_; - std::unique_ptr top_; -#ifdef VM_TRACE - std::unique_ptr vcd_; - uint64_t vcd_from_ = 0; -#endif - std::mutex mutex_; - memory_t memory_; - uint64_t cycle_ = 0; - uint64_t mem_latency_, mmio_cycles_; - FILE* trace_ = nullptr; - FILE* axi_trace_ = nullptr; - FILE* reg_trace_ = nullptr; - std::array ports_; - const std::array port_tag_ = {"", "1:", "2:", "3:", "4:", "5:", "6:", "7:"}; -}; - -foldsim_emul::foldsim_emul() : impl_(new Impl()) {} - -foldsim_emul::~foldsim_emul() { delete impl_; } - -int foldsim_emul::init() { return impl_->init(); } - -int foldsim_emul::reg_write(uint32_t offset, uint32_t value) { - return impl_->reg_write(offset, value); -} - -int foldsim_emul::reg_read(uint32_t offset, uint32_t* value) { - return impl_->reg_read(offset, value); -} - -int foldsim_emul::mem_write(uint64_t addr, const void* data, uint64_t size) { - return impl_->mem_write(addr, data, size); -} - -int foldsim_emul::mem_read(uint64_t addr, void* data, uint64_t size) { - return impl_->mem_read(addr, data, size); -} diff --git a/sim/foldsim/foldsim_emul.h b/sim/foldsim/foldsim_emul.h deleted file mode 100644 index 38ecbc988f..0000000000 --- a/sim/foldsim/foldsim_emul.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -namespace vortex { - -// The foldsim kernel (hw/rtl/afu/foldsim) simulated in-process, as the -// runtime's avedsim target reaches it: its control registers, and the device -// memory behind its AXI4 port. The kernel clock runs only while the host -// touches a register, each access costing the kernel cycles a register access -// over PCIe would (FOLDSIM_EMUL_MMIO_CYCLES, 250 by default), so a host that -// polls a register is what lets the target run. -class foldsim_emul { -public: - foldsim_emul(); - ~foldsim_emul(); - - int init(); - - int reg_write(uint32_t offset, uint32_t value); - int reg_read(uint32_t offset, uint32_t* value); - - // Device memory, by device address. - int mem_write(uint64_t addr, const void* data, uint64_t size); - int mem_read(uint64_t addr, void* data, uint64_t size); - -private: - class Impl; - Impl* impl_; -}; - -} // namespace vortex diff --git a/sw/common/foldsim_regs.h b/sw/common/foldsim_regs.h deleted file mode 100644 index e9949bd499..0000000000 --- a/sw/common/foldsim_regs.h +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The foldsim simulator's register map (hw/rtl/afu/foldsim/foldsim_regs.vh). -// Byte offsets on the kernel's AXI4-Lite control port. 0x040-0x0BF is left -// empty: on the V80 the PCIe block's MSI-X table sits at that offset of the BAR -// the first kernel is mapped at, and answers those accesses itself. - -#pragma once - -#define FOLDSIM_ID 0x464F4C44u -#define FOLDSIM_VERSION 5u - -#define FOLDSIM_REG_ID 0x000u -#define FOLDSIM_REG_VERSION 0x004u -#define FOLDSIM_REG_CTRL 0x008u // bit 0: the target runs -#define FOLDSIM_REG_STATUS 0x00Cu -#define FOLDSIM_REG_CMD_ADDR 0x010u // DCR address of the next command -#define FOLDSIM_REG_CMD_DATA 0x014u // DCR data, or reset cycles -#define FOLDSIM_REG_CMD 0x018u // write: queue a FOLDSIM_CMD_* -#define FOLDSIM_REG_DCR_RSP 0x01Cu // the last DCR read's response -#define FOLDSIM_REG_TCYCLES_LO 0x020u // target cycles since the kernel's reset -#define FOLDSIM_REG_TCYCLES_HI 0x024u -#define FOLDSIM_REG_HCYCLES_LO 0x028u // host (kernel clock) cycles -#define FOLDSIM_REG_HCYCLES_HI 0x02Cu -#define FOLDSIM_REG_MSTALL_LO 0x030u // host cycles waiting for device memory -#define FOLDSIM_REG_MSTALL_HI 0x034u -#define FOLDSIM_REG_NUM_CLASSES 0x038u // classes of threaded models -#define FOLDSIM_REG_MEM_BASE_LO 0x0C0u // device address of target address 0 -#define FOLDSIM_REG_MEM_BASE_HI 0x0C4u -#define FOLDSIM_REG_FINISH(c) (0x100u + 8u * (c)) // threads of class c finished, lo/hi - -// The memory model: each memory port's timing as a DRAM channel's banks -// (hw/rtl/afu/foldsim/foldsim_mem_bank.sv). Its settings take effect at the -// next target reset; a reset command's CMD_ADDR is the FOLDSIM_MEM_REFRESH_* -// the target has from that reset on. Times are DRAM clocks. -#define FOLDSIM_REG_MEM_REFRESH 0x0D8u // read only: the FOLDSIM_MEM_REFRESH_* in effect -#define FOLDSIM_REG_MEM_CAPS 0x0DCu // read only: FOLDSIM_MEM_CAPS_* -#define FOLDSIM_REG_MEM_CLOCK 0x0E0u // [15:0] data rate (MT/s), [31:16] core clock (MHz) -#define FOLDSIM_REG_MEM_LATENCY 0x0E4u // fixed read latency, target cycles -#define FOLDSIM_REG_MEM_QUEUE 0x0E8u // [15:0] requests in flight (0: slots only), - // [23:16] read queue, [31:24] write queue (bursts) -#define FOLDSIM_REG_MEM_GEOM 0x0ECu // log2 of: [1:0] pseudo-channels, [3:2] bank groups, - // [5:4] banks per group, [9:6] bursts per row, - // [11:10] bursts per row visit; [12] two-clock ACT -#define FOLDSIM_REG_MEM_WMARK 0x0F0u // write bursts: [7:0] to writes at, [15:8] back below -#define FOLDSIM_REG_MEM_REFI 0x0F4u // refresh: DRAM clocks from one to the next -#define FOLDSIM_REG_MEM_RFC 0x0F8u // refresh: DRAM clocks a refreshed bank may not activate -#define FOLDSIM_REG_MEM_TIMING(i) (0x800u + 4u * (i)) // FOLDSIM_MEM_T_*, 12 bits each -#define FOLDSIM_REG_MEM_STATS(p, i) (0xC00u + 0x40u * (p) + 8u * (i)) // port p's FOLDSIM_MEM_STAT_*, lo/hi - -#define FOLDSIM_MEM_REFRESH_OFF 0u -#define FOLDSIM_MEM_REFRESH_BANK 1u // per bank, the banks in turn -#define FOLDSIM_MEM_REFRESH_ALL 3u // all banks at once - -#define FOLDSIM_MEM_CAPS_PORTS(c) ((c) & 0xffu) // memory ports, a channel each -#define FOLDSIM_MEM_CAPS_HORIZON(c) (1u << ((((c) >> 8) & 0xfu) + 4u)) // ring, DRAM clocks -#define FOLDSIM_MEM_CAPS_SLOTS(c) (((c) >> 12) & 0xffu) // read slots per port -#define FOLDSIM_MEM_CAPS_TIME_W(c) ((((c) >> 28) & 0xfu) + 12u) // timing bits - -enum { - FOLDSIM_MEM_T_BL, FOLDSIM_MEM_T_CCDS, FOLDSIM_MEM_T_CCDL, FOLDSIM_MEM_T_CL, FOLDSIM_MEM_T_CWL, - FOLDSIM_MEM_T_RCDRD, FOLDSIM_MEM_T_RCDWR, FOLDSIM_MEM_T_RP, FOLDSIM_MEM_T_RAS, FOLDSIM_MEM_T_RC, - FOLDSIM_MEM_T_RTP, FOLDSIM_MEM_T_WR, FOLDSIM_MEM_T_RRDS, FOLDSIM_MEM_T_RRDL, FOLDSIM_MEM_T_FAW, - FOLDSIM_MEM_T_WTRS, FOLDSIM_MEM_T_WTRL, FOLDSIM_MEM_T_RTW_EXTRA, FOLDSIM_MEM_NUM_TIMINGS -}; - -enum { - FOLDSIM_MEM_STAT_READS, FOLDSIM_MEM_STAT_WRITES, FOLDSIM_MEM_STAT_HITS, FOLDSIM_MEM_STAT_MISSES, - FOLDSIM_MEM_STAT_CONFLICTS, FOLDSIM_MEM_STAT_LATENCY, FOLDSIM_MEM_STAT_REFRESHES, - FOLDSIM_MEM_STAT_FULL, FOLDSIM_MEM_NUM_STATS -}; - -#define FOLDSIM_STATUS_BUSY 0x1u // queued, running, or memory in flight -#define FOLDSIM_STATUS_FULL 0x2u // the command queue is full -#define FOLDSIM_STATUS_RSP 0x4u // a DCR read has its response -#define FOLDSIM_STATUS_MEM_ERR 0x8u // the target made a burst - -#define FOLDSIM_CMD_DCR_WRITE 1u -#define FOLDSIM_CMD_DCR_READ 2u -#define FOLDSIM_CMD_START 3u -#define FOLDSIM_CMD_RESET 4u diff --git a/sw/runtime/foldsim/Makefile b/sw/runtime/foldsim/Makefile index 64e7473fb9..72ac7e7a0d 100644 --- a/sw/runtime/foldsim/Makefile +++ b/sw/runtime/foldsim/Makefile @@ -13,6 +13,8 @@ SRC_DIR := $(VORTEX_HOME)/sw/runtime/foldsim CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror CXXFLAGS += -I$(INC_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) -I$(RT_COMMON_DIR) +# The shell's register map and in-process host, from the Foldsim installation. +CXXFLAGS += -I$(FOLDSIM_SHELL)/include CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += -fPIC CXXFLAGS += $(CONFIGS) @@ -22,7 +24,7 @@ LDFLAGS += -shared -pthread -Wl,-rpath,'$$ORIGIN' SRCS := $(SRC_DIR)/vortex.cpp $(RT_COMMON_DIR)/utils.cpp $(SIM_COMMON_DIR)/cmd_processor.cpp ifeq ($(TARGET), avedsim) -CXXFLAGS += -DFOLDSIM_EMUL -I$(SIM_DIR)/foldsim +CXXFLAGS += -DFOLDSIM_EMUL LDFLAGS += -L$(DESTDIR) -lfoldsim_emul EMUL := $(DESTDIR)/libfoldsim_emul.so else ifeq ($(TARGET), hw) diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp index ea099c3f60..eec4a8d5f2 100644 --- a/sw/runtime/foldsim/vortex.cpp +++ b/sw/runtime/foldsim/vortex.cpp @@ -17,7 +17,7 @@ // functional CommandProcessor model, and its hooks reach the foldsim // simulator. Device memory is the simulator's, in device memory at // FOLDSIM_REG_MEM_BASE; the target's DCR, start and busy are its command queue -// and status (sw/common/foldsim_regs.h). +// and status (Foldsim's shell, include/foldsim_regs.h). // // TARGET=hw runs the image on a V80 through VRT: registers over AXI4-Lite, // device memory over a QDMA queue. TARGET=avedsim simulates the same kernel @@ -85,7 +85,7 @@ class transport_t { } private: - foldsim_emul sim_; + foldsim::foldsim_emul sim_; }; #else From ad4e800d65a5e05e211373af7c9238e1074001b2 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 02:18:30 -0700 Subject: [PATCH 276/336] ci: name the logic-depth screen Beacon The tool that counts the logic levels of every register-to-register path before Vivado was called "the logic screen". Give it a proper name, Beacon, everywhere it is driven or documented: - ci/logic_screen.py -> ci/beacon.py; its `screen` subcommand is `run` - hw/scripts/screen_summary.py -> hw/scripts/beacon_summary.py - the DUT flow's `screen` target and `-screen` -> `beacon` and `-beacon` (hw/syn/xilinx/dut, and TARGET=beacon in hw/syn/yosys) - outputs beacon_summary.csv, beacon_units.json, beacon_paths.txt and the beacon/ build subdirectory; SCREEN_PATH_LEVELS -> BEACON_PATH_LEVELS - synth_gate's Screen backend -> Beacon (--tool=beacon), so the baselines live in ci/baselines/synthesis/beacon/ (none is recorded yet) - the logic_gate workflow's job, artifact globs and path filters - docs/designs/logic_screen.md -> docs/designs/beacon.md, its two SVGs, and the prose in AGENTS.md, the CI and Foldsim design docs Kept as they are: the gate's name and files (logic_gate.py, logic_gate.yaml, logic_gate.yml); "screen" in the graphics sense (screen space, full-screen, vp_screen); and the Foldsim pass names foldsim-screen-map and foldsim-screen-map-asic, which live in the separately built foldsim-opt. Each file that invokes them names them once (BEACON_MAP_PASS in the Makefiles, MAP_PASS/MAP_PASS_ASIC in ci/beacon.py, elab_done in synth_gate.py), so the Foldsim rename that follows is a one-line change per file. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/logic_gate.yml | 16 +-- AGENTS.md | 4 +- ci/{logic_screen.py => beacon.py} | 127 +++++++++--------- ci/logic_gate.py | 16 +-- ci/synth_gate.py | 35 ++--- ci/testcases/logic_gate.yaml | 18 +-- ...creen_compiler.svg => beacon_compiler.svg} | 2 +- ...ic_screen_system.svg => beacon_system.svg} | 28 ++-- docs/designs/{logic_screen.md => beacon.md} | 114 ++++++++-------- docs/designs/continuous_integration.md | 26 ++-- docs/designs/foldsim.md | 6 +- .../{screen_summary.py => beacon_summary.py} | 48 +++---- hw/scripts/circt_elaborate.sh | 2 +- hw/scripts/xilinx_cone_probe.tcl | 2 +- hw/syn/xilinx/dut/Makefile | 6 +- hw/syn/xilinx/dut/common.mk | 31 ++--- hw/syn/yosys/Makefile | 26 ++-- hw/syn/yosys/liberty_stage.py | 4 +- hw/syn/yosys/sta_endpoints.tcl | 4 +- 19 files changed, 260 insertions(+), 255 deletions(-) rename ci/{logic_screen.py => beacon.py} (86%) rename docs/assets/img/{logic_screen_compiler.svg => beacon_compiler.svg} (98%) rename docs/assets/img/{logic_screen_system.svg => beacon_system.svg} (87%) rename docs/designs/{logic_screen.md => beacon.md} (84%) rename hw/scripts/{screen_summary.py => beacon_summary.py} (80%) diff --git a/.github/workflows/logic_gate.yml b/.github/workflows/logic_gate.yml index 349b631f4c..c9ceb4791d 100644 --- a/.github/workflows/logic_gate.yml +++ b/.github/workflows/logic_gate.yml @@ -3,9 +3,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"). # # logic_gate — logic-level regression gate on hosted runners. -# Screens the DUT catalog (ci/testcases/logic_gate.yaml) with -# foldsim-estimate-timing and asserts the depth of every RTL unit against the -# golden results in ci/baselines/synthesis/screen/, so an RTL change that makes +# Runs Beacon over the DUT catalog (ci/testcases/logic_gate.yaml), through +# foldsim-estimate-timing, and asserts the depth of every RTL unit against the +# golden results in ci/baselines/synthesis/beacon/, so an RTL change that makes # a unit deeper is named on the change that does it, not the next night. # # Why its own workflow rather than a cell in ci.yml: it needs foldsim, which the @@ -28,7 +28,7 @@ on: - 'ci/synth_gate.py' - 'ci/logic_gate.py' - 'ci/testcases/logic_gate.yaml' - - 'ci/baselines/synthesis/screen/**' + - 'ci/baselines/synthesis/beacon/**' push: branches: [master] paths: @@ -39,7 +39,7 @@ on: - 'ci/synth_gate.py' - 'ci/logic_gate.py' - 'ci/testcases/logic_gate.yaml' - - 'ci/baselines/synthesis/screen/**' + - 'ci/baselines/synthesis/beacon/**' workflow_dispatch: inputs: builds: @@ -54,7 +54,7 @@ env: CCACHE_DISABLE: 1 jobs: - screen: + beacon: runs-on: ubuntu-22.04 timeout-minutes: 90 steps: @@ -112,8 +112,8 @@ jobs: logic_gate.json logic_gate_report.md build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/build.log - build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/screen_* - build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/screen/report.txt + build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/beacon_* + build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/beacon/report.txt if-no-files-found: warn - name: Verdict diff --git a/AGENTS.md b/AGENTS.md index 83cc984da4..c553eb0a55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,8 +86,8 @@ See [docs/testing.md](docs/testing.md) and [docs/debugging.md](docs/debugging.md - **RTL coverage path is `xrt`, not `rtlsim`.** When discussing or planning RTL verification, `xrt` is the canonical path — `rtlsim` bypasses the AFU surface. `rtlsim` remains useful for fast iteration on processor RTL; `xrt` is what proves the full integration. - **`ci/regression.sh` is the canonical source of tested configurations.** Use it to discover supported parameter combinations before inventing ad hoc ones. - **Perf-regression baselines (`ci/baselines/perf/*.json`) are golden data — never hand-edit them, and never "fix" a red perf gate by bumping the number.** They are regenerated only by `pytest ci -m perf_gate --update-baselines` (a human-run, reviewed step), and CI must never pass that flag. A `perf_gate` failure means real cycles moved: root-cause it, or — if the change is intended — regenerate the baseline so the diff shows the perf delta for review. Same rule as image goldens and `known_issue:`. -- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,screen}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/logic_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `logic_gate` means a unit got deeper, and its reasons name the unit. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). -- **Screen the logic depth of an RTL change before committing it.** A change to a pipeline, arbiter, queue or datapath can add logic levels that only the nightly `fpga_gate` would find. Run `make -C hw/syn/xilinx/dut -screen`, or `ci/logic_gate.py -b `, for the DUTs it touches. A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. See [docs/designs/logic_screen.md](docs/designs/logic_screen.md). +- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,beacon}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/logic_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `logic_gate` means a unit got deeper, and its reasons name the unit. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). +- **Check the logic depth of an RTL change with Beacon before committing it.** A change to a pipeline, arbiter, queue or datapath can add logic levels that only the nightly `fpga_gate` would find. Run `make -C hw/syn/xilinx/dut -beacon`, or `ci/logic_gate.py -b `, for the DUTs it touches. A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. See [docs/designs/beacon.md](docs/designs/beacon.md). - **SimX is the RTL's timing model — keep them in lockstep.** Any change that moves RTL cycles (pipeline structure, arbitration, queue depths, cache/memory behavior) must land together with the matching SimX timing-model update, and vice-versa. The `model_parity` CI gate enforces this: a `check: model_parity` case (`ci/testcases/core.yaml` + per-extension parity categories) runs the same app/args/configs on simx and rtlsim and asserts exact retired-instruction match plus cycle agreement within the case tolerance (default 5%). Never widen a tolerance to absorb a divergence — model the behavior. When adding a hardware feature, add or extend a parity case that exercises it. - **When RTL debugging stalls, switch to the SimX-as-oracle pattern.** For numerical bugs, deep pipeline races, or any failure mode where rtlsim is "close but wrong": (1) build/extend the SimX C++ model so it mirrors the *new* RTL architecture and gets to PASS; (2) add matching trace dumps to both SimX and RTL (cycle, FU events, SRAM addresses+data, hazards) — same CSV format on both sides; (3) diff trace files — the first divergence is the bug. Don't keep guessing from output values; localize via trace diff. See [docs/debugging.md](docs/debugging.md#simx-as-oracle-for-rtl-debug). diff --git a/ci/logic_screen.py b/ci/beacon.py similarity index 86% rename from ci/logic_screen.py rename to ci/beacon.py index b9efe48dd9..0cde0c9723 100755 --- a/ci/logic_screen.py +++ b/ci/beacon.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""logic_screen -- run foldsim's logic screen on a Vortex netlist, and read -its results by RTL unit or against Vivado. +"""beacon -- run Beacon, foldsim's logic-level count, on a Vortex netlist, and +read its results by RTL unit or against Vivado. - screen map an elaborated CIRCT netlist and count the logic levels of - every end point (foldsim-screen-map, foldsim-estimate-timing) + run map an elaborated CIRCT netlist and count the logic levels of + every end point (foldsim-estimate-timing) summarize the deep end points grouped by RTL unit, optionally beside Vivado's failing ones paths every deep path op by op, by RTL unit (one per pair of @@ -27,20 +27,20 @@ buffers which elastic buffers pass a deep ready chain or a computed output through: evidence for an OUT_BUF (OUT_REG) change - ci/logic_screen.py screen gen/design.mlir runs/full - ci/logic_screen.py screen runs/full/mapped.mlirbc runs/socket \\ + ci/beacon.py run gen/design.mlir runs/full + ci/beacon.py run runs/full/mapped.mlirbc runs/socket \\ top=VX_socket__lift black-boxes=VX_core__lift - ci/logic_screen.py summarize runs/full --min-levels 12 - ci/logic_screen.py paths runs/full --unit lsu_unit - ci/logic_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut - ci/logic_screen.py screen --keep VX_elastic_buffer design.mlir runs/eb \\ + ci/beacon.py summarize runs/full --min-levels 12 + ci/beacon.py paths runs/full --unit lsu_unit + ci/beacon.py validate runs/socket ep.csv --native VX_socket__lift=dut + ci/beacon.py run --keep VX_elastic_buffer design.mlir runs/eb \\ port-modules=VX_elastic_buffer - ci/logic_screen.py buffers runs/eb + ci/beacon.py buffers runs/eb A Vivado end point table is a CSV of `slack,levels,logic,net,start,end`, one -row per failing end point (a gate's ep.csv). The DUT flow and logic_gate -screen with the Makefiles instead (make -C hw/syn/xilinx/dut -screen, -ci/logic_gate.py); see docs/designs/logic_screen.md. +row per failing end point (a gate's ep.csv). The DUT flow and logic_gate run +Beacon through the Makefiles instead (make -C hw/syn/xilinx/dut -beacon, +ci/logic_gate.py); see docs/designs/beacon.md. """ import argparse import collections @@ -61,6 +61,9 @@ 'foldsim_target': 'foldsim_target_model_copy', } +MAP_PASS = 'foldsim-screen-map' +MAP_PASS_ASIC = 'foldsim-screen-map-asic' + VIVADO_CELL = re.compile( r'_reg((\[\d+\])*(_replica(_\d+)?)?(_[fb]ret(_\d+)?)*(_rep(_\d+)?)?' r'|_\d+_\d+_\d+_\d+|_bram_\d+)$') @@ -106,7 +109,7 @@ def ep_key(cell_path, native): def numpy(): - # Only the comparisons with Vivado need numpy; screening does not. + # Only the comparisons with Vivado need numpy; counting levels does not. import numpy as np return np @@ -129,9 +132,9 @@ def rank(x): return float(np.corrcoef(ra, rb)[0, 1]) -def endpoints(screen): - """The rows of a screen's end point table, as dicts.""" - with open(f'{screen}/endpoints.csv') as f: +def endpoints(run_dir): + """The rows of a run's end point table, as dicts.""" + with open(f'{run_dir}/endpoints.csv') as f: yield from csv.DictReader(f) @@ -174,8 +177,8 @@ def unit(key): return INDEX.sub('_*', d) -def cmd_screen(args): - """Map (unless the input is already mapped bytecode), then screen.""" +def cmd_run(args): + """Map (unless the input is already mapped bytecode), then count levels.""" opt = os.environ.get("FOLDSIM_OPT") or "foldsim-opt" os.makedirs(args.out, exist_ok=True) asic = any(o == "family=asap7" for o in args.options) @@ -186,7 +189,7 @@ def cmd_screen(args): threads = ["--mlir-disable-threading"] if args.no_threads else [] if not args.design.endswith(".mlirbc"): mapped = os.path.join(args.out, "mapped.mlirbc") - pipeline = "foldsim-screen-map-asic" if asic else "foldsim-screen-map" + pipeline = MAP_PASS_ASIC if asic else MAP_PASS if args.keep: # Keep these modules' instances through inlining, so their ports # can be reported (ports=, port-modules=). @@ -202,7 +205,7 @@ def cmd_screen(args): if any(o.startswith("port-modules=") for o in args.options): files.append("ports=%s/ports.csv" % args.out) opts = " ".join(files + args.options) - steps.append(("screen", [opt, mapped, "-o", os.devnull, + steps.append(("levels", [opt, mapped, "-o", os.devnull, "--pass-pipeline=builtin.module(" "foldsim-estimate-timing{%s})" % opts])) for name, cmd in steps: @@ -218,7 +221,7 @@ def cmd_screen(args): def cmd_validate(args): np = numpy() - inst = load_instances(f'{args.screen}/instances.csv') + inst = load_instances(f'{args.run_dir}/instances.csv') classes = FOLD_CLASSES sub = None if args.native: @@ -232,7 +235,7 @@ def cmd_validate(args): # that place a register in the generate scope that assigns it rather # than the one that declares it. by_dir = collections.defaultdict(dict) - for row in endpoints(args.screen): + for row in endpoints(args.run_dir): cls, p = paths[int(row['end_inst'])] if cls is None: continue @@ -301,22 +304,22 @@ def fallback(key): print(f'ep rows {rows}: fold shell / channels {shell_rows}, ' f'unmatched {unmatched_rows}, matched {rows - shell_rows - unmatched_rows} ' f'in {len(keys)} registers/memories') - print(f'screen keys in the classes: {len(ours)}') - print(f'Spearman(screen levels, Vivado levels) = {spearman(a, lv):.3f}') - print(f'Spearman(screen levels, -slack) = {spearman(a, -sl):.3f}') + print(f'Beacon keys in the classes: {len(ours)}') + print(f'Spearman(Beacon levels, Vivado levels) = {spearman(a, lv):.3f}') + print(f'Spearman(Beacon levels, -slack) = {spearman(a, -sl):.3f}') ratio = a[lv > 0] / lv[lv > 0] - print(f'screen/Vivado levels: median {np.median(ratio):.2f}, ' + print(f'Beacon/Vivado levels: median {np.median(ratio):.2f}, ' f'p10 {np.percentile(ratio, 10):.2f}, p90 {np.percentile(ratio, 90):.2f}') - print('\nVivado levels -> screen levels (registers/memories):') + print('\nVivado levels -> Beacon levels (registers/memories):') for v in range(0, int(lv.max()) + 1): sel = a[lv == v] if len(sel): - print(f' {v:3d}: n={len(sel):6d} screen median {np.median(sel):5.1f} ' + print(f' {v:3d}: n={len(sel):6d} Beacon median {np.median(sel):5.1f} ' f'p10 {np.percentile(sel, 10):5.1f} p90 {np.percentile(sel, 90):5.1f}') if args.native: - # The ep.csv of a gate lists every failing end point, so a screened + # The ep.csv of a gate lists every failing end point, so a measured # register without one meets timing. scr = np.array([v[0] for v in ours.values()]) fails = np.array([k in viv for k in ours]) @@ -334,7 +337,7 @@ def fallback(key): if args.json: summary = { 'rows': rows, 'shell': shell_rows, 'unmatched': unmatched_rows, - 'registers': len(keys), 'screened': len(ours), + 'registers': len(keys), 'measured': len(ours), 'rho_levels': round(spearman(a, lv), 3), 'rho_slack': round(spearman(a, -sl), 3), 'ratio': round(float(np.median(ratio)), 2), @@ -354,17 +357,17 @@ def fallback(key): print(f' {n:6d} {cls}: {k}') order = np.argsort(-a, kind='mergesort') - print(f'\ndeepest screened registers with failing Vivado end points:') + print(f'\ndeepest measured registers with failing Vivado end points:') for i in order[:args.top]: k = keys[i] - print(f' screen {a[i]:3d} vivado {lv[i]:3d} slack {sl[i]:7.3f} ' + print(f' beacon {a[i]:3d} vivado {lv[i]:3d} slack {sl[i]:7.3f} ' f'n {viv[k][2]:5d} {k[0]}: {k[1]}') def cmd_summarize(args): - inst = load_instances(f'{args.screen}/instances.csv') + inst = load_instances(f'{args.run_dir}/instances.csv') classes = FOLD_CLASSES if args.native: module, prefix = args.native.split('=') @@ -374,7 +377,7 @@ def cmd_summarize(args): groups = collections.defaultdict( lambda: {'n': 0, 'max': -1, 'hist': collections.Counter(), 'worst': None}) - for row in endpoints(args.screen): + for row in endpoints(args.run_dir): levels = int(row[args.metric]) if levels < args.min_levels: continue @@ -425,7 +428,7 @@ def cmd_summarize(args): missed = sorted((k for k in viv if k not in groups), key=lambda k: -viv[k][0]) print(f'\nVivado units at or above {args.min_levels} levels the ' - f'screen ranks below it ({len(missed)}):') + f'Beacon ranks below it ({len(missed)}):') for k in missed[:args.top]: print(f'{viv[k][0]:8d} {viv[k][1]:4d} {k[0]}: {k[1]}') @@ -435,18 +438,18 @@ def cmd_paths(args): """Every path of at least --min-levels, by unit, op by op.""" sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'hw', 'scripts')) - import screen_summary - count, _ = screen_summary.write_paths(sys.stdout, args.screen, + import beacon_summary + count, _ = beacon_summary.write_paths(sys.stdout, args.run_dir, args.min_levels, args.unit) if not count: - print('(the screen lists paths from its path-levels= on, 12 by ' + print('(Beacon lists paths from its path-levels= on, 12 by ' 'default)') -def load_ports(screen): - """instance -> {port: (direction, levels)} from a screen's ports.csv.""" +def load_ports(run_dir): + """instance -> {port: (direction, levels)} from a run's ports.csv.""" ports = collections.defaultdict(dict) - with open(f'{screen}/ports.csv') as f: + with open(f'{run_dir}/ports.csv') as f: for row in csv.DictReader(f): ports[(row['instance'], row['module'])][row['port']] = ( row['direction'], int(row['levels'])) @@ -456,7 +459,7 @@ def load_ports(screen): def cmd_ports(args): """The deepest instance of each unit, port by port.""" units = {} - for (inst, module), ports in load_ports(args.screen).items(): + for (inst, module), ports in load_ports(args.run_dir).items(): deepest = max(levels for _, levels in ports.values()) key = (INDEX.sub('_*', norm(inst)), module.rstrip('0123456789').rstrip('_')) if key not in units or deepest > units[key][0]: @@ -480,7 +483,7 @@ def cmd_buffers(args): registered: OUT_REG would register it. """ rows = [] - for (inst, module), ports in load_ports(args.screen).items(): + for (inst, module), ports in load_ports(args.run_dir).items(): if not module.startswith('VX_elastic_buffer'): continue get = lambda p: ports.get(p, ('', -1))[1] @@ -517,12 +520,12 @@ def cmd_buffers(args): def cmd_retime_split(args): np = numpy() - screen, ep, native, metric = args.screen, args.ep, args.native, args.metric + run_dir, ep, native, metric = args.run_dir, args.ep, args.native, args.metric module, prefix = native.split('=') - inst = load_instances(f'{screen}/instances.csv') + inst = load_instances(f'{run_dir}/instances.csv') paths = class_paths(inst, {module: prefix}) ours = collections.defaultdict(int) - for row in endpoints(screen): + for row in endpoints(run_dir): cls, p = paths[int(row['end_inst'])] base = row['end'].rsplit('.', 1)[0] key = (cls, norm(f'{p}/{base}' if p else base)) @@ -546,8 +549,8 @@ def cmd_retime_split(args): continue ratio = np.median(a[b > 0] / b[b > 0]) if (b > 0).any() else float('nan') print(f'retimed={retimed!s:5}: {len(keys):5d} registers, Spearman ' - f'{spearman(a, b):.3f}, median screen/Vivado {ratio:.2f}, ' - f'screen-Vivado median {np.median(a - b):+.1f}') + f'{spearman(a, b):.3f}, median Beacon/Vivado {ratio:.2f}, ' + f'Beacon-Vivado median {np.median(a - b):+.1f}') @@ -556,7 +559,7 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="cmd", required=True) - p = sub.add_parser("screen", help="map and screen a netlist") + p = sub.add_parser("run", help="map a netlist and count its levels") p.add_argument("design", help="elaborated netlist (.mlir), or mapped bytecode (.mlirbc)") p.add_argument("out", help="output directory") p.add_argument("options", nargs="*", @@ -567,51 +570,51 @@ def main(): p.add_argument("--keep", action="append", default=[], metavar="MODULE", help="keep MODULE's instances through inlining, to report " "their ports with port-modules=MODULE (repeatable)") - p.set_defaults(fn=cmd_screen) + p.set_defaults(fn=cmd_run) p = sub.add_parser("summarize", help="deep end points by RTL unit") - p.add_argument('screen') + p.add_argument('run_dir') p.add_argument('--min-levels', type=int, default=10) p.add_argument('--ep', help="a Vivado end point table to group beside") p.add_argument('--native', metavar='MODULE=PREFIX') p.add_argument('--top', type=int, default=40) p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], - help="the screen's column to group by") + help="the run's column to group by") p.set_defaults(fn=cmd_summarize) p = sub.add_parser("paths", help="deep paths op by op, by RTL unit") - p.add_argument('screen') + p.add_argument('run_dir') p.add_argument('--min-levels', type=int, default=12) p.add_argument('--unit', help="only the units whose name contains this") p.set_defaults(fn=cmd_paths) p = sub.add_parser("validate", help="agreement per register with Vivado") - p.add_argument('screen') + p.add_argument('run_dir') p.add_argument('ep') p.add_argument('--top', type=int, default=30) p.add_argument('--native', metavar='MODULE=PREFIX[:SUBPATH]', - help='native gate: the screened top and the harness ' + help='native gate: the measured top and the harness ' 'prefix of its cells (e.g. VX_socket__lift=dut); with ' - 'SUBPATH, only the screened end points below it, which ' + 'SUBPATH, only the measured end points below it, which ' 'the harness instantiates as PREFIX') p.add_argument('--json', help='also write the agreement to this file') p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], - help="the screen's column to compare") + help="the run's column to compare") p.set_defaults(fn=cmd_validate) p = sub.add_parser("ports", help="port depths of chosen modules, by unit") - p.add_argument('screen') + p.add_argument('run_dir') p.add_argument('--min-levels', type=int, default=8) p.add_argument('--top', type=int, default=40) p.set_defaults(fn=cmd_ports) p = sub.add_parser("buffers", help="elastic buffers that pass deep paths") - p.add_argument('screen') + p.add_argument('run_dir') p.add_argument('--min-levels', type=int, default=8) p.set_defaults(fn=cmd_buffers) p = sub.add_parser("retime-split", help="agreement split by Vivado's retiming") - p.add_argument('screen') + p.add_argument('run_dir') p.add_argument('ep') p.add_argument('native', metavar='MODULE=PREFIX') p.add_argument('metric', nargs='?', default='levels', choices=['levels', 'retimed']) diff --git a/ci/logic_gate.py b/ci/logic_gate.py index d220e423a6..61a2df84e8 100755 --- a/ci/logic_gate.py +++ b/ci/logic_gate.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""logic_gate — logic-level regression gate (the logic screen). +"""logic_gate — logic-level regression gate (Beacon). Gates the depth of every RTL unit of the DUT catalog: foldsim-estimate-timing counts the logic levels of every register-to-register path of the CIRCT netlist @@ -9,11 +9,11 @@ gating machinery is shared with the synthesis gates and lives in ci/synth_gate.py; this entry point only pins the tool. - Spec : ci/testcases/logic_gate.yaml (what to screen) - Baselines : ci/baselines/synthesis/screen/.json (last accepted levels) + Spec : ci/testcases/logic_gate.yaml (what to measure) + Baselines : ci/baselines/synthesis/beacon/.json (last accepted levels) Builds : hw/syn/xilinx/dut/logic_gate_ (no Vivado) - Metrics : /screen_summary.csv, screen_units.json - (written by hw/syn/xilinx/dut's `screen` target) + Metrics : /beacon_summary.csv, beacon_units.json + (written by hw/syn/xilinx/dut's `beacon` target) It needs CIRCT and foldsim (ci/toolchain_install.sh --circt --foldsim); the Xilinx environment is not used. @@ -24,8 +24,8 @@ ci/logic_gate.py -b cache -b tcu # explicit builds ci/logic_gate.py --update-baseline # re-record (human-reviewed, never in CI) -The same screen runs by hand through the flow's own entry point -- -`make -C hw/syn/xilinx/dut cache-screen` -- so the gate adds no second way to +Beacon runs by hand through the flow's own entry point -- +`make -C hw/syn/xilinx/dut cache-beacon` -- so the gate adds no second way to build. See docs/designs/continuous_integration.md 3.5. @@ -38,4 +38,4 @@ import synth_gate # noqa: E402 if __name__ == "__main__": - sys.exit(synth_gate.main(default_tool="screen")) + sys.exit(synth_gate.main(default_tool="beacon")) diff --git a/ci/synth_gate.py b/ci/synth_gate.py index b042d863cf..9e5dab60f2 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -12,11 +12,11 @@ --tool=xilinx Vivado synthesis + place-and-route -> ci/fpga_gate.py --tool=yosys Yosys + OpenSTA on an ASIC PDK -> ci/asic_gate.py - --tool=screen logic levels per RTL unit, no synthesis -> ci/logic_gate.py + --tool=beacon logic levels per RTL unit, no synthesis -> ci/logic_gate.py Spec : ci/testcases/.yaml (what to build) Baselines : ci/baselines/synthesis//.json (last accepted metrics) - Metrics : /synth_summary.csv (screen_summary.csv for the screen), + Metrics : /synth_summary.csv (beacon_summary.csv for Beacon), written by the flow itself Same division as every other check in the catalog: the yaml is hand-authored, @@ -457,9 +457,9 @@ def describe_baseline(self, base): base.get("cell_area_um2") or 0) -class Screen(Tool): - """The logic screen: logic levels per RTL unit (hw/syn/xilinx/dut, target - `screen`). +class Beacon(Tool): + """Beacon: logic levels per RTL unit (hw/syn/xilinx/dut, target + `beacon`). foldsim-estimate-timing ranks every register-to-register path of the CIRCT netlist the Xilinx flow elaborates by its logic levels, counted as Vivado @@ -469,11 +469,11 @@ class Screen(Tool): Depth is deterministic -- the same RTL and tools give the same levels -- so the gate has no tolerance: a unit that gets deeper fails, by name, until it - is fixed or a human re-records it. What it gates is depth, not Fmax: the - screen does not see wires (docs/designs/continuous_integration.md 3.5). + is fixed or a human re-records it. What it gates is depth, not Fmax: + Beacon does not see wires (docs/designs/continuous_integration.md 3.5). """ - name = "screen" + name = "beacon" gate = "logic_gate" flow = "hw/syn/xilinx/dut" @@ -482,14 +482,15 @@ class Screen(Tool): gated = ("max_levels",) integral = metrics key_metric = "max_levels" - summary = "screen_summary.csv" + summary = "beacon_summary.csv" tolerance = 0.0 columns = (("max_lv", "max_levels", "%d", 7), ("deep", "deep_endpoints", "%d", 9), ("watch", "watch_endpoints", "%d", 9), ("ends", "endpoints", "%d", 9)) - # Elaboration and mapping of the widest DUTs; the screen itself is seconds. + # Elaboration and mapping of the widest DUTs; the level count itself is + # seconds. weights = {"top": 600, "vortex": 600, "gfx": 600, "core": 300, "tensor": 300, "tcu": 300} fallback_weight = 120 @@ -503,13 +504,13 @@ class Screen(Tool): ("gen_sources.sh", "sources"), ("circt_elaborate.sh", "elaboration"), (elab_done, "mapping"), - ("foldsim-estimate-timing", "screen"), - ("screen_summary.py", "reporting"), + ("foldsim-estimate-timing", "levels"), + ("beacon_summary.py", "reporting"), ) error_re = re.compile(r"^(?:ERROR|FATAL|Error)\b|: error:|^\S+: \*\*\* ") - UNITS = "screen_units.json" - PATHS = "screen_paths.txt" + UNITS = "beacon_units.json" + PATHS = "beacon_paths.txt" def env(self, args): return {"circt": tool_version("circt-verilog", "--version", @@ -546,7 +547,7 @@ def build_command(self, dut_root, work, build, env, args): os.makedirs(work, exist_ok=True) shutil.copy2(os.path.join(dut_root, "build.mk"), os.path.join(work, "Makefile")) - return ["make", "-C", work, "screen"], dict(DUT=build["dut"], + return ["make", "-C", work, "beacon"], dict(DUT=build["dut"], CONFIGS=build["configs"]) def extras(self, work): @@ -604,7 +605,7 @@ def describe_baseline(self, base): return "max %d levels" % base["max_levels"] -TOOLS = {t.name: t for t in (Xilinx(), Yosys(), Screen())} +TOOLS = {t.name: t for t in (Xilinx(), Yosys(), Beacon())} def tool_version(name, flag, package=None, line_prefix=None): @@ -1350,7 +1351,7 @@ def main(argv=None, default_tool="xilinx"): % default_tool) ap.add_argument("--threshold", type=float, default=None, metavar="F", help="global gate tolerance, fraction (default: the " - "tool's, %.2f for synthesis, 0 for the screen). " + "tool's, %.2f for synthesis, 0 for Beacon). " "Overridden per-metric by --metric-threshold and " "per-build by the catalog's \"thresholds\"." % TOLERANCE) diff --git a/ci/testcases/logic_gate.yaml b/ci/testcases/logic_gate.yaml index 0c0e3b1f68..af6f6e7f1b 100644 --- a/ci/testcases/logic_gate.yaml +++ b/ci/testcases/logic_gate.yaml @@ -1,6 +1,6 @@ -# Logic-level regression gate (the logic screen): the deepest end point of -# every RTL unit must not get deeper than a checked-in golden result -# (ci/baselines/synthesis/screen/.json). See ci/logic_gate.py. +# Logic-level regression gate (Beacon): the deepest end point of every RTL +# unit must not get deeper than a checked-in golden result +# (ci/baselines/synthesis/beacon/.json). See ci/logic_gate.py. # # Sibling of fpga_gate (fpga_gate.yaml) and asic_gate (asic_gate.yaml): same # discipline -- golden baseline, results only ever re-recorded by a human -- @@ -13,7 +13,7 @@ # folded (g_banks_3_bank -> g_banks_*_bank). Every unit at or above the # watch threshold (8 levels at 300 MHz) is recorded; a unit that gets deeper # fails, and so does a new unit at or above the fix threshold (12). -# - No tolerance. The screen is deterministic, so any change in depth is a +# - No tolerance. Beacon is deterministic, so any change in depth is a # real one; a shallower unit reads IMPROVED, to be locked in by a re-record. # - The tool versions (CIRCT, foldsim) are part of config_hash: a new model # reads STALE, not as a regression. @@ -29,7 +29,7 @@ # The builds are fpga_gate.yaml's, DUT for DUT and config for config (all but # `top`, see below), so where both gates run a DUT they measure the same RTL, and a unit this gate flags # can be read against the Fmax gate's critical paths. clock_mhz is the Fmax -# gate's target, recorded for that comparison; the screen's thresholds are +# gate's target, recorded for that comparison; Beacon's thresholds are # 300 MHz ones. # # Tier `timing` is OPT-IN (OPT_IN_TIERS in testcase.py): it needs foldsim, @@ -37,7 +37,7 @@ # installs it and runs the gate. # # Run one by name: ci/logic_gate.py -b tcu -# By hand, no gate: make -C hw/syn/xilinx/dut tcu-screen +# By hand, no gate: make -C hw/syn/xilinx/dut tcu-beacon # Re-record: ci/logic_gate.py --update-baseline (human-reviewed, never CI) # --------------------------------------------------------------------------- category: logic_gate @@ -58,11 +58,11 @@ tests: # --------------------------------------------------------------------------- # builds — read by ci/logic_gate.py, ignored by the pytest catalog loader. # -# dut hw/syn/xilinx/dut/catalog.mk target to screen +# dut hw/syn/xilinx/dut/catalog.mk target to measure # clock_mhz the Fmax gate's target for this DUT (recorded, not asserted) # configs EXTRA defines, appended to the catalog's # group which golden file the results land in -# known_issue tracked expected failure: still screens and reports, does not +# known_issue tracked expected failure: still measures and reports, does not # fail the run # --------------------------------------------------------------------------- builds: @@ -84,7 +84,7 @@ builds: clock_mhz: 300 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE -# fpga_gate's `top` is not screened: CIRCT's import leaves the command +# fpga_gate's `top` is not measured: CIRCT's import leaves the command # processor's AXI-Lite response ports (VX_cp_core) as LLHD references, which # no HW/Comb/Seq tool reads. It joins this list when the import lowers them. diff --git a/docs/assets/img/logic_screen_compiler.svg b/docs/assets/img/beacon_compiler.svg similarity index 98% rename from docs/assets/img/logic_screen_compiler.svg rename to docs/assets/img/beacon_compiler.svg index 14f9df9cde..ea03690c04 100644 --- a/docs/assets/img/logic_screen_compiler.svg +++ b/docs/assets/img/beacon_compiler.svg @@ -22,7 +22,7 @@ - Logic screen — the two passes + Beacon — the two passes diff --git a/docs/assets/img/logic_screen_system.svg b/docs/assets/img/beacon_system.svg similarity index 87% rename from docs/assets/img/logic_screen_system.svg rename to docs/assets/img/beacon_system.svg index bd8cbbe674..f9414e7868 100644 --- a/docs/assets/img/logic_screen_system.svg +++ b/docs/assets/img/beacon_system.svg @@ -26,7 +26,7 @@ - Logic screen — where each piece lives, and what flows between them + Beacon — where each piece lives, and what flows between them @@ -38,21 +38,21 @@ the DUT catalog: hw/syn/xilinx/dut/catalog.mk - hw/syn/xilinx/dut — target screen + hw/syn/xilinx/dut — target beacon gen_sources.sh (hw/rtl, toml defines) circt_elaborate.sh -f mlir -x xilinx_ip_stubs.sv gen_platform.py --device=$(DEVICE) --emit=family - hw/scripts/screen_summary.py - screen_summary.csv, screen_units.json - screen_paths.txt (deep paths by unit) + hw/scripts/beacon_summary.py + beacon_summary.csv, beacon_units.json + beacon_paths.txt (deep paths by unit) units = instance paths, generate indices folded - ci/logic_gate.py (synth_gate.py Screen backend) + ci/logic_gate.py (synth_gate.py Beacon backend) ci/testcases/logic_gate.yaml - ci/baselines/synthesis/screen/*.json + ci/baselines/synthesis/beacon/*.json .github/workflows/logic_gate.yml PASS / FAIL per unit, zero tolerance on depth @@ -80,12 +80,12 @@ optional: retime-levels=, trace=, black-boxes= - Outputs (per DUT, screen/) + Outputs (per DUT, beacon/) endpoints.csv one row per end point bit instances.csv the elaborated instance tree report.txt histogram, deepest paths op by op paths.jsonl every deep path, source lines - mapped.mlirbc kept for re-screening other tops + mapped.mlirbc kept to measure other tops @@ -99,22 +99,22 @@ RTL timing work (native-timing loop) - screen a whole GPU in ~1 min before a + run Beacon on a whole GPU in ~1 min before a 20-40 min Vivado run; fix list = units at ≥ 12 levels, watch list ≥ 8 (300 MHz) - ci/logic_screen.py, hw/scripts/xilinx_*.tcl - screen summarize validate retime-split + ci/beacon.py, hw/scripts/xilinx_*.tcl + run summarize validate retime-split xilinx_cone_dump.tcl xilinx_cone_probe.tcl xilinx_wire_probe.tcl - screen ↔ Vivado agreement, and the Vivado + Beacon ↔ Vivado agreement, and the Vivado probes for what levels cannot show fpga_gate (nightly, Vivado) same DUTs and configurations: a unit the - screen flags is checked against its + Beacon flags is checked against its critical paths; Fmax stays its job diff --git a/docs/designs/logic_screen.md b/docs/designs/beacon.md similarity index 84% rename from docs/designs/logic_screen.md rename to docs/designs/beacon.md index 1b4bb1071d..81bb6b9049 100644 --- a/docs/designs/logic_screen.md +++ b/docs/designs/beacon.md @@ -1,11 +1,11 @@ -# Logic screen — logic depth of every path, before Vivado +# Beacon — logic depth of every path, before Vivado -The logic screen counts the logic levels of every register-to-register path +Beacon counts the logic levels of every register-to-register path of a Vortex design the way Vivado counts them, in seconds and without Vivado. It names the RTL units whose paths are too deep for the target clock, gates their depth on every RTL change (`logic_gate`), and is the first step -of RTL timing work: screen, fix the deep units, then confirm with one Vivado -run. +of RTL timing work: run Beacon, fix the deep units, then confirm with one +Vivado run. | § | contents | |---|---| @@ -26,7 +26,7 @@ run. It answers one question per register: **how many logic levels is its worst input path?** At 300 MHz on the V80, failing end points have a median of 7–8 levels, and between half and four fifths of the registers at 12 or more -levels fail, depending on the gate (§7). So the screen reports: +levels fail, depending on the gate (§7). So Beacon reports: - **fix list:** units with end points at ≥ 12 levels; - **watch list:** ≥ 8 levels. @@ -36,9 +36,9 @@ It does not predict slack, because it does not see wires (§9). One DUT of the synthesis catalog, from a configured build tree: ```bash -make -C hw/syn/xilinx/dut tcu-screen -# build_tcu/screen_paths.txt every path at >= 8 levels, op by op, by unit -# build_tcu/screen_units.json deepest end point per RTL unit +make -C hw/syn/xilinx/dut tcu-beacon +# build_tcu/beacon_paths.txt every path at >= 8 levels, op by op, by unit +# build_tcu/beacon_units.json deepest end point per RTL unit ``` The whole catalog as a gate, and a baseline update (a human step): @@ -48,17 +48,17 @@ ci/logic_gate.py -j 2 ci/logic_gate.py -b om --update-baseline ``` -Any CIRCT netlist, with `ci/logic_screen.py` (subcommands `screen`, +Any CIRCT netlist, with `ci/beacon.py` (subcommands `run`, `summarize`, `paths`, `validate`, `retime-split`, `ports`, `buffers`; `foldsim-opt` from the `PATH` or `FOLDSIM_OPT`): ```bash -ci/logic_screen.py screen design.mlir runs/full # map, then screen -ci/logic_screen.py screen runs/full/mapped.mlirbc runs/socket \ +ci/beacon.py run design.mlir runs/full # map, then count levels +ci/beacon.py run runs/full/mapped.mlirbc runs/socket \ top=VX_socket__lift black-boxes=VX_core__lift # one unit of a mapped design -ci/logic_screen.py summarize runs/full --min-levels 12 # deep end points by RTL unit -ci/logic_screen.py paths runs/full --unit lsu_unit # its paths, op by op (§5) -ci/logic_screen.py validate runs/socket ep.csv --native VX_socket__lift=dut +ci/beacon.py summarize runs/full --min-levels 12 # deep end points by RTL unit +ci/beacon.py paths runs/full --unit lsu_unit # its paths, op by op (§5) +ci/beacon.py validate runs/socket ep.csv --native VX_socket__lift=dut ``` Or the two passes directly: @@ -87,7 +87,7 @@ foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( ## 2. System architecture -![Logic screen system](../assets/img/logic_screen_system.svg) +![Beacon system](../assets/img/beacon_system.svg) The engine is two compiler passes of **foldsim** (`foldsim-opt`), built on CIRCT like the rest of foldsim, and shipped in the foldsim prebuilt. Vortex @@ -96,23 +96,23 @@ never builds it. The integration is in Vortex: | Piece | Where | Role | |---|---|---| | `foldsim-screen-map`, `foldsim-estimate-timing` | foldsim `lib/Transforms/EstimateTiming.cpp` | map to LUTs; count levels | -| `ci/logic_screen.py` | `ci/` | screen any netlist; read it by unit; compare with Vivado | +| `ci/beacon.py` | `ci/` | run Beacon on any netlist; read it by unit; compare with Vivado | | `xilinx_cone_dump.tcl`, `xilinx_cone_probe.tcl`, `xilinx_wire_probe.tcl` | `hw/scripts` | Vivado probes for what levels cannot show (§8) | -| target `screen` | `hw/syn/xilinx/dut/common.mk` | one DUT: sources → netlist → screen → summary | +| target `beacon` | `hw/syn/xilinx/dut/common.mk` | one DUT: sources → netlist → levels → summary | | `circt_elaborate.sh -f mlir` | `hw/scripts` | the lowered HW/Comb/Seq netlist, no `firtool` | | `gen_platform.py --emit=family` | `hw/scripts` | the family of `DEVICE`, from `hw/syn/xilinx/platforms` | -| `screen_summary.py` | `hw/scripts` | `screen_summary.csv` and per-unit `screen_units.json` | -| `Screen` backend, `logic_gate.py` | `ci/synth_gate.py`, `ci/` | the gate, beside `fpga_gate` and `asic_gate` | +| `beacon_summary.py` | `hw/scripts` | `beacon_summary.csv` and per-unit `beacon_units.json` | +| `Beacon` backend, `logic_gate.py` | `ci/synth_gate.py`, `ci/` | the gate, beside `fpga_gate` and `asic_gate` | | `logic_gate.yml` | `.github/workflows` | runs it on pull requests that touch RTL | -The screen reads the same CIRCT import that foldsim and the Yosys flow +Beacon reads the same CIRCT import that foldsim and the Yosys flow use (`circt-verilog --ir-hw`), elaborated from the sources and defines `gen_sources.sh` writes for Vivado, with the Xilinx floating-point IPs as black boxes (`xilinx_ip_stubs.sv`). ## 3. Compiler architecture -![Logic screen compiler](../assets/img/logic_screen_compiler.svg) +![Beacon compiler](../assets/img/beacon_compiler.svg) ### 3.1 `foldsim-screen-map` @@ -157,7 +157,7 @@ and one **node** per (context, value, bit). It never enumerates paths: predecessors back; the walk also yields the path's largest fanout and its module crossings. -Time and memory are linear in the design's bits: the full GPU screens in +Time and memory are linear in the design's bits: the full GPU runs in 16–21 s and 2.7 GB, a DUT of the catalog in seconds. ## 4. The model @@ -202,8 +202,8 @@ family is an error. ### 4.3 ASAP7 -For the ASIC flow (`hw/syn/yosys`, ASAP7 7.5-track RVT, TT corner) the -screen counts **cell stages**, not LUT levels. `foldsim-screen-map-asic` +For the ASIC flow (`hw/syn/yosys`, ASAP7 7.5-track RVT, TT corner) +Beacon counts **cell stages**, not LUT levels. `foldsim-screen-map-asic` maps the same balanced and-inverter graph to 3-input cuts, each one cell (NAND, AOI, MAJ, XOR), and `family=asap7` models the word-level operators as an ASIC builds them: @@ -252,10 +252,10 @@ candidates retiming is unlikely to absorb. | `endpoints.csv` | one row per end point bit: `levels,retimed,kind,start_kind,fanout,crossings,start_inst,start,end_inst,end` | | `instances.csv` | the instance tree: `id,parent,instance,module` (the `*_inst` columns' ids) | | `report.txt` | counts, combinational loops, ops without a model, the histogram of end points by levels, the `top-paths` deepest paths op by op, and the deepest path of each `trace=` match | -| `screen_summary.csv` | the DUT flow's one row: max levels, end points ≥ 12, ≥ 8, all | +| `beacon_summary.csv` | the DUT flow's one row: max levels, end points ≥ 12, ≥ 8, all | | `paths.jsonl` | `paths=`: one JSON line per pair of start and end registers whose deepest bit has at least `path-levels` levels: that bit's levels, the pair's bit count, start, end, and `ops`, the path from the end back to the start as `[levels, kind, fanout, instance id, name, source]` | -| `screen_units.json` | the deepest end point of every unit at ≥ 8 levels, and the 10 deepest paths | -| `screen_paths.txt` | the DUT flow's `paths.jsonl` at ≥ 8 levels, by unit (`ci/logic_screen.py paths` for any screen) | +| `beacon_units.json` | the deepest end point of every unit at ≥ 8 levels, and the 10 deepest paths | +| `beacon_paths.txt` | the DUT flow's `paths.jsonl` at ≥ 8 levels, by unit (`ci/beacon.py paths` for any run) | A path in `report.txt`, deepest first, one op per line: levels at the op, its kind, the fanout of its bit, and its instance path and name: @@ -304,35 +304,35 @@ On the full GPU, the 3,283 pairs at ≥ 12 levels fold into 169 paths in ### 6.1 The DUT flow `hw/syn/xilinx/dut` builds every DUT of `catalog.mk` for Vivado; its -`screen` target screens the same DUT with the same sources, defines and +`beacon` target runs Beacon on the same DUT with the same sources, defines and `DEVICE` (the U55C by default): -1. `gen_sources.sh` into `screen/sources.f`, a filelist of `hw/rtl` with +1. `gen_sources.sh` into `beacon/sources.f`, a filelist of `hw/rtl` with the defines (not preprocessed copies, so source lines are the repository's); -2. `circt_elaborate.sh -f mlir` into `screen/netlist.mlir`, the FP IPs as +2. `circt_elaborate.sh -f mlir` into `beacon/netlist.mlir`, the FP IPs as black boxes, with every op's source location; 3. `foldsim-screen-map`, then `foldsim-estimate-timing` with `family=$(gen_platform.py --device=$(DEVICE) --emit=family)`; -4. `screen_summary.py`: `screen_summary.csv`, `screen_units.json` and - `screen_paths.txt` (§5). +4. `beacon_summary.py`: `beacon_summary.csv`, `beacon_units.json` and + `beacon_paths.txt` (§5). The ASIC flow has the same target, with the same sources, defines and SRAM black boxes as its synthesis, and `family=$(PDK)`: ```bash -make -C hw/syn/yosys/dut om TARGET=screen -# om_VX_om_core_top/screen/report.txt: the deepest paths and the period +make -C hw/syn/yosys/dut om TARGET=beacon +# om_VX_om_core_top/beacon/report.txt: the deepest paths and the period ``` ### 6.2 `logic_gate` The third synthesis gate, beside `fpga_gate` (Vivado, nightly) and `asic_gate` (Yosys + OpenSTA, nightly); see -[continuous_integration.md §3.5](continuous_integration.md). It screens +[continuous_integration.md §3.5](continuous_integration.md). It runs Beacon on `fpga_gate`'s builds (`ci/testcases/logic_gate.yaml`, all but `top`, whose `VX_cp_core` keeps LLHD reference ports) in about four minutes, and checks -each against `ci/baselines/synthesis/screen/`: +each against `ci/baselines/synthesis/beacon/`: - a **unit** that gets deeper fails, by name; - a **new unit** at or above the fix threshold (12) fails; @@ -340,7 +340,7 @@ each against `ci/baselines/synthesis/screen/`: to lock in by re-recording; - end point counts are recorded, not gated (they grow with lanes and warps). -A failing build names its `screen_paths.txt`, where the failing unit is a +A failing build names its `beacon_paths.txt`, where the failing unit is a heading with its paths op by op; CI uploads it with the build's logs. Depth is deterministic, so the tolerance is zero. The baseline fingerprint @@ -350,22 +350,22 @@ by a human (`--update-baseline`), as for the other gates. ### 6.3 RTL timing work -Screen first, then Vivado: +Beacon first, then Vivado: ```bash -ci/logic_screen.py screen gen/design.mlir runs/full -ci/logic_screen.py summarize runs/full --min-levels 12 # the fix list, by unit +ci/beacon.py run gen/design.mlir runs/full +ci/beacon.py summarize runs/full --min-levels 12 # the fix list, by unit ``` -The screen ranks the units; one Vivado run confirms the fix. Compare a fix +Beacon ranks the units; one Vivado run confirms the fix. Compare a fix against a native gate (`hw/syn/xilinx/dut` or the OOC gates), not the folded design (§9). ## 7. Validation -Against Vivado, per RTL register or memory: the screen's deepest bit +Against Vivado, per RTL register or memory: Beacon's deepest bit against Vivado's deepest failing path and worst slack, on seven V80 datasets -with no new Vivado runs (`ci/logic_screen.py validate`): +with no new Vivado runs (`ci/beacon.py validate`): | Dataset | Vivado run | Spearman vs levels | Spearman vs slack | ≥ 12: precision / recall | |---|---|---|---|---| @@ -377,7 +377,7 @@ with no new Vivado runs (`ci/logic_screen.py validate`): | core | native gate, 3.333 ns, placed | 0.93 | 0.62 | 0.81 / 0.13 | | compute slice | native gate, 3.333 ns, placed | 0.87 | 0.75 | 0.58 / 0.57 | -The screen's depths run about 1.25× Vivado's on the native gates. The U55C +Beacon's depths run about 1.25× Vivado's on the native gates. The U55C profile was checked against `fpga_gate`'s critical paths: it agrees with the V80 profile wherever no DSP is on the path, and the remaining disagreement is the same on both devices. @@ -386,7 +386,7 @@ disagreement is the same on both devices. `hw/syn/yosys` builds them, flip-flops named after their registers, `hw/syn/yosys/sta_endpoints.tcl` for the slack of each): -| DUT | Registers | Spearman, stages vs arrival | Worst path: OpenSTA → screen | +| DUT | Registers | Spearman, stages vs arrival | Worst path: OpenSTA → Beacon | |---|---|---|---| | om | 93 | 0.91 | 827 → 879 ps | | cache | 526 | 0.82 | 1450 → 1311 ps | @@ -398,21 +398,21 @@ Per register, the fitted delay per stage is 17–25 ps, the library's 24. The worst path is within a mean 8 % on these five, from which the path term was fitted; on the six larger DUTs, the predicted Fmax is within a mean 13 % (−32 % to +3 %) of the asic_gate baselines, some of which predate the -current RTL. A DUT screens in 2–52 s, where Yosys and OpenSTA take +current RTL. Beacon runs a DUT in 2–52 s, where Yosys and OpenSTA take 2.5–37 min. ## 8. Diagnostics beyond levels -**Port depths.** `port-modules=` names modules whose instances' ports the -screen reports: the levels at which each input arrives and the levels behind +**Port depths.** `port-modules=` names modules whose instances' ports +Beacon reports: the levels at which each input arrives and the levels behind each output. Small modules are inlined by the mapping, so keep them with -`screen --keep`: +`run --keep`: ```bash -ci/logic_screen.py screen --keep VX_elastic_buffer design.mlir runs/eb \ +ci/beacon.py run --keep VX_elastic_buffer design.mlir runs/eb \ port-modules=VX_elastic_buffer -ci/logic_screen.py buffers runs/eb # the OUT_BUF advisor -ci/logic_screen.py ports runs/eb # every kept module, by unit +ci/beacon.py buffers runs/eb # the OUT_BUF advisor +ci/beacon.py ports runs/eb # every kept module, by unit ``` - The **buffer advisor** lists the `VX_elastic_buffer`s that pass a deep @@ -429,8 +429,8 @@ ci/logic_screen.py ports runs/eb # every kept module, by unit **Vivado probes.** -`hw/scripts` has three Vivado Tcl probes for questions the -screen cannot answer, run on a checkpoint +`hw/scripts` has three Vivado Tcl probes for questions +Beacon cannot answer, run on a checkpoint (`vivado -mode batch -source -tclargs ...`): | Probe | Answers | @@ -444,12 +444,12 @@ screen cannot answer, run on a checkpoint - **Depth, not slack.** On the placed 3.333 ns socket gate, wires are 83 % of the worst paths' delay; levels explain 1 % of net delay, and the real placement two thirds. No pre-placement wire term ranked slack better than - levels on every held-out gate, so the screen has none. Route-dominated + levels on every held-out gate, so Beacon has none. Route-dominated failures with few levels are invisible to it: recall at ≥ 12 levels is 0.13–0.57. - **Retiming and packing.** Vivado retimes and packs across what the RTL separates: a popcount, its adders and a compare become a LUT6 tree with a - retimed register inside. The screen's depths are RTL stage depths, about + retimed register inside. Beacon's depths are RTL stage depths, about 1.25× Vivado's. - **The fold.** In foldsim's folded designs, the fold turns some registers into LUTRAM and cuts some chains, so its paths differ from the RTL's. The @@ -457,7 +457,7 @@ screen cannot answer, run on a checkpoint - **Black boxes.** Vendor IPs and `black-boxes=` modules end and start paths at their pins: their insides are not timed. - **Devices.** The V80 and the U55C (Vivado), and ASAP7 RVT TT (the Yosys - flow). The ASAP7 numbers are the screen's period estimate, a sizing + flow). The ASAP7 numbers are Beacon's period estimate, a sizing guide, not a sign-off: they come within about 10–30 % of OpenSTA. - **Toolchain.** The catalog maps only with three CIRCT Synth fixes (zero-width mux, constant SOP cut, a failing longest-path query), which diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index 10428f8ebc..d988c0520e 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -283,12 +283,12 @@ catches one that makes a unit **deeper**, in seconds and without synthesis. | | `fpga_gate` | `asic_gate` | `logic_gate` | |---|---|---|---| | flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | CIRCT → `foldsim-estimate-timing` (logic levels as Vivado counts them on `DEVICE`'s family) | -| design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | `hw/syn/xilinx/dut`, target `screen` | +| design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | `hw/syn/xilinx/dut`, target `beacon` | | gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | `max_levels` against the baseline, and the deepest end point of every RTL unit | | also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | end point bits at ≥ 12 and ≥ 8 levels, the deepest paths | | comparable when | same device, optimization level, xlen | same PDK, VT, corner, xlen | same CIRCT, foldsim, xlen | | runner | self-hosted, licensed Vivado | hosted, one job per design | hosted, one job for the catalog | -| default tolerance | Fmax 2 %, LUT 5 % | Fmax 2 %, cell area 5 % | none: the screen is deterministic | +| default tolerance | Fmax 2 %, LUT 5 % | Fmax 2 %, cell area 5 % | none: Beacon is deterministic | None is a pytest cell. A cell is a build tree plus a driver; a gate build is neither, and the synthesis gates are hours long. They are standalone scripts @@ -313,12 +313,12 @@ entry points that pin the tool. | file | written by | holds | |---|---|---| | `ci/testcases/{fpga,asic,timing}_gate.yaml` | a person | per build: a design, a target clock, an area budget (synthesis gates), `CONFIGS`, a group, optional `thresholds` and `known_issue` | -| `ci/baselines/synthesis/{xilinx,yosys,screen}/.json` | the gate | measured metrics, the config hash, the tool versions and environment | +| `ci/baselines/synthesis/{xilinx,yosys,beacon}/.json` | the gate | measured metrics, the config hash, the tool versions and environment | Groups are `core`, `tensor`, `graphics`, `dxa` and `vm`. `asic_gate` uses the same group names and gates a subset of the builds (§4.5), so where the two gates share a design, a divergence between the tools on the same module is a -finding, not noise. `logic_gate` screens exactly `fpga_gate`'s builds, so a +finding, not noise. `logic_gate` measures exactly `fpga_gate`'s builds, so a unit it flags can be read against the Fmax gate's critical paths. The config hash ties the two files together: edit the spec, and the gate @@ -367,14 +367,14 @@ so the flow reports the signed worst slack instead. #### What `logic_gate` asserts -The screen ([logic_screen.md](logic_screen.md)) counts logic levels, not +Beacon ([beacon.md](beacon.md)) counts logic levels, not slack: it does not see wires, so it gates depth, which it predicts (Spearman 0.50–0.93 against Vivado per register on the V80 gates), and leaves Fmax to `fpga_gate`. It counts them for the family of the DUT flow's `DEVICE` (the U55C by default), which `gen_platform.py --emit=family` reads from `hw/syn/xilinx/platforms`; the -device is part of the baseline fingerprint, as it is for `fpga_gate`. The -screen has profiles for the V80 and the U55C only, and fails on any other +device is part of the baseline fingerprint, as it is for `fpga_gate`. +Beacon has profiles for the V80 and the U55C only, and fails on any other `DEVICE`. A **unit** is an end point's instance path with generate indices folded (`g_banks_3_bank` → `g_banks_*_bank`). @@ -387,8 +387,8 @@ screen has profiles for the V80 and the U55C only, and fails on any other There is no tolerance: the same RTL and tools give the same levels. A new CIRCT or foldsim can map the same RTL differently, so both versions are in the config -hash and a tool change reads `STALE`. `make -C hw/syn/xilinx/dut -screen` -runs one screen by hand; its `screen_paths.txt` lists every deep path op by op, +hash and a tool change reads `STALE`. `make -C hw/syn/xilinx/dut -beacon` +runs Beacon on one DUT by hand; its `beacon_paths.txt` lists every deep path op by op, by unit, with source lines, and a failing build names it. #### Running a multi-hour sweep @@ -541,13 +541,13 @@ adding it: an over-budget design costs a silent job kill, not a red gate. ### 4.6 `logic_gate.yml` On every pull request and push to master that touches the RTL, the DUT -wrappers, the screen's scripts, the gate or its baselines; and by manual -dispatch. One hosted job screens the whole catalog: it restores the toolchain, +wrappers, Beacon's scripts, the gate or its baselines; and by manual +dispatch. One hosted job runs Beacon on the whole catalog: it restores the toolchain, installs foldsim (`ci/toolchain_install.sh --foldsim`), runs the catalog's `run:` command with `--report`, and renders the report with [`synth_report.py`](../../ci/synth_report.py) into the workflow summary. The -artifacts are the report and, per design, the build log, the summary and the -screen's deepest paths. +artifacts are the report and, per design, the build log, the summary and +Beacon's deepest paths. --- diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 09ee21d3da..39a4805d9c 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -26,9 +26,9 @@ on it, and runs orders of magnitude faster than Verilator. The whole flow is one IR from SystemVerilog to Verilog: `circt-verilog` imports Vortex, Foldsim's passes cut it into models and compose the simulator, `firtool` writes it out, and the card's native flow (IP-XACT -packaging, `slashkit link`) builds the image. (`foldsim-opt` also carries the -logic screen, which counts the logic levels of a Vortex netlist before -Vivado: [logic_screen.md](logic_screen.md).) Foldsim is a project of its own +packaging, `slashkit link`) builds the image. (`foldsim-opt` also carries +Beacon, which counts the logic levels of a Vortex netlist before Vivado: +[beacon.md](beacon.md).) Foldsim is a project of its own that builds on CIRCT, as POCL builds on LLVM: the `foldsim` driver and `foldsim-opt` come from the toolchain (`$(TOOLDIR)/foldsim`), CIRCT from `$(TOOLDIR)/circt`. Nothing else is in the flow: no Yosys, no sv2v. diff --git a/hw/scripts/screen_summary.py b/hw/scripts/beacon_summary.py similarity index 80% rename from hw/scripts/screen_summary.py rename to hw/scripts/beacon_summary.py index ba16ec4d74..d92fa92e30 100644 --- a/hw/scripts/screen_summary.py +++ b/hw/scripts/beacon_summary.py @@ -1,21 +1,21 @@ #!/usr/bin/env python3 -"""Summarize a logic screen (foldsim-estimate-timing) for logic_gate. +"""Summarize a Beacon run (foldsim-estimate-timing) for logic_gate. -The screen writes one row per end point bit: the logic levels of its longest +Beacon writes one row per end point bit: the logic levels of its longest path, its start and its end. ci/logic_gate.py gates depth per RTL unit, so this collapses the table into what it compares: - screen_summary.csv one row: the deepest end point, the end point bits at or + beacon_summary.csv one row: the deepest end point, the end point bits at or above the fix and watch thresholds, and all end points - screen_units.json the deepest end point of every unit at or above the + beacon_units.json the deepest end point of every unit at or above the watch threshold, and the deepest paths (diagnostics) - screen_paths.txt if the screen wrote paths.jsonl: every path at or above + beacon_paths.txt if Beacon wrote paths.jsonl: every path at or above the watch threshold, op by op, by unit (write_paths) A unit is an end point's instance path with generate indices folded (g_banks_3_bank -> g_banks_*_bank), so one deep bank reads as one unit. - usage: screen_summary.py --top [--output-dir ] + usage: beacon_summary.py --top [--output-dir ] """ import argparse @@ -27,7 +27,7 @@ # 300 MHz on the V80: failing end points have a median of 7-8 levels, and # about 55-60% of the registers at 12 or more fail -# (docs/designs/logic_screen.md §7). +# (docs/designs/beacon.md §7). FIX_LEVELS = 12 WATCH_LEVELS = 8 NUM_PATHS = 10 @@ -43,7 +43,7 @@ def norm(name): def instance_paths(path): - """Instance id -> path from the top, from the screen's instances.csv.""" + """Instance id -> path from the top, from a run's instances.csv.""" rows = {} with open(path) as fh: for row in csv.DictReader(fh): @@ -69,16 +69,16 @@ def unit(path): def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - ap.add_argument("screen") + ap.add_argument("run_dir") ap.add_argument("--top", required=True) ap.add_argument("--output-dir", default=".") args = ap.parse_args() - paths = instance_paths(os.path.join(args.screen, "instances.csv")) + paths = instance_paths(os.path.join(args.run_dir, "instances.csv")) units = collections.defaultdict(int) counts = collections.Counter() deepest = [] - with open(os.path.join(args.screen, "endpoints.csv")) as fh: + with open(os.path.join(args.run_dir, "endpoints.csv")) as fh: for row in csv.DictReader(fh): levels = int(row["levels"]) counts["endpoints"] += 1 @@ -105,28 +105,28 @@ def main(): "endpoints": counts["endpoints"]} if not units: # Nothing reaches the watch threshold; the deepest is still recorded. - with open(os.path.join(args.screen, "endpoints.csv")) as fh: + with open(os.path.join(args.run_dir, "endpoints.csv")) as fh: summary["max_levels"] = max( (int(r["levels"]) for r in csv.DictReader(fh)), default=0) - with open(os.path.join(args.output_dir, "screen_summary.csv"), "w", + with open(os.path.join(args.output_dir, "beacon_summary.csv"), "w", newline="") as fh: w = csv.DictWriter(fh, fieldnames=COLUMNS) w.writeheader() w.writerow(summary) - with open(os.path.join(args.output_dir, "screen_units.json"), "w") as fh: + with open(os.path.join(args.output_dir, "beacon_units.json"), "w") as fh: json.dump({"fix_levels": FIX_LEVELS, "watch_levels": WATCH_LEVELS, "units": dict(sorted(units.items())), "deepest_paths": ["%d %s -> %s" % d for d in deepest]}, fh, indent=2) fh.write("\n") - print("screen: %s max %d levels, %d end point bits >= %d, %d units >= %d" + print("beacon: %s max %d levels, %d end point bits >= %d, %d units >= %d" % (args.top, summary["max_levels"], summary["deep_endpoints"], FIX_LEVELS, len(units), WATCH_LEVELS)) - if os.path.exists(os.path.join(args.screen, "paths.jsonl")): - with open(os.path.join(args.output_dir, "screen_paths.txt"), "w") as fh: - n, u = write_paths(fh, args.screen, WATCH_LEVELS, paths=paths) - print("screen: %d paths >= %d levels in %d units -> screen_paths.txt" + if os.path.exists(os.path.join(args.run_dir, "paths.jsonl")): + with open(os.path.join(args.output_dir, "beacon_paths.txt"), "w") as fh: + n, u = write_paths(fh, args.run_dir, WATCH_LEVELS, paths=paths) + print("beacon: %d paths >= %d levels in %d units -> beacon_paths.txt" % (n, WATCH_LEVELS, u)) @@ -134,21 +134,21 @@ def fold(name): return INDEX.sub("_*", norm(name)) -def write_paths(out, screen, min_levels, unit_filter=None, paths=None): - """Write the screen's deep paths (paths.jsonl) by unit, op by op. +def write_paths(out, run_dir, min_levels, unit_filter=None, paths=None): + """Write a run's deep paths (paths.jsonl) by unit, op by op. - The screen lists one path per pair of start and end registers; this folds + Beacon lists one path per pair of start and end registers; this folds the pairs whose names differ only in generate indices and bits, as units fold them, keeping the deepest. Returns (paths, units) written. """ if paths is None: - paths = instance_paths(os.path.join(screen, "instances.csv")) + paths = instance_paths(os.path.join(run_dir, "instances.csv")) def full(ctx, name): return "%s/%s" % (paths[ctx], name) if paths[ctx] else name units = collections.defaultdict(dict) - with open(os.path.join(screen, "paths.jsonl")) as fh: + with open(os.path.join(run_dir, "paths.jsonl")) as fh: for line in fh: p = json.loads(line) if p["levels"] < min_levels: diff --git a/hw/scripts/circt_elaborate.sh b/hw/scripts/circt_elaborate.sh index dcae42f694..387cf9af86 100755 --- a/hw/scripts/circt_elaborate.sh +++ b/hw/scripts/circt_elaborate.sh @@ -35,7 +35,7 @@ # v2005 writes Verilog-2005 for tools with a Verilog frontend (yosys): no # packed arrays or structs, no local variables, no size casts; -f mlir # writes the HW/Comb/Seq netlist itself, for tools built on CIRCT (the -# logic screen), and exports nothing. +# Beacon), and exports nothing. # # The sources are a filelist (-l, as gen_sources.sh writes it: +define+, # +incdir+ and files in compile order), or include directories whose sources diff --git a/hw/scripts/xilinx_cone_probe.tcl b/hw/scripts/xilinx_cone_probe.tcl index 1a9e6c7c15..861f6dc210 100644 --- a/hw/scripts/xilinx_cone_probe.tcl +++ b/hw/scripts/xilinx_cone_probe.tcl @@ -14,7 +14,7 @@ # xilinx_cone_probe.tcl ...: for each from|to pair of cell # patterns ("-" for any), up to 500 setup paths between them; prints the # level histogram, the endpoints of the deepest and the deepest path cell by -# cell. Answers whether a cone the screen reports exists in Vivado's netlist, +# cell. Answers whether a cone Beacon reports exists in Vivado's netlist, # and how deep Vivado built it. set dcp [lindex $argv 0] set out [lindex $argv 1] diff --git a/hw/syn/xilinx/dut/Makefile b/hw/syn/xilinx/dut/Makefile index ba42d37ff4..54188f0227 100644 --- a/hw/syn/xilinx/dut/Makefile +++ b/hw/syn/xilinx/dut/Makefile @@ -23,11 +23,11 @@ $(DUTS): cp build.mk $(PREFIX)_$@/Makefile $(MAKE) -C $(PREFIX)_$@ DUT=$@ clean && $(MAKE) -C $(PREFIX)_$@ DUT=$@ > $(PREFIX)_$@/build.log 2>&1 & -# The logic screen of one DUT, in the foreground: make tcu-screen -%-screen: +# Beacon on one DUT, in the foreground: make tcu-beacon +%-beacon: mkdir -p $(PREFIX)_$* cp build.mk $(PREFIX)_$*/Makefile - $(MAKE) -C $(PREFIX)_$* DUT=$* clean screen + $(MAKE) -C $(PREFIX)_$* DUT=$* clean beacon # Re-run power analysis on an existing build without rebuilding the design. # Usage: make -power SAIF_FILE= [SAIF_INST=] diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index 723c1495a9..76e30624ab 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -84,30 +84,31 @@ power: TOOL_DIR=$(SCRIPT_DIR) SAIF_FILE=$(SAIF_FILE) SAIF_INST=$(SAIF_INST) \ $(VIVADO) -mode batch -source $(SCRIPT_DIR)/xilinx_power_analysis.tcl -# The logic screen (foldsim-estimate-timing): the logic levels of every +# Beacon (foldsim-estimate-timing): the logic levels of every # register-to-register path of the design, counted as Vivado counts them on # DEVICE's family (hw/syn/xilinx/platforms), by RTL unit, in seconds and # without Vivado. It elaborates the same sources and # defines as gen-sources into a CIRCT netlist of its own, with the Xilinx # floating-point cores as black boxes. ci/logic_gate.py gates -# screen_summary.csv and screen_units.json; screen_paths.txt has every path -# of at least SCREEN_PATH_LEVELS (screen_summary.py's watch threshold), op by +# beacon_summary.csv and beacon_units.json; beacon_paths.txt has every path +# of at least BEACON_PATH_LEVELS (beacon_summary.py's watch threshold), op by # op, by unit. -SCREEN_PATH_LEVELS ?= 8 -screen: - mkdir -p screen - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -Oscreen/sources.f $(RTL_PKGS) - $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t $(TOP_LEVEL_ENTITY) -l screen/sources.f \ - -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r screen/memories.rpt -o screen/netlist.mlir - $(FOLDSIM_OPT) screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-screen-map)' \ - --emit-bytecode -o screen/mapped.mlirbc +BEACON_PATH_LEVELS ?= 8 +BEACON_MAP_PASS := foldsim-screen-map +beacon: + mkdir -p beacon + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -Obeacon/sources.f $(RTL_PKGS) + $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t $(TOP_LEVEL_ENTITY) -l beacon/sources.f \ + -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r beacon/memories.rpt -o beacon/netlist.mlir + $(FOLDSIM_OPT) beacon/netlist.mlir --pass-pipeline='builtin.module($(BEACON_MAP_PASS))' \ + --emit-bytecode -o beacon/mapped.mlirbc family=$$(python3 $(SCRIPT_DIR)/gen_platform.py --device=$(DEVICE) --emit=family) && \ - $(FOLDSIM_OPT) screen/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$$family output=screen/endpoints.csv instances=screen/instances.csv report=screen/report.txt paths=screen/paths.jsonl path-levels=$(SCREEN_PATH_LEVELS)})" - python3 $(SCRIPT_DIR)/screen_summary.py screen --top $(TOP_LEVEL_ENTITY) + $(FOLDSIM_OPT) beacon/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$$family output=beacon/endpoints.csv instances=beacon/instances.csv report=beacon/report.txt paths=beacon/paths.jsonl path-levels=$(BEACON_PATH_LEVELS)})" + python3 $(SCRIPT_DIR)/beacon_summary.py beacon --top $(TOP_LEVEL_ENTITY) clean: ifndef RESUME - rm -rf project_1 screen screen_summary.csv screen_units.json screen_paths.txt + rm -rf project_1 beacon beacon_summary.csv beacon_units.json beacon_paths.txt rm -rf .Xil rm -f *.rpt rm -f *.log @@ -117,4 +118,4 @@ else @echo "RESUME is defined, skipping clean." endif -.PHONY: all gen-sources build screen clean power +.PHONY: all gen-sources build beacon clean power diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index fa781974ea..3ae8c82c63 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -189,7 +189,7 @@ CFLAGS += $(RTL_INCLUDE) # Build targets all: timing -.PHONY: all synthesis techmap timing screen gate-sim gate-saif normalize-saif asap7 asap7-models clean +.PHONY: all synthesis techmap timing beacon gate-sim gate-saif normalize-saif asap7 asap7-models clean asap7: $(ASAP7_INSTALL) --install-dir=$(ASAP7_DIR) --vt=$(ASAP7_VT) @@ -285,19 +285,19 @@ timing: $(SYN_SRC_DEPS) $(SYN_LIB_DEPS) | $(BUILD_DIR) @$(SRC_DIR)/synth_summary.py --reports $(BUILD_DIR)/reports \ --top $(TOP_LEVEL_ENTITY) --clock-mhz $(CLOCK_FREQ) -# The logic screen (foldsim-estimate-timing, family=$(PDK)): the gate stages -# of every register-to-register path, with the same sources and SRAM black -# boxes as synthesis, in seconds and without Yosys; report.txt converts the -# deepest path to a period with the library's delays. -# docs/designs/logic_screen.md. -screen: $(BUILD_DIR)/src - mkdir -p $(BUILD_DIR)/screen +# Beacon (foldsim-estimate-timing, family=$(PDK)): the gate stages of every +# register-to-register path, with the same sources and SRAM black boxes as +# synthesis, in seconds and without Yosys; report.txt converts the deepest +# path to a period with the library's delays. docs/designs/beacon.md. +BEACON_MAP_PASS := foldsim-screen-map-asic +beacon: $(BUILD_DIR)/src + mkdir -p $(BUILD_DIR)/beacon $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -b'$(BB_MODULES)' \ - -o$(BUILD_DIR)/screen/netlist.mlir - $(FOLDSIM_OPT) $(BUILD_DIR)/screen/netlist.mlir --pass-pipeline='builtin.module(foldsim-screen-map-asic)' \ - --emit-bytecode -o $(BUILD_DIR)/screen/mapped.mlirbc - $(FOLDSIM_OPT) $(BUILD_DIR)/screen/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$(PDK) output=$(BUILD_DIR)/screen/endpoints.csv instances=$(BUILD_DIR)/screen/instances.csv report=$(BUILD_DIR)/screen/report.txt})' - @grep "^deepest path" $(BUILD_DIR)/screen/report.txt + -o$(BUILD_DIR)/beacon/netlist.mlir + $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/netlist.mlir --pass-pipeline='builtin.module($(BEACON_MAP_PASS))' \ + --emit-bytecode -o $(BUILD_DIR)/beacon/mapped.mlirbc + $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$(PDK) output=$(BUILD_DIR)/beacon/endpoints.csv instances=$(BUILD_DIR)/beacon/instances.csv report=$(BUILD_DIR)/beacon/report.txt})' + @grep "^deepest path" $(BUILD_DIR)/beacon/report.txt gate-sim: techmap asap7-models @if [ "$(PDK)" != "asap7" ]; then echo "ERROR: gate-sim requires PDK=asap7 functional models" >&2; exit 1; fi diff --git a/hw/syn/yosys/liberty_stage.py b/hw/syn/yosys/liberty_stage.py index d275695047..d7646515a5 100755 --- a/hw/syn/yosys/liberty_stage.py +++ b/hw/syn/yosys/liberty_stage.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The delays the logic screen's ASIC profile (foldsim-estimate-timing -family=asap7) converts stages with, read from a Liberty library. +"""The delays Beacon's ASIC profile (foldsim-estimate-timing family=asap7) +converts stages with, read from a Liberty library. usage: liberty_stage.py [--fanout N] [--suffix _ASAP7_75t_R] diff --git a/hw/syn/yosys/sta_endpoints.tcl b/hw/syn/yosys/sta_endpoints.tcl index 870da049e6..7a29f67a60 100644 --- a/hw/syn/yosys/sta_endpoints.tcl +++ b/hw/syn/yosys/sta_endpoints.tcl @@ -12,8 +12,8 @@ # limitations under the License. # sta_endpoints.tcl: OpenSTA, the worst setup slack at the data input of -# every flip-flop, for comparing the logic screen per register with an -# ASIC flow (a CSV in the end point table format of ci/logic_screen.py). +# every flip-flop, for comparing Beacon per register with an ASIC flow (a +# CSV in the end point table format of ci/beacon.py). # env: LIB NETLIST TOP SDC OUT proc env {name} { return $::env($name) } read_liberty [env LIB] From 81630e222efb325c8e984037191b80a093d036e6 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 02:19:13 -0700 Subject: [PATCH 277/336] ci/beacon.py: import json for validate --json validate --json writes the agreement summary with json.dump, but the module never imported json, so the option raised NameError. Co-Authored-By: Claude Opus 5.5 --- ci/beacon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/beacon.py b/ci/beacon.py index 0cde0c9723..1ec6bbb360 100755 --- a/ci/beacon.py +++ b/ci/beacon.py @@ -45,6 +45,7 @@ import argparse import collections import csv +import json import os import re import resource From 8224b97f773c1a63610e425f3d6c2db313f68cae Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 02:26:07 -0700 Subject: [PATCH 278/336] check_ip_contracts.py: run on Python 3.10 Its summary line reused the f-string's quote inside a replacement field, which only Python 3.12 accepts. On 3.10 (Ubuntu 22.04's python3) the script fails to parse, so every flow that checks the Xilinx IP contracts stops, the Foldsim runtime build among them. Co-Authored-By: Claude Opus 5.5 --- hw/scripts/check_ip_contracts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hw/scripts/check_ip_contracts.py b/hw/scripts/check_ip_contracts.py index 9218021c50..c20ff7f8c6 100755 --- a/hw/scripts/check_ip_contracts.py +++ b/hw/scripts/check_ip_contracts.py @@ -128,7 +128,8 @@ def main(): print(f'ERROR: {e}', file=sys.stderr) if errors: sys.exit(1) - print(f'xilinx IP contracts: {", ".join(f"{n} {contracts[n]["fixed_latency"]}" for n in RTL_ASSERTS)} consistent') + latencies = ", ".join(f"{n} {contracts[n]['fixed_latency']}" for n in RTL_ASSERTS) + print(f'xilinx IP contracts: {latencies} consistent') if __name__ == '__main__': From 5dffc5b0a0595a86e3bc6da3effbc1e47991ec9a Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 02:58:37 -0700 Subject: [PATCH 279/336] beacon: load the target Beacon maps to and times on Beacon's two foldsim passes now take one target, named after the FPGA family (hw/syn/xilinx/platforms) or the ASIC PDK (hw/syn/yosys): foldsim-beacon-map{target=} maps to that target's cells and foldsim-estimate-timing{target=} counts levels on it. Each flow passes the name it already has: the DUT flow the family of DEVICE, the Yosys flow PDK, ci/beacon.py run its target= (versal by default). The flows no longer pick the LUT6 or the 3-input mapper by hand, which sent any PDK but asap7 to the FPGA mapper, and U200/U250/Zynq devices, which had no profile, now have targets. Requires foldsim with foldsim-beacon-map (vortexgpgpu/foldsim 9237556); the foldsim prebuilt must be refreshed before this is pushed. On the U55C core netlist, the new passes give byte-identical end points, instances and paths to foldsim-screen-map with family=. Co-Authored-By: Claude Opus 5.5 --- ci/beacon.py | 14 ++++----- ci/synth_gate.py | 2 +- docs/assets/img/beacon_compiler.svg | 4 +-- docs/assets/img/beacon_system.svg | 4 +-- docs/designs/beacon.md | 48 +++++++++++++++++------------ hw/syn/xilinx/dut/common.mk | 8 ++--- hw/syn/yosys/Makefile | 7 ++--- hw/syn/yosys/liberty_stage.py | 2 +- 8 files changed, 49 insertions(+), 40 deletions(-) diff --git a/ci/beacon.py b/ci/beacon.py index 1ec6bbb360..110e2e923a 100755 --- a/ci/beacon.py +++ b/ci/beacon.py @@ -62,9 +62,6 @@ 'foldsim_target': 'foldsim_target_model_copy', } -MAP_PASS = 'foldsim-screen-map' -MAP_PASS_ASIC = 'foldsim-screen-map-asic' - VIVADO_CELL = re.compile( r'_reg((\[\d+\])*(_replica(_\d+)?)?(_[fb]ret(_\d+)?)*(_rep(_\d+)?)?' r'|_\d+_\d+_\d+_\d+|_bram_\d+)$') @@ -182,7 +179,9 @@ def cmd_run(args): """Map (unless the input is already mapped bytecode), then count levels.""" opt = os.environ.get("FOLDSIM_OPT") or "foldsim-opt" os.makedirs(args.out, exist_ok=True) - asic = any(o == "family=asap7" for o in args.options) + # Map to the cells of the target the levels are counted on. + target = next((o for o in args.options if o.startswith("target=")), + "target=versal") mapped = args.design steps = [] # MLIR maps modules in parallel: the full GPU maps in 53 s and 18 GB, or @@ -190,7 +189,7 @@ def cmd_run(args): threads = ["--mlir-disable-threading"] if args.no_threads else [] if not args.design.endswith(".mlirbc"): mapped = os.path.join(args.out, "mapped.mlirbc") - pipeline = MAP_PASS_ASIC if asic else MAP_PASS + pipeline = "foldsim-beacon-map{%s}" % target if args.keep: # Keep these modules' instances through inlining, so their ports # can be reported (ports=, port-modules=). @@ -564,8 +563,9 @@ def main(): p.add_argument("design", help="elaborated netlist (.mlir), or mapped bytecode (.mlirbc)") p.add_argument("out", help="output directory") p.add_argument("options", nargs="*", - help="foldsim-estimate-timing options (top=, family=, black-boxes=, " - "trace=, retime-levels=, ...); family=asap7 maps for the ASIC library") + help="foldsim-estimate-timing options (top=, target=, black-boxes=, " + "trace=, retime-levels=, ...); the design is mapped to the " + "cells of target= (versal by default)") p.add_argument("--no-threads", action="store_true", help="map on one thread: half the peak memory, twice the time") p.add_argument("--keep", action="append", default=[], metavar="MODULE", diff --git a/ci/synth_gate.py b/ci/synth_gate.py index 9e5dab60f2..9e7c457de0 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -498,7 +498,7 @@ class Beacon(Tool): # The netlist exists once foldsim-opt starts mapping it; a typo in a define # or an include dies in gen_sources.sh or circt_elaborate.sh before that. - elab_done = "foldsim-screen-map" + elab_done = "foldsim-beacon-map" elab_label = "elaboration" phases = ( ("gen_sources.sh", "sources"), diff --git a/docs/assets/img/beacon_compiler.svg b/docs/assets/img/beacon_compiler.svg index ea03690c04..b9b8c0a057 100644 --- a/docs/assets/img/beacon_compiler.svg +++ b/docs/assets/img/beacon_compiler.svg @@ -26,7 +26,7 @@ - foldsim-screen-map (per module) + foldsim-beacon-map (per module) 1. hw-flatten-modules: inline modules < 300 ops @@ -119,7 +119,7 @@ endpoints.csv, instances.csv, report.txt - Device profiles (family=): the FPGAs differ only in their DSPs + Targets (target=): the FPGAs differ in their DSPs versal DSP58 27×24 in 1 out 6 cascade 3 virtexuplusHBM DSP48E2 27×18 in 1 out 5 cascade 2 asap7 cell stages (3-input cuts), prefix adders, Dadda products diff --git a/docs/assets/img/beacon_system.svg b/docs/assets/img/beacon_system.svg index f9414e7868..ad70ab047a 100644 --- a/docs/assets/img/beacon_system.svg +++ b/docs/assets/img/beacon_system.svg @@ -68,14 +68,14 @@ foldsim-opt (foldsim, on CIRCT) - foldsim-screen-map + foldsim-beacon-map inline small modules, balance, lower to AIG, map to 6-input LUTs; adders, compares, multipliers and dividers stay word-level foldsim-estimate-timing per-bit arrival over the elaborated instances, - levels as Vivado counts them on family= + levels as Vivado counts them on target= versal (V80) | virtexuplusHBM (U55C) optional: retime-levels=, trace=, black-boxes= diff --git a/docs/designs/beacon.md b/docs/designs/beacon.md index 81bb6b9049..a91141b372 100644 --- a/docs/designs/beacon.md +++ b/docs/designs/beacon.md @@ -65,9 +65,9 @@ Or the two passes directly: ```bash foldsim-opt design.mlir --emit-bytecode -o mapped.mlirbc \ - --pass-pipeline='builtin.module(foldsim-screen-map)' + --pass-pipeline='builtin.module(foldsim-beacon-map{target=virtexuplusHBM})' foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( - foldsim-estimate-timing{top=VX_core_top family=virtexuplusHBM + foldsim-estimate-timing{top=VX_core_top target=virtexuplusHBM output=endpoints.csv instances=instances.csv report=report.txt top-paths=50 trace=lsu_unit})' ``` @@ -75,7 +75,7 @@ foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( | `foldsim-estimate-timing` option | meaning | |---|---| | `top=` | the top module (default: the one nothing instantiates) | -| `family=` | `versal` (V80, default), `virtexuplusHBM` (U55C), or `asap7` (ASIC, §4.3) | +| `target=` | the Beacon target (§4.2): `versal` (V80, default), `virtexuplus`, `virtexuplusHBM` (U55C), `zynquplus`, or `asap7` (ASIC, §4.3); map with `foldsim-beacon-map` for the same target | | `black-boxes=` | modules to treat as external, as a harness stubs them | | `output=`, `instances=`, `report=` | the files of §5 | | `min-levels=` | write only end points at or above this many levels | @@ -95,7 +95,7 @@ never builds it. The integration is in Vortex: | Piece | Where | Role | |---|---|---| -| `foldsim-screen-map`, `foldsim-estimate-timing` | foldsim `lib/Transforms/EstimateTiming.cpp` | map to LUTs; count levels | +| `foldsim-beacon-map`, `foldsim-estimate-timing` | foldsim `lib/Transforms/Beacon.cpp`, `EstimateTiming.cpp` | map to the target's cells; count levels | | `ci/beacon.py` | `ci/` | run Beacon on any netlist; read it by unit; compare with Vivado | | `xilinx_cone_dump.tcl`, `xilinx_cone_probe.tcl`, `xilinx_wire_probe.tcl` | `hw/scripts` | Vivado probes for what levels cannot show (§8) | | target `beacon` | `hw/syn/xilinx/dut/common.mk` | one DUT: sources → netlist → levels → summary | @@ -114,9 +114,15 @@ black boxes (`xilinx_ip_stubs.sv`). ![Beacon compiler](../assets/img/beacon_compiler.svg) -### 3.1 `foldsim-screen-map` +### 3.1 `foldsim-beacon-map` -A pipeline of CIRCT's synthesis passes plus three of foldsim's, per module: +Both passes load the same **target** from foldsim's target table +(`foldsim/Beacon.h`): its cells for the mapping, its DSPs and delays for the +level count. A target is named after its FPGA family or its PDK, so every +flow passes the name it already has: the DUT flow the family +`gen_platform.py --emit=family` gives for `DEVICE`, the ASIC flow `PDK`, +`ci/beacon.py run` its `target=`. The mapping is a pipeline of CIRCT's +synthesis passes plus three of foldsim's, per module: 1. **Inline small modules** (`hw-flatten-modules`, < 300 ops) so LUTs pack across buffers, arbiters and switches, as Vivado's flattened synthesis @@ -137,7 +143,8 @@ A pipeline of CIRCT's synthesis passes plus three of foldsim's, per module: reproduce. 5. **Depth-optimize** the graph: timing-aware variadic lowering, maximum AND cover, SOP balancing, structural hashing. -6. **Map to 6-input LUTs** (`synth-generic-lut-mapper`, timing strategy). +6. **Map to the target's cells** (`synth-generic-lut-mapper`, timing + strategy): 6-input LUTs on the FPGA families, 3-input cuts on ASAP7. The full GPU maps in about a minute and 18–20 GB. @@ -182,30 +189,32 @@ multiplexers per two more. ### 4.2 Devices -The V80 and the U55C both have LUT6s, 8-bit carry blocks (LOOKAHEAD8, -CARRY8) and 64-word distributed RAMs; they differ only in their DSPs. The -profile is selected by `family=`, with the names of +Versal and the UltraScale+ families all have LUT6s, 8-bit carry blocks +(LOOKAHEAD8, CARRY8) and 64-word distributed RAMs; they differ in their DSPs. +The target is selected by `target=`, with the family names of `hw/syn/xilinx/platforms`: -| `family=` | Device | DSP multiplier | fabric → registered product | product → fabric | per cascaded DSP | +| `target=` | Device | DSP multiplier | fabric → registered product | product → fabric | per cascaded DSP | |---|---|---|---|---|---| | `versal` | V80 | DSP58, 27 × 24 | 1 | 6 | 3 | -| `virtexuplusHBM` | U55C | DSP48E2, 27 × 18 | 1 | 5 | 2 | +| `virtexuplus` | U200, U250 | DSP48E2, 27 × 18 | 1 | 5 | 2 | +| `virtexuplusHBM` | U50, U55C, U280 | DSP48E2, 27 × 18 | 1 | 5 | 2 | +| `zynquplus` | Zynq UltraScale+ | DSP48E2, 27 × 18 | 1 | 5 | 2 | A product takes ⌈a/A⌉·⌈b/B⌉ DSPs (the better orientation). The cell counts are Vivado's: on Versal, M_DATA, ALUMUX, ALUADD and OUTPUT out of a DSP, and ALUMUX, ALUADD and OUTPUT through each cascaded one. A register fed by the product is the DSP's own (MREG or PREG), and so is a LUT that reads that register back, its clock enable or feedback; any other load pays the output -stage. The fabric-to-product value is fitted on the V80 data. Any other -family is an error. +stage. The fabric-to-product value is fitted on the V80 data. A name that +is not a target is an error that lists the targets. ### 4.3 ASAP7 For the ASIC flow (`hw/syn/yosys`, ASAP7 7.5-track RVT, TT corner) -Beacon counts **cell stages**, not LUT levels. `foldsim-screen-map-asic` +Beacon counts **cell stages**, not LUT levels. `target=asap7` maps the same balanced and-inverter graph to 3-input cuts, each one cell -(NAND, AOI, MAJ, XOR), and `family=asap7` models the word-level operators +(NAND, AOI, MAJ, XOR), and models the word-level operators as an ASIC builds them: | Element | Stages | @@ -312,13 +321,14 @@ On the full GPU, the 3,283 pairs at ≥ 12 levels fold into 169 paths in repository's); 2. `circt_elaborate.sh -f mlir` into `beacon/netlist.mlir`, the FP IPs as black boxes, with every op's source location; -3. `foldsim-screen-map`, then `foldsim-estimate-timing` with - `family=$(gen_platform.py --device=$(DEVICE) --emit=family)`; +3. `foldsim-beacon-map`, then `foldsim-estimate-timing`, both with + `target=` the family `gen_platform.py --device=$(DEVICE) --emit=family` + names; 4. `beacon_summary.py`: `beacon_summary.csv`, `beacon_units.json` and `beacon_paths.txt` (§5). The ASIC flow has the same target, with the same sources, defines and SRAM -black boxes as its synthesis, and `family=$(PDK)`: +black boxes as its synthesis, and `target=$(PDK)`: ```bash make -C hw/syn/yosys/dut om TARGET=beacon diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index 76e30624ab..fac58b65d0 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -94,16 +94,16 @@ power: # of at least BEACON_PATH_LEVELS (beacon_summary.py's watch threshold), op by # op, by unit. BEACON_PATH_LEVELS ?= 8 -BEACON_MAP_PASS := foldsim-screen-map +# The Beacon target: the family of the device, as hw/syn/xilinx/platforms names it. +BEACON_TARGET = $(shell python3 $(SCRIPT_DIR)/gen_platform.py --device=$(DEVICE) --emit=family) beacon: mkdir -p beacon $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -Obeacon/sources.f $(RTL_PKGS) $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t $(TOP_LEVEL_ENTITY) -l beacon/sources.f \ -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r beacon/memories.rpt -o beacon/netlist.mlir - $(FOLDSIM_OPT) beacon/netlist.mlir --pass-pipeline='builtin.module($(BEACON_MAP_PASS))' \ + $(FOLDSIM_OPT) beacon/netlist.mlir --pass-pipeline='builtin.module(foldsim-beacon-map{target=$(BEACON_TARGET)})' \ --emit-bytecode -o beacon/mapped.mlirbc - family=$$(python3 $(SCRIPT_DIR)/gen_platform.py --device=$(DEVICE) --emit=family) && \ - $(FOLDSIM_OPT) beacon/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$$family output=beacon/endpoints.csv instances=beacon/instances.csv report=beacon/report.txt paths=beacon/paths.jsonl path-levels=$(BEACON_PATH_LEVELS)})" + $(FOLDSIM_OPT) beacon/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) target=$(BEACON_TARGET) output=beacon/endpoints.csv instances=beacon/instances.csv report=beacon/report.txt paths=beacon/paths.jsonl path-levels=$(BEACON_PATH_LEVELS)})" python3 $(SCRIPT_DIR)/beacon_summary.py beacon --top $(TOP_LEVEL_ENTITY) clean: diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index 3ae8c82c63..b2c3351ac8 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -285,18 +285,17 @@ timing: $(SYN_SRC_DEPS) $(SYN_LIB_DEPS) | $(BUILD_DIR) @$(SRC_DIR)/synth_summary.py --reports $(BUILD_DIR)/reports \ --top $(TOP_LEVEL_ENTITY) --clock-mhz $(CLOCK_FREQ) -# Beacon (foldsim-estimate-timing, family=$(PDK)): the gate stages of every +# Beacon (foldsim-estimate-timing, target=$(PDK)): the gate stages of every # register-to-register path, with the same sources and SRAM black boxes as # synthesis, in seconds and without Yosys; report.txt converts the deepest # path to a period with the library's delays. docs/designs/beacon.md. -BEACON_MAP_PASS := foldsim-screen-map-asic beacon: $(BUILD_DIR)/src mkdir -p $(BUILD_DIR)/beacon $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -b'$(BB_MODULES)' \ -o$(BUILD_DIR)/beacon/netlist.mlir - $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/netlist.mlir --pass-pipeline='builtin.module($(BEACON_MAP_PASS))' \ + $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/netlist.mlir --pass-pipeline='builtin.module(foldsim-beacon-map{target=$(PDK)})' \ --emit-bytecode -o $(BUILD_DIR)/beacon/mapped.mlirbc - $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) family=$(PDK) output=$(BUILD_DIR)/beacon/endpoints.csv instances=$(BUILD_DIR)/beacon/instances.csv report=$(BUILD_DIR)/beacon/report.txt})' + $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) target=$(PDK) output=$(BUILD_DIR)/beacon/endpoints.csv instances=$(BUILD_DIR)/beacon/instances.csv report=$(BUILD_DIR)/beacon/report.txt})' @grep "^deepest path" $(BUILD_DIR)/beacon/report.txt gate-sim: techmap asap7-models diff --git a/hw/syn/yosys/liberty_stage.py b/hw/syn/yosys/liberty_stage.py index d7646515a5..b1ff614ae4 100755 --- a/hw/syn/yosys/liberty_stage.py +++ b/hw/syn/yosys/liberty_stage.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The delays Beacon's ASIC profile (foldsim-estimate-timing family=asap7) +"""The delays Beacon's ASIC profile (foldsim-estimate-timing target=asap7) converts stages with, read from a Liberty library. usage: liberty_stage.py [--fanout N] [--suffix _ASAP7_75t_R] From 88d32ff0211dfd99e9093d736339e4a32e129228 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 03:17:30 -0700 Subject: [PATCH 280/336] cache: drop the zero init of the dirty-byte mask RAMs The byte-enable store was zeroed at power-up (4f173e313) so that a never-installed way read as clean for the "clean victim has no dirty bytes" simulation assert. 1de9126e6 qualified that assert on a valid victim, and every consumer of the mask reads it only for a resident line or a valid victim; a line's mask is cleared on install and on flush. The mask of an invalid way is undefined by design, so the init has no reader: FPGA block and distributed RAMs power up at zero anyway, and an ASIC does not implement it. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_data.sv | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index 247a847dd1..71a96d1292 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -162,9 +162,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( .DATAW (LINE_SIZE), .SIZE (DATA_RAM_DEPTH), .OUT_REG (1), - .RDW_MODE ("R"), - .INIT_ENABLE (1), - .INIT_VALUE (0) + .RDW_MODE ("R") ) byteen_store ( .clk (clk), .reset (reset), @@ -201,9 +199,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( .SIZE (DATA_RAM_DEPTH), .OUT_REG (0), .LUTRAM (1), - .RDW_MODE ("W"), - .INIT_ENABLE (1), - .INIT_VALUE (0) + .RDW_MODE ("W") ) byteen_store ( .clk (clk), .reset (reset), From 9500630e06b22bf804c2a1c38790df047ee4538a Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 03:21:18 -0700 Subject: [PATCH 281/336] beacon validate: skip vendor-IP internals; match split, retimed and struct registers On the U55C core gate, 64,234 of 175,895 Vivado end point rows found no Beacon register, so the agreement covered two thirds of the design. 47,630 of them are registers inside the Xilinx floating-point cores, which Beacon treats as black boxes: they are now counted apart, every end point below an instance that drives a Beacon `ip` start point. The rest were Vivado cell names the leaf pattern missed: split block RAMs (ram_reg_), retimed and replicated registers (_bret__, _rep__) and struct-field registers (uop_data_reg[PC][3]). Unmatched rows drop to 724; matched registers go from 811 to 849, with Spearman against Vivado levels 0.751 (0.767 on the smaller set) and against slack 0.249 (0.257). Co-Authored-By: Claude Opus 5.5 --- ci/beacon.py | 48 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/ci/beacon.py b/ci/beacon.py index 110e2e923a..62bcc80799 100755 --- a/ci/beacon.py +++ b/ci/beacon.py @@ -63,8 +63,8 @@ } VIVADO_CELL = re.compile( - r'_reg((\[\d+\])*(_replica(_\d+)?)?(_[fb]ret(_\d+)?)*(_rep(_\d+)?)?' - r'|_\d+_\d+_\d+_\d+|_bram_\d+)$') + r'_reg((\[\w+\])*(_replica(_\d+)?)?(_[fb]ret(_+\d+)?)*(_rep(_+\d+)?)?' + r'|_\d+(_\d+_\d+_\d+)?|_bram_\d+)$') FOLD_SUFFIX = re.compile( r'(_ext(_\d+)?|_fold|_read(_\d+)?|_[fb]ret(_\d+)?|_replica(_\d+)?)+$') FOLD_SHELL = re.compile(r'^fold\d+_') @@ -235,17 +235,32 @@ def cmd_validate(args): # that place a register in the generate scope that assigns it rather # than the one that declares it. by_dir = collections.defaultdict(dict) - for row in endpoints(args.run_dir): - cls, p = paths[int(row['end_inst'])] + # Vendor IPs, black boxes to Beacon: Vivado's end points inside them have + # no counterpart. + ips = set() + + def class_key(inst_id, name): + cls, p = paths[int(inst_id)] if cls is None: - continue + return None # Registers of inlined modules are named /. - base = row['end'].rsplit('.', 1)[0] + base = name.rsplit('.', 1)[0] full = f'{p}/{base}' if p else base if sub: if not full.startswith(sub + '/'): - continue + return None full = full[len(sub) + 1:] + return cls, full + + for row in endpoints(args.run_dir): + if row['start_kind'] == 'ip': + ip = class_key(row['start_inst'], row['start']) + if ip: + ips.add((ip[0], norm(ip[1]))) + ck = class_key(row['end_inst'], row['end']) + if ck is None: + continue + cls, full = ck key = (cls, norm(full)) v = ours[key] if not v[1]: @@ -264,8 +279,13 @@ def fallback(key): best = (name, cand) return best[1] if best else key + def in_ip(key): + cls, k = key + parts = k.split('/') + return any((cls, '/'.join(parts[:i])) in ips for i in range(1, len(parts))) + viv = collections.defaultdict(lambda: [0, 0.0, 0]) # max lv, min slack, n - rows = unmatched_rows = shell_rows = 0 + rows = unmatched_rows = shell_rows = ip_rows = 0 unmatched = collections.Counter() with open(args.ep) as f: r = csv.reader(f) @@ -276,6 +296,9 @@ def fallback(key): if key is None or key[0] not in classes.values(): shell_rows += 1 continue + if in_ip(key): + ip_rows += 1 + continue if key not in ours and key[1].endswith('_rdata'): # The read register of a synchronous RAM is the memory's # read port in the design. @@ -294,7 +317,8 @@ def fallback(key): keys = sorted(viv) if not keys: print(f'ep rows {rows}: fold shell / channels {shell_rows}, ' - f'unmatched {unmatched_rows}, matched 0') + f'inside vendor IPs {ip_rows}, unmatched {unmatched_rows}, ' + f'matched 0') for (cls, k), n in unmatched.most_common(args.top): print(f' {n:6d} {cls}: {k}') return @@ -302,7 +326,8 @@ def fallback(key): lv = np.array([viv[k][0] for k in keys]) sl = np.array([viv[k][1] for k in keys]) print(f'ep rows {rows}: fold shell / channels {shell_rows}, ' - f'unmatched {unmatched_rows}, matched {rows - shell_rows - unmatched_rows} ' + f'inside vendor IPs {ip_rows}, unmatched {unmatched_rows}, ' + f'matched {rows - shell_rows - ip_rows - unmatched_rows} ' f'in {len(keys)} registers/memories') print(f'Beacon keys in the classes: {len(ours)}') print(f'Spearman(Beacon levels, Vivado levels) = {spearman(a, lv):.3f}') @@ -336,7 +361,8 @@ def fallback(key): if args.json: summary = { - 'rows': rows, 'shell': shell_rows, 'unmatched': unmatched_rows, + 'rows': rows, 'shell': shell_rows, 'vendor_ip': ip_rows, + 'unmatched': unmatched_rows, 'registers': len(keys), 'measured': len(ours), 'rho_levels': round(spearman(a, lv), 3), 'rho_slack': round(spearman(a, -sl), 3), From a738efb9dd129495fe871b56a1bb64b9aed0eed1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 03:27:05 -0700 Subject: [PATCH 282/336] lsu: classify address windows before the stack interleave The I/O, local-memory and OM windows were decoded from the AGU's final address, so the stack interleave sat in front of the request attribute path. The interleave maps the stack window onto itself and leaves every other address unchanged, so a window disjoint from the stack window classifies the linear address exactly as it does the interleaved one. The AGU now also outputs the linear address, the block decodes windows from it, and static asserts require the I/O, local-memory and OM windows to stay disjoint from the stack window. No functional or cycle change. Beacon on the core: the LSU request path goes from 20 to 15 levels and the core's deepest path from 20 to 18. rtlsim functional cases pass; lint clean at XLEN 32, XLEN 64 with EXT_A, and NUM_THREADS=32 with OM. Not run: the Vivado core gate. Adapted from foldsim 36b2cde55, deferred during the cherry-pick because it predated the stack window alignment (367891fec). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_lsu_agu.sv | 7 +++++-- hw/rtl/core/VX_lsu_block.sv | 34 ++++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/hw/rtl/core/VX_lsu_agu.sv b/hw/rtl/core/VX_lsu_agu.sv index 204db31184..c921308993 100644 --- a/hw/rtl/core/VX_lsu_agu.sv +++ b/hw/rtl/core/VX_lsu_agu.sv @@ -31,7 +31,10 @@ module VX_lsu_agu import VX_gpu_pkg::*; ( input wire [`VX_CFG_XLEN-1:0] stride, // rs2 (pack stride) input wire [11:0] offset, // immediate offset; [1:0] = pack uop_idx input wire [1:0] pack, // 0 = plain, non-zero = pack-load - output wire [`VX_CFG_XLEN-1:0] addr + output wire [`VX_CFG_XLEN-1:0] addr, + // the address before the stack interleave, which maps the stack window + // onto itself: any window disjoint from it classifies both alike + output wire [`VX_CFG_XLEN-1:0] lin_addr ); wire is_pack = (pack != 2'b00); wire [1:0] uop_idx = offset[1:0]; @@ -53,7 +56,7 @@ module VX_lsu_agu import VX_gpu_pkg::*; ( .sum (csa_sum), .carry (csa_carry) ); - wire [`VX_CFG_XLEN-1:0] lin_addr = csa_sum[`VX_CFG_XLEN-1:0] + csa_carry[`VX_CFG_XLEN-1:0]; + assign lin_addr = csa_sum[`VX_CFG_XLEN-1:0] + csa_carry[`VX_CFG_XLEN-1:0]; `UNUSED_VAR ({csa_sum[`VX_CFG_XLEN+1:`VX_CFG_XLEN], csa_carry[`VX_CFG_XLEN+1:`VX_CFG_XLEN]}) `ifdef VX_CFG_LSU_STACK_INTERLEAVE_ENABLE diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 6cf3b9446d..84f9399256 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -101,16 +101,38 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire req_fence_rel = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_aq; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr; + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] lin_addr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr VX_lsu_agu lsu_agu ( - .base (execute_if.data.rs1_data[i]), - .stride (execute_if.data.rs2_data[i]), - .offset (execute_if.data.op_args.lsu.offset), - .pack (execute_if.data.op_args.lsu.pack), - .addr (full_addr[i]) + .base (execute_if.data.rs1_data[i]), + .stride (execute_if.data.rs2_data[i]), + .offset (execute_if.data.op_args.lsu.offset), + .pack (execute_if.data.op_args.lsu.pack), + .addr (full_addr[i]), + .lin_addr (lin_addr[i]) ); end + // The address windows are classified on the address before the stack + // interleave, which keeps the interleave off the request's attribute + // path. The interleave maps the stack window onto itself, so this holds + // for every window outside it. +`ifdef VX_CFG_LSU_STACK_INTERLEAVE_ENABLE + `STATIC_ASSERT((`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) <= STACK_WINDOW_BOTTOM) + || (`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >= STACK_WINDOW_TOP), + ("the I/O window overlaps the stack window")) +`ifdef VX_CFG_LMEM_ENABLE + `STATIC_ASSERT((`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >= STACK_WINDOW_TOP) + || ((`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) + (`VX_CFG_XLEN'(1) << `VX_CFG_LMEM_LOG_SIZE)) <= STACK_WINDOW_BOTTOM), + ("the local memory window overlaps the stack window")) +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + `STATIC_ASSERT((`VX_CFG_XLEN'(`VX_MEM_OM_END_ADDR) <= STACK_WINDOW_BOTTOM) + || (`VX_CFG_XLEN'(`VX_MEM_OM_BASE_ADDR) >= STACK_WINDOW_TOP), + ("the OM aperture overlaps the stack window")) +`endif +`endif + // address type + AMO classification — per-lane mem_bus_attr_t // carried through the scheduler's USER channel and out via the // lsu_mem_if's per-lane req_data.user field. @@ -118,7 +140,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( mem_bus_attr_t [NUM_LANES-1:0] mem_req_attr_struct; wire [NUM_LANES-1:0][MEM_ATTR_WIDTH-1:0] mem_req_attr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_mem_req_attr - wire [MEM_ADDRW-1:0] block_addr = full_addr[i][MEM_ASHIFT +: MEM_ADDRW]; + wire [MEM_ADDRW-1:0] block_addr = lin_addr[i][MEM_ASHIFT +: MEM_ADDRW]; // is I/O address wire [MEM_ADDRW-1:0] io_addr_start = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >> MEM_ASHIFT); wire [MEM_ADDRW-1:0] io_addr_end = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) >> MEM_ASHIFT); From 10d15af467034206e122530bb05f655829485d1d Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 03:50:58 -0700 Subject: [PATCH 283/336] vm: do not re-reserve the VAs of a range already identity-mapped install_identity_map reserved its range in the VA allocator and ignored the result. A range inside one mapped before, as the 64-bit kernel image at 0x180000000 is inside the page-table and stack region init() maps, failed that reservation, and the allocator printed "Error: address range overlaps with existing allocation" on every 64-bit VM run that loads a kernel. The manager now remembers the ranges it reserved and skips a reservation they already cover. No mapping changes. Verified: gtx1080 profile, simx, sgemm -n64 passes without the message. Co-Authored-By: Claude Opus 5.5 --- sw/runtime/common/vm.cpp | 12 +++++++++++- sw/runtime/common/vm.h | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sw/runtime/common/vm.cpp b/sw/runtime/common/vm.cpp index 0922d8cfd1..2286093e9d 100644 --- a/sw/runtime/common/vm.cpp +++ b/sw/runtime/common/vm.cpp @@ -358,7 +358,17 @@ int VMManager::install_identity_map(uint64_t addr, uint64_t size) { constexpr uint32_t IDENTITY_PTE_FLAGS = PTE_V | PTE_R | PTE_W | PTE_X | PTE_U | PTE_A | PTE_D; - (void)virtual_mem_->reserve(addr, size); + // A range inside one already mapped, as the kernel image is inside the + // page-table and stack region init() maps, holds its VAs already. + bool reserved = false; + for (auto [start, end] : identity_ranges_) { + if (addr >= start && addr + size <= end) { + reserved = true; + break; + } + } + if (!reserved && virtual_mem_->reserve(addr, size) == 0) + identity_ranges_.emplace_back(addr, addr + size); uint64_t cur = addr; uint64_t end = addr + size; diff --git a/sw/runtime/common/vm.h b/sw/runtime/common/vm.h index 581ecd43fe..58a0cbdf2c 100644 --- a/sw/runtime/common/vm.h +++ b/sw/runtime/common/vm.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -129,6 +130,9 @@ class VMManager { // Heap mapped once with superpages (VORTEX_VM_MEGAPAGE_HEAP). bool megapage_heap_; + + // Identity-mapped ranges [start, end) whose VAs are reserved. + std::vector> identity_ranges_; }; } // namespace vortex From 0b0d7ca4d7a2b8d2b31e634c1d406d6ed08cc2e4 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 14:35:55 -0700 Subject: [PATCH 284/336] lmem: warp-wide local memory unit with dual pass engines VX_local_mem served LSU requests lane by lane through a request crossbar, an 18-level path that bounded the core gate. VX_lmem_unit takes a whole warp request per cycle per engine and serves it in passes: each pass gives every bank to the lowest pending lane mapping to it, and every pending read lane of the same word rides along; leftover lanes replay, and a request retires before the next starts, so the banks see every lane in program order. With several LSU blocks a second, read-only engine works the banks' second port (VX_1rw1r_ram), so two warps' reads go through each cycle. Stores and atomics use the first engine alone and are ordered against every read. Each block's response leaves from its own register, with no added latency (the response queue is bypassed when empty). DMA (DXA writes, TCU reads) takes whole bank rows. SimX models the same passes, engines and response registers. Beacon on the core against the parent: the LMEM's deepest path goes from 18 to 9 levels, and registers at 8+ levels from 789 to 337. lmem_merge is replaced by lmem_access, which adds cross-block, shared-counter and LR/SC cases. The AMO parity case skips the LR/SC phase (-s): its retry count depends on timing, so instruction counts cannot match; the lmem-access-amo regression cases still run it on simx and rtlsim. Validated (agent, snapshot of this tree): Verilator lint of lmem_unit at three shapes, XLEN 32 and 64 rtlsim builds, testcase lint; lmem_access regression on simx and rtlsim; model parity for lmem_access and sgemm2-lmem. Not run: perf_gate (cycles move), and the fpga/asic gates. The lmem gate budgets (fpga 26822 LUT from a Vivado run of this DUT; asic area stale) and the logic_gate lmem entry need a recorded baseline. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 4 - ci/testcases/asic_gate.yaml | 9 + ci/testcases/core.yaml | 30 +- ci/testcases/fpga_gate.yaml | 9 + ci/testcases/logic_gate.yaml | 8 + ci/testcases/regression.yaml | 35 +- docs/assets/img/amo_architecture.svg | 4 +- docs/assets/img/lsu_architecture.svg | 10 +- docs/designs/atomic_memory_operations.md | 4 +- docs/designs/lsu_pipeline_design.md | 31 +- docs/hardware_library.md | 176 +-- hw/rtl/core/VX_backend_slice.sv | 4 +- hw/rtl/core/VX_compute_slice.sv | 4 +- hw/rtl/core/VX_core.sv | 13 +- hw/rtl/core/VX_execute.sv | 4 +- hw/rtl/core/VX_execute_slice.sv | 4 +- hw/rtl/core/VX_lsu_block.sv | 4 +- hw/rtl/core/VX_lsu_mem_path.sv | 30 +- hw/rtl/core/VX_lsu_unit.sv | 14 +- hw/rtl/core/VX_mem_unit.sv | 37 +- hw/rtl/dxa/VX_dxa_pkg.sv | 4 +- hw/rtl/libs/VX_1rw1r_ram.sv | 319 +++++ hw/rtl/mem/VX_lmem_engine.sv | 442 +++++++ hw/rtl/mem/VX_lmem_unit.sv | 961 +++++++++++++++ hw/rtl/mem/VX_local_mem.sv | 912 -------------- hw/syn/altera/dut/catalog.mk | 4 +- hw/syn/xilinx/dut/catalog.mk | 4 +- hw/syn/yosys/dut/catalog.mk | 8 +- hw/unittest/Makefile | 6 +- hw/unittest/{local_mem => lmem_unit}/Makefile | 4 +- hw/unittest/lmem_unit/VX_lmem_unit_top.sv | 155 +++ hw/unittest/{local_mem => lmem_unit}/main.cpp | 0 hw/unittest/local_mem/VX_local_mem_top.sv | 116 -- hw/unittest/mem_unit/VX_mem_unit_top.sv | 9 +- hw/unittest/tensor/VX_tensor_top.sv | 9 +- sim/simx/core.cpp | 30 +- sim/simx/mem/local_mem.cpp | 1067 +++++++++-------- sim/simx/mem/local_mem.h | 20 +- tests/regression/Makefile | 2 +- .../{lmem_merge => lmem_access}/Makefile | 2 +- tests/regression/lmem_access/common.h | 60 + tests/regression/lmem_access/kernel.cpp | 220 ++++ .../{lmem_merge => lmem_access}/main.cpp | 75 +- tests/regression/lmem_merge/common.h | 22 - tests/regression/lmem_merge/kernel.cpp | 90 -- 45 files changed, 3064 insertions(+), 1911 deletions(-) create mode 100644 hw/rtl/libs/VX_1rw1r_ram.sv create mode 100644 hw/rtl/mem/VX_lmem_engine.sv create mode 100644 hw/rtl/mem/VX_lmem_unit.sv delete mode 100644 hw/rtl/mem/VX_local_mem.sv rename hw/unittest/{local_mem => lmem_unit}/Makefile (93%) create mode 100644 hw/unittest/lmem_unit/VX_lmem_unit_top.sv rename hw/unittest/{local_mem => lmem_unit}/main.cpp (100%) delete mode 100644 hw/unittest/local_mem/VX_local_mem_top.sv rename tests/regression/{lmem_merge => lmem_access}/Makefile (91%) create mode 100644 tests/regression/lmem_access/common.h create mode 100644 tests/regression/lmem_access/kernel.cpp rename tests/regression/{lmem_merge => lmem_access}/main.cpp (54%) delete mode 100644 tests/regression/lmem_merge/common.h delete mode 100644 tests/regression/lmem_merge/kernel.cpp diff --git a/VX_config.toml b/VX_config.toml index e297ef35ad..378e1c2771 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -320,10 +320,6 @@ lmem_bytes_per_bank = 2048 lmem_min_bytes = 16384 VX_CFG_LMEM_NUM_BANKS = "expr: $VX_CFG_NUM_LSU_LANES" VX_CFG_LMEM_LOG_SIZE = "expr: clog2(max($lmem_min_bytes, $VX_CFG_LMEM_NUM_BANKS * $lmem_bytes_per_bank))" -# Same-word reads of one LSU block's lanes share one bank access, and the -# response returns to all of them: each bank serves one distinct word per cycle -# to every lane reading it. -VX_CFG_LMEM_READ_MERGE = 1 [tcu] # string enum: 'DPI'|'DSP'|'BHF'|'TFR'|'FPNEW'; TFR is the universal default. On diff --git a/ci/testcases/asic_gate.yaml b/ci/testcases/asic_gate.yaml index 6166cf9a39..678a863f9f 100644 --- a/ci/testcases/asic_gate.yaml +++ b/ci/testcases/asic_gate.yaml @@ -123,6 +123,15 @@ builds: area_budget_um2: 84624 configs: "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8" +# Warp-wide local memory with two LSU blocks: both pass engines, the +# dual-port banks the read engine reads through, and the atomic logic. +- id: lmem + group: core + dut: lmem + clock_mhz: 400 + area_budget_um2: 15867 + configs: "-DVX_CFG_NUM_THREADS=16 -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_EXT_A_ENABLE" + # Multi-core system top: 2 cores with atomics and virtual memory behind the AFU # wrapper. Closes the cross-module boundaries (socket arbiters, socket MMU, # cluster mem network) that the per-unit DUTs do not. diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 26cd790c1e..e38cc8f049 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -244,19 +244,39 @@ tests: via: blackbox app: uniform_ld configs: -DVX_CFG_NUM_THREADS=16 -# Same-word local-memory reads of one LSU block, served by one bank access and -# returned to every lane. -- id: parity-lmem-merge +# Warp-wide local memory: broadcast reads, bank-conflict replays, sub-word and +# same-word stores, with one LSU block and with two, where the read engine +# works the banks' second port. +- id: parity-lmem-access check: model_parity via: blackbox - app: lmem_merge + app: lmem_access configs: -DVX_CFG_NUM_THREADS=16 -- id: parity-sgemm2-lmem-merge +- id: parity-lmem-access-iw2 + check: model_parity + via: blackbox + app: lmem_access + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_ISSUE_WIDTH=2 +# -s leaves out the LR/SC phase, whose retry count depends on timing; the +# lmem-access-amo* regression cases run it. +- id: parity-lmem-access-amo-iw2 + check: model_parity + via: blackbox + app: lmem_access + args: -n16 -s + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 +- id: parity-sgemm2-lmem check: model_parity via: blackbox app: sgemm2 args: -n32 -t4 configs: -DVX_CFG_NUM_THREADS=16 +- id: parity-sgemm2-lmem-iw2 + check: model_parity + via: blackbox + app: sgemm2 + args: -n64 -t16 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=2 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity diff --git a/ci/testcases/fpga_gate.yaml b/ci/testcases/fpga_gate.yaml index 4f0d04a661..65750815c6 100644 --- a/ci/testcases/fpga_gate.yaml +++ b/ci/testcases/fpga_gate.yaml @@ -88,6 +88,15 @@ builds: lut_budget: 171712 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE +# Warp-wide local memory with two LSU blocks: both pass engines, the +# dual-port banks the read engine reads through, and the atomic logic. +- id: lmem + group: core + dut: lmem + clock_mhz: 300 + lut_budget: 26822 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_EXT_A_ENABLE + # Multi-core system top: 2 cores at NT=NW=4, atomics + virtual memory (MMU). # Closes the cross-module boundaries (socket arbiters, socket MMU, cluster mem # network) that the per-unit DUTs do not; 250 MHz (routing-bound, misses 300). diff --git a/ci/testcases/logic_gate.yaml b/ci/testcases/logic_gate.yaml index af6f6e7f1b..56b3061da7 100644 --- a/ci/testcases/logic_gate.yaml +++ b/ci/testcases/logic_gate.yaml @@ -84,6 +84,14 @@ builds: clock_mhz: 300 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE +# Warp-wide local memory with two LSU blocks: both pass engines, the +# dual-port banks the read engine reads through, and the atomic logic. +- id: lmem + group: core + dut: lmem + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_EXT_A_ENABLE + # fpga_gate's `top` is not measured: CIRCT's import leaves the command # processor's AXI-Lite response ports (VX_cp_core) as LLHD references, which # no HW/Comb/Seq tool reads. It joins this list when the import lowers them. diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index b83322e89b..099df6752b 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -18,33 +18,48 @@ tests: - rtlsim dir: tests/regression target: run-{driver} -# Local-memory read merging: shared-word reads, partial masks, lanes of -# different LSU blocks, and per-lane order against the lane's own stores. -- id: lmem-merge-1 +# Local memory: shared-word reads (one lane to the whole warp, and lanes of +# different LSU blocks), partial masks, bank-conflict replays, sub-word loads +# and stores, same-word stores, and per-lane order against the lane's own +# stores and atomics. +- id: lmem-access-1 via: blackbox drivers: - simx - rtlsim - app: lmem_merge + app: lmem_access shape: threads: 16 -- id: lmem-merge-2 +# Two LSU blocks: the read engine on the banks' second port. +- id: lmem-access-2 via: blackbox drivers: - simx - app: lmem_merge + - rtlsim + app: lmem_access configs: -DVX_CFG_ISSUE_WIDTH=2 shape: threads: 32 warps: 8 -# The same test with merging off: every lane takes its own bank access. -- id: lmem-merge-off +# With atomics: a warp's reads around its own atomic adds to the word. +- id: lmem-access-amo + via: blackbox + drivers: + - simx + - rtlsim + app: lmem_access + configs: -DVX_CFG_EXT_A_ENABLE + shape: + threads: 16 +# Atomics with two LSU blocks: the read engine drains before an atomic enters +# the first engine. +- id: lmem-access-amo-iw2 via: blackbox drivers: - simx - rtlsim - app: lmem_merge - configs: -DVX_CFG_LMEM_READ_MERGE=0 + app: lmem_access + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 shape: threads: 16 - id: occupancy-1 diff --git a/docs/assets/img/amo_architecture.svg b/docs/assets/img/amo_architecture.svg index 6751a3d332..b9d02886a8 100644 --- a/docs/assets/img/amo_architecture.svg +++ b/docs/assets/img/amo_architecture.svg @@ -86,8 +86,8 @@ Commit point 1 — shared (local) memory - VX_local_mem bank - single-ported SRAM: an atomic owns its bank for 3 cycles + VX_lmem_unit bank + an atomic owns its bank for 3 cycles (first port; nothing else is issued) ① read the word ② register it, resolve the reservation ③ VX_amo_alu result written back under the request byteen VX_amo_unit: min(AMO_RS_SIZE, 4) stations, 3-bit hold credit diff --git a/docs/assets/img/lsu_architecture.svg b/docs/assets/img/lsu_architecture.svg index ce3db01776..782e4efae4 100644 --- a/docs/assets/img/lsu_architecture.svg +++ b/docs/assets/img/lsu_architecture.svg @@ -152,11 +152,11 @@ streams local path - VX_lsu_adapter, one per block → VX_local_mem - banked SRAM: lanes to different banks are served - in the same cycle - a second port serves the DXA and tensor - direct-memory clients + whole warp requests → VX_lmem_unit + a pass per cycle: each bank to its lowest lane, + same-word reads broadcast, conflicts replay + several LSU blocks: a read engine on the second + bank port; DXA and tensor DMA take whole rows per-core shared memory global path diff --git a/docs/designs/atomic_memory_operations.md b/docs/designs/atomic_memory_operations.md index 3388132790..500aac5ff8 100644 --- a/docs/designs/atomic_memory_operations.md +++ b/docs/designs/atomic_memory_operations.md @@ -8,7 +8,7 @@ cache bank and the local-memory bank), the reservation stations, and the SimX model. RTL: [`VX_amo_alu.sv`](../../hw/rtl/cache/VX_amo_alu.sv), [`VX_amo_unit.sv`](../../hw/rtl/cache/VX_amo_unit.sv), [`VX_cache_amo.sv`](../../hw/rtl/cache/VX_cache_amo.sv), -[`VX_local_mem.sv`](../../hw/rtl/mem/VX_local_mem.sv). SimX: +[`VX_lmem_unit.sv`](../../hw/rtl/mem/VX_lmem_unit.sv). SimX: [`sim/simx/amo/`](../../sim/simx/amo/), [`sim/simx/mem/cache.cpp`](../../sim/simx/mem/cache.cpp), [`sim/simx/mem/local_mem.cpp`](../../sim/simx/mem/local_mem.cpp). @@ -321,7 +321,7 @@ deferring, at most, one cycle longer than strictly needed. ## 7. Commit at the local-memory bank Shared (local) memory is a per-core SRAM, not a cache, so there is no line -to fetch and no level below: [`VX_local_mem.sv`](../../hw/rtl/mem/VX_local_mem.sv) +to fetch and no level below: [`VX_lmem_unit.sv`](../../hw/rtl/mem/VX_lmem_unit.sv) commits in place. [`VX_lmem_switch.sv`](../../hw/rtl/mem/VX_lmem_switch.sv) passes the attribute through unchanged on the local path. diff --git a/docs/designs/lsu_pipeline_design.md b/docs/designs/lsu_pipeline_design.md index 97268076f3..fe0b1f0f1d 100644 --- a/docs/designs/lsu_pipeline_design.md +++ b/docs/designs/lsu_pipeline_design.md @@ -305,14 +305,29 @@ The attribute passes through untouched on both paths. ### 5.2 Local path -Each LSU block has its **own** adapter into -[`VX_local_mem`](../../hw/rtl/mem/VX_local_mem.sv). A shared adapter would -close a loop through its pack and unpack buffers and deadlock with more than -one block. - -Local memory is a banked SRAM, so lanes addressing different banks are served -in the same cycle. Its second port is shared by the direct-memory clients — -the DXA writes and the tensor unit's reads. +Each LSU block hands its local requests whole to +[`VX_lmem_unit`](../../hw/rtl/mem/VX_lmem_unit.sv), the warp-wide local +memory. A block arbiter takes one request at a time into a pass engine +([`VX_lmem_engine`](../../hw/rtl/mem/VX_lmem_engine.sv)), which compares the +lanes' words once and then serves the request in bank-conflict-free passes: +every bank goes to its lowest pending lane, every pending lane reading that +lane's word rides along (a broadcast), and the remaining lanes replay. A +request retires before the next one starts, so every lane's accesses reach the +banks in program order. A request without bank conflicts takes the banks for a +single cycle, so one such request issues every cycle; the response +carries every lane the pass served. + +With several LSU blocks, a second engine for plain reads only works the banks' +second port, so two warps' reads go through in a cycle. Stores and atomics use +the first engine alone and wait for the read engine to drain; the read engine +takes no read while a store or atomic sits in the first engine, while one +waited at an input the cycle before, or in a cycle the first engine may be +taking one. Each engine's block is chosen a cycle ahead, so the ready a block +sees comes from registers. + +The direct-memory clients (the DXA writes and the tensor unit's reads) take +whole bank rows on the first port: a DMA request seen in one cycle stops the +passes and gets the banks the next. ### 5.3 Global path diff --git a/docs/hardware_library.md b/docs/hardware_library.md index b9ccc4182f..094d08460a 100644 --- a/docs/hardware_library.md +++ b/docs/hardware_library.md @@ -2,7 +2,7 @@ ## Overview -The Vortex hardware IP library (`hw/rtl/libs/`) provides a comprehensive collection of 76 reusable, parameterized SystemVerilog IP modules used throughout the Vortex GPGPU processor. These modules cover arbitration, buffering, arithmetic, memory, interconnect, and debug infrastructure, forming the foundational building blocks of the Vortex microarchitecture. +The Vortex hardware IP library (`hw/rtl/libs/`) provides a comprehensive collection of 77 reusable, parameterized SystemVerilog IP modules used throughout the Vortex GPGPU processor. These modules cover arbitration, buffering, arithmetic, memory, interconnect, and debug infrastructure, forming the foundational building blocks of the Vortex microarchitecture. All modules are parameterized for data width, depth, and implementation strategy, with support for both FPGA (Xilinx/Intel) and ASIC synthesis flows. @@ -1259,6 +1259,31 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn --- +### VX_1rw1r_ram + +**File:** `VX_1rw1r_ram.sv` + +**Purpose:** RAM with one read/write port and one read-only port, both with a registered read. + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `DATAW` | - | Data width in bits | +| `SIZE` | - | Number of entries | +| `WRENW` | 1 | Write enable granularity | +| `LUTRAM` | 0 | Force LUTRAM implementation | +| `RESET_RAM` | 0 | Clear the contents on reset (simulation) | +| `INIT_ENABLE` | 0 | Enable initialization | +| `INIT_FILE` | "" | Initialization memory file | + +**Features:** +- Port A reads or writes (read-first); port B reads, each with its own address +- A port B read of the word port A writes in the same cycle is undefined; callers must not issue one +- FPGA: the two ports of a true dual-port block RAM, or distributed RAM with `LUTRAM` +- ASIC: two `VX_dp_ram` copies that both take every write, one per read port, since the macro libraries have no read/write + read macro +- Used by the warp-wide local memory's second, read-only pass engine + +--- + ### VX_rom **File:** `VX_rom.sv` @@ -1811,77 +1836,78 @@ Infrastructure for runtime debug observation and serial trace extraction. | # | Module | Category | Area | Latency | |---|--------|----------|------|---------| -| 1 | VX_allocator | Allocator | Small | 0 cycles | -| 2 | VX_avs_adapter | Memory Adapter | Large | Multi-cycle | -| 3 | VX_axi_adapter | Memory Adapter | Large | Multi-cycle | -| 4 | VX_axi_write_ack | Memory Adapter | Small | 0-1 cycles | -| 5 | VX_bits_concat | Bit Manip | Zero | 0 cycles | -| 6 | VX_bits_insert | Bit Manip | Zero | 0 cycles | -| 7 | VX_bits_remove | Bit Manip | Zero | 0 cycles | -| 8 | VX_bypass_buffer | Buffer | Small | 0-1 cycles | -| 9 | VX_clockgate | Sync | Minimal | 0 cycles | -| 10 | VX_csa_32 | CSA | Small | 1 gate | -| 11 | VX_csa_42 | CSA | Medium | 2 gates | -| 12 | VX_csa_mod4 | CSA | Varies | O(log N) | -| 13 | VX_csa_tree | CSA | Varies | O(log N) | -| 14 | VX_cyclic_arbiter | Arbiter | Small | 0 cycles | -| 15 | VX_demux | Mux/Demux | Small | 0 cycles | -| 16 | VX_divider | Arithmetic | Large | Configurable | -| 17 | VX_dp_ram | Memory | Varies | 0-1 cycles | -| 18 | VX_edge_trigger | Sync | Minimal | 1 cycle | -| 19 | VX_elastic_adapter | Buffer | Minimal | 0-1 cycles | -| 20 | VX_elastic_buffer | Buffer | Varies | Configurable | -| 21 | VX_fifo_queue | Memory | Varies | 0-1 cycles | -| 22 | VX_find_first | Encoder | Small | O(log N) | -| 23 | VX_fold_mul | Arithmetic | Medium | Configurable | -| 24 | VX_generic_arbiter | Arbiter | Varies | 0 cycles | -| 25 | VX_gto_arbiter | Arbiter | Medium | 0 cycles | -| 26 | VX_index_buffer | Memory | Medium | 0-1 cycles | -| 27 | VX_index_queue | Memory | Medium | 0-1 cycles | -| 28 | VX_ks_adder | Arithmetic | Medium | O(log N) | -| 29 | VX_lzc | Encoder | Small | O(log N) | -| 30 | VX_matrix_arbiter | Arbiter | Medium | 0 cycles | -| 31 | VX_mem_bank_adapter | Memory Adapter | Large | Multi-cycle | -| 32 | VX_mem_coalescer | Memory Adapter | Large | Multi-cycle | -| 33 | VX_mem_data_adapter | Memory Adapter | Medium | Multi-cycle | -| 34 | VX_mem_scheduler | Scheduler | Large | Multi-cycle | -| 35 | VX_multiplier | Arithmetic | Varies | Configurable | -| 36 | VX_mux | Mux/Demux | Small | 0 cycles | -| 37 | VX_nz_iterator | Stream Util | Small | 0-1 cycles | -| 38 | VX_onehot_encoder | Encoder | Small | O(log N) | -| 39 | VX_onehot_mux | Mux/Demux | Small | 0 cycles | -| 40 | VX_onehot_shift | Bit Manip | Small | 0 cycles | -| 41 | VX_pe_serializer | Stream Util | Medium | Multi-cycle | -| 42 | VX_pending_size | Counter | Small | 0-1 cycles | -| 43 | VX_pipe_buffer | Buffer | Varies | N cycles | -| 44 | VX_pipe_register | Buffer | Varies | N cycles | -| 45 | VX_popcount | Arithmetic | Small | O(log N) | -| 46 | VX_priority_arbiter | Arbiter | Small | 0 cycles | -| 47 | VX_priority_encoder | Encoder | Small | O(log N) | -| 48 | VX_reduce_tree | Arithmetic | Small | O(log N) | -| 49 | VX_reset_relay | Sync | Small | 1 cycle | -| 50 | VX_rom | Memory | Varies | 0-1 cycles | -| 51 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | -| 52 | VX_scan | Encoder | Medium | O(log N) | -| 53 | VX_scope_switch | Debug | Small | 0 cycles | -| 54 | VX_scope_tap | Debug | Large | Multi-cycle | -| 55 | VX_serial_div | Arithmetic | Small | Multi-cycle | -| 56 | VX_serial_mul | Arithmetic | Small | Multi-cycle | -| 57 | VX_shift_register | Buffer | Varies | N cycles | -| 58 | VX_skid_buffer | Buffer | Small | 0-1 cycles | -| 59 | VX_sp_ram | Memory | Varies | 0-1 cycles | -| 60 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | -| 61 | VX_stream_buffer | Buffer | Small | 0-1 cycles | -| 62 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | -| 63 | VX_stream_fork | Stream Util | Small | 0 cycles | -| 64 | VX_stream_join | Stream Util | Small | 0 cycles | -| 65 | VX_stream_omega | Interconnect | Large | O(log N) | -| 66 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | -| 67 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | -| 68 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | -| 69 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | -| 70 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | -| 71 | VX_ticket_lock | Allocator | Small | 0-1 cycles | -| 72 | VX_toggle_buffer | Buffer | Small | 1 cycle | -| 73 | VX_transpose | Bit Manip | Zero | 0 cycles | -| 74 | VX_wallace_mul | Arithmetic | Large | O(log N) | +| 1 | VX_1rw1r_ram | Memory | Varies | 1 cycle | +| 2 | VX_allocator | Allocator | Small | 0 cycles | +| 3 | VX_avs_adapter | Memory Adapter | Large | Multi-cycle | +| 4 | VX_axi_adapter | Memory Adapter | Large | Multi-cycle | +| 5 | VX_axi_write_ack | Memory Adapter | Small | 0-1 cycles | +| 6 | VX_bits_concat | Bit Manip | Zero | 0 cycles | +| 7 | VX_bits_insert | Bit Manip | Zero | 0 cycles | +| 8 | VX_bits_remove | Bit Manip | Zero | 0 cycles | +| 9 | VX_bypass_buffer | Buffer | Small | 0-1 cycles | +| 10 | VX_clockgate | Sync | Minimal | 0 cycles | +| 11 | VX_csa_32 | CSA | Small | 1 gate | +| 12 | VX_csa_42 | CSA | Medium | 2 gates | +| 13 | VX_csa_mod4 | CSA | Varies | O(log N) | +| 14 | VX_csa_tree | CSA | Varies | O(log N) | +| 15 | VX_cyclic_arbiter | Arbiter | Small | 0 cycles | +| 16 | VX_demux | Mux/Demux | Small | 0 cycles | +| 17 | VX_divider | Arithmetic | Large | Configurable | +| 18 | VX_dp_ram | Memory | Varies | 0-1 cycles | +| 19 | VX_edge_trigger | Sync | Minimal | 1 cycle | +| 20 | VX_elastic_adapter | Buffer | Minimal | 0-1 cycles | +| 21 | VX_elastic_buffer | Buffer | Varies | Configurable | +| 22 | VX_fifo_queue | Memory | Varies | 0-1 cycles | +| 23 | VX_find_first | Encoder | Small | O(log N) | +| 24 | VX_fold_mul | Arithmetic | Medium | Configurable | +| 25 | VX_generic_arbiter | Arbiter | Varies | 0 cycles | +| 26 | VX_gto_arbiter | Arbiter | Medium | 0 cycles | +| 27 | VX_index_buffer | Memory | Medium | 0-1 cycles | +| 28 | VX_index_queue | Memory | Medium | 0-1 cycles | +| 29 | VX_ks_adder | Arithmetic | Medium | O(log N) | +| 30 | VX_lzc | Encoder | Small | O(log N) | +| 31 | VX_matrix_arbiter | Arbiter | Medium | 0 cycles | +| 32 | VX_mem_bank_adapter | Memory Adapter | Large | Multi-cycle | +| 33 | VX_mem_coalescer | Memory Adapter | Large | Multi-cycle | +| 34 | VX_mem_data_adapter | Memory Adapter | Medium | Multi-cycle | +| 35 | VX_mem_scheduler | Scheduler | Large | Multi-cycle | +| 36 | VX_multiplier | Arithmetic | Varies | Configurable | +| 37 | VX_mux | Mux/Demux | Small | 0 cycles | +| 38 | VX_nz_iterator | Stream Util | Small | 0-1 cycles | +| 39 | VX_onehot_encoder | Encoder | Small | O(log N) | +| 40 | VX_onehot_mux | Mux/Demux | Small | 0 cycles | +| 41 | VX_onehot_shift | Bit Manip | Small | 0 cycles | +| 42 | VX_pe_serializer | Stream Util | Medium | Multi-cycle | +| 43 | VX_pending_size | Counter | Small | 0-1 cycles | +| 44 | VX_pipe_buffer | Buffer | Varies | N cycles | +| 45 | VX_pipe_register | Buffer | Varies | N cycles | +| 46 | VX_popcount | Arithmetic | Small | O(log N) | +| 47 | VX_priority_arbiter | Arbiter | Small | 0 cycles | +| 48 | VX_priority_encoder | Encoder | Small | O(log N) | +| 49 | VX_reduce_tree | Arithmetic | Small | O(log N) | +| 50 | VX_reset_relay | Sync | Small | 1 cycle | +| 51 | VX_rom | Memory | Varies | 0-1 cycles | +| 52 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | +| 53 | VX_scan | Encoder | Medium | O(log N) | +| 54 | VX_scope_switch | Debug | Small | 0 cycles | +| 55 | VX_scope_tap | Debug | Large | Multi-cycle | +| 56 | VX_serial_div | Arithmetic | Small | Multi-cycle | +| 57 | VX_serial_mul | Arithmetic | Small | Multi-cycle | +| 58 | VX_shift_register | Buffer | Varies | N cycles | +| 59 | VX_skid_buffer | Buffer | Small | 0-1 cycles | +| 60 | VX_sp_ram | Memory | Varies | 0-1 cycles | +| 61 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | +| 62 | VX_stream_buffer | Buffer | Small | 0-1 cycles | +| 63 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | +| 64 | VX_stream_fork | Stream Util | Small | 0 cycles | +| 65 | VX_stream_join | Stream Util | Small | 0 cycles | +| 66 | VX_stream_omega | Interconnect | Large | O(log N) | +| 67 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | +| 68 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | +| 69 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | +| 70 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | +| 71 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | +| 72 | VX_ticket_lock | Allocator | Small | 0-1 cycles | +| 73 | VX_toggle_buffer | Buffer | Small | 1 cycle | +| 74 | VX_transpose | Bit Manip | Zero | 0 cycles | +| 75 | VX_wallace_mul | Arithmetic | Large | O(log N) | diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv index 2d7059d12a..42c9630e8b 100644 --- a/hw/rtl/core/VX_backend_slice.sv +++ b/hw/rtl/core/VX_backend_slice.sv @@ -66,7 +66,7 @@ import VX_tcu_pkg::*; output wire lsu_store_empty, output wire lsu_busy, `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], + VX_lsu_mem_if.master lmem_if, `endif VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], @@ -152,7 +152,7 @@ import VX_tcu_pkg::*; .lsu_store_empty (lsu_store_empty), .lsu_busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lmem_bus_if), + .lmem_if (lmem_if), `endif .dcache_bus_if (dcache_bus_if), `ifdef VX_CFG_EXT_DXA_ENABLE diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index cbfce89ffd..4d8c0b6619 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -69,7 +69,7 @@ import VX_tcu_pkg::*; output wire lsu_store_empty, output wire lsu_busy, `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], + VX_lsu_mem_if.master lmem_if, `endif VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], @@ -305,7 +305,7 @@ import VX_tcu_pkg::*; .lsu_store_empty (lsu_store_empty), .lsu_busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lmem_bus_if), + .lmem_if (lmem_if), `endif .dcache_bus_if (dcache_bus_if), `ifdef VX_CFG_EXT_DXA_ENABLE diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index e7bf27571d..c68f1eda23 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -85,10 +85,11 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty; `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if #( + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_lmem_bus_if[LSU_NUM_REQS](); + ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); `endif VX_mem_bus_if #( .DATA_SIZE (DCACHE_WORD_SIZE), @@ -450,7 +451,7 @@ import VX_tcu_pkg::*; .lsu_store_empty (lsu_store_empty[i]), .lsu_busy (lsu_busy[i]), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + .lmem_if (lsu_lmem_if[i]), `endif .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]), `ifdef VX_CFG_EXT_DXA_ENABLE @@ -669,7 +670,7 @@ import VX_tcu_pkg::*; .lsu_store_empty (lsu_store_empty[i]), .lsu_busy (lsu_busy[i]), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + .lmem_if (lsu_lmem_if[i]), `endif .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]), `ifdef VX_CFG_EXT_DXA_ENABLE @@ -750,7 +751,7 @@ import VX_tcu_pkg::*; .lsu_store_empty (lsu_store_empty), .lsu_busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lsu_lmem_bus_if), + .lmem_if (lsu_lmem_if), `endif .dcache_bus_if (lsu_dcache_bus_if), @@ -819,7 +820,7 @@ import VX_tcu_pkg::*; .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lsu_lmem_bus_if), + .lmem_if (lsu_lmem_if), `endif .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 330e01a254..7a7b394cdd 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -38,7 +38,7 @@ import VX_tcu_pkg::*; output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_busy, `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], + VX_lsu_mem_if.master lmem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], @@ -132,7 +132,7 @@ import VX_tcu_pkg::*; .store_empty (lsu_store_empty), .busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lmem_bus_if), + .lmem_if (lmem_if), `endif .dcache_bus_if (dcache_bus_if) ); diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index 5542697bc2..b7e6d65892 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -55,7 +55,7 @@ import VX_tcu_pkg::*; output wire lsu_store_empty, output wire lsu_busy, `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], + VX_lsu_mem_if.master lmem_if, `endif VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], @@ -196,7 +196,7 @@ import VX_tcu_pkg::*; .store_empty (lsu_store_empty), .busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lmem_bus_if), + .lmem_if (lmem_if), `endif .dcache_bus_if (dcache_bus_if) ); diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 84f9399256..43f764107e 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -45,7 +45,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( output wire busy, `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], + VX_lsu_mem_if.master lmem_if, `endif VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS] ); @@ -448,7 +448,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( .store_empty (store_empty), .busy (busy), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lmem_bus_if), + .lmem_if (lmem_if), `endif .dcache_bus_if (dcache_bus_if) ); diff --git a/hw/rtl/core/VX_lsu_mem_path.sv b/hw/rtl/core/VX_lsu_mem_path.sv index 3b3a255e0c..6cef8767b3 100644 --- a/hw/rtl/core/VX_lsu_mem_path.sv +++ b/hw/rtl/core/VX_lsu_mem_path.sv @@ -14,7 +14,8 @@ `include "VX_define.vh" // The memory path of one LSU block: its memory scheduler, the global/local -// switch, the local-memory adapter, the coalescer and the D-cache adapter. +// switch, the coalescer and the D-cache adapter. Local memory takes the +// switch's local requests whole. // The D-cache port carries one extra tag LSB: requests leave with it clear, and // a response with it set is the core's flush token, not this block's. module VX_lsu_mem_path import VX_gpu_pkg::*; #( @@ -41,7 +42,7 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( output wire busy, `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], + VX_lsu_mem_if.master lmem_if, `endif VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS] ); @@ -118,12 +119,6 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( `ifdef VX_CFG_LMEM_ENABLE - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_lmem_if(); - VX_lmem_switch #( .GLOBAL_OUT_BUF(1), .LOCAL_OUT_BUF(1), @@ -134,24 +129,7 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( .reset (reset), .lsu_in_if (lsu_mem_if), .global_out_if(lsu_dcache_if), - .local_out_if (lsu_lmem_if) - ); - - // A dedicated local-memory adapter per block: sharing one adapter's - // unpack/pack buffers between blocks would deadlock. - VX_lsu_adapter #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH), - .TAG_SEL_BITS (LSU_TAG_WIDTH - UUID_WIDTH), - .ARBITER ("P"), - .REQ_OUT_BUF (3), - .RSP_OUT_BUF (0) - ) lmem_adapter ( - .clk (clk), - .reset (reset), - .lsu_mem_if (lsu_lmem_if), - .mem_bus_if (lmem_bus_if) + .local_out_if (lmem_if) ); `else diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index ccd14130a3..408b3aec8e 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -45,7 +45,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] busy, `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], + VX_lsu_mem_if.master lmem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] ); @@ -75,16 +75,6 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( ) per_block_result_if[BLOCK_SIZE](); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if #( - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) block_lmem_bus_if[NUM_LANES](); - for (genvar j = 0; j < NUM_LANES; ++j) begin : g_lmem_bus_if - `ASSIGN_VX_MEM_BUS_IF (lmem_bus_if[block_idx * NUM_LANES + j], block_lmem_bus_if[j]); - end - `endif - VX_mem_bus_if #( .DATA_SIZE (DCACHE_WORD_SIZE), .TAG_WIDTH (DCACHE_TAG_WIDTH) @@ -115,7 +105,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( .store_empty (store_empty[block_idx]), .busy (busy[block_idx]), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (block_lmem_bus_if), + .lmem_if (lmem_if[block_idx]), `endif .dcache_bus_if (block_dcache_bus_if) ); diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index ef42d48fe1..6c4f5d09b2 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -35,7 +35,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( // the LSU blocks' memory paths `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if.slave lmem_bus_if [LSU_NUM_REQS], + VX_lsu_mem_if.slave lmem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif VX_mem_bus_if.slave lsu_dcache_bus_if [DCACHE_NUM_REQS], @@ -164,27 +164,26 @@ module VX_mem_unit import VX_gpu_pkg::*; #( end - VX_local_mem #( - .INSTANCE_ID (`SFORMATF(("%s-lmem", INSTANCE_ID))), - .SIZE (1 << `VX_CFG_LMEM_LOG_SIZE), - .NUM_REQS (LSU_NUM_REQS), - .NUM_BANKS (`VX_CFG_LMEM_NUM_BANKS), - .WORD_SIZE (LSU_WORD_SIZE), - .ADDR_WIDTH (LMEM_ADDR_WIDTH), - .TAG_WIDTH (LSU_TAG_WIDTH), - .DMA_ENABLE (LMEM_DMA_EN), + VX_lmem_unit #( + .INSTANCE_ID (`SFORMATF(("%s-lmem", INSTANCE_ID))), + .SIZE (1 << `VX_CFG_LMEM_LOG_SIZE), + .NUM_BLOCKS (`VX_CFG_NUM_LSU_BLOCKS), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .NUM_BANKS (`VX_CFG_LMEM_NUM_BANKS), + .WORD_SIZE (LSU_WORD_SIZE), + .ADDR_WIDTH (LMEM_ADDR_WIDTH), + .TAG_WIDTH (LSU_TAG_WIDTH), + .DMA_ENABLE (LMEM_DMA_EN), .DMA_TAG_WIDTH (LMEM_DMA_TAG_WIDTH), - .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), - .READ_MERGE (`VX_CFG_LMEM_READ_MERGE ? `VX_CFG_NUM_LSU_LANES : 0), - .OUT_BUF (3) - ) local_mem ( - .clk (clk), - .reset (reset), + .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) + ) lmem_unit ( + .clk (clk), + .reset (reset), `ifdef PERF_ENABLE - .lmem_perf (lmem_perf), + .lmem_perf (lmem_perf), `endif - .dma_bus_if (lmem_dma_if), - .lsu_bus_if (lmem_bus_if) + .lsu_mem_if (lmem_if), + .dma_bus_if (lmem_dma_if) ); `else diff --git a/hw/rtl/dxa/VX_dxa_pkg.sv b/hw/rtl/dxa/VX_dxa_pkg.sv index 5712b499de..f87414b545 100644 --- a/hw/rtl/dxa/VX_dxa_pkg.sv +++ b/hw/rtl/dxa/VX_dxa_pkg.sv @@ -24,8 +24,8 @@ package VX_dxa_pkg; localparam DXA_LMEM_WORD_SIZE = `VX_CFG_LMEM_NUM_BANKS * (`VX_CFG_XLEN / 8); localparam DXA_LMEM_ADDR_W = LMEM_DMA_ADDR_WIDTH; - // LMEM byte-address width: an LMEM-relative byte offset, same as - // VX_local_mem's ADDR_WIDTH = `CLOG2(SIZE) = `VX_CFG_LMEM_LOG_SIZE. + // LMEM byte-address width: an LMEM-relative byte offset into the + // (1 << `VX_CFG_LMEM_LOG_SIZE)-byte local memory. localparam DXA_SMEM_ADDR_W = `VX_CFG_LMEM_LOG_SIZE; localparam DXA_DESC_SLOT_BITS = `CLOG2(`VX_DCR_DXA_DESC_COUNT); diff --git a/hw/rtl/libs/VX_1rw1r_ram.sv b/hw/rtl/libs/VX_1rw1r_ram.sv new file mode 100644 index 0000000000..fef0aabad1 --- /dev/null +++ b/hw/rtl/libs/VX_1rw1r_ram.sv @@ -0,0 +1,319 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_platform.vh" + +`ifndef ASIC + `define RAM_INITIALIZATION \ + if (INIT_ENABLE != 0) begin : g_init \ + if (INIT_FILE != "") begin : g_file \ + initial $readmemh(INIT_FILE, ram); \ + end else begin : g_value \ + initial begin \ + for (integer i = 0; i < SIZE; ++i) begin : g_i \ + ram[i] = INIT_VALUE; \ + end \ + end \ + end \ + end +`else + `define RAM_INITIALIZATION +`endif + +`ifdef SIMULATION + `define RAM_RESET_BLOCK if (RESET_RAM && reset) begin \ + for (integer i = 0; i < SIZE; ++i) begin \ + ram[i] <= DATAW'(INIT_VALUE); \ + end \ + end else +`else + `define RAM_RESET_BLOCK +`endif + +`define RAM_WRITE_ALL `RAM_RESET_BLOCK \ + if (write_a) begin \ + ram[addr_a] <= wdata_a; \ + end + +`ifdef QUARTUS + `define RAM_ARRAY_WREN reg [WRENW-1:0][WSELW-1:0] ram [0:SIZE-1]; + `define RAM_WRITE_WREN `RAM_RESET_BLOCK \ + if (write_a) begin \ + for (integer i = 0; i < WRENW; ++i) begin \ + if (wren_a[i]) begin \ + ram[addr_a][i] <= wdata_a[i * WSELW +: WSELW]; \ + end \ + end \ + end +`else + `define RAM_ARRAY_WREN reg [DATAW-1:0] ram [0:SIZE-1]; + `define RAM_WRITE_WREN `RAM_RESET_BLOCK \ + if (write_a) begin \ + for (integer i = 0; i < WRENW; ++i) begin \ + if (wren_a[i]) begin \ + ram[addr_a][i * WSELW +: WSELW] <= wdata_a[i * WSELW +: WSELW]; \ + end \ + end \ + end +`endif + +// A RAM with one read/write port (A) and one read-only port (B), both with a +// registered read: the two ports of a true dual-port block RAM. Port A reads +// before it writes. A port B read of the word port A writes in the same cycle +// returns an undefined value on block RAM; the caller must not issue one. +// +// The ASIC libraries have no read/write + read macro, so on ASIC the RAM is two +// VX_dp_ram copies, one per read port, that both take every write: each maps +// onto the VX_dp_ram_asic macro the synthesis flows and their SRAM cost model +// already know. +`TRACING_OFF +module VX_1rw1r_ram #( + parameter DATAW = 1, + parameter SIZE = 1, + parameter WRENW = 1, + parameter LUTRAM = 0, + parameter RESET_RAM = 0, + parameter INIT_ENABLE = 0, + parameter INIT_FILE = "", + parameter [DATAW-1:0] INIT_VALUE = 0, + parameter ADDRW = `LOG2UP(SIZE) +) ( + input wire clk, + input wire reset, + + input wire read_a, + input wire write_a, + input wire [WRENW-1:0] wren_a, + input wire [ADDRW-1:0] addr_a, + input wire [DATAW-1:0] wdata_a, + output wire [DATAW-1:0] rdata_a, + + input wire read_b, + input wire [ADDRW-1:0] addr_b, + output wire [DATAW-1:0] rdata_b +); + localparam WSELW = DATAW / WRENW; + + `STATIC_ASSERT(!(WRENW * WSELW != DATAW), ("invalid parameter")) + +`ifdef SYNTHESIS +`ifdef ASIC + `UNUSED_PARAM (LUTRAM) + VX_dp_ram #( + .DATAW (DATAW), + .SIZE (SIZE), + .WRENW (WRENW), + .OUT_REG (1), + .RDW_MODE ("R"), + .RESET_RAM (RESET_RAM), + .INIT_ENABLE (INIT_ENABLE), + .INIT_FILE (INIT_FILE), + .INIT_VALUE (INIT_VALUE) + ) ram_a ( + .clk (clk), + .reset (reset), + .read (read_a), + .write (write_a), + .wren (wren_a), + .waddr (addr_a), + .wdata (wdata_a), + .raddr (addr_a), + .rdata (rdata_a) + ); + VX_dp_ram #( + .DATAW (DATAW), + .SIZE (SIZE), + .WRENW (WRENW), + .OUT_REG (1), + .RDW_MODE ("R"), + .RESET_RAM (RESET_RAM), + .INIT_ENABLE (INIT_ENABLE), + .INIT_FILE (INIT_FILE), + .INIT_VALUE (INIT_VALUE) + ) ram_b ( + .clk (clk), + .reset (reset), + .read (read_b), + .write (write_a), + .wren (wren_a), + .waddr (addr_a), + .wdata (wdata_a), + .raddr (addr_b), + .rdata (rdata_b) + ); +`else + localparam FORCE_BRAM = !LUTRAM && `FORCE_BRAM(SIZE, DATAW); + if (FORCE_BRAM) begin : g_bram + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_WREN + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end else begin : g_no_wren + `UNUSED_VAR (wren_a) + `NO_RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_ALL + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end + end else if (LUTRAM) begin : g_lutram + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `USE_FAST_BRAM `RAM_ARRAY_WREN + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_WREN + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end else begin : g_no_wren + `UNUSED_VAR (wren_a) + `NO_RW_RAM_CHECK `USE_FAST_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_ALL + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end + end else begin : g_auto + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_WREN + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end else begin : g_no_wren + `UNUSED_VAR (wren_a) + `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_ALL + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end + end +`endif +`else + // simulation + `UNUSED_PARAM (LUTRAM) + reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + + if (WRENW != 1) begin : g_wren + reg [DATAW-1:0] wdata_n; + always @(*) begin + wdata_n = ram[addr_a]; + for (integer i = 0; i < WRENW; ++i) begin + if (wren_a[i]) begin + wdata_n[i * WSELW +: WSELW] = wdata_a[i * WSELW +: WSELW]; + end + end + end + always @(posedge clk) begin + `RAM_RESET_BLOCK + if (write_a) begin + ram[addr_a] <= wdata_n; + end + end + end else begin : g_no_wren + `UNUSED_VAR (wren_a) + always @(posedge clk) begin + `RAM_WRITE_ALL + end + end + + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; +`endif + +endmodule +`TRACING_ON diff --git a/hw/rtl/mem/VX_lmem_engine.sv b/hw/rtl/mem/VX_lmem_engine.sv new file mode 100644 index 0000000000..ef083fc8e0 --- /dev/null +++ b/hw/rtl/mem/VX_lmem_engine.sv @@ -0,0 +1,442 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One pass engine of the warp-wide local memory. It takes a whole warp request +// and gives it to the banks in passes: in a pass every bank goes to the lowest +// pending lane that maps to it, and a plain read also serves every pending lane +// reading that lane's word. Lanes left over replay in the next passes, and the +// request retires before the next one starts. +// +// S0 the engine's input queue of two requests; the head's lanes' words are +// compared pairwise +// S1 the request, its comparison matrices, its pending lanes and the next +// pass's bank winners; a pass moves the winners' rows and payloads into +// the bank registers +// then the banks access their SRAM, two cycles after the pass each served +// lane takes its bank's word into the response register, and from there the +// response enters the response queue, or passes it when it is empty, on its +// way to its block's response register. +// +// The pair matrices are computed once per request, and each pass's bank +// winners and whether it is the last are registered ahead of it, so nothing +// combinational sits in front of the bank registers but the winners' row and +// payload mux. The first pass's winners come from S0; a replay finds the next +// winners in the cycle after a pass, so a conflicting request replays at one +// pass per two cycles. The last pass frees S1 in its own cycle. +module VX_lmem_engine import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 4, + parameter NUM_BANKS = 4, + parameter WORD_SIZE = 4, + // word address width + parameter ADDR_WIDTH = 12, + parameter TAG_WIDTH = 8, + // per-lane write payload carried to the banks (data, byte enables, atomic sideband) + parameter PAYLOAD_WIDTH = 1, + // request attributes common to every lane + parameter ATTR_WIDTH = 1, + parameter NUM_BLOCKS = 1, + parameter RSP_QUEUE_SIZE = 8, + + parameter WORD_WIDTH = WORD_SIZE * 8, + parameter BLOCK_WIDTH = `UP(`CLOG2(NUM_BLOCKS)), + parameter DONE_WIDTH = `CLOG2(NUM_BLOCKS + 1), + parameter BANK_ADDR_WIDTH = ADDR_WIDTH - `CLOG2(NUM_BANKS) +) ( + input wire clk, + input wire reset, + + // request + input wire req_valid, + input wire [BLOCK_WIDTH-1:0] req_block, + // reads and atomics respond, stores do not + input wire req_rw, + // a plain read: its lanes of one word share a bank access + input wire req_read, + input wire req_amo, + input wire [ATTR_WIDTH-1:0] req_attr, + input wire [NUM_LANES-1:0] req_mask, + input wire [NUM_LANES-1:0][ADDR_WIDTH-1:0] req_addr, + input wire [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] req_payload, + input wire [TAG_WIDTH-1:0] req_tag, + output wire req_ready, + + // the engine holds a request; the one it holds is a store or an atomic + output wire busy, + output wire holds_write, + + // the banks are free for a pass this cycle + input wire pass_en, + // an atomic pass went out + output wire pass_amo, + + // bank requests, registered + output wire [NUM_BANKS-1:0] bank_valid, + output wire bank_rw, + output wire bank_amo, + output wire [ATTR_WIDTH-1:0] bank_attr, + output wire [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] bank_addr, + output wire [NUM_BANKS-1:0][PAYLOAD_WIDTH-1:0] bank_payload, + + // the banks' read data, two cycles after the pass + input wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] bank_rdata, + + // the response queue's head, taken into a block's response register; + // rsp_done counts this engine's responses the blocks took this cycle + output wire rsp_valid, + output wire [BLOCK_WIDTH-1:0] rsp_block, + output wire [NUM_LANES-1:0] rsp_mask, + output wire [NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data, + output wire [TAG_WIDTH-1:0] rsp_tag, + input wire rsp_take, + input wire [DONE_WIDTH-1:0] rsp_done, + + // perf: lanes served by this cycle's pass, whether they store, and + // whether lanes were left over + output wire [NUM_LANES-1:0] perf_served, + output wire perf_store, + output wire perf_replay +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS); + localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); + localparam CREDIT_WIDTH = `CLOG2(RSP_QUEUE_SIZE + 1); + localparam RSP_DATAW = BLOCK_WIDTH + NUM_LANES + NUM_LANES * WORD_WIDTH + TAG_WIDTH; + + `STATIC_ASSERT(ADDR_WIDTH > BANK_SEL_BITS, ("invalid parameter")) + + // S0: the engine's input queue. Holding two requests, it takes one in any + // cycle that starts with at most one, so its ready is a register and the + // head stays put while S1 is busy. + + localparam S0_DATAW = BLOCK_WIDTH + 3 + ATTR_WIDTH + NUM_LANES + + NUM_LANES * (ADDR_WIDTH + PAYLOAD_WIDTH) + TAG_WIDTH; + + wire s0_valid; + wire [BLOCK_WIDTH-1:0] s0_block; + wire s0_rw; + wire s0_read; + wire s0_amo; + wire [ATTR_WIDTH-1:0] s0_attr; + wire [NUM_LANES-1:0] s0_mask; + wire [NUM_LANES-1:0][ADDR_WIDTH-1:0] s0_addr; + wire [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] s0_payload; + wire [TAG_WIDTH-1:0] s0_tag; + wire s1_free; + + VX_elastic_buffer #( + .DATAW (S0_DATAW), + .SIZE (2), + .OUT_REG (1) + ) s0_buf ( + .clk (clk), + .reset (reset), + .valid_in (req_valid), + .ready_in (req_ready), + .data_in ({req_block, req_rw, req_read, req_amo, req_attr, req_mask, req_addr, req_payload, req_tag}), + .data_out ({s0_block, s0_rw, s0_read, s0_amo, s0_attr, s0_mask, s0_addr, s0_payload, s0_tag}), + .valid_out (s0_valid), + .ready_out (s1_free) + ); + + // pairwise comparison: same_bank[i][j] and same_word[i][j] for lower lanes j < i + + wire [NUM_LANES-1:0][NUM_LANES-1:0] s0_same_bank; + wire [NUM_LANES-1:0][NUM_LANES-1:0] s0_same_word; + wire [NUM_LANES-1:0][NUM_LANES-1:0] s0_conflict; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_cmp_i + for (genvar j = 0; j < NUM_LANES; ++j) begin : g_cmp_j + if (j < i) begin : g_lower + if (BANK_SEL_BITS != 0) begin : g_banks + assign s0_same_bank[i][j] = (s0_addr[i][BANK_SEL_BITS-1:0] == s0_addr[j][BANK_SEL_BITS-1:0]); + end else begin : g_one_bank + assign s0_same_bank[i][j] = 1'b1; + end + assign s0_same_word[i][j] = (s0_addr[i] == s0_addr[j]); + // two lanes that need different passes + assign s0_conflict[i][j] = s0_mask[i] && s0_mask[j] && s0_same_bank[i][j] + && ~(s0_read && s0_same_word[i][j]); + end else begin : g_none + assign s0_same_bank[i][j] = 1'b0; + assign s0_same_word[i][j] = 1'b0; + assign s0_conflict[i][j] = 1'b0; + end + end + end + wire s0_one_pass = ~(| s0_conflict); + + // the first pass's bank winners: each bank's lowest lane + wire [NUM_LANES-1:0] s0_first; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_s0_first + assign s0_first[i] = s0_mask[i] && ~(| (s0_mask & s0_same_bank[i])); + end + + // S1 + + reg s1_valid; + // the bank winners of the next pass are in s1_winners + reg s1_ready; + // the next pass serves every pending lane + reg s1_last; + reg [NUM_LANES-1:0] s1_winners; + reg [BLOCK_WIDTH-1:0] s1_block; + reg s1_rw; + reg s1_read; + reg s1_amo; + reg [ATTR_WIDTH-1:0] s1_attr; + reg [NUM_LANES-1:0] s1_pending; + reg [NUM_LANES-1:0][ADDR_WIDTH-1:0] s1_addr; + reg [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] s1_payload; + reg [TAG_WIDTH-1:0] s1_tag; + reg [NUM_LANES-1:0][NUM_LANES-1:0] s1_same_bank; + reg [NUM_LANES-1:0][NUM_LANES-1:0] s1_same_word; + + // A pass serves the registered bank winners and the pending read lanes of + // their words. After a pass that leaves lanes pending, the next cycle finds + // the next winners among them and whether the pass after serves them all: + // a request replays at one pass per two cycles, and its last pass frees S1. + + wire [NUM_LANES-1:0] served; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_served + assign served[i] = s1_winners[i] || (s1_read && s1_pending[i] && (| (s1_winners & s1_same_word[i]))); + end + + wire [NUM_LANES-1:0] winners_n; + wire [NUM_LANES-1:0][NUM_LANES-1:0] pending_conflict; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_winners_n + assign winners_n[i] = s1_pending[i] && ~(| (s1_pending & s1_same_bank[i])); + for (genvar j = 0; j < NUM_LANES; ++j) begin : g_conflict + assign pending_conflict[i][j] = s1_pending[i] && s1_pending[j] && s1_same_bank[i][j] + && ~(s1_read && s1_same_word[i][j]); + end + end + wire last_n = ~(| pending_conflict); + + reg [CREDIT_WIDTH-1:0] credits; + wire has_rsp = ~s1_rw; + wire pass_fire = s1_valid && s1_ready && pass_en && (~has_rsp || (credits != '0)); + wire s1_release = pass_fire && s1_last; + assign s1_free = ~s1_valid || s1_release; + wire s1_load = s0_valid && s1_free; + + always @(posedge clk) begin + if (reset) begin + s1_valid <= 1'b0; + end else if (s1_load) begin + s1_valid <= 1'b1; + end else if (s1_release) begin + s1_valid <= 1'b0; + end + if (s1_load) begin + s1_ready <= 1'b1; + s1_last <= s0_one_pass; + s1_winners <= s0_first; + s1_pending <= s0_mask; + s1_block <= s0_block; + s1_rw <= s0_rw; + s1_read <= s0_read; + s1_amo <= s0_amo; + s1_attr <= s0_attr; + s1_addr <= s0_addr; + s1_payload <= s0_payload; + s1_tag <= s0_tag; + s1_same_bank <= s0_same_bank; + s1_same_word <= s0_same_word; + end else if (pass_fire) begin + s1_ready <= 1'b0; + s1_pending <= s1_pending & ~served; + end else if (~s1_ready) begin + s1_ready <= 1'b1; + s1_last <= last_n; + s1_winners <= winners_n; + end + end + + // stores and atomics held between S0's two entries and S1 + reg [1:0] num_writes; + wire write_in = req_valid && req_ready && ~req_read; + wire write_out = s1_release && ~s1_read; + always @(posedge clk) begin + if (reset) begin + num_writes <= '0; + end else begin + num_writes <= num_writes + 2'(write_in) - 2'(write_out); + end + end + + assign busy = s0_valid || s1_valid; + assign holds_write = (num_writes != '0); + assign pass_amo = pass_fire && s1_amo; + + // bank registers: each bank takes its first lane's row and payload + + wire [NUM_LANES-1:0][BANK_SEL_WIDTH-1:0] s1_bank; + wire [NUM_LANES-1:0][BANK_ADDR_WIDTH+PAYLOAD_WIDTH-1:0] s1_bank_data; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lane_addr + if (BANK_SEL_BITS != 0) begin : g_banks + assign s1_bank[i] = s1_addr[i][BANK_SEL_BITS-1:0]; + end else begin : g_one_bank + assign s1_bank[i] = '0; + end + assign s1_bank_data[i] = {s1_addr[i][BANK_SEL_BITS +: BANK_ADDR_WIDTH], s1_payload[i]}; + end + + reg [NUM_BANKS-1:0] bank_valid_r; + reg bank_rw_r; + reg bank_amo_r; + reg [ATTR_WIDTH-1:0] bank_attr_r; + reg [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] bank_addr_r; + reg [NUM_BANKS-1:0][PAYLOAD_WIDTH-1:0] bank_payload_r; + + for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_banks + wire [NUM_LANES-1:0] sel; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_sel + assign sel[i] = s1_winners[i] && (s1_bank[i] == BANK_SEL_WIDTH'(b)); + end + + wire [BANK_ADDR_WIDTH-1:0] sel_addr; + wire [PAYLOAD_WIDTH-1:0] sel_payload; + + VX_onehot_mux #( + .DATAW (BANK_ADDR_WIDTH + PAYLOAD_WIDTH), + .N (NUM_LANES) + ) bank_mux ( + .data_in (s1_bank_data), + .sel_in (sel), + .data_out ({sel_addr, sel_payload}) + ); + + always @(posedge clk) begin + if (reset) begin + bank_valid_r[b] <= 1'b0; + end else begin + bank_valid_r[b] <= pass_fire && (| sel); + end + bank_addr_r[b] <= sel_addr; + bank_payload_r[b] <= sel_payload; + end + end + + always @(posedge clk) begin + bank_rw_r <= s1_rw; + bank_amo_r <= s1_amo; + bank_attr_r <= s1_attr; + end + + assign bank_valid = bank_valid_r; + assign bank_rw = bank_rw_r; + assign bank_amo = bank_amo_r; + assign bank_attr = bank_attr_r; + assign bank_addr = bank_addr_r; + assign bank_payload = bank_payload_r; + + // response: the pass's lanes and banks follow it to the SRAM output + + wire m2_valid; + wire [BLOCK_WIDTH-1:0] m2_block; + wire [NUM_LANES-1:0] m2_mask; + wire [NUM_LANES-1:0][BANK_SEL_WIDTH-1:0] m2_bank; + wire [TAG_WIDTH-1:0] m2_tag; + + VX_pipe_register #( + .DATAW (1 + BLOCK_WIDTH + NUM_LANES + NUM_LANES * BANK_SEL_WIDTH + TAG_WIDTH), + .RESETW (1), + .DEPTH (2) + ) rsp_meta_pipe ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({pass_fire && has_rsp, s1_block, served, s1_bank, s1_tag}), + .data_out ({m2_valid, m2_block, m2_mask, m2_bank, m2_tag}) + ); + + wire [NUM_LANES-1:0][WORD_WIDTH-1:0] m2_data; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rsp_data + assign m2_data[i] = bank_rdata[m2_bank[i]]; + end + + // The lane mux reads the SRAM outputs; its result is registered before it + // travels to the response queue. + wire m3_valid; + wire [BLOCK_WIDTH-1:0] m3_block; + wire [NUM_LANES-1:0] m3_mask; + wire [NUM_LANES-1:0][WORD_WIDTH-1:0] m3_data; + wire [TAG_WIDTH-1:0] m3_tag; + + VX_pipe_register #( + .DATAW (RSP_DATAW + 1), + .RESETW (1) + ) rsp_data_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({m2_valid, m2_block, m2_mask, m2_data, m2_tag}), + .data_out ({m3_valid, m3_block, m3_mask, m3_data, m3_tag}) + ); + + // The queue has a slot for every pass that responds and is in flight + // toward its block, so it never fills. A response that finds the queue + // empty passes straight on to its block's register. + wire rsp_empty; + wire [RSP_DATAW-1:0] rsp_head; + wire rsp_push = m3_valid && ~(rsp_empty && rsp_take); + wire rsp_pop = ~rsp_empty && rsp_take; + + VX_fifo_queue #( + .DATAW (RSP_DATAW), + .DEPTH (RSP_QUEUE_SIZE), + .OUT_REG (0), + .LUTRAM (1) + ) rsp_queue ( + .clk (clk), + .reset (reset), + .push (rsp_push), + .pop (rsp_pop), + .data_in ({m3_block, m3_mask, m3_data, m3_tag}), + .data_out (rsp_head), + .empty (rsp_empty), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + assign rsp_valid = ~rsp_empty || m3_valid; + assign {rsp_block, rsp_mask, rsp_data, rsp_tag} = rsp_empty ? {m3_block, m3_mask, m3_data, m3_tag} : rsp_head; + + // a response's slot is free again once its block has taken it on + wire credit_take = pass_fire && has_rsp; + always @(posedge clk) begin + if (reset) begin + credits <= CREDIT_WIDTH'(RSP_QUEUE_SIZE); + end else begin + credits <= credits - CREDIT_WIDTH'(credit_take) + CREDIT_WIDTH'(rsp_done); + end + end + + assign perf_served = pass_fire ? served : '0; + assign perf_store = s1_rw; + assign perf_replay = pass_fire && ~s1_last; + +`ifdef DBG_TRACE_MEM + always @(posedge clk) begin + if (pass_fire) begin + `TRACE(2, ("%t: %s pass: rw=%b, amo=%b, pending=%b, served=%b, tag=0x%0h\n", + $time, INSTANCE_ID, s1_rw, s1_amo, s1_pending, served, s1_tag)) + end + end +`endif + +endmodule diff --git a/hw/rtl/mem/VX_lmem_unit.sv b/hw/rtl/mem/VX_lmem_unit.sv new file mode 100644 index 0000000000..a48400fb16 --- /dev/null +++ b/hw/rtl/mem/VX_lmem_unit.sv @@ -0,0 +1,961 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Warp-wide local memory. It takes whole warp requests from the LSU blocks and +// serves each in bank-conflict-free passes (VX_lmem_engine): same-word reads +// are broadcast to every lane reading the word, conflicting lanes replay, and +// requests retire in order, so every lane's accesses reach the banks in program +// order. +// +// With several LSU blocks a second engine, for plain reads only, works the +// banks' second port, so two warps' reads go through each cycle. Stores and +// atomics use the first engine alone: one enters it only once the read engine +// has drained, and the read engine takes no read while a store or atomic sits in +// the first engine, while one waited at an input the cycle before, or in a cycle +// the first engine may be taking one. A write therefore never meets a read of +// the other port in a bank, and keeps its order against every read. +// +// The DMA port (DXA writes, TCU reads) takes whole bank rows. A DMA request seen +// in one cycle gets the banks the next: no pass is issued in a cycle a DMA +// request is seen, and it is accepted in the cycle that follows. An atomic owns +// its bank for three cycles (read, capture, write back), so no pass or DMA access +// is issued in the two cycles after an atomic pass. +module VX_lmem_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + + // Size in bytes + parameter SIZE = (1024*16), + // Number of LSU blocks + parameter NUM_BLOCKS = 1, + // Lanes of a warp request + parameter NUM_LANES = 4, + // Number of banks + parameter NUM_BANKS = 4, + // Size of a word in bytes + parameter WORD_SIZE = `VX_CFG_XLEN/8, + // Word address width + parameter ADDR_WIDTH = `CLOG2(SIZE / WORD_SIZE), + // Request tag size + parameter TAG_WIDTH = 16, + + // Enable DMA port + parameter DMA_ENABLE = 0, + parameter DMA_TAG_WIDTH = 1, + + // Synthesize atomic-op + LR/SC logic in the banks + parameter AMO_ENABLE = 0 +) ( + input wire clk, + input wire reset, + +`ifdef PERF_ENABLE + output lmem_perf_t lmem_perf, +`endif + + // LSU blocks' warp requests + VX_lsu_mem_if.slave lsu_mem_if [NUM_BLOCKS], + + // DMA read/write port + VX_mem_bus_if.slave dma_bus_if +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam WORD_WIDTH = WORD_SIZE * 8; + localparam NUM_WORDS = SIZE / WORD_SIZE; + localparam WORDS_PER_BANK = NUM_WORDS / NUM_BANKS; + localparam BANK_ADDR_WIDTH = `CLOG2(WORDS_PER_BANK); + localparam BLOCK_BITS = `CLOG2(NUM_BLOCKS); + localparam BLOCK_WIDTH = `UP(BLOCK_BITS); + localparam DUAL = (NUM_BLOCKS > 1); + // A pass's response reaches its block's response register four cycles + // after the pass at the earliest, and its slot is free for a pass the cycle + // after the block takes it: eight slots keep one pass per cycle going. + localparam RSP_QUEUE_SIZE = 8; + localparam DONE_WIDTH = `CLOG2(NUM_BLOCKS + 1); + localparam AMO_OP_WIDTH = $bits(amo_op_e); + // Compare-and-swap adds a comparand: the request's data word already + // carries the swap value, so the third operand needs its own room. +`ifdef VX_CFG_EXT_ZACAS_ENABLE + localparam AMO_CMP_WIDTH = `VX_CFG_XLEN; +`else + localparam AMO_CMP_WIDTH = 0; +`endif + // per lane: the hart id that holds a reservation, and the comparand + localparam LANE_SB_WIDTH = AMO_ENABLE ? (HART_ID_WIDTH + AMO_CMP_WIDTH) : 0; + localparam PAYLOAD_WIDTH = WORD_WIDTH + WORD_SIZE + LANE_SB_WIDTH; + // common to all lanes: the atomic op and its signedness + localparam ATTR_WIDTH = AMO_OP_WIDTH + 1; + + `STATIC_ASSERT(ADDR_WIDTH == (BANK_ADDR_WIDTH + `CLOG2(NUM_BANKS)), ("invalid parameter")) + `STATIC_ASSERT(`IS_POW2(NUM_BANKS), ("invalid parameter")) + + // the blocks' requests + + wire [NUM_BLOCKS-1:0] in_valid; + wire [NUM_BLOCKS-1:0] in_rw; + wire [NUM_BLOCKS-1:0] in_amo; + wire [NUM_BLOCKS-1:0] in_read; + wire [NUM_BLOCKS-1:0][ATTR_WIDTH-1:0] in_attr; + wire [NUM_BLOCKS-1:0][NUM_LANES-1:0] in_mask; + wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][ADDR_WIDTH-1:0] in_addr; + wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] in_payload; + wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] in_tag; + wire [NUM_BLOCKS-1:0] in_ready; + + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_in + // the atomic op is the same on every lane of a request + mem_bus_attr_t attr0; + assign attr0 = mem_bus_attr_t'(lsu_mem_if[k].req_data.user[0]); + assign in_valid[k] = lsu_mem_if[k].req_valid; + assign in_rw[k] = lsu_mem_if[k].req_data.rw; + assign in_amo[k] = AMO_ENABLE ? attr0.amo.amo_valid : 1'b0; + assign in_read[k] = ~in_rw[k] && ~in_amo[k]; + assign in_attr[k] = AMO_ENABLE ? {attr0.amo.amo_op, attr0.amo.amo_unsigned} : '0; + assign in_mask[k] = lsu_mem_if[k].req_data.mask; + assign in_tag[k] = lsu_mem_if[k].req_data.tag; + `UNUSED_VAR (attr0) + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lanes + assign in_addr[k][i] = lsu_mem_if[k].req_data.addr[i][ADDR_WIDTH-1:0]; + if (AMO_ENABLE != 0) begin : g_amo + mem_bus_attr_t lane_attr; + assign lane_attr = mem_bus_attr_t'(lsu_mem_if[k].req_data.user[i]); + `ifdef VX_CFG_EXT_ZACAS_ENABLE + assign in_payload[k][i] = {lane_attr.amo.hart_id, lane_attr.amo.amo_cmp, + lsu_mem_if[k].req_data.byteen[i], lsu_mem_if[k].req_data.data[i]}; + `else + assign in_payload[k][i] = {lane_attr.amo.hart_id, + lsu_mem_if[k].req_data.byteen[i], lsu_mem_if[k].req_data.data[i]}; + `endif + `UNUSED_VAR (lane_attr) + end else begin : g_no_amo + assign in_payload[k][i] = {lsu_mem_if[k].req_data.byteen[i], lsu_mem_if[k].req_data.data[i]}; + `UNUSED_VAR (lsu_mem_if[k].req_data.user[i]) + end + `UNUSED_VAR (lsu_mem_if[k].req_data.addr[i]) + end + assign lsu_mem_if[k].req_ready = in_ready[k]; + end + + // the engines + + wire pass_en; + wire a_pass_amo; + + wire a_req_valid; + wire a_req_ready; + wire a_busy; + wire a_holds_write; + wire [BLOCK_WIDTH-1:0] a_req_block; + wire b_req_valid; + wire b_req_ready; + wire [BLOCK_WIDTH-1:0] b_req_block; + + wire [NUM_BANKS-1:0] a_bank_valid; + wire a_bank_rw; + wire a_bank_amo; + wire [ATTR_WIDTH-1:0] a_bank_attr; + wire [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] a_bank_addr; + wire [NUM_BANKS-1:0][PAYLOAD_WIDTH-1:0] a_bank_payload; + wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] a_bank_rdata; + + wire a_rsp_valid; + wire a_rsp_take; + wire [DONE_WIDTH-1:0] a_rsp_done; + wire [BLOCK_WIDTH-1:0] a_rsp_block; + wire [NUM_LANES-1:0] a_rsp_mask; + wire [NUM_LANES-1:0][WORD_WIDTH-1:0] a_rsp_data; + wire [TAG_WIDTH-1:0] a_rsp_tag; + + wire [NUM_LANES-1:0] a_perf_served; + wire a_perf_store; + wire a_perf_replay; + + wire b_busy; + wire [NUM_BANKS-1:0] b_bank_valid; + wire [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] b_bank_addr; + wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] b_bank_rdata; + + wire b_rsp_valid; + wire b_rsp_take; + wire [DONE_WIDTH-1:0] b_rsp_done; + wire [BLOCK_WIDTH-1:0] b_rsp_block; + wire [NUM_LANES-1:0] b_rsp_mask; + wire [NUM_LANES-1:0][WORD_WIDTH-1:0] b_rsp_data; + wire [TAG_WIDTH-1:0] b_rsp_tag; + + wire [NUM_LANES-1:0] b_perf_served; + wire b_perf_replay; + + // Block arbitration. The ready each block sees comes from registers only: + // each engine's block is chosen a cycle ahead, from the requests waiting + // then, and the engine takes it if it is still there. A block that was + // just served may present a new request of either kind, so a choice of it + // counts as a store: the first engine takes a store only while the read + // engine is idle, and the read engine takes nothing in that cycle. + + wire [NUM_BLOCKS-1:0] in_fire = in_valid & in_ready; + + if (DUAL) begin : g_dual_arb + + reg a_sel_valid_r; + reg [BLOCK_WIDTH-1:0] a_sel_r; + reg a_sel_read_r; + reg b_sel_valid_r; + reg [BLOCK_WIDTH-1:0] b_sel_r; + reg write_waiting_r; + + wire a_take = a_sel_valid_r && a_req_ready && (a_sel_read_r || ~b_busy); + wire b_take = b_sel_valid_r && b_req_ready && ~a_holds_write && ~write_waiting_r + && ~(a_take && ~a_sel_read_r); + + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_ready + assign in_ready[k] = (a_take && (a_sel_r == BLOCK_WIDTH'(k))) + || (b_take && (b_sel_r == BLOCK_WIDTH'(k))); + end + + assign a_req_valid = a_take && in_valid[a_sel_r]; + assign a_req_block = a_sel_r; + assign b_req_valid = b_take && in_valid[b_sel_r]; + assign b_req_block = b_sel_r; + + // the next cycle's choices, from the requests waiting now + wire [NUM_BLOCKS-1:0] known_read = in_valid & in_read & ~in_fire; + wire b_busy_n = b_busy || (b_req_valid && b_req_ready); + wire [NUM_BLOCKS-1:0] a_eligible = known_read | (in_valid & {NUM_BLOCKS{~b_busy_n}}); + + wire a_next_valid; + wire [BLOCK_WIDTH-1:0] a_next; + wire [NUM_BLOCKS-1:0] a_next_onehot; + + VX_generic_arbiter #( + .NUM_REQS (NUM_BLOCKS), + .TYPE ("R") + ) a_arbiter ( + .clk (clk), + .reset (reset), + .requests (a_eligible), + .grant_index (a_next), + .grant_onehot (a_next_onehot), + .grant_valid (a_next_valid), + .grant_ready (1'b1) + ); + + wire [NUM_BLOCKS-1:0] b_eligible = known_read & ~a_next_onehot; + + wire b_next_valid; + wire [BLOCK_WIDTH-1:0] b_next; + wire [NUM_BLOCKS-1:0] b_next_onehot; + + VX_generic_arbiter #( + .NUM_REQS (NUM_BLOCKS), + .TYPE ("R") + ) b_arbiter ( + .clk (clk), + .reset (reset), + .requests (b_eligible), + .grant_index (b_next), + .grant_onehot (b_next_onehot), + .grant_valid (b_next_valid), + .grant_ready (1'b1) + ); + `UNUSED_VAR (b_next_onehot) + + always @(posedge clk) begin + if (reset) begin + a_sel_valid_r <= 1'b0; + b_sel_valid_r <= 1'b0; + write_waiting_r <= 1'b0; + end else begin + a_sel_valid_r <= a_next_valid; + b_sel_valid_r <= b_next_valid; + write_waiting_r <= (| (in_valid & ~in_read & ~in_fire)); + end + a_sel_r <= a_next; + a_sel_read_r <= known_read[a_next]; + b_sel_r <= b_next; + end + + end else begin : g_single_arb + + // one block: its request goes straight to the first engine + assign a_req_valid = in_valid[0]; + assign a_req_block = '0; + assign in_ready[0] = a_req_ready; + assign b_req_valid = 1'b0; + assign b_req_block = '0; + `UNUSED_VAR (in_fire) + `UNUSED_VAR (b_req_ready) + `UNUSED_VAR (b_busy) + + end + + VX_lmem_engine #( + .INSTANCE_ID (`SFORMATF(("%s-engine0", INSTANCE_ID))), + .NUM_LANES (NUM_LANES), + .NUM_BANKS (NUM_BANKS), + .WORD_SIZE (WORD_SIZE), + .ADDR_WIDTH (ADDR_WIDTH), + .TAG_WIDTH (TAG_WIDTH), + .PAYLOAD_WIDTH (PAYLOAD_WIDTH), + .ATTR_WIDTH (ATTR_WIDTH), + .NUM_BLOCKS (NUM_BLOCKS), + .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE) + ) engine_a ( + .clk (clk), + .reset (reset), + .req_valid (a_req_valid), + .req_block (a_req_block), + .req_rw (in_rw[a_req_block]), + .req_read (in_read[a_req_block]), + .req_amo (in_amo[a_req_block]), + .req_attr (in_attr[a_req_block]), + .req_mask (in_mask[a_req_block]), + .req_addr (in_addr[a_req_block]), + .req_payload (in_payload[a_req_block]), + .req_tag (in_tag[a_req_block]), + .req_ready (a_req_ready), + .busy (a_busy), + .holds_write (a_holds_write), + .pass_en (pass_en), + .pass_amo (a_pass_amo), + .bank_valid (a_bank_valid), + .bank_rw (a_bank_rw), + .bank_amo (a_bank_amo), + .bank_attr (a_bank_attr), + .bank_addr (a_bank_addr), + .bank_payload (a_bank_payload), + .bank_rdata (a_bank_rdata), + .rsp_valid (a_rsp_valid), + .rsp_block (a_rsp_block), + .rsp_mask (a_rsp_mask), + .rsp_data (a_rsp_data), + .rsp_tag (a_rsp_tag), + .rsp_take (a_rsp_take), + .rsp_done (a_rsp_done), + .perf_served (a_perf_served), + .perf_store (a_perf_store), + .perf_replay (a_perf_replay) + ); + `UNUSED_VAR (a_busy) + + if (DUAL) begin : g_read_engine + + wire [NUM_LANES-1:0][0:0] b_req_payload = '0; + wire b_bank_rw; + wire b_bank_amo; + wire b_holds_write; + wire b_pass_amo; + wire b_perf_store; + wire [ATTR_WIDTH-1:0] b_bank_attr; + wire [NUM_BANKS-1:0][0:0] b_bank_payload; + + VX_lmem_engine #( + .INSTANCE_ID (`SFORMATF(("%s-engine1", INSTANCE_ID))), + .NUM_LANES (NUM_LANES), + .NUM_BANKS (NUM_BANKS), + .WORD_SIZE (WORD_SIZE), + .ADDR_WIDTH (ADDR_WIDTH), + .TAG_WIDTH (TAG_WIDTH), + .PAYLOAD_WIDTH (1), + .ATTR_WIDTH (ATTR_WIDTH), + .NUM_BLOCKS (NUM_BLOCKS), + .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE) + ) engine_b ( + .clk (clk), + .reset (reset), + .req_valid (b_req_valid), + .req_block (b_req_block), + .req_rw (1'b0), + .req_read (1'b1), + .req_amo (1'b0), + .req_attr ('0), + .req_mask (in_mask[b_req_block]), + .req_addr (in_addr[b_req_block]), + .req_payload (b_req_payload), + .req_tag (in_tag[b_req_block]), + .req_ready (b_req_ready), + .busy (b_busy), + .holds_write (b_holds_write), + .pass_en (pass_en), + .pass_amo (b_pass_amo), + .bank_valid (b_bank_valid), + .bank_rw (b_bank_rw), + .bank_amo (b_bank_amo), + .bank_attr (b_bank_attr), + .bank_addr (b_bank_addr), + .bank_payload (b_bank_payload), + .bank_rdata (b_bank_rdata), + .rsp_valid (b_rsp_valid), + .rsp_block (b_rsp_block), + .rsp_mask (b_rsp_mask), + .rsp_data (b_rsp_data), + .rsp_tag (b_rsp_tag), + .rsp_take (b_rsp_take), + .rsp_done (b_rsp_done), + .perf_served (b_perf_served), + .perf_store (b_perf_store), + .perf_replay (b_perf_replay) + ); + `UNUSED_VAR ({b_bank_rw, b_bank_amo, b_holds_write, b_pass_amo, b_bank_attr, b_bank_payload, b_perf_store}) + + end else begin : g_no_read_engine + + assign b_busy = 1'b0; + assign b_bank_valid = '0; + assign b_bank_addr = '0; + assign b_rsp_valid = 1'b0; + assign b_rsp_block = '0; + assign b_rsp_mask = '0; + assign b_rsp_data = '0; + assign b_rsp_tag = '0; + assign b_perf_served = '0; + assign b_perf_replay = 1'b0; + `UNUSED_VAR (b_bank_rdata) + `UNUSED_VAR (b_rsp_take) + `UNUSED_VAR (b_rsp_done) + `UNUSED_VAR (a_holds_write) + `UNUSED_VAR (b_req_valid) + `UNUSED_VAR (b_req_block) + assign b_req_ready = 1'b0; + + end + + // Responses. Each block has a response register, which takes the first + // engine's queue head ahead of the read engine's when it is empty or is + // being emptied. A response's queue slot is free again once its block has + // taken it on. + + wire [NUM_BLOCKS-1:0] a_load; + wire [NUM_BLOCKS-1:0] b_load; + wire [NUM_BLOCKS-1:0] a_done; + wire [NUM_BLOCKS-1:0] b_done; + + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_rsp + reg rsp_valid_r; + reg rsp_from_b_r; + reg [NUM_LANES-1:0] rsp_mask_r; + reg [NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data_r; + reg [TAG_WIDTH-1:0] rsp_tag_r; + + wire rsp_fire = rsp_valid_r && lsu_mem_if[k].rsp_ready; + wire rsp_free = ~rsp_valid_r || lsu_mem_if[k].rsp_ready; + wire a_hit = a_rsp_valid && (a_rsp_block == BLOCK_WIDTH'(k)); + wire b_hit = b_rsp_valid && (b_rsp_block == BLOCK_WIDTH'(k)); + + assign a_load[k] = rsp_free && a_hit; + assign b_load[k] = rsp_free && ~a_hit && b_hit; + assign a_done[k] = rsp_fire && ~rsp_from_b_r; + assign b_done[k] = rsp_fire && rsp_from_b_r; + + always @(posedge clk) begin + if (reset) begin + rsp_valid_r <= 1'b0; + end else if (rsp_free) begin + rsp_valid_r <= a_hit || b_hit; + end + if (rsp_free) begin + rsp_from_b_r <= ~a_hit; + rsp_mask_r <= a_hit ? a_rsp_mask : b_rsp_mask; + rsp_data_r <= a_hit ? a_rsp_data : b_rsp_data; + rsp_tag_r <= a_hit ? a_rsp_tag : b_rsp_tag; + end + end + + assign lsu_mem_if[k].rsp_valid = rsp_valid_r; + assign lsu_mem_if[k].rsp_data.mask = rsp_mask_r; + assign lsu_mem_if[k].rsp_data.data = rsp_data_r; + assign lsu_mem_if[k].rsp_data.tag = rsp_tag_r; + end + + `POP_COUNT(a_rsp_done, a_done); + `POP_COUNT(b_rsp_done, b_done); + assign a_rsp_take = (| a_load); + assign b_rsp_take = (| b_load); + + // DMA and atomic slots + + reg [1:0] amo_stall_r; + wire amo_stall = (amo_stall_r != 2'd0); + always @(posedge clk) begin + if (reset) begin + amo_stall_r <= 2'd0; + end else if (a_pass_amo) begin + amo_stall_r <= 2'd2; + end else if (amo_stall) begin + amo_stall_r <= amo_stall_r - 2'd1; + end + end + + wire dma_valid = DMA_ENABLE && dma_bus_if.req_valid; + reg dma_slot_r; + always @(posedge clk) begin + if (reset) begin + dma_slot_r <= 1'b0; + end else begin + dma_slot_r <= dma_valid && ~amo_stall; + end + end + assign pass_en = ~dma_valid && ~amo_stall; + + // DMA port handshake + // rw=0 reads : accepted when the response pipe-buffer has space. + // rw=1 writes : no response issued. + + wire dma_rsp_buf_ready; + wire dma_fire = dma_slot_r && dma_valid && (dma_bus_if.req_data.rw || dma_rsp_buf_ready); + wire dma_rd_fire = dma_fire && ~dma_bus_if.req_data.rw; + wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] per_bank_rdata_a; + + if (DMA_ENABLE) begin : g_dma_enable + `UNUSED_VAR (dma_bus_if.req_data.attr) + + assign dma_bus_if.req_ready = dma_slot_r && (dma_bus_if.req_data.rw || dma_rsp_buf_ready); + + // the tag follows the SRAM read to its output register + VX_pipe_buffer #( + .DATAW (DMA_TAG_WIDTH) + ) dma_rsp_buf ( + .clk (clk), + .reset (reset), + .valid_in (dma_rd_fire), + .ready_in (dma_rsp_buf_ready), + .data_in (dma_bus_if.req_data.tag), + .valid_out (dma_bus_if.rsp_valid), + .data_out (dma_bus_if.rsp_data.tag), + .ready_out (dma_bus_if.rsp_ready) + ); + + // The response buffer holds only the tag; the data is the live SRAM + // output, which the next pass's reads overwrite. While a response is + // back-pressured, serve a copy of the row latched on its first cycle. + reg [NUM_BANKS-1:0][WORD_WIDTH-1:0] dma_rsp_hold_data_r; + reg dma_rsp_hold_valid_r; + wire dma_rsp_consumed = dma_bus_if.rsp_valid && dma_bus_if.rsp_ready; + always @(posedge clk) begin + if (reset) begin + dma_rsp_hold_valid_r <= 1'b0; + end else if (dma_rsp_consumed) begin + dma_rsp_hold_valid_r <= 1'b0; + end else if (dma_bus_if.rsp_valid && ~dma_rsp_hold_valid_r) begin + dma_rsp_hold_data_r <= per_bank_rdata_a; + dma_rsp_hold_valid_r <= 1'b1; + end + end + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_dma_rsp_data + assign dma_bus_if.rsp_data.data[i*WORD_WIDTH +: WORD_WIDTH] = + dma_rsp_hold_valid_r ? dma_rsp_hold_data_r[i] : per_bank_rdata_a[i]; + end + + end else begin : g_no_dma + assign dma_rsp_buf_ready = 1'b0; + assign dma_bus_if.req_ready = 1'b0; + assign dma_bus_if.rsp_valid = 1'b0; + assign dma_bus_if.rsp_data = '0; + `UNUSED_VAR (dma_bus_if.req_valid) + `UNUSED_VAR (dma_bus_if.req_data) + `UNUSED_VAR (dma_bus_if.rsp_ready) + end + + // banks + + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_data_store + + wire dma_wr_b = dma_fire && dma_bus_if.req_data.rw + && (| dma_bus_if.req_data.byteen[i*WORD_SIZE +: WORD_SIZE]); + + wire lsu_active = a_bank_valid[i]; + + wire [WORD_WIDTH-1:0] req_data; + wire [WORD_SIZE-1:0] req_byteen; + wire [HART_ID_WIDTH-1:0] req_hart_id; + `ifdef VX_CFG_EXT_ZACAS_ENABLE + wire [AMO_CMP_WIDTH-1:0] req_amo_cmp; + `endif + if (AMO_ENABLE != 0) begin : g_payload_amo + `ifdef VX_CFG_EXT_ZACAS_ENABLE + assign {req_hart_id, req_amo_cmp, req_byteen, req_data} = a_bank_payload[i]; + `else + assign {req_hart_id, req_byteen, req_data} = a_bank_payload[i]; + `endif + end else begin : g_payload + assign {req_byteen, req_data} = a_bank_payload[i]; + assign req_hart_id = '0; + `ifdef VX_CFG_EXT_ZACAS_ENABLE + assign req_amo_cmp = '0; + `endif + end + + // Bank-facing AMO interface, tied off when the A extension is absent so + // none of the atomic hardware is synthesized. + wire amo_wb_store; // owed atomic write-back + wire [BANK_ADDR_WIDTH-1:0] amo_wb_addr; + wire [WORD_SIZE-1:0] amo_wb_byteen; + wire [WORD_WIDTH-1:0] amo_wb_data; + wire sc_resolving; // SC outcome replaces the response + wire sc_fail; + wire sc_lane_hi; // outcome lane of a resolving .W SC + + if (AMO_ENABLE != 0) begin : g_amo + + // Atomics. An atomic carries rw=0, so the read returning the old + // value is the ordinary read path. What the bank owes is the write + // of the computed value. + // + // The old word is registered before the ALU sees it. Feeding the ALU + // straight from the SRAM output puts the BRAM clock-to-out, the adder + // and the BRAM setup in a single period; the register splits that + // into BRAM-out-to-flop and flop-through-adder-to-BRAM-in. + // + // An atomic owns its bank for three cycles: read, capture, write back. + reg amo_rd_valid_r; + reg amo_wb_valid_r; + reg [WORD_WIDTH-1:0] amo_old_r; + reg [BANK_ADDR_WIDTH-1:0] amo_wb_addr_r; + reg [WORD_SIZE-1:0] amo_wb_byteen_r; + reg [AMO_OP_WIDTH-1:0] amo_wb_op_r; + reg amo_wb_unsigned_r; + reg [WORD_WIDTH-1:0] amo_wb_rhs_r; + `ifdef VX_CFG_EXT_ZACAS_ENABLE + reg [AMO_CMP_WIDTH-1:0] amo_wb_cmp_r; + `endif + + wire [AMO_OP_WIDTH-1:0] req_amo_op = a_bank_attr[1 +: AMO_OP_WIDTH]; + wire req_amo_unsigned = a_bank_attr[0]; + + wire amo_accept = lsu_active && a_bank_amo; + + // Reservation commit stage. Every accepted request that can touch a + // reservation -- an atomic, or a plain store -- reaches it one cycle + // after acceptance, so a single address port serves both. The + // look-ahead read is driven from the acceptance cycle, which is what + // makes the registered reservation entry land here. + wire is_lr_in = a_bank_amo && (amo_op_e'(req_amo_op) == AMO_OP_LR); + wire is_sc_in = a_bank_amo && (amo_op_e'(req_amo_op) == AMO_OP_SC); + + reg res_valid_r; + reg res_is_lr_r; + reg res_is_sc_r; + reg res_is_amo_r; + reg res_is_store_r; + reg [BANK_ADDR_WIDTH-1:0] res_addr_r; + reg [HART_ID_WIDTH-1:0] res_hart_id_r; + + always @(posedge clk) begin + if (reset) begin + res_valid_r <= 1'b0; + end else begin + res_valid_r <= lsu_active; + end + if (lsu_active) begin + res_is_lr_r <= is_lr_in; + res_is_sc_r <= is_sc_in; + res_is_amo_r <= a_bank_amo; + res_is_store_r <= a_bank_rw; + res_addr_r <= a_bank_addr[i]; + res_hart_id_r <= req_hart_id; + end + end + + // A store-conditional fails when its reservation is gone. The + // outcome is known at the commit stage, one cycle before the + // write-back, so it can both suppress the write and be returned as + // this request's response. + wire res_check; + assign sc_fail = res_is_sc_r && ~res_check; + assign sc_resolving = res_valid_r && res_is_sc_r; + + // The SC outcome must land in the addressed lane: the LSU extracts + // the subword by address, so a .W outcome left at bit 0 would read + // as 0 ("success") for every upper-lane SC, including a failed one + // whose store was suppressed. + if (WORD_WIDTH == 64) begin : g_sc_lane + reg res_lane_hi_r; + always @(posedge clk) begin + if (lsu_active) begin + res_lane_hi_r <= ~req_byteen[0]; + end + end + assign sc_lane_hi = res_lane_hi_r; + end else begin : g_sc_lane0 + assign sc_lane_hi = 1'b0; + end + + // Everything that ends up writing the word: a plain store, a + // read-modify-write atomic (which carries rw=0 and so is not a store + // by the request flag), and a store-conditional that found its + // reservation. + wire res_commits_store = res_is_store_r + || (res_is_amo_r && ~res_is_lr_r && ~res_is_sc_r) + || (res_is_sc_r && res_check); + + reg amo_wb_is_lr_r; + reg amo_wb_sc_fail_r; + always @(posedge clk) begin + if (amo_rd_valid_r) begin + amo_wb_is_lr_r <= res_is_lr_r; + amo_wb_sc_fail_r <= sc_fail; + end + end + + // A load-reserved leaves the word alone, and a failed + // store-conditional must not store; every other atomic writes its + // computed value back. + assign amo_wb_store = amo_wb_valid_r && ~amo_wb_is_lr_r && ~amo_wb_sc_fail_r; + + always @(posedge clk) begin + if (reset) begin + amo_rd_valid_r <= 1'b0; + amo_wb_valid_r <= 1'b0; + end else begin + amo_rd_valid_r <= amo_accept; + amo_wb_valid_r <= amo_rd_valid_r; + end + if (amo_rd_valid_r) begin + amo_old_r <= per_bank_rdata_a[i]; + end + if (amo_accept) begin + amo_wb_addr_r <= a_bank_addr[i]; + amo_wb_byteen_r <= req_byteen; + amo_wb_op_r <= req_amo_op; + amo_wb_unsigned_r <= req_amo_unsigned; + amo_wb_rhs_r <= req_data; + `ifdef VX_CFG_EXT_ZACAS_ENABLE + amo_wb_cmp_r <= req_amo_cmp; + `endif + end + end + + wire [63:0] amo_new_word; + wire [63:0] amo_ret_word; + + // Lane-align the RMW operands. The bank word is XLEN wide, so on a + // 64-bit hart a .W atomic addresses one of two 32-bit lanes; the + // ALU computes right-aligned. Store data arrives lane-positioned + // (same packing as a plain store), so old and rhs shift down by the + // lane offset and the result shifts back up under the request + // byteen. The comparand arrives right-aligned from rs3 and needs no + // shift. The addressed lane is recovered from the byteen: an atomic + // is naturally aligned, so a .W to the upper lane never covers byte + // 0 and a .D covers the full word. + wire [63:0] amo_old_lane; + wire [63:0] amo_rhs_lane; + wire [1:0] amo_width; + if (WORD_WIDTH == 64) begin : g_amo_lane + wire lane_hi = ~amo_wb_byteen_r[0]; + assign amo_width = (&amo_wb_byteen_r) ? 2'd3 : 2'd2; + assign amo_old_lane = lane_hi ? {32'h0, amo_old_r[63:32]} : amo_old_r; + assign amo_rhs_lane = lane_hi ? {32'h0, amo_wb_rhs_r[63:32]} : amo_wb_rhs_r; + assign amo_wb_data = lane_hi ? {amo_new_word[31:0], 32'h0} : amo_new_word; + end else begin : g_amo_word + assign amo_width = 2'd2; + assign amo_old_lane = 64'(amo_old_r); + assign amo_rhs_lane = 64'(amo_wb_rhs_r); + assign amo_wb_data = amo_new_word[WORD_WIDTH-1:0]; + end + // VX_CFG_AMO_RS_SIZE sizes ONE shared LLC bank's contention; deployed + // per lmem bank it multiplies by NUM_LMEM_BANKS x NUM_CORES. After + // the holder-credit protocol a hot word maps to exactly one station, + // so depth only reduces false eviction between distinct reserved + // words aliasing in one bank -- and a spurious SC failure is + // architecturally legal. Clamp the depth. + // A credit is spent per refused LR, not per cycle: a warp's LR + // reaches a bank one lane per pass, so between a holder's LR and its + // SC every other hart of the core can be refused, and a refused hart + // can retry once more. The budget covers two refusals per hart; a + // smaller one lets one warp's own LR burst displace its holder and a + // contended retry loop stops making progress. + VX_amo_unit #( + .NUM_RES_ENTRIES (`MIN(`VX_CFG_AMO_RS_SIZE, 4)), + .HOLD_CREDIT_BITS (`CLOG2(`VX_CFG_NUM_WARPS * `VX_CFG_NUM_THREADS) + 1), + .LINE_ADDR_BITS (BANK_ADDR_WIDTH), + .DATA_WIDTH (WORD_WIDTH) + ) amo_unit ( + .clk (clk), + .reset (reset), + .pipe_stall (1'b0), + .compute_op (amo_op_e'(amo_wb_op_r)), + .compute_unsigned (amo_wb_unsigned_r), + .compute_width (amo_width), + .compute_old (amo_old_lane), + .compute_rhs (amo_rhs_lane), + `ifdef VX_CFG_EXT_ZACAS_ENABLE + .compute_cmp (64'(amo_wb_cmp_r)), + `else + .compute_cmp (64'b0), + `endif + .compute_new_word (amo_new_word), + .compute_ret_word (amo_ret_word), + // A load-reserved claims the word; a store-conditional gives up + // its own reservation whether it succeeds or fails; any committing + // store breaks whoever else holds one. + .res_reserve (res_valid_r && res_is_lr_r), + .res_clear (res_valid_r && res_is_sc_r), + .res_invalidate (res_valid_r && res_commits_store), + .res_hart_id (res_hart_id_r), + .res_line_addr (res_addr_r), + .res_line_addr_n (a_bank_addr[i]), + .res_check (res_check) + ); + `UNUSED_VAR (amo_ret_word) + if (WORD_WIDTH < 64) begin : g_amo_hi_unused + `UNUSED_VAR (amo_new_word[63:WORD_WIDTH]) + end + + assign amo_wb_addr = amo_wb_addr_r; + assign amo_wb_byteen = amo_wb_byteen_r; + + end else begin : g_no_amo + assign amo_wb_store = 1'b0; + assign amo_wb_addr = '0; + assign amo_wb_byteen = '0; + assign amo_wb_data = '0; + assign sc_resolving = 1'b0; + assign sc_fail = 1'b0; + assign sc_lane_hi = 1'b0; + `UNUSED_VAR (req_hart_id) + `ifdef VX_CFG_EXT_ZACAS_ENABLE + `UNUSED_VAR (req_amo_cmp) + `endif + end + + // Port A address / write-data / write-enable mux: DMA, an owed atomic + // write-back, or the first engine's pass. They never coincide. + wire [BANK_ADDR_WIDTH-1:0] bank_sram_addr; + wire [WORD_WIDTH-1:0] bank_sram_wdata; + wire [WORD_SIZE-1:0] bank_sram_wren; + + assign bank_sram_addr = dma_fire ? BANK_ADDR_WIDTH'(dma_bus_if.req_data.addr) + : amo_wb_store ? amo_wb_addr + : a_bank_addr[i]; + assign bank_sram_wdata = dma_fire ? dma_bus_if.req_data.data[i*WORD_WIDTH +: WORD_WIDTH] + : amo_wb_store ? amo_wb_data + : req_data; + assign bank_sram_wren = dma_fire ? dma_bus_if.req_data.byteen[i*WORD_SIZE +: WORD_SIZE] + : amo_wb_store ? amo_wb_byteen + : req_byteen; + + wire sram_read = dma_rd_fire || (lsu_active && ~a_bank_rw); + wire sram_write = dma_wr_b || amo_wb_store || (lsu_active && a_bank_rw); + + if (DUAL) begin : g_dual_port + VX_1rw1r_ram #( + .DATAW (WORD_WIDTH), + .SIZE (WORDS_PER_BANK), + .WRENW (WORD_SIZE) + ) lmem_store ( + .clk (clk), + .reset (reset), + .read_a (sram_read), + .write_a (sram_write), + .wren_a (bank_sram_wren), + .addr_a (bank_sram_addr), + .wdata_a (bank_sram_wdata), + .rdata_a (per_bank_rdata_a[i]), + .read_b (b_bank_valid[i]), + .addr_b (b_bank_addr[i]), + .rdata_b (b_bank_rdata[i]) + ); + end else begin : g_single_port + VX_sp_ram #( + .DATAW (WORD_WIDTH), + .SIZE (WORDS_PER_BANK), + .WRENW (WORD_SIZE), + .OUT_REG (1), + .RDW_MODE ("R") + ) lmem_store ( + .clk (clk), + .reset (reset), + .read (sram_read), + .write (sram_write), + .wren (bank_sram_wren), + .addr (bank_sram_addr), + .wdata (bank_sram_wdata), + .rdata (per_bank_rdata_a[i]) + ); + assign b_bank_rdata[i] = '0; + `UNUSED_VAR (b_bank_valid[i]) + `UNUSED_VAR (b_bank_addr[i]) + end + + // A store-conditional returns its outcome, not the word it read: zero + // for success, one for failure, positioned in the addressed lane for + // the LSU's subword extraction. The outcome is resolved at the commit + // stage, the cycle the response takes the SRAM output. + assign a_bank_rdata[i] = sc_resolving + ? (sc_lane_hi ? (WORD_WIDTH'(sc_fail) << (WORD_WIDTH/2)) + : WORD_WIDTH'(sc_fail)) + : per_bank_rdata_a[i]; + end + + if (AMO_ENABLE == 0) begin : g_no_amo_attr + `UNUSED_VAR ({a_bank_amo, a_bank_attr}) + end + +`ifdef PERF_ENABLE + wire [`CLOG2(NUM_LANES+1)-1:0] perf_a_served; + wire [`CLOG2(NUM_LANES+1)-1:0] perf_b_served; + `POP_COUNT(perf_a_served, a_perf_served); + `POP_COUNT(perf_b_served, b_perf_served); + + wire [NUM_BLOCKS-1:0] perf_crsp_stall_per_blk; + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_perf_crsp + assign perf_crsp_stall_per_blk[k] = lsu_mem_if[k].rsp_valid && ~lsu_mem_if[k].rsp_ready; + end + wire [`CLOG2(NUM_BLOCKS+1)-1:0] perf_crsp_stalls_per_cycle; + `POP_COUNT(perf_crsp_stalls_per_cycle, perf_crsp_stall_per_blk); + + reg [PERF_CTR_BITS-1:0] perf_reads; + reg [PERF_CTR_BITS-1:0] perf_writes; + reg [PERF_CTR_BITS-1:0] perf_bank_stalls; + reg [PERF_CTR_BITS-1:0] perf_crsp_stalls; + + always @(posedge clk) begin + if (reset) begin + perf_reads <= '0; + perf_writes <= '0; + perf_bank_stalls <= '0; + perf_crsp_stalls <= '0; + end else begin + perf_reads <= perf_reads + PERF_CTR_BITS'(a_perf_store ? 0 : perf_a_served) + PERF_CTR_BITS'(perf_b_served); + perf_writes <= perf_writes + PERF_CTR_BITS'(a_perf_store ? perf_a_served : 0); + perf_bank_stalls <= perf_bank_stalls + PERF_CTR_BITS'(a_perf_replay) + PERF_CTR_BITS'(b_perf_replay); + perf_crsp_stalls <= perf_crsp_stalls + PERF_CTR_BITS'(perf_crsp_stalls_per_cycle); + end + end + + assign lmem_perf.reads = perf_reads; + assign lmem_perf.writes = perf_writes; + assign lmem_perf.bank_stalls = perf_bank_stalls; + assign lmem_perf.crsp_stalls = perf_crsp_stalls; +`else + `UNUSED_VAR ({a_perf_served, a_perf_store, a_perf_replay, b_perf_served, b_perf_replay}) +`endif + +`ifdef DBG_TRACE_MEM + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_trace + always @(posedge clk) begin + if (lsu_mem_if[k].req_valid && lsu_mem_if[k].req_ready) begin + `TRACE(2, ("%t: %s req[%0d]: rw=%b, mask=%b, tag=0x%0h\n", + $time, INSTANCE_ID, k, lsu_mem_if[k].req_data.rw, lsu_mem_if[k].req_data.mask, lsu_mem_if[k].req_data.tag)) + end + if (lsu_mem_if[k].rsp_valid && lsu_mem_if[k].rsp_ready) begin + `TRACE(2, ("%t: %s rsp[%0d]: mask=%b, tag=0x%0h\n", + $time, INSTANCE_ID, k, lsu_mem_if[k].rsp_data.mask, lsu_mem_if[k].rsp_data.tag)) + end + end + end +`endif + +endmodule diff --git a/hw/rtl/mem/VX_local_mem.sv b/hw/rtl/mem/VX_local_mem.sv deleted file mode 100644 index 82059d8454..0000000000 --- a/hw/rtl/mem/VX_local_mem.sv +++ /dev/null @@ -1,912 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_define.vh" - -module VX_local_mem import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - - // Size of cache in bytes - parameter SIZE = (1024*16*8), - - // Number of Word requests per cycle - parameter NUM_REQS = 4, - // Number of banks - parameter NUM_BANKS = 4, - - // Address width - parameter ADDR_WIDTH = `CLOG2(SIZE), - // Size of a word in bytes - parameter WORD_SIZE = `VX_CFG_XLEN/8, - - // Request tag size - parameter TAG_WIDTH = 16, - - // Enable DMA port - parameter DMA_ENABLE = 0, - parameter DMA_TAG_WIDTH = 1, - - // Synthesize atomic-op + LR/SC logic in the banks - parameter AMO_ENABLE = 0, - - // Merge same-word reads within groups of this many adjacent ports (0: off) - parameter READ_MERGE = 0, - - // Response buffer - parameter OUT_BUF = 0 - ) ( - input wire clk, - input wire reset, - - // PERF -`ifdef PERF_ENABLE - output lmem_perf_t lmem_perf, -`endif - - // LSU read/write port - VX_mem_bus_if.slave lsu_bus_if [NUM_REQS], - - // DMA read/write port - VX_mem_bus_if.slave dma_bus_if -); - `UNUSED_SPARAM (INSTANCE_ID) - - localparam REQ_SEL_BITS = `CLOG2(NUM_REQS); - localparam REQ_SEL_WIDTH = `UP(REQ_SEL_BITS); - localparam WORD_WIDTH = WORD_SIZE * 8; - localparam NUM_WORDS = SIZE / WORD_SIZE; - localparam WORDS_PER_BANK = NUM_WORDS / NUM_BANKS; - localparam BANK_ADDR_WIDTH = `CLOG2(WORDS_PER_BANK); - localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS); - localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); - // AMO sideband carried to the banks: valid + op + unsigned + hart_id. The - // operand width comes from the request's byteen; hart_id identifies the - // reserver so a store-conditional can tell its own reservation from - // another hart's. - // Compare-and-swap adds a comparand: the request's data word already - // carries the swap value, so the third operand needs its own room. -`ifdef VX_CFG_EXT_ZACAS_ENABLE - localparam AMO_CMP_WIDTH = `VX_CFG_XLEN; -`else - localparam AMO_CMP_WIDTH = 0; -`endif - localparam AMO_SB_WIDTH = 1 + $bits(amo_op_e) + 1 + HART_ID_WIDTH + AMO_CMP_WIDTH; - // A merged read carries its group's merged-port mask under its tag, through - // the bank, to the response multicast. - localparam MERGE_GROUP = (READ_MERGE != 0) ? `MIN(READ_MERGE, NUM_REQS) : 1; - localparam DO_READ_MERGE = (MERGE_GROUP > 1); - localparam MERGE_BITS = DO_READ_MERGE ? MERGE_GROUP : 0; - localparam BANK_TAG_WIDTH = TAG_WIDTH + MERGE_BITS; - localparam REQ_DATAW = 1 + BANK_ADDR_WIDTH + WORD_SIZE + WORD_WIDTH + BANK_TAG_WIDTH + AMO_SB_WIDTH; - localparam RSP_DATAW = WORD_WIDTH + TAG_WIDTH; - - `STATIC_ASSERT(ADDR_WIDTH == (BANK_ADDR_WIDTH + `CLOG2(NUM_BANKS)), ("invalid parameter")) - `STATIC_ASSERT(!DO_READ_MERGE || (`IS_POW2(MERGE_GROUP) && `IS_DIVISBLE(NUM_REQS, MERGE_GROUP)), ("invalid parameter: read merge group")) - - // bank selection - - wire [NUM_REQS-1:0][BANK_SEL_WIDTH-1:0] req_bank_idx; - if (NUM_BANKS > 1) begin : g_req_bank_idx - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_bank_idxs - assign req_bank_idx[i] = lsu_bus_if[i].req_data.addr[0 +: BANK_SEL_BITS]; - end - end else begin : g_req_bank_idx_0 - assign req_bank_idx = 0; - end - - // bank addressing - - wire [NUM_REQS-1:0][BANK_ADDR_WIDTH-1:0] req_bank_addr; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_bank_addr - assign req_bank_addr[i] = lsu_bus_if[i].req_data.addr[BANK_SEL_BITS +: BANK_ADDR_WIDTH]; - end - - // AMO sideband per request, from the shared mem-bus attr. - wire [NUM_REQS-1:0] req_amo_valid; - wire [NUM_REQS-1:0][$bits(amo_op_e)-1:0] req_amo_op; - wire [NUM_REQS-1:0] req_amo_unsigned; - wire [NUM_REQS-1:0][HART_ID_WIDTH-1:0] req_amo_hart_id; -`ifdef VX_CFG_EXT_ZACAS_ENABLE - wire [NUM_REQS-1:0][AMO_CMP_WIDTH-1:0] req_amo_cmp; -`endif - // Gated by AMO_ENABLE: the widths stay constant and the zeros constant-fold - // through the crossbar payload (the VX_cache_bank pattern). - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_amo - mem_bus_attr_t lane_attr; - assign lane_attr = mem_bus_attr_t'(lsu_bus_if[i].req_data.attr); - assign req_amo_valid[i] = AMO_ENABLE ? lane_attr.amo.amo_valid : 1'b0; - assign req_amo_op[i] = AMO_ENABLE ? lane_attr.amo.amo_op : '0; - assign req_amo_unsigned[i] = AMO_ENABLE ? lane_attr.amo.amo_unsigned : 1'b0; - assign req_amo_hart_id[i] = AMO_ENABLE ? lane_attr.amo.hart_id : '0; - `ifdef VX_CFG_EXT_ZACAS_ENABLE - assign req_amo_cmp[i] = AMO_ENABLE ? lane_attr.amo.amo_cmp : '0; - `endif - `UNUSED_VAR (lane_attr) - end - - // bank requests dispatch - - wire [NUM_BANKS-1:0] per_bank_req_valid; - wire [NUM_BANKS-1:0] per_bank_req_rw; - wire [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] per_bank_req_addr; - wire [NUM_BANKS-1:0][WORD_SIZE-1:0] per_bank_req_byteen; - wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] per_bank_req_data; - wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_req_tag; - wire [NUM_BANKS-1:0] per_bank_req_amo_valid; - wire [NUM_BANKS-1:0][$bits(amo_op_e)-1:0] per_bank_req_amo_op; - wire [NUM_BANKS-1:0] per_bank_req_amo_unsigned; - wire [NUM_BANKS-1:0][HART_ID_WIDTH-1:0] per_bank_req_amo_hart_id; -`ifdef VX_CFG_EXT_ZACAS_ENABLE - wire [NUM_BANKS-1:0][AMO_CMP_WIDTH-1:0] per_bank_req_amo_cmp; -`endif - wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_req_idx; - wire [NUM_BANKS-1:0] per_bank_req_ready; - - wire [NUM_BANKS-1:0][REQ_DATAW-1:0] per_bank_req_data_aos; - - wire [NUM_REQS-1:0] req_valid_in; - wire [NUM_REQS-1:0][REQ_DATAW-1:0] req_data_in; - wire [NUM_REQS-1:0] req_ready_in; - -`ifdef PERF_ENABLE - wire [PERF_CTR_BITS-1:0] perf_collisions; -`endif - - // Read merging: plain reads of the same word with the same tag from ports - // of one group in the same cycle are served by one bank access. The lowest - // such port leads; the others stay out of the request crossbar and are - // accepted in the cycle their leader is, so every port keeps its own - // request order into the bank. - wire [NUM_REQS-1:0] req_follower; - wire [NUM_REQS-1:0][BANK_TAG_WIDTH-1:0] req_bank_tag; - if (DO_READ_MERGE) begin : g_read_merge - wire [NUM_REQS-1:0] plain_read; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_plain_read - assign plain_read[i] = lsu_bus_if[i].req_valid && ~lsu_bus_if[i].req_data.rw && ~req_amo_valid[i]; - end - // same_word[i][j]: port i reads what a lower port j of its group reads - wire [NUM_REQS-1:0][NUM_REQS-1:0] same_word; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_same_i - for (genvar j = 0; j < NUM_REQS; ++j) begin : g_same_j - if (j < i && (j / MERGE_GROUP) == (i / MERGE_GROUP)) begin : g_cmp - assign same_word[i][j] = plain_read[i] && plain_read[j] - && (lsu_bus_if[i].req_data.addr == lsu_bus_if[j].req_data.addr) - && (lsu_bus_if[i].req_data.tag == lsu_bus_if[j].req_data.tag); - end else begin : g_none - assign same_word[i][j] = 1'b0; - end - end - end - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_merge - localparam GROUP_BASE = (i / MERGE_GROUP) * MERGE_GROUP; - wire [MERGE_GROUP-1:0] merged_mask; - wire [NUM_REQS-1:0] leader_fire; - for (genvar k = 0; k < MERGE_GROUP; ++k) begin : g_merged_mask - assign merged_mask[k] = same_word[GROUP_BASE + k][i]; - end - for (genvar j = 0; j < NUM_REQS; ++j) begin : g_leader_fire - assign leader_fire[j] = same_word[i][j] && ~req_follower[j] && req_ready_in[j]; - end - assign req_bank_tag[i] = {lsu_bus_if[i].req_data.tag, merged_mask}; - assign req_follower[i] = (| same_word[i]); - assign lsu_bus_if[i].req_ready = req_follower[i] ? (| leader_fire) : req_ready_in[i]; - end - end else begin : g_no_read_merge - assign req_follower = '0; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_bank_tag - assign req_bank_tag[i] = lsu_bus_if[i].req_data.tag; - assign lsu_bus_if[i].req_ready = req_ready_in[i]; - end - end - - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_data_in - // a merged read enters the crossbar once, as its leader - assign req_valid_in[i] = lsu_bus_if[i].req_valid && ~req_follower[i]; - assign req_data_in[i] = { - req_amo_valid[i], - req_amo_op[i], - req_amo_unsigned[i], - req_amo_hart_id[i], - `ifdef VX_CFG_EXT_ZACAS_ENABLE - req_amo_cmp[i], - `endif - lsu_bus_if[i].req_data.rw, - req_bank_addr[i], - lsu_bus_if[i].req_data.data, - lsu_bus_if[i].req_data.byteen, - req_bank_tag[i] - }; - end - - VX_stream_xbar #( - .NUM_INPUTS (NUM_REQS), - .NUM_OUTPUTS (NUM_BANKS), - .DATAW (REQ_DATAW), - .PERF_CTR_BITS (PERF_CTR_BITS), - .ARBITER ("P"), - // over more than 12 ports the arbiters are a tree; slices of four keep - // its depth and shorten the grant each request waits on - .MAX_FANOUT ((NUM_REQS > 12) ? 4 : `MAX_FANOUT), - .OUT_BUF (3) // output should be registered for the data_store addressing - ) req_xbar ( - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .collisions (perf_collisions), - `else - `UNUSED_PIN (collisions), - `endif - .valid_in (req_valid_in), - .data_in (req_data_in), - .sel_in (req_bank_idx), - .ready_in (req_ready_in), - .valid_out (per_bank_req_valid), - .data_out (per_bank_req_data_aos), - .sel_out (per_bank_req_idx), - .ready_out (per_bank_req_ready) - ); - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_req_data_soa - assign { - per_bank_req_amo_valid[i], - per_bank_req_amo_op[i], - per_bank_req_amo_unsigned[i], - per_bank_req_amo_hart_id[i], - `ifdef VX_CFG_EXT_ZACAS_ENABLE - per_bank_req_amo_cmp[i], - `endif - per_bank_req_rw[i], - per_bank_req_addr[i], - per_bank_req_data[i], - per_bank_req_byteen[i], - per_bank_req_tag[i] - } = per_bank_req_data_aos[i]; - end - - // banks access (declared here so g_dma_enable can reference per_bank_rsp_data) - - wire [NUM_BANKS-1:0] per_bank_rsp_valid; - wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] per_bank_rsp_data; - // Back-pressure-safe copy of the LSU response data (see g_data_store). - wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] bank_lsu_rsp_data; - wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_rsp_idx; - wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_rsp_tag; - wire [NUM_BANKS-1:0] per_bank_rsp_ready; - - // DMA port handshake - // rw=0 reads : accepted when the response pipe-buffer has space. - // rw=1 writes : always accepted; no response issued. - // DMA has priority over LSU at every bank SRAM. - - wire dma_rsp_buf_ready; // driven by pipe-buffer or tied 0 when disabled - - if (DMA_ENABLE) begin : g_dma_enable - `UNUSED_VAR (dma_bus_if.req_data.attr) - - assign dma_bus_if.req_ready = dma_bus_if.req_data.rw || dma_rsp_buf_ready; - - wire dma_rd_fire = dma_bus_if.req_valid && ~dma_bus_if.req_data.rw && dma_rsp_buf_ready; - - // Delay tag by 1 cycle to align with SRAM OUT_REG latency - VX_pipe_buffer #( - .DATAW (DMA_TAG_WIDTH) - ) dma_rsp_buf ( - .clk (clk), - .reset (reset), - .valid_in (dma_rd_fire), - .ready_in (dma_rsp_buf_ready), - .data_in (dma_bus_if.req_data.tag), - .valid_out (dma_bus_if.rsp_valid), - .data_out (dma_bus_if.rsp_data.tag), - .ready_out (dma_bus_if.rsp_ready) - ); - - // Pack all bank SRAM outputs into the read response. - // - // Same back-pressure hazard as the LSU side: dma_rsp_buf buffers only - // the tag/valid; the data (all banks) is the live SRAM OUT_REG. While a - // DMA read response is back-pressured, an interleaving LSU read - // (~dma_active) re-drives a bank's OUT_REG and corrupts the held DMA - // line. Latch the whole line on the first response-valid cycle (rdata - // is still valid then) and serve the latched copy while stalled. One - // DMA response spans all banks, so this is a single full-line hold. - reg [NUM_BANKS-1:0][WORD_WIDTH-1:0] dma_rsp_hold_data_r; - reg dma_rsp_hold_valid_r; - wire dma_rsp_consumed = dma_bus_if.rsp_valid && dma_bus_if.rsp_ready; - always @(posedge clk) begin - if (reset) begin - dma_rsp_hold_valid_r <= 1'b0; - end else if (dma_rsp_consumed) begin - dma_rsp_hold_valid_r <= 1'b0; - end else if (dma_bus_if.rsp_valid && ~dma_rsp_hold_valid_r) begin - dma_rsp_hold_data_r <= per_bank_rsp_data; - dma_rsp_hold_valid_r <= 1'b1; - end - end - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_dma_rsp_data - assign dma_bus_if.rsp_data.data[i*WORD_WIDTH +: WORD_WIDTH] = - dma_rsp_hold_valid_r ? dma_rsp_hold_data_r[i] : per_bank_rsp_data[i]; - end - - end else begin : g_no_dma - assign dma_rsp_buf_ready = 1'b0; - assign dma_bus_if.req_ready = 1'b0; - assign dma_bus_if.rsp_valid = 1'b0; - assign dma_bus_if.rsp_data = '0; - `UNUSED_VAR (dma_bus_if.req_valid) - `UNUSED_VAR (dma_bus_if.req_data) - `UNUSED_VAR (dma_bus_if.rsp_ready) - end - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_data_store - wire bank_rsp_valid, bank_rsp_ready; - - // DMA active signals (priority over LSU) - wire dma_wr_b = DMA_ENABLE - && dma_bus_if.req_valid - && dma_bus_if.req_data.rw - && (|dma_bus_if.req_data.byteen[i*WORD_SIZE +: WORD_SIZE]); - - wire dma_rd_b = DMA_ENABLE - && dma_bus_if.req_valid - && ~dma_bus_if.req_data.rw - && dma_rsp_buf_ready; - - wire dma_active = dma_wr_b | dma_rd_b; - - wire lsu_active = per_bank_req_valid[i] && per_bank_req_ready[i]; - - // Bank-facing AMO interface, tied off when the A extension is absent so - // none of the atomic hardware is synthesized (the dcache counterpart is - // AMO_ENABLE in VX_cache_bank). - wire amo_busy; // an atomic owns the bank port - wire amo_wb_store; // owed atomic write-back - wire [BANK_ADDR_WIDTH-1:0] amo_wb_addr; - wire [WORD_SIZE-1:0] amo_wb_byteen; - wire [WORD_WIDTH-1:0] amo_wb_data; - wire sc_resolving; // SC outcome replaces the response - wire sc_fail; - wire sc_lane_hi; // outcome lane of a resolving .W SC - - if (AMO_ENABLE != 0) begin : g_amo - - // Atomics. The LSU sends an AMO with rw=0, so the read returning the old - // value is the ordinary read path and needs no change. What the bank owes - // is the write of the computed value. - // - // The old word is registered before the ALU sees it. Feeding the ALU - // straight from the SRAM output puts the BRAM clock-to-out, the adder and - // the BRAM setup in a single period; the register splits that into - // BRAM-out-to-flop and flop-through-adder-to-BRAM-in, which is what keeps - // the atomic adder off the critical path. - // - // The port is single-ported, so an atomic owns its bank for three cycles: - // read, capture, write back. - reg amo_rd_valid_r; - reg amo_wb_valid_r; - reg [WORD_WIDTH-1:0] amo_old_r; - reg [BANK_ADDR_WIDTH-1:0] amo_wb_addr_r; - reg [WORD_SIZE-1:0] amo_wb_byteen_r; - reg [$bits(amo_op_e)-1:0] amo_wb_op_r; - reg amo_wb_unsigned_r; - reg [WORD_WIDTH-1:0] amo_wb_rhs_r; - `ifdef VX_CFG_EXT_ZACAS_ENABLE - reg [AMO_CMP_WIDTH-1:0] amo_wb_cmp_r; - `endif - - wire amo_accept = lsu_active && per_bank_req_amo_valid[i]; - - // Reservation commit stage. Every accepted request that can touch a - // reservation -- an atomic, or a plain store -- reaches it one cycle - // after acceptance, so a single address port serves both. The look-ahead - // read is driven from the acceptance cycle, which is what makes the - // registered reservation entry land here. - wire is_lr_in = per_bank_req_amo_valid[i] - && (amo_op_e'(per_bank_req_amo_op[i]) == AMO_OP_LR); - wire is_sc_in = per_bank_req_amo_valid[i] - && (amo_op_e'(per_bank_req_amo_op[i]) == AMO_OP_SC); - - reg res_valid_r; - reg res_is_lr_r; - reg res_is_sc_r; - reg res_is_amo_r; - reg res_is_store_r; - reg [BANK_ADDR_WIDTH-1:0] res_addr_r; - reg [HART_ID_WIDTH-1:0] res_hart_id_r; - - always @(posedge clk) begin - if (reset) begin - res_valid_r <= 1'b0; - end else begin - res_valid_r <= lsu_active; - end - if (lsu_active) begin - res_is_lr_r <= is_lr_in; - res_is_sc_r <= is_sc_in; - res_is_amo_r <= per_bank_req_amo_valid[i]; - res_is_store_r <= per_bank_req_rw[i]; - res_addr_r <= per_bank_req_addr[i]; - res_hart_id_r <= per_bank_req_amo_hart_id[i]; - end - end - - // A store-conditional fails when its reservation is gone. The outcome is - // known at the commit stage, one cycle before the write-back, so it can - // both suppress the write and be returned as this request's response. - wire res_check; - assign sc_fail = res_is_sc_r && ~res_check; - assign sc_resolving = res_valid_r && res_is_sc_r; - - // The SC outcome must land in the addressed lane: the LSU extracts - // the subword by address, so a .W outcome left at bit 0 would read - // as 0 ("success") for every upper-lane SC, including a failed one - // whose store was suppressed. - if (WORD_WIDTH == 64) begin : g_sc_lane - reg res_lane_hi_r; - always @(posedge clk) begin - if (lsu_active) begin - res_lane_hi_r <= ~per_bank_req_byteen[i][0]; - end - end - assign sc_lane_hi = res_lane_hi_r; - end else begin : g_sc_lane0 - assign sc_lane_hi = 1'b0; - end - - // Everything that ends up writing the word: a plain store, a - // read-modify-write atomic (which carries rw=0 and so is not a store by - // the request flag), and a store-conditional that found its reservation. - wire res_commits_store = res_is_store_r - || (res_is_amo_r && ~res_is_lr_r && ~res_is_sc_r) - || (res_is_sc_r && res_check); - - reg amo_wb_is_lr_r; - reg amo_wb_sc_fail_r; - always @(posedge clk) begin - if (amo_rd_valid_r) begin - amo_wb_is_lr_r <= res_is_lr_r; - amo_wb_sc_fail_r <= sc_fail; - end - end - - // A load-reserved leaves the word alone, and a failed store-conditional - // must not store; every other atomic writes its computed value back. - assign amo_wb_store = amo_wb_valid_r && ~amo_wb_is_lr_r && ~amo_wb_sc_fail_r; - assign amo_busy = amo_rd_valid_r || amo_wb_valid_r; - - always @(posedge clk) begin - if (reset) begin - amo_rd_valid_r <= 1'b0; - amo_wb_valid_r <= 1'b0; - end else begin - amo_rd_valid_r <= amo_accept; - amo_wb_valid_r <= amo_rd_valid_r; - end - if (amo_rd_valid_r) begin - amo_old_r <= per_bank_rsp_data[i]; - end - if (amo_accept) begin - amo_wb_addr_r <= per_bank_req_addr[i]; - amo_wb_byteen_r <= per_bank_req_byteen[i]; - amo_wb_op_r <= per_bank_req_amo_op[i]; - amo_wb_unsigned_r <= per_bank_req_amo_unsigned[i]; - amo_wb_rhs_r <= per_bank_req_data[i]; - `ifdef VX_CFG_EXT_ZACAS_ENABLE - amo_wb_cmp_r <= per_bank_req_amo_cmp[i]; - `endif - end - end - - // old_word is this cycle's SRAM output -- the same value the response - // carries back, which is why ret_word is not used here. - wire [63:0] amo_new_word, amo_ret_word; - - // Lane-align the RMW operands. The bank word is XLEN wide, so on a - // 64-bit hart a .W atomic addresses one of two 32-bit lanes; the - // ALU computes right-aligned. Store data arrives lane-positioned - // (same packing as a plain store), so old and rhs shift down by - // the lane offset and the result shifts back up under the request - // byteen. The comparand arrives right-aligned from rs3 and needs - // no shift. The addressed lane is recovered from the byteen: an - // atomic is naturally aligned, so a .W to the upper lane never - // covers byte 0 and a .D covers the full word. - wire [63:0] amo_old_lane, amo_rhs_lane; - wire [1:0] amo_width; - if (WORD_WIDTH == 64) begin : g_amo_lane - wire lane_hi = ~amo_wb_byteen_r[0]; - assign amo_width = (&amo_wb_byteen_r) ? 2'd3 : 2'd2; - assign amo_old_lane = lane_hi ? {32'h0, amo_old_r[63:32]} : amo_old_r; - assign amo_rhs_lane = lane_hi ? {32'h0, amo_wb_rhs_r[63:32]} : amo_wb_rhs_r; - assign amo_wb_data = lane_hi ? {amo_new_word[31:0], 32'h0} : amo_new_word; - end else begin : g_amo_word - assign amo_width = 2'd2; - assign amo_old_lane = 64'(amo_old_r); - assign amo_rhs_lane = 64'(amo_wb_rhs_r); - assign amo_wb_data = amo_new_word[WORD_WIDTH-1:0]; - end - // VX_CFG_AMO_RS_SIZE sizes ONE shared LLC bank's contention; deployed - // per lmem bank it multiplies by NUM_LMEM_BANKS x NUM_CORES. After the - // holder-credit protocol a hot word maps to exactly one station, so - // depth only reduces false eviction between distinct reserved words - // aliasing in one bank -- and a spurious SC failure is architecturally - // legal. Clamp the depth. - // A credit is spent per refused LR, not per cycle: a warp's LR reaches - // the bank as one request per lane, back to back, so between a holder's - // LR and its SC every other hart of the core can be refused, and a - // refused hart can retry once more. The budget covers two refusals per - // hart; a smaller one lets one warp's own LR burst displace its holder - // and a contended retry loop stops making progress. - VX_amo_unit #( - .NUM_RES_ENTRIES (`MIN(`VX_CFG_AMO_RS_SIZE, 4)), - .HOLD_CREDIT_BITS (`CLOG2(`VX_CFG_NUM_WARPS * `VX_CFG_NUM_THREADS) + 1), - .LINE_ADDR_BITS (BANK_ADDR_WIDTH), - .DATA_WIDTH (WORD_WIDTH) - ) amo_unit ( - .clk (clk), - .reset (reset), - .pipe_stall (1'b0), - .compute_op (amo_op_e'(amo_wb_op_r)), - .compute_unsigned (amo_wb_unsigned_r), - .compute_width (amo_width), - .compute_old (amo_old_lane), - .compute_rhs (amo_rhs_lane), - `ifdef VX_CFG_EXT_ZACAS_ENABLE - .compute_cmp (64'(amo_wb_cmp_r)), - `else - .compute_cmp (64'b0), - `endif - .compute_new_word (amo_new_word), - .compute_ret_word (amo_ret_word), - // A load-reserved claims the word; a store-conditional gives up its - // own reservation whether it succeeds or fails; any committing store - // breaks whoever else holds one. - .res_reserve (res_valid_r && res_is_lr_r), - .res_clear (res_valid_r && res_is_sc_r), - .res_invalidate (res_valid_r && res_commits_store), - .res_hart_id (res_hart_id_r), - .res_line_addr (res_addr_r), - .res_line_addr_n (per_bank_req_addr[i]), - .res_check (res_check) - ); - `UNUSED_VAR (amo_ret_word) - if (WORD_WIDTH < 64) begin : g_amo_hi_unused - `UNUSED_VAR (amo_new_word[63:WORD_WIDTH]) - end - - assign amo_wb_addr = amo_wb_addr_r; - assign amo_wb_byteen = amo_wb_byteen_r; - - end else begin : g_no_amo - assign amo_busy = 1'b0; - assign amo_wb_store = 1'b0; - assign amo_wb_addr = '0; - assign amo_wb_byteen = '0; - assign amo_wb_data = '0; - assign sc_resolving = 1'b0; - assign sc_fail = 1'b0; - assign sc_lane_hi = 1'b0; - `UNUSED_VAR (per_bank_req_amo_valid[i]) - `UNUSED_VAR (per_bank_req_amo_op[i]) - `UNUSED_VAR (per_bank_req_amo_unsigned[i]) - `UNUSED_VAR (per_bank_req_amo_hart_id[i]) - `ifdef VX_CFG_EXT_ZACAS_ENABLE - `UNUSED_VAR (per_bank_req_amo_cmp[i]) - `endif - end - - // SRAM address / write-data / write-enable mux: DMA first, then an owed - // atomic write-back, then the incoming request - wire [BANK_ADDR_WIDTH-1:0] bank_sram_addr; - wire [WORD_WIDTH-1:0] bank_sram_wdata; - wire [WORD_SIZE-1:0] bank_sram_wren; - - assign bank_sram_addr = dma_active ? BANK_ADDR_WIDTH'(dma_bus_if.req_data.addr) - : amo_wb_store ? amo_wb_addr - : per_bank_req_addr[i]; - assign bank_sram_wdata = dma_wr_b ? dma_bus_if.req_data.data[i*WORD_WIDTH +: WORD_WIDTH] - : amo_wb_store ? amo_wb_data - : per_bank_req_data[i]; - assign bank_sram_wren = dma_wr_b ? dma_bus_if.req_data.byteen[i*WORD_SIZE +: WORD_SIZE] - : amo_wb_store ? amo_wb_byteen - : per_bank_req_byteen[i]; - - VX_sp_ram #( - .DATAW (WORD_WIDTH), - .SIZE (WORDS_PER_BANK), - .WRENW (WORD_SIZE), - .OUT_REG (1), - .RDW_MODE ("R") - ) lmem_store ( - .clk (clk), - .reset (reset), - .read (dma_rd_b || (lsu_active && ~per_bank_req_rw[i])), - .write (dma_wr_b || amo_wb_store || (lsu_active && per_bank_req_rw[i])), - .wren (bank_sram_wren), - .addr (bank_sram_addr), - .wdata (bank_sram_wdata), - .rdata (per_bank_rsp_data[i]) - ); - - // Read-during-write hazard: stalls LSU reads to an address written last cycle - // (SRAM OUT_REG + RDW_MODE="R" returns stale data on same-cycle read-after-write). - // DMA reads bypass this check. - - reg [BANK_ADDR_WIDTH-1:0] last_wr_addr; - reg last_wr_valid; - always @(posedge clk) begin - if (reset) begin - last_wr_valid <= 0; - end else begin - last_wr_valid <= dma_wr_b || amo_wb_store || (lsu_active && per_bank_req_rw[i]); - end - last_wr_addr <= bank_sram_addr; - end - wire is_rdw_hazard = last_wr_valid && ~per_bank_req_rw[i] && (per_bank_req_addr[i] == last_wr_addr); - - // LSU response valid / request ready — blocked by DMA and RDW hazards - - assign bank_rsp_valid = per_bank_req_valid[i] - && ~dma_active - && ~amo_busy - && ~per_bank_req_rw[i] - && ~is_rdw_hazard; - - assign per_bank_req_ready[i] = ~dma_active - && ~amo_busy - && (bank_rsp_ready || per_bank_req_rw[i]) - && ~is_rdw_hazard; - - // Delay tag/idx to align with SRAM 1-cycle output latency - VX_pipe_buffer #( - .DATAW (REQ_SEL_WIDTH + BANK_TAG_WIDTH) - ) bram_buf ( - .clk (clk), - .reset (reset), - .valid_in (bank_rsp_valid), - .ready_in (bank_rsp_ready), - .data_in ({per_bank_req_idx[i], per_bank_req_tag[i]}), - .data_out ({per_bank_rsp_idx[i], per_bank_rsp_tag[i]}), - .valid_out (per_bank_rsp_valid[i]), - .ready_out (per_bank_rsp_ready[i]) - ); - - // A store-conditional returns its outcome, not the word it read: zero - // for success, one for failure, positioned in the addressed lane for - // the LSU's subword extraction. The outcome is resolved at the commit - // stage, which is the same cycle the response carries the SRAM output, - // so it substitutes here rather than needing a response of its own. - wire [WORD_WIDTH-1:0] bank_rsp_word = sc_resolving - ? (sc_lane_hi ? (WORD_WIDTH'(sc_fail) << (WORD_WIDTH/2)) - : WORD_WIDTH'(sc_fail)) - : per_bank_rsp_data[i]; - - // Back-pressure-safe LSU response data. The bank SRAM OUT_REG - // (per_bank_rsp_data) is read live by the response xbar, but the - // response valid/tag is elastic-buffered, so a back-pressured LSU - // response can sit valid for >1 cycle. During that wait an interleaving - // DMA read on this bank re-drives the shared OUT_REG, corrupting the - // held LSU response (WGMMA+DXA issues concurrent LSU A-reads and DMA - // B-reads to the same banks — the only workload that does). Latch the - // data on the first response-valid cycle (rdata is still valid then) - // and serve the latched copy while the response is stalled. The latched - // copy is the substituted word, so a stalled SC keeps its outcome. - reg [WORD_WIDTH-1:0] lsu_rsp_hold_data_r; - reg lsu_rsp_hold_valid_r; - always @(posedge clk) begin - if (reset) begin - lsu_rsp_hold_valid_r <= 1'b0; - end else if (per_bank_rsp_valid[i] && per_bank_rsp_ready[i]) begin - lsu_rsp_hold_valid_r <= 1'b0; // response consumed - end else if (per_bank_rsp_valid[i] && ~lsu_rsp_hold_valid_r) begin - lsu_rsp_hold_data_r <= bank_rsp_word; // capture before DMA can clobber - lsu_rsp_hold_valid_r <= 1'b1; - end - end - assign bank_lsu_rsp_data[i] = lsu_rsp_hold_valid_r ? lsu_rsp_hold_data_r - : bank_rsp_word; - - end - - // bank responses gather - - wire [NUM_REQS-1:0] rsp_valid_out; - wire [NUM_REQS-1:0][RSP_DATAW-1:0] rsp_data_out; - wire [NUM_REQS-1:0] rsp_ready_out; - - if (DO_READ_MERGE) begin : g_rsp_mcast - // A merged read's response goes to its leader's port and to every - // merged port: each output takes it when its arbiter grants it, and the - // bank's response is released once all its targets have taken it. - localparam GROUP_BITS = `CLOG2(MERGE_GROUP); - wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_pending; - wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_taken; - wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_valid; - wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_ready; - wire [NUM_BANKS-1:0][RSP_DATAW-1:0] per_bank_rsp_pdata; - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_banks - wire [MERGE_GROUP-1:0] merged_mask = per_bank_rsp_tag[i][MERGE_GROUP-1:0]; - wire [REQ_SEL_WIDTH-1:0] group_base = (per_bank_rsp_idx[i] >> GROUP_BITS) << GROUP_BITS; - wire [NUM_REQS-1:0] targets = (NUM_REQS'(1) << per_bank_rsp_idx[i]) - | (NUM_REQS'(merged_mask) << group_base); - reg [NUM_REQS-1:0] sent_r; - assign per_bank_rsp_pdata[i] = {bank_lsu_rsp_data[i], per_bank_rsp_tag[i][MERGE_BITS +: TAG_WIDTH]}; - assign lane_pending[i] = targets & ~sent_r & {NUM_REQS{per_bank_rsp_valid[i]}}; - for (genvar j = 0; j < NUM_REQS; ++j) begin : g_taken - assign per_output_valid[j][i] = lane_pending[i][j]; - assign lane_taken[i][j] = lane_pending[i][j] && per_output_ready[j][i]; - end - assign per_bank_rsp_ready[i] = per_bank_rsp_valid[i] && ((lane_pending[i] & ~lane_taken[i]) == '0); - always @(posedge clk) begin - if (reset) begin - sent_r <= '0; - end else if (per_bank_rsp_valid[i]) begin - sent_r <= per_bank_rsp_ready[i] ? '0 : (sent_r | lane_taken[i]); - end - end - end - for (genvar j = 0; j < NUM_REQS; ++j) begin : g_outputs - VX_stream_arb #( - .NUM_INPUTS (NUM_BANKS), - .NUM_OUTPUTS (1), - .DATAW (RSP_DATAW), - .ARBITER ("P"), - .OUT_BUF (OUT_BUF) - ) rsp_arb ( - .clk (clk), - .reset (reset), - .valid_in (per_output_valid[j]), - .data_in (per_bank_rsp_pdata), - .ready_in (per_output_ready[j]), - .valid_out (rsp_valid_out[j]), - .data_out (rsp_data_out[j]), - `UNUSED_PIN (sel_out), - .ready_out (rsp_ready_out[j]) - ); - end - end else begin : g_rsp_xbar - wire [NUM_BANKS-1:0][RSP_DATAW-1:0] per_bank_rsp_data_aos; - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_rsp_data_aos - assign per_bank_rsp_data_aos[i] = {bank_lsu_rsp_data[i], per_bank_rsp_tag[i]}; - end - - VX_stream_xbar #( - .NUM_INPUTS (NUM_BANKS), - .NUM_OUTPUTS (NUM_REQS), - .DATAW (RSP_DATAW), - .ARBITER ("P"), // this priority arbiter has negligeable impact on performance - .OUT_BUF (OUT_BUF) - ) rsp_xbar ( - .clk (clk), - .reset (reset), - `UNUSED_PIN (collisions), - .sel_in (per_bank_rsp_idx), - .valid_in (per_bank_rsp_valid), - .data_in (per_bank_rsp_data_aos), - .ready_in (per_bank_rsp_ready), - .valid_out (rsp_valid_out), - .data_out (rsp_data_out), - .ready_out (rsp_ready_out), - `UNUSED_PIN (sel_out) - ); - end - - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_lsu_bus_if - assign lsu_bus_if[i].rsp_valid = rsp_valid_out[i]; - assign lsu_bus_if[i].rsp_data = rsp_data_out[i]; - assign rsp_ready_out[i] = lsu_bus_if[i].rsp_ready; - end - -`ifdef PERF_ENABLE - // per cycle: reads, writes - wire [`CLOG2(NUM_REQS+1)-1:0] perf_reads_per_cycle; - wire [`CLOG2(NUM_REQS+1)-1:0] perf_writes_per_cycle; - wire [`CLOG2(NUM_REQS+1)-1:0] perf_crsp_stall_per_cycle; - - wire [NUM_REQS-1:0] req_rw; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_rw - assign req_rw[i] = lsu_bus_if[i].req_data.rw; - end - - wire [NUM_REQS-1:0] perf_reads_per_req, perf_writes_per_req; - wire [NUM_REQS-1:0] perf_crsp_stall_per_req = rsp_valid_out & ~rsp_ready_out; - - // a merged read counts once, as the bank access that serves it - `BUFFER(perf_reads_per_req, req_valid_in & req_ready_in & ~req_rw); - `BUFFER(perf_writes_per_req, req_valid_in & req_ready_in & req_rw); - - `POP_COUNT(perf_reads_per_cycle, perf_reads_per_req); - `POP_COUNT(perf_writes_per_cycle, perf_writes_per_req); - `POP_COUNT(perf_crsp_stall_per_cycle, perf_crsp_stall_per_req); - - reg [PERF_CTR_BITS-1:0] perf_reads; - reg [PERF_CTR_BITS-1:0] perf_writes; - reg [PERF_CTR_BITS-1:0] perf_crsp_stalls; - - always @(posedge clk) begin - if (reset) begin - perf_reads <= '0; - perf_writes <= '0; - perf_crsp_stalls <= '0; - end else begin - perf_reads <= perf_reads + PERF_CTR_BITS'(perf_reads_per_cycle); - perf_writes <= perf_writes + PERF_CTR_BITS'(perf_writes_per_cycle); - perf_crsp_stalls <= perf_crsp_stalls + PERF_CTR_BITS'(perf_crsp_stall_per_cycle); - end - end - - assign lmem_perf.reads = perf_reads; - assign lmem_perf.writes = perf_writes; - assign lmem_perf.bank_stalls = perf_collisions; - assign lmem_perf.crsp_stalls = perf_crsp_stalls; - -`endif - -`ifdef DBG_TRACE_MEM - - wire [NUM_BANKS-1:0][TAG_WIDTH-UUID_WIDTH-1:0] per_bank_req_tag_value; - wire [NUM_BANKS-1:0][UUID_WIDTH-1:0] per_bank_req_uuid; - - wire [NUM_BANKS-1:0][TAG_WIDTH-UUID_WIDTH-1:0] per_bank_rsp_tag_value; - wire [NUM_BANKS-1:0][UUID_WIDTH-1:0] per_bank_rsp_uuid; - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_req_uuid - assign per_bank_req_tag_value[i] = per_bank_req_tag[i][MERGE_BITS +: (TAG_WIDTH-UUID_WIDTH)]; - assign per_bank_rsp_tag_value[i] = per_bank_rsp_tag[i][MERGE_BITS +: (TAG_WIDTH-UUID_WIDTH)]; - if (UUID_WIDTH != 0) begin : g_uuid - assign per_bank_req_uuid[i] = per_bank_req_tag[i][BANK_TAG_WIDTH-1 -: UUID_WIDTH]; - assign per_bank_rsp_uuid[i] = per_bank_rsp_tag[i][BANK_TAG_WIDTH-1 -: UUID_WIDTH]; - end else begin : g_no_uuid - assign per_bank_req_uuid[i] = 0; - assign per_bank_rsp_uuid[i] = 0; - end - end - - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_trace - always @(posedge clk) begin - if (lsu_bus_if[i].req_valid && lsu_bus_if[i].req_ready) begin - if (lsu_bus_if[i].req_data.rw) begin - `TRACE(2, ("%t: %s core-wr-req[%0d]: addr=0x%0h, byteen=0x%h, data=0x%h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, lsu_bus_if[i].req_data.addr, lsu_bus_if[i].req_data.byteen, lsu_bus_if[i].req_data.data, lsu_bus_if[i].req_data.tag.value, lsu_bus_if[i].req_data.tag.uuid)) - end else begin - `TRACE(2, ("%t: %s core-rd-req[%0d]: addr=0x%0h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, lsu_bus_if[i].req_data.addr, lsu_bus_if[i].req_data.tag.value, lsu_bus_if[i].req_data.tag.uuid)) - end - end - if (lsu_bus_if[i].rsp_valid && lsu_bus_if[i].rsp_ready) begin - `TRACE(2, ("%t: %s core-rd-rsp[%0d]: data=0x%h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, lsu_bus_if[i].rsp_data.data, lsu_bus_if[i].rsp_data.tag.value, lsu_bus_if[i].rsp_data.tag.uuid)) - end - end - end - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_bank_trace - always @(posedge clk) begin - if (per_bank_req_valid[i] && per_bank_req_ready[i]) begin - if (per_bank_req_rw[i]) begin - `TRACE(2, ("%t: %s bank-wr-req[%0d]: addr=0x%0h, byteen=0x%h, data=0x%h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, per_bank_req_addr[i], per_bank_req_byteen[i], per_bank_req_data[i], per_bank_req_tag_value[i], per_bank_req_uuid[i])) - end else begin - `TRACE(2, ("%t: %s bank-rd-req[%0d]: addr=0x%0h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, per_bank_req_addr[i], per_bank_req_tag_value[i], per_bank_req_uuid[i])) - end - end - if (per_bank_rsp_valid[i] && per_bank_rsp_ready[i]) begin - `TRACE(2, ("%t: %s bank-rd-rsp[%0d]: data=0x%h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, per_bank_rsp_data[i], per_bank_rsp_tag_value[i], per_bank_rsp_uuid[i])) - end - end - end - -`endif - -endmodule diff --git a/hw/syn/altera/dut/catalog.mk b/hw/syn/altera/dut/catalog.mk index 74563d021f..233d580c83 100644 --- a/hw/syn/altera/dut/catalog.mk +++ b/hw/syn/altera/dut/catalog.mk @@ -58,8 +58,8 @@ issue_PRJ := VX_issue_top issue_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm $(FPU_INC) \ -I$(IP_CACHE_DIR) -I$(UNITTEST_DIR)/issue -lmem_PRJ := VX_local_mem_top -lmem_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(UNITTEST_DIR)/local_mem +lmem_PRJ := VX_lmem_unit_top +lmem_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(UNITTEST_DIR)/lmem_unit mem_unit_PRJ := VX_mem_unit_top mem_unit_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/core -I$(RTL_DIR)/fpu \ diff --git a/hw/syn/xilinx/dut/catalog.mk b/hw/syn/xilinx/dut/catalog.mk index a00e1f681d..12167b355b 100644 --- a/hw/syn/xilinx/dut/catalog.mk +++ b/hw/syn/xilinx/dut/catalog.mk @@ -87,8 +87,8 @@ gfx_PKG = $(RTL_DIR)/fpu/VX_fpu_pkg.sv $(RTL_DIR)/dxa/VX_dxa_pkg.sv $(RTL_DIR)/ issue_PRJ := VX_issue_top issue_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm $(FPU_INC) -I$(UNITTEST_DIR)/issue -lmem_PRJ := VX_local_mem_top -lmem_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(UNITTEST_DIR)/local_mem +lmem_PRJ := VX_lmem_unit_top +lmem_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(UNITTEST_DIR)/lmem_unit mem_unit_PRJ := VX_mem_unit_top mem_unit_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/core -I$(RTL_DIR)/fpu -I$(UNITTEST_DIR)/mem_unit diff --git a/hw/syn/yosys/dut/catalog.mk b/hw/syn/yosys/dut/catalog.mk index 75ae90439c..bef32bddfa 100644 --- a/hw/syn/yosys/dut/catalog.mk +++ b/hw/syn/yosys/dut/catalog.mk @@ -9,7 +9,7 @@ # synthesizes, so both gates measure the same modules and a divergence between # them is meaningful rather than an artefact of two DUT lists. -DUTS := cache core tcu gfx tex raster om rtu dxa mufu vm tensor top vortex +DUTS := cache core lmem tcu gfx tex raster om rtu dxa mufu vm tensor top vortex UNITTEST_DIR ?= $(VORTEX_HOME)/hw/unittest AFU_RTL_DIR ?= $(VORTEX_HOME)/hw/rtl/afu @@ -22,6 +22,12 @@ core_TOP := VX_core_top core_INC := -I$(UNITTEST_DIR)/core core_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE +# Warp-wide local memory at the configured core shape; its gate builds set the +# shape, as the Xilinx lmem DUT's do. +lmem_TOP := VX_lmem_unit_top +lmem_INC := -I$(UNITTEST_DIR)/lmem_unit +lmem_CFG := + tcu_TOP := VX_tcu_unit_top tcu_INC := -I$(UNITTEST_DIR)/tcu_unit # EXT_F_DISABLE mirrors the Xilinx catalog: this DUT is the tensor datapath, and diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index c6c1f5f4e1..f8e4a6ea79 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -4,7 +4,7 @@ all: $(MAKE) -C cache $(MAKE) -C core $(MAKE) -C issue - $(MAKE) -C local_mem + $(MAKE) -C lmem_unit $(MAKE) -C mem_unit $(MAKE) -C tcu_fedp $(MAKE) -C cta_dispatcher @@ -39,7 +39,7 @@ run: $(MAKE) -C cache run $(MAKE) -C core run $(MAKE) -C issue run - $(MAKE) -C local_mem run + $(MAKE) -C lmem_unit run $(MAKE) -C mem_unit run $(MAKE) -C tcu_fedp run $(MAKE) -C cta_dispatcher run @@ -188,7 +188,7 @@ clean: $(MAKE) -C cache clean $(MAKE) -C core clean $(MAKE) -C issue clean - $(MAKE) -C local_mem clean + $(MAKE) -C lmem_unit clean $(MAKE) -C mem_unit clean $(MAKE) -C tcu_fedp clean $(MAKE) -C cta_dispatcher clean diff --git a/hw/unittest/local_mem/Makefile b/hw/unittest/lmem_unit/Makefile similarity index 93% rename from hw/unittest/local_mem/Makefile rename to hw/unittest/lmem_unit/Makefile index e2488d0855..58eeddf360 100644 --- a/hw/unittest/local_mem/Makefile +++ b/hw/unittest/lmem_unit/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk -PROJECT := local_mem +PROJECT := lmem_unit RTL_DIR := $(VORTEX_HOME)/hw/rtl DPI_DIR := $(VORTEX_HOME)/hw/dpi @@ -20,6 +20,6 @@ RTL_PKGS := $(RTL_DIR)/VX_gpu_pkg.sv RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs RTL_INCLUDE += -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(SRC_DIR) -TOP := VX_local_mem_top +TOP := VX_lmem_unit_top include ../common.mk \ No newline at end of file diff --git a/hw/unittest/lmem_unit/VX_lmem_unit_top.sv b/hw/unittest/lmem_unit/VX_lmem_unit_top.sv new file mode 100644 index 0000000000..0e6dabe6f2 --- /dev/null +++ b/hw/unittest/lmem_unit/VX_lmem_unit_top.sv @@ -0,0 +1,155 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Synthesis wrapper of the warp-wide local memory at the configured core +// shape: one warp request port per LSU block, and the row-wide DMA port. The +// requests arrive through a register each, as from the LSU blocks' local-path +// buffers and the DMA arbiter in the core; out-of-context synthesis leaves +// paths from the ports untimed, so this is what puts the unit's request +// decoding on the timing report. +module VX_lmem_unit_top import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter SIZE = (1 << `VX_CFG_LMEM_LOG_SIZE), + parameter NUM_BLOCKS = `VX_CFG_NUM_LSU_BLOCKS, + parameter NUM_LANES = `VX_CFG_NUM_LSU_LANES, + parameter NUM_BANKS = `VX_CFG_LMEM_NUM_BANKS, + parameter WORD_SIZE = LSU_WORD_SIZE, + parameter TAG_WIDTH = LSU_TAG_WIDTH, + parameter ADDR_WIDTH = `CLOG2(SIZE / WORD_SIZE), + parameter WORD_WIDTH = WORD_SIZE * 8, + parameter DMA_ADDR_WIDTH = ADDR_WIDTH - `CLOG2(NUM_BANKS), + parameter DMA_DATA_SIZE = NUM_BANKS * WORD_SIZE, + parameter DMA_TAG_WIDTH = LMEM_DMA_TAG_WIDTH +) ( + input wire clk, + input wire reset, + + // warp requests + input wire [NUM_BLOCKS-1:0] req_valid, + input wire [NUM_BLOCKS-1:0] req_rw, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0] req_mask, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][WORD_SIZE-1:0] req_byteen, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][ADDR_WIDTH-1:0] req_addr, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][MEM_ATTR_WIDTH-1:0] req_attr, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][WORD_WIDTH-1:0] req_data, + input wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] req_tag, + output wire [NUM_BLOCKS-1:0] req_ready, + + // responses + output wire [NUM_BLOCKS-1:0] rsp_valid, + output wire [NUM_BLOCKS-1:0][NUM_LANES-1:0] rsp_mask, + output wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data, + output wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] rsp_tag, + input wire [NUM_BLOCKS-1:0] rsp_ready, + + // DMA requests + input wire dma_req_valid, + input wire dma_req_rw, + input wire [DMA_ADDR_WIDTH-1:0] dma_req_addr, + input wire [DMA_DATA_SIZE*8-1:0] dma_req_data, + input wire [DMA_DATA_SIZE-1:0] dma_req_byteen, + input wire [DMA_TAG_WIDTH-1:0] dma_req_tag, + output wire dma_req_ready, + + // DMA responses + output wire dma_rsp_valid, + output wire [DMA_DATA_SIZE*8-1:0] dma_rsp_data, + output wire [DMA_TAG_WIDTH-1:0] dma_rsp_tag, + input wire dma_rsp_ready +); + localparam REQ_DATAW = NUM_LANES + 1 + NUM_LANES * (LSU_ADDR_WIDTH + WORD_WIDTH + WORD_SIZE + MEM_ATTR_WIDTH) + TAG_WIDTH; + localparam DMA_REQ_DATAW = 1 + DMA_ADDR_WIDTH + DMA_DATA_SIZE * 8 + DMA_DATA_SIZE + MEM_ATTR_WIDTH + DMA_TAG_WIDTH; + + VX_lsu_mem_if #( + .NUM_LANES (NUM_LANES), + .DATA_SIZE (WORD_SIZE), + .TAG_WIDTH (TAG_WIDTH) + ) lsu_mem_if[NUM_BLOCKS](); + + VX_mem_bus_if #( + .DATA_SIZE (NUM_BANKS * WORD_SIZE), + .TAG_WIDTH (DMA_TAG_WIDTH), + .ADDR_WIDTH (DMA_ADDR_WIDTH) + ) dma_bus_if(); + + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_blocks + wire [NUM_LANES-1:0][LSU_ADDR_WIDTH-1:0] lsu_addr; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_addr + assign lsu_addr[i] = LSU_ADDR_WIDTH'(req_addr[k][i]); + end + + VX_elastic_buffer #( + .DATAW (REQ_DATAW), + .SIZE (1) + ) req_buf ( + .clk (clk), + .reset (reset), + .valid_in (req_valid[k]), + .ready_in (req_ready[k]), + .data_in ({req_mask[k], req_rw[k], lsu_addr, req_data[k], req_byteen[k], req_attr[k], req_tag[k]}), + .data_out (lsu_mem_if[k].req_data), + .valid_out (lsu_mem_if[k].req_valid), + .ready_out (lsu_mem_if[k].req_ready) + ); + + assign rsp_valid[k] = lsu_mem_if[k].rsp_valid; + assign rsp_mask[k] = lsu_mem_if[k].rsp_data.mask; + assign rsp_data[k] = lsu_mem_if[k].rsp_data.data; + assign rsp_tag[k] = lsu_mem_if[k].rsp_data.tag; + assign lsu_mem_if[k].rsp_ready = rsp_ready[k]; + end + + VX_elastic_buffer #( + .DATAW (DMA_REQ_DATAW), + .SIZE (1) + ) dma_req_buf ( + .clk (clk), + .reset (reset), + .valid_in (dma_req_valid), + .ready_in (dma_req_ready), + .data_in ({dma_req_rw, dma_req_addr, dma_req_data, dma_req_byteen, MEM_ATTR_WIDTH'(0), dma_req_tag}), + .data_out (dma_bus_if.req_data), + .valid_out (dma_bus_if.req_valid), + .ready_out (dma_bus_if.req_ready) + ); + + assign dma_rsp_valid = dma_bus_if.rsp_valid; + assign dma_rsp_data = dma_bus_if.rsp_data.data; + assign dma_rsp_tag = dma_bus_if.rsp_data.tag; + assign dma_bus_if.rsp_ready = dma_rsp_ready; + + VX_lmem_unit #( + .INSTANCE_ID (INSTANCE_ID), + .SIZE (SIZE), + .NUM_BLOCKS (NUM_BLOCKS), + .NUM_LANES (NUM_LANES), + .NUM_BANKS (NUM_BANKS), + .WORD_SIZE (WORD_SIZE), + .ADDR_WIDTH (ADDR_WIDTH), + .TAG_WIDTH (TAG_WIDTH), + .DMA_ENABLE (1), + .DMA_TAG_WIDTH (DMA_TAG_WIDTH), + .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) + ) lmem_unit ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (lmem_perf), + `endif + .lsu_mem_if (lsu_mem_if), + .dma_bus_if (dma_bus_if) + ); + +endmodule diff --git a/hw/unittest/local_mem/main.cpp b/hw/unittest/lmem_unit/main.cpp similarity index 100% rename from hw/unittest/local_mem/main.cpp rename to hw/unittest/lmem_unit/main.cpp diff --git a/hw/unittest/local_mem/VX_local_mem_top.sv b/hw/unittest/local_mem/VX_local_mem_top.sv deleted file mode 100644 index cf323cb596..0000000000 --- a/hw/unittest/local_mem/VX_local_mem_top.sv +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_define.vh" - -module VX_local_mem_top import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - - // Size of cache in bytes - parameter SIZE = (1024*16*8), - - // Number of Word requests per cycle - parameter NUM_REQS = 4, - // Number of banks - parameter NUM_BANKS = 4, - - // Size of a word in bytes - parameter WORD_SIZE = `VX_CFG_XLEN/8, - - // Request tag size - parameter TAG_WIDTH = 16 + UUID_WIDTH, - - // Address width - parameter NUM_WORDS = SIZE / WORD_SIZE, - parameter WORDS_PER_BANK = NUM_WORDS / NUM_BANKS, - parameter BANK_ADDR_WIDTH = `CLOG2(WORDS_PER_BANK), - parameter ADDR_WIDTH = BANK_ADDR_WIDTH + `CLOG2(NUM_BANKS) - ) ( - input wire clk, - input wire reset, - - // Core request - input wire [NUM_REQS-1:0] mem_req_valid, - input wire [NUM_REQS-1:0] mem_req_rw, - input wire [NUM_REQS-1:0][WORD_SIZE-1:0] mem_req_byteen, - input wire [NUM_REQS-1:0][ADDR_WIDTH-1:0] mem_req_addr, - input wire [NUM_REQS-1:0][MEM_ATTR_WIDTH-1:0] mem_req_attr, - input wire [NUM_REQS-1:0][WORD_SIZE*8-1:0] mem_req_data, - input wire [NUM_REQS-1:0][TAG_WIDTH-1:0] mem_req_tag, - output wire [NUM_REQS-1:0] mem_req_ready, - - // Core response - output wire [NUM_REQS-1:0] mem_rsp_valid, - output wire [NUM_REQS-1:0][WORD_SIZE*8-1:0] mem_rsp_data, - output wire [NUM_REQS-1:0][TAG_WIDTH-1:0] mem_rsp_tag, - input wire [NUM_REQS-1:0] mem_rsp_ready -); - VX_mem_bus_if #( - .DATA_SIZE (WORD_SIZE), - .TAG_WIDTH (TAG_WIDTH), - .ADDR_WIDTH(ADDR_WIDTH) - ) lsu_bus_if[NUM_REQS](); - - VX_mem_bus_if #( - .DATA_SIZE (NUM_BANKS * WORD_SIZE), - .TAG_WIDTH (LMEM_DMA_TAG_WIDTH), - .ADDR_WIDTH(BANK_ADDR_WIDTH) - ) dma_bus_if(); - - // LSU request - for (genvar i = 0; i < NUM_REQS; ++i) begin - assign lsu_bus_if[i].req_valid = mem_req_valid[i]; - assign lsu_bus_if[i].req_data.rw = mem_req_rw[i]; - assign lsu_bus_if[i].req_data.byteen = mem_req_byteen[i]; - assign lsu_bus_if[i].req_data.addr = mem_req_addr[i]; - assign lsu_bus_if[i].req_data.attr = mem_req_attr[i]; - assign lsu_bus_if[i].req_data.data = mem_req_data[i]; - assign lsu_bus_if[i].req_data.tag = mem_req_tag[i]; - assign mem_req_ready[i] = lsu_bus_if[i].req_ready; - end - - // LSU response - for (genvar i = 0; i < NUM_REQS; ++i) begin - assign mem_rsp_valid[i] = lsu_bus_if[i].rsp_valid; - assign mem_rsp_data[i] = lsu_bus_if[i].rsp_data.data; - assign mem_rsp_tag[i] = lsu_bus_if[i].rsp_data.tag; - assign lsu_bus_if[i].rsp_ready = mem_rsp_ready[i]; - end - - // DMA port not exercised by this testbench - assign dma_bus_if.req_valid = '0; - assign dma_bus_if.req_data = '0; - assign dma_bus_if.rsp_ready = '1; - `UNUSED_VAR (dma_bus_if.req_ready) - `UNUSED_VAR (dma_bus_if.rsp_valid) - `UNUSED_VAR (dma_bus_if.rsp_data) - - VX_local_mem #( - .INSTANCE_ID(INSTANCE_ID), - .SIZE (SIZE), - .NUM_REQS (NUM_REQS), - .NUM_BANKS (NUM_BANKS), - .WORD_SIZE (WORD_SIZE), - .ADDR_WIDTH (ADDR_WIDTH), - .TAG_WIDTH (TAG_WIDTH), - .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), - .READ_MERGE (`VX_CFG_LMEM_READ_MERGE ? NUM_REQS : 0), - .OUT_BUF (3) - ) local_mem ( - .clk (clk), - .reset (reset), - .lsu_bus_if (lsu_bus_if), - .dma_bus_if (dma_bus_if) - ); - -endmodule diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index 1f39567731..12d186a47d 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -46,10 +46,11 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( output wire [DCACHE_NUM_REQS-1:0] mem_rsp_ready ); `ifdef VX_CFG_LMEM_ENABLE - VX_mem_bus_if #( + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_lmem_bus_if[LSU_NUM_REQS](); + ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); `endif VX_mem_bus_if #( @@ -96,7 +97,7 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( `UNUSED_PIN (store_empty), `UNUSED_PIN (busy), `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + .lmem_if (lsu_lmem_if[i]), `endif .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]) ); @@ -144,7 +145,7 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( .lmem_perf (lmem_perf), `endif `ifdef VX_CFG_LMEM_ENABLE - .lmem_bus_if (lsu_lmem_bus_if), + .lmem_if (lsu_lmem_if), `endif .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_if), diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index b3059bcdec..741521a4dc 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -222,10 +222,11 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( wire [NUM_LSU_BLKS-1:0] block_sched_empty; wire [NUM_LSU_BLKS-1:0] block_store_empty; - VX_mem_bus_if #( + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_lmem_bus_if[LSU_NUM_REQS](); + ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); VX_mem_bus_if #( .DATA_SIZE (DCACHE_WORD_SIZE), @@ -263,7 +264,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .sched_empty (block_sched_empty[block_idx]), .store_empty (block_store_empty[block_idx]), `UNUSED_PIN (busy), - .lmem_bus_if (lsu_lmem_bus_if[block_idx * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + .lmem_if (lsu_lmem_if[block_idx]), .dcache_bus_if (lsu_dcache_bus_if[block_idx * DCACHE_CHANNELS +: DCACHE_CHANNELS]) ); end @@ -298,7 +299,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .dxa_lmem_bus_if (per_core_dxa_lmem_bus_if[c]), .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif - .lmem_bus_if (lsu_lmem_bus_if), + .lmem_if (lsu_lmem_if), .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), .flush_l1_only (1'b0), diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index ec8af29b71..7479a64a02 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -154,18 +154,17 @@ class Core::Impl { } } - // create local memory. + // create local memory: it takes each LSU block's warp requests whole. // The DXA writes and the TCU tile-buffer reads share the row-wide DMA // port, round-robin. snprintf(sname, 100, "%s-lmem", name.c_str()); local_mem_ = LocalMem::Create(sname, LocalMem::Config{ (1 << VX_CFG_LMEM_LOG_SIZE), LSU_WORD_SIZE, - LSU_NUM_REQS, + VX_CFG_NUM_LSU_BLOCKS, + LSU_CHANNELS, log2ceil(VX_CFG_LMEM_NUM_BANKS), - false, - VX_CFG_EXT_DXA_ENABLED + VX_CFG_TCU_WGMMA_ENABLED, - (VX_CFG_LMEM_READ_MERGE != 0 && LSU_CHANNELS > 1) ? LSU_CHANNELS : 0 // read_merge: one LSU block's lanes + VX_CFG_EXT_DXA_ENABLED + VX_CFG_TCU_WGMMA_ENABLED }); // create lmem switch @@ -174,25 +173,10 @@ class Core::Impl { lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1); } - // create per-block lmem adapters - std::vector lsu_lmem_adapter(VX_CFG_NUM_LSU_BLOCKS); + // connect lmem switch to local memory for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - snprintf(sname, 100, "%s-lsu_lmem_adapter%d", name.c_str(), b); - lsu_lmem_adapter.at(b) = LsuMemAdapter::Create(sname, LSU_CHANNELS, 1); - } - - // connect lmem switch to per-block adapters - for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - lmem_switch_.at(b)->ReqOutLmem.bind(&lsu_lmem_adapter.at(b)->ReqIn); - lsu_lmem_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInLmem); - } - - // connect per-block lmem adapters to local memory ports - for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - for (uint32_t c = 0; c < LSU_CHANNELS; ++c) { - lsu_lmem_adapter.at(b)->ReqOut.at(c).bind(&local_mem_->Inputs.at(b * LSU_CHANNELS + c)); - local_mem_->Outputs.at(b * LSU_CHANNELS + c).bind(&lsu_lmem_adapter.at(b)->RspIn.at(c)); - } + lmem_switch_.at(b)->ReqOutLmem.bind(&local_mem_->ReqIn.at(b)); + local_mem_->RspOut.at(b).bind(&lmem_switch_.at(b)->RspInLmem); } // Per-channel request/response endpoints facing the dcache ports. diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 64434e76bd..337b574605 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -16,10 +16,8 @@ #include #include #include -#include +#include #include -#include -#include "types.h" #if VX_CFG_EXT_A_ENABLED #include "amo_ops.h" #include "amo_unit.h" @@ -27,548 +25,577 @@ using namespace vortex; -// The lane-to-bank crossbar under read merging. Each bank grants its lowest -// requesting lane per cycle, as the plain crossbar does. A plain read of the -// same word with the same tag as a lower lane of its LSU block follows the -// lowest such lane: it stays out of arbitration, is accepted when that leader -// is, and the bank's response returns to every one of them. -class LmemMergeXbar : public SimObject { -public: - using Ptr = std::shared_ptr; - - std::vector> ReqIn; // lanes - std::vector> RspOut; // lanes - std::vector> ReqOut; // banks - std::vector> RspIn; // banks - - LmemMergeXbar(const SimContext& ctx, const char* name, uint32_t num_inputs, uint32_t num_banks, - uint32_t group, uint32_t lg2_word, uint32_t req_delay, uint32_t rsp_delay) - : SimObject(ctx, name) - , ReqIn(num_inputs, this) - , RspOut(num_inputs, this) - , ReqOut(num_banks, this) - , RspIn(num_banks, this) - , group_(group) - , lg2_word_(lg2_word) - , lg2_inputs_(log2ceil(num_inputs)) - , req_delay_(req_delay) - , rsp_delay_(rsp_delay) - , followers_(num_banks) - , sent_(num_banks, 0) - { - __assert(group_ >= 1 && group_ <= 64 && (num_inputs % group_) == 0, - "a merge group must be at most 64 lanes dividing the ports"); - } - - uint64_t collisions() const { return collisions_; } - -protected: - void on_reset() { - for (auto& f : followers_) { - f.clear(); - } - std::fill(sent_.begin(), sent_.end(), 0); - } - - void on_tick() { - uint32_t I = ReqIn.size(); - uint32_t B = ReqOut.size(); - uint32_t R = 1 << lg2_inputs_; - - // responses: each lane takes one bank per cycle, lowest first, and a - // bank's response leaves once every lane it serves has taken it - std::vector targets(B, 0); - std::vector bases(B, 0); - bool idle = true; - for (uint32_t b = 0; b < B; ++b) { - if (RspIn.at(b).empty()) { - continue; - } - idle = false; - auto& rsp = RspIn.at(b).peek(); - uint32_t lane = rsp.tag & (R - 1); - bases.at(b) = (lane / group_) * group_; - uint64_t mask = 0; - auto it = followers_.at(b).find(rsp.tag); - if (it != followers_.at(b).end()) { - mask = it->second; - } - targets.at(b) = (mask | (1ull << (lane - bases.at(b)))) & ~sent_.at(b); - } - for (uint32_t o = 0; o < I; ++o) { - for (uint32_t b = 0; b < B; ++b) { - if (o < bases.at(b) || o >= bases.at(b) + group_ - || !(targets.at(b) & (1ull << (o - bases.at(b))))) { - continue; - } - MemRsp out(RspIn.at(b).peek()); - out.tag >>= lg2_inputs_; - if (RspOut.at(o).try_send(out, rsp_delay_)) { - DT(4, this->name() << " rsp" << b << "_" << o << ": " << out); - sent_.at(b) |= 1ull << (o - bases.at(b)); - } - break; - } - } - for (uint32_t b = 0; b < B; ++b) { - if (targets.at(b) != 0 && (targets.at(b) & ~sent_.at(b)) == 0) { - followers_.at(b).erase(RspIn.at(b).peek().tag); - RspIn.at(b).pop(); - sent_.at(b) = 0; - } - } - - // requests: find each follower's leader, then let each bank grant its - // lowest requesting lane that follows no other. A lane presents one - // request per cycle, so a lane granted or released this cycle sits out - // the remaining banks even when its next request is already queued. - std::vector lead(I, -1); - std::vector taken(I, false); - for (uint32_t base = 0; base < I; base += group_) { - for (uint32_t i = base; i < base + group_; ++i) { - if (!this->is_plain_read(i)) { - continue; - } - auto& req = ReqIn.at(i).peek(); - for (uint32_t j = base; j < i; ++j) { - if (!this->is_plain_read(j)) { - continue; - } - auto& lreq = ReqIn.at(j).peek(); - if ((lreq.addr >> lg2_word_) == (req.addr >> lg2_word_) && lreq.tag == req.tag) { - lead.at(i) = j; - break; - } - } - } - } - for (uint32_t o = 0; o < B; ++o) { - int32_t g = -1; - bool has_collision = false; - for (uint32_t i = 0; i < I; ++i) { - if (taken.at(i) || ReqIn.at(i).empty() || lead.at(i) != -1 || this->bank_of(ReqIn.at(i).peek()) != o) { - continue; - } - if (g != -1) { - has_collision = true; - break; - } - g = i; - } - if (g == -1) { - continue; - } - idle = false; - MemReq out(ReqIn.at(g).peek()); - out.tag = (out.tag << lg2_inputs_) | g; - if (ReqOut.at(o).try_send(out, req_delay_)) { - DT(4, this->name() << " req" << g << "_" << o << ": " << out); - uint32_t base = (g / group_) * group_; - uint64_t mask = 0; - for (uint32_t i = base; i < base + group_; ++i) { - if (lead.at(i) == g && !taken.at(i)) { - DT(3, this->name() << " read-merge: lane=" << i << ", leader=" << g << ", " << ReqIn.at(i).peek()); - mask |= 1ull << (i - base); - ReqIn.at(i).pop(); - taken.at(i) = true; - } - } - if (mask != 0) { - followers_.at(o)[out.tag] = mask; - } - ReqIn.at(g).pop(); - taken.at(g) = true; - } - collisions_ += has_collision; - } - - for (uint32_t i = 0; i < I && idle; ++i) { - idle = ReqIn.at(i).empty(); - } - if (idle) { - this->tick_sleep(); - } - } - - uint32_t bank_of(const MemReq& req) const { - return (uint32_t)((req.addr >> lg2_word_) & (ReqOut.size() - 1)); - } - - bool is_plain_read(uint32_t i) const { - return !ReqIn.at(i).empty() && ReqIn.at(i).peek().op == MemOp::LD; - } - -private: - uint32_t group_; - uint32_t lg2_word_; - uint32_t lg2_inputs_; - uint32_t req_delay_; - uint32_t rsp_delay_; - std::vector> followers_; // [bank] leader's tag -> lanes riding it, from the group base - std::vector sent_; // [bank] lanes that took the response, from the group base - uint64_t collisions_ = 0; - - friend class SimObject; -}; - +// A request's timing, in cycles from the one its engine takes it in: +// 0 the engine takes the request into its input queue of two; it takes one +// only in a cycle that starts with at most one queued +// 1 the lanes' words are compared and the request moves on to be issued +// 2 its first pass is issued to the bank registers +// 3 the banks are accessed +// 4 each served lane takes its bank's word +// 5 the response reaches its block's response register +// 6 the response register goes to the LSU block +// With several LSU blocks, the block an engine takes from is chosen the cycle +// before (see arbitrate()), and a second, read-only engine works the banks' +// second port, so two warps' reads go through each cycle. Stores and atomics +// take the first engine alone: one enters it only once the read engine has +// drained, and the read engine takes no read while a store or atomic sits in +// the first engine, while one waited at an input the cycle before, or in a +// cycle the first engine may be taking one, so every write keeps its order +// against every read. +// The model does a pass's bank accesses when it is issued. Every access keeps +// its order relative to the others, so this is the order the banks see. class LocalMem::Impl { protected: - LocalMem* simobject_; - Config config_; - RAM ram_; - uint32_t addr_bits_; + // a pass issued at cycle t has its response at the response queue's head + // at t + 3, from where the block's response register takes it + static constexpr uint32_t RSP_LATENCY = 3; + // an engine's response slots, held from the pass until the block takes the + // response from its response register + static constexpr uint32_t RSP_QUEUE_SIZE = 8; #if VX_CFG_EXT_A_ENABLED - AmoUnit amo_unit_; - // Per-bank atomic occupancy. A bank has a single port and the value read out - // is registered before the update is computed, so an atomic owns its bank - // for three cycles -- read, capture, write back -- and it accepts nothing - // else meanwhile. - static constexpr uint32_t AMO_BANK_STALL_CYCLES = 2; - std::vector amo_busy_; - // The word the atomic now occupying the bank writes back, and whether it - // writes at all: a load-reserved never writes and a failed - // store-conditional must not, so neither leaves a shadow behind. - std::vector amo_wb_addr_; - std::vector amo_rdw_pending_; + // An atomic reads its word, registers it, then writes the new value back: + // the two cycles after its pass the banks take nothing else. + static constexpr uint32_t AMO_STALL_CYCLES = 2; #endif - // A read of a word written the previous cycle (by DMA, a store or an atomic's - // write-back) would see the old value, so the bank holds it off one cycle. - // rdw_* records this cycle's writes; rdw_cur_* holds last cycle's. - std::vector rdw_addr_; - std::vector rdw_valid_; - std::vector rdw_cur_addr_; - std::vector rdw_cur_valid_; - MemCrossBar::Ptr mem_xbar_; - LmemMergeXbar::Ptr merge_xbar_; - mutable PerfStats perf_stats_; - uint32_t dma_last_grant_ = 0; - - uint64_t to_local_addr(uint64_t addr) { - return bit_getw(addr, 0, addr_bits_-1); - } - - // The word a bank is addressed by. Two byte addresses falling in one word - // are the same access as far as the bank is concerned. - uint64_t bank_word(uint64_t addr) { - return to_local_addr(addr) >> log2ceil(config_.line_size); - } -public: - Impl(LocalMem* simobject, const Config& config) - : simobject_(simobject) - , config_(config) - , ram_(config.capacity) - , addr_bits_(log2ceil(config.capacity)) + struct Request { + LsuReq req; + uint32_t block; + Request() : req(0), block(0) {} + }; + + struct RspEntry { + uint64_t ready; + uint32_t block; + LsuRsp rsp; + }; + + static constexpr uint32_t S0_SIZE = 2; + + struct Engine { + std::deque s0; + bool s1_valid = false; + Request s1; + BitVector<> pending; + // the next pass's bank winners are known: a pass that leaves lanes + // pending is followed by a cycle that finds the next ones + bool ready = false; + std::deque rsp_queue; + }; + + LocalMem* simobject_; + Config config_; + RAM ram_; + uint32_t addr_bits_; + uint32_t lg2_word_; #if VX_CFG_EXT_A_ENABLED - , amo_unit_(VX_CFG_AMO_RS_SIZE < 2 ? 2u : (uint32_t)VX_CFG_AMO_RS_SIZE) - , amo_busy_(1 << config.B, 0) - , amo_wb_addr_(1 << config.B, 0) - , amo_rdw_pending_(1 << config.B, false) + AmoUnit amo_unit_; #endif - , rdw_addr_(1 << config.B, 0) - , rdw_valid_(1 << config.B, false) - , rdw_cur_addr_(1 << config.B, 0) - , rdw_cur_valid_(1 << config.B, false) - { - char sname[100]; - snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); - uint32_t lg2_line_size = log2ceil(config_.line_size); - uint32_t num_banks = 1 << config.B; - // The request crossbar registers its output toward the banks, one cycle - // ahead of the SRAM read. The response crossbar's register is the - // adapter's response stage. An arbiter over more than 1.5x the fan-out - // limit is split into a buffered tree, one cycle deeper. The request - // side arbitrates the lanes at each bank, the response side the banks - // at each lane. - auto tree_delay = [](uint32_t requesters) -> uint32_t { - constexpr uint32_t kMaxFanout = 8; - return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; - }; - if (config.read_merge != 0) { - merge_xbar_ = LmemMergeXbar::Create(sname, config.num_reqs, num_banks, config.read_merge, - lg2_line_size, 1 + tree_delay(config.num_reqs), tree_delay(num_banks)); - for (uint32_t i = 0; i < config.num_reqs; ++i) { - simobject->Inputs.at(i).bind(&merge_xbar_->ReqIn.at(i)); - merge_xbar_->RspOut.at(i).bind(&simobject->Outputs.at(i)); - } - return; - } - mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::Priority, config.num_reqs, num_banks, - [lg2_line_size, num_banks](const MemCrossBar::ReqType& req) { - // Custom logic to calculate the output index using bank interleaving - return (uint32_t)((req.addr >> lg2_line_size) & (num_banks-1)); - }, 1 + tree_delay(config.num_reqs), tree_delay(num_banks)); - for (uint32_t i = 0; i < config.num_reqs; ++i) { - simobject->Inputs.at(i).bind(&mem_xbar_->ReqIn.at(i)); - mem_xbar_->RspOut.at(i).bind(&simobject->Outputs.at(i)); - } - } - - SimChannel& bank_req(uint32_t b) { - return merge_xbar_ ? merge_xbar_->ReqOut.at(b) : mem_xbar_->ReqOut.at(b); - } - - SimChannel& bank_rsp(uint32_t b) { - return merge_xbar_ ? merge_xbar_->RspIn.at(b) : mem_xbar_->RspIn.at(b); - } - - virtual ~Impl() {} - - void reset() { - // perf_stats_ survives: the MPM counters run from device reset. - dma_last_grant_ = config_.dma_clients ? (config_.dma_clients - 1) : 0; + uint32_t stall_cycles_; + std::vector engines_; + struct RspReg { + bool valid = false; + uint32_t engine = 0; + LsuRsp rsp; + RspReg() : rsp(0) {} + }; + std::vector rsp_regs_; // per block + std::vector arb_last_; // per engine, its round-robin arbiter's last choice + struct Selection { + bool valid = false; + uint32_t block = 0; + bool read = false; // the chosen block's request is known to be a plain read + }; + Selection a_sel_; + Selection b_sel_; + bool write_waiting_r_; + bool dma_slot_; + uint32_t dma_last_grant_; + mutable PerfStats perf_stats_; + + uint64_t to_local_addr(uint64_t addr) const { + return bit_getw(addr, 0, addr_bits_ - 1); + } + + // A bank serves one word per cycle; byte addresses within one word are the + // same access as far as the bank is concerned. + uint64_t bank_word(uint64_t addr) const { + return to_local_addr(addr) >> lg2_word_; + } + + uint32_t bank_of(uint64_t addr) const { + return (uint32_t)(bank_word(addr) & ((1u << config_.B) - 1)); + } + + static bool is_plain_read(const LsuReq& req) { + return !req.is_write() && !req.is_amo(); + } + + static bool holds_write(const Engine& e) { + for (auto& r : e.s0) { + if (!is_plain_read(r.req)) { + return true; + } + } + return e.s1_valid && !is_plain_read(e.s1.req); + } + + // The lanes one pass serves: each bank goes to its lowest pending lane, and + // a plain read also serves every pending lane reading the same word. A + // store or an atomic takes its bank alone, so lanes writing one word land + // in lane order and the highest lane's bytes are the ones left. + BitVector<> select_pass(const LsuReq& req, const BitVector<>& pending) const { + uint32_t num_banks = 1u << config_.B; + std::vector winner(num_banks, -1); + BitVector<> served(pending.size()); + bool join = is_plain_read(req); + for (uint32_t i = 0; i < pending.size(); ++i) { + if (!pending.test(i)) { + continue; + } + uint32_t b = this->bank_of(req.addrs.at(i)); + if (winner.at(b) == -1) { + winner.at(b) = i; + served.set(i); + } else if (join && this->bank_word(req.addrs.at(i)) == this->bank_word(req.addrs.at(winner.at(b)))) { + served.set(i); + } + } + return served; + } + + // Do one lane's bank access and return the word the lane gets back, if any. + std::shared_ptr access(const LsuReq& req, uint32_t i) { + uint64_t la = to_local_addr(req.addrs.at(i)); + uint64_t line_addr = la & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); #if VX_CFG_EXT_A_ENABLED - amo_unit_.reset(); - std::fill(amo_busy_.begin(), amo_busy_.end(), 0); - std::fill(amo_rdw_pending_.begin(), amo_rdw_pending_.end(), false); + uint32_t hart_id = make_hart_id(req.cid, req.wid, req.tids.at(i)); + if (req.is_amo()) { + // Shared-memory atomics read-modify-write in place and return the old + // word; LR/SC reservations are kept per hart. + const uint32_t bmask = VX_CFG_MEM_BLOCK_SIZE - 1; + const uint32_t byte_off = (uint32_t)(la & bmask); + const uint64_t byteen = req.byteen.at(i); + const uint32_t wbytes = byteen ? (uint32_t)__builtin_popcountll(byteen) : 4u; + const uint8_t width = (wbytes >= 8) ? 3 : 2; + const bool is_lr = (req.op == MemOp::AMO_LR); + const bool is_sc = (req.op == MemOp::AMO_SC); + const bool sc_fail = is_sc && !amo_unit_.check(hart_id, la); + const bool do_store = !is_lr && !sc_fail; + + uint8_t obuf[8] = {0}; + ram_.read(obuf, la, wbytes); + const uint64_t old_word = amo_load_word(obuf, 0, width); + const uint64_t rhs = req.data.at(i) + ? amo_load_word(req.data.at(i)->data(), byte_off, width) : 0; + auto rmw = amo_compute(req.op, width, old_word, rhs, + req.flags.amo_unsigned, req.amo_cmp.at(i)); + + auto rsp_data = make_mem_block(); + std::memset(rsp_data->data(), 0, rsp_data->size()); + const uint64_t ret_word = is_sc ? (sc_fail ? 1ull : 0ull) : rmw.ret_word; + amo_store_word(rsp_data->data(), byte_off, width, ret_word); + + // An SC gives up its own reservation whether it succeeds or fails; an + // LR claims one. A committed store breaks every other hart's + // reservation on the word, or a second hart's stale reservation would + // survive the first hart's successful SC and one update would be lost. + if (is_lr) { + amo_unit_.reserve(hart_id, la); + } else if (is_sc) { + amo_unit_.clear(hart_id, la); + } + if (do_store) { + uint8_t sbuf[8]; + amo_store_word(sbuf, 0, width, rmw.new_word); + ram_.write(sbuf, la, wbytes); + amo_unit_.invalidate(la, hart_id); + } + // an atomic responds, so it counts as a read + ++perf_stats_.reads; + return rsp_data; + } #endif - std::fill(rdw_valid_.begin(), rdw_valid_.end(), false); - std::fill(rdw_cur_valid_.begin(), rdw_cur_valid_.end(), false); - } - - // Serve one DMA client's row access this cycle and return the banks it - // occupies. Clients are served round-robin, starting after the last one - // served, and DMA wins every bank it touches over the LSU: a read touches - // all banks, a write those its byte enables cover. - uint64_t dma_tick() { - uint32_t num_banks = (1 << config_.B); - uint32_t lg2_line_size = log2ceil(config_.line_size); - for (uint32_t i = 0; i < config_.dma_clients; ++i) { - uint32_t c = (dma_last_grant_ + 1 + i) % config_.dma_clients; - bool pending = false; - bool rsp_ready = true; - for (uint32_t p = 0; p < DMA_PORTS; ++p) { - auto& in = simobject_->DmaInputs.at(c * DMA_PORTS + p); - if (in.empty()) { - continue; - } - pending = true; - if (!in.peek().is_write() && simobject_->DmaOutputs.at(c * DMA_PORTS + p).full()) { - rsp_ready = false; - } - } - if (!pending) { - continue; - } - if (!rsp_ready) { - return 0; - } - uint64_t banks = 0; - for (uint32_t p = 0; p < DMA_PORTS; ++p) { - auto& in = simobject_->DmaInputs.at(c * DMA_PORTS + p); - if (in.empty()) { - continue; - } - auto& req = in.peek(); - uint64_t line_addr = to_local_addr(req.addr) & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); - if (req.is_write()) { - for (uint32_t b = 0; req.data && b < VX_CFG_MEM_BLOCK_SIZE; ++b) { - if (req.byteen & (1ull << b)) { - uint8_t value = (*req.data)[b]; - ram_.write(&value, line_addr + b, 1); - uint32_t bank = ((line_addr + b) >> lg2_line_size) & (num_banks - 1); - banks |= 1ull << bank; - rdw_addr_[bank] = this->bank_word(line_addr + b); - rdw_valid_[bank] = true; - } - } + if (req.is_write()) { + for (uint32_t b = 0; req.data.at(i) && b < VX_CFG_MEM_BLOCK_SIZE; ++b) { + if (req.byteen.at(i) & (1ull << b)) { + uint8_t value = (*req.data.at(i))[b]; + ram_.write(&value, line_addr + b, 1); + } + } #if VX_CFG_EXT_A_ENABLED - amo_unit_.invalidate(to_local_addr(req.addr), req.hart_id); + // A plain store breaks a reservation on the word it writes, as an + // atomic's commit does, or a store between a hart's load-reserved and + // its store-conditional would go unnoticed. + amo_unit_.invalidate(la, hart_id); #endif - ++perf_stats_.writes; - } else { - MemRsp rsp{req.tag, req.hart_id, req.uuid}; - auto rsp_data = make_mem_block(); - ram_.read(rsp_data->data(), line_addr, VX_CFG_MEM_BLOCK_SIZE); - rsp.data = rsp_data; - // The request arrived over a registered channel; the read - // data returns the cycle after it was issued. - simobject_->DmaOutputs.at(c * DMA_PORTS + p).send(rsp, 0); - banks = (num_banks >= 64) ? ~uint64_t(0) : ((uint64_t(1) << num_banks) - 1); - ++perf_stats_.reads; - } - DT(4, simobject_->name() << "-dma" << c << " req : " << req); - in.pop(); - } - dma_last_grant_ = c; - return banks; - } - return 0; - } - - void tick() { - rdw_cur_addr_.swap(rdw_addr_); - rdw_cur_valid_.swap(rdw_valid_); - std::fill(rdw_valid_.begin(), rdw_valid_.end(), false); - - uint64_t dma_banks = this->dma_tick(); - - // process bank requets from xbar - uint32_t num_banks = (1 << config_.B); - for (uint32_t i = 0; i < num_banks; ++i) { + ++perf_stats_.writes; + return nullptr; + } + auto rsp_data = make_mem_block(); + ram_.read(rsp_data->data(), line_addr, VX_CFG_MEM_BLOCK_SIZE); + ++perf_stats_.reads; + return rsp_data; + } + + bool dma_pending() const { + for (auto& in : simobject_->DmaInputs) { + if (!in.empty()) { + return true; + } + } + return false; + } + + // Serve one DMA client's row access. Clients are served round-robin, + // starting after the last one served. + void dma_fire() { + for (uint32_t i = 0; i < config_.dma_clients; ++i) { + uint32_t c = (dma_last_grant_ + 1 + i) % config_.dma_clients; + bool pending = false; + bool rsp_ready = true; + for (uint32_t p = 0; p < DMA_PORTS; ++p) { + auto& in = simobject_->DmaInputs.at(c * DMA_PORTS + p); + if (in.empty()) { + continue; + } + pending = true; + if (!in.peek().is_write() && simobject_->DmaOutputs.at(c * DMA_PORTS + p).full()) { + rsp_ready = false; + } + } + if (!pending) { + continue; + } + if (!rsp_ready) { + return; + } + for (uint32_t p = 0; p < DMA_PORTS; ++p) { + auto& in = simobject_->DmaInputs.at(c * DMA_PORTS + p); + if (in.empty()) { + continue; + } + auto& req = in.peek(); + uint64_t line_addr = to_local_addr(req.addr) & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); + if (req.is_write()) { + for (uint32_t b = 0; req.data && b < VX_CFG_MEM_BLOCK_SIZE; ++b) { + if (req.byteen & (1ull << b)) { + uint8_t value = (*req.data)[b]; + ram_.write(&value, line_addr + b, 1); + } + } #if VX_CFG_EXT_A_ENABLED - // Cycles an in-flight atomic owns; the bank serves nothing else. - if (amo_busy_[i] != 0) { - if (0 == --amo_busy_[i] && amo_rdw_pending_[i]) { - // write-back just issued - rdw_addr_[i] = amo_wb_addr_[i]; - rdw_valid_[i] = true; - } - continue; - } + amo_unit_.invalidate(to_local_addr(req.addr), req.hart_id); #endif - if ((dma_banks >> i) & 1) { - continue; - } - auto& xbar_req_out = this->bank_req(i); - if (xbar_req_out.empty()) - continue; - - auto& bank_req = xbar_req_out.peek(); - - // A read of the word written last cycle waits a cycle. An atomic - // reads the bank for its old value, so it counts as a reader here - // even though the request carries a write flag; that is what makes - // contending atomics on one word cost four cycles. + } else { + MemRsp rsp{req.tag, req.hart_id, req.uuid}; + auto rsp_data = make_mem_block(); + ram_.read(rsp_data->data(), line_addr, VX_CFG_MEM_BLOCK_SIZE); + rsp.data = rsp_data; + // The request arrived over a registered channel; the read data + // returns the cycle after it was issued. + simobject_->DmaOutputs.at(c * DMA_PORTS + p).send(rsp, 0); + } + DT(4, simobject_->name() << "-dma" << c << " req : " << req); + in.pop(); + } + dma_last_grant_ = c; + return; + } + } + + // Issue an engine's next pass. Returns true when it was the request's last, + // which frees S1 for a new request in the same cycle. The response queue + // occupancy is the one from the start of the cycle. + bool issue_pass(Engine& e, uint32_t rsp_count) { + auto& req = e.s1.req; + bool has_rsp = !req.is_write() || req.is_amo(); + if (has_rsp && rsp_count >= RSP_QUEUE_SIZE) { + return false; + } + auto served = this->select_pass(req, e.pending); + LsuRsp rsp(config_.num_lanes); + rsp.tag = req.tag; + rsp.cid = req.cid; + rsp.uuid = req.uuid; + for (uint32_t i = 0; i < config_.num_lanes; ++i) { + if (!served.test(i)) { + continue; + } + auto data = this->access(req, i); + if (has_rsp) { + rsp.mask.set(i); + rsp.data.at(i) = data; + } + e.pending.reset(i); + } + DT(4, simobject_->name() << " pass: served=" << served << ", " << req); + if (has_rsp) { + e.rsp_queue.push_back({SimPlatform::instance().cycles() + RSP_LATENCY, e.s1.block, std::move(rsp)}); + } #if VX_CFG_EXT_A_ENABLED - const bool reads_bank = !bank_req.is_write() || memop_is_atomic(bank_req.op); -#else - const bool reads_bank = !bank_req.is_write(); + if (req.is_amo()) { + stall_cycles_ = AMO_STALL_CYCLES; + } #endif - if (rdw_cur_valid_[i] - && reads_bank - && this->bank_word(bank_req.addr) == rdw_cur_addr_[i]) { - continue; - } + if (e.pending.none()) { + e.s1_valid = false; + return true; + } + ++perf_stats_.bank_stalls; + e.ready = false; + return false; + } + + void take(Engine& e, uint32_t b) { + auto& in = simobject_->ReqIn.at(b); + Request r; + r.req = in.peek(); + r.block = b; + DT(4, simobject_->name() << "-req" << b << ": " << r.req); + e.s0.push_back(std::move(r)); + in.pop(); + } + + // The lowest eligible block after the last one chosen, round-robin. + int32_t pick(const std::vector& eligible, uint32_t& last) { + for (uint32_t i = 0; i < config_.num_blocks; ++i) { + uint32_t b = (last + 1 + i) % config_.num_blocks; + if (eligible.at(b)) { + last = b; + return b; + } + } + return -1; + } + + // Each engine takes the block chosen for it the cycle before, if its + // request is still there, then the choices for the next cycle are made from + // the requests waiting at the start of this one. A block served this cycle + // may present a new request of either kind next cycle, so a choice of it + // counts as a store: the first engine takes a store only while the read + // engine is idle, and the read engine takes nothing in that cycle. The read + // engine takes a read only while no store or atomic waited at an input the + // cycle before or sits in the first engine. + void arbitrate(const std::vector& s0_ready, bool a_write, bool b_busy) { + uint32_t n = config_.num_blocks; + std::vector in_valid(n); + std::vector in_read(n); + std::vector fired(n, false); + for (uint32_t b = 0; b < n; ++b) { + auto& in = simobject_->ReqIn.at(b); + in_valid.at(b) = !in.empty(); + in_read.at(b) = in_valid.at(b) && is_plain_read(in.peek()); + } + bool a_take = a_sel_.valid && s0_ready.at(0) && (a_sel_.read || !b_busy); + bool b_take = b_sel_.valid && s0_ready.at(1) && !a_write && !write_waiting_r_ + && !(a_take && !a_sel_.read); + bool b_fired = false; + if (a_take && in_valid.at(a_sel_.block)) { + this->take(engines_.at(0), a_sel_.block); + fired.at(a_sel_.block) = true; + } + if (b_take && in_valid.at(b_sel_.block)) { + this->take(engines_.at(1), b_sel_.block); + fired.at(b_sel_.block) = true; + b_fired = true; + } + + std::vector known_read(n); + std::vector a_elig(n); + std::vector b_elig(n); + bool b_busy_n = b_busy || b_fired; + bool write_waiting = false; + for (uint32_t b = 0; b < n; ++b) { + known_read.at(b) = in_read.at(b) && !fired.at(b); + a_elig.at(b) = known_read.at(b) || (in_valid.at(b) && !b_busy_n); + write_waiting = write_waiting || (in_valid.at(b) && !in_read.at(b) && !fired.at(b)); + } + int32_t a_next = this->pick(a_elig, arb_last_.at(0)); + for (uint32_t b = 0; b < n; ++b) { + b_elig.at(b) = known_read.at(b) && ((int32_t)b != a_next); + } + int32_t b_next = this->pick(b_elig, arb_last_.at(1)); + a_sel_.valid = (a_next != -1); + a_sel_.block = (a_next != -1) ? a_next : 0; + a_sel_.read = (a_next != -1) && known_read.at(a_next); + b_sel_.valid = (b_next != -1); + b_sel_.block = (b_next != -1) ? b_next : 0; + write_waiting_r_ = write_waiting; + } +public: + Impl(LocalMem* simobject, const Config& config) + : simobject_(simobject) + , config_(config) + , ram_(config.capacity) + , addr_bits_(log2ceil(config.capacity)) + , lg2_word_(log2ceil(config.word_size)) #if VX_CFG_EXT_A_ENABLED - // Shared-memory atomics: read-modify-write in place and ALWAYS - // return the old word. An AMO has a destination register, so the - // LSU reserved a response slot; a missing response hangs the warp - // (this was the prior behaviour — LMEM had no AMO path). LR/SC - // reservations are tracked per hart for correct contended - // compare-swap, mirroring the dcache AMO path used for global/SSBO. - if (memop_is_atomic(bank_req.op)) { - const uint32_t bmask = VX_CFG_MEM_BLOCK_SIZE - 1; - const uint64_t la = to_local_addr(bank_req.addr); - const uint32_t byte_off = (uint32_t)(la & bmask); - const uint32_t wbytes = bank_req.byteen - ? (uint32_t)__builtin_popcountll(bank_req.byteen) : 4u; - const uint8_t width = (wbytes >= 8) ? 3 : 2; - const MemOp op = bank_req.op; - const bool is_lr = (op == MemOp::AMO_LR); - const bool is_sc = (op == MemOp::AMO_SC); - const bool sc_fail = is_sc && !amo_unit_.check(bank_req.hart_id, la); - const bool do_store = !is_lr && !sc_fail; - - uint8_t obuf[8] = {0}; - ram_.read(obuf, la, wbytes); - const uint64_t old_word = amo_load_word(obuf, 0, width); - const uint64_t rhs = bank_req.data - ? amo_load_word(bank_req.data->data(), byte_off, width) : 0; - auto rmw = amo_compute(op, width, old_word, rhs, - bank_req.flags.amo_unsigned, - bank_req.amo_cmp); - - // Build the response before mutating any state so a full RspIn - // can stall-retry cleanly. - MemRsp bank_rsp{bank_req.tag, bank_req.hart_id, bank_req.uuid}; - auto rsp_data = make_mem_block(); - std::memset(rsp_data->data(), 0, rsp_data->size()); - const uint64_t ret_word = is_sc ? (sc_fail ? 1ull : 0ull) : rmw.ret_word; - amo_store_word(rsp_data->data(), byte_off, width, ret_word); - bank_rsp.data = rsp_data; - if (!this->bank_rsp(i).try_send(bank_rsp)) - continue; // stall; no state mutated yet - - // An SC gives up its own reservation whether it succeeds or - // fails; an LR claims one. Everything else is a plain RMW, - // whose commit breaks the other harts' reservations below. - if (is_lr) amo_unit_.reserve(bank_req.hart_id, la); - else if (is_sc) amo_unit_.clear(bank_req.hart_id, la); - if (do_store) { - uint8_t sbuf[8]; - amo_store_word(sbuf, 0, width, rmw.new_word); - ram_.write(sbuf, la, wbytes); - // Any committed store breaks every other hart's reservation - // on the word. Without this a second hart's stale - // reservation survives the first hart's successful SC, so - // both report success and one update is lost. - amo_unit_.invalidate(la, bank_req.hart_id); - } - perf_stats_.reads += !do_store; - perf_stats_.writes += do_store; - DT(4, simobject_->name() << "-bank" << i << " amo : " << bank_req); - amo_wb_addr_[i] = this->bank_word(bank_req.addr); - amo_rdw_pending_[i] = do_store; - xbar_req_out.pop(); - amo_busy_[i] = AMO_BANK_STALL_CYCLES; // capture, then write back - continue; - } + , amo_unit_(VX_CFG_AMO_RS_SIZE < 2 ? 2u : (uint32_t)VX_CFG_AMO_RS_SIZE) #endif - - // Apply byte-enabled writes from TLM payload to local RAM. - if (bank_req.is_write() && bank_req.data) { - uint64_t line_addr = to_local_addr(bank_req.addr) & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); - for (uint32_t b = 0; b < VX_CFG_MEM_BLOCK_SIZE; ++b) { - if (bank_req.byteen & (1ull << b)) { - uint8_t value = (*bank_req.data)[b]; - ram_.write(&value, line_addr + b, 1); - } - } - rdw_addr_[i] = this->bank_word(bank_req.addr); - rdw_valid_[i] = true; + , stall_cycles_(0) + , engines_((config.num_blocks > 1) ? 2 : 1) + , rsp_regs_(config.num_blocks) + , arb_last_(engines_.size(), 0) + , write_waiting_r_(false) + , dma_slot_(false) + , dma_last_grant_(0) + { + for (auto& e : engines_) { + e.pending = BitVector<>(config.num_lanes); + } + } + + virtual ~Impl() {} + + void reset() { + // perf_stats_ survives: the MPM counters run from device reset. #if VX_CFG_EXT_A_ENABLED - // A plain store breaks a reservation on the word it writes, the - // same as an atomic's commit does; otherwise a store between a - // hart's load-reserved and its store-conditional would go - // unnoticed and the conditional would wrongly succeed. - amo_unit_.invalidate(to_local_addr(bank_req.addr), bank_req.hart_id); + amo_unit_.reset(); #endif - } - - // Loads always respond. Stores respond when configured globally OR - // the request opts in via MEM_FLAG_STRSP. - if (!bank_req.is_write() || config_.write_reponse || bank_req.flags.strsp) { - // send xbar response — for reads, capture the line payload. - MemRsp bank_rsp{bank_req.tag, bank_req.hart_id, bank_req.uuid}; - if (!bank_req.is_write()) { - auto rsp_data = make_mem_block(); - uint64_t line_addr = to_local_addr(bank_req.addr) & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); - ram_.read(rsp_data->data(), line_addr, VX_CFG_MEM_BLOCK_SIZE); - bank_rsp.data = rsp_data; - } - if (!this->bank_rsp(i).try_send(bank_rsp)) - continue; // stall - } - - DT(4, simobject_->name() << "-bank" << i << " req : " << bank_req); - - // update perf counters - perf_stats_.reads += !bank_req.is_write(); - perf_stats_.writes += bank_req.is_write(); - - // remove input - xbar_req_out.pop(); - } - } - - const PerfStats& perf_stats() const { - perf_stats_.bank_stalls = merge_xbar_ ? merge_xbar_->collisions() : mem_xbar_->collisions(); - return perf_stats_; - } - + stall_cycles_ = 0; + for (auto& e : engines_) { + e.s0.clear(); + e.s1_valid = false; + e.pending.reset(); + e.rsp_queue.clear(); + } + for (auto& r : rsp_regs_) { + r.valid = false; + } + std::fill(arb_last_.begin(), arb_last_.end(), config_.num_blocks - 1); + a_sel_ = Selection(); + b_sel_ = Selection(); + write_waiting_r_ = false; + dma_slot_ = false; + dma_last_grant_ = config_.dma_clients ? (config_.dma_clients - 1) : 0; + } + + void tick() { + uint32_t num_engines = engines_.size(); + uint64_t now = SimPlatform::instance().cycles(); + + // Responses. Each block's response register goes to the block; a free + // one then takes the first engine's queue head ahead of the read engine's. + // An engine's slots count the responses in its queue and in the registers. + std::vector rsp_count(num_engines); + for (uint32_t k = 0; k < num_engines; ++k) { + rsp_count.at(k) = engines_.at(k).rsp_queue.size(); + } + for (auto& r : rsp_regs_) { + rsp_count.at(r.engine) += r.valid; + } + for (uint32_t b = 0; b < config_.num_blocks; ++b) { + auto& r = rsp_regs_.at(b); + if (r.valid && simobject_->RspOut.at(b).try_send(r.rsp, 0)) { + DT(4, simobject_->name() << "-rsp" << b << ": " << r.rsp); + r.valid = false; + } + } + for (uint32_t k = 0; k < num_engines; ++k) { + auto& e = engines_.at(k); + if (e.rsp_queue.empty() || e.rsp_queue.front().ready > now) { + continue; + } + auto& r = rsp_regs_.at(e.rsp_queue.front().block); + if (r.valid) { + continue; + } + r.valid = true; + r.engine = k; + r.rsp = std::move(e.rsp_queue.front().rsp); + e.rsp_queue.pop_front(); + } + + // the arbiter's view of the engines, as registered at the cycle start + bool a_write = holds_write(engines_.at(0)); + bool b_busy = (num_engines > 1) && (!engines_.at(1).s0.empty() || engines_.at(1).s1_valid); + std::vector s0_ready(num_engines); + for (uint32_t k = 0; k < num_engines; ++k) { + s0_ready.at(k) = (engines_.at(k).s0.size() < S0_SIZE); + } + + // S1: a DMA access, or the engines' passes. A DMA request seen in one + // cycle takes the banks the next; no pass is issued while one is seen. + // An engine whose last pass left lanes pending spends this cycle finding + // the next winners. + std::vector s1_ready(num_engines); + std::vector can_pass(num_engines); + for (uint32_t k = 0; k < num_engines; ++k) { + auto& e = engines_.at(k); + s1_ready.at(k) = !e.s1_valid; + can_pass.at(k) = e.s1_valid && e.ready; + if (e.s1_valid && !e.ready) { + e.ready = true; + } + } + if (stall_cycles_ != 0) { + --stall_cycles_; + dma_slot_ = false; + } else { + bool dma_valid = this->dma_pending(); + if (dma_slot_) { + this->dma_fire(); + } + dma_slot_ = dma_valid; + if (!dma_valid) { + for (uint32_t k = 0; k < num_engines; ++k) { + if (can_pass.at(k)) { + s1_ready.at(k) = this->issue_pass(engines_.at(k), rsp_count.at(k)); + } + } + } + } + + // S0 -> S1, with the single-pass check + for (uint32_t k = 0; k < num_engines; ++k) { + auto& e = engines_.at(k); + if (s1_ready.at(k) && !e.s0.empty()) { + e.s1 = std::move(e.s0.front()); + e.s0.pop_front(); + e.s1_valid = true; + e.pending = e.s1.req.mask; + e.ready = true; + } + } + + if (num_engines == 1) { + // one block: its request goes straight to the engine + auto& in = simobject_->ReqIn.at(0); + if (s0_ready.at(0) && !in.empty()) { + this->take(engines_.at(0), 0); + } + } else { + this->arbitrate(s0_ready, a_write, b_busy); + } + + bool idle = (stall_cycles_ == 0) && !dma_slot_ && !a_sel_.valid && !b_sel_.valid; + for (auto& e : engines_) { + idle = idle && e.s0.empty() && !e.s1_valid && e.rsp_queue.empty(); + } + for (auto& r : rsp_regs_) { + idle = idle && !r.valid; + } + if (idle && !this->dma_pending()) { + simobject_->tick_sleep(); + } + } + + const PerfStats& perf_stats() const { + return perf_stats_; + } }; /////////////////////////////////////////////////////////////////////////////// LocalMem::LocalMem(const SimContext& ctx, const char* name, const Config& config) - : SimObject(ctx, name) - , Inputs(config.num_reqs, this) - , Outputs(config.num_reqs, this) - , DmaInputs(config.dma_clients * DMA_PORTS, this) - , DmaOutputs(config.dma_clients * DMA_PORTS, this) - , impl_(new Impl(this, config)) + : SimObject(ctx, name) + , ReqIn(config.num_blocks, this) + , RspOut(config.num_blocks, this) + , DmaInputs(config.dma_clients * DMA_PORTS, this) + , DmaOutputs(config.dma_clients * DMA_PORTS, this) + , impl_(new Impl(this, config)) {} LocalMem::~LocalMem() { diff --git a/sim/simx/mem/local_mem.h b/sim/simx/mem/local_mem.h index 7e725d30f9..b887b95798 100644 --- a/sim/simx/mem/local_mem.h +++ b/sim/simx/mem/local_mem.h @@ -17,6 +17,13 @@ namespace vortex { +// Warp-wide local memory. It takes whole warp requests from the LSU blocks, +// one per cycle per engine (two engines with several LSU blocks), and serves +// each in passes: each pass gives every bank to the lowest pending lane that +// maps to it, and every pending read lane of the same word rides along. Lanes +// left over replay in later passes, one every other cycle, and a request +// retires before the next one starts, so the banks see every lane in program +// order. class LocalMem : public SimObject { public: // A DMA access covers one full bank row. A row wider than a mem_block @@ -27,12 +34,11 @@ class LocalMem : public SimObject { struct Config { uint32_t capacity; - uint32_t line_size; - uint32_t num_reqs; - uint32_t B; // log2 number of banks - bool write_reponse; + uint32_t word_size; + uint32_t num_blocks; // LSU blocks, one warp request port each + uint32_t num_lanes; // lanes of a warp request + uint32_t B; // log2 number of banks uint32_t dma_clients; // row-wide DMA masters, served round-robin - uint32_t read_merge; // same-word reads merge within groups of this many adjacent ports (0: off) }; struct PerfStats { @@ -48,8 +54,8 @@ class LocalMem : public SimObject { } }; - std::vector> Inputs; - std::vector> Outputs; + std::vector> ReqIn; + std::vector> RspOut; // DMA port: client c uses channels [c * DMA_PORTS, (c + 1) * DMA_PORTS). std::vector> DmaInputs; diff --git a/tests/regression/Makefile b/tests/regression/Makefile index eb32835a84..025260ccdf 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax uniform_ld lmem_merge mufu zfhmin \ + softmax uniform_ld lmem_access mufu zfhmin \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/lmem_merge/Makefile b/tests/regression/lmem_access/Makefile similarity index 91% rename from tests/regression/lmem_merge/Makefile rename to tests/regression/lmem_access/Makefile index 5177c093ca..c62a891e30 100644 --- a/tests/regression/lmem_merge/Makefile +++ b/tests/regression/lmem_access/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk -PROJECT := lmem_merge +PROJECT := lmem_access SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) diff --git a/tests/regression/lmem_access/common.h b/tests/regression/lmem_access/common.h new file mode 100644 index 0000000000..02db3dd69b --- /dev/null +++ b/tests/regression/lmem_access/common.h @@ -0,0 +1,60 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define LMEM_MARK 0x80000000u + +// Local-memory layout per block, in words: read-only words first, then one +// RAW, one WAR and one AMO word per warp and iteration, then a read-only +// bank-conflict region of warp_size^2 words, then per warp and iteration +// SUB_WORDS(warp_size) words of sub-word stores and one same-word store word, +// then one cross-warp word per iteration, then the atomic counter and the +// LR/SC counter that every thread of the block updates. +#define RO_WORDS 128 + +// retry bound of the LR/SC increment +#define LRSC_MAX_RETRIES 4096 + +// words holding one byte store of each lane, four lanes to a word +#define SUB_WORDS(w) (((w) + 3) / 4) + +// The byte a lane stores into its sub-word store word, and the word a lane +// stores into the warp's same-word store word. +static inline uint8_t sub_byte(uint32_t slot, uint32_t lane) { + return (uint8_t)(0x40 + slot * 7 + lane); +} +static inline uint32_t same_word(uint32_t slot, uint32_t lane) { + return 0x80000000u | (slot << 8) | lane; +} + +// The value one warp stores into iteration k's cross-warp word. +static inline uint32_t xwarp_word(uint32_t k) { + return 0xC0000000u | (k << 12) | 0x5A5u; +} + +// Initial value of word i: every byte differs and the high bytes are set, so +// sub-word loads see distinct, sign-extended values. +static inline uint32_t ro_word(uint32_t i) { + return (i * 0x9E3779B1u) ^ 0xA5C30000u; +} + +typedef struct { + uint32_t num_iters; // iterations per warp + uint32_t warp_size; // threads per warp + uint32_t lmem_words; // local-memory words per block + uint32_t raw_base; // first RAW word + uint32_t war_base; // first WAR word + uint32_t amo_base; // first AMO word + uint32_t cf_base; // first bank-conflict word + uint32_t sub_base; // first sub-word store word + uint32_t same_base; // first same-word store word + uint32_t xwarp_base; // first cross-warp word + uint32_t count_word; // atomic counter + uint32_t lrsc_word; // LR/SC counter + uint32_t lrsc; // run the LR/SC phase + uint64_t err_addr; // out: per-thread error count + uint64_t dump_addr; // out: each block's final local memory +} kernel_arg_t; + +#endif diff --git a/tests/regression/lmem_access/kernel.cpp b/tests/regression/lmem_access/kernel.cpp new file mode 100644 index 0000000000..fa0d26f26c --- /dev/null +++ b/tests/regression/lmem_access/kernel.cpp @@ -0,0 +1,220 @@ +#include +#include "common.h" + +// Local-memory reads that share words, around single-lane stores and +// atomics. Per iteration k, lane s = k % W takes a turn: +// +// shared words - lanes in groups of 1, 2, 4, ... W read one word; then 2 +// and 4 distinct words per warp, in one bank (32 words +// apart) and in different banks: every lane must get its +// own word; a third of the lanes read one word (partial +// thread mask); every warp reads the same word (lanes of +// different LSU blocks) +// conflicts - every lane reads its own word of one bank, so the request +// replays over W passes +// sub-words - every lane loads a byte (signed and unsigned) and a +// half-word of one shared word, at its own offset +// RAW - lane s stores its word, then the whole warp reads it: +// lane s must see its own store +// WAR - the whole warp reads a word, then lane s stores it: lane +// s's read must not see its own later store +// AMO - the whole warp reads a word, every lane adds 1 to it +// atomically, then the whole warp reads it again: each lane's +// first read must not see its own add, and its last must +// sub-word - every lane stores one byte, four lanes to a word, then +// stores reads its word back: it must hold all four lanes' bytes +// same-word - every lane stores its own value to one word, then the +// stores whole warp reads it: the highest lane's store is the last +// cross-warp - one lane of warp k % NW stores iteration k's word while +// the other warps, on other LSU blocks, read it: each read +// returns the old word or the stored one +// counter - every thread adds 1 to one word of the block atomically +// each iteration: no add is lost +// After the iterations, every thread increments one more word of the block +// through a load-reserved/store-conditional loop: no increment is lost. The +// host can leave this phase out (its retries make the instruction count +// depend on timing). +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = reinterpret_cast(__local_mem()); + auto err = reinterpret_cast(arg->err_addr); + auto dump = reinterpret_cast(arg->dump_addr); + const uint32_t W = arg->warp_size; + const uint32_t lid = threadIdx.x; + const uint32_t bsize = blockDim.x; + const uint32_t tid = blockIdx.x * bsize + lid; + const uint32_t lane = lid % W; + const uint32_t warp = lid / W; + const uint32_t NW = bsize / W; + const uint32_t words = arg->lmem_words; + + for (uint32_t i = lid; i < words; i += bsize) { + lmem[i] = ro_word(i); + } + __syncthreads(); + + uint32_t errors = 0; + for (uint32_t k = 0; k < arg->num_iters; ++k) { + uint32_t s = k % W; + uint32_t slot = warp * arg->num_iters + k; + + uint32_t base = (slot * 7) % (RO_WORDS / 2); + uint32_t shared_idx[5] = { + base + (lane >> (k % 6)) % (RO_WORDS / 2), + base + (lane & 1) * 32, + base + (lane & 3) * 16, + base + (lane & 1), + base + (lane & 3), + }; + for (uint32_t p = 0; p < 5; ++p) { + uint32_t idx = shared_idx[p] % RO_WORDS; + if (lmem[idx] != ro_word(idx)) { + ++errors; + } + } + if (lane % 3 == k % 3) { + uint32_t idx = (base + 5) % RO_WORDS; + if (lmem[idx] != ro_word(idx)) { + ++errors; + } + } + { + uint32_t idx = (k * 5) % RO_WORDS; + if (lmem[idx] != ro_word(idx)) { + ++errors; + } + } + + { + uint32_t idx = arg->cf_base + lane * W + s; + if (lmem[idx] != ro_word(idx)) { + ++errors; + } + } + + { + uint32_t idx = (base + 9) % RO_WORDS; + uint32_t v = ro_word(idx); + uint32_t bo = lane & 3; + uint32_t ho = lane & 1; + auto lmem_u8 = reinterpret_cast(lmem); + auto lmem_s8 = reinterpret_cast(lmem); + auto lmem_u16 = reinterpret_cast(lmem); + if (lmem_u8[idx * 4 + bo] != ((v >> (8 * bo)) & 0xff)) { + ++errors; + } + if (lmem_s8[idx * 4 + bo] != (int8_t)(v >> (8 * bo))) { + ++errors; + } + if (lmem_u16[idx * 2 + ho] != ((v >> (16 * ho)) & 0xffff)) { + ++errors; + } + } + + uint32_t ri = arg->raw_base + slot; + if (lane == s) { + lmem[ri] = LMEM_MARK + ri; + } + uint32_t rv = lmem[ri]; + if (lane == s && rv != LMEM_MARK + ri) { + ++errors; + } + + uint32_t wi = arg->war_base + slot; + uint32_t wv = lmem[wi]; + if (lane == s) { + lmem[wi] = LMEM_MARK + wi; + } + if (lane == s && wv != ro_word(wi)) { + ++errors; + } + + #if VX_CFG_EXT_A_ENABLED + uint32_t ai = arg->amo_base + slot; + uint32_t a0 = lmem[ai]; + uint32_t old = __atomic_fetch_add(const_cast(&lmem[ai]), 1u, __ATOMIC_RELAXED); + uint32_t a1 = lmem[ai]; + // Other lanes' adds may land on either side of this lane's reads; this + // lane's own add must not reach its earlier read and must reach its later + // one, and returns a value no other add has passed over. + if (old - ro_word(ai) >= W || (old - a0) >= W || (a1 - old - 1) >= W) { + ++errors; + } + #endif + + { + uint32_t si = arg->sub_base + slot * SUB_WORDS(W) + lane / 4; + auto lmem_u8 = reinterpret_cast(lmem); + lmem_u8[si * 4 + (lane & 3)] = sub_byte(slot, lane); + uint32_t v = lmem[si]; + uint32_t expected = ro_word(si); + for (uint32_t b = 0; b < 4; ++b) { + uint32_t l = (lane & ~3u) + b; + if (l < W) { + expected = (expected & ~(0xffu << (8 * b))) | ((uint32_t)sub_byte(slot, l) << (8 * b)); + } + } + if (v != expected) { + ++errors; + } + } + + { + uint32_t si = arg->same_base + slot; + lmem[si] = same_word(slot, lane); + if (lmem[si] != same_word(slot, W - 1)) { + ++errors; + } + } + + { + uint32_t xi = arg->xwarp_base + k; + if (warp == k % NW) { + if (lane == s) { + lmem[xi] = xwarp_word(k); + } + } else { + uint32_t v = lmem[xi]; + if (v != ro_word(xi) && v != xwarp_word(k)) { + ++errors; + } + } + } + + #if VX_CFG_EXT_A_ENABLED + { + uint32_t ci = arg->count_word; + uint32_t old = __atomic_fetch_add(const_cast(&lmem[ci]), 1u, __ATOMIC_RELAXED); + if (old - ro_word(ci) >= bsize * arg->num_iters) { + ++errors; + } + } + #endif + } + +#if VX_CFG_EXT_A_ENABLED + if (arg->lrsc) { + // The retry is written out, not left to the compiler's compare-and-swap + // expansion, whose inner retry branch is not reconverged per lane. It is + // bounded, so a store-conditional that never succeeds leaves the count + // short instead of hanging the kernel. + auto word = const_cast(&lmem[arg->lrsc_word]); + uint32_t old; + uint32_t sc_fail; + uint32_t guard = 0; + do { + asm volatile ("lr.w %0, (%1)" : "=r"(old) : "r"(word) : "memory"); + asm volatile ("sc.w %0, %2, (%1)" : "=r"(sc_fail) : "r"(word), "r"(old + 1) : "memory"); + ++guard; + } while (sc_fail && guard < LRSC_MAX_RETRIES); + if (sc_fail || old - ro_word(arg->lrsc_word) >= bsize) { + ++errors; + } + } +#endif + err[tid] = errors; + + __syncthreads(); + for (uint32_t i = lid; i < words; i += bsize) { + dump[blockIdx.x * words + i] = lmem[i]; + } +} diff --git a/tests/regression/lmem_merge/main.cpp b/tests/regression/lmem_access/main.cpp similarity index 54% rename from tests/regression/lmem_merge/main.cpp rename to tests/regression/lmem_access/main.cpp index 391bb57bda..3b60946c55 100644 --- a/tests/regression/lmem_merge/main.cpp +++ b/tests/regression/lmem_access/main.cpp @@ -1,5 +1,6 @@ -// lmem_merge — local-memory reads that share words must each get their own -// word, and keep each thread's order with its own stores. +// lmem_access — local-memory reads that share words must each get their own +// word, stores of several lanes to one word must all land, and every thread +// keeps its order with its own stores. #include #include "common.h" @@ -24,15 +25,20 @@ namespace { const char* kernel_file = "kernel.vxbin"; uint32_t num_iters = 16; +// The LR/SC phase retries a contended store-conditional, so its instruction +// count depends on timing; -s leaves it out where instruction counts must +// match, as in the model-parity runs. +bool skip_lrsc = false; void parse_args(int argc, char** argv) { int c; - while ((c = getopt(argc, argv, "n:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:k:sh")) != -1) { switch (c) { case 'n': num_iters = std::atoi(optarg); break; case 'k': kernel_file = optarg; break; + case 's': skip_lrsc = true; break; default: - std::cout << "Usage: [-n iterations] [-k kernel] [-h]" << std::endl; + std::cout << "Usage: [-n iterations] [-k kernel] [-s (no LR/SC phase)] [-h]" << std::endl; std::exit(c == 'h' ? 0 : -1); } } @@ -49,7 +55,10 @@ int main(int argc, char** argv) { CHECK(vx_device_open(0, &dev)); // One block per core holding all its warps, two waves of blocks. - uint64_t nt = 0, nw = 0, nc = 0, lmem_size = 0; + uint64_t nt = 0; + uint64_t nw = 0; + uint64_t nc = 0; + uint64_t lmem_size = 0; CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); @@ -58,12 +67,15 @@ int main(int argc, char** argv) { const uint32_t block = static_cast(nt * nw); const uint32_t grid = static_cast(nc * 2); const uint32_t num_threads = block * grid; - // shrink the iteration count until a block's RAW and WAR words fit - while (num_iters > 1 && (RO_WORDS + 2 * nw * num_iters) * sizeof(uint32_t) > lmem_size) { + // shrink the iteration count until a block's per-iteration words fit + const uint32_t cf_words = W * W; + const uint32_t slot_words = 4 + SUB_WORDS(W); + while (num_iters > 1 && (RO_WORDS + slot_words * nw * num_iters + cf_words + num_iters + 2) * sizeof(uint32_t) > lmem_size) { num_iters /= 2; } - const uint32_t words = RO_WORDS + 2 * static_cast(nw) * num_iters; - std::cout << "lmem_merge: iters=" << num_iters << " block=" << block << " grid=" << grid + const uint32_t slots = static_cast(nw) * num_iters; + const uint32_t words = RO_WORDS + slot_words * slots + cf_words + num_iters + 2; + std::cout << "lmem_access: iters=" << num_iters << " block=" << block << " grid=" << grid << " words=" << words << std::endl; if (words * sizeof(uint32_t) > lmem_size) { std::cout << "local memory too small\nFAILED!" << std::endl; @@ -74,7 +86,8 @@ int main(int argc, char** argv) { vx_queue_h q = nullptr; CHECK(vx_queue_create(dev, &qi, &q)); - vx_buffer_h err_buf = nullptr, dump_buf = nullptr; + vx_buffer_h err_buf = nullptr; + vx_buffer_h dump_buf = nullptr; CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); CHECK(vx_buffer_create(dev, grid * words * sizeof(uint32_t), VX_MEM_WRITE, &dump_buf)); @@ -88,7 +101,15 @@ int main(int argc, char** argv) { kernel_arg.warp_size = W; kernel_arg.lmem_words = words; kernel_arg.raw_base = RO_WORDS; - kernel_arg.war_base = RO_WORDS + static_cast(nw) * num_iters; + kernel_arg.war_base = RO_WORDS + slots; + kernel_arg.amo_base = RO_WORDS + 2 * slots; + kernel_arg.cf_base = RO_WORDS + 3 * slots; + kernel_arg.sub_base = kernel_arg.cf_base + cf_words; + kernel_arg.same_base = kernel_arg.sub_base + SUB_WORDS(W) * slots; + kernel_arg.xwarp_base = kernel_arg.same_base + slots; + kernel_arg.count_word = kernel_arg.xwarp_base + num_iters; + kernel_arg.lrsc_word = kernel_arg.count_word + 1; + kernel_arg.lrsc = skip_lrsc ? 0 : 1; CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); CHECK(vx_buffer_address(dump_buf, &kernel_arg.dump_addr)); @@ -102,8 +123,10 @@ int main(int argc, char** argv) { li.block_dim[0]= block; li.lmem_size = words * sizeof(uint32_t); - std::vector h_err(num_threads), h_dump(grid * words); - vx_event_h launch_ev = nullptr, read_ev = nullptr; + std::vector h_err(num_threads); + std::vector h_dump(grid * words); + vx_event_h launch_ev = nullptr; + vx_event_h read_ev = nullptr; CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); CHECK(vx_enqueue_read(q, h_dump.data(), dump_buf, 0, grid * words * sizeof(uint32_t), 1, &launch_ev, &read_ev)); @@ -119,10 +142,32 @@ int main(int argc, char** argv) { } errors += h_err[t]; } - // every RAW and WAR word was stored by its lane; the read-only words kept their value + // every RAW and WAR word was stored by its lane, every sub-word store word + // holds its lanes' bytes, every same-word store word the highest lane's + // store, every cross-warp word its warp's store, and the counters every + // thread's increments; the read-only words kept their value for (uint32_t b = 0; b < grid; ++b) { for (uint32_t i = 0; i < words; ++i) { - uint32_t expected = (i < RO_WORDS) ? (i * 3 + 2) : (LMEM_MARK + i); + uint32_t expected = ro_word(i); + if (i >= kernel_arg.raw_base && i < kernel_arg.amo_base) { + expected = LMEM_MARK + i; + } else if (i >= kernel_arg.amo_base && i < kernel_arg.cf_base && VX_CFG_EXT_A_ENABLED) { + expected += W; + } else if (i >= kernel_arg.sub_base && i < kernel_arg.same_base) { + uint32_t slot = (i - kernel_arg.sub_base) / SUB_WORDS(W); + uint32_t lane0 = ((i - kernel_arg.sub_base) % SUB_WORDS(W)) * 4; + for (uint32_t k = 0; k < 4 && lane0 + k < W; ++k) { + expected = (expected & ~(0xffu << (8 * k))) | ((uint32_t)sub_byte(slot, lane0 + k) << (8 * k)); + } + } else if (i >= kernel_arg.same_base && i < kernel_arg.xwarp_base) { + expected = same_word(i - kernel_arg.same_base, W - 1); + } else if (i >= kernel_arg.xwarp_base && i < kernel_arg.count_word) { + expected = xwarp_word(i - kernel_arg.xwarp_base); + } else if (i == kernel_arg.count_word && VX_CFG_EXT_A_ENABLED) { + expected += block * num_iters; + } else if (i == kernel_arg.lrsc_word && VX_CFG_EXT_A_ENABLED && kernel_arg.lrsc) { + expected += block; + } uint32_t actual = h_dump[b * words + i]; if (actual != expected) { if (errors < 16) { diff --git a/tests/regression/lmem_merge/common.h b/tests/regression/lmem_merge/common.h deleted file mode 100644 index 99d53a6627..0000000000 --- a/tests/regression/lmem_merge/common.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef _COMMON_H_ -#define _COMMON_H_ - -#include - -#define LMEM_MARK 0x80000000u - -// Local-memory layout per block, in words: read-only words first, then one -// RAW and one WAR word per warp and iteration. -#define RO_WORDS 128 - -typedef struct { - uint32_t num_iters; // iterations per warp - uint32_t warp_size; // threads per warp - uint32_t lmem_words; // local-memory words per block - uint32_t raw_base; // first RAW word - uint32_t war_base; // first WAR word - uint64_t err_addr; // out: per-thread error count - uint64_t dump_addr; // out: each block's final local memory -} kernel_arg_t; - -#endif diff --git a/tests/regression/lmem_merge/kernel.cpp b/tests/regression/lmem_merge/kernel.cpp deleted file mode 100644 index 474a4aed44..0000000000 --- a/tests/regression/lmem_merge/kernel.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include -#include "common.h" - -// Local-memory reads that share words, around single-lane stores. Per -// iteration k, lane s = k % W takes a turn: -// -// shared words - lanes in groups of 1, 2, 4, ... W read one word; then 2 -// and 4 distinct words per warp, in one bank (32 words -// apart) and in different banks: every lane must get its -// own word; a third of the lanes read one word (partial -// thread mask); every warp reads the same word (lanes of -// different LSU blocks) -// RAW - lane s stores its word, then the whole warp reads it: -// lane s must see its own store -// WAR - the whole warp reads a word, then lane s stores it: lane -// s's read must not see its own later store -__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - auto lmem = reinterpret_cast(__local_mem()); - auto err = reinterpret_cast(arg->err_addr); - auto dump = reinterpret_cast(arg->dump_addr); - const uint32_t W = arg->warp_size; - const uint32_t lid = threadIdx.x; - const uint32_t bsize = blockDim.x; - const uint32_t tid = blockIdx.x * bsize + lid; - const uint32_t lane = lid % W; - const uint32_t warp = lid / W; - const uint32_t words = arg->lmem_words; - - for (uint32_t i = lid; i < words; i += bsize) { - lmem[i] = i * 3 + 2; - } - __syncthreads(); - - uint32_t errors = 0; - for (uint32_t k = 0; k < arg->num_iters; ++k) { - uint32_t s = k % W; - uint32_t slot = warp * arg->num_iters + k; - - uint32_t base = (slot * 7) % (RO_WORDS / 2); - uint32_t shared_idx[5] = { - base + (lane >> (k % 6)) % (RO_WORDS / 2), - base + (lane & 1) * 32, - base + (lane & 3) * 16, - base + (lane & 1), - base + (lane & 3), - }; - for (uint32_t p = 0; p < 5; ++p) { - uint32_t idx = shared_idx[p] % RO_WORDS; - if (lmem[idx] != idx * 3 + 2) { - ++errors; - } - } - if (lane % 3 == k % 3) { - uint32_t idx = (base + 5) % RO_WORDS; - if (lmem[idx] != idx * 3 + 2) { - ++errors; - } - } - { - uint32_t idx = (k * 5) % RO_WORDS; - if (lmem[idx] != idx * 3 + 2) { - ++errors; - } - } - - uint32_t ri = arg->raw_base + slot; - if (lane == s) { - lmem[ri] = LMEM_MARK + ri; - } - uint32_t rv = lmem[ri]; - if (lane == s && rv != LMEM_MARK + ri) { - ++errors; - } - - uint32_t wi = arg->war_base + slot; - uint32_t wv = lmem[wi]; - if (lane == s) { - lmem[wi] = LMEM_MARK + wi; - } - if (lane == s && wv != wi * 3 + 2) { - ++errors; - } - } - err[tid] = errors; - - __syncthreads(); - for (uint32_t i = lid; i < words; i += bsize) { - dump[blockIdx.x * words + i] = lmem[i]; - } -} From 372589e8eaf8d6fd6d26df4adf19f99ae7becaee Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 14:36:08 -0700 Subject: [PATCH 285/336] rtu: size the RT-cache tag for every ray context The RT-cache request tag carried clog2(RTCACHE_MSHR_SIZE) bits of context id, 4 at the default, while the RTU issues up to VX_CFG_RTU_NUM_CTX contexts: at 32 threads and more, ids aliased and responses returned to the wrong ray. The tag is now CLOG2(VX_CFG_RTU_NUM_CTX) bits, the same width at the default. Validated: rtlsim rt_smoke at the default config with 32 threads (VM off) passes, 61 instructions / 880 cycles. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index c1bd73d290..192d1aa688 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1951,7 +1951,8 @@ package VX_gpu_pkg; localparam RTCACHE_ADDR_WIDTH = (`VX_CFG_MEM_ADDR_WIDTH - `CLOG2(RTCACHE_WORD_SIZE)); localparam RTCACHE_LINE_SIZE = `VX_CFG_L1_LINE_SIZE; localparam RTCACHE_NUM_REQS = 1; - localparam RTCACHE_TAG_ID_BITS = `CLOG2(`VX_CFG_RTCACHE_MSHR_SIZE); + // The core-side tag carries the RTU context index. + localparam RTCACHE_TAG_ID_BITS = `CLOG2(`VX_CFG_RTU_NUM_CTX); localparam RTCACHE_TAG_WIDTH = (UUID_WIDTH + RTCACHE_TAG_ID_BITS); localparam RTCACHE_BUS_TAG_WIDTH = (RTCACHE_TAG_WIDTH + 1); localparam RTCACHE_MEM_DATA_WIDTH= (RTCACHE_LINE_SIZE * 8); From 01c637ec9161ed64a825324f595c56faa665587f Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 14:36:19 -0700 Subject: [PATCH 286/336] config: give the F64 divide and square root their RTL latency on FPGAs With D (XLEN 64, FLEN 64) VX_fdiv_unit and VX_fsqrt_unit use the RTL cores, since the vendor IP is F32 only, but fdiv/fsqrt_latency_dsp still handed them the IP's latency (28 on Vivado, 15/10 on Quartus) where the cores need 32. Simulation never builds the DSP FPU type, so the STATIC_ASSERT that catches it never ran; synthesis indexed past the end of the 28-stage pipeline (slang rejects it: element 29 of reg[27:0]). The vendor latency now applies only without D. 32-bit builds are unchanged (checked with gen_config: Vivado 28/28, Quartus 15/10). Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/VX_config.toml b/VX_config.toml index 378e1c2771..ccfa148a0e 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -142,8 +142,9 @@ VX_CFG_FMA_LATENCY = "expr: 16 if $fpu_dsp_vivado else (4 if $fpu_dsp_quartus e # STD timing; DPI is a fast STD and takes the same values. fdiv_latency_std = "expr: 32 if $VX_CFG_EXT_D_ENABLE else 17" fsqrt_latency_std = "expr: $fdiv_latency_std" -fdiv_latency_dsp = "expr: 15 if $QUARTUS else (28 if $VIVADO else (32 if $VX_CFG_EXT_D_ENABLE else 17))" -fsqrt_latency_dsp = "expr: 10 if $QUARTUS else (28 if $VIVADO else (32 if $VX_CFG_EXT_D_ENABLE else 17))" +# The vendor divide and square-root IP is F32 only; with D they are the RTL cores. +fdiv_latency_dsp = "expr: 32 if $VX_CFG_EXT_D_ENABLE else (15 if $QUARTUS else (28 if $VIVADO else 17))" +fsqrt_latency_dsp = "expr: 32 if $VX_CFG_EXT_D_ENABLE else (10 if $QUARTUS else (28 if $VIVADO else 17))" VX_CFG_FDIV_LATENCY = "expr: 16 if $VX_CFG_FPU_TYPE_FPNEW else ($fdiv_latency_dsp if $VX_CFG_FPU_TYPE_DSP else $fdiv_latency_std)" VX_CFG_FSQRT_LATENCY= "expr: 16 if $VX_CFG_FPU_TYPE_FPNEW else ($fsqrt_latency_dsp if $VX_CFG_FPU_TYPE_DSP else $fsqrt_latency_std)" From 98dbc20aae9b68b3f8eba359ad00957a4a48a111 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 14:36:20 -0700 Subject: [PATCH 287/336] core DUT: drive kernel launches from ports The core synthesis wrapper tied the KMU launch bus off, so the CTA dispatcher never fired and synthesis removed it: no gate (fpga, asic, logic) has ever timed it. The launch bus is now a set of DUT ports, as DCR already is. Beacon then shows the dispatcher at 17 levels (tail_r through the slot normalize and pre-wrap compares and the slot * stride product into cur_lmem_base_r) and about a dozen paths at 13-15, real on the full GPU where the KMU drives it. Their fix is separate. The core gate's area and timing baselines move with this change and need re-recording. Co-Authored-By: Claude Opus 5.5 --- hw/unittest/core/VX_core_top.sv | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/hw/unittest/core/VX_core_top.sv b/hw/unittest/core/VX_core_top.sv index d5aef6f3d4..04eb76192a 100644 --- a/hw/unittest/core/VX_core_top.sv +++ b/hw/unittest/core/VX_core_top.sv @@ -72,6 +72,15 @@ module VX_core_top import VX_gpu_pkg::*; output wire dcr_rsp_valid, output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + // Kernel launches: without them the CTA dispatcher is constant and + // synthesis removes it. + input wire kmu_valid, + input wire kmu_kind, + input wire kmu_eop, + input wire [KMU_DEST_W-1:0] kmu_dest, + input wire [KMU_DATAW-1:0] kmu_data, + output wire kmu_ready, + // Status output wire busy ); @@ -136,11 +145,12 @@ module VX_core_top import VX_gpu_pkg::*; `endif VX_kmu_bus_if kmu_bus_if(); - assign kmu_bus_if.valid = 1'b0; - assign kmu_bus_if.kind = KMU_KIND_COMPUTE; - assign kmu_bus_if.eop = 1'b0; - assign kmu_bus_if.dest = '0; - assign kmu_bus_if.data = '0; + assign kmu_bus_if.valid = kmu_valid; + assign kmu_bus_if.kind = kmu_kind; + assign kmu_bus_if.eop = kmu_eop; + assign kmu_bus_if.dest = kmu_dest; + assign kmu_bus_if.data = kmu_data; + assign kmu_ready = kmu_bus_if.ready; VX_dcr_bus_if dcr_bus_if(); From 6ca1897c1936b88fc3fbe05c9079fb98f5da4783 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 14:36:31 -0700 Subject: [PATCH 288/336] beacon: compare a change against its parent; fix the Vivado agreement Beacon becomes the inner loop of RTL timing work (beacon.md 6.3): a reference run of the parent, then each edit compared with it register by register. - ci/beacon.py diff: registers that got deeper or shallower, new and gone deep registers, units whose deepest register moved, and the mean depth of the 50 deepest. It fails on a register 4+ levels deeper, one that reaches the fix threshold (12) from below, or a new one at it. Against routed U55C runs of eight core revisions, +4 levels lost slack every time (n = 170), +1..+3 is within Vivado's run-to-run spread, and the top-50 mean tracks the change in WNS (rho 0.56) where the deepest path does not. - ci/logic_gate.py --against DIR: runs the diff for every build against the same build in a reference tree and fails on a regression (DEEPER). - ci/beacon.py validate: the minimum slack started at 0.0, clamping positive slacks (rho with slack 0.25 -> 0.40 on the p2lsulmem core); precision and recall counted every row under 1 ns as failing. Replication suffixes in any order, dual-port LUTRAM halves and registered LUTRAM reads now match, and --native TOP= reads a DUT build's table, which has no harness prefix. Co-Authored-By: Claude Opus 5.5 --- ci/beacon.py | 166 ++++++++++++++++++++++++++++++++++++++--- ci/synth_gate.py | 44 +++++++++++ docs/designs/beacon.md | 55 +++++++++++--- 3 files changed, 243 insertions(+), 22 deletions(-) diff --git a/ci/beacon.py b/ci/beacon.py index 62bcc80799..e9a976dadb 100755 --- a/ci/beacon.py +++ b/ci/beacon.py @@ -21,6 +21,8 @@ Vivado's failing ones paths every deep path op by op, by RTL unit (one per pair of registers, generate copies folded) + diff a change against its parent revision: the registers that got + deeper or shallower, by register and by unit validate agreement per register with a Vivado end point table retime-split the same agreement, split by whether Vivado retimed the paths ports the depth at the ports of chosen modules' instances, by unit @@ -32,6 +34,7 @@ top=VX_socket__lift black-boxes=VX_core__lift ci/beacon.py summarize runs/full --min-levels 12 ci/beacon.py paths runs/full --unit lsu_unit + ci/beacon.py diff runs/parent runs/change --min-levels 8 ci/beacon.py validate runs/socket ep.csv --native VX_socket__lift=dut ci/beacon.py run --keep VX_elastic_buffer design.mlir runs/eb \\ port-modules=VX_elastic_buffer @@ -62,9 +65,11 @@ 'foldsim_target': 'foldsim_target_model_copy', } +# Vivado appends replication and retiming suffixes in any order +# (_rep__0_replica_1). VIVADO_CELL = re.compile( - r'_reg((\[\w+\])*(_replica(_\d+)?)?(_[fb]ret(_+\d+)?)*(_rep(_+\d+)?)?' - r'|_\d+(_\d+_\d+_\d+)?|_bram_\d+)$') + r'_reg((\[\w+\])*(_replica(_\d+)?|_[fb]ret(_+\d+)?|_rep(_+\d+)?)*' + r'|_\d+(_\d+)*|_bram_\d+)$') FOLD_SUFFIX = re.compile( r'(_ext(_\d+)?|_fold|_read(_\d+)?|_[fb]ret(_\d+)?|_replica(_\d+)?)+$') FOLD_SHELL = re.compile(r'^fold\d+_') @@ -82,6 +87,9 @@ def ep_key(cell_path, native): Native: //[/]/, being the harness prefix. """ + # The halves of a dual-port LUTRAM (RAM32X1D/RAM64X1D: DP, SP) sit + # below the memory cell. + cell_path = re.sub(r'/(DP|SP)(/[^/]+)$', r'\2', cell_path) parts = cell_path.split('/') if native: cls = parts[0] @@ -224,9 +232,14 @@ def cmd_validate(args): inst = load_instances(f'{args.run_dir}/instances.csv') classes = FOLD_CLASSES sub = None + harness = '' if args.native: module, prefix = args.native.split('=') prefix, _, sub = prefix.partition(':') + # A DUT build's table names its cells from the DUT's top down, with + # no harness prefix. + harness = '' if prefix else 'dut/' + prefix = prefix or 'dut' classes = {module: prefix} paths = class_paths(inst, classes) @@ -284,7 +297,7 @@ def in_ip(key): parts = k.split('/') return any((cls, '/'.join(parts[:i])) in ips for i in range(1, len(parts))) - viv = collections.defaultdict(lambda: [0, 0.0, 0]) # max lv, min slack, n + viv = collections.defaultdict(lambda: [0, float('inf'), 0]) # max lv, min slack, n rows = unmatched_rows = shell_rows = ip_rows = 0 unmatched = collections.Counter() with open(args.ep) as f: @@ -292,7 +305,7 @@ def in_ip(key): next(r) for slack, levels, logic, net, start, end in r: rows += 1 - key = ep_key(end, args.native) + key = ep_key(harness + end, args.native) if key is None or key[0] not in classes.values(): shell_rows += 1 continue @@ -303,6 +316,10 @@ def in_ip(key): # The read register of a synchronous RAM is the memory's # read port in the design. key = (key[0], key[1][:-len('_rdata')]) + if (key not in ours and key[1].endswith('_rdata_r') + and (key[0], key[1][:-len('_rdata_r')] + '_ram') in ours): + # So is the registered read of a synchronous LUTRAM. + key = (key[0], key[1][:-len('_rdata_r')] + '_ram') if key not in ours: key = fallback(key) if key not in ours: @@ -344,12 +361,11 @@ def in_ip(key): f'p10 {np.percentile(sel, 10):5.1f} p90 {np.percentile(sel, 90):5.1f}') if args.native: - # The ep.csv of a gate lists every failing end point, so a measured - # register without one meets timing. + # The table lists the end points near or past the period; a measured + # register fails when its worst slack is negative. scr = np.array([v[0] for v in ours.values()]) - fails = np.array([k in viv for k in ours]) - print('\nflag registers at >= T levels (gate lists all failing end ' - 'points):') + fails = np.array([k in viv and viv[k][1] < 0 for k in ours]) + print('\nflag registers at >= T levels (failing: negative slack):') print(' T flagged precision recall') for t in range(4, 21, 2): flagged = scr >= t @@ -499,6 +515,119 @@ def cmd_ports(args): print(f' {direction:>3} {port:24s} {levels:3d}') +BIT_INDEX = re.compile(r'(\[\d+\])+$') + + +def register_depths(run_dir, metric): + """register -> (levels, deepest start) over its bits, for a run.""" + inst = load_instances(f'{run_dir}/instances.csv') + paths = class_paths(inst, FOLD_CLASSES) + regs = {} + for row in endpoints(run_dir): + levels = int(row[metric]) + cls, p = paths[int(row['end_inst'])] + end = row['end'] + full = f'{p}/{end}' if p else end + key = (cls or '', BIT_INDEX.sub('', full)) + if key not in regs or levels > regs[key][0]: + sp = paths[int(row['start_inst'])][1] + regs[key] = (levels, f'{sp}/{row["start"]}' if sp else row['start']) + return regs + + +def cmd_diff(args): + """A change's effect on depth: every register at or above --min-levels in + either run whose depth moved, worst first. Exits 1 on a regression: a + register --fail-delta or more levels deeper, one that reaches --fix-levels + from below, or one only the change has at --fix-levels. + + Against routed Vivado runs of the same DUT's revisions, a register that + got four or more levels deeper lost slack every time, while a rise of one + to three is within Vivado's own spread from run to run. The mean depth of + the deepest registers tracks the change in worst slack; the single deepest + one does not.""" + before = register_depths(args.before, args.metric) + after = register_depths(args.after, args.metric) + + def deepest(regs): + return max((v[0] for v in regs.values()), default=0) + + def watched(regs): + return sum(1 for v in regs.values() if v[0] >= args.min_levels) + + def top_mean(regs, n=50): + depths = sorted((v[0] for v in regs.values()), reverse=True)[:n] + return sum(depths) / len(depths) if depths else 0.0 + + print(f'deepest path {deepest(before)} -> {deepest(after)} levels; ' + f'mean of the 50 deepest {top_mean(before):.1f} -> {top_mean(after):.1f}; ' + f'registers at or above {args.min_levels} levels ' + f'{watched(before)} -> {watched(after)}') + + deeper, shallower, new, gone = [], [], [], [] + for key, (lv, start) in after.items(): + if key not in before: + if lv >= args.min_levels: + new.append((lv, key, start)) + continue + old = before[key][0] + if lv > old and lv >= args.min_levels: + deeper.append((lv, old, key, start)) + elif lv < old and old >= args.min_levels: + shallower.append((lv, old, key, start)) + for key, (lv, start) in before.items(): + if key not in after and lv >= args.min_levels: + gone.append((lv, key, start)) + + def name(key): + return f'{key[0]}: {key[1]}' if key[0] else key[1] + + def show(title, rows, moved): + if not rows: + return + print(f'\n{title} ({len(rows)})') + for row in rows[:args.top]: + if moved: + lv, old, key, start = row + print(f'{old:4d} -> {lv:<4d} {name(key)}') + else: + lv, key, start = row + print(f'{lv:12d} {name(key)}') + print(f'{"":>13}from {start}') + + deeper.sort(key=lambda r: (-r[0], r[1] - r[0])) + shallower.sort(key=lambda r: (-r[1], r[0] - r[1])) + new.sort(key=lambda r: -r[0]) + gone.sort(key=lambda r: -r[0]) + regress = [r for r in deeper if r[0] - r[1] >= args.fail_delta + or (r[0] >= args.fix_levels > r[1])] + regress_keys = {r[2] for r in regress} + show(f'REGRESSION: {args.fail_delta} or more levels deeper, or now at ' + f'{args.fix_levels}', regress, True) + show('deeper by less (watch)', [r for r in deeper if r[2] not in regress_keys], True) + show('new at or above the watch', new, False) + show('shallower', shallower, True) + show('no longer present', gone, False) + + # The same movement by unit: the deepest register of each unit. + units = collections.defaultdict(lambda: [0, 0]) + for i, regs in enumerate((before, after)): + for (cls, reg), (lv, _) in regs.items(): + u = units[(cls, unit(norm(reg)))] + u[i] = max(u[i], lv) + moved = sorted(((b, a, k) for k, (b, a) in units.items() + if a != b and max(a, b) >= args.min_levels), + key=lambda r: (-max(r[0], r[1]), r[0] - r[1])) + if moved: + print(f'\nunits whose deepest register moved ({len(moved)})') + for b, a, (cls, u) in moved[:args.top]: + print(f'{b:4d} -> {a:<4d} {cls + ": " if cls else ""}{u}') + + # A register the reference lacks may be a renamed one, so it fails only at + # the fix threshold, as a new unit does in logic_gate. + return 1 if regress or any(r[0] >= args.fix_levels for r in new) else 0 + + def cmd_buffers(args): """VX_elastic_buffer instances whose ready chain or output is deep. @@ -623,7 +752,8 @@ def main(): help='native gate: the measured top and the harness ' 'prefix of its cells (e.g. VX_socket__lift=dut); with ' 'SUBPATH, only the measured end points below it, which ' - 'the harness instantiates as PREFIX') + 'the harness instantiates as PREFIX; an empty PREFIX (VX_core_top=) for ' + 'a DUT build, whose cells have no harness prefix') p.add_argument('--json', help='also write the agreement to this file') p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], help="the run's column to compare") @@ -635,6 +765,22 @@ def main(): p.add_argument('--top', type=int, default=40) p.set_defaults(fn=cmd_ports) + p = sub.add_parser("diff", help="a change's effect on depth, register by register") + p.add_argument('before', help="run directory of the reference (the parent revision)") + p.add_argument('after', help="run directory of the change") + p.add_argument('--min-levels', type=int, default=8, + help="watch registers at or above this depth in either run") + p.add_argument('--fix-levels', type=int, default=12, + help="a register that reaches this depth from below, or " + "only the change has at it, fails") + p.add_argument('--fail-delta', type=int, default=4, + help="a register this many levels deeper fails; a smaller " + "rise is listed to watch") + p.add_argument('--top', type=int, default=40) + p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], + help="the runs' column to compare") + p.set_defaults(fn=cmd_diff) + p = sub.add_parser("buffers", help="elastic buffers that pass deep paths") p.add_argument('run_dir') p.add_argument('--min-levels', type=int, default=8) diff --git a/ci/synth_gate.py b/ci/synth_gate.py index 9e7c457de0..f99da2b0f5 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -1330,6 +1330,40 @@ def cell(key, fmt): return failures +def compare_against(results, ref_dir, tool): + """Each finished build's Beacon end points against the same build in + ref_dir, register by register; return the builds a register got deeper + in, as failing verdicts.""" + script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "beacon.py") + failures = [] + for r in results: + b, work = r["build"], r.get("work") + if not r["metrics"] or not work: + continue + ref = os.path.join(build_work(b, ref_dir, tool), "beacon") + cur = os.path.join(work, "beacon") + if not os.path.exists(os.path.join(ref, "endpoints.csv")): + print("\n%s: no reference run in %s" % (b["id"], ref)) + continue + out = os.path.join(work, "beacon_diff.txt") + with open(out, "w") as fh: + m = r["metrics"] + rc = subprocess.run([sys.executable, script, "diff", ref, cur, + "--min-levels", str(m.get("watch_levels") or 8), + "--fix-levels", str(m.get("fix_levels") or 12)], + stdout=fh, stderr=subprocess.STDOUT).returncode + with open(out) as fh: + head = fh.readline().strip() + print("\n%s against %s: %s%s\n see %s" + % (b["id"], ref_dir, head, " -- REGRESSION" if rc == 1 else "", out)) + if rc == 1: + failures.append((b["id"], "DEEPER", ["a register got deeper than in " + "the reference: %s" % out])) + elif rc: + failures.append((b["id"], "BUILD-FAIL", ["beacon.py diff failed: %s" % out])) + return failures + + # --------------------------------------------------------------------------- def main(argv=None, default_tool="xilinx"): @@ -1371,6 +1405,12 @@ def main(argv=None, default_tool="xilinx"): ap.add_argument("--build-dir", metavar="DIR", help="build tree to synthesize in (default: " "/build_, configured on demand)") + ap.add_argument("--against", metavar="DIR", + help="Beacon only: the build tree of a reference run, " + "typically the parent revision's; each build's end " + "points are compared register by register with the " + "same build there (ci/beacon.py diff), and a register " + "that got deeper fails the run") ap.add_argument("--xlen", type=int, default=32, choices=(32, 64), help="ISA width (default: 32)") ap.add_argument("--opt-level", type=int, default=3, metavar="N", @@ -1441,6 +1481,8 @@ def main(argv=None, default_tool="xilinx"): return 0 tool.check_env() + if args.against and tool.name != "beacon": + sys.exit("ERROR: --against compares Beacon runs; use it with --tool=beacon") if args.jobs < 1: sys.exit("ERROR: --jobs must be >= 1") @@ -1467,6 +1509,8 @@ def main(argv=None, default_tool="xilinx"): gated = tuple(m for m in (args.gate.split(",") if args.gate else tool.gated) if m) outcomes = report(results, env, args, gated, tool, args.update_baseline) failures = [o for o in outcomes if o[1] not in PASSING] + if args.against: + failures += compare_against(results, args.against, tool) ok = [r for r in results if r["metrics"]] if args.update_baseline and ok: diff --git a/docs/designs/beacon.md b/docs/designs/beacon.md index a91141b372..d7a90fe7d5 100644 --- a/docs/designs/beacon.md +++ b/docs/designs/beacon.md @@ -358,18 +358,49 @@ is the CIRCT and foldsim versions, `DEVICE` and `XLEN`: a new foldsim reads as STALE, not as a regression. Baselines are golden data, re-recorded only by a human (`--update-baseline`), as for the other gates. -### 6.3 RTL timing work - -Beacon first, then Vivado: - -```bash -ci/beacon.py run gen/design.mlir runs/full -ci/beacon.py summarize runs/full --min-levels 12 # the fix list, by unit -``` - -Beacon ranks the units; one Vivado run confirms the fix. Compare a fix -against a native gate (`hw/syn/xilinx/dut` or the OOC gates), not the -folded design (§9). +### 6.3 RTL timing work: Beacon first + +Beacon is the inner loop of any RTL change that can move timing. A DUT takes +minutes in Beacon and hours in Vivado or Yosys, so a deeper path is found and +fixed while the change is still open, and synthesis runs once, to confirm. + +1. **Reference.** Beacon on the parent revision, once per base, in a build + tree of its own: + + ```bash + git worktree add ../ref HEAD + ../ref/ci/logic_gate.py -b core -b lmem --build-dir $PWD/build32_beacon_ref + ``` + +2. **Change.** The same builds on the working tree, each compared with the + reference register by register: + + ```bash + ci/logic_gate.py -b core -b lmem --against build32_beacon_ref + ``` + + Besides the per-unit verdict against the golden baseline, each build + writes `beacon_diff.txt` (`ci/beacon.py diff`). It lists the registers + that got deeper, new deep registers, and the ones that got shallower, then + the units whose deepest register moved. A register that got deeper fails + the run (`DEEPER`), and so does a new one at the fix threshold (12). A new + one below it is listed but passes, since a rename looks the same. + +3. **Fix.** `ci/beacon.py paths /beacon --unit ` prints a deep + unit's paths op by op. Edit, and repeat step 2 until nothing got deeper, + or until each deeper path is one the change intends. + +4. **Confirm.** Run synthesis once per change, not once per edit: the + `fpga_gate` or `asic_gate` builds of the DUTs whose depth moved. Then feed + the result back. `ci/beacon.py validate /beacon ep.csv` compares the + run with Vivado's end point table, and a disagreement there is a Beacon bug + to fix in foldsim. + +Beacon counts logic levels; it does not see wire delay, placement, SLR +crossings, congestion or area. A change that moves those (a wide fan-out, a +floorplan, a larger RAM) still needs synthesis before it lands. Compare a fix +against a native gate (`hw/syn/xilinx/dut` or the OOC gates), not the folded +design (§9). ## 7. Validation From 1e0d51d095d02797bf617b7bc6e7869894f91b31 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 14:36:46 -0700 Subject: [PATCH 289/336] profiles: model the GPUs' full feature set and pin them to XLEN 64 The GPU profiles only set the core, cache and memory shape, so they ran without the units every part has. Each GPU profile now also enables VM, atomics with Zacas, Zfhmin, MUFU at the part's SFU throughput, 32 shared memory banks and the part's shared-memory size, plus its tensor formats, DXA (TMA), RTU (RT cores) and the graphics units where the part has them, each with its data-sheet reference. The real parts are 64-bit machines, so a profile may name an "xlen": configure --profile picks XLEN from it and rejects a conflicting --xlen, and a build tree of the other width rejects the profile at make time (gen_config.py --xlen / --profile-xlen). tinygpu names none and builds at either width. rtx1080 becomes gtx1080, the part's real name. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- ci/gen_config.py | 21 +++++++-- config.mk.in | 2 +- configure | 15 ++++++- docs/simulation.md | 17 +++++--- profiles/a100.json | 30 ++++++++++++- profiles/gtx1080.json | 58 +++++++++++++++++++++++++ profiles/h100.json | 31 ++++++++++++- profiles/rtx1080.json | 36 --------------- profiles/rtx5080.json | 39 ++++++++++++++++- profiles/v100.json | 28 +++++++++++- tests/unittest/dram_bank_model/main.cpp | 2 +- 12 files changed, 227 insertions(+), 56 deletions(-) create mode 100644 profiles/gtx1080.json delete mode 100644 profiles/rtx1080.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 59407123e5..35ab713ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,10 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, ### Added -- **Hardware profiles.** `profiles/.json` names a machine (`tinygpu`, `rtx1080`, `v100`, `a100`, `h100`, `rtx5080`) that every flow +- **Hardware profiles.** `profiles/.json` names a machine (`tinygpu`, `gtx1080`, `v100`, `a100`, `h100`, `rtx5080`) that every flow builds through `configure --profile=` or `blackbox.sh --profile=`; explicit `CONFIGS` win over the profile. + The GPU profiles are 64-bit machines with VM, atomics, MUFU, shared memory and the tensor, DXA, ray-tracing and graphics units of + the real part; a profile's `xlen` sets `configure`'s XLEN, and a build tree of the other width rejects it. See [docs/simulation.md](docs/simulation.md#hardware-profiles). - **Per-partition fetch (`VX_CFG_FETCH_WIDTH`, default 1).** The front end splits into FETCH_WIDTH slices, each with its own warp scheduler, fetch, decode and icache port, serving the warps with `wid % FETCH_WIDTH == f`; CTA dispatch, barriers and warp control diff --git a/ci/gen_config.py b/ci/gen_config.py index 316c8d0a97..58aa770a45 100755 --- a/ci/gen_config.py +++ b/ci/gen_config.py @@ -484,7 +484,7 @@ def set_enum(name: str, val: Any) -> None: # ----------------------------- PROFILE_SECTIONS = ("core", "chip", "platform") -PROFILE_FIELDS = {"name", "extends", "description", "reference", "sources"} | set(PROFILE_SECTIONS) +PROFILE_FIELDS = {"name", "extends", "description", "xlen", "reference", "sources"} | set(PROFILE_SECTIONS) @dataclass @@ -495,6 +495,7 @@ class Profile: sections: Dict[str, Set[str]] # section -> knobs it sets reference: Dict[str, Any] sources: Any + xlen: Optional[int] # ISA width the machine needs (None: any) def _profile_path(name: str, profiles_dir: str) -> str: @@ -519,7 +520,7 @@ def load_profile(name: str, profiles_dir: str, _seen: Tuple[str, ...] = ()) -> P base = load_profile(data["extends"], profiles_dir, _seen + (name,)) else: base = Profile(name="", chain=[], values={}, sections={s: set() for s in PROFILE_SECTIONS}, - reference={}, sources=None) + reference={}, sources=None, xlen=None) values = dict(base.values) sections = {s: set(base.sections[s]) for s in PROFILE_SECTIONS} @@ -532,8 +533,11 @@ def load_profile(name: str, profiles_dir: str, _seen: Tuple[str, ...] = ()) -> P reference = dict(base.reference) reference.update(data.get("reference", {})) sources = data.get("sources", base.sources) + xlen = data.get("xlen", base.xlen) + if xlen not in (None, 32, 64): + raise ValueError(f"Profile '{name}': xlen must be 32 or 64") return Profile(name=data.get("name", name), chain=base.chain + [name], values=values, - sections=sections, reference=reference, sources=sources) + sections=sections, reference=reference, sources=sources, xlen=xlen) def validate_profile(prof: Profile, toml_defs: Dict[str, Any], enums: Dict[str, EnumSpec], @@ -1621,6 +1625,10 @@ def main(argv: List[str]) -> int: ap.add_argument("--profile", default="", help="hardware profile: profiles/.json next to the config, or a .json path") ap.add_argument("--profiles-dir", default=None, help="profile directory (default: /profiles)") + ap.add_argument("--xlen", type=int, default=None, + help="the build's XLEN; a profile that names another xlen is rejected") + ap.add_argument("--profile-xlen", action="store_true", + help="print the profile's xlen (empty if it names none) and exit") ap.add_argument("--merge-configs", action="store_true", help="print --cflags with the profile folded in (explicit -D flags win) and exit") args, unknown = ap.parse_known_args(argv) @@ -1641,6 +1649,13 @@ def main(argv: List[str]) -> int: prof = load_profile(args.profile, profiles_dir) validate_profile(prof, toml_defs, enums, params_set | set(builtins.keys())) + if args.profile_xlen: + sys.stdout.write(f"{prof.xlen if prof and prof.xlen else ''}\n") + return 0 + if prof is not None and prof.xlen is not None and args.xlen is not None and args.xlen != prof.xlen: + raise ValueError(f"Profile '{prof.name}' models a {prof.xlen}-bit machine but this build is XLEN={args.xlen}: " + f"use a build{prof.xlen} tree (configure --xlen={prof.xlen})") + if args.merge_configs: out = merge_configs(prof, args.cflags, enums) if prof else args.cflags sys.stdout.write(out.strip() + "\n") diff --git a/config.mk.in b/config.mk.in index d2987da576..5c26a2b871 100644 --- a/config.mk.in +++ b/config.mk.in @@ -69,7 +69,7 @@ VRT_HOME ?= /usr/local # CONFIGS win over the profile. PROFILE ?= @PROFILE@ ifneq ($(strip $(PROFILE)),) -override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --merge-configs --cflags='$(CONFIGS)') +override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --xlen=$(XLEN) --merge-configs --cflags='$(CONFIGS)') ifneq ($(.SHELLSTATUS),0) $(error invalid hardware profile '$(PROFILE)') endif diff --git a/configure b/configure index 0a37713f77..848dcef99e 100755 --- a/configure +++ b/configure @@ -177,7 +177,7 @@ usage() { } while [[ "$#" -gt 0 ]]; do case $1 in - --xlen=*) XLEN="${1#*=}" ;; + --xlen=*) XLEN="${1#*=}"; XLEN_SET=1 ;; --tooldir=*) TOOLDIR="${1#*=}" ;; --osversion=*) OSVERSION="${1#*=}" ;; --prefix=*|--installdir=*) PREFIX="${1#*=}" ;; @@ -194,6 +194,19 @@ if [ -n "$PROFILE" ] && [ ! -f "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" & exit -1 fi +# a profile that names an xlen picks it, and an explicit --xlen must agree +if [ -n "$PROFILE" ]; then + PROFILE_XLEN=$(python3 "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/ci/gen_config.py" \ + --config="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/VX_config.toml" --profile="$PROFILE" --profile-xlen) || exit -1 + if [ -n "$PROFILE_XLEN" ]; then + if [ -n "$XLEN_SET" ] && [ "$XLEN" != "$PROFILE_XLEN" ]; then + echo "Error: profile '$PROFILE' models a $PROFILE_XLEN-bit machine; --xlen=$XLEN disagrees" + exit -1 + fi + XLEN=$PROFILE_XLEN + fi +fi + # check OS if [ "$OSVERSION" == "unsupported" ]; then echo "Error: Unsupported OS." diff --git a/docs/simulation.md b/docs/simulation.md index b0ffa3eea2..0c4ea4271a 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -231,11 +231,18 @@ machine from it: SimX, rtlsim, the FPGA and ASIC flows, and the kernels. | Profile | Machine | |---|---| | `tinygpu` | 1 core, 2 warps x 2 threads, shared memory | -| `rtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 2 MB L2, GDDR5X as 16 GDDR6 channels | -| `v100` | Tesla V100 (Volta): 80 cores, 4 MB L2, 32 HBM2 channels at 1752 MT/s | -| `a100` | A100 (Ampere): 108 cores, 32 MB L2, HBM2 at the card's 1555 GB/s | -| `h100` | H100 SXM5 (Hopper): 132 cores, 32 MB L2, HBM3 at the card's 3.35 TB/s | -| `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 MB L2, GDDR7 bandwidth on the GDDR6 model | +| `gtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 96 KB shared memory (64 KB modelled), 2 MB L2, GDDR5X as 16 GDDR6 channels; no tensor cores | +| `v100` | Tesla V100 (Volta): 80 cores x 64 warps, 64 KB shared memory, 4 MB L2, 32 HBM2 channels at 1752 MT/s; FP16 tensor cores | +| `a100` | A100 (Ampere): 108 cores x 64 warps, 128 KB shared memory, 32 MB L2, HBM2 at 1555 GB/s; FP16/TF32/INT8/INT4 tensor cores with 2:4 sparsity | +| `h100` | H100 SXM5 (Hopper): 132 cores x 64 warps, 128 KB shared memory, 32 MB L2, HBM3 at 3.35 TB/s; FP16/TF32/FP8/INT8 tensor cores with sparsity and wgmma, TMA as DXA | +| `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 KB shared memory, 64 MB L2, GDDR7 bandwidth on the GDDR6 model; FP16/TF32/FP8/FP4/MX/INT8 tensor cores with sparsity, TMA as DXA, RT cores as RTU | + +Every GPU profile also turns on virtual memory, the A extension with Zacas, +Zfhmin, MUFU at the part's special-function throughput, and 32 shared-memory +banks. The real parts are 64-bit machines, so these profiles set `"xlen": 64`: +`configure --profile=` picks XLEN from the profile (an explicit, +conflicting `--xlen` is an error), and a `build32` tree rejects them. A profile +without an `xlen` field, like `tinygpu`, builds at either width. Select one per build with `../configure --profile=`, or per run: diff --git a/profiles/a100.json b/profiles/a100.json index a9f778afe1..bdf6a6f8f2 100644 --- a/profiles/a100.json +++ b/profiles/a100.json @@ -1,12 +1,28 @@ { "name": "a100", "description": "NVIDIA A100 SXM4 (GA100, Ampere, sm_80) at its 1410 MHz boost clock: one Vortex core per SM.", + "xlen": 64, "core": { "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, "VX_CFG_DCACHE_SIZE": 131072, - "VX_CFG_FETCH_WIDTH": 4 + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 17, + "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_TF32_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_INT4_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -34,6 +50,16 @@ "PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_QUEUE_SIZE": "Fixed DRAM latency and per-channel request queue of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)", "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA A100 Tensor Core GPU Architecture whitepaper)", - "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "164 KB maximum shared memory per SM (CUDA Programming Guide, compute capability 8.0), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the 192 KB L1/shared pool rounded down: the driver gives a kernel the L1 its shared-memory use leaves", + "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Third-generation tensor cores: FP16/BF16, TF32, INT8 and INT4, with 2:4 structured sparsity (A100 whitepaper). A100 has cp.async but no TMA, so no DXA", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Off: the datacenter part is built for compute (no display outputs; the raster and ROP hardware is reduced or absent), so the profile models its compute machine", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 8.0)", + "SIMD_WIDTH": "Left at NUM_THREADS (32) although the part has 16 FP32 / 16 INT32 lanes per SM processing block, so a warp instruction issues over two cycles (A100 whitepaper): the RTL TCU needs full-warp operands (NUM_TCU_LANES == NUM_THREADS) and VX_lane_dispatch cannot widen a 16-lane dispatch to 32 lanes, so SIMD_WIDTH=16 with EXT_TCU_ENABLE does not build in rtlsim. The tensor cores are kept; the ALU/FPU/LSU issue width is 2x the part's" } } diff --git a/profiles/gtx1080.json b/profiles/gtx1080.json new file mode 100644 index 0000000000..9834525c10 --- /dev/null +++ b/profiles/gtx1080.json @@ -0,0 +1,58 @@ +{ + "name": "gtx1080", + "description": "NVIDIA GeForce GTX 1080 (GP104, Pascal, sm_61) at its 1607 MHz base clock: one Vortex core per SM.", + "xlen": 64, + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 32768, + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 32, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 20, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 2097152, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 16, + "VX_CFG_CP_LLC_CLIENT": 1 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, + "VX_CFG_PLATFORM_CLOCK_RATE": 1607, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 320000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 10000 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "20 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA GeForce GTX 1080 whitepaper)", + "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at 10000 MT/s: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own instruction buffer and warp scheduler (NVIDIA GeForce GTX 1080 (GP104) whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "96 KB dedicated shared memory per SM, separate from the 48 KB L1/texture (GTX 1080 whitepaper), rounded down to the power of two the LMEM accepts", + "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE": "Pascal has no tensor cores and no RT cores", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", + "NUM_MUFU_LANES": "32 special-function results per clock per SM (8 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 6.1)" + } +} diff --git a/profiles/h100.json b/profiles/h100.json index 5100b09169..00eab71817 100644 --- a/profiles/h100.json +++ b/profiles/h100.json @@ -1,12 +1,30 @@ { "name": "h100", "description": "NVIDIA H100 SXM5 (GH100, Hopper, sm_90) at its 1980 MHz boost clock: one Vortex core per SM.", + "xlen": 64, "core": { "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, "VX_CFG_DCACHE_SIZE": 262144, - "VX_CFG_FETCH_WIDTH": 4 + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 17, + "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_TF32_ENABLE": true, + "VX_CFG_TCU_FP8_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_TCU_WGMMA_ENABLE": true, + "VX_CFG_EXT_DXA_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -31,6 +49,15 @@ "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM3 stacks x 16 64-bit channels at 5236 MT/s = 40 128-bit channel pairs, modelled as 32 at 6545 MT/s for a power-of-two interleave: 32 x 16 B x 6545 MT/s = 3.35 TB/s, the card's bandwidth; latency not calibrated", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)", "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", - "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "228 KB maximum shared memory per SM (CUDA Programming Guide, compute capability 9.0), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the whole 256 KB L1/shared pool: the driver gives a kernel the L1 its shared-memory use leaves", + "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Fourth-generation tensor cores: FP16/BF16, TF32, FP8 and INT8 (no INT4), 2:4 structured sparsity, warpgroup MMA (wgmma); the Tensor Memory Accelerator is the DXA (H100 whitepaper)", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Off: the datacenter part is built for compute (no display outputs; the raster and ROP hardware is reduced or absent), so the profile models its compute machine", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 9.0)" } } diff --git a/profiles/rtx1080.json b/profiles/rtx1080.json deleted file mode 100644 index 3c95172e2c..0000000000 --- a/profiles/rtx1080.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "rtx1080", - "description": "NVIDIA GeForce GTX 1080 (GP104, Pascal, sm_61) at its 1607 MHz base clock: one Vortex core per SM.", - "core": { - "VX_CFG_NUM_WARPS": 64, - "VX_CFG_NUM_THREADS": 32, - "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 32768, - "VX_CFG_FETCH_WIDTH": 4 - }, - "chip": { - "VX_CFG_NUM_CLUSTERS": 1, - "VX_CFG_NUM_CORES": 20, - "VX_CFG_L2_ENABLE": true, - "VX_CFG_L2_SIZE": 2097152, - "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 16, - "VX_CFG_CP_LLC_CLIENT": 1 - }, - "platform": { - "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, - "VX_CFG_PLATFORM_CLOCK_RATE": 1607, - "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 320000, - "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", - "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 10000 - }, - "sources": { - "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "20 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA GeForce GTX 1080 whitepaper)", - "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", - "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at 10000 MT/s: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)", - "FETCH_WIDTH": "4 SM processing blocks, each with its own instruction buffer and warp scheduler (NVIDIA GeForce GTX 1080 (GP104) whitepaper)", - "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" - } -} diff --git a/profiles/rtx5080.json b/profiles/rtx5080.json index f703f3e82b..4f250455ef 100644 --- a/profiles/rtx5080.json +++ b/profiles/rtx5080.json @@ -1,13 +1,38 @@ { "name": "rtx5080", "description": "NVIDIA GeForce RTX 5080 (GB203, Blackwell, sm_120) at its 2295 MHz base clock: one Vortex core per SM.", + "xlen": 64, "core": { "VX_CFG_NUM_WARPS": 48, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, "VX_CFG_DCACHE_SIZE": 131072, "VX_CFG_DCACHE_MSHR_SIZE": 24, - "VX_CFG_FETCH_WIDTH": 4 + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_TF32_ENABLE": true, + "VX_CFG_TCU_FP8_ENABLE": true, + "VX_CFG_TCU_FP4_ENABLE": true, + "VX_CFG_TCU_MX_ENABLE": true, + "VX_CFG_TCU_MXFP4_ENABLE": true, + "VX_CFG_TCU_NVFP4_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_EXT_DXA_ENABLE": true, + "VX_CFG_EXT_RTU_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_RTU_TLAS_ENABLE": true }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -62,6 +87,16 @@ "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_DATA_RATE": "30 Gbps GDDR7 x8 channels modelled as GDDR6 x16 channels at 15000 MT/s: the same channel count, 32 B burst time and 960 GB/s (32 x 2 B x 15 GT/s)", "PLATFORM_MEMORY_LATENCY": "675 core cycles of PHY/NoC latency, calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference' with the same chase (random cyclic permutation, 128 B stride, one thread, cold L2 with CP_LLC_CLIENT=0; cycle difference between 128 KiB and 256 KiB footprints over the 1024 extra loads: 802.1). The L2 link latency is not modelled, so an L2 hit is about 79 cycles against the card's 354. Recalibrate when the memory model changes", "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA RTX Blackwell GPU Architecture whitepaper)", - "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "100 KB maximum shared memory per SM (compute capability 12.0 device query: 102400 B per SM), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the whole 128 KB L1/shared pool: the driver gives a kernel the L1 its shared-memory use leaves", + "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Fifth-generation tensor cores: FP16/BF16, TF32, FP8, FP4 (NVFP4, MXFP4) and the MX formats, INT8, with 2:4 structured sparsity; no wgmma on sm_120. TMA is the DXA; the fourth-generation RT cores are the RTU (RTX Blackwell whitepaper)", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", + "RTU_TLAS_ENABLE": "RT cores traverse two-level acceleration structures (instance TLAS over BLAS), as DXR/Vulkan ray tracing requires", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 12.0)" } } diff --git a/profiles/v100.json b/profiles/v100.json index 4bccc586bd..cf8a79ab7a 100644 --- a/profiles/v100.json +++ b/profiles/v100.json @@ -1,12 +1,26 @@ { "name": "v100", "description": "NVIDIA Tesla V100 SXM2 (GV100, Volta, sm_70) at its 1530 MHz boost clock: one Vortex core per SM.", + "xlen": 64, "core": { "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, "VX_CFG_DCACHE_SIZE": 131072, - "VX_CFG_FETCH_WIDTH": 4 + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -31,6 +45,16 @@ "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "4 HBM2 stacks x 8 128-bit channels = 32 channels at 1752 MT/s: 32 x 16 B x 1752 MT/s = 897 GB/s (whitepaper)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)", "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA Tesla V100 GPU Architecture whitepaper)", - "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "96 KB maximum shared memory per SM (CUDA Programming Guide, compute capability 7.0), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the whole 128 KB L1/shared pool: the driver gives a kernel the L1 its shared-memory use leaves, so a kernel without shared memory sees the full pool", + "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Volta tensor cores: FP16 inputs with FP32 accumulation only (Volta whitepaper). The TCU's FP16 format group also carries BF16, which Volta lacks: no knob separates them", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 7.0)", + "SIMD_WIDTH": "Left at NUM_THREADS (32) although the part has 16 FP32 / 16 INT32 lanes per SM processing block, so a warp instruction issues over two cycles (Volta whitepaper): the RTL TCU needs full-warp operands (NUM_TCU_LANES == NUM_THREADS) and VX_lane_dispatch cannot widen a 16-lane dispatch to 32 lanes, so SIMD_WIDTH=16 with EXT_TCU_ENABLE does not build in rtlsim. The tensor cores are kept; the ALU/FPU/LSU issue width is 2x the part's" } } diff --git a/tests/unittest/dram_bank_model/main.cpp b/tests/unittest/dram_bank_model/main.cpp index c52421105a..26458cdace 100644 --- a/tests/unittest/dram_bank_model/main.cpp +++ b/tests/unittest/dram_bank_model/main.cpp @@ -62,7 +62,7 @@ const Machine kMachines[] = { {"v100", {DramSim::Type::HBM2, 1, 64, 1530, 1752, 0, 0}}, {"a100", {DramSim::Type::HBM2, 1, 64, 1410, 3038, 190, 257}}, {"h100", {DramSim::Type::HBM3, 1, 64, 1980, 6545, 0, 0}}, - {"rtx1080", {DramSim::Type::GDDR6, 1, 64, 1607, 10000, 0, 0}}, + {"gtx1080", {DramSim::Type::GDDR6, 1, 64, 1607, 10000, 0, 0}}, {"rtx5080", {DramSim::Type::GDDR6, 1, 64, 2295, 15000, 339, 0}}, }; From 9ae6dee7a4850665769f3c53fc59c36eadb0ccc3 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 20:40:42 -0700 Subject: [PATCH 290/336] docs: core partitioning design document The core partitioning section in microarchitecture.md was one dense paragraph and a table, with no figure, while every other subsystem has its own illustrated design document. core_partitioning.md covers how FW, IW and the per-unit block counts slice the warps, the three structures the widths select (compute slices, backend slices, stage-sliced), the frontend crossbar, the core-wide services, configuration examples including the U55C and rtx5080 shapes, and the performance and area trade-offs. microarchitecture.md keeps a summary and links to it. Co-Authored-By: Claude Opus 5.5 --- .../img/core_partitioning_structures.svg | 192 ++++++++++++++++++ docs/designs/core_partitioning.md | 184 +++++++++++++++++ docs/designs/microarchitecture.md | 41 +--- 3 files changed, 385 insertions(+), 32 deletions(-) create mode 100644 docs/assets/img/core_partitioning_structures.svg create mode 100644 docs/designs/core_partitioning.md diff --git a/docs/assets/img/core_partitioning_structures.svg b/docs/assets/img/core_partitioning_structures.svg new file mode 100644 index 0000000000..40a1198394 --- /dev/null +++ b/docs/assets/img/core_partitioning_structures.svg @@ -0,0 +1,192 @@ + + + + + + + + + + + Core partitioning at issue width 4: the widths select the structure + + Compute slices + FW = IW = every B = 4 + + core-wide: CTA dispatch, barriers, retirement, + SFU/TCU shared services, icache · dcache · LMEM + + Backend slices + FW = 2 < IW = every B = 4 + + core-wide: CTA dispatch, barriers, retirement, + SFU/TCU shared services, icache · dcache · LMEM + + Stage-sliced + FW = IW = 4, but the SFU has B = 1 + + core-wide: CTA dispatch, barriers, retirement, + SFU/TCU shared services, icache · dcache · LMEM + + compute slice 0 + + frontend + slice + wid mod 4 = 0 + + + backend + slice + issue + ALU · FPU + LSU · SFU + TCU + commit + + + compute slice 1 + + frontend + slice + wid mod 4 = 1 + + + backend + slice + issue + ALU · FPU + LSU · SFU + TCU + commit + + + compute slice 2 + + frontend + slice + wid mod 4 = 2 + + + backend + slice + issue + ALU · FPU + LSU · SFU + TCU + commit + + + compute slice 3 + + frontend + slice + wid mod 4 = 3 + + + backend + slice + issue + ALU · FPU + LSU · SFU + TCU + commit + + + frontend slice 0 + wid mod 2 = 0 + own icache port + + + frontend slice 1 + wid mod 2 = 1 + own icache port + + + VX_frontend_xbar: combinational routing + + + backend + slice 0 + wid mod 4 = 0 + issue + 1 block of + every unit + commit + + + + backend + slice 1 + wid mod 4 = 1 + issue + 1 block of + every unit + commit + + + + backend + slice 2 + wid mod 4 = 2 + issue + 1 block of + every unit + commit + + + + backend + slice 3 + wid mod 4 = 3 + issue + 1 block of + every unit + commit + + + frontend + slice 0 + wid mod 4 = 0 + + + frontend + slice 1 + wid mod 4 = 1 + + + frontend + slice 2 + wid mod 4 = 2 + + + frontend + slice 3 + wid mod 4 = 3 + + + VX_frontend_xbar: combinational routing + + + VX_issue + 4 issue slices: ibuffer, scoreboard, operands + + + VX_execute + ALU, FPU, LSU, TCU: 4 blocks (wid mod 4) + SFU: 1 block, takes the 4 issue slices in turn + + + VX_commit: 4 commit slices + + diff --git a/docs/designs/core_partitioning.md b/docs/designs/core_partitioning.md new file mode 100644 index 0000000000..5034f058c4 --- /dev/null +++ b/docs/designs/core_partitioning.md @@ -0,0 +1,184 @@ +# Core Partitioning — Design + +**Scope:** how a Vortex core divides its warps among parallel pipelines — the +fetch width, the issue width and the per-unit block counts, the three core +structures they select, the frontend crossbar that joins slices of different +widths, and the services that stay core-wide. Covers the RTL +([`VX_core.sv`](../../hw/rtl/core/VX_core.sv), +[`VX_frontend_slice.sv`](../../hw/rtl/core/VX_frontend_slice.sv), +[`VX_frontend_xbar.sv`](../../hw/rtl/core/VX_frontend_xbar.sv), +[`VX_backend_slice.sv`](../../hw/rtl/core/VX_backend_slice.sv), +[`VX_compute_slice.sv`](../../hw/rtl/core/VX_compute_slice.sv), +[`VX_issue.sv`](../../hw/rtl/core/VX_issue.sv), +[`VX_execute.sv`](../../hw/rtl/core/VX_execute.sv), +[`VX_commit.sv`](../../hw/rtl/core/VX_commit.sv), +[`VX_lane_dispatch.sv`](../../hw/rtl/core/VX_lane_dispatch.sv)) and the +configuration knobs in [`VX_config.toml`](../../VX_config.toml). + +The pipeline stages themselves are described in +[`microarchitecture.md`](microarchitecture.md); the CTA dispatcher that feeds +warps to the slices in [`cta_dispatch_architecture.md`](cta_dispatch_architecture.md); +the LSU blocks in [`lsu_pipeline_design.md`](lsu_pipeline_design.md). + +![Core partitioning](../assets/img/core_partitioning_structures.svg) + +--- + +## 1. Slicing by warp id + +A core holds `VX_CFG_NUM_WARPS` warps. Every parallel structure in it owns a +fixed subset of them, selected by the low bits of the warp id: + +| Width | Knob | Owner of warp `wid` | Unit of replication | +|---|---|---|---| +| FW | `VX_CFG_FETCH_WIDTH` | frontend slice `wid mod FW` | warp scheduler, fetch, decode, one icache port | +| IW | `VX_CFG_ISSUE_WIDTH` | issue slice `wid mod IW` | ibuffer, scoreboard, operand collector, commit | +| B | `VX_CFG_NUM_{ALU,FPU,LSU,SFU,TCU}_BLOCKS` | block `wid mod B` of that unit | one block of the execution unit | + +The widths must nest: FW and every B divide IW. FW is a power of two +(`VX_CFG_FETCH_WIDTH` defaults to the lowest set bit of IW), so `wid mod FW` +and `wid mod B` are bit slices of `wid mod IW`, and a warp never changes +slice. Each structure names its warps locally — a frontend slice's warp +`wfs` is core warp `wfs·FW + slice_id` — and instructions leave a slice +carrying the core-wide `wid`. + +Defaults: IW is `up(NUM_WARPS / 16)`, so a core of up to 16 warps has one +issue slice; FW follows IW; the ALU, FPU, LSU and TCU have one block per +issue slice (B = IW); the SFU has one block (`VX_CFG_NUM_SFU_BLOCKS = 1`). + +## 2. Three structures + +[`VX_gpu_pkg.sv`](../../hw/rtl/VX_gpu_pkg.sv) derives the structure from +the widths; there is no switch for it. + +``` +BACKEND_SLICES = every enabled unit has B == IW (ALU, LSU, SFU, FPU if F, TCU if TCU) +COMPUTE_SLICES = BACKEND_SLICES && FW == IW +``` + +| Widths | Structure | Modules | +|---|---|---| +| every B = IW = FW | **compute slices** | IW × `VX_compute_slice`, no crossbar | +| every B = IW, FW < IW | **backend slices** | FW × `VX_frontend_slice`, `VX_frontend_xbar`, IW × `VX_backend_slice` | +| some unit has B ≠ IW | **stage-sliced** | FW × `VX_frontend_slice`, `VX_frontend_xbar`, `VX_issue` / `VX_execute` / `VX_commit` | + +All three run the same warps through the same stages with the same +timing; they differ in how many instances there are and what connects them. +With IW = FW = 1 and the default block counts, every condition holds and the +core is a single compute slice. + +### 2.1 Compute slices + +A compute slice ([`VX_compute_slice.sv`](../../hw/rtl/core/VX_compute_slice.sv)) +is a frontend slice and a backend slice joined directly: one end-to-end +pipeline for the warps `wid mod IW == i`, the counterpart of an NVIDIA SM +processing block. Frontend slice *i* serves only issue slice *i* and the +blocks of slice *i*, so there is nothing to route between slices. Only the +core scheduler, the icache, the memory system and the shared SFU/TCU services +are reached through its ports. This is the structure to floorplan by: each +slice is self-contained apart from those ports. + +Compute slices with IW > 1 have one SFU block per slice, so each slice's +warp-control requests are arbitrated against the other slices', and a slice +retires its own warps when its exit request is granted (`SLICE_RETIRE`). +With a single slice the core retires them from the registered request. + +### 2.2 Backend slices + +When FW < IW, frontend slice *f* feeds issue slices `f, f+FW, f+2·FW, …`. +The back end is still IW identical slices +([`VX_backend_slice.sv`](../../hw/rtl/core/VX_backend_slice.sv)): issue, +execute (one block of every unit) and commit for one issue slice. Between +them sits the frontend crossbar (§3). + +### 2.3 Stage-sliced + +When some unit has fewer blocks than issue slices — by default the SFU — a +block serves several issue slices, so issue and execute cannot be cut into +identical end-to-end slices. The core is cut by stage instead: +[`VX_issue`](../../hw/rtl/core/VX_issue.sv) holds IW issue slices, +[`VX_execute`](../../hw/rtl/core/VX_execute.sv) holds B blocks of each unit, +and [`VX_commit`](../../hw/rtl/core/VX_commit.sv) holds IW commit slices +([`VX_commit_slice`](../../hw/rtl/core/VX_commit_slice.sv)). Each unit's +[`VX_lane_dispatch`](../../hw/rtl/core/VX_lane_dispatch.sv) groups the IW issue +ports into IW / B batches: block *b* takes the issue slices `b, b+B, …` in +turn, which is the `wid mod B` ownership of §1. + +## 3. The frontend crossbar + +[`VX_frontend_xbar.sv`](../../hw/rtl/core/VX_frontend_xbar.sv) connects FW +frontend slices to IW issue slices and to the ALU and SFU blocks whenever the +core is not built from compute slices. It is purely combinational: it adds +no cycles, so a configuration's timing does not depend on which structure the +widths select. + +| Direction | Traffic | Routed by | +|---|---|---| +| frontend → issue | decoded instructions | issue slice *i* takes frontend slice `i mod FW` | +| issue → frontend | issue and commit events, per-warp drain status | the frontend slice that owns the warp | +| ALU blocks → frontend | branch resolution | the owning frontend slice | +| SFU blocks → frontend | warp control, CSR reads of scheduler state, RT unlocks | the owning frontend slice; a read is answered by the slice owning the read `wid` | + +## 4. Core-wide services + +Some state belongs to the whole core, not to a slice. Each is reached through +one port per slice or per block, in every structure. + +| Service | Module | Why core-wide | +|---|---|---| +| CTA launch | [`VX_cta_dispatch`](../../hw/rtl/core/VX_cta_dispatch.sv) in the core scheduler, one lane per frontend slice (`NUM_LANES = FW`) | a CTA's warps span slices | +| barriers, wspawn, warp retirement | [`VX_warp_shared_unit`](../../hw/rtl/core/VX_warp_shared_unit.sv) | a barrier counts warps of every slice | +| DXA, TEX and RTU buses | [`VX_sfu_shared`](../../hw/rtl/core/VX_sfu_shared.sv) | one accelerator of each kind per core; the SFU blocks are arbitrated onto it | +| WGMMA tile buffers | [`VX_tcu_shared`](../../hw/rtl/tcu/VX_tcu_shared.sv) | a warpgroup's B operand is common to its TCU blocks | +| icache, dcache, local memory | [`VX_mem_unit`](../../hw/rtl/core/VX_mem_unit.sv) | one memory system per core; the icache has one request port per frontend slice (`ICACHE_NUM_REQS = FW`) | + +`VX_warp_shared_unit` takes one request per cycle, round-robin. Its +requesters register their requests toward the arbiter; a sole requester is +connected directly and its request is registered inside the unit instead, so +a request takes effect at the same distance from its instruction either way. + +## 5. Configurations + +| Configuration | NW | IW | FW | B (ALU/FPU/LSU/TCU, SFU) | Structure | +|---|---|---|---|---|---| +| default | 4–16 | 1 | 1 | 1, 1 | one compute slice | +| `logic_gate` core (`NT=NW=16`) | 16 | 1 | 1 | 1, 1 | one compute slice | +| U55C image C (`ISSUE_WIDTH=4`) | 8 | 4 | 4 | 4, 1 | stage-sliced | +| `rtx5080` profile | 48 | 4 | 4 | 4, 1 | stage-sliced | +| `rtx5080` with `NUM_SFU_BLOCKS=4` | 48 | 4 | 4 | 4, 4 | 4 compute slices | +| IW = 4, `FETCH_WIDTH=1` | — | 4 | 1 | 4, 4 | backend slices | + +To get compute slices at IW > 1, give the SFU one block per issue slice +(`-DVX_CFG_NUM_SFU_BLOCKS=`). Lowering a block count below IW trades that +unit's throughput and area against arbitration, and moves the core to the +stage-sliced structure. + +## 6. Performance and area + +**Fetch width.** Each frontend slice schedules and fetches its own warps on its +own icache port, so FW sets how many instructions the core can fetch per +cycle. At FW > 1, slices often fetch from the same icache line in the same +cycle; with `VX_CFG_ICACHE_READ_MERGE` (on by default, no effect at FW = 1) +those reads share one bank access and each slice gets its own word. The +merge cut sgemm cycles by 12.1% at IW = FW = 4, and by 25.8% on one +`rtx5080` core in rtlsim. + +**Issue width and blocks.** IW sets how many instructions issue per cycle; +each issue slice has its own ibuffer, scoreboard, operand collector and +commit port. A unit with B < IW issues at most B instructions of that unit +per cycle. + +**Area.** A frontend slice replicates the warp scheduler, fetch and decode; a +backend slice, the issue logic and one block of every unit. The crossbar is +combinational routing only. Compute slices replicate the most — every unit +in every slice — and have the shortest connections, since a slice talks to +the rest of the core only through the core-wide services. The stage-sliced +structure shares the units with B < IW at the cost of the dispatch and commit +arbitration between their issue slices. + +## 7. SimX + +SimX models the timing, not the structure: its core has one pipeline +parameterized by the same widths, and `model_parity` holds it to the RTL in +every structure. A change that moves cycles in one structure moves the SimX +model with it (see [`AGENTS.md`](../../AGENTS.md)). diff --git a/docs/designs/microarchitecture.md b/docs/designs/microarchitecture.md index 10516946c5..44eb6c4313 100644 --- a/docs/designs/microarchitecture.md +++ b/docs/designs/microarchitecture.md @@ -77,38 +77,15 @@ Vortex has a 6-stage pipeline: ### Core partitioning -The core splits its warps into slices, each a constant bit-slice of the warp id: -`VX_CFG_FETCH_WIDTH` (FW) frontend slices (`wid mod FW`), `VX_CFG_ISSUE_WIDTH` (IW) -issue slices (`wid mod IW`), and B blocks per execution unit (`wid mod B`), with -FW and B dividing IW. A slice owns its warps' pipeline state; the core scheduler -(CTA launch, barriers, wspawn, retirement), the shared SFU/TCU services and the -memory unit stay core-wide. - -The structure follows from the widths: - -| Widths | Structure | -|---|---| -| every B = IW = FW | IW compute slices (`VX_compute_slice`), no frontend crossbar | -| every B = IW, FW < IW | FW frontend slices, the frontend crossbar, IW backend slices (`VX_backend_slice`) | -| some unit has B ≠ IW | FW frontend slices, the frontend crossbar, stage-sliced `VX_issue` / `VX_execute` / `VX_commit` | - -- **Frontend slice** (`VX_frontend_slice`): warp scheduler, fetch and decode for - `wid mod FW`, on its own icache port. -- **Backend slice** (`VX_backend_slice`): issue, execute (one block of every - unit) and commit for one issue slice. -- **Compute slice** (`VX_compute_slice`): a frontend slice and its backend slice, - joined directly: one end-to-end pipeline for `wid mod IW`. This is the - counterpart of an NVIDIA SM processing block. -- **Frontend crossbar** (`VX_frontend_xbar`): combinational routing between FW - frontend slices and the back end whenever the core is not built from - compute slices. It sends each instruction to its issue slice, and each - branch, warp-control and CSR event to the frontend slice that owns the warp. -- **Core-wide services**, reached through one port per block in every structure: - - `VX_sfu_shared` arbitrates the SFU blocks onto the core's single DXA, TEX - and RTU buses. - - `VX_tcu_shared` holds the state that spans TCU blocks: the WGMMA tile - buffers, whose B operand is common to a warpgroup's blocks, and the sparse - metadata AGU. +The core splits its warps among parallel pipelines by warp id: +`VX_CFG_FETCH_WIDTH` frontend slices (`wid mod FW`), `VX_CFG_ISSUE_WIDTH` issue +slices (`wid mod IW`), and B blocks per execution unit (`wid mod B`). The +widths select the structure: IW compute slices when every width is equal, FW +frontend slices behind a combinational frontend crossbar otherwise, and a +stage-sliced issue/execute/commit when some unit has fewer blocks than issue +slices. CTA launch, barriers, warp retirement, the shared SFU/TCU services and +the memory system stay core-wide. See +[core_partitioning.md](core_partitioning.md). ### Vortex clustering architecture - Sockets From 591d871080252e2e4d16e0bd49b98614bb4f76fd Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 20:40:52 -0700 Subject: [PATCH 291/336] tcu: drive each DSM lane mask from its own process VX_tcu_dsm wrote vld_mask_per_k[k] from one always_comb per generate iteration, so N processes drove slices of a single variable. CIRCT cannot lower a multiply-driven variable to wires and leaves an LLHD signal in the netlist, so Beacon (and the Yosys flow) failed on any build with VX_CFG_TCU_DSM_ENABLE. Each iteration now computes its own vld_mask_k and assigns it to the array element; the logic is unchanged. Verified: the TCU DUT with DSM, MX and FP4 enabled elaborates through CIRCT and runs through Beacon with no LLHD ops left. Not simulated: no CI case enables DSM. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/tcu/VX_tcu_dsm.sv | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/hw/rtl/tcu/VX_tcu_dsm.sv b/hw/rtl/tcu/VX_tcu_dsm.sv index a8f88a88b7..cebc13b747 100644 --- a/hw/rtl/tcu/VX_tcu_dsm.sv +++ b/hw/rtl/tcu/VX_tcu_dsm.sv @@ -26,7 +26,7 @@ module VX_tcu_dsm import VX_tcu_pkg::*; #( output wire [TCU_MAX_INPUTS-1:0] vld_mask ); - logic [N-1:0][7:0] vld_mask_per_k; + wire [N-1:0][7:0] vld_mask_per_k; wire [TCU_MAX_INPUTS-1:0] vld_mask_w = vld_mask_per_k; for (genvar k = 0; k < N; ++k) begin : g_k @@ -56,47 +56,48 @@ module VX_tcu_dsm import VX_tcu_pkg::*; #( end `UNUSED_VAR({a_tf32_nz, b_tf32_nz, a_f16_nz, b_f16_nz, a_i8_nz, b_i8_nz, a_i4_nz, b_i4_nz}) + logic [7:0] vld_mask_k; always_comb begin - vld_mask_per_k[k] = '1; + vld_mask_k = '1; case (fmt_s) `ifdef VX_CFG_TCU_TF32_ENABLE TCU_TF32_ID: begin - vld_mask_per_k[k] = {8{a_tf32_nz && b_tf32_nz}}; + vld_mask_k = {8{a_tf32_nz && b_tf32_nz}}; end `endif `ifdef VX_CFG_TCU_FP16_ENABLE TCU_FP16_ID, TCU_BF16_ID: begin - vld_mask_per_k[k][0 +: 4] = {4{a_f16_nz[0] && b_f16_nz[0]}}; - vld_mask_per_k[k][4 +: 4] = {4{a_f16_nz[1] && b_f16_nz[1]}}; + vld_mask_k[0 +: 4] = {4{a_f16_nz[0] && b_f16_nz[0]}}; + vld_mask_k[4 +: 4] = {4{a_f16_nz[1] && b_f16_nz[1]}}; end `endif `ifdef VX_CFG_TCU_FP8_ENABLE TCU_FP8_ID, TCU_BF8_ID: begin for (int e = 0; e < 4; ++e) begin - vld_mask_per_k[k][e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; + vld_mask_k[e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; end end `ifdef VX_CFG_TCU_MX_ENABLE TCU_MXFP8_ID, TCU_MXBF8_ID: begin for (int e = 0; e < 4; ++e) begin - vld_mask_per_k[k][e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; + vld_mask_k[e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; end end `ifdef VX_CFG_TCU_FP4_ENABLE `ifdef VX_CFG_TCU_MXFP4_ENABLE TCU_MXFP4_ID: begin for (int e = 0; e < 8; ++e) begin - vld_mask_per_k[k][e] = a_i4_nz[e] && b_i4_nz[e]; + vld_mask_k[e] = a_i4_nz[e] && b_i4_nz[e]; end end `endif `ifdef VX_CFG_TCU_NVFP4_ENABLE TCU_NVFP4_ID: begin for (int e = 0; e < 8; ++e) begin - vld_mask_per_k[k][e] = a_i4_nz[e] && b_i4_nz[e]; + vld_mask_k[e] = a_i4_nz[e] && b_i4_nz[e]; end end `endif @@ -107,7 +108,7 @@ module VX_tcu_dsm import VX_tcu_pkg::*; #( TCU_I8_ID, TCU_U8_ID: begin for (int e = 0; e < 4; ++e) begin - vld_mask_per_k[k][e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; + vld_mask_k[e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; end end `endif @@ -115,15 +116,16 @@ module VX_tcu_dsm import VX_tcu_pkg::*; #( TCU_I4_ID, TCU_U4_ID: begin for (int e = 0; e < 8; ++e) begin - vld_mask_per_k[k][e] = a_i4_nz[e] && b_i4_nz[e]; + vld_mask_k[e] = a_i4_nz[e] && b_i4_nz[e]; end end `endif default: begin - vld_mask_per_k[k] = '1; + vld_mask_k = '1; end endcase end + assign vld_mask_per_k[k] = vld_mask_k; end // Optional output register (aligns vld_mask with the FEDP operand pipe). From 9ca22116555ebaa4405617731b4b73dcea6a2000 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 21:30:51 -0700 Subject: [PATCH 292/336] beacon: learned calibration ranking of near-critical registers Logic levels rank Vivado's near-critical registers (slack under 0.3 ns at 300 MHz on the U55C) at about half the ceiling of one Vivado run predicting another: AP 0.35-0.44 and a Spearman of 0.31 against slack, over 13 routed builds. `ci/beacon.py calibrate` trains gradient-boosted trees on 38 per-register features, all from Beacon's own outputs (depth, width, end and start kinds, fan-out, hierarchy span, the deepest path's op mix), and writes them as plain JSON trees; `score` evaluates them in pure Python and ranks a run's registers by predicted slack and the near-critical and failing probabilities. Held out by RTL lineage, the near-critical AP goes from 0.44 to 0.77 and the Spearman from 0.30 to 0.65; held out by whole DUT, to 0.64. The model does not follow a design-wide slowdown as well as the top-50 mean of levels, so it is advisory: diff and logic_gate keep gating on levels. The model trained on the full corpus ships as hw/scripts/beacon_calibration_u55c.json. Verified: the JSON trees reproduce scikit-learn's predictions to 4.5e-8 on the 13 builds; score runs on a core DUT run in 7 s. Co-Authored-By: Claude Opus 5.5 --- ci/beacon.py | 344 ++++++++++++++++++++++++ docs/designs/beacon.md | 98 ++++++- hw/scripts/beacon_calibration_u55c.json | 1 + 3 files changed, 440 insertions(+), 3 deletions(-) create mode 100644 hw/scripts/beacon_calibration_u55c.json diff --git a/ci/beacon.py b/ci/beacon.py index e9a976dadb..652834e4bd 100755 --- a/ci/beacon.py +++ b/ci/beacon.py @@ -28,6 +28,10 @@ ports the depth at the ports of chosen modules' instances, by unit buffers which elastic buffers pass a deep ready chain or a computed output through: evidence for an OUT_BUF (OUT_REG) change + calibrate train the learned calibration (scikit-learn) on DUT builds: + Beacon runs and the Vivado end point tables of the same sources + score rank a run's registers by the calibrated near-critical + probability, with predicted slack (pure Python; advisory) ci/beacon.py run gen/design.mlir runs/full ci/beacon.py run runs/full/mapped.mlirbc runs/socket \\ @@ -39,6 +43,8 @@ ci/beacon.py run --keep VX_elastic_buffer design.mlir runs/eb \\ port-modules=VX_elastic_buffer ci/beacon.py buffers runs/eb + ci/beacon.py calibrate cal.json --build build_core/beacon ep.csv core + ci/beacon.py score build_core/beacon cal.json --top 30 A Vivado end point table is a CSV of `slack,levels,logic,net,start,end`, one row per failing end point (a gate's ep.csv). The DUT flow and logic_gate run @@ -49,6 +55,7 @@ import collections import csv import json +import math import os import re import resource @@ -708,6 +715,322 @@ def cmd_retime_split(args): f'Beacon-Vivado median {np.median(a - b):+.1f}') +# Learned calibration (calibrate, score): per-register features of a run, +# all from Beacon's own outputs. The p_*/n_* columns come from the deepest +# path in paths.jsonl, for registers at or above the model's path levels; +# below that they are missing, as in training. +CAL_FEATURES = ( + 'lev', 'lev_mean', 'log2_bits', 'frac_deep', 'log2_nbits_deep', + 'k_d', 'k_wd', 'k_wctl', 'k_rd', 'k_port', + 's_reg', 's_port', 's_bram', 's_ip', 's_other', + 'log2_sfo', 'log2_sfomax', 'log2_in_fo', 'log2_in_fo_deep', 'cross', 'xmax', + 'hier_depth', 'log2_lca_bits', 'lca_up', + 'p_lut', 'p_carry', 'p_cmp', 'p_lutram', 'p_bram', 'p_dsp', + 'n_carry', 'n_cmp', 'p_maxfo', 'p_sumlog_fo', 'p_nfo64', 'p_ninst', 'p_nsrc', + 'hand', +) +CAL_OPS = ('lut', 'carry', 'cmp', 'lutram', 'bram', 'dsp') +CAL_KINDS = {'d': 'd', 'wd': 'wd', 'wa': 'wctl', 'we': 'wctl', 'wm': 'wctl', + 'ra': 'rd', 're': 'rd'} +CAL_NEAR = 0.3 # ns: the near-critical set the model ranks +CAL_CENSOR = 1.0 # ns: a gate's ep.csv lists end points under 1 ns only + + +def log2p(x): + return math.log2(1 + max(0, x)) + + +def register_features(run_dir, path_levels): + """(features, by_dir, ips) of a run: register -> {feature: value}, all + end point bits under one name (a memory's ports included), keyed by its + normalized path below the run's top; the instance -> {base name: key} + map and the vendor IP instances, for matching Vivado's names.""" + inst = load_instances(f'{run_dir}/instances.csv') + memo = {} + + def ipath(ctx): + if ctx not in memo: + parent, name, _ = inst[ctx] + p = '' if ctx == 0 else ipath(parent) + memo[ctx] = '' if ctx == 0 else (f'{p}/{name}' if p else name) + return memo[ctx] + + def full(ctx, name): + p = ipath(int(ctx)) + return f'{p}/{name}' if p else name + + sys.setrecursionlimit(10000) + regs, ips = {}, set() + by_dir = collections.defaultdict(dict) + for row in endpoints(run_dir): + if row['start_kind'] == 'ip': + ips.add(norm(full(row['start_inst'], row['start'].rsplit('.', 1)[0]))) + f = full(row['end_inst'], row['end'].rsplit('.', 1)[0]) + key = norm(f) + lv, fo, sfo = int(row['levels']), int(row.get('in_fanout') or 0), int(row['fanout']) + r = regs.get(key) + if r is None: + r = regs[key] = dict(lev=-1, bits=0, sum=0, sfomax=0, fo=0, xmax=0, lvs=[]) + d, _, b = f.rpartition('/') + by_dir[norm(d)][norm(b.rsplit('.', 1)[-1])] = key + r['bits'] += 1 + r['sum'] += lv + r['lvs'].append(lv) + r['fo'] = max(r['fo'], fo) + r['xmax'] = max(r['xmax'], int(row['crossings'])) + if row['start_kind'] == 'reg': + r['sfomax'] = max(r['sfomax'], sfo) + if lv > r['lev']: + r.update(lev=lv, kind=row['kind'], skind=row['start_kind'], sfo=sfo, + fo_deep=fo, cross=int(row['crossings']), + start=norm(full(row['start_inst'], row['start']))) + + paths = {} + if os.path.exists(f'{run_dir}/paths.jsonl'): + with open(f'{run_dir}/paths.jsonl') as f: + for line in f: + p = json.loads(line) + key = norm(full(p['end_inst'], p['end'].rsplit('.', 1)[0])) + if key not in paths or p['levels'] > paths[key]['levels']: + paths[key] = p + + # Bits below each instance: the span of a register and its deepest start. + under = collections.Counter() + for k, r in regs.items(): + parts = k.split('/')[:-1] + for i in range(len(parts) + 1): + under['/'.join(parts[:i])] += r['bits'] + + feats = {} + for k, r in regs.items(): + lv = r['lev'] + a, c = k.split('/')[:-1], r['start'].split('/')[:-1] + n = 0 + while n < min(len(a), len(c)) and a[n] == c[n]: + n += 1 + deep = sum(1 for v in r['lvs'] if v >= lv - 1) + kind = CAL_KINDS.get(r['kind'], 'port') + skind = r['skind'] if r['skind'] in ('reg', 'port', 'bram', 'ip') else 'other' + x = dict(lev=lv, lev_mean=r['sum'] / r['bits'], log2_bits=log2p(r['bits']), + frac_deep=deep / r['bits'], log2_nbits_deep=log2p(deep), + log2_sfo=log2p(r['sfo']), log2_sfomax=log2p(r['sfomax']), + log2_in_fo=log2p(r['fo']), log2_in_fo_deep=log2p(r['fo_deep']), + cross=r['cross'], xmax=r['xmax'], hier_depth=len(a), + log2_lca_bits=log2p(under['/'.join(a[:n])]), lca_up=len(a) - n) + for v in ('d', 'wd', 'wctl', 'rd', 'port'): + x['k_' + v] = float(kind == v) + for v in ('reg', 'port', 'bram', 'ip', 'other'): + x['s_' + v] = float(skind == v) + p = paths.get(k) if lv >= path_levels else None + if p is None: + x.update((f, float('nan')) for f in CAL_FEATURES if f.startswith(('p_', 'n_'))) + x['hand'] = lv + else: + ops = p['ops'] # [levels, kind, fanout, instance, name, source], end first + cost, count = collections.Counter(), collections.Counter() + for i in range(len(ops) - 1): + op = ops[i][1] if ops[i][1] in CAL_OPS else 'lut' + cost[op] += ops[i][0] - ops[i + 1][0] + count[op] += 1 + fos = [o[2] for o in ops[1:-1]] + x.update((f'p_{op}', cost[op]) for op in CAL_OPS) + x.update(n_carry=count['carry'], n_cmp=count['cmp'], + p_maxfo=log2p(max(fos, default=0)), + p_sumlog_fo=sum(log2p(v) for v in fos), + p_nfo64=sum(1 for v in fos if v >= 64), + p_ninst=len({o[3] for o in ops}), + p_nsrc=len({o[5].split(':')[0] for o in ops})) + # Against Vivado's path delay, a carry level costs about 0.42 of + # a LUT level and a compare 0.15. + x['hand'] = lv - 0.58 * cost['carry'] - 0.85 * cost['cmp'] + feats[k] = x + return feats, by_dir, ips + + +def vivado_slacks(ep, regs, by_dir, ips): + """register -> worst slack of a DUT build's end point table (cells named + from the DUT's top down), matched to a run's registers as validate does.""" + def in_ip(k): + parts = k.split('/') + return any('/'.join(parts[:i]) in ips for i in range(1, len(parts))) + + def fallback(k): + d, _, leaf = k.rpartition('/') + best = None + for name, cand in by_dir.get(d, {}).items(): + if (leaf == name or leaf.endswith('_' + name)) and ( + best is None or len(name) > len(best[0])): + best = (name, cand) + return best[1] if best else k + + slack = {} + with open(ep) as f: + r = csv.reader(f) + next(r) + for s, levels, logic, net, start, end in r: + key = ep_key('top/' + end, True) + if key is None or in_ip(key[1]): + continue + k = key[1] + if k not in regs and k.endswith('_rdata'): + k = k[:-len('_rdata')] + if k not in regs and k.endswith('_rdata_r') and k[:-len('_rdata_r')] + '_ram' in regs: + k = k[:-len('_rdata_r')] + '_ram' + if k not in regs: + k = fallback(k) + if k in regs: + slack[k] = min(slack.get(k, float('inf')), float(s)) + return slack + + +def average_precision(score, positive): + """Average precision of a ranking, tied scores taken together.""" + np = numpy() + order = np.argsort(-np.asarray(score, float), kind='mergesort') + s, y = np.asarray(score, float)[order], np.asarray(positive, bool)[order] + tp = np.cumsum(y) + last = np.r_[s[1:] != s[:-1], True] + prec, rec = tp[last] / (np.nonzero(last)[0] + 1), tp[last] / max(y.sum(), 1) + return float(np.sum(prec * np.diff(np.r_[0.0, rec]))) + + +def hgb_json(m, link): + """A fitted HistGradientBoosting model as plain trees: a node goes left + when its feature is <= t, or is missing and m is set; a leaf has f -1.""" + trees = [] + for (pred,) in m._predictors: + n = pred.nodes + leaf = n['is_leaf'].astype(bool) + trees.append(dict( + f=[-1 if lf else int(i) for i, lf in zip(n['feature_idx'], leaf)], + t=[0 if lf else float(t) for t, lf in zip(n['num_threshold'], leaf)], + m=[int(v) for v in n['missing_go_to_left']], + l=[int(v) for v in n['left']], r=[int(v) for v in n['right']], + v=[float(f'{v:.7g}') if lf else 0 for v, lf in zip(n['value'], leaf)])) + return dict(link=link, base=float(m._baseline_prediction.ravel()[0]), trees=trees) + + +def hgb_predict(model, x): + """Evaluate hgb_json trees on one feature row (a list).""" + raw = model['base'] + for t in model['trees']: + i = 0 + f = t['f'] + while f[i] >= 0: + v = x[f[i]] + i = t['l'][i] if (t['m'][i] if v != v else v <= t['t'][i]) else t['r'][i] + raw += t['v'][i] + return 1 / (1 + math.exp(-raw)) if model['link'] == 'logistic' else raw + + +def cal_models(X, slack, weight, features): + """Fit the slack regressor and the near-critical and failing classifiers: + gradient-boosted trees, never ranking a deeper register (lev, hand) safer.""" + from sklearn.ensemble import (HistGradientBoostingClassifier, + HistGradientBoostingRegressor) + kw = dict(max_iter=300, learning_rate=0.05, max_leaf_nodes=15, min_samples_leaf=50, + l2_regularization=1.0, early_stopping=False, random_state=0) + up = [1 if f in ('lev', 'hand') else 0 for f in features] + reg = HistGradientBoostingRegressor(monotonic_cst=[-v for v in up], **kw) + near = HistGradientBoostingClassifier(monotonic_cst=up, **kw) + fail = HistGradientBoostingClassifier(monotonic_cst=up, **kw) + return (reg.fit(X, slack, sample_weight=weight), + near.fit(X, slack < CAL_NEAR, sample_weight=weight), + fail.fit(X, slack < 0, sample_weight=weight)) + + +def cmd_calibrate(args): + """Train the calibration on DUT builds: each a Beacon run and the Vivado + end point table of the same sources. Writes the model as JSON.""" + np = numpy() + builds = [] + for b in args.build: + if len(b) not in (2, 3): + sys.exit('--build RUN_DIR EP_CSV [GROUP]') + run_dir, ep = b[:2] + feats, by_dir, ips = register_features(run_dir, args.path_levels) + slacks = vivado_slacks(ep, feats, by_dir, ips) + keys = sorted(feats) + X = np.array([[feats[k][f] for f in CAL_FEATURES] for k in keys], float) + y = np.array([min(slacks.get(k, CAL_CENSOR), CAL_CENSOR) for k in keys]) + measured = np.array([k in slacks for k in keys]) + builds.append(dict(run=run_dir, group=b[2] if len(b) == 3 else run_dir, + X=X, y=y, measured=measured)) + print(f'{run_dir}: {len(keys)} registers, {measured.sum()} in the table, ' + f'{(y < 0).sum()} failing, {(y < CAL_NEAR).sum()} under {CAL_NEAR} ns') + + def fit(train): + X = np.vstack([b['X'] for b in train]) + y = np.concatenate([b['y'] for b in train]) + # Every build weighs the same, whatever its register count. + w = np.concatenate([np.full(len(b['y']), len(y) / len(train) / len(b['y'])) + for b in train]) + return cal_models(X, y, w, CAL_FEATURES) + + if args.holdout: + print('\nleave one build out (builds of its group left out of training):') + print(f'{"":40} near-critical AP Spearman vs -slack') + print(f'{"build":40} {"levels":>7} {"model":>7} {"levels":>7} {"model":>7}') + for b in builds: + train = [o for o in builds if o['group'] != b['group']] + if not train: + continue + reg, near, _ = fit(train) + lev, m, y = b['X'][:, 0], b['measured'], b['y'] + ap = [average_precision(s, y < CAL_NEAR) if (y < CAL_NEAR).any() else float('nan') + for s in (lev, near.predict_proba(b['X'])[:, 1])] + rho = [spearman(s[m], -y[m]) for s in (lev, -reg.predict(b['X']))] + name = '/'.join(os.path.normpath(b['run']).split('/')[-2:]) + print(f'{name[-40:]:40} {ap[0]:7.2f} {ap[1]:7.2f} {rho[0]:7.2f} {rho[1]:7.2f}') + + import sklearn + reg, near, fail = fit(builds) + model = dict( + features=list(CAL_FEATURES), path_levels=args.path_levels, near=CAL_NEAR, + censor=CAL_CENSOR, sklearn=sklearn.__version__, + builds=[dict(run=b['run'], group=b['group'], registers=len(b['y']), + failing=int((b['y'] < 0).sum())) for b in builds], + slack=hgb_json(reg, 'identity'), p_near=hgb_json(near, 'logistic'), + p_fail=hgb_json(fail, 'logistic')) + with open(args.model, 'w') as f: + json.dump(model, f, separators=(',', ':')) + print(f'\nwrote {args.model} ({os.path.getsize(args.model) // 1024} KB, ' + f'{len(builds)} builds)') + + +def cmd_score(args): + """Rank a run's registers by the calibrated near-critical probability, + with the predicted worst slack. Advisory: the gates stay on levels.""" + with open(args.model) as f: + model = json.load(f) + feats, _, _ = register_features(args.run_dir, model['path_levels']) + names = model['features'] + if any(math.isnan(x['p_lut']) for x in feats.values() + if x['lev'] >= model['path_levels']): + print(f'warning: registers at or above {model["path_levels"]} levels lack ' + f'a path in paths.jsonl; rerun with path-levels={model["path_levels"]}') + rows = [] + for k, x in feats.items(): + v = [x[n] for n in names] + rows.append((hgb_predict(model['p_near'], v), hgb_predict(model['slack'], v), + hgb_predict(model['p_fail'], v), x['lev'], k)) + rows.sort(key=lambda r: (-r[0], r[1])) + low = sorted(r[1] for r in rows)[:50] + print(f'{len(rows)} registers; P(slack < {model["near"]} ns) >= 0.5: ' + f'{sum(r[0] >= 0.5 for r in rows)}; mean of the 50 lowest predicted ' + f'slacks {sum(low) / max(1, len(low)):+.3f} ns (model trained on ' + f'{len(model["builds"])} builds)') + print(f'{"P(near)":>7} {"slack":>6} {"P(fail)":>7} {"lv":>3} register') + for p, s, pf, lv, k in [r for r in rows if r[3] >= args.min_levels][:args.top]: + print(f'{p:7.2f} {s:+6.2f} {pf:7.2f} {lv:3d} {k}') + if args.csv: + with open(args.csv, 'w', newline='') as f: + w = csv.writer(f) + w.writerow(['register', 'levels', 'p_near', 'slack', 'p_fail']) + for p, s, pf, lv, k in rows: + w.writerow([k, lv, f'{p:.4f}', f'{s:.4f}', f'{pf:.4f}']) + + def main(): ap = argparse.ArgumentParser(description=__doc__, @@ -793,6 +1116,27 @@ def main(): p.add_argument('metric', nargs='?', default='levels', choices=['levels', 'retimed']) p.set_defaults(fn=cmd_retime_split) + p = sub.add_parser("calibrate", help="train the learned slack calibration") + p.add_argument('model', help="the model JSON to write") + p.add_argument('--build', nargs='+', action='append', required=True, + metavar='RUN_DIR EP_CSV [GROUP]', + help="a DUT build: its Beacon run and Vivado end point table; " + "builds of one GROUP (one RTL lineage) are held out together " + "(repeatable)") + p.add_argument('--path-levels', type=int, default=8, + help="path features from this depth on (the runs' path-levels)") + p.add_argument('--holdout', action='store_true', + help="also print a leave-one-build-out check against levels") + p.set_defaults(fn=cmd_calibrate) + + p = sub.add_parser("score", help="registers by calibrated near-critical risk") + p.add_argument('run_dir') + p.add_argument('model', help="a model JSON from calibrate") + p.add_argument('--min-levels', type=int, default=0) + p.add_argument('--top', type=int, default=40) + p.add_argument('--csv', help="also write every register's prediction here") + p.set_defaults(fn=cmd_score) + args = ap.parse_args() return args.fn(args) or 0 diff --git a/docs/designs/beacon.md b/docs/designs/beacon.md index d7a90fe7d5..ba009fd413 100644 --- a/docs/designs/beacon.md +++ b/docs/designs/beacon.md @@ -15,7 +15,7 @@ Vivado run. | [4](#4-the-model) | The model: levels per element, devices | | [5](#5-outputs) | Outputs | | [6](#6-integration) | Integration: the DUT flow, `logic_gate`, timing work | -| [7](#7-validation) | Validation against Vivado | +| [7](#7-validation) | Validation against Vivado; the learned calibration (§7.1) | | [8](#8-diagnostics) | Diagnostics beyond levels | | [9](#9-invariants-and-limitations) | Invariants and limitations | @@ -49,7 +49,8 @@ ci/logic_gate.py -b om --update-baseline ``` Any CIRCT netlist, with `ci/beacon.py` (subcommands `run`, -`summarize`, `paths`, `validate`, `retime-split`, `ports`, `buffers`; +`summarize`, `paths`, `validate`, `retime-split`, `ports`, `buffers`, +`calibrate`, `score`; `foldsim-opt` from the `PATH` or `FOLDSIM_OPT`): ```bash @@ -442,6 +443,95 @@ was fitted; on the six larger DUTs, the predicted Fmax is within a mean current RTL. Beacon runs a DUT in 2–52 s, where Yosys and OpenSTA take 2.5–37 min. +### 7.1 Learned calibration (advisory) + +Levels rank Vivado's near-critical registers at about half the ceiling set +by one Vivado run predicting another. `ci/beacon.py score` ranks them with a +calibration learned from routed U55C builds instead: gradient-boosted trees +on 38 per-register features, all from Beacon's own outputs (no Vivado input): + +- **depth:** levels, mean levels over the bits, the share of bits within one + level of the deepest; +- **width:** bits, and bits within one level of the deepest; +- **end and start kind:** D, RAM write data or control, RAM read; register, + port, BRAM, IP; +- **fan-out:** the deepest start's, the cone's (`in_fanout`), the largest + along the deepest path, nets of 64 or more loads on it; +- **span:** hierarchy depth, crossings, the bits under the lowest common + instance of the register and its deepest start; +- **the deepest path's op mix** (`paths.jsonl`, at 8 levels or more): levels + spent in LUT, carry, compare, LUTRAM, BRAM and DSP ops, distinct + instances and source files, and the levels with a carry level costed at + 0.42 of a LUT level and a compare at 0.15. + +The model is three tree ensembles: the worst slack (capped at 1 ns, the +`ep.csv` horizon), the probability of slack under 0.3 ns (near-critical) and +under 0 ns. A register never ranks safer for being deeper (a monotone +constraint on levels). `calibrate` trains them with scikit-learn and writes +them as plain JSON trees; `score` evaluates them in pure Python, in about +4 s for a core. The model trained on the 13 builds below ships as +[`hw/scripts/beacon_calibration_u55c.json`](../../hw/scripts/beacon_calibration_u55c.json); +it applies to the U55C at 300 MHz, the target it was trained on. + +```bash +ci/beacon.py score build_core/beacon hw/scripts/beacon_calibration_u55c.json \ + --top 30 --csv score.csv +ci/beacon.py calibrate cal.json --holdout \ + --build build_core/beacon build_core/ep.csv core \ + --build build_cache/beacon build_cache/ep.csv cache +``` + +Each `--build` is a DUT build's Beacon run (`path-levels=8`, as the DUT flow +writes it) and the end point table of the same sources. `--holdout` prints +the check below for the given builds. + +**Validation.** 13 routed U55C builds at 3.333 ns: 8 core revisions, 4 lmem, +1 cache, 19 k registers. Each build is held out and scored by a model +trained on the builds outside its group. Revisions of one RTL lineage share +most of their registers, so they are held out together: + +- core with the coretiming lineage (origin, it1, it2, lsuagu); +- core with the lmemwarp lineage (v1, v2, final, p2lsulmem); +- cache; +- lmem. + +The stricter test holds out the whole DUT (core trained on cache and lmem +only). + +| held out | near-critical AP: levels | levels + 0.35·log2(fan-out) | carry/compare cost | model, lineage out | model, DUT out | Spearman vs −slack: levels / model | +|---|---|---|---|---|---|---| +| core origin | 0.36 | 0.38 | 0.41 | 0.72 | 0.65 | 0.15 / 0.64 | +| core it1 | 0.40 | 0.42 | 0.42 | 0.75 | 0.65 | 0.18 / 0.63 | +| core it2 | 0.44 | 0.46 | 0.48 | 0.77 | 0.68 | 0.24 / 0.68 | +| core lsuagu | 0.68 | 0.70 | 0.70 | 0.80 | 0.71 | 0.33 / 0.54 | +| core lmemwarp v1 | 0.37 | 0.40 | 0.48 | 0.75 | 0.56 | 0.27 / 0.66 | +| core lmemwarp v2 | 0.44 | 0.50 | 0.54 | 0.82 | 0.61 | 0.35 / 0.73 | +| core lmemwarp | 0.37 | 0.41 | 0.52 | 0.81 | 0.59 | 0.32 / 0.64 | +| core p2lsulmem | 0.55 | 0.63 | 0.60 | 0.87 | 0.65 | 0.40 / 0.78 | +| cache | 0.47 | 0.57 | 0.52 | 0.65 | 0.65 | 0.50 / 0.56 | +| **mean** | **0.44** | **0.50** | **0.52** | **0.77** | **0.64** | **0.30 / 0.65** | + +On the three lmem builds, with 4–5 near-critical registers each, the AP goes +from 0.02–0.03 to 0.63–0.68. One Vivado run predicting another of nearly +the same RTL (it1, it2) reaches 0.80. AP for failing registers goes from +0.21 to 0.44; that set is mostly placement noise at a WNS near −0.1 ns. + +- **Width carries most of it.** Shuffling the bit counts costs 0.29 of AP. + This is partly selection: a wide register shows the worst of many bits. + Within registers of one deep bit, though, the AP still goes from 0.47 to + 0.79. +- **Span, the op mix and fan-out carry the rest** (0.03–0.08 each). The op + mix matters most on the cache (0.27). +- **The DUT-out column is the honest number for a new DUT.** The lineage-out + column is the one for a new revision of a DUT already trained on. +- **It does not track a whole design's slowdown.** The mean of the 50 lowest + predicted slacks follows the change in WNS over 10 revision pairs worse + than the top-50 mean of levels: Spearman 0.15 (lineage out) or 0.42 (DUT + out), against 0.53. The lsuagu regression, −1.3 ns, moves it by 0.02 ns. + +So the model is a ranking for where to look first, not a gate. `diff` and +`logic_gate` keep gating on levels. + ## 8. Diagnostics beyond levels **Port depths.** `port-modules=` names modules whose instances' ports @@ -487,7 +577,9 @@ Beacon cannot answer, run on a checkpoint placement two thirds. No pre-placement wire term ranked slack better than levels on every held-out gate, so Beacon has none. Route-dominated failures with few levels are invisible to it: recall at ≥ 12 levels is - 0.13–0.57. + 0.13–0.57. The learned calibration (§7.1) ranks near-critical registers + better from width, span and fan-out, but it is trained on three U55C DUTs + and is advisory. - **Retiming and packing.** Vivado retimes and packs across what the RTL separates: a popcount, its adders and a compare become a LUT6 tree with a retimed register inside. Beacon's depths are RTL stage depths, about diff --git a/hw/scripts/beacon_calibration_u55c.json b/hw/scripts/beacon_calibration_u55c.json new file mode 100644 index 0000000000..51be12a671 --- /dev/null +++ b/hw/scripts/beacon_calibration_u55c.json @@ -0,0 +1 @@ +{"features":["lev","lev_mean","log2_bits","frac_deep","log2_nbits_deep","k_d","k_wd","k_wctl","k_rd","k_port","s_reg","s_port","s_bram","s_ip","s_other","log2_sfo","log2_sfomax","log2_in_fo","log2_in_fo_deep","cross","xmax","hier_depth","log2_lca_bits","lca_up","p_lut","p_carry","p_cmp","p_lutram","p_bram","p_dsp","n_carry","n_cmp","p_maxfo","p_sumlog_fo","p_nfo64","p_ninst","p_nsrc","hand"],"path_levels":8,"near":0.3,"censor":1.0,"sklearn":"1.9.1","builds":[{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c0_origin/beacon","group":"coreA","registers":2289,"failing":104},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c1_ct_it1/beacon","group":"coreA","registers":2290,"failing":25},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c3_ct_it2/beacon","group":"coreA","registers":2290,"failing":92},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c6_lsuagu/beacon","group":"coreA","registers":2304,"failing":709},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c2_lw_v1/beacon","group":"coreB","registers":1687,"failing":93},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c4_lw_v2/beacon","group":"coreB","registers":1690,"failing":112},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c5_lw/beacon","group":"coreB","registers":1685,"failing":98},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c7_lsulmem/beacon","group":"coreB","registers":1686,"failing":114},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/k1_p2erase/beacon","group":"cache","registers":775,"failing":10},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/l0_merge3/beacon","group":"lmem","registers":163,"failing":0},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/l1_r1/beacon","group":"lmem","registers":460,"failing":0},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/l2_r2/beacon","group":"lmem","registers":471,"failing":0},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/l3_final/beacon","group":"lmem","registers":477,"failing":0}],"slack":{"link":"identity","base":0.7503466214366413,"trees":[{"f":[2,0,21,-1,1,-1,-1,32,23,32,-1,24,-1,-1,-1,37,-1,-1,37,9,21,22,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,5.5,0,4.145833333333334,0,0,9.271599002932193,3.5,3.9474088816539714,0,7.5,0,0,0,8.34,0,0,6.970000000000001,0.5,5.5,11.775312925643524,0,0,0,0,17.094543568094064,0,0],"m":[1,1,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,12,0,0,0,16,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,7,4,0,6,0,0,15,14,11,0,13,0,0,0,17,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.01009299,0,0.007409015,-0.001117181,0,0,0,-0.008838531,0,-0.02555701,-0.04193794,-0.00553998,0,-0.003177707,-0.01679376,0,0,0,0,0.0003354531,-0.01197272,-0.02062047,0.01242137,0,-0.03062118,-0.06987605]},{"f":[2,0,21,20,-1,-1,1,-1,-1,32,23,32,-1,24,-1,-1,-1,37,-1,-1,37,9,21,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,5.5,0.5,0,0,4.145833333333334,0,0,9.271599002932193,3.5,3.9474088816539714,0,7.5,0,0,0,8.34,0,0,6.970000000000001,0.5,5.5,0,0,0,17.094543568094064,0,0],"m":[1,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,12,0,14,0,0,0,18,0,0,21,22,23,0,0,0,27,0,0],"r":[20,9,6,5,0,0,8,0,0,17,16,13,0,15,0,0,0,19,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0.01015324,0.00311568,0,0.007038721,-0.001062828,0,0,0,-0.008398592,0,-0.02428303,-0.03984905,-0.005264266,0,-0.003018921,-0.01595749,0,0,0,-0.006789299,-0.01959018,0.01180336,0,-0.02909147,-0.06644545]},{"f":[2,0,21,-1,1,-1,-1,32,23,32,-1,-1,-1,16,-1,22,-1,-1,37,9,21,-1,22,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,5.5,0,4.145833333333334,0,0,9.271599002932193,3.5,3.9474088816539714,0,0,0,6.044394119358453,0,17.07907323966382,0,0,6.970000000000001,0.5,5.5,0,8.32532955673354,0,0,0,17.094543568094064,0,0],"m":[1,1,1,0,1,0,0,0,1,0,0,0,0,1,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,0,0,14,0,16,0,0,19,20,21,0,23,0,0,0,27,0,0],"r":[18,7,4,0,6,0,0,13,12,11,0,0,0,15,0,17,0,0,26,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0.009109054,0,0.006686935,-0.001011141,0,0,0,-0.00798055,-0.02966998,-0.005002274,0,-0.001011141,0,-0.007254686,-0.0369035,0,0,0,-0.006450212,0,-0.002746125,-0.01999366,0.01121609,0,-0.02763817,-0.06318328]},{"f":[2,0,21,-1,1,-1,-1,32,21,21,32,-1,-1,-1,-1,37,-1,-1,37,9,21,22,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,5.5,0,4.145833333333334,0,0,9.271599002932193,9.5,7.5,3.9474088816539714,0,0,0,0,8.34,0,0,6.970000000000001,0.5,5.5,11.775312925643524,0,0,0,0,17.094543568094064,0,0],"m":[1,1,1,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,0,0,0,0,16,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,7,4,0,6,0,0,15,14,13,12,0,0,0,0,17,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.008653654,0,0.00635273,-0.0009534893,0,0,0,0,-0.00826245,-0.02606419,-0.0009534893,-0.05729316,0,-0.002631972,-0.01481627,0,0,0,0,0.0009788293,-0.01071442,-0.01768159,0.01065804,0,-0.02625748,-0.06008126]},{"f":[2,0,16,-1,-1,32,21,21,32,-1,-1,-1,-1,16,-1,22,-1,-1,37,9,21,-1,-1,-1,22,23,-1,-1,-1],"t":[5.228690439536267,6.5,11.620219825507487,0,0,9.271599002932193,9.5,7.5,3.9474088816539714,0,0,0,0,6.044394119358453,0,17.07907323966382,0,0,6.970000000000001,0.5,5.5,0,0,0,17.094543568094064,1.5,0,0,0],"m":[1,1,1,0,0,0,1,1,0,0,0,0,0,1,0,1,0,0,1,1,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,0,0,0,14,0,16,0,0,19,20,21,0,0,0,25,26,0,0,0],"r":[18,5,4,0,0,13,12,11,10,0,0,0,0,15,0,17,0,0,24,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0.007646069,0.0002043261,0,0,0,0,-0.007851071,-0.02476316,-0.0009035235,-0.05450668,0,-0.0009035235,0,-0.006611584,-0.03497537,0,0,0,-0.005822199,-0.01679813,0.01012775,0,0,-0.0185915,-0.02751632,-0.05713154]},{"f":[2,0,1,-1,16,-1,-1,32,23,32,-1,24,-1,-1,-1,-1,19,9,37,23,21,-1,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,4.208333333333334,0,7.266786540694901,0,0,9.271599002932193,3.5,3.9474088816539714,0,7.5,0,0,0,0,0.5,0.5,6.970000000000001,2.5,3.5,0,0,0,0,0,17.07907323966382,0,0],"m":[1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,12,0,0,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,7,4,0,6,0,0,15,14,11,0,13,0,0,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0.007703011,0,0.00711944,-0.0008404133,0,0,0,-0.006781262,0,-0.01905988,-0.03367628,-0.003831588,-0.003822916,0,0,0,0,0,-0.001641005,-0.01474928,-0.0005012965,-0.02175248,0.009990955,0,-0.02559911,-0.05844007]},{"f":[2,0,21,19,-1,-1,-1,32,21,21,32,-1,-1,-1,-1,-1,19,9,37,4,16,-1,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,3.5,0.5,0,0,0,9.271599002932193,9.5,7.5,3.9474088816539714,0,0,0,0,0,0.5,0.5,6.970000000000001,6.569535826647579,4.523561956057013,0,0,0,0,0,17.07907323966382,0,0],"m":[1,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,0,0,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,7,6,5,0,0,0,15,14,13,12,0,0,0,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0.008377874,-0.0008047369,0.004674679,0,0,0,0,-0.007131571,-0.0223218,-0.0008047369,-0.0509243,-0.003631874,0,0,0,0,0,0.008127961,-0.008601503,-0.01357791,-0.02066651,0.009493772,0,-0.02432059,-0.05558786]},{"f":[2,0,16,-1,-1,37,23,-1,-1,16,19,23,-1,-1,-1,-1,19,9,37,23,21,-1,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,11.696967526234287,0,0,7.05,0.5,0,0,8.618385502258606,2.5,5.5,0,0,0,0,0.5,0.5,6.970000000000001,2.5,3.5,0,0,0,0,0,17.07907323966382,0,0],"m":[1,1,1,0,0,1,0,0,0,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,10,11,12,0,0,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,5,4,0,0,9,8,0,0,15,14,13,0,0,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0.006556431,-0.0007025786,0,0,-0.008177587,-0.0007631222,0,0,0,-0.02454818,-0.008721901,-0.008721901,-0.008721901,0,0,0,0,0,-0.001211505,-0.01346069,-0.0002570622,-0.01963477,0.009021331,0,-0.02310591,-0.05287487]},{"f":[2,0,1,-1,16,-1,-1,37,23,-1,-1,16,19,23,-1,-1,-1,-1,37,9,21,-1,-1,-1,22,23,-1,-1,-1],"t":[5.228690439536267,6.5,4.208333333333334,0,7.266786540694901,0,0,7.05,0.5,0,0,8.618385502258606,2.5,5.5,0,0,0,0,6.970000000000001,0.5,5.5,0,0,0,17.094543568094064,1.5,0,0,0],"m":[1,1,1,0,0,0,0,1,0,0,0,1,1,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,12,13,14,0,0,0,0,19,20,21,0,0,0,25,26,0,0,0],"r":[18,7,4,0,6,0,0,11,10,0,0,17,16,15,0,0,0,0,24,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0.006664328,0,0.006088676,-0.0006931119,0,0,-0.007769287,-0.0006931119,0,0,0,-0.02332295,-0.00821929,-0.00821929,-0.00821929,0,0,0,-0.004418344,-0.01380193,0.008205545,0,0,-0.01516843,-0.02278726,-0.04745487]},{"f":[2,0,21,-1,-1,37,23,-1,-1,21,22,-1,23,-1,-1,-1,19,9,2,37,25,-1,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,6.5,0,0,7.05,0.5,0,0,9.5,12.613886328427084,0,1.5,0,0,0,0.5,0.5,10.510264838663748,6.970000000000001,6.5,0,0,0,0,0,17.07907323966382,0,0],"m":[1,1,1,0,0,1,0,0,0,1,0,0,1,0,0,0,1,1,1,1,1,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,10,11,0,13,0,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,5,4,0,0,9,8,0,0,15,12,0,14,0,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0.006324955,0.002321033,0,0,-0.007381373,-0.0006285159,0,0,-0.007743111,0,-0.02348585,-0.00796087,-0.04611934,0,0,0,0,0,-0.006690392,0.01385997,-0.01760044,-0.03040429,0.008210748,0,-0.02108128,-0.04797821]},{"f":[2,1,1,-1,16,-1,-1,23,22,0,-1,21,-1,-1,25,-1,-1,-1,19,1,-1,22,2,16,-1,-1,-1,-1,-1],"t":[5.228690439536267,6.196924603174603,4.208333333333334,0,7.924812503605781,0,0,4.5,12.278990886161445,9.5,0,5.5,0,0,0.5,0,0,0,0.5,0.8611111111111112,0,13.87915215758784,8.9274737487839,4.523561956057013,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,12,0,0,15,0,0,0,19,20,0,22,23,24,0,0,0,0,0],"r":[18,7,4,0,6,0,0,17,14,11,0,13,0,0,16,0,0,0,28,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0.006037799,0,0.004552922,-0.003502164,0,0,0,-0.001289892,0,-0.008345555,-0.03867145,0,-0.02430005,-0.01160658,0.000666555,0,0,0.004763966,0,0,0,0.006605067,-0.006528114,-0.01764439,-0.01770441,-0.02119219]},{"f":[2,1,16,-1,1,-1,-1,23,22,0,-1,21,-1,-1,-1,22,-1,-1,19,9,2,22,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,6.196924603174603,6.930737337562887,0,4.208333333333334,0,0,4.5,12.278990886161445,9.5,0,5.5,0,0,0,17.094543568094064,0,0,0.5,0.5,10.510264838663748,13.87915215758784,0,0,0,0,1.5,0,0],"m":[1,1,1,0,1,0,0,1,0,1,0,1,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,12,0,0,0,16,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,7,4,0,6,0,0,15,14,11,0,13,0,0,0,17,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.006029469,0,0.004683011,-0.0014152,0,0,0,-0.001225502,0,-0.00793004,-0.03677452,-0.01443037,0,0.003006789,-0.01327387,0,0,0,0,-0.00530783,-0.01590745,-0.02822703,0.007581285,0,-0.007342449,-0.0224426]},{"f":[2,0,21,20,-1,-1,-1,37,21,-1,-1,21,22,-1,23,-1,-1,-1,37,1,2,-1,-1,21,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,6.5,0.5,0,0,0,7.05,4.5,0,0,9.5,12.613886328427084,0,1.5,0,0,0,6.970000000000001,1.9735450749013448,9.051177541662383,0,0,3.5,0,0,17.094543568094064,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,15,0,0,0,19,20,21,0,0,24,0,0,27,0,0],"r":[18,7,6,5,0,0,0,11,10,0,0,17,14,0,16,0,0,0,26,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.005899725,0.0002879526,0.001736505,0,0,-0.006203783,-0.0005244386,0,0,-0.006625508,0,-0.0207102,-0.006628663,-0.04119741,0,0,0,0.003812352,-0.01242497,0,-0.002328426,-0.0113312,0,-0.01676075,-0.04116548]},{"f":[2,1,1,-1,4,-1,21,-1,-1,23,22,-1,-1,17,-1,-1,19,2,37,1,-1,-1,-1,-1,23,-1,22,-1,-1],"t":[5.228690439536267,6.196924603174603,2.68018018018018,0,3.084962500721156,0,2.5,0,0,3.5,15.212004812197344,0,0,7.65814076587311,0,0,0.5,8.9274737487839,6.970000000000001,2.9861172120945896,0,0,0,0,1.5,0,17.07907323966382,0,0],"m":[1,1,0,0,1,0,0,0,0,1,1,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,0,0,14,0,0,17,18,19,20,0,0,0,0,25,0,27,0,0],"r":[16,9,4,0,6,0,8,0,0,13,12,0,0,15,0,0,24,23,22,21,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0.005847049,0,0.004039003,0,0.004687338,-0.005289506,0,0,-0.007283113,-0.020264,0,0.001437225,-0.01705139,0,0,0,0,0.003890422,-0.005437378,-0.01396088,-0.01538907,0,-0.006343413,0,-0.01932653,-0.04278667]},{"f":[2,32,1,-1,22,-1,23,-1,22,-1,17,-1,-1,21,33,-1,-1,-1,19,1,-1,22,2,16,-1,-1,-1,-1,-1],"t":[5.228690439536267,2.69615871138938,4.208333333333334,0,12.199665042528586,0,0.5,0,12.738408632106918,0,7.723832212141186,0,0,9.5,22.05815630213804,0,0,0,0.5,0.8611111111111112,0,13.87915215758784,8.9274737487839,4.523561956057013,0,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,0,9,0,11,0,0,14,15,0,0,0,19,20,0,22,23,24,0,0,0,0,0],"r":[18,13,4,0,6,0,8,0,10,0,12,0,0,17,16,0,0,0,28,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0.004970444,0,0.004405795,0,-0.01045918,0,-0.0110591,0,0.004143893,-0.02416683,0,0,-0.01178025,-0.003587426,-0.03904595,0,0,0.004448402,0,0,0,0.006428722,-0.005202046,-0.01493739,-0.01466711,-0.01747733]},{"f":[2,32,1,4,-1,-1,22,-1,23,-1,22,-1,22,-1,-1,21,4,-1,-1,-1,37,1,2,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,2.69615871138938,4.208333333333334,4.0437314206251695,0,0,12.199665042528586,0,0.5,0,12.738408632106918,0,17.094543568094064,0,0,9.5,2.453445297804259,0,0,0,6.970000000000001,2.01685992705101,8.9274737487839,0,0,0,17.094543568094064,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,1,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,9,0,11,0,13,0,0,16,17,0,0,0,21,22,23,0,0,0,27,0,0],"r":[20,15,6,5,0,0,8,0,10,0,12,0,14,0,0,19,18,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0.005258508,0.0003526751,0,0.004185645,0,-0.00993719,0,-0.01050879,0,0.004239104,-0.01781273,0,0,-0.004826696,-0.0139212,-0.03714693,0,0,0,0.002982685,-0.009066476,-0.008790707,0,-0.0144571,-0.03659876]},{"f":[2,32,1,-1,22,-1,23,-1,17,16,-1,-1,-1,21,33,21,-1,-1,-1,-1,20,2,37,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,2.69615871138938,4.208333333333334,0,12.199665042528586,0,0.5,0,7.723832212141186,7.924812503605781,0,0,0,9.5,22.05815630213804,7.5,0,0,0,0,0.5,8.9274737487839,6.970000000000001,0,0,0,1.5,0,0],"m":[1,1,1,0,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,0,9,10,0,0,0,14,15,16,0,0,0,0,21,22,23,0,0,0,27,0,0],"r":[20,13,4,0,6,0,8,0,12,11,0,0,0,19,18,17,0,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0.004485881,0,0.003976495,0,-0.009441247,0,0,0.004725008,-0.006421602,-0.02080143,0,0,0,-0.01234529,0.01628808,-0.003109551,-0.03534027,0,0,0,0.000392129,-0.01290246,-0.01322363,0,-0.008515076,-0.01818194]},{"f":[2,32,1,-1,22,-1,23,-1,22,-1,22,-1,-1,21,4,-1,-1,-1,20,4,22,2,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,2.69615871138938,4.208333333333334,0,12.199665042528586,0,0.5,0,12.738408632106918,0,17.094543568094064,0,0,9.5,2.453445297804259,0,0,0,0.5,10.510264838663748,15.439474430023438,8.9274737487839,0,0,0,0,14.116600814324018,0,0],"m":[1,1,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,1,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,0,9,0,11,0,0,14,15,0,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,13,4,0,6,0,8,0,10,0,12,0,0,17,16,0,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.004261611,0,0.003777797,0,-0.008970055,0,-0.009646439,0,0.003884403,-0.01652813,0,0,-0.004287279,-0.01275551,-0.03362148,0,0,0,0,0.0007934911,-0.008767322,-0.01127094,-0.02392104,0,-0.01001035,-0.01952533]},{"f":[2,37,1,4,-1,0,-1,-1,22,-1,-1,22,1,-1,21,-1,-1,-1,32,2,37,-1,22,-1,-1,-1,4,-1,-1],"t":[5.228690439536267,7.05,4.208333333333334,4.0437314206251695,0,1.5,0,0,12.199665042528586,0,0,12.613886328427084,9.107199999999999,0,5.5,0,0,0,6.474913355379556,8.9274737487839,3.97,0,13.747003318603246,0,0,0,6.16978583919235,0,0],"m":[1,1,1,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,1,1,0,0,1,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,0,0,12,13,0,15,0,0,0,19,20,21,0,23,0,0,0,27,0,0],"r":[18,11,8,5,0,7,0,0,10,0,0,17,14,0,16,0,0,0,26,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0.004558292,0,0.004096726,-0.002796903,0,0.003598501,-0.001821469,0,0,-0.002796903,0,-0.005840014,-0.03047734,-0.01300204,0,0,0,0.002492647,0,-0.003283161,-0.0115814,-0.01332112,0,-0.01481442,-0.03537206]},{"f":[2,37,1,4,-1,-1,22,-1,-1,22,1,-1,21,-1,-1,-1,1,-1,22,2,32,25,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,7.05,4.208333333333334,4.0437314206251695,0,0,12.199665042528586,0,0,12.613886328427084,9.107199999999999,0,5.5,0,0,0,0.8611111111111112,0,13.87915215758784,10.510264838663748,6.474913355379556,6.5,0,0,0,0,17.094543568094064,0,0],"m":[1,1,1,1,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,13,0,0,0,17,0,19,20,21,22,0,0,0,0,27,0,0],"r":[16,9,6,5,0,0,8,0,0,15,12,0,14,0,0,0,18,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0.004330405,-0.0001684425,0,0.003418689,-0.001730433,0,0,-0.002630842,0,-0.005549247,-0.02898235,-0.01235304,0,0.004733279,0,0,0,0,-0.00408254,0.0137212,-0.01792513,-0.02215867,0,-0.01297751,-0.03181022]},{"f":[2,32,1,-1,22,16,-1,2,23,-1,-1,-1,-1,15,23,-1,-1,-1,20,4,22,2,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,3.9474088816539714,4.208333333333334,0,17.094543568094064,7.266786540694901,0,1.292481250360578,0.5,0,0,0,0,8.618385502258606,3.5,0,0,0,0.5,10.510264838663748,15.439474430023438,8.9274737487839,0,0,0,0,1.5,0,0],"m":[1,1,1,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,0,8,9,0,0,0,0,14,15,0,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,13,4,0,12,7,0,11,10,0,0,0,0,17,16,0,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.003646412,0,0,0.002874395,0,0,-0.004794221,0.00615717,-0.006353009,-0.01592655,0,0,-0.01367633,0.0005443959,-0.002331011,0,0,0,0,0.0008808274,-0.007476909,-0.009479863,-0.02111318,0,-0.006971701,-0.01517641]},{"f":[30,21,22,0,-1,-1,24,33,2,-1,-1,-1,-1,-1,2,21,1,16,-1,-1,-1,2,22,-1,-1,15,-1,-1,-1],"t":[1.5,9.5,14.03140051979143,8.5,0,0,7.5,20.752541988539424,4.80643424864552,0,0,0,0,0,8.9274737487839,6.5,6.015625,12.683433292835403,0,0,0,3.39067985676233,17.094543568094064,0,0,10.501837184902296,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,0,0,1,1,0,0,1,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,0,0,15,16,17,18,0,0,0,22,23,0,0,26,0,0,0],"r":[14,13,6,5,0,0,12,11,10,0,0,0,0,0,28,21,20,19,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.001229939,-0.00736486,0,0,0,0.01325826,-0.008519137,-0.01633367,-0.01513572,-0.03641548,0,0,0,0,0.00384211,-0.004528237,-0.003734257,0,0,0.001578458,-0.01257666,0,-0.007824104,0.004617118,-0.01150193]},{"f":[30,21,22,0,-1,-1,24,-1,-1,-1,2,21,20,16,-1,-1,22,-1,-1,2,22,16,-1,-1,-1,1,-1,-1,-1],"t":[1.5,9.5,14.03140051979143,8.5,0,0,7.5,0,0,0,8.9274737487839,6.5,0.5,12.683433292835403,0,0,12.199665042528586,0,0,3.39067985676233,17.094543568094064,7.774787059601174,0,0,0,1.000936329588015,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,0,11,12,13,14,0,0,17,0,0,20,21,22,0,0,0,26,0,0,0],"r":[10,9,6,5,0,0,8,0,0,0,28,19,16,15,0,0,18,0,0,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0.001168594,-0.006996992,0,-0.005562268,-0.01438006,-0.03462948,0,0,0,0,0.003888598,-0.004302426,0,0.00368211,-0.005466634,0,0,0,0.003531525,-0.002169117,-0.01195347,0,0.002802234,-0.008142368,-0.01092768]},{"f":[30,21,22,0,-1,-1,4,-1,-1,-1,2,21,19,16,-1,-1,22,-1,-1,2,22,16,-1,-1,-1,15,-1,-1,-1],"t":[1.5,9.5,14.03140051979143,8.5,0,0,6.08730682361797,0,0,0,8.9274737487839,6.5,0.5,12.683433292835403,0,0,12.199665042528586,0,0,3.39067985676233,17.094543568094064,7.774787059601174,0,0,0,10.501837184902296,0,0,0],"m":[0,1,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,1,1,1,0,0,0,1,0,0,0],"l":[1,2,3,4,0,0,7,0,0,0,11,12,13,14,0,0,17,0,0,20,21,22,0,0,0,26,0,0,0],"r":[10,9,6,5,0,0,8,0,0,0,28,19,16,15,0,0,18,0,0,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0.001110308,-0.006647498,0,-0.009672096,-0.02687236,-0.03293108,0,0,0,0,0.003682637,-0.004087876,0,0.003498318,-0.005499489,0,0,0,0.003355056,-0.002060779,-0.01136116,0,-0.007140122,0.004390287,-0.01038209]},{"f":[2,32,21,-1,1,4,-1,22,-1,-1,23,-1,22,16,-1,-1,-1,18,-1,-1,1,-1,22,2,32,-1,-1,-1,-1],"t":[5.228690439536267,3.9474088816539714,2.5,0,4.208333333333334,4.0437314206251695,0,15.756821747580634,0,0,0.5,0,17.094543568094064,7.924812503605781,0,0,0,7.980000966034041,0,0,0.8611111111111112,0,13.87915215758784,10.510264838663748,6.474913355379556,0,0,0,0],"m":[1,1,0,0,1,1,0,1,0,0,0,0,1,1,0,0,0,1,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,0,11,0,13,14,0,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,17,4,0,10,7,0,9,0,0,12,0,16,15,0,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0.003975376,0,0,0.003007934,0,-0.0004294658,-0.01563132,0,-0.004459527,0,0,0.004172534,-0.002212977,-0.01304426,0,-0.006546589,0.008278223,0,0.004560486,0,0,0,-0.002481657,-0.01537794,-0.01839156,-0.01108207]},{"f":[37,2,1,16,-1,-1,22,-1,23,-1,22,-1,-1,9,-1,-1,4,21,22,4,-1,-1,15,-1,33,-1,-1,-1,-1],"t":[7.05,5.065928480304397,4.208333333333334,10.486835021563051,0,0,12.199665042528586,0,3.5,0,16.819976309430963,0,0,0.5,0,0,6.16978583919235,9.5,12.613886328427084,2.453445297804259,0,0,5.044394119358453,0,20.752541988539424,0,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,9,0,11,0,0,14,0,0,17,18,19,20,0,0,23,0,25,0,0,0,0],"r":[16,13,6,5,0,0,8,0,10,0,12,0,0,15,0,0,28,27,22,21,0,0,24,0,26,0,0,0,0],"v":[0,0,0,0,0.003284177,-0.0008901339,0,0.002873133,0,-0.003307776,0,0.007570542,-0.0009312196,0,-0.003307776,0.005985948,0,0,0,0,-0.003307776,-0.007348128,0,-0.01736287,0,-0.003307776,-0.01083695,-0.02550077,-0.02684005]},{"f":[2,32,21,-1,1,4,-1,22,-1,-1,23,-1,22,-1,-1,18,-1,-1,20,4,33,21,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,3.9474088816539714,2.5,0,4.208333333333334,4.0437314206251695,0,15.756821747580634,0,0,0.5,0,17.094543568094064,0,0,7.980000966034041,0,0,0.5,10.510264838663748,26.99635432609064,7.5,0,0,0,0,1.5,0,0],"m":[1,1,0,0,1,1,0,1,0,0,0,0,1,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,0,11,0,13,0,0,16,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,15,4,0,10,7,0,9,0,0,12,0,14,0,0,17,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.003633094,0,0,0.002720713,0,-0.0005149661,-0.01500218,0,-0.004169768,0,0.001908659,-0.01235126,0,-0.005860888,0.008086778,0,0,0,0,0.0008364615,-0.006063507,-0.0121728,-0.0174813,0,-0.005100672,-0.01212888]},{"f":[2,37,21,-1,1,4,-1,-1,21,-1,-1,22,21,-1,-1,21,-1,-1,20,2,1,-1,2,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,7.05,2.5,0,4.208333333333334,4.0437314206251695,0,0,8.5,0,0,15.439474430023438,8.5,0,0,7.5,0,0,0.5,10.510264838663748,0.8611111111111112,0,5.538407798525416,0,0,0,14.116600814324018,0,0],"m":[1,1,0,0,1,1,0,0,1,0,0,1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,0,9,0,0,12,13,0,0,16,0,0,19,20,21,0,23,0,0,0,27,0,0],"r":[18,11,4,0,8,7,0,0,10,0,0,15,14,0,0,17,0,0,26,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0.003496748,0,0,0.002584704,-0.001699859,0,-0.001261285,0.007513785,0,0,-0.002491027,-0.02276248,0,-0.01420882,-0.001699859,0,0,0,0.005680821,0,0.01287367,-0.003544033,-0.01563138,0,-0.00624597,-0.0131535]},{"f":[30,21,22,0,-1,-1,4,15,-1,33,18,-1,-1,-1,-1,-1,2,16,22,-1,16,-1,-1,2,-1,0,-1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,6.16978583919235,5.044394119358453,0,20.752541988539424,1.292481250360578,0,0,0,0,0,8.9274737487839,6.930737337562887,17.094543568094064,0,6.539158811108031,0,0,1.292481250360578,0,4.5,0,0,0],"m":[0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,1,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,4,0,0,7,8,0,10,11,0,0,0,0,0,17,18,19,0,21,0,0,24,0,26,0,0,0],"r":[16,15,6,5,0,0,14,9,0,13,12,0,0,0,0,0,28,23,20,0,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,0.004036614,-0.005061085,0,0,-0.01573609,0,0,-0.003675963,0.01174653,-0.009628935,-0.02249793,-0.02839957,0,0,0,0.00258567,0,-0.00165613,-0.02315412,0,0.002367423,0,0.0002966802,-0.00550038,-0.008660792]},{"f":[30,21,22,0,-1,-1,4,21,-1,2,-1,-1,-1,-1,2,21,1,16,-1,-1,-1,2,-1,1,-1,17,-1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,6.16978583919235,7.5,0,5.065928480304397,0,0,0,0,8.9274737487839,6.5,6.015625,12.683433292835403,0,0,0,3.245926548164837,0,1.000936329588015,0,8.496347487943302,0,0,0],"m":[0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,1,1,1,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,4,0,0,7,8,0,10,0,0,0,0,15,16,17,18,0,0,0,22,0,24,0,26,0,0,0],"r":[14,13,6,5,0,0,12,9,0,11,0,0,0,0,28,21,20,19,0,0,0,23,0,25,0,27,0,0,0],"v":[0,0,0,0,0.003835476,-0.004808298,0,0,-0.009573816,0,0.008457708,-0.006238086,-0.02139129,-0.02700671,0,0,0,0,0.002724847,-0.003548554,-0.002803576,0,0.0004969368,0,0.002569624,0,-0.008739302,-0.001020736,-0.008228388]},{"f":[30,21,22,0,-1,-1,24,36,-1,-1,22,-1,-1,-1,2,16,22,-1,16,-1,-1,2,-1,0,-1,-1,1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,7.5,3.5,0,0,17.094543568094064,0,0,0,8.9274737487839,6.930737337562887,17.094543568094064,0,6.539158811108031,0,0,1.292481250360578,0,4.5,0,0,2.01685992705101,0,0],"m":[0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,1,1,0,1,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,0,0,15,16,17,0,19,0,0,22,0,24,0,0,27,0,0],"r":[14,13,6,5,0,0,10,9,0,0,12,0,0,0,26,21,18,0,20,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0.00364436,-0.004568138,0,0,-0.01152198,0.001375796,0,-0.008921283,-0.02371989,-0.02568217,0,0,0,0.00235863,0,-0.001610228,-0.02204955,0,0.002201136,0,0.0002461764,-0.005154955,0,-0.004627697,-0.01275135]},{"f":[2,1,16,-1,22,-1,-1,4,1,21,-1,-1,-1,-1,21,4,-1,-1,1,-1,22,23,1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.196924603174603,11.620219825507487,0,7.25627765006036,0,0,2.453445297804259,8.375,8.5,0,0,0,0,3.5,3.753897320099348,0,0,0.6824793747286149,0,12.121442107629743,2.5,8.046875,0,0,0,17.094543568094064,0,0],"m":[1,1,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,0,0,0,15,16,0,0,19,0,21,22,23,0,0,0,27,0,0],"r":[14,7,4,0,6,0,0,13,12,11,0,0,0,0,18,17,0,0,20,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0.002054476,0,-0.0105251,-0.000386021,0,0,0,-9.511392e-06,0.0102789,-0.005045263,-0.008013701,0,0,0.01337196,0.001247589,0,0.003447043,0,0,0,-0.003356382,-0.01246699,0.003929056,0,-0.007934483,-0.02228827]},{"f":[2,1,21,-1,23,-1,-1,4,15,-1,15,-1,15,-1,37,-1,-1,15,-1,-1,21,-1,1,2,21,-1,-1,-1,-1],"t":[5.228690439536267,5.3560606060606055,7.5,0,4.5,0,0,3.084962500721156,5.044394119358453,0,6.044394119358453,0,6.820178962415188,0,9.205,0,0,6.882643049361842,0,0,3.5,0,4.92309205350118,8.9274737487839,7.5,0,0,0,0],"m":[1,1,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,11,0,13,0,15,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,7,4,0,6,0,0,17,10,0,12,0,14,0,16,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0.002275021,0,-0.004177678,0.003519658,0,0,-0.006222774,0,0.01119996,0,-0.008562748,0,0.001360111,-0.006921017,0,-0.001487876,-0.009120422,0,0.003203332,0,0,0,0.002151788,-0.003518152,-0.008850262,-0.008606413]},{"f":[2,1,21,22,-1,2,-1,-1,23,22,-1,-1,-1,4,18,-1,-1,-1,21,-1,1,-1,22,23,-1,-1,15,-1,-1],"t":[5.228690439536267,5.489664082687339,7.5,16.819976309430963,0,3.245926548164837,0,0,4.5,2.084962500721156,0,0,0,3.084962500721156,2.160964047443681,0,0,0,3.5,0,0.6824793747286149,0,11.488914557581817,2.5,0,0,11.696967526234287,0,0],"m":[1,1,1,1,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,0,14,15,0,0,0,19,0,21,0,23,24,0,0,27,0,0],"r":[18,13,8,5,0,7,0,0,12,11,0,0,0,17,16,0,0,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0.002383514,0,-0.0004580709,-0.02308646,0,0,0.005687868,-0.005240605,0.003344041,0,0,-0.003091417,0.001533092,-0.005513615,0,0.003043517,0,0.003283517,0,0,-0.003618434,0.004543914,0,-0.008223074,-0.001120926]},{"f":[30,21,22,0,-1,-1,-1,-1,2,16,22,2,-1,-1,16,-1,-1,15,-1,2,-1,1,-1,-1,1,2,-1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,0,0,8.9274737487839,6.930737337562887,17.094543568094064,7.754129516000594,0,0,6.539158811108031,0,0,7.451211111832329,0,1.292481250360578,0,4.208333333333334,0,0,2.01685992705101,9.266718101290426,0,0,0],"m":[0,1,0,0,0,0,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,0,0,9,10,11,12,0,0,15,0,0,18,0,20,0,22,0,0,25,26,0,0,0],"r":[8,7,6,5,0,0,0,0,24,17,14,13,0,0,16,0,0,19,0,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0.00365745,-0.003729314,-0.006227947,-0.02355188,0,0,0,0,0.001917375,0.01242073,0,-0.001593158,-0.02063145,0,-0.003767092,0,0.002647624,0,0.0007469128,-0.004844642,0,0,-0.007496685,0.004457542,-0.01123601]},{"f":[2,32,23,22,-1,-1,-1,21,-1,4,1,-1,-1,22,-1,-1,21,-1,1,2,21,-1,-1,22,-1,-1,22,-1,-1],"t":[5.228690439536267,10.463013322934223,3.5,14.158358850608169,0,0,0,2.5,0,4.0437314206251695,4.208333333333334,0,0,14.247293309538719,0,0,3.5,0,4.92309205350118,8.9274737487839,7.5,0,0,10.896650035079205,0,0,17.094543568094064,0,0],"m":[1,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,0,10,11,0,0,14,0,0,17,0,19,20,21,0,0,24,0,0,27,0,0],"r":[16,7,6,5,0,0,0,9,0,13,12,0,0,15,0,0,18,0,26,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-0.003060156,-0.009889142,0.001226775,0,0.00254678,0,0,0.001816088,-0.0004061465,0,-0.001092494,-0.0150007,0,0.002937343,0,0,0,0.002029989,-0.00300024,0,-0.00177435,-0.01062384,0,-0.006879078,-0.02079873]},{"f":[2,32,16,22,-1,-1,22,-1,-1,18,1,-1,-1,-1,21,-1,1,-1,22,21,16,-1,22,-1,-1,-1,15,-1,-1],"t":[5.228690439536267,3.9474088816539714,11.620219825507487,17.094543568094064,0,0,7.25627765006036,0,0,7.980000966034041,12.069111969111969,0,0,0,3.5,0,0.6824793747286149,0,11.488914557581817,8.5,6.882643049361842,0,7.95646805166752,0,0,0,11.696967526234287,0,0],"m":[1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,0,0,15,0,17,0,19,20,21,0,23,0,0,0,27,0,0],"r":[14,9,6,5,0,0,8,0,0,13,12,0,0,0,16,0,18,0,26,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0.001509454,-0.007707629,0,-0.00995907,-0.0006758821,0,0,-0.003248729,-0.0150121,0.007828831,0,0.002790798,0,0.003100529,0,0,0,0.007240898,0,-0.01625131,-0.002213952,-0.009468699,0,-0.007304109,-0.0007449547]},{"f":[30,21,22,0,-1,-1,15,-1,15,-1,-1,-1,2,21,22,-1,21,1,-1,2,-1,-1,-1,-1,1,2,-1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,5.044394119358453,0,6.169925001442312,0,0,0,8.9274737487839,6.5,13.87915215758784,0,4.5,2.532051282051282,0,3.084962500721156,0,0,0,0,2.01685992705101,9.266718101290426,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,0,0,1,1,1,0,1,0,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,7,0,9,0,0,0,13,14,15,0,17,18,0,20,0,0,0,0,25,26,0,0,0],"r":[12,11,6,5,0,0,8,0,10,0,0,0,24,23,16,0,22,19,0,21,0,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0.003656149,-0.003272386,0,-0.01223436,0,0.008137118,-0.006057241,-0.02189131,0,0,0,0.002054349,0,0,0.003741753,0,-0.003677748,-0.01116781,0.002291743,-0.0007441683,0,0,-0.006614021,0.004533797,-0.01007423]},{"f":[2,1,-1,4,1,22,-1,22,-1,15,-1,15,-1,-1,-1,-1,21,25,-1,-1,1,-1,22,16,-1,-1,22,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,8.55,12.199665042528586,0,12.738408632106918,0,6.044394119358453,0,7.118941072723508,0,0,0,0,3.5,4.5,0,0,0.6824793747286149,0,12.199665042528586,2.321928094887362,0,0,17.094543568094064,0,0],"m":[1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,8,0,10,0,12,0,0,0,0,17,18,0,0,21,0,23,24,0,0,27,0,0],"r":[16,3,0,15,14,7,0,9,0,11,0,13,0,0,0,0,20,19,0,0,22,0,26,25,0,0,28,0,0],"v":[0,0,0.001926072,0,0,0,0.003446589,0,-0.006563533,0,0.005993552,0,-0.01395208,0.0003140402,-0.003877671,-0.00659403,0,0,0.001216004,0.01293189,0,0.002577709,0,0,0.003963392,-0.003341359,0,-0.006555259,-0.0183946]},{"f":[2,1,-1,4,37,22,-1,2,-1,-1,-1,-1,21,25,-1,-1,1,-1,22,16,-1,1,16,-1,-1,-1,22,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,8.05,12.199665042528586,0,1.792481250360578,0,0,0,0,3.5,4.5,0,0,0.6824793747286149,0,12.199665042528586,2.321928094887362,0,9.107199999999999,5.78135971352466,0,0,0,17.094543568094064,0,0],"m":[1,1,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,8,0,0,0,0,13,14,0,0,17,0,19,20,0,22,23,0,0,0,27,0,0],"r":[12,3,0,11,10,7,0,9,0,0,0,0,16,15,0,0,18,0,26,21,0,25,24,0,0,0,28,0,0],"v":[0,0,0.001829781,0,0,0,0.003274413,0,-0.001785509,0.002536214,-0.004220216,-0.006265041,0,0,0.001155242,0.01229396,0,0.002449089,0,0,0.00376596,0,0,-0.007013876,-0.001399042,-0.009909512,0,-0.006227685,-0.01749064]},{"f":[2,1,-1,4,15,2,-1,-1,15,-1,15,16,-1,-1,2,-1,-1,-1,21,25,-1,-1,1,-1,22,16,-1,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,5.044394119358453,1.792481250360578,0,0,6.044394119358453,0,6.857980995127572,6.539158811108031,0,0,1.292481250360578,0,0,0,3.5,4.5,0,0,0.6824793747286149,0,12.199665042528586,1.584962500721156,0,0,0],"m":[1,1,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,0,11,12,0,0,15,0,0,0,19,20,0,0,23,0,25,26,0,0,0],"r":[18,3,0,17,8,7,0,0,10,0,14,13,0,0,16,0,0,0,22,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0.001738303,0,0,0,-0.006360621,0.001795956,0,0.006298109,0,0,-0.002986104,-0.0205756,0,0.001939921,-0.001931708,-0.005952465,0,0,0.001097517,0.01168749,0,0.002326888,0,0,0.006134287,-0.002733068,-0.006268198]},{"f":[2,1,16,19,-1,22,-1,23,-1,-1,22,-1,-1,-1,21,25,-1,-1,1,-1,22,25,21,-1,-1,17,-1,-1,-1],"t":[4.0437314206251695,8.7375,11.620219825507487,1.5,0,12.199665042528586,0,1.5,0,0,7.25627765006036,0,0,0,3.5,4.5,0,0,0.6824793747286149,0,12.199665042528586,0.5,7.5,0,0,7.781300095563516,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,11,0,0,0,15,16,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[14,13,10,5,0,7,0,9,0,0,12,0,0,0,18,17,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.001640322,0,0.004152283,0,-0.005264568,0.002792203,0,-0.00897769,-0.0006652157,-0.003776245,0,0,0.001042676,0.01111094,0,0.002210783,0,0,0,0.00110064,-0.003403931,0,-0.01976826,-0.004592869,-0.005954963]},{"f":[2,1,16,19,-1,22,-1,23,-1,-1,-1,-1,21,-1,17,2,-1,-1,2,18,22,-1,2,-1,-1,23,-1,-1,-1],"t":[4.0437314206251695,8.7375,11.620219825507487,1.5,0,12.199665042528586,0,1.5,0,0,0,0,3.5,0,1.292481250360578,6.08730682361797,0,0,9.883406784235138,6.4178278454199,12.613886328427084,0,5.065928480304397,0,0,3.5,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,0,0,13,0,15,16,0,0,19,20,21,0,23,0,0,26,0,0,0],"r":[12,11,10,5,0,7,0,9,0,0,0,0,14,0,18,17,0,0,28,25,22,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0.001558314,0,0.003945174,0,-0.005001719,0.002653361,-0.002269258,-0.003587737,0,0.001423325,0,0,0.003996343,-0.002016102,0,0,0,-0.002608369,0,-0.01364276,-0.005637507,0,-0.002159899,0.005270627,-0.01238126]},{"f":[2,1,-1,4,15,2,-1,-1,15,-1,15,16,-1,-1,2,-1,-1,-1,21,-1,1,-1,22,37,16,-1,-1,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,5.044394119358453,1.792481250360578,0,0,6.044394119358453,0,6.857980995127572,6.539158811108031,0,0,1.292481250360578,0,0,0,3.5,0,0.6824793747286149,0,12.121442107629743,9.075,1.584962500721156,0,0,0,0],"m":[1,1,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,0,11,12,0,0,15,0,0,0,19,0,21,0,23,24,25,0,0,0,0],"r":[18,3,0,17,8,7,0,0,10,0,14,13,0,0,16,0,0,0,20,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0.001507727,0,0,0,-0.005724268,0.00186293,0,0.005867675,0,0,-0.002749277,-0.01968999,0,0.001801844,-0.001918824,-0.005379656,0,0.001352201,0,0.002106287,0,0,0,0.005811512,-0.001737976,-0.01453875,-0.005204944]},{"f":[2,1,16,-1,21,-1,-1,-1,21,32,-1,-1,17,2,16,-1,-1,-1,2,18,2,-1,22,-1,-1,23,-1,-1,-1],"t":[4.0437314206251695,8.7375,11.620219825507487,0,7.5,0,0,0,3.5,4.60106191191523,0,0,1.292481250360578,6.08730682361797,7.679480099505446,0,0,0,9.883406784235138,6.4178278454199,5.40929108874043,0,17.094543568094064,0,0,3.5,0,0,0],"m":[1,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,5,0,0,0,9,10,0,0,13,14,15,0,0,0,19,20,21,0,23,0,0,26,0,0,0],"r":[8,7,4,0,6,0,0,0,12,11,0,0,18,17,16,0,0,0,28,25,22,0,24,0,0,27,0,0,0],"v":[0,0,0,0.001147308,0,-0.0005396475,-0.008503063,-0.003314915,0,0,0.01042591,0.0008518863,0,0,0,0.01191455,0.001421286,-0.001861348,0,0,0,-0.007418408,0,-0.002801377,-0.01597622,0,-0.001912025,0.005118711,-0.01161514]},{"f":[30,21,4,27,22,-1,-1,-1,16,-1,-1,-1,2,16,2,-1,-1,21,-1,15,22,-1,-1,22,-1,-1,16,-1,-1],"t":[1.5,9.5,6.16978583919235,0.5,16.98413870862295,0,0,0,7.442943495848728,0,0,0,8.9274737487839,2.584962500721156,7.640240405159187,0,0,2.5,0,5.169925001442312,16.819976309430963,0,0,17.07907323966382,0,0,11.868050853745158,0,0],"m":[0,1,1,0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,0,0,0,9,0,0,0,13,14,15,0,0,18,0,20,21,0,0,24,0,0,27,0,0],"r":[12,11,8,7,6,0,0,0,10,0,0,0,26,17,16,0,0,19,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.001859192,0.0122916,-0.008121665,0,-0.0001857106,-0.01809779,-0.01931443,0,0,0,0.001923977,0.0143707,0,0.001656344,0,0,-0.002135974,-0.01338066,0,0.0003097886,-0.0116041,0,-0.005389397,0.004822595]},{"f":[30,21,-1,-1,2,16,4,22,-1,15,23,-1,-1,17,-1,-1,-1,15,-1,21,-1,15,-1,-1,1,2,-1,-1,-1],"t":[1.5,9.5,0,0,8.9274737487839,6.930737337562887,7.754129516000594,16.819976309430963,0,6.0,2.5,0,0,7.6181566135214025,0,0,0,7.451211111832329,0,7.5,0,7.774787059601174,0,0,2.01685992705101,9.266718101290426,0,0,0],"m":[0,1,0,0,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,0,0,5,6,7,8,0,10,11,0,0,14,0,0,0,18,0,20,0,22,0,0,25,26,0,0,0],"r":[4,3,0,0,24,17,16,9,0,13,12,0,0,15,0,0,0,19,0,21,0,23,0,0,28,27,0,0,0],"v":[0,0,-0.002178626,-0.01836716,0,0,0,0,0.001460641,0,0,-0.016822,-0.004470897,0,0.001257074,-0.008033794,0.01366689,0,-0.00284892,0,0.001061453,0,0.001485448,-0.005577142,0,0,-0.004974966,0.004824269,-0.007869908]},{"f":[2,1,-1,4,18,4,23,-1,15,-1,-1,-1,-1,-1,21,-1,22,2,1,-1,21,4,-1,-1,-1,-1,22,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,2.160964047443681,1.792481250360578,3.5,0,8.076815597050832,0,0,0,0,0,3.5,0,16.765402642760364,9.883406784235138,0.3808851224105462,0,8.5,9.193524834446507,0,0,0,0,16.786010507999382,0,0],"m":[1,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,7,0,9,0,0,0,0,0,15,0,17,18,19,0,21,22,0,0,0,0,27,0,0],"r":[14,3,0,13,12,11,8,0,10,0,0,0,0,0,16,0,26,25,20,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0.001296431,0,0,0,0,-0.004742594,0,0.007903301,-0.0013694,0.001704019,0.001328091,-0.005045252,0,0.001183818,0,0,0,0.004328119,0,0,-0.001635213,0.00848965,-0.007257641,-0.009286119,0,-0.03042415,-0.004559684]},{"f":[37,4,19,35,21,-1,4,22,-1,-1,16,-1,-1,-1,22,2,-1,-1,-1,15,11,-1,-1,-1,36,24,-1,-1,-1],"t":[8.05,3.9534452978042594,1.5,1.5,6.5,0,1.792481250360578,2.084962500721156,0,0,10.501837184902296,0,0,0,12.199665042528586,1.792481250360578,0,0,0,10.376125389276174,0.5,0,0,0,5.5,7.5,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,11,0,0,0,15,16,0,0,0,20,21,0,0,0,25,26,0,0,0],"r":[24,19,14,13,6,0,10,9,0,0,12,0,0,0,18,17,0,0,0,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0.001413787,0,0,0.004463777,-0.001504616,0,0.006182006,-0.0004171584,0.009903235,0,0,0.004909221,-0.001744468,-0.001744468,0,0,-0.001744468,0.006379062,0.001664868,0,0,-0.01167812,-0.002473844,-0.02423254]},{"f":[1,21,-1,16,22,-1,-1,22,-1,-1,4,33,-1,33,-1,22,-1,-1,22,24,21,-1,-1,-1,32,17,-1,-1,-1],"t":[2.01685992705101,7.5,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,13.249860669831508,0,17.460839126440824,0,17.094543568094064,0,0,12.199665042528586,9.5,5.5,0,0,0,6.208926257442949,2.69615871138938,0,0,0],"m":[0,1,0,1,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,0,8,0,0,11,12,0,14,0,16,0,0,19,20,21,0,0,0,25,26,0,0,0],"r":[10,3,0,7,6,0,0,9,0,0,18,13,0,15,0,17,0,0,24,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0.001514115,0,0,0.00614862,-0.002591541,0,-0.01551936,0.001354442,0,0,-0.005173294,0,0.01158753,0,0.0005208784,-0.007487303,0,0,0,0.001581861,-0.002530958,-0.009962296,0,0,0.004169317,-0.003195852,-0.005922454]},{"f":[1,21,19,2,-1,-1,-1,16,22,-1,-1,22,-1,-1,4,33,-1,33,-1,-1,22,24,1,-1,-1,-1,32,-1,-1],"t":[2.01685992705101,7.5,1.5,9.193524834446507,0,0,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,13.249860669831508,0,17.460839126440824,0,0,12.199665042528586,9.5,2.9733542319749215,0,0,0,6.208926257442949,0,0],"m":[0,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,0,15,16,0,18,0,0,21,22,23,0,0,0,27,0,0],"r":[14,7,6,5,0,0,0,11,10,0,0,13,0,0,20,17,0,19,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0.001435946,0.00930363,-0.006581797,0,0,0.005843148,-0.002462223,0,-0.0147512,0.001288402,0,0,-0.004915742,0,0.01101505,0.0003566795,0,0,0,-0.007407482,0.0003047911,-0.009467931,0,-0.0003277002,-0.005626538]},{"f":[30,21,-1,-1,2,21,23,22,17,-1,-1,1,-1,21,17,18,-1,-1,1,-1,-1,-1,2,-1,-1,3,-1,-1,-1],"t":[1.5,9.5,0,0,10.510264838663748,6.5,3.5,13.87915215758784,11.578476357671452,0,0,2.532051282051282,0,4.5,6.9827117829050644,5.065928480304397,0,0,5.015625,0,0,0,4.128693921346326,0,0,0.026802550058364013,0,0,0],"m":[0,1,0,0,1,1,1,1,1,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,8,9,0,0,12,0,14,15,16,0,0,19,0,0,0,23,0,0,26,0,0,0],"r":[4,3,0,0,28,25,22,11,10,0,0,13,0,21,18,17,0,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,-0.001793758,-0.01657933,0,0,0,0,0,0.001144265,-0.009538767,0,0.00284544,0,0,0,-0.004935764,0.00393552,0,-0.02279681,-0.007585419,0.0006713617,0,0.002871791,0.01086618,0,-0.005913337,-0.0004683477,-0.008264888]},{"f":[1,21,2,2,-1,-1,-1,16,22,-1,-1,22,-1,-1,4,-1,22,24,1,-1,-1,-1,32,17,-1,0,-1,-1,-1],"t":[2.01685992705101,7.5,9.250296472946918,9.075473573191719,0,0,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,0,12.199665042528586,9.5,2.9733542319749215,0,0,0,6.208926257442949,2.69615871138938,0,8.5,0,0,0],"m":[0,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,0,15,0,17,18,19,0,0,0,23,24,0,26,0,0,0],"r":[14,7,6,5,0,0,0,11,10,0,0,13,0,0,16,0,22,21,20,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0.001337367,-0.005673278,0.009029398,0,0,0.00557612,-0.002261276,0,-0.01392979,0.001256718,0,0.0003046911,0,0,0,-0.007013127,0.0002690045,-0.008861565,0,0,0.004069179,0,0.008365866,-0.00466798,-0.005258962]},{"f":[25,4,0,-1,-1,-1,2,33,16,17,2,-1,-1,-1,22,15,22,21,-1,-1,-1,16,22,-1,-1,-1,-1,-1,-1],"t":[2.5,6.40929108874043,11.5,0,0,0,9.051177541662383,12.53085435233031,2.584962500721156,11.503187353537466,7.640240405159187,0,0,0,17.07907323966382,5.169925001442312,16.819976309430963,3.5,0,0,0,6.930737337562887,16.96990787537721,0,0,0,0,0,0],"m":[0,1,1,0,0,0,1,1,0,1,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,9,10,11,0,0,0,15,16,17,18,0,0,0,22,23,0,0,0,0,0,0],"r":[6,5,4,0,0,0,28,27,14,13,12,0,0,0,26,21,20,19,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,-0.001094566,-0.006290324,-0.01483648,0,0,0,0,0,0.001593409,0.01308531,-0.01030069,0,0,0,0,0.0007053689,-0.002403899,-0.0117903,0,0,0.001589859,-0.01049135,-0.0001711212,-0.01119273,0.007886674,-0.003873363]},{"f":[37,4,19,33,17,-1,-1,16,-1,-1,22,2,-1,-1,-1,15,-1,-1,36,33,-1,32,-1,15,-1,22,-1,-1,-1],"t":[8.05,3.9534452978042594,1.5,27.893589438525005,10.482680779086738,0,0,9.862637357558794,0,0,12.199665042528586,1.792481250360578,0,0,0,10.376125389276174,0,0,5.5,13.895530557358477,0,8.110850196876331,0,9.92481250360578,0,16.765402642760364,0,0,0],"m":[1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,12,0,0,0,16,0,0,19,20,0,22,0,24,0,26,0,0,0],"r":[18,15,10,7,6,0,0,9,0,0,14,13,0,0,0,17,0,0,28,21,0,23,0,25,0,27,0,0,0],"v":[0,0,0,0,0,0.007673583,0.00101322,0,0.0009128435,-0.001187236,0,0,0.004615235,-0.001405008,-0.001405008,0,-0.001405008,0.001652688,0,0,-0.007351764,0,-0.001405008,0,-0.01226727,0,-0.001405008,-0.007735816,-0.02189021]},{"f":[1,21,2,-1,-1,16,22,-1,-1,22,-1,-1,4,-1,22,21,16,-1,-1,-1,32,19,20,-1,-1,-1,15,-1,-1],"t":[2.01685992705101,7.5,9.221514256038809,0,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,0,12.199665042528586,8.5,4.523561956057013,0,0,0,6.208926257442949,0.5,0.5,0,0,0,7.451211111832329,0,0],"m":[0,1,1,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,0,13,0,15,16,17,0,0,0,21,22,23,0,0,0,27,0,0],"r":[12,5,4,0,0,9,8,0,0,11,0,0,14,0,20,19,18,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0.001061363,0.008722559,0,0,0.005233079,-0.002092173,0,-0.01310443,0.001273905,0,0.0002759241,0,0,0,0.007124086,-0.000793602,-0.01333961,0,0,0,-0.0001593836,-0.01286875,0.008387075,0,-0.006772819,-0.002805109]},{"f":[21,-1,2,1,-1,4,32,23,22,33,-1,15,-1,-1,-1,-1,-1,20,22,-1,2,-1,-1,16,-1,18,-1,-1,-1],"t":[2.5,0,8.9274737487839,4.92309205350118,0,3.084962500721156,10.463013322934223,3.5,12.199665042528586,29.75825500004914,0,9.92481250360578,0,0,0,0,0,0.5,12.613886328427084,0,5.065928480304397,0,0,6.0,0,8.03615005796417,0,0,0],"m":[0,0,1,1,0,1,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,0,0,0,18,19,0,21,0,0,24,0,26,0,0,0],"r":[2,0,28,5,0,17,16,15,14,11,0,13,0,0,0,0,0,23,20,0,22,0,0,25,0,27,0,0,0],"v":[0,0.001156793,0,0,0.0004807002,0,0,0,0,0,0.002556343,0,-0.0206506,-0.001238944,-0.003218045,0.001143224,0.006810297,0,0,0.002983373,0,-0.01382329,-0.001115191,0,0.003678295,0,-0.006872176,0.003732961,-0.00401651]},{"f":[33,2,16,17,-1,-1,15,22,21,-1,-1,-1,32,-1,22,33,32,-1,-1,-1,17,-1,-1,-1,15,-1,2,-1,-1],"t":[29.297756302860655,10.510264838663748,2.584962500721156,11.78143940991899,0,0,5.169925001442312,14.116600814324018,3.5,0,0,0,3.753897320099348,0,17.094543568094064,27.408661170593152,7.602820271428284,0,0,0,8.729619742794132,0,0,0,9.92481250360578,0,2.453445297804259,0,0],"m":[1,1,0,1,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,0,15,16,17,0,0,0,21,0,0,0,25,0,27,0,0],"r":[24,23,6,5,0,0,12,11,10,0,0,0,14,0,20,19,18,0,0,0,22,0,0,0,26,0,28,0,0],"v":[0,0,0,0,0.001505028,-0.01082044,0,0,0,0.0006799388,-0.002203598,-0.005162837,0,0.0139484,0,0,0,0.0003750346,-0.004413337,0.007187929,0,-0.01493915,0.004211367,-0.007211902,0,-0.02132823,0,0.001733127,-0.004824015]},{"f":[1,21,2,2,-1,-1,-1,16,22,-1,-1,22,-1,-1,4,-1,22,21,23,-1,-1,-1,32,17,-1,0,-1,-1,-1],"t":[2.01685992705101,7.5,9.221514256038809,9.075473573191719,0,0,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,0,12.199665042528586,8.5,2.5,0,0,0,6.208926257442949,2.69615871138938,0,8.5,0,0,0],"m":[0,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,0,15,0,17,18,19,0,0,0,23,24,0,26,0,0,0],"r":[14,7,6,5,0,0,0,11,10,0,0,13,0,0,16,0,22,21,20,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0.001071841,-0.005260897,0.008515776,0,0,0.004903847,-0.001911219,0,-0.01235507,0.001180854,0,0.0002477719,0,0,0,-0.0006684176,0.007248656,-0.01228555,0,0,0.003810277,0,0.007786059,-0.004079266,-0.004405532]},{"f":[33,2,16,17,-1,-1,15,22,21,-1,-1,-1,32,-1,22,33,32,-1,-1,-1,17,-1,-1,-1,15,-1,2,-1,-1],"t":[29.297756302860655,10.510264838663748,2.584962500721156,11.78143940991899,0,0,5.169925001442312,16.819976309430963,3.5,0,0,0,3.753897320099348,0,17.094543568094064,27.408661170593152,7.602820271428284,0,0,0,8.729619742794132,0,0,0,9.92481250360578,0,2.453445297804259,0,0],"m":[1,1,0,1,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,0,15,16,17,0,0,0,21,0,0,0,25,0,27,0,0],"r":[24,23,6,5,0,0,12,11,10,0,0,0,14,0,20,19,18,0,0,0,22,0,0,0,26,0,28,0,0],"v":[0,0,0,0,0.001388901,-0.01025748,0,0,0,0.0004850435,-0.002437618,-0.01139385,0,0.01321672,0,0,0,0.0003587073,-0.004109776,0.006823695,0,-0.01411461,0.004072127,-0.006751438,0,-0.02017845,0,0.001634424,-0.004481073]},{"f":[21,-1,2,1,-1,4,22,25,-1,-1,23,25,-1,-1,15,-1,15,-1,15,-1,-1,16,2,22,-1,-1,-1,-1,-1],"t":[2.5,0,8.9274737487839,4.92309205350118,0,4.80643424864552,12.121442107629743,0.5,0,0,3.5,1.5,0,0,6.044394119358453,0,6.584962500721156,0,8.618385502258606,0,0,6.882643049361842,5.8698903048866296,14.03140051979143,0,0,0,0,0],"m":[0,0,1,1,0,1,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,12,0,0,15,0,17,0,19,0,0,22,23,24,0,0,0,0,0],"r":[2,0,28,5,0,21,10,9,0,0,14,13,0,0,16,0,18,0,20,0,0,27,26,25,0,0,0,0,0],"v":[0,0.0009996875,0,0,0.0004428828,0,0,0,0.002592051,-0.009257566,0,0,-0.003094432,0.000810815,0,0.007034391,0,-0.008505137,0,0.004004897,-0.0009194988,0,0,0,0.0009088086,-0.01101109,0.007242427,-0.006014867,-0.003538759]},{"f":[2,1,2,2,21,-1,-1,-1,-1,4,18,4,36,-1,-1,-1,33,-1,32,-1,22,-1,-1,16,32,-1,-1,-1,-1],"t":[10.510264838663748,5.417948717948718,9.266718101290426,9.051177541662383,7.5,0,0,0,0,3.084962500721156,2.160964047443681,1.792481250360578,4.5,0,0,0,17.67212302715558,0,9.271599002932193,0,17.094543568094064,0,0,6.882643049361842,3.9474088816539714,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,4,5,0,0,0,0,10,11,12,13,0,0,0,17,0,19,0,21,0,0,24,25,0,0,0,0],"r":[28,9,8,7,6,0,0,0,0,23,16,15,14,0,0,0,18,0,20,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0.0007532304,-0.0009515619,-0.004837303,0.008293815,0,0,0,0,-0.003903181,0.002120943,0.002219091,0,0.01341943,0,-0.00402969,0,0.002114557,-0.007333309,0,0,0.004962241,-0.00316557,-0.003819077,-0.006248942]},{"f":[33,2,1,-1,16,-1,15,22,21,-1,-1,-1,3,-1,32,-1,22,2,-1,2,-1,-1,17,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,10.510264838663748,0.39811320754716983,0,2.584962500721156,0,5.169925001442312,16.819976309430963,3.5,0,0,0,0.030393996247654785,0,5.845435504646156,0,17.07907323966382,1.292481250360578,0,1.792481250360578,0,0,8.729619742794132,0,0,0,9.92481250360578,0,0],"m":[1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,9,0,0,0,13,0,15,0,17,18,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,4,0,6,0,12,11,10,0,0,0,14,0,16,0,22,19,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0.002790976,0,0.001128073,0,0,0,0.0003492307,-0.002246819,-0.01085951,0,-0.006975157,0,0.00886741,0,0,0.0009542799,0,-0.00368098,-0.0001658797,0,-0.01387025,0.001953063,-0.005938513,0,-0.01866556,-0.001272873]},{"f":[33,2,4,2,25,33,-1,22,27,-1,16,-1,15,-1,-1,16,-1,-1,-1,-1,-1,-1,15,-1,2,36,-1,-1,-1],"t":[29.297756302860655,10.510264838663748,9.266718101290426,9.051177541662383,6.5,22.05815630213804,0,17.094543568094064,0.5,0,6.930737337562887,0,7.451211111832329,0,0,6.539158811108031,0,0,0,0,0,0,9.92481250360578,0,2.453445297804259,3.5,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,11,0,13,0,0,16,0,0,0,0,0,0,23,0,25,26,0,0,0],"r":[22,21,20,19,18,7,0,15,10,0,12,0,14,0,0,17,0,0,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.001523565,0,0,0.003308941,0,0.0007008705,0,-0.002298461,0.0001921743,0,-0.0003739234,-0.01337694,0.009514322,-0.004055407,0.006891412,-0.005643505,0,-0.01774742,0,0,0.007150196,-0.001131879,-0.003900971]},{"f":[4,25,19,-1,2,36,-1,33,-1,33,-1,21,-1,-1,-1,-1,1,-1,1,-1,22,32,-1,17,-1,-1,32,-1,-1],"t":[3.9534452978042594,5.5,1.5,0,3.084962500721156,2.5,0,11.327499782570463,0,17.833701323482934,0,4.5,0,0,0,0,2.01685992705101,0,2.9733542319749215,0,12.199665042528586,8.341935934311302,0,10.543869317296533,0,0,6.208926257442949,0,0],"m":[1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,10,0,12,0,0,0,0,17,0,19,0,21,22,0,24,0,0,27,0,0],"r":[16,15,4,0,14,7,0,9,0,11,0,13,0,0,0,0,18,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0,0.000446794,0,0,0.005358727,0,-0.008957329,0,0.01143143,0,-0.002009784,0.001966057,-0.00426696,0.00872164,0,0.0008400854,0,-0.006513873,0,0,-0.004725644,0,0.002213686,-0.003893726,0,-0.0005258926,-0.004154154]},{"f":[33,2,1,-1,4,2,25,16,-1,15,22,-1,-1,32,-1,22,2,-1,-1,17,-1,-1,-1,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,10.510264838663748,0.39811320754716983,0,9.266718101290426,9.034782574103549,6.5,2.584962500721156,0,5.169925001442312,16.819976309430963,0,0,6.021684556978844,0,17.07907323966382,1.292481250360578,0,0,8.729619742794132,0,0,0,0,0,0,9.92481250360578,0,0],"m":[1,1,0,0,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,10,11,0,0,14,0,16,17,0,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,4,0,24,23,22,9,0,13,12,0,0,15,0,19,18,0,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0.002612715,0,0,0,0,0.001061694,0,0,-0.001243468,-0.01001092,0,0.005271949,0,0,0.000839201,-0.0005695343,0,-0.0127569,0.002292555,0.008616354,-0.004972703,0.006566009,-0.005231688,0,-0.01685008,-0.00114727]},{"f":[37,21,-1,22,21,2,17,-1,33,-1,-1,-1,23,-1,22,-1,22,-1,-1,0,-1,23,-1,2,-1,-1,21,-1,-1],"t":[11.26,2.5,0,12.199665042528586,6.5,6.451187557243013,9.075473573191719,0,35.98439522565276,0,0,0,1.5,0,8.133841411710264,0,11.415473622904775,0,0,2.5,0,3.5,0,4.0437314206251695,0,0,8.5,0,0],"m":[1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0],"l":[1,2,0,4,5,6,7,0,9,0,0,0,13,0,15,0,17,0,0,20,0,22,0,24,0,0,27,0,0],"r":[26,3,0,19,12,11,8,0,10,0,0,0,14,0,16,0,18,0,0,21,0,23,0,25,0,0,28,0,0],"v":[0,0,0.0007869015,0,0,0,0,0.00168839,0,0.004044465,-0.001560101,0.005703986,0,-0.001816922,0,0.01276758,0,-0.002226985,0.002175376,0,0.0008173675,0,-0.002226985,0,-0.0001956399,-0.001908261,0,-0.002226985,-0.02416666]},{"f":[33,2,1,-1,4,2,25,16,-1,15,22,-1,-1,32,-1,22,2,-1,-1,17,-1,-1,-1,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,10.510264838663748,0.39811320754716983,0,9.266718101290426,9.034782574103549,6.5,2.584962500721156,0,5.169925001442312,16.819976309430963,0,0,5.845435504646156,0,17.07907323966382,1.292481250360578,0,0,8.729619742794132,0,0,0,0,0,0,9.92481250360578,0,0],"m":[1,1,0,0,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,10,11,0,0,14,0,16,17,0,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,4,0,24,23,22,9,0,13,12,0,0,15,0,19,18,0,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0.002451664,0,0,0,0,0.0009746228,0,0,-0.001158073,-0.009431339,0,0.006930941,0,0,0.000825467,-0.0004877418,0,-0.01207556,0.002231383,0.008054735,-0.004623655,0.006203605,-0.004884785,0,-0.01553315,-0.001107599]},{"f":[1,21,-1,20,16,16,33,-1,0,-1,-1,-1,-1,15,-1,-1,16,16,-1,-1,22,25,-1,16,-1,-1,25,-1,-1],"t":[5.489664082687339,7.5,0,0.5,13.56045225766691,10.794415866350105,23.76434662895582,0,2.5,0,0,0,0,7.774787059601174,0,0,6.108524456778169,5.044394119358453,0,0,12.199665042528586,0.5,0,8.618385502258606,0,0,1.5,0,0],"m":[1,1,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,0,9,0,0,0,0,14,0,0,17,18,0,0,21,22,0,24,0,0,27,0,0],"r":[16,3,0,13,12,11,8,0,10,0,0,0,0,15,0,0,20,19,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0.0005680782,0,0,0,0,0.007122008,0,0.0003006681,-0.004260023,-0.006761044,0.006968803,0,0.001605728,-0.003212158,0,0,-0.001012244,0.007182276,0,0,0.001109443,0,-0.01789238,-0.00258908,0,-0.004105288,0.001269927]},{"f":[37,32,21,-1,22,21,17,4,-1,-1,22,-1,-1,-1,18,0,-1,-1,17,-1,32,-1,-1,-1,36,33,-1,-1,-1],"t":[8.05,10.41931383093672,2.5,0,12.199665042528586,6.5,9.075473573191719,5.599836172418183,0,0,11.876328901062104,0,0,0,7.519373778108656,1.5,0,0,8.571721503073015,0,8.341935934311302,0,0,0,5.5,13.895530557358477,0,0,0],"m":[1,1,0,0,0,0,1,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,0,11,0,0,0,15,16,0,0,19,0,21,0,0,0,25,26,0,0,0],"r":[24,23,4,0,14,13,10,9,0,0,12,0,0,0,18,17,0,0,20,0,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0.0007275386,0,0,0,0,0.001690162,0.007337166,0,-0.0008277944,0.003586534,-0.0007600075,0,0,0.001105116,-0.0008277944,0,0.002775703,0,0.01170319,-0.0008277944,0.006034722,0,0,-0.005809022,-0.0008277944,-0.01536133]},{"f":[4,25,-1,-1,1,22,-1,22,-1,22,-1,-1,1,-1,22,21,17,-1,-1,-1,32,-1,2,1,22,-1,-1,-1,-1],"t":[3.9534452978042594,5.5,0,0,2.01685992705101,7.499052785859982,0,8.133841411710264,0,8.882387211598338,0,0,2.9733542319749215,0,12.199665042528586,8.5,10.543869317296533,0,0,0,6.208926257442949,0,6.011183906514227,4.031939779270633,16.819976309430963,0,0,0,0],"m":[1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,0,0,5,6,0,8,0,10,0,0,13,0,15,16,17,0,0,0,21,0,23,24,25,0,0,0,0],"r":[4,3,0,0,12,7,0,9,0,11,0,0,14,0,20,19,18,0,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0.0001986951,0.007290981,0,0,-0.007019721,0,0.01019044,0,-0.007760467,0.001094859,0,-0.005800614,0,0,0,0.001646995,-0.002989115,-0.0105119,0,-0.0002854441,0,0,0,-0.001541281,-0.01183116,-0.009389766,-0.002030217]},{"f":[33,2,4,2,25,22,-1,1,-1,16,15,-1,-1,18,-1,15,-1,-1,-1,-1,-1,-1,15,-1,2,36,-1,-1,-1],"t":[29.297756302860655,10.510264838663748,9.266718101290426,9.051177541662383,6.5,12.199665042528586,0,4.92309205350118,0,6.044394119358453,5.044394119358453,0,0,1.292481250360578,0,7.118941072723508,0,0,0,0,0,0,9.92481250360578,0,2.453445297804259,3.5,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,10,11,0,0,14,0,16,0,0,0,0,0,0,23,0,25,26,0,0,0],"r":[22,21,20,19,18,7,0,9,0,13,12,0,0,15,0,17,0,0,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0.0007124532,0,0.0003326575,0,0,-0.002615865,0.002131132,0,-0.0008685852,0,-0.009300819,-0.002681429,0.007193294,-0.003279583,0.005917901,-0.004404349,0,-0.01397297,0,0,0.006508585,-0.001023271,-0.003387104]},{"f":[22,37,23,1,-1,22,4,-1,-1,-1,4,2,19,-1,-1,-1,-1,21,-1,-1,34,36,17,17,-1,-1,-1,-1,-1],"t":[17.094543568094064,11.26,3.5,2.01685992705101,0,12.199665042528586,4.951187557243013,0,0,0,6.451187557243013,4.0437314206251695,0.5,0,0,0,0,8.5,0,0,1.5,4.5,8.729619742794132,8.076805701955248,0,0,0,0,0],"m":[1,1,1,0,0,0,1,0,0,0,1,1,1,0,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,0,11,12,13,0,0,0,0,18,0,0,21,22,23,24,0,0,0,0,0],"r":[20,17,10,5,0,9,8,0,0,0,16,15,14,0,0,0,0,19,0,0,28,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0.0004954022,0,0,0.000848469,-0.001507728,-0.001709019,0,0,0,0.0004988783,0.003394667,-0.00165446,0.007306584,0,-0.001933888,-0.02086708,0,0,0,0,0.0005464116,-0.02393084,0.00536996,0.008909601,-0.0129082]},{"f":[1,-1,2,23,22,21,-1,-1,-1,2,-1,-1,22,21,16,2,-1,-1,-1,24,22,-1,-1,-1,22,-1,32,-1,-1],"t":[0.39811320754716983,0,4.0437314206251695,3.5,16.978341770814396,6.5,0,0,0,2.69615871138938,0,0,16.765402642760364,8.5,1.584962500721156,8.9274737487839,0,0,0,5.5,10.224917391507956,0,0,0,16.786010507999382,0,6.337096134072842,0,0],"m":[0,0,1,1,1,1,0,0,0,1,0,0,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,0,0,13,14,15,16,0,0,0,20,21,0,0,0,25,0,27,0,0],"r":[2,0,12,9,8,7,0,0,0,11,0,0,24,19,18,17,0,0,0,23,22,0,0,0,26,0,28,0,0],"v":[0,0.00191388,0,0,0,0,0.0002957072,-0.001729166,-0.006930577,0,0.0008949916,0.006127296,0,0,0,0,0.006992286,-0.002910239,-0.0003879226,0,0,-0.01088958,-0.0004722067,-0.01683201,0,-0.02744455,0,0.004263812,-0.003101888]},{"f":[21,-1,37,22,16,-1,-1,23,1,-1,-1,4,2,33,-1,17,19,21,-1,-1,-1,-1,-1,3,-1,-1,21,-1,-1],"t":[2.5,0,11.26,12.199665042528586,10.501837184902296,0,0,3.5,1.5277777777777777,0,0,6.451187557243013,4.0437314206251695,22.152182868153965,0,8.496347487943302,0.5,7.5,0,0,0,0,0,0.99414994940175,0,0,7.5,0,0],"m":[0,0,1,0,1,0,0,1,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,14,0,16,17,18,0,0,0,0,0,24,0,0,27,0,0],"r":[2,0,26,7,6,0,0,11,10,0,0,23,22,15,0,21,20,19,0,0,0,0,0,25,0,0,28,0,0],"v":[0,0.0006483386,0,0,0,0.0008827369,-0.001535239,0,0,0.0008352234,-0.002051928,0,0,0,0.01117172,0,0,0,0.001963069,-0.001110176,0.003162222,-0.002200939,-0.002011756,0,-0.0002649433,0.01182671,0,-0.002200939,-0.01805595]},{"f":[22,2,4,2,25,37,32,23,1,-1,-1,4,-1,-1,-1,21,-1,-1,-1,-1,-1,-1,32,2,16,-1,-1,-1,-1],"t":[17.094543568094064,10.510264838663748,9.266718101290426,9.051177541662383,6.5,11.26,10.463013322934223,3.5,5.489664082687339,0,0,6.451187557243013,0,0,0,6.5,0,0,0,0,0,0,8.341935934311302,1.292481250360578,6.539158811108031,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,12,0,0,0,16,0,0,0,0,0,0,23,24,25,0,0,0,0],"r":[22,21,20,19,18,15,14,11,10,0,0,13,0,0,0,17,0,0,0,0,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0.0002171799,-0.001321003,0,0.0006050795,0.007868288,0.004187485,0,-0.001975396,-0.01806708,0.006458903,-0.003053104,0.005508971,-0.004069214,0,0,0,6.061457e-06,-0.01021453,0.00533298,-0.01104303]},{"f":[33,23,4,-1,22,15,16,15,15,17,-1,-1,-1,-1,-1,4,-1,25,-1,-1,-1,-1,15,-1,2,36,-1,-1,-1],"t":[29.297756302860655,8.5,4.0437314206251695,0,16.725912049439096,10.376125389276174,7.266786540694901,5.78135971352466,5.672425341971495,10.269270673170688,0,0,0,0,0,4.128693921346326,0,0.5,0,0,0,0,9.92481250360578,0,2.453445297804259,3.5,0,0,0],"m":[1,1,1,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,9,10,0,0,0,0,0,16,0,18,0,0,0,0,23,0,25,26,0,0,0],"r":[22,21,4,0,20,15,14,13,12,11,0,0,0,0,0,17,0,19,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0.0002478288,0,0,0,0,0,0,0.0003624386,-0.004614507,-0.006174394,0.003151608,-0.003116141,0,-0.005455446,0,0.01123074,0.001970949,-0.003764736,0.005835956,0,-0.01197659,0,0,0.00606068,-0.00105788,-0.003042919]},{"f":[22,2,4,2,25,37,32,23,-1,4,-1,-1,-1,21,-1,-1,-1,-1,-1,-1,18,32,-1,17,26,-1,-1,-1,-1],"t":[17.094543568094064,10.510264838663748,9.266718101290426,9.051177541662383,6.5,11.26,10.463013322934223,3.5,0,6.451187557243013,0,0,0,6.5,0,0,0,0,0,0,8.729619742794132,8.341935934311302,0,8.571721503073015,0.5,0,0,0,0],"m":[1,1,1,1,1,1,1,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,10,0,0,0,14,0,0,0,0,0,0,21,22,0,24,25,0,0,0,0],"r":[20,19,18,17,16,13,12,9,0,11,0,0,0,15,0,0,0,0,0,0,28,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,0,0,-9.325821e-05,0,0.0005834576,0.007419645,0.003912122,0,-0.00182454,-0.01673545,0.0061287,-0.002883515,0.005282127,-0.003836576,0,0,0.003162166,0,0,-0.009270798,0.0007116455,-0.01929915,0.005504659]},{"f":[1,22,-1,-1,4,25,-1,-1,22,24,-1,21,16,2,-1,-1,15,16,-1,17,-1,-1,-1,37,-1,-1,4,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,4.0437314206251695,6.5,0,0,16.725912049439096,3.5,0,8.5,1.584962500721156,8.9274737487839,0,0,10.376125389276174,7.266786540694901,0,8.025009334276167,0,0,0,6.05,0,0,6.434604125275413,0,0],"m":[0,1,0,0,1,1,0,0,1,0,0,1,0,1,0,0,1,1,0,1,0,0,0,1,0,0,1,0,0],"l":[1,2,0,0,5,6,0,0,9,10,0,12,13,14,0,0,17,18,0,20,0,0,0,24,0,0,27,0,0],"r":[4,3,0,0,8,7,0,0,26,11,0,23,16,15,0,0,22,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0.001304559,0.01016061,0,0,0.0001081318,0.006181318,0,0,0.007538666,0,0,0,0.006555191,-0.002557596,0,0,-0.0001690259,0,-0.005093689,-0.0005222063,0.001068374,0,-0.002514307,-0.01132932,0,-0.004450536,0.005251337]},{"f":[21,-1,2,4,1,-1,16,16,-1,-1,16,17,-1,17,16,-1,-1,-1,4,-1,22,-1,-1,-1,17,16,-1,-1,-1],"t":[2.5,0,8.9274737487839,7.997154617983022,4.92309205350118,0,6.044394119358453,5.78135971352466,0,0,6.832890014164741,6.434604125275413,0,8.729619742794132,6.247927513443585,0,0,0,1.292481250360578,0,11.406459975416917,0,0,0,12.18494601563336,11.868050853745158,0,0,0],"m":[0,0,1,1,1,0,0,1,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,0,7,8,0,0,11,12,0,14,15,0,0,0,19,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,23,6,0,10,9,0,0,18,13,0,17,16,0,0,0,20,0,22,0,0,0,28,27,0,0,0],"v":[0,0.0006011699,0,0,0,0.0002071042,0,0,-0.0003854929,0.005081981,0,0,-0.001190481,0,0,-0.006331103,-0.02530961,0.003152736,0,0.001589913,0,0.0001301891,-0.003396627,0.0097837,0,0,-0.002266734,0.006799971,-0.009500039]},{"f":[33,23,22,33,16,-1,1,-1,-1,-1,22,15,16,22,36,-1,-1,-1,-1,-1,-1,-1,15,-1,2,36,-1,-1,-1],"t":[29.297756302860655,8.5,12.199665042528586,23.925614337747046,10.462502272597265,0,1.000936329588015,0,0,0,12.921158001713904,10.457380879072534,4.247927513443585,12.613886328427084,2.5,0,0,0,0,0,0,0,9.92481250360578,0,2.453445297804259,3.5,0,0,0],"m":[1,1,0,1,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,12,13,14,15,0,0,0,0,0,0,0,23,0,25,26,0,0,0],"r":[22,21,10,9,6,0,8,0,0,0,20,19,18,17,16,0,0,0,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0.0007762797,0,-0.010548,-0.000545887,0.006018649,0,0,0,0,0,0.00556791,-0.002257897,-0.00718133,-0.00560397,0.003895453,-3.079856e-05,0.005495829,0,-0.01075617,0,0,0.005631557,-0.001046427,-0.002839746]},{"f":[1,22,-1,-1,4,25,-1,-1,22,24,-1,21,16,2,-1,-1,15,16,-1,-1,-1,22,-1,37,-1,-1,4,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,4.0437314206251695,6.5,0,0,16.725912049439096,3.5,0,8.5,1.584962500721156,8.9274737487839,0,0,10.376125389276174,7.266786540694901,0,0,0,10.224917391507956,0,6.05,0,0,6.434604125275413,0,0],"m":[0,1,0,0,1,1,0,0,1,0,0,1,0,1,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,0,0,5,6,0,0,9,10,0,12,13,14,0,0,17,18,0,0,0,22,0,24,0,0,27,0,0],"r":[4,3,0,0,8,7,0,0,26,11,0,21,16,15,0,0,20,19,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0.001245427,0.009588393,0,0,9.64203e-05,0.005846403,0,0,0.007183351,0,0,0,0.006121296,-0.002214316,0,0,-0.0001676299,-0.002872553,0.001068845,0,-0.01054629,0,-0.0004329909,-0.01014548,0,-0.004197824,0.004970097]},{"f":[9,4,-1,2,18,-1,-1,32,15,15,16,3,-1,17,21,-1,-1,-1,-1,-1,16,-1,-1,25,22,-1,-1,-1,-1],"t":[0.5,4.0437314206251695,0,4.128693921346326,5.918419679874358,0,0,6.208926257442949,5.78135971352466,5.672425341971495,6.409390936137702,0.5803571428571428,0,10.269270673170688,8.5,0,0,0,0,0,6.6865005271832185,0,0,1.5,15.439474430023438,0,0,0,0],"m":[1,1,0,0,1,0,0,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,10,11,12,0,14,15,0,0,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,3,0,7,6,0,0,23,20,19,18,13,0,17,16,0,0,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0.0001500812,0,0,-0.005064084,0.002398397,0,0,0,0,0,0.004552382,0,0,0.0006636755,-0.004530538,-0.003800189,-0.002865784,-0.005597185,0,0.005666625,0.0001265824,0,0,-0.0003221986,-0.008724003,-0.01189795,0.002598201]},{"f":[21,17,-1,-1,23,1,-1,22,21,-1,4,-1,36,-1,15,-1,-1,-1,4,2,2,17,-1,-1,-1,-1,3,-1,-1],"t":[2.5,10.090748425878532,0,0,3.5,2.01685992705101,0,12.199665042528586,3.5,0,1.292481250360578,0,3.5,0,9.92481250360578,0,0,0,6.451187557243013,4.0437314206251695,2.69615871138938,8.496347487943302,0,0,0,0,0.8775684931506849,0,0],"m":[0,1,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,0,8,9,0,11,0,13,0,15,0,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[4,3,0,0,18,7,0,17,10,0,12,0,14,0,16,0,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0.0004265545,0.004202454,0,0,0.0003790881,0,0,0.003885634,0,0.001199864,0,-0.0006461387,0,-0.01269688,-0.002636263,-0.002047483,0,0,0,0,0.001088166,-0.004428085,0.00549888,-0.001631657,0,-0.002583763,0.007803228]},{"f":[23,22,1,-1,3,22,22,-1,-1,-1,23,22,1,-1,22,-1,-1,15,-1,-1,-1,34,18,15,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,0.39811320754716983,0,0.05927318295739348,16.765402642760364,16.73920708776685,0,0,0,3.5,16.978341770814396,5.489664082687339,0,15.717370311433935,0,0,6.0,0,0,0,1.5,8.729619742794132,6.584962500721156,0,0,0,0,0],"m":[1,1,0,0,0,1,1,0,0,0,1,1,1,0,1,0,0,1,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,11,12,13,0,15,0,0,18,0,0,0,22,23,24,0,0,0,0,0],"r":[28,21,4,0,10,9,8,0,0,0,20,17,14,0,16,0,0,19,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0.001508723,0,0,0,-0.001610429,0.004631914,-0.02330886,0,0,0,0.0001542587,0,-0.0007078582,-0.00654202,0,-0.01135952,0.005269829,0.0008519239,0,0,0,-0.01059023,0.002315926,0.006558252,-0.01062329,0.004780946]},{"f":[21,-1,2,4,2,4,23,16,22,4,-1,-1,15,-1,32,18,-1,-1,-1,-1,-1,-1,22,3,-1,-1,-1,-1,-1],"t":[2.5,0,10.510264838663748,9.266718101290426,8.9274737487839,7.997154617983022,0.5,10.501837184902296,13.415167820124083,7.044021426237043,0,0,6.0,0,7.014643613484123,1.292481250360578,0,0,0,0,0,0,16.765402642760364,0.008766718459616732,0,0,0,0,0],"m":[0,0,1,1,1,1,0,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,10,0,0,13,0,15,16,0,0,0,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,27,22,21,20,19,12,11,0,0,14,0,18,17,0,0,0,0,0,0,26,25,0,0,0,0,0],"v":[0,0.0005243369,0,0,0,0,0,0,0,0,0.0002513836,-0.009223684,0,0.00157535,0,0,-0.002771734,-0.00809283,0.003625687,-0.004581997,0.0001705055,0.009019764,0,0,0.002139303,-0.003640743,-0.01084763,0.006449432,-0.003942209]},{"f":[33,33,33,33,24,-1,22,-1,-1,25,1,-1,-1,24,-1,4,-1,-1,33,-1,-1,22,17,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,23.58371303411409,20.752541988539424,13.895530557358477,4.5,0,12.613886328427084,0,0,0.5,7.0202978971962615,0,0,6.5,0,3.9534452978042594,0,0,21.16864017939033,0,0,16.838461650489297,7.781300095563516,0,0,0,9.92481250360578,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,0,16,0,0,19,0,0,22,23,0,0,0,27,0,0],"r":[26,21,18,9,6,0,8,0,0,13,12,0,0,15,0,17,0,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,5.929308e-05,0,-0.0001950208,-0.005759816,0,0,0.002807465,-0.005453767,0,0.001351511,0,0.01936757,0.003057229,0,-0.009129101,-0.001318068,0,0,0.002198293,0.007807967,-0.00260431,0,-0.009764884,-0.0007906171]},{"f":[23,22,9,2,25,37,-1,-1,-1,22,17,22,-1,-1,17,-1,22,-1,-1,-1,-1,34,18,15,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,0.5,9.051177541662383,6.5,11.26,0,0,0,16.752707246237605,9.229707804263573,11.681676585676911,0,0,10.460454442916898,0,12.457649623491124,0,0,0,0,1.5,8.729619742794132,6.584962500721156,0,0,0,0,0],"m":[1,1,1,1,1,1,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,12,0,0,15,0,17,0,0,0,0,22,23,24,0,0,0,0,0],"r":[28,21,20,9,8,7,0,0,0,19,14,13,0,0,16,0,18,0,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0,5.271143e-05,-0.003322685,0.005068241,0,0,0,-0.0008451611,-0.006473262,0,0.008714518,0,-0.006371915,0.0004759029,-0.01040257,0.002322836,0,0,0,-0.009988319,0.002066623,0.005950914,-0.01002393,0.004625229]},{"f":[23,21,23,22,-1,1,-1,15,17,-1,-1,33,-1,23,15,-1,1,-1,-1,22,-1,-1,27,-1,-1,37,-1,-1,-1],"t":[8.5,6.5,3.5,13.87915215758784,0,2.532051282051282,0,5.044394119358453,4.583709072915869,0,0,17.460839126440824,0,0.5,6.0,0,6.015625,0,0,16.978341770814396,0,0,0.5,0,0,11.05,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,9,0,0,12,0,14,15,0,17,0,0,20,0,0,23,0,0,26,0,0,0],"r":[28,25,22,5,0,7,0,11,10,0,0,13,0,19,16,0,18,0,0,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0.0003278223,0,0.002096721,0,0,-0.003957058,-0.01296868,0,0.00876085,0,0,0.004368465,0,-0.008734542,-0.00317443,0,0.0006059423,-0.006422286,0,0.003237538,-0.006090678,0,-0.0004226292,-0.01514355,0.004396751]},{"f":[9,4,-1,2,18,-1,-1,2,-1,32,16,-1,1,15,16,15,-1,-1,-1,-1,33,-1,-1,25,22,-1,-1,-1,-1],"t":[0.5,4.0437314206251695,0,4.128693921346326,5.918419679874358,0,0,4.80643424864552,0,6.208926257442949,1.584962500721156,0,6.241071428571429,10.043027283594547,7.651051691178929,6.022367813028454,0,0,0,0,8.065928480304397,0,0,1.5,15.439474430023438,0,0,0,0],"m":[1,1,0,0,1,0,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,5,0,0,8,0,10,11,0,13,14,15,16,0,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,3,0,7,6,0,0,9,0,23,12,0,20,19,18,17,0,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0.0001399263,0,0,-0.004712361,0.002222478,0,0.003396663,0,0,0.002633634,0,0,0,0,-0.000602017,0.003283105,-0.002578898,0.001622494,0,-0.01016229,-0.001963616,0,0,-0.0004682529,-0.007479389,-0.01072496,0.002275016]},{"f":[1,22,-1,-1,3,22,22,-1,-1,-1,4,-1,22,-1,22,-1,20,22,-1,17,-1,-1,15,21,17,-1,-1,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,0.05927318295739348,16.765402642760364,16.73920708776685,0,0,0,4.951187557243013,0,7.499052785859982,0,8.32532955673354,0,0.5,8.882387211598338,0,11.78143940991899,0,0,9.92481250360578,7.5,7.080487187463968,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,0,13,0,15,0,17,18,0,20,0,0,23,24,25,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,12,0,14,0,16,0,22,19,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,0.0009699434,0.00866803,0,0,0,-0.001526242,0.004514474,-0.02185101,0,0.0001533866,0,-0.006224256,0,0.008686094,0,0,-0.008579685,0,0.0005215116,-0.007499898,0,0,0,-0.00212912,-0.009693096,-0.001318355,0.002333094]},{"f":[33,33,33,33,24,-1,22,-1,-1,25,1,-1,-1,24,-1,4,-1,-1,33,-1,-1,22,17,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,23.58371303411409,20.752541988539424,13.895530557358477,4.5,0,12.613886328427084,0,0,0.5,7.0202978971962615,0,0,6.5,0,3.9534452978042594,0,0,21.16864017939033,0,0,16.838461650489297,7.781300095563516,0,0,0,9.92481250360578,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,0,16,0,0,19,0,0,22,23,0,0,0,27,0,0],"r":[26,21,18,9,6,0,8,0,0,13,12,0,0,15,0,17,0,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,4.700283e-05,0,-5.867216e-05,-0.005430246,0,0,0.002668604,-0.004832824,0,0.001331974,0,0.01809867,0.002932088,0,-0.008466775,-0.001194389,0,0,0.002156275,0.007429345,-0.002351484,0,-0.008954511,-0.0007174627]},{"f":[23,22,2,1,2,-1,-1,3,22,22,-1,-1,-1,23,22,-1,-1,-1,-1,32,-1,17,17,26,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,10.655783256170313,0.39811320754716983,6.08730682361797,0,0,0.05927318295739348,16.765402642760364,16.73920708776685,0,0,0,3.5,16.978341770814396,0,0,0,0,8.341935934311302,0,8.729619742794132,8.076805701955248,0.5,0,0,0,0,0],"m":[1,1,1,0,1,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,10,0,0,0,14,15,0,0,0,0,20,0,22,23,24,0,0,0,0,0],"r":[28,19,18,7,6,0,0,13,12,11,0,0,0,17,16,0,0,0,0,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.0007651724,0.007164905,0,0,0,-0.001447314,0.004309099,-0.02077402,0,0,-8.806794e-05,-0.005350104,0.0007921799,-0.003309924,0,0.003772032,0,0,0,-0.007594293,0.0006331756,-0.01997443,0.001945778,0.004216414]},{"f":[21,17,-1,-1,23,17,22,-1,15,-1,32,-1,-1,-1,18,17,22,-1,21,-1,19,-1,-1,17,-1,22,-1,-1,-1],"t":[2.5,10.090748425878532,0,0,0.5,10.543869317296533,13.415167820124083,0,6.0,0,7.014643613484123,0,0,0,7.033254845462222,4.0437314206251695,8.133841411710264,0,7.5,0,0.5,0,0,5.918419679874358,0,14.116600814324018,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,0,0,0,15,16,17,0,19,0,21,0,0,24,0,26,0,0,0],"r":[4,3,0,0,14,13,8,0,10,0,12,0,0,0,28,23,18,0,20,0,22,0,0,25,0,27,0,0,0],"v":[0,0,0.0003418093,0.003862594,0,0,0,0.0001616235,0,0.001334532,0,-0.00480974,0.00327652,-0.004224149,0,0,0,0.01078442,0,0.001105006,0,-0.002661307,0.004180352,0,-0.003270458,0,0.000643998,-0.001990601,0.001083208]},{"f":[33,33,33,33,24,-1,-1,33,2,-1,-1,33,-1,-1,-1,22,21,-1,-1,32,-1,23,-1,-1,15,-1,22,-1,-1],"t":[29.297756302860655,23.58371303411409,20.752541988539424,13.895530557358477,4.5,0,0,17.460839126440824,5.538407798525416,0,0,18.93170853178854,0,0,0,15.439474430023438,6.5,0,0,6.337096134072842,0,7.5,0,0,9.92481250360578,0,9.945253865134848,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,12,0,0,0,16,17,0,0,20,0,22,0,0,25,0,27,0,0],"r":[24,15,14,7,6,0,0,11,10,0,0,13,0,0,0,19,18,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,4.314559e-05,-0.002764437,0,0,0.01014657,0.001390744,0,-0.001941549,0.006371583,-0.001907509,0,0,0.007328932,0.0004279037,0,0.002354332,0,-0.009641126,-0.0007013178,0,-0.008591856,0,0.002336876,-0.001814009]},{"f":[22,22,22,37,32,16,15,-1,33,-1,-1,2,-1,-1,-1,21,-1,-1,-1,15,-1,1,-1,-1,17,17,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.978341770814396,11.26,10.463013322934223,7.266786540694901,5.78135971352466,0,19.167172160526107,0,0,1.292481250360578,0,0,0,7.5,0,0,0,6.0,0,4.008411214953271,0,0,8.729619742794132,8.571721503073015,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,0,12,0,0,0,16,0,0,0,20,0,22,0,0,25,26,0,0,0],"r":[24,19,18,15,14,11,8,0,10,0,0,13,0,0,0,17,0,0,0,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.000155136,0,0.006947173,0.0008623198,0,0.0006292189,-0.0008782106,0.003601141,0,-0.001392002,-0.01448979,-0.005453681,0,-0.007613949,0,-2.627749e-06,0.00361854,0,0,-0.001379502,-0.01120737,0.002764111]},{"f":[23,21,23,22,-1,-1,-1,37,22,22,-1,-1,22,-1,18,18,18,4,0,-1,-1,-1,-1,-1,22,-1,-1,-1,-1],"t":[8.5,6.5,3.5,13.87915215758784,0,0,0,11.05,7.5621616439616925,6.671801068872866,0,0,8.133841411710264,0,10.651680794759452,6.584884225259038,5.303665156874805,1.792481250360578,2.5,0,0,0,0,0,10.305472761501687,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,0,0,13,0,15,16,17,18,19,0,0,0,0,0,25,0,0,0,0],"r":[28,7,6,5,0,0,0,27,12,11,0,0,14,0,24,23,22,21,20,0,0,0,0,0,26,0,0,0,0],"v":[0,0,0,0,0.0002824937,-0.0008827823,0.002382819,0,0,0,0.001112901,-0.005268795,0,0.008136677,0,0,0,0,0,0.001587054,-0.002304918,0.0005089214,-0.003515407,0.001417443,0,0.001283972,-0.004576182,-0.01311168,0.00401676]},{"f":[9,4,-1,2,22,-1,-1,15,16,17,4,-1,20,15,15,-1,-1,-1,35,-1,-1,-1,32,-1,-1,16,-1,-1,-1],"t":[0.5,4.0437314206251695,0,4.128693921346326,11.876328901062104,0,0,10.376125389276174,7.651051691178929,11.503187353537466,4.80643424864552,0,1.5,5.78135971352466,5.672425341971495,0,0,0,2.5,0,0,0,6.474913355379556,0,0,10.486835021563051,0,0,0],"m":[1,1,0,0,0,0,0,1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,10,11,0,13,14,15,0,0,0,19,0,0,0,23,0,0,26,0,0,0],"r":[28,3,0,7,6,0,0,25,22,21,12,0,18,17,16,0,0,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0.0001224165,0,0,0.001455868,-0.004570556,0,0,0,0,0.003760708,0,0,0,-0.0001049058,-0.004247879,0.003187398,0,-0.007661725,-0.001041668,-0.008511198,0,-0.002015208,-0.01035709,0,0.005428551,0.0001698345,0.002017734]},{"f":[33,32,33,33,33,33,-1,25,-1,24,-1,4,-1,-1,-1,33,-1,22,17,-1,-1,-1,1,-1,-1,-1,2,-1,-1],"t":[37.98858650139573,10.463013322934223,29.569691584072146,23.58371303411409,20.752541988539424,13.895530557358477,0,0.5,0,6.5,0,3.9534452978042594,0,0,0,24.145246550704165,0,15.439474430023438,7.198837316474133,0,0,0,9.107199999999999,0,0,0,2.453445297804259,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,10,0,12,0,0,0,16,0,18,19,0,0,0,23,0,0,0,27,0,0],"r":[26,25,22,15,14,7,0,9,0,11,0,13,0,0,0,17,0,21,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-2.022886e-05,0,-0.0005912325,0,0.001127476,0,0.01660061,0.002576072,-0.002039654,0,0.007614003,0,0,0.001107133,0.008363542,-0.002015461,0,-0.0002151488,-0.007497097,0.004744692,0,0.0003507168,-0.004360782]},{"f":[23,22,22,22,16,-1,2,17,-1,-1,1,-1,-1,22,-1,1,-1,-1,22,-1,-1,32,-1,17,17,-1,-1,-1,-1],"t":[8.5,17.094543568094064,17.069460217922263,16.978341770814396,7.266786540694901,0,1.292481250360578,10.620215166896223,0,0,4.92309205350118,0,0,16.98920034885663,0,4.008411214953271,0,0,17.078739781790915,0,0,8.341935934311302,0,8.729619742794132,8.076805701955248,0,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,11,0,0,14,0,16,0,0,19,0,0,22,0,24,25,0,0,0,0],"r":[28,21,18,13,6,0,10,9,0,0,12,0,0,15,0,17,0,0,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0.0002578127,0,0,0.001029445,-0.003076566,0,-7.239991e-06,-0.00211741,0,-0.005784198,0,-0.002436457,0.003022392,0,0.004025647,-0.0001478317,0,0.003215843,0,0,-0.001927052,-0.01814494,0.001914057,0.003857838]},{"f":[9,2,4,2,25,4,4,-1,22,-1,22,-1,22,16,-1,-1,1,-1,-1,18,-1,22,-1,-1,-1,-1,-1,-1,-1],"t":[0.5,10.510264838663748,9.266718101290426,9.051177541662383,6.5,7.950787011501063,4.951187557243013,0,7.499052785859982,0,8.133841411710264,0,9.40390385294334,9.207014320177532,0,0,6.241071428571429,0,0,1.292481250360578,0,12.35227706330556,0,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,11,0,13,14,0,0,17,0,0,20,0,22,0,0,0,0,0,0,0],"r":[28,27,26,25,24,19,8,0,10,0,12,0,16,15,0,0,18,0,0,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,9.675194e-05,0,-0.004952584,0,0.007284744,0,0,-0.01057729,-0.0007084206,0,0.0002194433,-0.00314305,0,0.008738726,0,-0.005298821,0.005292597,0.004901166,-0.002534666,0.004478233,-0.003200173,0.001916448]},{"f":[23,22,16,-1,37,15,22,-1,-1,25,16,22,-1,-1,22,-1,-1,-1,-1,32,-1,17,17,26,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,1.584962500721156,0,12.52,5.78135971352466,14.116600814324018,0,0,2.5,6.108524456778169,16.964783007086943,0,0,16.786010507999382,0,0,0,0,8.341935934311302,0,8.729619742794132,8.076805701955248,0.5,0,0,0,0,0],"m":[1,1,0,0,1,0,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,12,0,0,15,0,0,0,0,20,0,22,23,24,0,0,0,0,0],"r":[28,19,4,0,18,9,8,0,0,17,14,13,0,0,16,0,0,0,0,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0.0007184988,0,0,0,-0.0002756943,-0.002381241,0,0,0,0.002751195,-0.002592785,0,-0.000156794,0.001520778,0.008629519,-0.005130602,0,0.0031251,0,0,0,-0.006838514,0.0007578724,-0.01722602,0.001859946,0.003682919]},{"f":[21,16,-1,-1,23,17,22,-1,15,-1,32,18,-1,1,-1,-1,-1,-1,18,17,22,-1,21,-1,19,-1,-1,-1,-1],"t":[2.5,10.066089190457772,0,0,0.5,10.543869317296533,13.415167820124083,0,6.0,0,7.014643613484123,1.292481250360578,0,5.015625,0,0,0,0,6.65017628016411,4.0437314206251695,8.133841411710264,0,7.5,0,0.5,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,1,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,12,0,14,0,0,0,0,19,20,21,0,23,0,25,0,0,0,0],"r":[4,3,0,0,18,17,8,0,10,0,16,13,0,15,0,0,0,0,28,27,22,0,24,0,26,0,0,0,0],"v":[0,0,0.0002885846,0.003985366,0,0,0,0.0001313876,0,0.001221423,0,0,-0.002147747,0,-0.01250567,-0.004725385,0.003115967,-0.003894374,0,0,0,0.00962093,0,0.00100448,0,-0.002432377,0.003791176,-0.001118853,0.0008390306]},{"f":[22,15,2,-1,24,-1,17,-1,-1,17,1,-1,-1,-1,22,15,16,36,22,-1,-1,-1,-1,-1,4,-1,3,-1,-1],"t":[12.199665042528586,10.501837184902296,1.292481250360578,0,9.5,0,7.5507416557826375,0,0,10.269270673170688,1.3336158192090395,0,0,0,12.921158001713904,10.457380879072534,9.047123912114026,2.5,12.613886328427084,0,0,0,0,0,7.950787011501063,0,0.9640400250156348,0,0],"m":[0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,0,0,0,15,16,17,18,19,0,0,0,0,0,25,0,27,0,0],"r":[14,9,4,0,6,0,8,0,0,13,12,0,0,0,24,23,22,21,20,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0.001301026,0,0.0002070964,0,-0.00859763,-0.0007894666,0,0,-0.01114649,0.002022127,-0.00754695,0,0,0,0,0,0.005519477,-0.008970404,-0.002985038,-0.01081462,0.003236348,0,-7.603559e-05,0,-0.001200621,0.007371188]},{"f":[32,-1,37,32,1,36,-1,21,23,3,-1,1,-1,-1,22,2,-1,-1,-1,-1,21,-1,-1,-1,21,2,-1,-1,-1],"t":[2.69615871138938,0,11.26,10.463013322934223,5.417948717948718,4.5,0,7.5,0.5,0.9992038216560509,0,3.354166666666667,0,0,16.819976309430963,2.453445297804259,0,0,0,0,9.5,0,0,0,8.5,2.453445297804259,0,0,0],"m":[0,0,1,1,1,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,0,12,0,0,15,16,0,0,0,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,23,20,7,0,19,14,11,0,13,0,0,18,17,0,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,0.004040335,0,0,0,0,0.002027492,0,0,0,0.006140259,0,0.0001323856,-0.001241057,0,0,0.001839137,0.0004036889,-0.001241057,-0.000989651,0,-0.0006685517,0.00336795,0.003258862,0,0,-0.001241057,-0.004854798,-0.01207521]},{"f":[23,22,22,22,16,-1,2,-1,1,-1,-1,22,-1,1,-1,-1,22,-1,1,-1,-1,25,17,17,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,17.069460217922263,16.978341770814396,7.266786540694901,0,1.292481250360578,0,4.92309205350118,0,0,16.98920034885663,0,4.008411214953271,0,0,17.078739781790915,0,3.966632962588473,0,0,1.5,8.729619742794132,8.571721503073015,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,9,0,0,12,0,14,0,0,17,0,19,0,0,22,23,24,0,0,0,0,0],"r":[28,21,16,11,6,0,8,0,10,0,0,13,0,15,0,0,18,0,20,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.0002362128,0,0.000586864,0,-1.089922e-05,-0.001923253,0,-0.005473053,0,-0.002314521,0.002857811,0,0.003816348,0,-0.002907632,0.0016019,0,0,0,-0.0009098778,-0.01615407,0.002260491,0.002948029,0.003503356]},{"f":[22,15,16,-1,-1,17,1,-1,-1,2,-1,-1,22,15,15,22,36,-1,-1,-1,17,-1,-1,-1,2,25,-1,-1,-1],"t":[12.199665042528586,10.501837184902296,12.711021616791836,0,0,10.269270673170688,1.3336158192090395,0,0,1.292481250360578,0,0,12.921158001713904,10.457380879072534,4.247927513443585,12.613886328427084,2.5,0,0,0,1.792481250360578,0,0,0,9.293467526905953,2.5,0,0,0],"m":[0,1,1,0,0,1,0,0,0,1,0,0,0,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,0,13,14,15,16,17,0,0,0,21,0,0,0,25,26,0,0,0],"r":[12,5,4,0,0,9,8,0,0,11,0,0,24,23,20,19,18,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0.0005259108,-0.004051535,0,0,-0.01057788,0.001909341,0,-0.01115206,-0.002049552,0,0,0,0,0,0.005267462,-0.001952802,-0.00548722,0,-0.003240812,-0.008761213,0.003036129,0,0,-9.638163e-05,0.006034494,0.005287267]},{"f":[1,22,-1,-1,3,22,4,-1,24,-1,-1,-1,25,37,23,-1,-1,21,-1,-1,17,-1,36,22,-1,-1,3,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,0.05277777777777778,16.759160379664443,1.292481250360578,0,4.5,0,0,0,2.5,11.05,3.5,0,0,5.5,0,0,1.292481250360578,0,2.5,12.613886328427084,0,0,0.4843597262952102,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,0,0,13,14,15,0,0,18,0,0,21,0,23,24,0,0,27,0,0],"r":[4,3,0,0,12,11,8,0,10,0,0,0,20,17,16,0,0,19,0,0,22,0,26,25,0,0,28,0,0],"v":[0,0,0.000745041,0.007679012,0,0,0,9.807641e-05,0,1.89675e-05,-0.006747879,-0.007630329,0,0,0,-0.0001500136,0.0005397238,0,-0.001844344,-0.009719756,0,0.01133967,0,0,0.003224214,-0.008242932,0,0.001339661,-0.005935615]},{"f":[23,21,22,23,-1,4,4,21,-1,-1,21,-1,15,-1,-1,21,-1,-1,32,-1,17,17,33,-1,-1,-1,-1,-1,-1],"t":[8.5,10.5,17.094543568094064,3.5,0,6.451187557243013,3.9534452978042594,8.5,0,0,7.5,0,13.56045225766691,0,0,7.5,0,0,8.341935934311302,0,8.729619742794132,8.076805701955248,21.80651201897376,0,0,0,0,0,0],"m":[1,1,1,1,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,0,0,11,0,13,0,0,16,0,0,19,0,21,22,23,0,0,0,0,0,0],"r":[28,27,18,5,0,15,10,9,0,0,12,0,14,0,0,17,0,0,20,0,26,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-9.504106e-05,0,0,0,0.0005573634,0.003985367,0,-0.005227653,0,-0.0007727259,0.00592756,0,0.005967394,-0.002183809,0,0.003036698,0,0,0,0.000791736,-0.006525412,-0.01564245,0.001636235,-0.005161431,0.003344097]},{"f":[22,21,18,2,-1,-1,-1,15,4,-1,-1,-1,22,15,16,36,22,-1,-1,17,-1,-1,-1,-1,2,25,-1,-1,-1],"t":[12.199665042528586,7.5,10.616089240373544,9.08214658581559,0,0,0,10.690871009292314,5.918813966585702,0,0,0,12.921158001713904,10.457380879072534,9.047123912114026,2.5,12.613886328427084,0,0,1.292481250360578,0,0,0,0,9.293467526905953,2.5,0,0,0],"m":[0,1,1,1,0,0,0,1,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,0,13,14,15,16,17,0,0,20,0,0,0,0,25,26,0,0,0],"r":[12,7,6,5,0,0,0,11,10,0,0,0,24,23,22,19,18,0,0,21,0,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0.0005815968,0.005846891,-0.004611869,0,0,0.0007024086,-0.002745542,-0.006745449,0,0,0,0,0,0.004851495,-0.007835567,0,0.0002944513,-0.003711812,-0.009855806,0.002884939,0,0,-9.334699e-05,0.005463333,0.005002932]},{"f":[9,22,-1,23,15,16,-1,-1,22,1,-1,-1,-1,17,-1,18,2,17,17,-1,-1,-1,-1,2,17,-1,-1,-1,-1],"t":[0.5,14.03140051979143,0,1.5,7.118941072723508,6.247927513443585,0,0,15.717370311433935,4.008411214953271,0,0,0,4.128693921346326,0,7.383290334558488,5.459058925515887,7.252657678068118,6.531360383582522,0,0,0,0,1.292481250360578,8.496347487943302,0,0,0,0],"m":[1,1,0,0,1,1,0,0,1,1,0,0,0,0,0,1,0,1,1,0,0,0,0,1,1,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,10,0,0,0,14,0,16,17,18,19,0,0,0,0,24,25,0,0,0,0],"r":[28,3,0,13,8,7,0,0,12,11,0,0,0,15,0,23,22,21,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0.0001107018,0,0,0,-0.003004553,-0.01626281,0,0,-0.00231143,0.005390633,-0.009899776,0,0.0009732961,0,0,0,0,-0.004612622,0.001332836,-0.0114008,-0.0005078957,0,0,0.001072481,-0.002369264,0.003175984,0.001683987]},{"f":[33,33,12,32,-1,24,32,-1,17,-1,-1,-1,-1,22,21,33,-1,-1,-1,32,-1,23,-1,-1,22,-1,22,-1,-1],"t":[29.569691584072146,23.58371303411409,0.5,2.69615871138938,0,7.5,6.6765734127490415,0,2.69615871138938,0,0,0,0,15.439474430023438,6.5,27.807475161786478,0,0,0,6.337096134072842,0,7.5,0,0,9.945253865134848,0,11.876328901062104,0,0],"m":[1,1,1,0,0,1,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,9,0,0,0,0,14,15,16,0,0,0,20,0,22,0,0,25,0,27,0,0],"r":[24,13,12,5,0,11,8,0,10,0,0,0,0,19,18,17,0,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0.004771361,0,0,-1.669453e-06,0,0.001359434,0.01562034,-0.001672822,-0.003645227,0,0,0,0.009285386,0.002627192,0.0004316513,0,0.001982987,0,-0.007689853,-0.0004381716,0,0.002410789,0,-0.0094077,-0.001599303]},{"f":[22,-1,21,23,22,-1,16,0,-1,33,17,-1,-1,-1,-1,4,22,-1,18,-1,-1,-1,22,16,-1,-1,37,-1,-1],"t":[2.084962500721156,0,6.5,2.5,14.03140051979143,0,8.618385502258606,2.5,0,12.738373101969731,6.9827117829050644,0,0,0,0,5.89476682248518,16.978341770814396,0,2.160964047443681,0,0,0,7.382047033558342,10.372865060112588,0,0,11.05,0,0],"m":[0,0,1,1,1,0,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,5,0,7,8,0,10,11,0,0,0,0,16,17,0,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,22,15,6,0,14,9,0,13,12,0,0,0,0,21,18,0,20,0,0,0,26,25,0,0,28,0,0],"v":[0,0.002509542,0,0,0,0.0001689463,0,0,0.001489259,0,0,-0.003937566,-0.01850241,-0.001589133,0.0005244863,0,0,0.001664629,0,0.004105862,-0.0062718,0.006022016,0,0,-0.002887106,-0.009624479,0,-6.152185e-05,-0.009506687]},{"f":[23,21,22,22,22,23,-1,-1,22,-1,1,-1,-1,22,-1,1,-1,-1,33,15,17,-1,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,17.094543568094064,17.069460217922263,16.978341770814396,0.5,0,0,16.98920034885663,0,4.008411214953271,0,0,17.078739781790915,0,3.966632962588473,0,0,21.80651201897376,6.584962500721156,8.729619742794132,0,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,0,9,0,11,0,0,14,0,16,0,0,19,20,21,0,0,0,0,0,27,0,0],"r":[26,25,18,13,8,7,0,0,10,0,12,0,0,15,0,17,0,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.0002193993,0.0002808209,0,-0.005153638,0,-0.002171466,0.002625696,0,0.003593046,0,-0.002738257,0.001457345,0,0,0,-0.009859065,0.003686049,0.002248802,-0.006735072,-0.004865929,0,0.00647607,-0.0003697883]},{"f":[33,32,33,36,-1,15,2,2,-1,-1,-1,22,17,-1,-1,15,37,-1,22,-1,-1,-1,1,-1,-1,-1,2,-1,-1],"t":[37.98858650139573,10.463013322934223,29.807475161786478,3.5,0,6.539158811108031,3.245926548164837,1.792481250360578,0,0,0,15.439474430023438,7.198837316474133,0,0,13.56045225766691,7.92,0,16.765402642760364,0,0,0,9.107199999999999,0,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,0,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,0,0,0,12,13,0,0,16,17,0,19,0,0,0,23,0,0,0,27,0,0],"r":[26,25,22,5,0,11,10,9,0,0,0,15,14,0,0,21,18,0,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,-3.90737e-05,0,0,0,0.004088189,-0.004097751,0.01216514,0,0,0.001069895,0.007280478,0,0,0.001692154,0,-0.001470018,-0.008369518,0.005583218,0,7.052354e-05,-0.008957745,0.004215948,0,0.0003076701,-0.003549707]},{"f":[21,17,-1,-1,22,-1,22,-1,22,-1,22,18,2,-1,-1,17,-1,-1,22,17,1,-1,-1,36,22,-1,-1,-1,-1],"t":[2.5,10.090748425878532,0,0,6.779867784305249,0,6.930301023985015,0,7.082728314749225,0,12.199665042528586,10.609627990977145,1.292481250360578,0,0,10.682994316786338,0,0,12.921158001713904,1.292481250360578,6.015625,0,0,2.5,12.613886328427084,0,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,0,7,0,9,0,11,12,13,0,0,16,0,0,19,20,21,0,0,24,25,0,0,0,0],"r":[4,3,0,0,6,0,8,0,10,0,18,15,14,0,0,17,0,0,28,23,22,0,0,27,26,0,0,0,0],"v":[0,0,0.000241853,0.003320879,0,0.001195796,0,-0.01128507,0,-0.003501632,0,0,0,0.001515495,0.0001468721,0,-0.01120566,-0.0003871476,0,0,0,-0.001470656,0.006088181,0,0,0.004598721,-0.00747814,-0.004152115,-0.0001544794]},{"f":[1,22,-1,-1,3,22,4,-1,24,-1,-1,-1,23,17,18,-1,19,-1,-1,32,1,17,-1,-1,17,-1,-1,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,0.05277777777777778,16.759160379664443,1.292481250360578,0,4.5,0,0,0,0.5,6.924507036847409,5.108372929097653,0,0.5,0,0,7.014643613484123,2.997611464968153,11.651826559171445,0,0,7.430398005367313,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,0,0,13,14,15,0,17,0,0,20,21,22,0,0,25,0,0,0,0],"r":[4,3,0,0,12,11,8,0,10,0,0,0,28,19,16,0,18,0,0,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0.0006703231,0.007003,0,0,0,7.198779e-05,0,-3.043659e-05,-0.006337773,-0.007256128,0,0,0,-0.0002708806,0,0.001131137,0.008299261,0,0,0,0.001442156,-0.006275574,0,-0.005792673,-0.002304107,0.003688949,0.0002229141]},{"f":[33,32,33,1,15,15,36,-1,-1,22,22,-1,-1,-1,15,1,22,-1,-1,-1,-1,-1,34,-1,-1,24,-1,-1,-1],"t":[37.98858650139573,10.41931383093672,29.807475161786478,12.64516129032258,5.78135971352466,3.321928094887362,2.5,0,0,16.819976309430963,7.25627765006036,0,0,0,6.169925001442312,6.069196428571429,16.96990787537721,0,0,0,0,0,2.5,0,0,9.5,0,0,0],"m":[1,1,1,1,0,1,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,11,0,0,0,15,16,17,0,0,0,0,0,23,0,0,26,0,0,0],"r":[28,25,22,21,14,9,8,0,0,13,12,0,0,0,20,19,18,0,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0,0,0,0.000445985,-0.003238144,0,0,-0.003413072,-0.0005217705,-0.007644091,0,0,0,0.001546206,-0.006528232,0.005051286,1.845813e-05,0.003632365,0,-0.00954444,3.28917e-05,0,0.006434276,-0.001307442,-0.001743188]},{"f":[9,16,17,16,-1,-1,-1,16,22,17,-1,-1,-1,22,22,-1,-1,22,15,-1,-1,22,22,-1,-1,4,-1,-1,-1],"t":[0.5,7.924812503605781,11.78143940991899,7.741466986401147,0,0,0,8.060695931687555,15.717370311433935,6.219136028062415,0,0,0,6.930301023985015,6.5685756301391525,0,0,16.759160379664443,13.56045225766691,0,0,16.786010507999382,16.765402642760364,0,0,3.9534452978042594,0,0,0],"m":[1,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,0,0,0,14,15,0,0,18,19,0,0,22,23,0,0,26,0,0,0],"r":[28,7,6,5,0,0,0,13,12,11,0,0,0,17,16,0,0,21,20,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,7.090654e-05,0.001592563,-0.006537682,0,0,0,-0.00400971,0.001696953,-0.009340993,0,0,0.001775594,-0.01071731,0,0,-2.521112e-05,0.007061904,0,0,-0.001481205,-0.02068258,0,0.003575407,-0.001792585,0.001555201]},{"f":[22,1,16,-1,-1,16,-1,-1,33,22,-1,17,21,36,-1,3,-1,26,18,-1,-1,-1,16,-1,-1,-1,15,-1,-1],"t":[11.36027577861459,1.13125,10.794415866350105,0,0,12.711021616791836,0,0,29.807475161786478,11.424679641461715,0,12.18494601563336,8.5,3.5,0,0.3162042875157629,0,0.5,7.198837316474133,0,0,0,10.794415866350105,0,0,0,9.92481250360578,0,0],"m":[0,0,1,0,0,1,0,0,1,0,0,1,1,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,13,14,0,16,0,18,19,0,0,0,23,0,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,26,11,0,25,22,15,0,17,0,21,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.0002211887,-0.01049953,0,0.0007783227,-0.003367836,0,0,-0.003128726,0,0,0,-0.0001124732,0,0.009205188,0,0,-0.0005832406,0.006751133,0.006355479,0,0.002943048,-0.003190434,-0.00245371,0,-0.009374101,-0.001510996]},{"f":[1,1,-1,18,-1,-1,2,5,20,-1,-1,22,22,22,23,22,-1,-1,-1,-1,-1,32,-1,17,10,-1,-1,-1,-1],"t":[5.417948717948718,5.015625,0,4.245926548164837,0,0,9.068775776507199,0.5,0.5,0,0,17.094543568094064,16.98920034885663,16.98413870862295,3.5,15.717370311433935,0,0,0,0,0,8.341935934311302,0,8.729619742794132,0.5,0,0,0,0],"m":[1,1,0,1,0,0,1,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,0,0,12,13,14,15,16,0,0,0,0,0,22,0,24,25,0,0,0,0],"r":[6,3,0,5,0,0,28,11,10,0,0,21,20,19,18,17,0,0,0,0,0,23,0,27,26,0,0,0,0],"v":[0,0,6.721089e-05,0,0.001274698,0.008402274,0,0,0,-0.0006467162,-0.005010159,0,0,0,0,0,-0.0001689153,-0.004791333,0.001106801,-0.005977261,0.002961956,0,0.002946815,0,0,-0.00247976,-0.01510932,0.001831796,-0.006106365]},{"f":[22,-1,21,23,22,-1,16,0,-1,33,17,-1,-1,-1,-1,4,22,-1,18,-1,-1,-1,22,-1,37,18,-1,-1,-1],"t":[2.084962500721156,0,6.5,2.5,14.03140051979143,0,8.618385502258606,2.5,0,12.738373101969731,6.9827117829050644,0,0,0,0,5.89476682248518,16.96990787537721,0,1.292481250360578,0,0,0,7.382047033558342,0,11.05,10.620215166896223,0,0,0],"m":[0,0,1,1,1,0,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,0,7,8,0,10,11,0,0,0,0,16,17,0,19,0,0,0,23,0,25,26,0,0,0],"r":[2,0,22,15,6,0,14,9,0,13,12,0,0,0,0,21,18,0,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0.002326739,0,0,0,0.0001523433,0,0,0.001426153,0,0,-0.003654725,-0.01734284,-0.001387002,0.0004467789,0,0,0.001541864,0,0.003775341,-0.004943877,0.005612922,0,-0.003633471,0,0,0.0001112383,-0.002000224,-0.008264276]},{"f":[25,1,24,22,2,-1,-1,-1,21,-1,20,16,1,22,2,-1,-1,-1,17,-1,-1,21,-1,0,-1,-1,-1,-1,-1],"t":[6.5,5.417948717948718,7.5,16.725912049439096,6.584884225259038,0,0,0,7.5,0,0.5,9.250298417906333,2.876915188080302,8.32532955673354,5.14960400919364,0,0,0,7.344289078557851,0,0,8.5,0,2.5,0,0,0,0,0],"m":[1,1,0,1,1,0,0,0,1,0,1,1,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,0,11,12,13,14,15,0,0,0,19,0,0,22,0,24,0,0,0,0,0],"r":[28,27,8,7,6,0,0,0,10,0,26,21,18,17,16,0,0,0,20,0,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0.001216514,0.01158271,-0.004069929,0,0.0002255081,0,0,0,0,0,0.0006705586,0.01059693,-0.0006432798,0,-0.004488196,0.001017563,0,-0.005102984,0,0.004771253,-0.003224207,0.0004715571,-0.0003864214,0.003054236]},{"f":[23,21,22,-1,17,16,-1,-1,15,15,21,-1,25,-1,-1,17,-1,-1,21,-1,22,22,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,0,3.39067985676233,5.672425341971495,0,0,6.857980995127572,6.539158811108031,4.5,0,0.5,0,0,8.729619742794132,0,0,4.5,0,16.786010507999382,16.759160379664443,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,0,9,10,11,0,13,0,0,16,0,0,19,0,21,22,0,0,0,0,27,0,0],"r":[26,25,4,0,8,7,0,0,18,15,12,0,14,0,0,17,0,0,20,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.0001138627,0,0,-0.0009515492,0.001370636,0,0,0,-0.003844874,0,-0.001342534,0.006583699,0,-0.01722518,0.004264006,0,0.00651735,0,0,-0.0009888649,-0.0101666,0.0005828408,-0.004336184,0,0.006059314,-0.0004896122]},{"f":[33,32,23,17,18,-1,19,-1,-1,1,17,2,-1,-1,-1,1,17,-1,-1,15,-1,-1,-1,24,-1,-1,2,-1,-1],"t":[37.98858650139573,10.41931383093672,0.5,6.924507036847409,5.108372929097653,0,0.5,0,0,2.997611464968153,11.651826559171445,9.20088558729989,0,0,0,6.646978021978022,7.430398005367313,0,0,6.247927513443585,0,0,0,9.5,0,0,2.453445297804259,0,0],"m":[1,1,0,1,1,0,1,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,12,0,0,0,16,17,0,0,20,0,0,0,24,0,0,27,0,0],"r":[26,23,22,9,6,0,8,0,0,15,14,13,0,0,0,19,18,0,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.0002505383,0,0.001020107,0.007837077,0,0,0,0.0007260958,0.006451031,-0.005697246,0,0,-0.008742818,-0.002563929,0,-0.00271697,0.007363325,0.000186562,0,0.006014992,-0.001022474,0,0.0003591262,-0.003272633]},{"f":[32,-1,1,-1,33,2,-1,17,-1,-1,33,-1,15,15,17,1,-1,-1,-1,-1,15,17,15,-1,-1,-1,22,-1,-1],"t":[2.69615871138938,0,5.417948717948718,0,13.895530557358477,1.292481250360578,0,2.69615871138938,0,0,17.460839126440824,0,6.169925001442312,4.169925001442312,1.792481250360578,6.015625,0,0,0,0,6.820178962415188,8.729619742794132,6.539158811108031,0,0,0,16.98920034885663,0,0],"m":[0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,0,3,0,5,6,0,8,0,0,11,0,13,14,15,16,0,0,0,0,21,22,23,0,0,0,27,0,0],"r":[2,0,4,0,10,7,0,9,0,0,12,0,20,19,18,17,0,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0.003266462,0,0.0001111334,0,0,-0.004611595,0,0.003048882,-0.002267021,0,0.00588738,0,0,0,0,-0.002743834,0.0046181,-0.004541984,0.004605265,0,0,0,-0.002504499,-0.01182707,0.004155974,0,-0.0005920309,0.001440559]},{"f":[22,1,16,-1,-1,-1,33,22,-1,22,18,-1,22,-1,-1,22,16,16,-1,18,-1,-1,-1,2,-1,-1,15,-1,-1],"t":[11.36027577861459,1.13125,10.794415866350105,0,0,0,29.807475161786478,11.424679641461715,0,12.199665042528586,10.680359255641626,0,11.683871801979876,0,0,12.921158001713904,10.462502272597265,4.247927513443585,0,1.792481250360578,0,0,0,9.293467526905953,0,0,9.92481250360578,0,0],"m":[0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,0,1,1,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,10,11,0,13,0,0,16,17,18,0,20,0,0,0,24,0,0,27,0,0],"r":[6,5,4,0,0,0,26,9,0,15,12,0,14,0,0,23,22,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,-0.0002392599,-0.00961246,0.0006318935,0,0,-0.00298601,0,0,0.001099376,0,-0.01210346,-0.0005968885,0,0,0,-0.0008285111,0,-0.002724494,-0.008463136,0.002555135,0,-4.642508e-05,0.004703631,0,-0.008613534,-0.00137451]},{"f":[22,-1,21,23,22,-1,1,-1,15,-1,33,-1,-1,4,22,-1,18,-1,-1,-1,22,15,-1,-1,37,22,-1,-1,-1],"t":[2.084962500721156,0,6.5,2.5,14.03140051979143,0,2.532051282051282,0,5.044394119358453,0,19.167172160526107,0,0,5.89476682248518,16.96990787537721,0,1.292481250360578,0,0,0,7.5621616439616925,9.060695931687555,0,0,11.05,8.133841411710264,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,9,0,11,0,0,14,15,0,17,0,0,0,21,22,0,0,25,26,0,0,0],"r":[2,0,20,13,6,0,8,0,10,0,12,0,0,19,16,0,18,0,0,0,24,23,0,0,28,27,0,0,0],"v":[0,0.002209604,0,0,0,0.0001347799,0,0.001613741,0,-0.004349427,0,0.006890206,-0.002002828,0,0,0.001405166,0,0.003550636,-0.004666927,0.005284749,0,0,-0.002307635,-0.008837002,0,0,0.006668917,-0.0001234677,-0.007799607]},{"f":[23,21,25,1,24,22,2,-1,-1,-1,21,17,-1,-1,20,-1,-1,4,21,-1,-1,25,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,6.5,5.417948717948718,7.5,16.725912049439096,6.584884225259038,0,0,0,7.5,11.969133019829592,0,0,0.5,0,0,6.499295214872664,8.5,0,0,0.5,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,0,1,1,0,0,0,1,1,0,0,1,0,0,1,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,15,0,0,18,19,0,0,22,0,0,0,0,27,0,0],"r":[26,25,24,17,10,9,8,0,0,0,14,13,0,0,16,0,0,21,20,0,0,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0.001119559,0.01095692,-0.003983497,0,0,0.0002432535,-0.00297402,0,-0.001418748,0.0004017068,0,0,-0.0003979971,0.002303538,0,-0.0005404246,-0.01110717,0.003049287,-0.00412,0,0.005716597,-0.0004705366]},{"f":[22,-1,33,23,1,-1,22,0,-1,-1,15,22,-1,33,24,36,-1,-1,-1,22,-1,1,-1,-1,-1,-1,34,-1,-1],"t":[11.36027577861459,0,29.807475161786478,0.5,1.588659147869674,0,11.722778233171695,3.5,0,0,9.38586240064146,11.775312925643524,0,19.167172160526107,6.5,2.5,0,0,0,14.03140051979143,0,5.015625,0,0,0,0,2.5,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,0,11,12,0,14,15,16,0,0,0,20,0,22,0,0,0,0,27,0,0],"r":[2,0,26,25,6,0,10,9,0,0,24,13,0,19,18,17,0,0,0,21,0,23,0,0,0,0,28,0,0],"v":[0,0.0003383055,0,0,0,0.0003005355,0,0,-0.002632079,-0.009234796,0,0,0.003468886,0,0,0,0.004214829,-0.001859661,0.009879484,0,-0.001806677,0,-0.01468731,-0.002836158,0.003476943,0.0002366284,0,-0.006525231,-0.001078527]},{"f":[22,-1,23,17,17,17,3,3,-1,-1,22,17,18,-1,16,17,-1,-1,-1,-1,15,-1,32,-1,-1,-1,-1,-1,-1],"t":[2.084962500721156,0,0.5,11.651826559171445,10.69085411329249,10.616089240373544,0.9992038216560509,0.4226190476190476,0,0,13.827286379004683,9.845435504646158,5.108372929097653,0,12.225508720236002,7.980000966034041,0,0,0,0,6.0,0,7.014643613484123,0,0,0,0,0,0],"m":[0,0,0,1,1,1,0,1,0,0,1,1,1,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,12,13,0,15,16,0,0,0,0,21,0,23,0,0,0,0,0,0],"r":[2,0,28,27,26,25,10,9,0,0,20,19,14,0,18,17,0,0,0,0,22,0,24,0,0,0,0,0,0],"v":[0,0.002124718,0,0,0,0,0,0,0.000899093,0.006933367,0,0,0,-0.0002734773,0,0,0.001111434,0.005347789,-0.003174254,-0.001748001,0,0.001115941,0,-0.003479098,0.002844743,-0.006915884,0.003088691,-0.005107263,0.0001640008]},{"f":[4,-1,2,18,16,22,-1,-1,-1,-1,15,16,4,-1,-1,32,15,17,-1,-1,-1,-1,16,-1,22,15,-1,-1,-1],"t":[4.0437314206251695,0,4.128693921346326,5.918419679874358,7.118941072723508,16.819976309430963,0,0,0,0,10.376125389276174,7.651051691178929,4.80643424864552,0,0,6.474913355379556,9.756556322524087,7.723832212141186,0,0,0,0,10.486835021563051,0,16.759160379664443,13.56045225766691,0,0,0],"m":[1,0,0,1,1,1,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,0,0,0,11,12,13,0,0,16,17,18,0,0,0,0,23,0,25,26,0,0,0],"r":[2,0,10,9,8,7,0,0,0,0,22,15,14,0,0,21,20,19,0,0,0,0,24,0,28,27,0,0,0],"v":[0,9.72807e-05,0,0,0,0,0.0005393365,-0.00649996,-0.007236603,0.001963577,0,0,0,0.003126606,-0.0002115249,0,0,0,-0.003281519,0.0002860037,-0.006205648,-0.007595421,0,0.004622934,0,0,0.0003168406,0.007026837,-0.006354625]},{"f":[23,21,22,23,-1,-1,17,16,-1,-1,15,4,22,33,-1,15,4,-1,-1,-1,-1,2,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,3.5,0,0,3.39067985676233,5.672425341971495,0,0,7.366322214245816,4.951187557243013,16.819976309430963,17.460839126440824,0,6.247927513443585,3.39067985676233,0,0,0,0,5.8698903048866296,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,12,13,14,0,16,17,0,0,0,0,22,0,0,0,0,27,0,0],"r":[26,25,6,5,0,0,10,9,0,0,24,21,20,15,0,19,18,0,0,0,0,23,0,0,0,0,28,0,0],"v":[0,0,0,0,2.525409e-05,0.001471628,0,0,-0.0008500213,0.001176147,0,0,0,0,0.007483575,0,0,-0.00132291,0.003509125,-0.009210508,-0.005401652,0,-0.01004521,-0.0003642403,-0.000122171,-0.003956237,0,0.005459312,-0.000471872]},{"f":[15,15,36,-1,16,-1,-1,22,22,-1,-1,-1,32,-1,15,22,-1,22,-1,-1,0,26,1,-1,-1,-1,22,-1,-1],"t":[5.78135971352466,3.321928094887362,2.5,0,11.031356596255709,0,0,16.819976309430963,7.25627765006036,0,0,0,3.753897320099348,0,6.169925001442312,16.975010474608837,0,17.069460217922263,0,0,11.5,0.5,4.92309205350118,0,0,0,10.558733410935197,0,0],"m":[0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,0,13,0,15,16,0,18,0,0,21,22,23,0,0,0,27,0,0],"r":[12,7,4,0,6,0,0,11,10,0,0,0,14,0,20,17,0,19,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0.0003826748,0,-0.006157388,-3.276973e-05,0,0,-0.002918107,-0.0003977363,-0.006525911,0,0.009982826,0,0,0.00178727,0,-0.00639011,0.002065656,0,0,0,0.000328874,-0.0004170614,0.003289278,0,-0.001863945,-0.008852942]},{"f":[22,5,3,-1,-1,-1,22,22,36,-1,0,-1,17,-1,2,-1,-1,-1,2,2,4,25,-1,-1,-1,17,-1,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,0,12.921158001713904,12.613886328427084,2.5,0,3.5,0,1.792481250360578,0,1.292481250360578,0,0,0,9.293467526905953,9.051177541662383,7.950787011501063,2.5,0,0,0,9.44286754736589,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,9,0,11,0,13,0,15,0,0,0,19,20,21,22,0,0,0,26,0,0,0],"r":[6,5,4,0,0,0,18,17,10,0,12,0,14,0,16,0,0,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0,-0.000149462,-0.005989927,0.0003259359,0,0,0,0.004295521,0,0.0007139778,0,-0.001314398,0,-0.008289724,-0.001467687,-0.003350586,0,0,0,0,-6.939782e-05,0.006119372,0.007099082,0,-0.008962983,-0.0002786624,0.004348776]},{"f":[23,4,1,-1,-1,22,15,21,16,22,-1,-1,15,-1,17,-1,22,-1,-1,-1,-1,4,15,-1,-1,-1,15,-1,-1],"t":[8.5,4.0437314206251695,14.266666666666666,0,0,16.725912049439096,13.0003521774803,9.5,1.584962500721156,11.488914557581817,0,0,10.376125389276174,0,9.27958770372691,0,11.722778233171695,0,0,0,0,6.434604125275413,9.002815015607053,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,1,1,1,0,1,0,0,1,0,1,0,1,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,0,13,0,15,0,17,0,0,0,0,22,23,0,0,0,27,0,0],"r":[26,5,4,0,0,21,20,19,12,11,0,0,14,0,16,0,18,0,0,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,6.865711e-05,0.003754149,0,0,0,0,0,0.004269172,-0.001900848,0,-0.0005608302,0,-8.100782e-05,0,-0.0002282675,0.006496031,-0.00300301,0.006159161,0,0,-0.001514693,-0.006082287,0.003652847,0,0.005300254,-0.0004584858]},{"f":[23,18,22,18,-1,16,-1,-1,1,-1,1,22,-1,-1,15,-1,32,-1,-1,21,17,-1,-1,15,-1,2,-1,-1,-1],"t":[0.5,9.845435504646158,13.87915215758784,5.108372929097653,0,12.225508720236002,0,0,1.000936329588015,0,5.015625,14.03140051979143,0,0,5.906890595608519,0,7.014643613484123,0,0,4.5,10.090748425878532,0,0,9.264442600226602,0,1.292481250360578,0,0,0],"m":[0,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,0,11,12,0,0,15,0,17,0,0,20,21,0,0,24,0,26,0,0,0],"r":[28,19,8,5,0,7,0,0,10,0,14,13,0,0,16,0,18,0,0,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,-0.0001255309,0,0.001353687,-0.002898473,0,0.001865353,0,0,-0.003076776,-0.01364371,0,0.004347118,0,-0.00258723,0.002684676,0,0,-0.003101084,0.001729881,0,-0.001257143,0,-0.01107256,-0.002440401,0.0001610808]},{"f":[3,22,15,4,22,3,-1,-1,-1,-1,-1,-1,23,18,17,22,16,18,-1,-1,-1,15,-1,33,-1,-1,-1,-1,-1],"t":[0.034940494354592615,16.765402642760364,10.113742166049189,2.453445297804259,16.73920708776685,0.007093253968253968,0,0,0,0,0,0,0.5,9.845435504646158,9.27958770372691,13.87915215758784,11.696967526234287,6.65017628016411,0,0,0,6.0,0,18.059836468389907,0,0,0,0,0],"m":[0,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,0,13,14,15,16,17,18,0,0,0,22,0,24,0,0,0,0,0],"r":[12,11,10,9,8,7,0,0,0,0,0,0,28,27,26,21,20,19,0,0,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0.003940323,-0.001229854,0.004673495,-0.003234182,-0.005490523,-0.007922677,0,0,0,0,0,0,-1.447437e-05,0.00203141,-0.003255773,0,0.001062179,0,0.004619193,-0.002914414,0.004380231,-0.001501637,0.0002181123]},{"f":[22,-1,21,23,22,-1,32,-1,19,-1,-1,4,22,-1,4,22,-1,-1,2,-1,-1,-1,22,-1,22,-1,37,-1,-1],"t":[2.084962500721156,0,6.5,2.5,15.717370311433935,0,2.69615871138938,0,0.5,0,0,5.89476682248518,16.819976309430963,0,2.160964047443681,16.978341770814396,0,0,2.453445297804259,0,0,0,7.5621616439616925,0,8.32532955673354,0,11.05,0,0],"m":[0,0,1,1,1,0,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,0,7,0,9,0,0,12,13,0,15,16,0,0,19,0,0,0,23,0,25,0,27,0,0],"r":[2,0,22,11,6,0,8,0,10,0,0,21,14,0,18,17,0,0,20,0,0,0,24,0,26,0,28,0,0],"v":[0,0.00199048,0,0,0,9.128872e-05,0,-0.000316437,0,-0.00291417,-0.01124682,0,0,0.001822686,0,0,0.002678696,-0.003640472,0,-0.01409231,0.0001573585,0.004803204,0,-0.002766351,0,0.005773383,0,-0.0001311288,-0.007123591]},{"f":[33,32,4,4,-1,-1,33,2,22,-1,-1,15,15,16,-1,22,-1,-1,-1,21,-1,-1,-1,24,-1,-1,2,-1,-1],"t":[37.98858650139573,10.41931383093672,4.0437314206251695,3.5221970596792267,0,0,27.44694185105858,4.128693921346326,11.876328901062104,0,0,10.043027283594547,9.269126679149418,7.651051691178929,0,7.770639075561187,0,0,0,4.5,0,0,0,9.5,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,12,13,14,0,16,0,0,0,20,0,0,0,24,0,0,27,0,0],"r":[26,23,6,5,0,0,22,11,10,0,0,19,18,15,0,17,0,0,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,1.408731e-05,0.0012715,0,0,0,0.00145741,-0.003540538,0,0,0,6.612074e-05,0,-0.008140946,-0.0009289235,-0.005634328,0,0.005324949,0.000323571,-0.004950917,0,0.005666531,-0.0008281219,0,0.0004467666,-0.00301843]},{"f":[22,-1,22,22,22,-1,-1,16,-1,-1,22,-1,33,22,-1,22,18,-1,22,-1,-1,22,22,-1,-1,-1,34,-1,-1],"t":[4.926654777701837,0,7.082728314749225,6.779867784305249,5.9525054311951475,0,0,5.78135971352466,0,0,11.36027577861459,0,29.807475161786478,11.424679641461715,0,12.199665042528586,10.680359255641626,0,11.683871801979876,0,0,12.921158001713904,12.613886328427084,0,0,0,2.5,0,0],"m":[0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,0,11,0,13,14,0,16,17,0,19,0,0,22,23,0,0,0,27,0,0],"r":[2,0,10,7,6,0,0,9,0,0,12,0,26,15,0,21,18,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0.001014989,0,0,0,-0.003566221,0.0009036857,0,-0.002507699,-0.008121796,0,0.0004799374,0,0,-0.00257013,0,0,0.0009894666,0,-0.01050575,-0.0003519621,0,0,-0.0004909778,-0.003194478,-4.076873e-06,0,-0.006016436,-0.0009754483]},{"f":[3,22,15,4,22,3,-1,-1,-1,-1,-1,-1,23,18,17,22,-1,15,-1,33,-1,18,-1,1,-1,-1,-1,-1,-1],"t":[0.034940494354592615,16.765402642760364,10.113742166049189,2.453445297804259,16.73920708776685,0.007093253968253968,0,0,0,0,0,0,0.5,9.845435504646158,9.27958770372691,13.87915215758784,0,6.0,0,18.059836468389907,0,1.292481250360578,0,5.015625,0,0,0,0,0],"m":[0,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,0,13,14,15,16,0,18,0,20,0,22,0,24,0,0,0,0,0],"r":[12,11,10,9,8,7,0,0,0,0,0,0,28,27,26,17,0,19,0,21,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0.003737364,-0.001207884,0.004449675,-0.003099621,-0.005185214,-0.007545809,0,0,0,0,4.848573e-05,0,0.001009865,0,0.004388053,0,-0.001190317,0,-0.00953042,-0.002870919,0.004142386,-0.001424595,0.0002143892]},{"f":[15,15,22,-1,4,-1,-1,22,-1,16,16,22,-1,-1,15,-1,-1,-1,32,-1,1,21,-1,16,-1,-1,21,-1,-1],"t":[5.78135971352466,4.087462841250339,16.765402642760364,0,2.160964047443681,0,0,10.63843335887114,0,6.409390936137702,5.169925001442312,12.613886328427084,0,0,5.491853096329675,0,0,0,3.753897320099348,0,11.278421650655924,8.5,0,11.98370619265935,0,0,5.5,0,0],"m":[0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,0,10,11,12,0,0,15,0,0,0,19,0,21,22,0,24,0,0,27,0,0],"r":[18,7,4,0,6,0,0,9,0,17,14,13,0,0,16,0,0,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0,0.0001740755,0,-0.007310146,-0.0008007952,0,-0.003169831,0,0,0,0.0002312804,-0.002729693,0,0.001717169,-0.001112345,-0.00254345,0,0.009407714,0,0,5.793254e-05,0,0.002333521,-0.00462228,0,-0.0009006177,-0.009033052]},{"f":[23,21,4,21,-1,17,-1,-1,22,24,-1,21,18,-1,-1,-1,22,-1,22,-1,2,-1,16,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,4.0437314206251695,8.5,0,6.781300095563516,0,0,16.725912049439096,3.5,0,9.5,8.03615005796417,0,0,0,16.786010507999382,0,16.819976309430963,0,5.065928480304397,0,7.774787059601174,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,12,13,0,0,0,17,0,19,0,21,0,23,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,16,11,0,15,14,0,0,0,18,0,20,0,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,4.638912e-05,0,0.00279699,-0.00127028,0,0,0.004944411,0,0,-0.0003494766,0.0009899167,-0.002768984,0,-0.00574986,0,0.002527957,0,-0.006558926,0,0.002865991,-0.002898646,-0.003635073,0,0.005062457,-0.000537745]},{"f":[1,22,1,-1,2,-1,-1,-1,3,22,4,-1,-1,-1,25,37,-1,21,-1,-1,17,-1,36,22,-1,-1,16,-1,-1],"t":[0.39811320754716983,14.247293309538719,0.01933482600988898,0,4.95344529780426,0,0,0,0.05277777777777778,16.759160379664443,1.292481250360578,0,0,0,2.5,11.05,0,5.5,0,0,1.292481250360578,0,2.5,12.613886328427084,0,0,2.807354922057604,0,0],"m":[0,1,1,0,0,0,0,0,0,1,1,0,0,0,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,0,5,0,0,0,9,10,11,0,0,0,15,16,0,18,0,0,21,0,23,24,0,0,27,0,0],"r":[8,7,4,0,6,0,0,0,14,13,12,0,0,0,20,17,0,19,0,0,22,0,26,25,0,0,28,0,0],"v":[0,0,0,-0.0004415881,0,-0.0003182101,0.003490246,0.006250146,0,0,0,-8.737013e-05,-0.00333385,-0.006017934,0,0,-3.94821e-06,0,-0.001310489,-0.006720971,0,0.009020025,0,0,0.002509324,-0.006581241,0,-0.004672639,0.0004232735]},{"f":[32,33,33,12,32,-1,37,32,-1,-1,-1,-1,33,-1,22,17,-1,-1,22,-1,22,-1,-1,1,-1,-1,2,-1,-1],"t":[10.463013322934223,29.807475161786478,23.58371303411409,0.5,2.69615871138938,0,7.92,6.6765734127490415,0,0,0,0,24.145246550704165,0,15.439474430023438,7.198837316474133,0,0,16.98920034885663,0,17.083799521744545,0,0,9.107199999999999,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,0,1,1,0,0,0,0,0,0,1,1,0,0,1,0,1,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,15,16,0,0,19,0,21,0,0,24,0,0,27,0,0],"r":[26,23,12,11,6,0,10,9,0,0,0,0,14,0,18,17,0,0,20,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0.003394457,0,0,-6.755307e-06,0.004409221,-0.00108217,-0.003710138,0,0.005911191,0,0,0.0006396892,0.005730768,0,-0.003249621,0,0.0041483,-0.003445515,0,-4.920701e-05,-0.0042937,0,0.003904603,-0.001197929]},{"f":[22,-1,23,17,3,-1,36,-1,1,15,-1,-1,21,22,22,-1,-1,2,33,-1,17,17,-1,-1,-1,-1,-1,-1,-1],"t":[2.084962500721156,0,0.5,12.18494601563336,0.9992038216560509,0,2.5,0,3.354166666666667,11.683433292835403,0,0,3.5,12.199665042528586,11.67903991803059,0,0,2.453445297804259,18.059836468389907,0,5.281121212110536,1.292481250360578,0,0,0,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,0,0,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,9,10,0,0,13,14,15,0,0,18,19,0,21,22,0,0,0,0,0,0,0],"r":[2,0,28,27,6,0,8,0,12,11,0,0,26,17,16,0,0,25,20,0,24,23,0,0,0,0,0,0,0],"v":[0,0.001827714,0,0,0,0.001816488,0,0.002251916,0,0,7.22592e-05,-0.004063938,0,0,0,3.427187e-05,0.006371652,0,0,0.006265002,0,0,-0.0003079778,0.006578411,-0.002741484,-0.003268674,-0.002557759,-0.006289677,0.0001489423]},{"f":[23,22,-1,17,16,37,-1,-1,-1,15,4,22,4,18,-1,-1,-1,-1,2,-1,-1,22,22,-1,-1,-1,15,-1,-1],"t":[8.5,14.03140051979143,0,3.39067985676233,5.672425341971495,6.890000000000001,0,0,0,7.366322214245816,4.951187557243013,16.819976309430963,3.39067985676233,7.344289078557851,0,0,0,0,5.8698903048866296,0,0,16.786010507999382,16.759160379664443,0,0,0,6.539158811108031,0,0],"m":[1,1,0,0,0,1,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,0,0,10,11,12,13,14,0,0,0,0,19,0,0,22,23,0,0,0,27,0,0],"r":[26,3,0,9,8,7,0,0,0,21,18,17,16,15,0,0,0,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,8.80438e-05,0,0,0,0.0003240069,-0.002697001,0.001020087,0,0,0,0,0,-0.002148216,0.001678976,0.003944622,-0.00487594,0,-0.009377606,-0.0003912682,0,0,-0.0005805953,-0.009430026,0.0006740557,0,0.00490136,-0.0005182563]},{"f":[33,32,16,17,-1,-1,16,22,17,-1,-1,-1,22,18,-1,22,-1,17,-1,-1,-1,24,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,10.41931383093672,7.924812503605781,11.78143940991899,0,0,8.060695931687555,15.717370311433935,6.219136028062415,0,0,0,11.722778233171695,0.5,0,11.406459975416917,0,10.680359255641626,0,0,0,9.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,1,0,0,0,1,1,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,14,0,16,0,18,0,0,0,22,0,0,25,0,27,0,0],"r":[24,21,6,5,0,0,12,11,10,0,0,0,20,15,0,17,0,19,0,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0.0001110503,-0.005348278,0,0,0,-0.00323884,0.001838966,-0.00838379,0,0,-0.006660641,0,8.427872e-05,0,-0.001954677,-0.00840656,0.0002552836,0,0.005190857,-0.000747262,0,0.0004666525,0,-0.0003238991,-0.004187938]},{"f":[15,15,22,21,-1,-1,4,-1,-1,22,-1,16,-1,-1,32,-1,15,1,22,-1,-1,-1,0,26,-1,-1,22,-1,-1],"t":[5.78135971352466,4.087462841250339,16.765402642760364,8.5,0,0,2.160964047443681,0,0,10.63843335887114,0,6.409390936137702,0,0,3.753897320099348,0,6.169925001442312,6.069196428571429,16.96990787537721,0,0,0,11.5,0.5,0,0,10.558733410935197,0,0],"m":[0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,0,12,0,0,15,0,17,18,19,0,0,0,23,24,0,0,27,0,0],"r":[14,9,6,5,0,0,8,0,0,11,0,13,0,0,16,0,22,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0.0002437255,-0.002327731,0,-0.006942959,-0.00072086,0,-0.003057063,0,-0.0002749562,-0.002402105,0,0.008611349,0,0,0,0.001111783,-0.004837908,0.003089435,0,0,3.196332e-06,0.002941384,0,-0.00158424,-0.007566388]},{"f":[36,-1,22,2,-1,-1,22,16,16,22,36,-1,-1,-1,18,-1,-1,-1,2,2,4,25,-1,-1,-1,17,-1,-1,-1],"t":[1.5,0,12.199665042528586,1.292481250360578,0,0,12.921158001713904,10.462502272597265,4.247927513443585,12.613886328427084,2.5,0,0,0,1.792481250360578,0,0,0,9.293467526905953,9.051177541662383,7.950787011501063,2.5,0,0,0,9.44286754736589,0,0,0],"m":[0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,0,0,0,15,0,0,0,19,20,21,22,0,0,0,26,0,0,0],"r":[2,0,6,5,0,0,18,17,14,13,12,0,0,0,16,0,0,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,-0.00255039,0,0,0.000728523,-4.815958e-05,0,0,0,0,0,0.003848819,-0.001122114,-0.003735257,0,-0.002178583,-0.007648388,0.0020939,0,0,0,0,-5.999851e-05,0.005012247,0.006468246,0,-0.008250555,-0.0004346035,0.003964258]},{"f":[23,3,-1,36,-1,1,15,17,17,-1,-1,-1,16,-1,-1,21,-1,32,19,2,-1,-1,22,21,-1,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,0,2.5,0,3.354166666666667,11.683433292835403,11.78143940991899,10.69085411329249,0,0,0,11.756139298372927,0,0,1.5,0,6.337096134072842,0.5,1.292481250360578,0,0,12.199665042528586,3.5,0,0,0,0,0],"m":[0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,6,7,8,9,0,0,0,13,0,0,16,0,18,19,20,0,0,23,24,0,0,0,0,0],"r":[28,3,0,5,0,15,12,11,10,0,0,0,14,0,0,17,0,27,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0.001726954,0,0.00208007,0,0,0,0,2.366071e-05,0.002836776,-0.005019183,0,-0.01061265,0.0006011262,0,0.000727132,0,0,0,-0.00985168,-0.001685075,0,0,0.006674446,-0.000408383,-0.002023575,0.002314806,0.000145443]},{"f":[33,32,16,17,-1,-1,16,22,21,-1,-1,-1,22,18,-1,22,-1,17,-1,-1,32,-1,-1,24,-1,-1,2,-1,-1],"t":[37.98858650139573,10.41931383093672,7.924812503605781,11.78143940991899,0,0,8.060695931687555,15.717370311433935,5.5,0,0,0,11.722778233171695,0.5,0,11.406459975416917,0,1.792481250360578,0,0,4.999295214872664,0,0,9.5,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,14,0,16,0,18,0,0,21,0,0,24,0,0,27,0,0],"r":[26,23,6,5,0,0,12,11,10,0,0,0,20,15,0,17,0,19,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.0001071198,-0.004812333,0,0,0,0.002057969,-0.002844087,-0.007976935,0,0,-0.006183116,0,6.7705e-05,0,0.000455753,-0.005138359,0,-0.005441178,0.0003175409,0,0.00485426,-0.0007388556,0,0.000404014,-0.002674635]},{"f":[36,-1,22,2,-1,24,-1,17,-1,-1,22,22,36,-1,0,15,-1,-1,17,-1,2,24,-1,-1,-1,17,-1,-1,-1],"t":[1.5,0,12.199665042528586,1.292481250360578,0,9.5,0,7.5507416557826375,0,0,12.738408632106918,12.613886328427084,2.5,0,3.5,2.0,0,0,1.792481250360578,0,1.292481250360578,4.5,0,0,0,3.084962500721156,0,0,0],"m":[0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,12,13,0,15,16,0,0,19,0,21,22,0,0,0,26,0,0,0],"r":[2,0,10,5,0,7,0,9,0,0,28,25,14,0,18,17,0,0,20,0,24,23,0,0,0,27,0,0,0],"v":[0,-0.002508185,0,0,0.0006939558,0,9.094653e-05,0,-0.00690872,-0.0001562637,0,0,0,0.003594141,0,0,-0.001041918,0.001923645,0,-0.001041918,0,0,-0.004205539,-0.01139327,-0.001041918,0,-0.00203818,-0.006635357,-2.064016e-05]},{"f":[22,-1,17,16,-1,22,-1,-1,32,-1,33,2,-1,-1,17,17,-1,19,-1,-1,21,-1,18,4,-1,-1,18,-1,-1],"t":[14.03140051979143,0,3.39067985676233,5.672425341971495,0,16.903290731167747,0,0,5.22766361015228,0,20.752541988539424,4.80643424864552,0,0,7.392087364565857,6.864066550153108,0,1.5,0,0,5.5,0,8.496347487943302,5.065928480304397,0,0,9.26091905453048,0,0],"m":[1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,0,11,12,0,0,15,16,0,18,0,0,21,0,23,24,0,0,27,0,0],"r":[2,0,8,5,0,7,0,0,10,0,14,13,0,0,20,17,0,19,0,0,22,0,26,25,0,0,28,0,0],"v":[0,8.798333e-05,0,0,-0.0006454961,0,0.0007231869,0.004766882,0,-0.007975177,0,0,0.01016804,-0.002557033,0,0,-0.0013518,0,-0.01271856,-0.00198334,0,0.003696092,0,0,0.0007465461,-0.001742686,0,-0.005661424,-0.0002768557]},{"f":[23,3,-1,18,36,-1,17,16,15,-1,-1,-1,-1,18,15,-1,-1,-1,22,2,-1,1,-1,24,-1,1,-1,-1,-1],"t":[0.5,0.9992038216560509,0,9.845435504646158,2.5,0,9.27958770372691,11.696967526234287,10.342074667999139,0,0,0,0,10.69085411329249,10.501837184902296,0,0,0,14.03140051979143,1.292481250360578,0,5.417948717948718,0,11.5,0,6.951219512195122,0,0,0],"m":[0,0,0,1,0,0,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,0,7,8,9,0,0,0,0,14,15,0,0,0,19,20,0,22,0,24,0,26,0,0,0],"r":[18,3,0,13,6,0,12,11,10,0,0,0,0,17,16,0,0,0,28,21,0,23,0,25,0,27,0,0,0],"v":[0,0,0.001664041,0,0,0.002844147,0,0,0,-0.0002009058,0.003418599,-0.002784184,0.003386913,0,0,-0.00141477,-0.007496345,0.0006935442,0,0,0.001495873,0,0.0005189424,0,-0.0004706574,0,-0.001019517,-0.007720936,-0.0001876315]},{"f":[36,-1,33,32,15,-1,22,1,-1,18,-1,22,2,-1,-1,-1,21,-1,1,19,-1,-1,-1,24,-1,-1,2,-1,-1],"t":[1.5,0,37.98858650139573,10.41931383093672,9.756556322524087,0,11.722778233171695,1.3336158192090395,0,2.903677461028802,0,11.67903991803059,1.792481250360578,0,0,0,3.5,0,8.135583053967,1.5,0,0,0,9.5,0,0,2.453445297804259,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,10,0,12,13,0,0,0,17,0,19,20,0,0,0,24,0,0,27,0,0],"r":[2,0,26,23,6,0,16,9,0,11,0,15,14,0,0,0,18,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,-0.002462274,0,0,0,8.098606e-05,0,0,-0.006796089,0,0.0007374584,0,0,0.0004913761,-0.004055702,-0.008385,0,0.002753377,0,0,-0.0002735123,0.002435681,-0.004810857,0,0.00462602,-0.0006741461,0,0.0003244513,-0.002560796]},{"f":[22,22,16,-1,2,36,-1,-1,-1,16,-1,-1,22,1,22,16,-1,-1,-1,22,-1,-1,32,-1,17,-1,18,-1,-1],"t":[16.98920034885663,16.978341770814396,6.882643049361842,0,1.292481250360578,3.5,0,0,0,5.044394119358453,0,0,17.078739781790915,4.145833333333334,17.069460217922263,7.741466986401147,0,0,0,17.02942582914998,0,0,8.341935934311302,0,8.571721503073015,0,8.729619742794132,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,0,0,0,10,0,0,13,14,15,16,0,0,0,20,0,0,23,0,25,0,27,0,0],"r":[12,9,4,0,8,7,0,0,0,11,0,0,22,19,18,17,0,0,0,21,0,0,24,0,26,0,28,0,0],"v":[0,0,0,0.0001606567,0,0,0.00684168,0.0003925926,-0.0005865316,0,-0.005100752,-0.0009147154,0,0,0,0,-0.005145517,0.000330927,0.002053248,0,0.002260854,0.005774756,0,0.003585817,0,-0.0003572266,0,-0.008953791,0.0009740543]},{"f":[23,3,-1,1,-1,-1,37,19,35,17,22,-1,21,2,-1,22,-1,-1,-1,-1,-1,17,21,15,-1,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,0,3.354166666666667,0,0,7.92,1.5,1.5,1.292481250360578,8.133841411710264,0,7.5,2.160964047443681,0,16.819976309430963,0,0,0,0,0,7.430398005367313,5.5,6.247927513443585,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,1,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,11,0,13,14,0,16,0,0,0,0,0,22,23,24,0,0,0,0,0],"r":[6,3,0,5,0,0,28,21,20,19,12,0,18,15,0,17,0,0,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0.001607007,0,-3.848277e-06,-0.0006972725,0,0,0,0,0,0.008107373,0,0,0.00229567,0,0.0009929986,-0.0001696198,-0.0001696198,-0.0001280188,0.003616468,0,0,0,0.00163203,0.00837214,0.002345359,-0.0001573003,-0.0006037852]},{"f":[23,21,22,-1,15,17,1,-1,23,-1,-1,18,-1,-1,17,-1,17,-1,33,2,-1,-1,15,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,0,5.169925001442312,5.740899715832876,1.4587811431362971,0,1.5,0,0,7.67595272073847,0,0,4.128693921346326,0,5.597993649014255,0,20.752541988539424,4.80643424864552,0,0,7.366322214245816,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,12,0,0,15,0,17,0,19,20,0,0,23,0,0,0,27,0,0],"r":[26,25,4,0,14,11,8,0,10,0,0,13,0,0,16,0,18,0,22,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,8.595028e-05,0,0,0,0.003745452,0,-0.003869236,-0.0003355775,0,-0.01150837,-0.001168371,0,0.0005564687,0,-0.003653729,0,0,0.009474035,-0.004341657,0,-0.002737734,-3.747142e-05,-0.00336997,0,0.004598386,-0.000430929]},{"f":[36,-1,25,37,2,2,4,23,-1,-1,18,-1,22,-1,-1,-1,4,-1,-1,21,-1,-1,17,-1,22,36,-1,-1,-1],"t":[1.5,0,2.5,11.05,9.266718101290426,9.051177541662383,7.675607660927803,0.5,0,0,1.292481250360578,0,12.35227706330556,0,0,0,9.328332883600808,0,0,5.5,0,0,1.292481250360578,0,12.613886328427084,2.5,0,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,0,0,17,0,0,20,0,0,23,0,25,26,0,0,0],"r":[2,0,22,19,16,15,10,9,0,0,12,0,14,0,0,0,18,0,0,21,0,0,24,0,28,27,0,0,0],"v":[0,-0.002416572,0,0,0,0,0,0,-0.0002280844,0.0001807928,0,0.006358582,0,-0.001073983,0.004070212,-0.001073983,0,0.006983041,-0.0005830859,0,-0.001073983,-0.005462382,0,0.008695283,0,0,0.002317816,-0.003307292,-0.003819489]},{"f":[22,-1,15,15,15,15,-1,3,-1,16,-1,-1,-1,18,-1,-1,32,-1,15,-1,37,1,-1,37,-1,-1,34,-1,-1],"t":[2.084962500721156,0,5.78135971352466,5.491853096329675,5.169925001442312,4.087462841250339,0,0.49242424242424243,0,6.409390936137702,0,0,0,5.491496787347155,0,0,3.753897320099348,0,6.409390936137702,0,10.469999999999999,8.046875,0,8.2,0,0,3.5,0,0],"m":[0,0,0,1,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,10,0,0,0,14,0,0,17,0,19,0,21,22,0,24,0,0,27,0,0],"r":[2,0,16,13,12,7,0,9,0,11,0,0,0,15,0,0,18,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0.001684919,0,0,0,0,5.651923e-05,0,0.0002384111,0,-0.001550403,-0.004783213,0.001551055,0,-0.001706121,0.002144406,0,0.007864098,0,0.0008903465,0,0,-6.225777e-05,0,0.01057109,0.0002528688,0,-0.004895285,-0.001105996]},{"f":[22,5,3,-1,-1,21,-1,15,-1,-1,22,22,36,-1,0,15,-1,-1,17,-1,2,1,-1,-1,-1,17,-1,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,7.5,0,10.690871009292314,0,0,12.738408632106918,12.613886328427084,2.5,0,3.5,2.0,0,0,1.792481250360578,0,1.292481250360578,7.0046728971962615,0,0,0,3.084962500721156,0,0,0],"m":[0,0,1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,0,8,0,0,11,12,13,0,15,16,0,0,19,0,21,22,0,0,0,26,0,0,0],"r":[10,5,4,0,0,7,0,9,0,0,28,25,14,0,18,17,0,0,20,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,-0.0001468307,-0.005404109,0,0.0004421178,0,-6.288428e-05,-0.003692913,0,0,0,0.003230283,0,0,-0.001010346,0.001701726,0,-0.001010346,0,0,-0.01082138,-0.003816077,-0.001010346,0,-0.001927472,-0.006145998,-2.365662e-05]},{"f":[23,22,22,22,22,22,23,22,-1,1,-1,18,-1,-1,-1,-1,-1,15,-1,-1,22,-1,-1,33,-1,-1,15,-1,-1],"t":[8.5,17.094543568094064,17.069460217922263,16.819976309430963,16.786010507999382,16.765402642760364,4.5,14.03140051979143,0,1.596471647869674,0,6.434604125275413,0,0,0,0,0,6.0,0,0,17.078739781790915,0,0,21.80651201897376,0,0,6.539158811108031,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,10,0,12,0,0,0,0,0,18,0,0,21,0,0,24,0,0,27,0,0],"r":[26,23,20,17,16,15,14,9,0,11,0,13,0,0,0,0,0,19,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,4.225677e-05,0,0.001017944,0,-0.002686682,-0.000542882,0.001250202,-0.01894445,0.002401259,0,-0.003825596,-0.0006120831,0,0.003193012,-0.0001735555,0,-0.0002682975,-0.005530896,0,0.004385427,-0.0004145359]},{"f":[22,-1,22,22,22,-1,21,-1,-1,16,-1,-1,22,21,16,2,-1,-1,-1,17,-1,-1,33,22,-1,-1,34,-1,-1],"t":[4.926654777701837,0,7.082728314749225,6.779867784305249,5.9525054311951475,0,4.5,0,0,5.78135971352466,0,0,11.36027577861459,9.5,1.584962500721156,6.451187557243013,0,0,0,1.292481250360578,0,0,29.807475161786478,11.424679641461715,0,0,2.5,0,0],"m":[0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,14,15,16,0,0,0,20,0,0,23,24,0,0,27,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,22,19,18,17,0,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0.0008498236,0,0,0,-0.003274849,0,0.0002938391,0.004450577,0,-0.002053523,-0.006466291,0,0,0,0,0.0009426283,0.007958921,0.0003427967,0,-0.005450453,0.0007309898,0,0,-0.002213943,3.829187e-06,0,-0.005273507,-0.0008666482]},{"f":[23,-1,16,22,-1,22,22,4,23,-1,-1,16,-1,-1,-1,18,-1,-1,16,36,-1,18,-1,-1,22,-1,22,-1,-1],"t":[0.5,0,7.924812503605781,16.819976309430963,0,16.98920034885663,16.978341770814396,2.160964047443681,7.5,0,0,6.930737337562887,0,0,0,8.645081439392136,0,0,8.618385502258606,4.5,0,1.292481250360578,0,0,16.765402642760364,0,16.786010507999382,0,0],"m":[0,0,1,1,0,0,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,0,0,12,0,0,0,16,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,18,5,0,15,14,11,10,0,0,13,0,0,0,17,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.0001499175,0,0,0.000645102,0,0,0,0,0.002247252,-0.001744662,0,-0.001684299,-0.01348697,-0.004151574,0,0.001145494,-0.003734569,0,0,-0.00627844,0,0.004317572,-0.001910574,0,0.0002096801,0,-0.01752395,0.000782237]},{"f":[23,21,22,15,21,22,16,16,17,-1,-1,-1,17,16,-1,-1,-1,-1,-1,25,-1,-1,0,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,17.105450904090162,5.78135971352466,9.5,16.819976309430963,5.491853096329675,5.357552004618084,11.78143940991899,0,0,0,9.267956609922159,11.031356596255709,0,0,0,0,0,2.5,0,0,7.5,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,1,1,1,1,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,13,14,0,0,0,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,22,19,18,17,12,11,10,0,0,0,16,15,0,0,0,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,0.0001148219,-0.004662423,0.006367097,0,0,-0.0008971953,-0.00388907,0.0004099928,-0.003699911,-0.006177444,0,0.0001001513,0.004713483,0,5.767251e-05,-0.005293038,-0.003275385,0,0.00426665,-0.0003626714]},{"f":[36,-1,4,2,4,4,25,-1,17,-1,-1,22,-1,-1,17,-1,22,-1,-1,1,-1,1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,2.5,0,1.292481250360578,0,0,8.658207063149977,0,0,1.292481250360578,0,12.35227706330556,0,0,1.9735450749013448,0,4.757007376185458,0,0,11.683871801979876,0,13.054613167076562,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,0,15,0,17,0,0,20,0,22,0,0,25,0,27,0,0],"r":[2,0,24,19,14,11,8,0,10,0,0,13,0,0,16,0,18,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.002255962,0,0,0,0,0,-1.234583e-05,0,0.00824318,-0.0002052668,0,-0.007114486,0.0001055721,0,0.007744476,0,-0.003161132,0.00357478,0,-0.0003137307,0,-0.00627311,0.0004237805,0,0.006310347,0,-0.002065601,0.004293481]},{"f":[22,-1,23,17,2,21,22,-1,15,-1,-1,16,-1,22,1,-1,18,-1,1,-1,-1,-1,16,-1,16,-1,-1,-1,-1],"t":[2.084962500721156,0,0.5,12.18494601563336,7.6934196157931645,6.5,13.747003318603246,0,6.392317422778761,0,0,2.807354922057604,0,11.612768260520431,1.13125,0,3.084962500721156,0,2.9055555555555554,0,0,0,1.584962500721156,0,9.862637357558794,0,0,0,0],"m":[0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,14,15,0,17,0,19,0,0,0,23,0,25,0,0,0,0],"r":[2,0,28,27,22,11,8,0,10,0,0,13,0,21,16,0,18,0,20,0,0,0,24,0,26,0,0,0,0],"v":[0,0.001561701,0,0,0,0,0,7.341584e-06,0,0.0003634437,-0.002377779,0,-0.008746844,0,0,-0.003438381,0,0.004262687,0,0.0007781277,-0.005742483,-0.007257761,0,0.00739253,0,-0.001099774,0.003676724,-0.004797887,0.000121491]},{"f":[23,21,22,-1,17,-1,15,4,22,33,-1,15,4,-1,-1,-1,-1,2,-1,-1,15,18,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,0,3.39067985676233,0,7.366322214245816,4.951187557243013,16.819976309430963,17.460839126440824,0,6.247927513443585,3.39067985676233,0,0,0,0,5.8698903048866296,0,0,10.457380879072534,9.135888799148791,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,0,1,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,12,13,0,0,0,0,18,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,4,0,6,0,20,17,16,11,0,15,14,0,0,0,0,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,8.263441e-05,0,0.0002976935,0,0,0,0,0.006657913,0,0,-0.000768946,0.003152687,-0.00745933,-0.004121281,0,-0.008406182,-2.940014e-05,0,0,0.0001978888,0.005689106,-0.001304733,-0.003107577,0,0.004050013,-0.000350304]},{"f":[22,22,-1,18,-1,-1,22,1,22,16,-1,-1,-1,22,-1,-1,32,-1,18,18,22,-1,0,-1,22,-1,-1,-1,-1],"t":[16.98920034885663,16.98413870862295,0,1.292481250360578,0,0,17.078739781790915,4.145833333333334,17.069460217922263,7.741466986401147,0,0,0,17.02942582914998,0,0,8.341935934311302,0,8.645081439392136,7.383290334558488,17.07907323966382,0,3.5,0,17.094543568094064,0,0,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,0,0,0,14,0,0,17,0,19,20,21,0,23,0,25,0,0,0,0],"r":[6,3,0,5,0,0,16,13,12,11,0,0,0,15,0,0,18,0,28,27,22,0,24,0,26,0,0,0,0],"v":[0,0,-3.376369e-06,0,-0.006774533,-0.002406454,0,0,0,0,-0.00476065,0.0002941662,0.001778816,0,0.002066418,0.005164478,0,0.003064534,0,0,0,-0.003792044,0,0.004653216,0,0.0003845733,-0.004461048,0.001004062,-0.003895381]},{"f":[22,-1,22,22,22,-1,21,-1,-1,16,-1,-1,22,-1,33,22,-1,22,16,-1,15,-1,-1,-1,23,33,-1,-1,-1],"t":[4.926654777701837,0,7.082728314749225,6.779867784305249,5.9525054311951475,0,4.5,0,0,5.78135971352466,0,0,11.406459975416917,0,29.807475161786478,11.424679641461715,0,12.199665042528586,10.462502272597265,0,10.681238411777805,0,0,0,5.5,41.975696368331896,0,0,0],"m":[0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,0,15,16,0,18,19,0,21,0,0,0,25,26,0,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,14,0,24,17,0,23,20,0,22,0,0,0,28,27,0,0,0],"v":[0,0.0007900656,0,0,0,-0.003060302,0,0.0002926971,0.004292125,0,-0.001865108,-0.005913039,0,0.0003571797,0,0,-0.002168099,0,0,0.001124263,0,-0.004251059,-0.0001168441,-9.849168e-05,0,0,-0.00395163,-0.0001293913,0.000315461]},{"f":[23,18,2,15,17,18,18,-1,2,22,-1,24,-1,-1,-1,19,-1,-1,33,-1,-1,1,-1,-1,-1,21,-1,-1,-1],"t":[0.5,9.845435504646158,9.266718101290426,10.113742166049189,6.924507036847409,5.108372929097653,1.792481250360578,0,1.292481250360578,7.5621616439616925,0,4.5,0,0,0,0.5,0,0,16.993854061608424,0,0,1.9735450749013448,0,0,0,3.5,0,0,0],"m":[0,1,1,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,10,0,12,0,0,0,16,0,0,19,0,0,22,0,0,0,26,0,0,0],"r":[28,25,24,21,18,15,8,0,14,11,0,13,0,0,0,17,0,0,20,0,0,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0001100094,0,0,-0.0004201162,0,-0.003618756,-0.009923715,0.00233278,0,0.0006796708,0.006542802,0,-0.002360472,0.00111584,0,-0.003755078,0.002661486,0.004320589,0,0.0007939761,-0.001772633,0.0001214759]},{"f":[36,-1,4,2,4,4,2,-1,-1,22,-1,-1,18,-1,22,-1,-1,15,18,1,-1,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,6.451187557243013,0,0,8.658207063149977,0,0,1.292481250360578,0,12.35227706330556,0,0,10.462502272597265,1.292481250360578,1.3897752394988947,0,0,0,0,11.683871801979876,0,13.054613167076562,0,0],"m":[0,0,1,1,1,1,1,0,0,1,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,0,15,0,0,18,19,20,0,0,0,0,25,0,27,0,0],"r":[2,0,24,17,12,9,8,0,0,11,0,0,14,0,16,0,0,23,22,21,0,0,0,0,26,0,28,0,0],"v":[0,-0.002158154,0,0,0,0,0,-2.957378e-05,0.0006850247,0,-0.006563889,8.34556e-05,0,0.006617335,0,-0.003116315,0.003158495,0,0,0,-0.001053707,-0.004691999,0.002632037,-0.005669632,0,0.005783454,0,-0.00194664,0.004050389]},{"f":[33,32,15,15,15,-1,-1,-1,22,1,-1,18,-1,22,2,-1,-1,-1,21,-1,-1,24,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,8.005624549193879,7.924812503605781,0,0,0,11.722778233171695,1.3336158192090395,0,2.903677461028802,0,11.67903991803059,1.792481250360578,0,0,0,3.5,0,0,9.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,0,9,10,0,12,0,14,15,0,0,0,19,0,0,22,0,0,25,0,27,0,0],"r":[24,21,8,7,6,0,0,0,18,11,0,13,0,17,16,0,0,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,4.02069e-05,-0.002209834,0.001020459,0,0,-0.006030399,0,0.0006427194,0,0,0.0005280114,-0.003647899,-0.007735929,0,0.002482945,-0.0001947889,0,0.004316062,-0.0004726686,0,0.0003527961,0,0.0001197512,-0.003662279]},{"f":[15,15,-1,22,-1,16,22,1,-1,-1,16,1,-1,-1,-1,-1,32,-1,15,-1,37,1,-1,37,-1,-1,34,-1,-1],"t":[5.78135971352466,4.087462841250339,0,10.63843335887114,0,6.409390936137702,11.775312925643524,4.008411214953271,0,0,6.392317422778761,1.5277777777777777,0,0,0,0,3.753897320099348,0,6.409390936137702,0,10.469999999999999,8.046875,0,8.2,0,0,3.5,0,0],"m":[0,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,0,4,0,6,7,8,0,0,11,12,0,0,0,0,17,0,19,0,21,22,0,24,0,0,27,0,0],"r":[16,3,0,5,0,15,10,9,0,0,14,13,0,0,0,0,18,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0,7.750111e-05,0,-0.002673763,0,0,0,0.0005947175,0.005859088,0,0,0.001053874,-0.000919968,0.002386065,-0.002100026,0,0.007090718,0,0.0007969204,0,0,-4.87447e-05,0,0.009857951,0.0002760498,0,-0.004559945,-0.001018229]},{"f":[22,-1,21,15,17,1,-1,23,-1,-1,18,-1,-1,33,2,-1,22,2,-1,-1,-1,22,15,-1,15,-1,-1,-1,-1],"t":[14.03140051979143,0,9.5,5.169925001442312,5.740899715832876,1.4587811431362971,0,1.5,0,0,7.67595272073847,0,0,17.460839126440824,1.792481250360578,0,16.725912049439096,5.065928480304397,0,0,0,14.158358850608169,6.539158811108031,0,7.266786540694901,0,0,0,0],"m":[1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,0,0,14,15,0,17,18,0,0,0,22,23,0,25,0,0,0,0],"r":[2,0,28,13,10,7,0,9,0,0,12,0,0,21,16,0,20,19,0,0,0,27,24,0,26,0,0,0,0],"v":[0,7.386097e-05,0,0,0,0,0.003542722,0,-0.003501637,-0.0002069972,0,-0.0107353,-0.001112915,0,0,0.008680086,0,0,-0.001579797,0.006045088,-0.002815618,0,0,-0.0005727615,0,-0.01017883,-0.00196966,-7.224743e-05,0.00235909]},{"f":[23,-1,16,22,-1,22,22,4,23,-1,-1,16,-1,-1,-1,18,-1,-1,16,36,-1,18,-1,-1,22,-1,22,-1,-1],"t":[0.5,0,7.924812503605781,16.819976309430963,0,16.98920034885663,16.978341770814396,2.160964047443681,7.5,0,0,6.930737337562887,0,0,0,8.645081439392136,0,0,8.618385502258606,4.5,0,1.292481250360578,0,0,16.765402642760364,0,16.786010507999382,0,0],"m":[0,0,1,1,0,0,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,0,0,12,0,0,0,16,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,18,5,0,15,14,11,10,0,0,13,0,0,0,17,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.0001369256,0,0,0.0006094291,0,0,0,0,0.002141028,-0.001678012,0,-0.001525937,-0.01284968,-0.003799672,0,0.001029466,-0.003370063,0,0,-0.005942722,0,0.004038239,-0.001767516,0,0.0001978382,0,-0.01663751,0.0006432123]},{"f":[23,21,22,-1,15,17,1,-1,23,-1,-1,18,-1,-1,33,2,-1,-1,22,15,-1,21,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,0,5.169925001442312,5.740899715832876,1.4587811431362971,0,1.5,0,0,7.67595272073847,0,0,17.460839126440824,1.792481250360578,0,0,14.158358850608169,6.700439718141092,0,7.5,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,12,0,0,15,16,0,0,19,20,0,22,0,0,0,0,27,0,0],"r":[26,25,4,0,14,11,8,0,10,0,0,13,0,0,18,17,0,0,24,21,0,23,0,0,0,0,28,0,0],"v":[0,0,0,7.703898e-05,0,0,0,0.003367635,0,-0.003353729,-0.0002132143,0,-0.01022805,-0.001035555,0,0,0.00823629,1.849241e-05,0,0,-0.0005091394,0,-0.007789537,0.001081442,-1.012293e-05,-0.002990583,0,0.003821322,-0.0002921859]},{"f":[23,3,-1,32,22,16,15,16,-1,-1,-1,16,-1,22,15,1,1,-1,-1,-1,-1,-1,15,-1,-1,2,-1,-1,-1],"t":[0.5,0.9992038216560509,0,7.014643613484123,14.03140051979143,6.882643049361842,5.882643049361842,5.491853096329675,0,0,0,7.614709844115208,0,11.876328901062104,9.264442600226602,3.9935774037849088,1.13125,0,0,0,0,0,6.247927513443585,0,0,1.792481250360578,0,0,0],"m":[0,0,0,1,1,1,1,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,0,4,5,6,7,8,0,0,0,12,0,14,15,16,17,0,0,0,0,0,23,0,0,26,0,0,0],"r":[28,3,0,25,22,11,10,9,0,0,0,13,0,21,20,19,18,0,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0.001425263,0,0,0,0,0,1.875726e-05,-0.001628508,0.0007070939,0,-0.002263027,0,0,0,0,-0.002670426,0.002657354,-0.001510439,-0.002030759,0.001330384,0,-0.001548538,-0.008793492,0,0.004513271,0.000302646,0.0001177519]},{"f":[36,-1,4,2,4,4,2,-1,21,-1,-1,22,-1,-1,17,-1,22,-1,-1,1,-1,1,-1,-1,22,-1,15,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,6.451187557243013,0,5.5,0,0,8.658207063149977,0,0,1.292481250360578,0,12.35227706330556,0,0,1.9735450749013448,0,4.757007376185458,0,0,11.683871801979876,0,9.862637357558794,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,0,15,0,17,0,0,20,0,22,0,0,25,0,27,0,0],"r":[2,0,24,19,14,11,8,0,10,0,0,13,0,0,16,0,18,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.00209783,0,0,0,0,0,-2.786771e-05,0,0.002735719,0.0002540653,0,-0.006259572,0.000124987,0,0.006847725,0,-0.002689067,0.003130108,0,-0.0002232753,0,-0.005860855,0.0002805306,0,0.005567352,0,-0.002592908,0.00267481]},{"f":[22,-1,23,18,2,-1,-1,17,16,-1,-1,18,-1,-1,16,22,-1,-1,16,36,-1,18,-1,-1,22,-1,22,-1,-1],"t":[2.084962500721156,0,0.5,10.543869317296533,9.266718101290426,0,0,10.682994316786338,9.271463027904375,0,0,11.651826559171445,0,0,7.924812503605781,16.819976309430963,0,0,8.618385502258606,4.5,0,1.292481250360578,0,0,16.765402642760364,0,16.786010507999382,0,0],"m":[0,0,0,1,1,0,0,1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,0,0,15,16,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,14,7,6,0,0,11,10,0,0,13,0,0,18,17,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,0.001446506,0,0,0,-0.0001203019,0.004447053,0,0,-0.001707415,-0.008834474,0,0.00164285,-0.002757866,0,0,0.0005789364,-0.0003464138,0,0,-0.005645803,0,0.003860772,-0.001688811,0,0.0001822606,0,-0.0158303,0.0005827458]},{"f":[22,-1,22,-1,15,22,21,16,16,-1,-1,-1,-1,-1,32,-1,15,-1,22,-1,37,1,-1,37,-1,-1,34,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,5.78135971352466,16.819976309430963,9.5,5.491853096329675,5.357552004618084,0,0,0,0,0,3.753897320099348,0,6.409390936137702,0,6.930301023985015,0,10.469999999999999,8.046875,0,8.2,0,0,3.5,0,0],"m":[0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,0,0,0,0,0,15,0,17,0,19,0,21,22,0,24,0,0,27,0,0],"r":[2,0,4,0,14,13,12,11,10,0,0,0,0,0,16,0,18,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0.0007542944,0,-0.002886802,0,0,0,0,0,4.152573e-05,0.005982891,-0.0006653283,-0.004336417,-0.002994154,0,0.006652959,0,0.0007267351,0,-0.004439108,0,0,-1.939365e-05,0,0.00931669,0.0002897662,0,-0.0042452,-0.0009273663]},{"f":[1,1,-1,18,-1,-1,2,22,22,4,4,4,19,15,-1,-1,-1,20,-1,36,-1,-1,-1,-1,-1,22,-1,-1,-1],"t":[5.417948717948718,5.015625,0,4.245926548164837,0,0,9.068775776507199,16.98920034885663,16.98413870862295,6.499295214872664,5.89476682248518,2.453445297804259,0.5,6.6293566200796095,0,0,0,0.5,0,2.5,0,0,0,0,0,17.07907323966382,0,0,0],"m":[1,1,0,1,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,11,12,13,14,0,0,0,18,0,20,0,0,0,0,0,26,0,0,0],"r":[6,3,0,5,0,0,28,25,24,23,22,17,16,15,0,0,0,19,0,21,0,0,0,0,0,27,0,0,0],"v":[0,0,3.054826e-05,0,0.000956512,0.007231326,0,0,0,0,0,0,0,0,-0.004528667,-0.00010785,0.000724614,0,0.0002231075,0,-0.005093931,-0.001245992,0.002981015,-0.003742265,-0.004722568,0,0.003446213,-0.0007078999,-0.005587921]},{"f":[32,33,1,1,-1,2,17,2,17,19,-1,-1,-1,16,-1,-1,-1,-1,-1,1,17,22,-1,-1,-1,-1,4,-1,-1],"t":[10.463013322934223,29.75825500004914,12.64516129032258,5.417948717948718,0,9.068775776507199,2.160964047443681,1.292481250360578,1.792481250360578,0.5,0,0,0,6.523561956057013,0,0,0,0,0,9.107199999999999,7.6181566135214025,16.978341770814396,0,0,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,7,8,9,10,0,0,0,14,0,0,0,0,0,20,21,22,0,0,0,0,27,0,0],"r":[26,19,18,5,0,17,16,13,12,11,0,0,0,15,0,0,0,0,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,7.402612e-05,0,0,0,0,0,-0.003259259,0.0006726001,-0.006742363,0,0.001708553,-0.001261852,0.0002075453,-0.005314258,0.002664027,0,0,0,0.003615468,-0.000872668,-0.001348,-0.003562318,0,0.002818553,-0.001098805]},{"f":[23,21,22,22,22,22,22,23,-1,-1,-1,-1,15,-1,21,-1,4,-1,-1,22,-1,-1,0,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,17.105450904090162,17.069460217922263,16.819976309430963,16.786010507999382,16.765402642760364,4.5,0,0,0,0,6.0,0,5.5,0,1.792481250360578,0,0,17.078739781790915,0,0,7.5,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,13,0,15,0,17,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,22,19,12,11,10,9,0,0,0,0,14,0,16,0,18,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,-4.996044e-05,0.001230839,-0.01605699,0.002119695,0,-0.003204934,0,0.00122844,0,-0.0003060574,-0.002779332,0,0.002747903,-0.0003260818,0,7.280016e-05,-0.004831306,-0.002812094,0,0.003648395,-0.0002760084]},{"f":[22,5,3,-1,-1,19,-1,-1,22,22,36,-1,0,22,-1,-1,17,1,-1,-1,2,1,-1,-1,-1,4,-1,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,0.5,0,0,12.738408632106918,12.613886328427084,2.5,0,3.5,12.457649623491124,0,0,1.792481250360578,6.015625,0,0,1.292481250360578,7.0046728971962615,0,0,0,1.792481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,14,0,0,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[8,5,4,0,0,7,0,0,28,25,12,0,16,15,0,0,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,-5.634817e-05,-0.004902321,0,0.0001012255,0.000845752,0,0,0,0.002915845,0,0,-0.0006431069,0.002321333,0,0,-0.00322891,-0.000904178,0,0,-0.009654057,-0.003144261,-0.0009863841,0,-0.001518393,-0.006126326,-1.48295e-05]},{"f":[22,-1,22,22,22,-1,21,-1,-1,16,-1,-1,22,-1,33,22,-1,22,16,15,-1,-1,15,-1,-1,-1,34,-1,-1],"t":[4.926654777701837,0,7.082728314749225,6.779867784305249,5.9525054311951475,0,4.5,0,0,5.78135971352466,0,0,11.406459975416917,0,29.807475161786478,11.424679641461715,0,12.199665042528586,10.462502272597265,10.342074667999139,0,0,10.681238411777805,0,0,0,2.5,0,0],"m":[0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,0,15,16,0,18,19,20,0,0,23,0,0,0,27,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,14,0,26,17,0,25,22,21,0,0,24,0,0,0,28,0,0],"v":[0,0.0007103753,0,0,0,-0.002754747,0,0.0002539278,0.004153117,0,-0.001665104,-0.005173911,0,0.0003252665,0,0,-0.001954678,0,0,0,0.0006016322,0.003108028,0,-0.004028684,-1.427674e-06,-8.553815e-05,0,-0.004791301,-0.0007288458]},{"f":[23,3,-1,32,22,16,-1,16,-1,22,15,0,-1,-1,15,-1,17,2,-1,-1,17,-1,-1,-1,15,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,0,7.014643613484123,14.03140051979143,6.882643049361842,0,7.614709844115208,0,11.876328901062104,9.264442600226602,4.5,0,0,10.376125389276174,0,10.482680779086738,1.292481250360578,0,0,10.69085411329249,0,0,0,6.247927513443585,0,0,0,0],"m":[0,0,0,1,1,1,0,0,0,1,1,1,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,4,5,6,0,8,0,10,11,12,0,0,15,0,17,18,0,0,21,0,0,0,25,0,0,0,0],"r":[28,3,0,27,24,7,0,9,0,23,14,13,0,0,16,0,20,19,0,0,22,0,0,0,26,0,0,0,0],"v":[0,0,0.001367944,0,0,0,2.505001e-05,0,-0.002135941,0,0,0,0.001347011,-0.002243548,0,-0.003537883,0,0,0.002157003,-0.001692774,0,-0.007562946,-0.0004471385,0.001250076,0,-0.001488116,-0.008369004,0.001610831,0.0001091043]},{"f":[36,-1,4,2,4,4,2,-1,21,-1,-1,22,-1,-1,18,-1,22,-1,-1,1,-1,1,-1,-1,22,-1,15,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,6.451187557243013,0,5.5,0,0,8.658207063149977,0,0,1.292481250360578,0,12.35227706330556,0,0,1.9735450749013448,0,4.757007376185458,0,0,11.683871801979876,0,9.862637357558794,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,0,15,0,17,0,0,20,0,22,0,0,25,0,27,0,0],"r":[2,0,24,19,14,11,8,0,10,0,0,13,0,0,16,0,18,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.001958907,0,0,0,0,0,-2.906243e-05,0,0.002575849,0.00026768,0,-0.005863769,0.000130918,0,0.005974723,0,-0.002689116,0.002799475,0,-0.0001705474,0,-0.005583306,0.0004162532,0,0.005227215,0,-0.002406732,0.002711458]},{"f":[4,-1,18,16,15,15,18,-1,-1,-1,36,-1,-1,4,-1,18,-1,-1,16,16,17,-1,-1,-1,21,-1,16,-1,-1],"t":[4.0437314206251695,0,8.03615005796417,7.266786540694901,5.78135971352466,5.672425341971495,5.108372929097653,0,0,0,3.5,0,0,4.2849278041654735,0,5.065928480304397,0,0,9.862637357558794,4.247927513443585,10.166483743581448,0,0,0,5.5,0,11.620219825507487,0,0],"m":[1,0,1,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,0,0,14,0,16,0,0,19,20,21,0,0,0,25,0,27,0,0],"r":[2,0,18,13,10,9,8,0,0,0,12,0,0,15,0,17,0,0,24,23,22,0,0,0,26,0,28,0,0],"v":[0,6.40233e-05,0,0,0,0,0,-0.0002381315,0.002703239,-0.00247674,0,-0.001987354,0.002018326,0,-0.005125966,0,6.249178e-06,-0.002227198,0,0,0,0.004892178,-0.001049616,-0.001827069,0,0.003764637,0,-0.002423252,0.002004484]},{"f":[22,22,22,23,2,-1,22,22,22,-1,-1,-1,22,2,-1,-1,-1,-1,-1,-1,22,1,-1,-1,32,-1,18,-1,-1],"t":[16.98920034885663,16.978341770814396,16.975010474608837,7.5,1.292481250360578,0,16.819976309430963,16.786010507999382,16.765402642760364,0,0,0,16.903290731167747,2.453445297804259,0,0,0,0,0,0,17.078739781790915,4.008411214953271,0,0,8.341935934311302,0,8.645081439392136,0,0],"m":[1,1,1,1,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,7,8,9,0,0,0,13,14,0,0,0,0,0,0,21,22,0,0,25,0,27,0,0],"r":[20,19,18,17,6,0,12,11,10,0,0,0,16,15,0,0,0,0,0,0,24,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,0.0002207142,0,0,0,-6.072964e-05,-0.01512676,0.002558282,0,0,-0.01208095,-0.002155289,0.001014508,-0.003961666,0.002136193,-0.00297184,0,0,7.771822e-05,0.002857635,0,0.002805432,0,-0.0003595626,-0.003550199]},{"f":[23,21,1,33,22,22,-1,17,-1,-1,15,-1,-1,-1,4,22,-1,17,15,-1,-1,-1,24,-1,16,-1,-1,-1,-1],"t":[8.5,10.5,5.417948717948718,27.680602362796474,16.759160379664443,12.027747761539269,0,10.482680779086738,0,0,9.002815015607053,0,0,0,6.499295214872664,17.094543568094064,0,8.729619742794132,6.584962500721156,0,0,0,13.5,0,7.774787059601174,0,0,0,0],"m":[1,1,1,0,1,1,0,1,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,11,0,0,0,15,16,0,18,19,0,0,0,23,0,25,0,0,0,0],"r":[28,27,14,13,10,7,0,9,0,0,12,0,0,0,22,17,0,21,20,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0,0,0,0,0.0004262974,0,0.004656002,0.00107158,0,0.0008269194,-0.006394442,1.79123e-05,0,0,-4.327794e-05,0,0,-0.01080287,-0.001415317,0.002344932,0,-0.008710743,0,0.002373557,-0.003212886,-0.002655626,0.001703714]},{"f":[23,3,4,-1,-1,32,22,16,18,-1,-1,16,-1,22,15,0,-1,-1,-1,-1,33,0,-1,33,-1,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,3.753897320099348,0,0,7.014643613484123,14.03140051979143,6.882643049361842,3.6427011094311244,0,0,7.614709844115208,0,11.876328901062104,9.264442600226602,4.5,0,0,0,0,21.257266042509173,1.5,0,15.968216440900115,0,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,0,0,0,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,0,12,0,14,15,16,0,0,0,0,21,22,0,24,0,0,0,0,0],"r":[28,5,4,0,0,27,20,11,10,0,0,13,0,19,18,17,0,0,0,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0.0003820315,0.003918013,0,0,0,0,-0.000172795,0.0006588985,0,-0.00202995,0,0,0,0.001287546,-0.002131702,-0.001788794,0.001159494,0,0,0.001666224,0,-0.01018026,-0.002799454,0.001588703,0.001529847,0.0001074958]},{"f":[33,32,15,15,-1,-1,22,1,-1,18,-1,22,2,-1,-1,-1,21,-1,1,19,-1,-1,-1,36,-1,-1,3,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,8.005624549193879,0,0,11.722778233171695,1.3336158192090395,0,2.903677461028802,0,11.67903991803059,1.792481250360578,0,0,0,3.5,0,8.135583053967,1.5,0,0,0,3.5,0,0,0.5187999999999999,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,0,10,0,12,13,0,0,0,17,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,6,5,0,0,16,9,0,11,0,15,14,0,0,0,18,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-8.62322e-06,0.0008990051,0,0,-0.005460063,0,0.0007143942,0,0,0.0007202362,-0.003114002,-0.007094419,0,0.002120899,0,0,-0.0002511627,0.002048729,-0.004018284,0,0.004112792,-8.696082e-05,0,-0.002552372,-3.201305e-05]},{"f":[22,22,-1,17,-1,-1,22,1,22,16,-1,-1,18,-1,-1,-1,32,-1,18,18,22,-1,0,-1,22,-1,-1,-1,-1],"t":[16.98920034885663,16.98413870862295,0,1.292481250360578,0,0,17.078739781790915,4.008411214953271,17.069460217922263,7.741466986401147,0,0,6.076776015854056,0,0,0,8.341935934311302,0,8.645081439392136,7.383290334558488,17.07907323966382,0,3.5,0,17.094543568094064,0,0,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,0,0,13,0,0,0,17,0,19,20,21,0,23,0,25,0,0,0,0],"r":[6,3,0,5,0,0,16,15,12,11,0,0,14,0,0,0,18,0,28,27,22,0,24,0,26,0,0,0,0],"v":[0,0,-2.512974e-06,0,-0.00591185,-0.001773398,0,0,0,0,-0.004517477,-2.608488e-05,0,0.003900517,0.0001681185,0.002721446,0,0.002734387,0,0,0,-0.003533662,0,0.004406116,0,0.0004792144,-0.004006911,0.0009765292,-0.003252435]},{"f":[22,-1,23,18,17,32,22,-1,33,0,-1,33,-1,-1,-1,18,-1,-1,-1,21,17,-1,-1,15,-1,2,-1,-1,-1],"t":[2.084962500721156,0,0.5,9.314012697712199,9.27958770372691,7.014643613484123,14.03140051979143,0,21.257266042509173,1.5,0,15.968216440900115,0,0,0,1.292481250360578,0,0,0,4.5,10.009557117932596,0,0,9.264442600226602,0,1.292481250360578,0,0,0],"m":[0,0,0,1,1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,0,12,0,0,0,16,0,0,0,20,21,0,0,24,0,26,0,0,0],"r":[2,0,28,19,18,15,8,0,14,11,0,13,0,0,0,17,0,0,0,23,22,0,0,25,0,27,0,0,0],"v":[0,0.001297662,0,0,0,0,0,-7.061512e-05,0,0,0.001585537,0,-0.009681878,-0.002660953,0.00151122,0,-0.0004690263,0.004820542,0.003425479,0,0,-0.002553522,0.0007660554,0,-0.0007135855,0,-0.008094388,-0.001788625,0.0001037762]},{"f":[33,32,15,-1,22,1,-1,16,18,-1,-1,-1,23,-1,36,-1,15,23,-1,-1,-1,24,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,0,11.722778233171695,1.3336158192090395,0,11.193525360501216,10.463013322934223,0,0,0,0.5,0,3.5,0,10.066089190457772,1.5,0,0,0,9.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,8,9,0,0,0,13,0,15,0,17,18,0,0,0,22,0,0,25,0,27,0,0],"r":[24,21,4,0,12,7,0,11,10,0,0,0,14,0,16,0,20,19,0,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,5.259997e-05,0,0,-0.005192361,0,0,-0.001151076,-0.006546692,0.001132571,0,0.00213737,0,-0.005321718,0,0,7.600255e-05,-0.00658604,5.294126e-05,0,0.0035702,-0.000429869,0,0.0003584224,0,0.0004522913,-0.003477152]},{"f":[15,15,-1,22,-1,16,22,1,-1,-1,16,-1,15,-1,22,22,22,-1,-1,-1,-1,-1,32,-1,1,21,-1,-1,-1],"t":[5.78135971352466,4.087462841250339,0,10.63843335887114,0,6.409390936137702,11.775312925643524,4.008411214953271,0,0,5.169925001442312,0,5.357552004618084,0,13.747003318603246,13.689894219688643,13.129761635363703,0,0,0,0,0,3.753897320099348,0,12.069111969111969,8.5,0,0,0],"m":[0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,0,4,0,6,7,8,0,0,11,0,13,0,15,16,17,0,0,0,0,0,23,0,25,26,0,0,0],"r":[22,3,0,5,0,21,10,9,0,0,12,0,14,0,20,19,18,0,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,6.450417e-05,0,-0.002406574,0,0,0,0.0005449012,0.005206974,0,-0.001272448,0,0.001170977,0,0,0,-0.003463913,0.001278928,-0.003091262,0.001125258,-0.001924615,0,0.006520018,0,0,3.295861e-05,0.0012329,-0.002360277]},{"f":[36,-1,4,2,4,4,2,-1,-1,22,-1,-1,17,-1,18,-1,-1,15,17,23,-1,-1,-1,-1,22,-1,15,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,7.005613627711627,0,0,8.658207063149977,0,0,1.292481250360578,0,8.14580120349141,0,0,10.794415866350105,9.44286754736589,1.5,0,0,0,0,11.683871801979876,0,9.862637357558794,0,0],"m":[0,0,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,0,15,0,0,18,19,20,0,0,0,0,25,0,27,0,0],"r":[2,0,24,17,12,9,8,0,0,11,0,0,14,0,16,0,0,23,22,21,0,0,0,0,26,0,28,0,0],"v":[0,-0.001856066,0,0,0,0,0,-6.606193e-06,0.001380305,0,-0.005576411,0.0001794978,0,0.006216567,0,-0.002902851,0.002331707,0,0,0,-0.0004296602,-0.005558775,0.001476597,-0.005253501,0,0.004999836,0,-0.00221527,0.002628043]},{"f":[1,2,-1,-1,3,22,22,3,-1,2,-1,-1,-1,-1,25,37,-1,25,-1,-1,17,-1,1,-1,22,36,-1,-1,-1],"t":[0.39811320754716983,6.08730682361797,0,0,0.05277777777777778,16.759160379664443,16.73920708776685,0.007093253968253968,0,8.612669628062957,0,0,0,0,2.5,9.629999999999999,0,1.5,0,0,1.292481250360578,0,5.417948717948718,0,12.613886328427084,2.5,0,0,0],"m":[0,1,0,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,10,0,0,0,0,15,16,0,18,0,0,21,0,23,0,25,26,0,0,0],"r":[4,3,0,0,14,13,12,9,0,11,0,0,0,0,20,17,0,19,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,6.107634e-05,0.003117912,0,0,0,0,0.00197917,0,-0.001145052,-0.004067211,0.003267748,-0.003782064,0,0,8.086662e-06,0,-0.0007867851,-0.00540863,0,0.00624983,0,0.002134897,0,0,0.002428852,-0.00260524,-0.00520821]},{"f":[4,-1,18,16,15,15,22,-1,-1,1,-1,-1,32,-1,-1,4,-1,18,-1,15,-1,-1,16,-1,21,-1,16,-1,-1],"t":[4.0437314206251695,0,8.03615005796417,7.266786540694901,5.78135971352466,3.584962500721156,13.747003318603246,0,0,3.822916666666667,0,0,6.208926257442949,0,0,4.2849278041654735,0,5.065928480304397,0,10.457380879072534,0,0,9.862637357558794,0,5.5,0,11.620219825507487,0,0],"m":[1,0,1,0,1,1,1,0,0,1,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,0,0,16,0,18,0,20,0,0,23,0,25,0,27,0,0],"r":[2,0,22,15,12,9,8,0,0,11,0,0,14,0,0,17,0,19,0,21,0,0,24,0,26,0,28,0,0],"v":[0,5.946979e-05,0,0,0,0,0,5.320867e-05,0.004290571,0,-0.002526286,7.059604e-05,0,0.001968663,-0.001686263,0,-0.004828448,0,-2.154447e-06,0,-0.002842297,4.353141e-05,0,-0.0003605585,0,0.003417106,0,-0.002070403,0.001919204]},{"f":[23,-1,21,17,-1,22,36,20,-1,-1,17,-1,33,-1,3,-1,-1,37,15,22,-1,-1,-1,-1,37,18,-1,-1,-1],"t":[0.5,0,5.5,3.753897320099348,0,15.717370311433935,3.5,0.5,0,0,5.918419679874358,0,27.680602362796474,0,0.5187999999999999,0,0,7.92,6.0,16.819976309430963,0,0,0,0,11.73,6.562222230004293,0,0,0],"m":[0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,0,13,0,15,0,0,18,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,17,10,9,0,0,12,0,14,0,16,0,0,23,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.0001155589,0,0,0.0009379896,0,0,0,-0.0007366881,-0.004166354,0,-0.003333163,0,0.005354046,0,-0.0005402187,0.001323115,0,0,0,0.002023685,-0.003614597,0.00394732,-0.008514864,0,0,-0.0003350666,0.0004423764,-0.004667042]},{"f":[22,22,22,23,2,-1,22,22,22,23,-1,-1,-1,-1,22,2,-1,-1,-1,-1,-1,-1,22,1,-1,-1,32,-1,-1],"t":[16.98920034885663,16.978341770814396,16.975010474608837,7.5,1.292481250360578,0,16.819976309430963,16.786010507999382,16.765402642760364,4.5,0,0,0,0,16.903290731167747,2.453445297804259,0,0,0,0,0,0,17.078739781790915,4.145833333333334,0,0,8.341935934311302,0,0],"m":[1,1,1,1,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,9,10,0,0,0,0,15,16,0,0,0,0,0,0,23,24,0,0,27,0,0],"r":[22,21,20,19,6,0,14,13,12,11,0,0,0,0,18,17,0,0,0,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,0.000202116,0,0,0,0,-0.0001536971,0.001148689,-0.01416969,0.002395941,0,0,-0.01149336,-0.00200838,0.0009905481,-0.003800441,0.00204389,-0.002698425,0,0,0.0002791404,0.002819429,0,0.002588877,-0.000691701]},{"f":[2,2,21,-1,16,-1,-1,15,-1,-1,2,25,-1,-1,1,18,2,4,-1,-1,-1,22,16,-1,16,-1,-1,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,0,11.98370619265935,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,8.992857142857144,9.229707804263573,9.025136904799169,7.997154617983022,0,0,0,12.921158001713904,10.372865060112588,0,12.683433292835403,0,0,0,0],"m":[1,1,1,0,1,0,0,1,0,0,0,0,0,0,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,0,0,11,12,0,0,15,16,17,18,0,0,0,22,23,0,25,0,0,0,0],"r":[10,7,4,0,6,0,0,9,0,0,14,13,0,0,28,21,20,19,0,0,0,27,24,0,26,0,0,0,0],"v":[0,0,0,-6.15553e-05,0,0.001543059,-0.003638125,0,-0.0001740254,-0.008541512,0,0,0.0005738365,0.007923623,0,0,0,0,5.390971e-05,0.002387452,-0.001055135,0,0,-0.001194376,0,0.004187256,0.000543837,0.005935314,-0.003006729]},{"f":[22,-1,22,-1,23,18,2,33,33,2,-1,-1,-1,22,-1,15,33,-1,-1,-1,-1,17,17,-1,-1,17,-1,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,0.5,10.543869317296533,9.266718101290426,17.997194725799872,9.573602462471115,1.292481250360578,0,0,0,14.03140051979143,0,6.247927513443585,21.257266042509173,0,0,0,0,10.682994316786338,10.609627990977145,0,0,11.651826559171445,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,10,0,0,0,14,0,16,17,0,0,0,0,22,23,0,0,26,0,0,0],"r":[2,0,4,0,28,21,20,13,12,11,0,0,0,15,0,19,18,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0.0006658727,0,-0.002580365,0,0,0,0,0,0,-0.002358538,0.002123783,0.00588597,0,-9.564289e-05,0,0,-0.002266898,0.001443234,-0.007393831,0.003968681,0,0,-0.001516227,-0.00686993,0,0.001255861,-0.002241447,9.827273e-05]},{"f":[1,1,22,-1,-1,18,-1,-1,2,15,2,-1,-1,25,4,-1,18,4,-1,-1,17,-1,-1,16,22,-1,-1,-1,-1],"t":[5.417948717948718,5.015625,17.094543568094064,0,0,4.245926548164837,0,0,9.068775776507199,2.807354922057604,1.292481250360578,0,0,2.5,3.084962500721156,0,8.03615005796417,6.499295214872664,0,0,9.075473573191719,0,0,4.523561956057013,12.613886328427084,0,0,0,0],"m":[1,1,1,0,0,1,0,0,1,0,1,0,0,1,1,0,1,1,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,0,14,15,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,13,12,0,0,23,16,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,1.693447e-05,0.004057391,0,0.0008255357,0.006630055,0,0,0,-0.003554225,0.002638148,0,0,0.0001340823,0,0,-0.0007256883,-0.004629477,0,0.006554687,-0.0006097988,0,0,0.002679938,-0.004815045,0.006748637,-0.004610451]},{"f":[2,2,-1,15,-1,-1,2,25,-1,-1,18,2,4,15,22,-1,-1,-1,-1,22,-1,-1,22,16,-1,16,-1,-1,-1],"t":[5.40929108874043,5.2849278041654735,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,9.229707804263573,8.9274737487839,7.997154617983022,2.0,7.95646805166752,0,0,0,0,16.759160379664443,0,0,12.921158001713904,10.372865060112588,0,12.683433292835403,0,0,0],"m":[1,1,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,15,0,0,0,0,20,0,0,23,24,0,26,0,0,0],"r":[6,3,0,5,0,0,10,9,0,0,22,19,18,17,16,0,0,0,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0,-2.317966e-05,0,-0.0001711093,-0.008124201,0,0,0.0004799543,0.007458138,0,0,0,0,0,-0.0003741903,-0.005434918,0.0002268172,0.003927336,0,-0.0006717228,-0.005137457,0,0,-0.001069266,0,0.00401817,0.0005110619,0.004920637]},{"f":[3,22,-1,-1,3,16,22,-1,-1,-1,3,3,-1,-1,32,24,17,17,-1,21,-1,-1,-1,2,33,-1,-1,-1,-1],"t":[0.008766718459616732,16.725912049439096,0,0,0.034940494354592615,8.67595703294175,16.73920708776685,0,0,0,0.32902480033627574,0.1873015873015873,0,0,10.463013322934223,9.5,12.18494601563336,11.26283925556662,0,7.5,0,0,0,1.292481250360578,32.267154600431816,0,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,0,1,0,0,1,1,1,1,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,0,0,15,16,17,18,0,20,0,0,0,24,25,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,14,13,0,0,28,23,22,19,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0.002619128,-0.0009450154,0,0,0,-0.001098516,0.003155767,-0.004358264,0,0,0.0002742404,0.002502351,0,0,0,0,-1.475879e-05,0,0.003020009,-0.001226802,-0.001165221,0,0,0.002790453,-0.002792572,-0.007617853,0.001357107]},{"f":[1,2,-1,-1,3,22,22,3,-1,2,-1,-1,-1,-1,4,2,4,-1,-1,21,-1,-1,16,15,17,-1,-1,-1,-1],"t":[0.39811320754716983,6.08730682361797,0,0,0.05277777777777778,16.759160379664443,16.73920708776685,0.007093253968253968,0,8.612669628062957,0,0,0,0,9.266718101290426,9.034782574103549,7.675607660927803,0,0,8.5,0,0,10.696098170955864,4.247927513443585,9.293471190846834,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,10,0,0,0,0,15,16,17,0,0,20,0,0,23,24,25,0,0,0,0],"r":[4,3,0,0,14,13,12,9,0,11,0,0,0,0,22,19,18,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,7.285914e-05,0.002860556,0,0,0,0,0.001845954,0,-0.001032008,-0.003526576,0.003092576,-0.00321563,0,0,0,-1.183859e-05,0.001273564,0,-0.003819814,-0.0003171247,0,0,0,0.003575633,-0.0004490782,-0.002975847,0.005230289]},{"f":[15,15,15,-1,22,-1,-1,4,23,-1,-1,-1,15,33,-1,18,-1,-1,21,-1,18,-1,1,17,-1,23,-1,-1,-1],"t":[9.756556322524087,8.005624549193879,7.924812503605781,0,15.676331898425197,0,0,3.6427011094311244,0.5,0,0,0,10.066089190457772,29.807475161786478,0,8.03615005796417,0,0,3.5,0,10.620215166896223,0,3.9993913572732804,10.69085411329249,0,1.5,0,0,0],"m":[1,1,1,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,1,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,0,13,14,0,16,0,0,19,0,21,0,23,24,0,26,0,0,0],"r":[12,7,4,0,6,0,0,11,10,0,0,0,18,15,0,17,0,0,20,0,22,0,28,25,0,27,0,0,0],"v":[0,0,0,3.791362e-05,0,-0.00110294,-0.005971119,0,0,0.0004893282,0.002747139,-0.0008965329,0,0,0.0009774974,0,-0.004262534,-0.0005981094,0,0.002113823,0,-1.815557e-05,0,0,-0.004105838,0,-0.001195748,0.003823355,-0.007146985]},{"f":[22,-1,22,16,-1,-1,22,21,4,4,4,18,19,-1,-1,2,-1,2,-1,-1,-1,-1,22,-1,-1,17,-1,-1,-1],"t":[6.671801068872866,0,7.082728314749225,5.882643049361842,0,0,11.406459975416917,9.5,6.451187557243013,6.118903736861568,5.169368191286958,8.00281227459694,0.5,0,0,1.792481250360578,0,3.245926548164837,0,0,0,0,10.029977181607594,0,0,1.292481250360578,0,0,0],"m":[0,0,0,1,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,12,13,0,0,16,0,18,0,0,0,0,23,0,0,26,0,0,0],"r":[2,0,6,5,0,0,28,25,22,21,20,15,14,0,0,17,0,19,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0.0004961836,0,0,-0.0009555952,-0.004593545,0,0,0,0,0,0,0,0.000276634,0.003150145,0,0.001061889,0,-0.004150169,0.0002003943,0.003971824,-0.005698365,0,0.006090632,0.0002009888,0,-0.004197153,0.0004718811,-5.600173e-05]},{"f":[1,33,22,22,-1,-1,15,-1,-1,21,2,-1,16,-1,-1,15,17,1,-1,-1,-1,16,-1,22,-1,-1,2,-1,-1],"t":[5.417948717948718,27.680602362796474,16.759160379664443,12.027747761539269,0,0,9.002815015607053,0,0,7.5,9.221514256038809,0,10.681238411777805,0,0,10.043027283594547,5.83200138144325,2.9733542319749215,0,0,0,10.372865060112588,0,15.10428213920758,0,0,9.068775776507199,0,0],"m":[1,0,1,1,0,0,1,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,13,0,0,16,17,18,0,0,0,22,0,24,0,0,27,0,0],"r":[26,9,6,5,0,0,8,0,0,15,12,0,14,0,0,21,20,19,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0.0003448861,0.003090753,0,0.0006100274,-0.005808981,0,0,7.320514e-05,0,-0.0003145785,0.006275479,0,0,0,0.000142257,-0.004375167,0.0004914042,0,-0.007765623,0,-0.002247783,0.003056793,0,-0.0001333758,-0.004270565]},{"f":[23,22,22,23,-1,22,-1,22,-1,22,-1,-1,22,18,-1,-1,15,17,-1,-1,4,2,-1,-1,-1,24,-1,-1,-1],"t":[8.5,17.105450904090162,17.069460217922263,7.5,0,16.975010474608837,0,16.978341770814396,0,16.98920034885663,0,0,17.078739781790915,7.588709768994619,0,0,6.584962500721156,8.729619742794132,0,0,4.0437314206251695,2.69615871138938,0,0,0,8.5,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,10,0,0,13,14,0,0,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[28,25,12,5,0,7,0,9,0,11,0,0,16,15,0,0,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,1.423734e-06,0,-0.004697508,0,0.002432356,0,-0.005008482,0.001118318,0,0,0.003350518,0.0003939379,0,0,-0.006490984,0.00339545,0,0,0.0002819396,0.003980147,-0.001745674,0,-0.004191935,8.871429e-05,0.001627721]},{"f":[23,4,-1,2,22,-1,-1,2,-1,1,22,15,21,2,-1,-1,-1,21,-1,-1,-1,1,-1,21,-1,-1,15,-1,-1],"t":[8.5,4.0437314206251695,0,4.128693921346326,11.876328901062104,0,0,4.80643424864552,0,6.241071428571429,10.27595502635321,2.807354922057604,8.5,7.894792293346235,0,0,0,7.5,0,0,0,6.970655725248841,0,8.5,0,0,6.539158811108031,0,0],"m":[1,1,0,0,0,0,0,0,0,1,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,0,4,5,0,0,8,0,10,11,12,13,14,0,0,0,18,0,0,0,22,0,24,0,0,27,0,0],"r":[26,3,0,7,6,0,0,9,0,21,20,17,16,15,0,0,0,19,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,5.31212e-05,0,0,0.001582812,-0.002772966,0,0.001817281,0,0,0,0,0,-0.0001153289,0.004591304,-0.002788111,0,-0.008150073,-0.0008501973,0.0002206146,0,-0.004858338,0,-0.0004280567,-0.005071434,0,0.003151179,-0.0001727772]},{"f":[23,18,15,15,2,17,18,18,-1,-1,19,-1,-1,33,17,-1,1,-1,-1,-1,-1,-1,2,-1,4,-1,-1,-1,-1],"t":[0.5,9.314012697712199,10.113742166049189,9.756556322524087,9.266718101290426,6.924507036847409,3.6427011094311244,2.903677461028802,0,0,0.5,0,0,16.993854061608424,7.252657678068118,0,2.484375,0,0,0,0,0,1.292481250360578,0,2.160964047443681,0,0,0,0],"m":[0,1,1,1,1,1,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,11,0,0,14,15,0,17,0,0,0,0,0,23,0,25,0,0,0,0],"r":[28,27,22,21,20,13,10,9,0,0,12,0,0,19,16,0,18,0,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0,0,0,0,0,0,-8.493683e-05,-0.007255619,0,0.0005923105,0.005697045,0,0,-0.008417187,0,0.0005833898,-0.002071509,0.000635696,0.003615761,-0.002742794,0,0.004417772,0,-0.003022916,0.002052523,-0.0008419496,9.90589e-05]},{"f":[22,-1,22,16,-1,-1,22,-1,33,25,-1,24,-1,33,4,-1,-1,18,16,-1,-1,-1,23,33,1,-1,-1,-1,-1],"t":[6.671801068872866,0,7.082728314749225,5.882643049361842,0,0,11.406459975416917,0,29.807475161786478,0.5,0,6.5,0,20.752541988539424,3.9534452978042594,0,0,1.792481250360578,10.46352437327118,0,0,0,5.5,41.975696368331896,9.107199999999999,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,0,12,0,14,15,0,0,18,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,6,5,0,0,8,0,22,11,0,13,0,17,16,0,0,21,20,0,0,0,28,27,26,0,0,0,0],"v":[0,0.0004872621,0,0,-0.0009074962,-0.004248215,0,0.0002642563,0,0,-7.433526e-05,0,-0.0001957651,0,0,0.009572544,0.001937052,0,0,0.005958728,0.0003511154,-0.001649135,0,0,0,-0.002054344,-0.006182216,-4.113954e-05,0.0003373343]},{"f":[33,32,15,-1,22,1,-1,16,19,23,-1,-1,-1,17,-1,-1,23,-1,36,-1,-1,36,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,0,11.722778233171695,1.3336158192090395,0,10.794415866350105,0.5,0.5,0,0,0,10.457893746154532,0,0,0.5,0,3.5,0,0,3.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,8,9,10,0,0,0,14,0,0,17,0,19,0,0,22,0,0,25,0,27,0,0],"r":[24,21,4,0,16,7,0,13,12,11,0,0,0,15,0,0,18,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,5.216949e-05,0,0,-0.004686107,0,0,0,-0.004469003,-0.0004960295,0.0001897807,0,0.0023035,-0.001424896,0,0.001804464,0,-0.00454338,-9.952322e-05,0,0.003576452,-0.0001422549,0,0.0003916925,0,0.0003960844,-0.003214692]},{"f":[23,-1,21,17,-1,22,3,3,-1,-1,17,-1,19,-1,-1,37,15,22,-1,-1,-1,-1,37,18,-1,-1,23,-1,-1],"t":[0.5,0,5.5,3.753897320099348,0,15.717370311433935,0.8293269230769231,0.49242424242424243,0,0,5.918419679874358,0,0.5,0,0,7.92,6.0,16.819976309430963,0,0,0,0,10.780000000000001,6.531360383582522,0,0,2.5,0,0],"m":[0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,0,13,0,0,16,17,18,0,0,0,0,23,24,0,0,27,0,0],"r":[2,0,22,5,0,15,10,9,0,0,12,0,14,0,0,21,20,19,0,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.000105438,0,0,0.0008648204,0,0,0,-0.000337902,-0.003266267,0,-0.003001724,0,0.0006302115,0.002825696,0,0,0,0.001936129,-0.003330319,0.003583202,-0.007767065,0,0,-0.0002894373,0.0003875717,0,-0.006347622,-0.001718469]},{"f":[22,-1,22,-1,18,18,4,2,2,4,23,-1,-1,-1,1,1,-1,-1,-1,-1,16,-1,15,-1,-1,21,-1,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,12.18494601563336,11.26283925556662,9.328332883600808,9.293467526905953,9.051177541662383,8.049190845056867,0.5,0,0,0,3.172945205479452,1.99834912080135,0,0,0,0,4.247927513443585,0,10.486835021563051,0,0,7.5,0,0,0],"m":[0,0,0,0,1,1,1,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,7,8,9,10,11,0,0,0,15,16,0,0,0,0,21,0,23,0,0,26,0,0,0],"r":[2,0,4,0,28,25,20,19,14,13,12,0,0,0,18,17,0,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0.0006235917,0,-0.002470516,0,0,0,0,0,0,0,-0.000159454,0.000135301,0.002150293,0,0,-0.0006825407,-0.005605432,0.002346222,0.004313871,0,0.0003335978,0,-0.005070971,-0.0003859173,0,0.002565679,-0.001727665,-0.0009754503]},{"f":[23,1,-1,4,22,22,22,22,22,-1,2,2,-1,-1,-1,-1,-1,-1,17,15,-1,18,-1,-1,-1,22,-1,-1,-1],"t":[8.5,5.417948717948718,0,6.499295214872664,17.094543568094064,16.98920034885663,16.98413870862295,16.975010474608837,15.717370311433935,0,3.084962500721156,1.292481250360578,0,0,0,0,0,0,8.729619742794132,6.584962500721156,0,6.400854179458231,0,0,0,13.005894632618489,0,0,0],"m":[1,1,0,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,7,8,9,0,11,12,0,0,0,0,0,0,19,20,0,22,0,0,0,26,0,0,0],"r":[28,3,0,25,18,17,16,15,10,0,14,13,0,0,0,0,0,0,24,21,0,23,0,0,0,27,0,0,0],"v":[0,0,5.72748e-05,0,0,0,0,0,0,3.794952e-05,0,0,-0.002782658,0.001115292,-0.009220088,0.002550025,-0.004593088,0.001600853,0,0,-0.01000241,0,-0.00425015,0.002704006,0.002087418,0,-0.005063325,0.001906893,0.001478506]},{"f":[2,2,21,22,26,35,23,-1,19,-1,21,-1,-1,-1,-1,22,-1,-1,16,-1,-1,15,-1,-1,2,25,-1,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,15.717370311433935,0.5,2.5,0.5,0,1.5,0,5.5,0,0,0,0,15.756821747580634,0,0,11.98370619265935,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,0],"m":[1,1,1,1,1,1,1,0,1,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,11,0,0,0,0,16,0,0,19,0,0,22,0,0,25,26,0,0,0],"r":[24,21,18,15,14,13,8,0,10,0,12,0,0,0,0,17,0,0,20,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0001610973,0,0.0002075735,0,0.003281557,-0.0004554032,-0.00169941,0.002622239,0,-0.008395882,-0.0003217358,0,0.001461231,-0.003381918,0,-0.0001947923,-0.007644105,0,0,0.0004304302,0.006831269,0.0001087806]},{"f":[22,-1,22,16,-1,-1,22,21,4,4,4,18,19,-1,-1,2,-1,2,-1,-1,-1,-1,22,-1,17,-1,-1,-1,-1],"t":[6.671801068872866,0,7.082728314749225,5.882643049361842,0,0,11.406459975416917,9.5,6.451187557243013,6.118903736861568,5.169368191286958,8.00281227459694,0.5,0,0,1.792481250360578,0,3.245926548164837,0,0,0,0,10.029977181607594,0,1.292481250360578,0,0,0,0],"m":[0,0,0,1,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,12,13,0,0,16,0,18,0,0,0,0,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,28,27,22,21,20,15,14,0,0,17,0,19,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0.0004652803,0,0,-0.0008552314,-0.003902105,0,0,0,0,0,0,0,0.0002362414,0.00290352,0,0.0009914479,0,-0.003872716,0.0001348835,0.003622151,-0.005432537,0,0.005671494,0,-0.001377714,0.002016696,-0.001876631,-4.964883e-05]},{"f":[15,-1,15,21,-1,-1,21,-1,17,22,35,-1,22,-1,-1,37,-1,-1,17,-1,15,22,22,33,-1,-1,-1,-1,-1],"t":[9.756556322524087,0,10.066089190457772,6.5,0,0,3.5,0,2.160964047443681,16.725912049439096,2.5,0,15.439474430023438,0,0,6.890000000000001,0,0,3.245926548164837,0,13.0003521774803,16.975010474608837,16.765402642760364,29.297756302860655,0,0,0,0,0],"m":[1,0,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,11,0,13,0,0,16,0,0,19,0,21,22,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,8,0,18,15,12,0,14,0,0,17,0,0,20,0,28,27,26,25,0,0,0,0,0],"v":[0,5.180619e-05,0,0,-0.0008886015,-0.005584603,0,0.00190195,0,0,0,0.006423794,0,6.925315e-05,0.004174005,0,0.0004952473,-0.002798607,0,-0.004778292,0,0,0,0,0.002983937,-0.0005585038,-0.004761788,0.0009628373,-0.004362057]},{"f":[22,-1,18,18,4,2,2,4,23,17,-1,-1,-1,-1,1,1,-1,-1,-1,-1,16,-1,15,-1,-1,15,-1,-1,-1],"t":[2.084962500721156,0,12.18494601563336,11.26283925556662,9.328332883600808,9.293467526905953,9.051177541662383,8.049190845056867,0.5,6.924507036847409,0,0,0,0,3.172945205479452,1.99834912080135,0,0,0,0,4.247927513443585,0,10.486835021563051,0,0,11.683433292835403,0,0,0],"m":[0,0,1,1,1,1,1,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,8,9,10,0,0,0,0,15,16,0,0,0,0,21,0,23,0,0,26,0,0,0],"r":[2,0,28,25,20,19,14,13,12,11,0,0,0,0,18,17,0,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0.001122688,0,0,0,0,0,0,0,0,-2.584994e-05,-0.0008151712,0.0001274444,0.001984379,0,0,-0.0006379274,-0.005364049,0.002340735,0.004064017,0,0.0003016229,0,-0.004812568,-0.0003651372,0,0.002600089,-0.001313079,-0.000926264]},{"f":[1,1,22,-1,-1,18,-1,-1,2,22,22,22,4,4,4,-1,-1,-1,-1,-1,-1,17,15,-1,18,-1,-1,-1,-1],"t":[5.417948717948718,5.015625,17.094543568094064,0,0,4.245926548164837,0,0,9.068775776507199,17.094543568094064,16.98920034885663,16.98413870862295,6.499295214872664,5.89476682248518,2.453445297804259,0,0,0,0,0,0,8.729619742794132,6.584962500721156,0,6.400854179458231,0,0,0,0],"m":[1,1,1,0,0,1,0,0,1,1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,13,14,15,0,0,0,0,0,0,22,23,0,25,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,21,20,19,18,17,16,0,0,0,0,0,0,27,24,0,26,0,0,0,0],"v":[0,0,0,1.448704e-05,0.00398657,0,0.0007049818,0.006090449,0,0,0,0,0,0,0,0.0001812338,-0.0007516088,0.002446126,-0.003012204,-0.003620395,0.001823743,0,0,-0.008807832,0,-0.004028206,0.003181795,0.001867725,-0.003978094]},{"f":[2,2,21,-1,16,16,15,-1,-1,-1,-1,15,-1,-1,2,25,-1,-1,18,-1,15,21,-1,17,-1,-1,22,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,0,11.98370619265935,7.679480099505446,7.6653359171851765,0,0,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,9.229707804263573,0,9.862637357558794,6.5,0,10.620215166896223,0,0,11.876328901062104,0,0],"m":[1,1,1,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,7,0,0,0,0,12,0,0,15,16,0,0,19,0,21,22,0,24,0,0,27,0,0],"r":[14,11,4,0,10,9,8,0,0,0,0,13,0,0,18,17,0,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0,-5.516303e-05,0,0,0,0.001410478,-0.004718034,0.002934674,-0.003192754,0,-0.0001922784,-0.007275283,0,0,0.000431974,0.006348174,0,-7.018888e-05,0,0,-0.001309989,0,0.004858994,0.0002115158,0,0.0008724626,0.005013563]},{"f":[15,-1,15,17,-1,-1,21,-1,3,-1,18,18,17,22,35,-1,-1,-1,17,-1,-1,-1,1,17,-1,23,-1,-1,-1],"t":[9.756556322524087,0,10.066089190457772,1.292481250360578,0,0,3.5,0,0.034940494354592615,0,10.620215166896223,10.45532575654158,2.160964047443681,16.725912049439096,2.5,0,0,0,3.245926548164837,0,0,0,3.9993913572732804,10.69085411329249,0,1.5,0,0,0],"m":[1,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,0,3,4,0,0,7,0,9,0,11,12,13,14,15,0,0,0,19,0,0,0,23,24,0,26,0,0,0],"r":[2,0,6,5,0,0,8,0,10,0,22,21,18,17,16,0,0,0,20,0,0,0,28,25,0,27,0,0,0],"v":[0,5.046411e-05,0,0,-0.004032092,-0.0005160069,0,0.001786177,0,-0.003563951,0,0,0,0,0,0.006104158,0.0006066575,-0.0006295331,0,-0.00454419,-0.0003028042,0.00162581,0,0,-0.003830243,0,-0.001033698,0.003171959,-0.006569098]},{"f":[22,5,3,-1,-1,-1,22,23,2,2,17,1,-1,-1,37,-1,-1,22,1,-1,-1,-1,-1,4,-1,-1,4,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,0,12.738408632106918,1.5,8.049190845056867,1.292481250360578,1.792481250360578,6.015625,0,0,5.84,0,0,12.613886328427084,3.822916666666667,0,0,0,0,1.792481250360578,0,0,7.997154617983022,0,0],"m":[0,0,1,0,0,0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,0,0,0,7,8,9,10,11,12,0,0,15,0,0,18,19,0,0,0,0,24,0,0,27,0,0],"r":[6,5,4,0,0,0,26,23,22,17,14,13,0,0,16,0,0,21,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,-1.383821e-05,-0.004091174,0.0001882931,0,0,0,0,0,0,-0.001499788,0.005042337,0,-0.0001476961,-0.006221774,0,0,0.0001668783,0.00292483,-0.003527749,-0.003157403,0,-0.001267139,-0.006774015,0,-2.489274e-05,0.001304103]},{"f":[15,22,-1,-1,32,-1,15,22,3,1,-1,-1,-1,22,-1,-1,15,-1,37,1,-1,37,-1,35,-1,-1,34,-1,-1],"t":[5.78135971352466,16.819976309430963,0,0,3.753897320099348,0,6.409390936137702,16.96990787537721,0.5898428571428571,2.354717912147707,0,0,0,17.069460217922263,0,0,6.523561956057013,0,10.469999999999999,8.046875,0,8.2,0,2.5,0,0,3.5,0,0],"m":[0,1,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,0,7,8,9,10,0,0,0,14,0,0,17,0,19,20,0,22,0,24,0,0,27,0,0],"r":[4,3,0,0,6,0,16,13,12,11,0,0,0,15,0,0,18,0,26,21,0,23,0,25,0,0,28,0,0],"v":[0,0,-8.25407e-05,-0.002384293,0,0.005572919,0,0,0,0,0.0004928958,-0.005121378,0.0009820348,0,-0.004690085,0.00144013,0,-0.001843335,0,0,-9.640797e-06,0,0.00800198,0,0.001951461,-0.0007411463,0,-0.00355355,-0.0007411463]},{"f":[23,21,2,-1,16,-1,-1,37,20,22,2,22,21,-1,-1,-1,-1,-1,16,-1,16,18,-1,-1,17,-1,-1,-1,-1],"t":[8.5,7.5,9.221514256038809,0,10.696098170955864,0,0,11.05,0.5,16.98920034885663,1.292481250360578,10.224917391507956,8.5,0,0,0,0,0,6.044394119358453,0,7.330916878114617,8.729619742794132,0,0,7.198837316474133,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,12,13,0,0,0,0,0,19,0,21,22,0,0,25,0,0,0,0],"r":[28,7,4,0,6,0,0,27,18,17,16,15,14,0,0,0,0,0,20,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,2.426437e-05,0,-0.0004231369,0.006008079,0,0,0,0,0,0,-0.00237252,0.004510084,-0.00237252,-0.0003618477,0.0009552398,0,0.001487537,0,0,-0.00237252,0.003611851,0,0.001957837,7.909303e-06,-0.004568192,0.001426121]},{"f":[22,5,3,-1,-1,-1,22,23,15,16,22,16,1,0,-1,-1,17,-1,2,-1,-1,-1,-1,-1,-1,4,-1,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,0,12.738408632106918,1.5,6.523561956057013,4.247927513443585,12.613886328427084,4.169925001442312,6.015625,2.5,0,0,1.292481250360578,0,1.292481250360578,0,0,0,0,0,0,1.792481250360578,0,0,0],"m":[0,0,1,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,0,7,8,9,10,11,12,13,14,0,0,17,0,19,0,0,0,0,0,0,26,0,0,0],"r":[6,5,4,0,0,0,28,25,24,23,22,21,16,15,0,0,18,0,20,0,0,0,0,0,0,27,0,0,0],"v":[0,0,0,-2.808752e-05,-0.003888591,0.0001835416,0,0,0,0,0,0,0,0,-0.00027575,-0.00374682,0,0.004400697,0,-0.003960331,0.002883514,0.003657035,-0.004277495,-0.00321784,0.001475009,0,-0.001194264,-0.006465769,-2.464022e-06]},{"f":[2,2,17,17,21,21,-1,17,-1,-1,-1,-1,1,-1,17,-1,16,-1,16,-1,-1,15,-1,-1,2,25,-1,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.496347487943302,7.6181566135214025,8.5,7.5,0,5.14960400919364,0,0,0,0,1.000936329588015,0,8.729619742794132,0,10.46352437327118,0,10.696098170955864,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,0],"m":[1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,0,0,13,0,15,0,17,0,19,0,0,22,0,0,25,26,0,0,0],"r":[24,21,12,11,10,7,0,9,0,0,0,0,14,0,16,0,18,0,20,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,4.269253e-05,0,-0.00197257,0.0002767582,0.001349798,0.001259447,0,-0.005360318,0,-0.006611341,0,0.00046452,0,-0.003639731,-0.000506154,0,-0.0001873653,-0.00686644,0,0,0.0004488192,0.005986565,0.0001055307]},{"f":[22,-1,23,17,1,16,15,-1,-1,-1,17,18,4,-1,16,-1,-1,22,-1,-1,1,17,-1,-1,33,-1,-1,-1,-1],"t":[2.084962500721156,0,0.5,12.18494601563336,3.354166666666667,11.868050853745158,11.683433292835403,0,0,0,6.734566509914796,5.108372929097653,1.292481250360578,0,5.169925001442312,0,0,11.36027577861459,0,0,6.646978021978022,7.344289078557851,0,0,21.011466140970732,0,0,0,0],"m":[0,0,0,1,1,1,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,12,13,0,15,0,0,18,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,28,27,10,9,8,0,0,0,20,17,14,0,16,0,0,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,0.001114683,0,0,0,0,0,4.977636e-05,-0.005250238,0.003653365,0,0,0,-0.001570292,0,0.002797736,-0.0004623542,0,0.0003613881,0.006019331,0,0,-0.007635847,-0.001524005,0,0.003675583,-0.0003076365,-0.003715362,8.672043e-05]},{"f":[22,16,-1,-1,22,16,-1,-1,15,15,-1,22,-1,22,-1,17,-1,3,-1,-1,32,-1,15,-1,15,22,-1,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,5.882643049361842,0,0,5.169925001442312,4.087462841250339,0,11.681676585676911,0,13.747003318603246,0,5.740899715832876,0,0.5803571428571428,0,0,3.753897320099348,0,5.491853096329675,0,5.614709844115208,13.689894219688643,0,0,0],"m":[0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,0,12,0,14,0,16,0,18,0,0,21,0,23,0,25,26,0,0,0],"r":[4,3,0,0,8,7,0,0,20,11,0,13,0,15,0,17,0,19,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0.0002917329,0.002453899,0,0,-0.0008497275,-0.003556628,0,0,6.114679e-06,0,-0.003622733,0,1.378719e-05,0,-0.0003805548,0,-0.001799642,-0.008599622,0,0.005259014,0,0.001215117,0,0,0.0001545303,-0.002880906,5.930321e-05]},{"f":[1,1,22,-1,-1,18,-1,-1,2,15,2,-1,-1,25,4,-1,18,18,-1,-1,17,-1,-1,16,22,-1,-1,-1,-1],"t":[5.417948717948718,5.015625,17.094543568094064,0,0,4.245926548164837,0,0,9.068775776507199,2.807354922057604,1.292481250360578,0,0,2.5,3.084962500721156,0,8.03615005796417,7.252657678068118,0,0,9.075473573191719,0,0,4.523561956057013,12.613886328427084,0,0,0,0],"m":[1,1,1,0,0,1,0,0,1,0,1,0,0,1,1,0,1,1,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,0,14,15,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,13,12,0,0,23,16,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,1.49751e-05,0.00379999,0,0.0006499638,0.005608509,0,0,0,-0.003184064,0.002171383,0,0,0.0001262873,0,0,-0.0006033285,-0.003891125,0,0.005808203,-0.000405687,0,0,0.002307846,-0.004091085,0.005803455,-0.003972226]},{"f":[2,2,21,22,26,35,-1,-1,-1,22,-1,-1,16,16,15,-1,-1,-1,-1,15,-1,-1,2,25,-1,-1,15,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,15.717370311433935,0.5,2.5,0,0,0,15.756821747580634,0,0,11.98370619265935,7.679480099505446,7.6653359171851765,0,0,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,1.0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,0,0,13,14,15,0,0,0,0,20,0,0,23,24,0,0,27,0,0],"r":[22,19,12,9,8,7,0,0,0,11,0,0,18,17,16,0,0,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,0,3.350749e-05,-0.001583483,0.002483219,0,-0.007929158,-0.0002895361,0,0,0,0.001279168,-0.004456484,0.002744426,-0.003078512,0,-0.0001860581,-0.006534201,0,0,0.0003614753,0.005595703,0,-0.003015959,0.0001560262]},{"f":[23,21,23,-1,17,-1,-1,37,20,22,2,22,21,-1,-1,-1,-1,-1,16,-1,16,18,-1,-1,17,-1,-1,-1,-1],"t":[8.5,7.5,2.5,0,4.128693921346326,0,0,11.05,0.5,16.98920034885663,1.292481250360578,10.224917391507956,8.5,0,0,0,0,0,6.044394119358453,0,7.330916878114617,8.729619742794132,0,0,7.198837316474133,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,12,13,0,0,0,0,0,19,0,21,22,0,0,25,0,0,0,0],"r":[28,7,4,0,6,0,0,27,18,17,16,15,14,0,0,0,0,0,20,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,-2.725351e-05,0,0.001295237,-0.0001830231,0,0,0,0,0,0,-0.002299356,0.004150256,-0.002299356,-0.0003295456,0.0009401356,0,0.001347748,0,0,-0.002299356,0.003415504,0,0.001801719,-1.318236e-05,-0.00442298,0.001371301]},{"f":[22,22,22,22,22,22,15,22,-1,-1,-1,3,-1,-1,-1,-1,-1,22,-1,1,17,17,-1,-1,-1,-1,25,-1,-1],"t":[17.105450904090162,17.069460217922263,16.838461650489297,16.786010507999382,16.765402642760364,16.73920708776685,13.56045225766691,15.717370311433935,0,0,0,0.05927318295739348,0,0,0,0,0,17.078739781790915,0,7.9752100840336135,8.729619742794132,8.571721503073015,0,0,0,0,0.5,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,12,0,0,0,0,0,18,0,20,21,22,0,0,0,0,27,0,0],"r":[26,17,16,15,14,11,10,9,0,0,0,13,0,0,0,0,0,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,3.710794e-05,-0.0008976315,0.00421366,0,0.004789747,-0.00256763,-0.01337963,0.001206083,-0.0006498525,0,0.002206736,0,0,0,-0.0001773264,-0.008370778,0.001660131,0.002569259,0,5.209415e-05,-0.003598832]},{"f":[2,2,-1,15,-1,-1,2,25,-1,-1,18,2,4,15,22,-1,-1,-1,-1,-1,15,21,-1,17,-1,-1,22,-1,-1],"t":[5.40929108874043,5.2849278041654735,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,9.229707804263573,8.9274737487839,7.997154617983022,2.0,7.5621616439616925,0,0,0,0,0,9.862637357558794,6.5,0,10.620215166896223,0,0,11.876328901062104,0,0],"m":[1,1,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,15,0,0,0,0,0,21,22,0,24,0,0,27,0,0],"r":[6,3,0,5,0,0,10,9,0,0,20,19,18,17,16,0,0,0,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,-2.320754e-05,0,-0.0001834199,-0.006208414,0,0,0.0003981423,0.005315308,0,0,0,0,0,0.0003695009,-0.004701783,0.0002181119,0.003439896,-0.0009222464,0,0,-0.001171096,0,0.004582615,0.0001633835,0,0.0008389653,0.004688211]},{"f":[3,22,-1,-1,3,16,22,-1,-1,-1,12,2,-1,22,1,22,3,-1,-1,-1,26,-1,4,-1,-1,3,-1,-1,-1],"t":[0.008766718459616732,16.725912049439096,0,0,0.034940494354592615,8.67595703294175,16.73920708776685,0,0,0,0.5,6.451187557243013,0,16.975010474608837,5.417948717948718,16.832741648970774,0.09303583502366464,0,0,0,0.5,0,6.912876916441556,0,0,0.11587329992841804,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,0,14,15,16,17,0,0,0,21,0,23,0,0,26,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,13,0,25,20,19,18,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0.002401719,-0.001045312,0,0,0,-0.001005405,0.002983247,-0.003909601,0,0,-2.42268e-05,0,0,0,0,0.003713816,0.0005372382,-0.003709887,0,-0.005617071,0,0.001393748,-0.002907903,0,-0.0003851057,0.005486475,-0.001474831]},{"f":[22,22,22,22,22,23,22,-1,22,-1,-1,-1,-1,-1,23,-1,-1,22,-1,1,17,17,-1,-1,-1,-1,25,-1,-1],"t":[17.105450904090162,17.069460217922263,16.819976309430963,16.786010507999382,16.765402642760364,4.5,15.717370311433935,0,15.756821747580634,0,0,0,0,0,2.5,0,0,17.078739781790915,0,7.9752100840336135,8.729619742794132,8.571721503073015,0,0,0,0,0.5,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,0,9,0,0,0,0,0,15,0,0,18,0,20,21,22,0,0,0,0,27,0,0],"r":[26,17,14,13,12,11,8,0,10,0,0,0,0,0,16,0,0,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,1.073721e-05,0,-0.007090796,-0.0003762788,0.0007290118,-0.01273792,0.001673877,0,-0.003159788,-0.0003428538,0,0.002092554,0,0,0,-0.0001824375,-0.007966264,0.001569636,0.002435838,0,4.503003e-05,-0.003424499]},{"f":[3,22,-1,-1,3,16,22,-1,-1,-1,12,2,-1,22,1,22,24,-1,2,-1,-1,-1,26,-1,-1,3,-1,-1,-1],"t":[0.008766718459616732,16.725912049439096,0,0,0.034940494354592615,8.67595703294175,16.73920708776685,0,0,0,0.5,6.451187557243013,0,16.975010474608837,5.417948717948718,16.832741648970774,2.5,0,6.584884225259038,0,0,0,0.5,0,0,0.11587329992841804,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,0,14,15,16,17,0,19,0,0,0,23,0,0,26,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,13,0,25,22,21,18,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0.002291248,-0.0008694702,0,0,0,-0.0009652741,0.002925212,-0.003704524,0,0,-2.386906e-05,0,0,0,0,0.0004033116,0,0.0002709234,0.004832473,-0.003505679,0,-0.005330595,-0.0006507567,0,-0.0003467609,0.005223171,-0.001404839]},{"f":[21,-1,20,22,2,21,16,-1,15,-1,-1,0,-1,15,-1,-1,22,17,-1,2,-1,-1,-1,-1,16,-1,37,-1,-1],"t":[7.5,0,0.5,16.98920034885663,1.292481250360578,8.5,3.169925001442312,0,10.066089190457772,0,0,2.5,0,4.247927513443585,0,0,16.844368517741394,6.199638962254484,0,6.055241654908112,0,0,0,0,6.044394119358453,0,10.015,0,0],"m":[1,0,0,1,0,1,1,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,14,0,0,17,18,0,20,0,0,0,0,25,0,27,0,0],"r":[2,0,24,23,16,11,8,0,10,0,0,13,0,15,0,0,22,19,0,21,0,0,0,0,26,0,28,0,0],"v":[0,4.459584e-05,0,0,0,0,0,-0.00130775,0,-0.009758163,-0.00156189,0,0.002346582,0,5.880169e-05,-0.005027393,0,0,-0.0004490952,0,0.007268317,3.417251e-05,-0.003414384,0.0008008714,0,0.001294076,0,4.447286e-05,-0.003452668]},{"f":[22,16,-1,-1,22,16,-1,-1,22,21,16,2,-1,-1,22,22,16,-1,-1,1,-1,-1,3,-1,33,-1,-1,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,5.882643049361842,0,0,11.406459975416917,9.5,1.584962500721156,6.451187557243013,0,0,9.087343393278285,8.530347965843777,6.523561956057013,0,0,1.000936329588015,0,0,0.8516483516483516,0,36.27694515242878,0,0,0,0],"m":[0,1,0,0,0,1,0,0,0,1,0,1,0,0,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,12,0,0,15,16,17,0,0,20,0,0,23,0,25,0,0,0,0],"r":[4,3,0,0,8,7,0,0,28,27,14,13,0,0,22,19,18,0,0,21,0,0,24,0,26,0,0,0,0],"v":[0,0,0.0002907949,0.00234504,0,0,-0.0007950777,-0.003382913,0,0,0,0,0.0007993489,0.005803045,0,0,0,0.004357616,-8.796538e-05,0,-0.00792498,-0.002019864,0,-0.0008539805,0,0.0005192194,0.002707214,-0.001891717,-4.53513e-05]},{"f":[23,3,3,-1,-1,1,-1,32,21,-1,19,2,19,-1,-1,1,17,17,-1,-1,-1,-1,15,18,-1,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,0.4226190476190476,0,0,3.354166666666667,0,7.014643613484123,1.5,0,1.5,1.292481250360578,0.5,0,0,5.015625,9.26326872279839,4.208926257442949,0,0,0,0,6.0,2.160964047443681,0,0,0,0,0],"m":[0,0,1,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,9,0,11,12,13,0,0,16,17,18,0,0,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,7,0,27,10,0,22,15,14,0,0,21,20,19,0,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0.0002372097,0.0035608,0,1.848862e-05,0,0,0.0003924019,0,0,0,-0.008219651,-0.0007887033,0,0,0,-0.0002563617,-0.00642948,-0.000814085,0.0004448979,0,0,-3.967381e-05,0.004429961,-0.001070356,0.001183511,8.499547e-05]},{"f":[4,-1,2,22,-1,15,22,-1,-1,18,-1,-1,18,17,3,22,26,-1,-1,-1,22,-1,-1,4,16,-1,-1,-1,-1],"t":[4.0437314206251695,0,4.128693921346326,11.876328901062104,0,4.087462841250339,16.819976309430963,0,0,4.245926548164837,0,0,8.03615005796417,6.022197059679227,0.7083333333333333,16.725912049439096,2.5,0,0,0,15.212004812197344,0,0,9.037514954341955,6.0,0,0,0,0],"m":[1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,0,0,10,0,0,13,14,15,16,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,12,5,0,9,8,0,0,11,0,0,28,23,20,19,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,4.893153e-05,0,0,0.001599818,0,0,0.001710986,-0.003728545,0,-0.005987783,-0.001017,0,0,0,0,0,0.005276386,0.00189246,-0.003144359,0,-0.0002200253,0.003511108,0,0,0.0005735376,-0.001171411,0.002406942,0.000753473]},{"f":[2,3,22,-1,-1,3,22,15,-1,-1,-1,4,2,2,-1,21,-1,-1,1,1,-1,-1,-1,18,-1,22,-1,-1,-1],"t":[10.680359255641626,0.007093253968253968,16.725912049439096,0,0,0.034940494354592615,16.73920708776685,5.20945336562895,0,0,0,9.266718101290426,9.034782574103549,6.451187557243013,0,8.5,0,0,3.172945205479452,1.9735450749013448,0,0,0,9.293471190846834,0,12.457649623491124,0,0,0],"m":[1,0,0,0,0,0,1,1,0,0,0,1,1,1,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,0,12,13,14,0,16,0,0,19,20,0,0,0,24,0,26,0,0,0],"r":[28,5,4,0,0,11,10,9,0,0,0,23,18,15,0,17,0,0,22,21,0,0,0,25,0,27,0,0,0],"v":[0,0,0,0.003564556,-0.0008340175,0,0,0,-0.0007561786,-0.003387509,0.002024364,0,0,0,-2.171396e-05,0,0.0007446645,-0.001851714,0,0,-0.000634513,-0.004829356,0.001450535,0,0.004574692,0,-0.002500406,0.001114539,-0.001318829]},{"f":[22,16,-1,-1,22,16,-1,-1,22,5,2,-1,-1,-1,22,23,2,2,17,1,-1,-1,-1,-1,-1,4,-1,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,5.882643049361842,0,0,12.199665042528586,0.5,6.033380966193453,0,0,0,12.738408632106918,1.5,8.049190845056867,1.292481250360578,1.792481250360578,6.015625,0,0,0,0,0,1.792481250360578,0,0,0],"m":[0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,0,0,0,15,16,17,18,19,20,0,0,0,0,0,26,0,0,0],"r":[4,3,0,0,8,7,0,0,14,13,12,0,0,0,28,25,24,23,22,21,0,0,0,0,0,27,0,0,0],"v":[0,0,0.0002894852,0.00222781,0,0,-0.0007540835,-0.003185139,0,0,0,-0.002724958,0.00036517,0.000219924,0,0,0,0,0,0,-0.001185108,0.004851965,-0.002640886,0.0008430185,-0.002956571,0,-0.001093986,-0.006161894,-5.106164e-06]},{"f":[23,-1,21,17,-1,22,3,36,20,-1,24,-1,-1,-1,17,-1,19,-1,-1,18,22,-1,22,-1,-1,-1,37,-1,-1],"t":[0.5,0,5.5,3.753897320099348,0,15.717370311433935,0.8293269230769231,3.5,0.5,0,7.5,0,0,0,5.918419679874358,0,0.5,0,0,5.065928480304397,16.819976309430963,0,17.069460217922263,0,0,0,10.780000000000001,0,0],"m":[0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,8,9,0,11,0,0,0,15,0,17,0,0,20,21,0,23,0,0,0,27,0,0],"r":[2,0,26,5,0,19,14,13,10,0,12,0,0,0,16,0,18,0,0,25,22,0,24,0,0,0,28,0,0],"v":[0,-9.371156e-05,0,0,0.0007655581,0,0,0,0,-0.0002876986,0,-0.005671316,-0.001715398,0.001948459,0,-0.00268842,0,0.000528445,0.002484792,0,0,0.001930658,0,-0.002575768,0.001917232,-0.005836149,0,-3.53695e-05,-0.00288812]},{"f":[1,1,16,-1,9,-1,-1,2,7,-1,-1,-1,22,-1,23,24,-1,33,15,-1,-1,20,-1,22,33,-1,-1,-1,-1],"t":[0.39811320754716983,0.01933482600988898,10.372865060112588,0,0.5,0,0,5.8698903048866296,0.5,0,0,0,14.03140051979143,0,2.5,6.5,0,19.167172160526107,5.044394119358453,0,0,0.5,0,15.717370311433935,21.257266042509173,0,0,0,0],"m":[0,1,1,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,1,0,0,1,0,1,1,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,0,13,0,15,16,0,18,19,0,0,22,0,24,25,0,0,0,0],"r":[12,7,4,0,6,0,0,11,10,0,0,0,14,0,28,17,0,21,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0.0003725158,0,-0.004130278,-0.001029871,0,0,0.001815941,-0.0005044768,0.005077811,0,4.098783e-05,0,0,-0.00559702,0,0,-0.00125552,0.00623192,0,-0.0003336909,0,0,-0.003417441,0.001338367,-0.007705466,3.689923e-05]},{"f":[22,22,22,22,22,22,-1,-1,-1,-1,-1,22,18,-1,-1,4,17,15,-1,-1,-1,4,4,-1,-1,-1,25,-1,-1],"t":[17.105450904090162,17.069460217922263,16.819976309430963,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,0,17.078739781790915,7.65814076587311,0,0,1.292481250360578,8.729619742794132,6.584962500721156,0,0,0,4.0437314206251695,2.69615871138938,0,0,0,0.5,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,12,13,0,0,16,17,18,0,0,0,22,23,0,0,0,27,0,0],"r":[26,11,10,9,8,7,0,0,0,0,0,15,14,0,0,21,20,19,0,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-2.583778e-06,0.001877448,-0.0121006,0.001509886,-0.0005140415,0,0,0.002867953,0.0001637776,0,0,0,-0.006816056,-7.304717e-05,0.002144043,0,0,0.0003502251,0.00396651,-0.0008182621,0,1.493725e-05,-0.003270677]},{"f":[23,18,18,15,15,18,18,18,16,-1,-1,-1,33,-1,33,-1,19,-1,-1,-1,-1,2,-1,4,-1,-1,-1,-1,-1],"t":[0.5,9.314012697712199,9.267956609922159,10.342074667999139,9.756556322524087,7.298618922788638,3.5221970596792267,2.903677461028802,7.614709844115208,0,0,0,19.167172160526107,0,21.16864017939033,0,1.5,0,0,0,0,1.292481250360578,0,2.160964047443681,0,0,0,0,0],"m":[0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,13,0,15,0,17,0,0,0,0,22,0,24,0,0,0,0,0],"r":[28,27,26,21,20,19,12,11,10,0,0,0,14,0,16,0,18,0,0,0,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,-0.0001761862,0.000866238,-0.007947775,0,0.005494381,0,-0.003310037,0,0.0003431178,0.003281967,-0.002127285,-0.002338672,0,0.003882627,0,-0.002395308,0.001867202,0.003384091,-0.000721473,8.217465e-05]},{"f":[22,-1,22,-1,23,18,2,22,-1,0,-1,17,17,1,-1,2,-1,-1,-1,2,-1,-1,-1,17,15,-1,-1,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,0.5,10.543869317296533,9.266718101290426,13.689894219688643,0,1.5,0,3.5221970596792267,1.292481250360578,2.0015923566878984,0,2.453445297804259,0,0,0,1.292481250360578,0,0,0,10.682994316786338,9.264442600226602,0,0,0,0],"m":[0,0,0,0,0,1,1,1,0,0,0,1,1,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0],"l":[1,0,3,0,5,6,7,8,0,10,0,12,13,14,0,16,0,0,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,4,0,28,23,22,9,0,11,0,19,18,15,0,17,0,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0.0005421899,0,-0.002414802,0,0,0,0,7.582181e-05,0,0.0001621002,0,0,0,-0.002727973,0,-0.0005203504,-0.002928067,-0.006959846,0,-0.0005203504,-0.001832499,0.002980426,0,0,-0.0009883918,-0.005065854,5.905962e-05,7.806636e-05]},{"f":[33,33,33,33,33,33,12,1,-1,-1,-1,25,19,-1,-1,-1,-1,33,-1,-1,0,22,-1,-1,-1,-1,4,-1,-1],"t":[37.98858650139573,35.98606694718995,29.75825500004914,27.27007187651612,26.622428940176302,23.58371303411409,0.5,8.046875,0,0,0,1.5,0.5,0,0,0,0,27.807475161786478,0,0,8.5,16.978341770814396,0,0,0,0,3.753897320099348,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,12,13,0,0,0,0,18,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,20,17,16,11,10,9,0,0,0,15,14,0,0,0,0,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,-1.522162e-05,0.0007554014,-0.002908381,0,0,0.002291168,-0.001374452,0.005383822,-0.002358459,0,0.004598028,6.771728e-05,0,0,0.00195842,-0.0008969489,-0.002389769,0.001909219,0,-0.001571287,0.0007211925]},{"f":[4,-1,18,16,18,17,16,15,-1,-1,1,-1,-1,23,-1,-1,-1,4,-1,18,-1,15,19,-1,-1,22,-1,-1,-1],"t":[4.0437314206251695,0,8.03615005796417,7.651051691178929,6.434604125275413,6.4178278454199,6.022367813028454,3.584962500721156,0,0,3.9935774037849088,0,0,1.5,0,0,0,4.2849278041654735,0,6.118903736861568,0,10.457380879072534,0.5,0,0,15.782972563647261,0,0,0],"m":[1,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,14,0,0,0,18,0,20,0,22,23,0,0,26,0,0,0],"r":[2,0,28,17,16,13,10,9,0,0,12,0,0,15,0,0,0,19,0,21,0,25,24,0,0,27,0,0,0],"v":[0,4.790949e-05,0,0,0,0,0,0,0.0001461957,-0.001630439,0,0.004795219,0.0009288887,0,-0.003042312,-6.438843e-05,0.00208001,0,-0.004658157,0,-5.930549e-05,0,0,-0.009169392,-0.001223174,0,0.001667768,-0.003055979,0.000647258]},{"f":[22,22,22,22,22,-1,-1,-1,-1,22,18,-1,-1,1,17,17,22,18,-1,-1,0,-1,-1,-1,-1,-1,25,-1,-1],"t":[17.105450904090162,17.069460217922263,16.838461650489297,16.786010507999382,16.765402642760364,0,0,0,0,17.078739781790915,7.65814076587311,0,0,7.9752100840336135,8.729619742794132,8.571721503073015,17.07907323966382,7.383290334558488,0,0,6.5,0,0,0,0,0,0.5,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,0,0,10,11,0,0,14,15,16,17,18,0,0,21,0,0,0,0,0,27,0,0],"r":[26,9,8,7,6,0,0,0,0,13,12,0,0,25,24,23,20,19,0,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,1.696999e-05,-0.01150829,0.0009789716,-0.0005422665,0,0,0.002713771,0.0001446614,0,0,0,0,0,-0.003177422,0.0004218909,0,0.003834523,-0.0001118748,-0.007345586,0.001456651,0.002280492,0,1.907716e-06,-0.003130227]},{"f":[2,2,21,22,26,-1,-1,22,-1,3,-1,-1,16,16,15,-1,-1,-1,-1,15,-1,-1,2,17,-1,-1,15,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,15.717370311433935,0.5,0,0,15.756821747580634,0,0.5187999999999999,0,0,11.98370619265935,7.679480099505446,7.6653359171851765,0,0,0,0,5.672425341971495,0,0,5.538407798525416,1.292481250360578,0,0,1.0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,0,13,14,15,0,0,0,0,20,0,0,23,24,0,0,27,0,0],"r":[22,19,12,7,6,0,0,9,0,11,0,0,18,17,16,0,0,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,-2.750893e-06,0.002336563,0,-0.006768533,0,-0.004493435,-0.0002090814,0,0,0,0.001167651,-0.004269232,0.002604742,-0.002853839,0,-0.0002321347,-0.005814671,0,0,0.004027894,-0.0005345162,0,-0.002843354,0.0001468747]},{"f":[23,1,22,-1,-1,2,15,2,-1,-1,25,4,18,-1,-1,18,18,-1,-1,17,-1,-1,15,22,-1,-1,-1,-1,-1],"t":[8.5,5.417948717948718,17.083799521744545,0,0,8.049190845056867,2.807354922057604,1.292481250360578,0,0,2.5,3.084962500721156,7.980000966034041,0,0,8.03615005796417,7.252657678068118,0,0,9.075473573191719,0,0,4.523561956057013,12.613886328427084,0,0,0,0,0],"m":[1,1,1,0,0,1,0,1,0,0,1,1,1,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,11,12,13,0,0,16,17,0,0,20,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,10,9,0,0,22,15,14,0,0,19,18,0,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,4.091925e-05,0.003562964,0,0,0,-0.002985911,0.001996266,0,0,0,0.0002633189,-0.001622072,0,0,-0.0005702189,-0.004598052,0,0.006691126,-0.0004914829,0,0,0.002150164,-0.003919364,0.005209785,-0.003390583,0.001254584]},{"f":[2,2,17,17,15,16,-1,-1,-1,-1,1,-1,17,-1,16,-1,16,-1,-1,21,-1,-1,2,25,-1,-1,15,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.496347487943302,7.6181566135214025,10.457380879072534,9.756556322524087,0,0,0,0,1.000936329588015,0,8.729619742794132,0,10.46352437327118,0,10.696098170955864,0,0,7.5,0,0,5.538407798525416,7.5,0,0,1.0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,11,0,13,0,15,0,17,0,0,20,0,0,23,24,0,0,27,0,0],"r":[22,19,10,9,8,7,0,0,0,0,12,0,14,0,16,0,18,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,0,-1.03425e-05,-0.002162359,0.000709723,0.001065938,0,-0.004889359,0,-0.005728363,0,0.0004839481,0,-0.003434714,-0.0004512763,0,-0.005528154,-0.0002226544,0,0,0.0003757008,0.004586315,0,-0.002706148,0.0001470388]},{"f":[21,-1,20,22,2,21,16,17,-1,-1,15,-1,-1,22,-1,0,-1,22,-1,-1,22,22,-1,16,-1,-1,-1,-1,-1],"t":[7.5,0,0.5,16.98920034885663,1.292481250360578,8.5,3.169925001442312,1.292481250360578,0,0,10.066089190457772,0,0,10.224917391507956,0,2.5,0,14.116600814324018,0,0,16.844368517741394,12.738408632106918,0,7.330916878114617,0,0,0,0,0],"m":[1,0,0,1,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,14,0,16,0,18,0,0,21,22,0,24,0,0,0,0,0],"r":[2,0,28,27,20,13,10,9,0,0,12,0,0,15,0,17,0,19,0,0,26,23,0,25,0,0,0,0,0],"v":[0,4.282642e-05,0,0,0,0,0,0,-0.003757856,-0.0001512461,0,-0.00922063,-0.001534316,0,0.003843341,0,0.001152676,0,-0.005989992,-0.0007517035,0,0,-0.0005687404,0,0.005975231,8.973639e-05,-0.003211293,0.0007171386,0.0002032182]},{"f":[2,3,22,-1,-1,4,2,4,4,35,-1,-1,-1,16,17,-1,-1,-1,17,23,-1,-1,-1,16,1,-1,-1,-1,-1],"t":[10.680359255641626,0.007093253968253968,16.725912049439096,0,0,9.266718101290426,9.034782574103549,7.675607660927803,7.408266855279292,2.5,0,0,0,7.614709844115208,11.303485185891478,0,0,0,9.44286754736589,0.5,0,0,0,10.486835021563051,2.0015923566878984,0,0,0,0],"m":[1,0,0,0,0,1,1,1,1,1,0,0,0,1,1,0,0,0,1,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,0,0,14,15,0,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[28,5,4,0,0,23,18,13,12,11,0,0,0,17,16,0,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0.003316928,-0.0007130086,0,0,0,0,0,2.273724e-05,-0.0006666659,-0.003303114,0,0,0.006578861,-0.002912815,-0.001041211,0,0,-0.00104875,-0.004496265,0.001686783,0,0,0.0009398023,-0.002598756,0.00494728,-0.001279562]},{"f":[22,-1,22,-1,23,18,2,-1,-1,17,17,-1,-1,-1,21,17,-1,15,22,-1,22,-1,-1,15,-1,-1,37,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,0.5,10.543869317296533,9.266718101290426,0,0,10.682994316786338,10.609627990977145,0,0,0,5.5,3.753897320099348,0,6.523561956057013,16.96990787537721,0,17.069460217922263,0,0,10.462502272597265,0,0,10.780000000000001,0,0],"m":[0,0,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,0,5,6,7,0,0,10,11,0,0,0,15,16,0,18,19,0,21,0,0,24,0,0,27,0,0],"r":[2,0,4,0,14,9,8,0,0,13,12,0,0,0,26,17,0,23,20,0,22,0,0,25,0,0,28,0,0],"v":[0,0.0005305529,0,-0.002274515,0,0,0,-7.5219e-05,0.0028057,0,0,-0.001150688,-0.005647575,6.137202e-05,0,0,0.0007054219,0,0,-0.0004906985,0,-0.00721989,-0.0007997264,0,0.0007140637,-0.001636966,0,-3.657703e-05,-0.002722391]},{"f":[23,22,2,16,-1,-1,24,1,16,-1,1,-1,22,-1,1,-1,-1,4,23,-1,-1,-1,17,-1,-1,24,-1,-1,-1],"t":[8.5,17.105450904090162,1.292481250360578,5.614709844115208,0,0,9.5,8.102912599421547,6.882643049361842,0,4.994215473608098,0,16.819976309430963,0,5.015625,0,0,4.80643424864552,1.5,0,0,0,6.584884225259038,0,0,8.5,0,0,0],"m":[1,1,0,0,0,0,1,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,11,0,13,0,15,0,0,18,19,0,0,0,23,0,0,26,0,0,0],"r":[28,25,6,5,0,0,22,17,10,0,12,0,14,0,16,0,0,21,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.0002625687,0.0003622905,0,0,0,0.0001603206,0,-3.281596e-05,0,-0.0008253029,0,-0.01097344,-0.0008834916,0,0,0.002238568,0.007660542,-0.0007253174,0,-0.005431207,0.0002264183,0,-0.002997108,-0.0002472393,0.001236166]},{"f":[22,16,-1,-1,22,1,-1,-1,15,15,-1,22,-1,22,-1,17,-1,-1,32,-1,15,-1,15,22,22,-1,-1,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,1.000936329588015,0,0,5.169925001442312,4.087462841250339,0,11.681676585676911,0,14.03140051979143,0,5.740899715832876,0,0,3.753897320099348,0,5.491853096329675,0,5.614709844115208,13.689894219688643,13.129761635363703,0,0,0,0],"m":[0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,0,12,0,14,0,16,0,0,19,0,21,0,23,24,25,0,0,0,0],"r":[4,3,0,0,8,7,0,0,18,11,0,13,0,15,0,17,0,0,20,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0.0002777021,0.002129667,0,0,-0.002066991,-0.0002764294,0,0,-1.807745e-06,0,-0.003191819,0,1.846245e-05,0,-0.0001204575,-0.008049883,0,0.004560338,0,0.00111685,0,0,0,-0.003110715,0.001328955,-0.002582603,4.626319e-05]},{"f":[2,2,-1,16,-1,-1,2,25,-1,-1,18,2,4,15,22,-1,-1,-1,-1,-1,15,21,-1,16,-1,-1,2,-1,-1],"t":[5.40929108874043,5.2849278041654735,0,7.330916878114617,0,0,5.538407798525416,7.5,0,0,9.273793716589307,8.9274737487839,7.997154617983022,2.0,7.5621616439616925,0,0,0,0,0,9.862637357558794,6.5,0,10.681238411777805,0,0,10.41668595978746,0,0],"m":[1,1,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,15,0,0,0,0,0,21,22,0,24,0,0,27,0,0],"r":[6,3,0,5,0,0,10,9,0,0,20,19,18,17,16,0,0,0,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,-2.036912e-05,0,-0.0002148726,-0.0052646,0,0,0.0003883331,0.004310387,0,0,0,0,0,0.0005886734,-0.004237397,0.0002258678,0.003004743,-0.0008694977,0,0,-0.00108897,0,0.004422683,5.006568e-05,0,0.00568535,-0.0004277223]},{"f":[33,32,15,15,15,-1,-1,4,23,-1,24,-1,-1,2,-1,22,-1,-1,22,1,-1,-1,-1,36,-1,-1,4,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,8.005624549193879,7.924812503605781,0,0,3.6427011094311244,0.5,0,8.5,0,0,6.139515068224824,0,16.838461650489297,0,0,11.722778233171695,1.3336158192090395,0,0,0,3.5,0,0,3.753897320099348,0,0],"m":[1,1,1,1,1,0,0,1,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,11,0,0,14,0,16,0,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,18,7,6,0,0,13,10,0,12,0,0,15,0,17,0,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,2.240954e-05,-0.001422935,0,0,0.0004498425,0,0.003274682,5.023885e-05,0,-0.002705903,0,0.001991154,-0.00292824,0,0,-0.003794542,-0.0007131117,-1.670109e-05,0,0.003008605,-0.0002080645,0,-0.001515366,0.0007313202]},{"f":[22,16,-1,-1,22,1,-1,-1,18,18,2,4,4,15,-1,-1,-1,-1,19,22,22,-1,-1,-1,-1,-1,15,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,1.000936329588015,0,0,12.18494601563336,11.43339081173756,9.341519573083566,9.221342183932917,9.05390992190079,5.169925001442312,0,0,0,0,0.5,12.457649623491124,11.685405419153785,0,0,0,0,0,11.385323176175872,0,0],"m":[0,1,0,0,0,1,0,0,1,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,12,13,14,0,0,0,0,19,20,21,0,0,0,0,0,27,0,0],"r":[4,3,0,0,8,7,0,0,26,25,18,17,16,15,0,0,0,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0.0002633285,0.002046508,0,0,-0.001937941,-0.0002615277,0,0,0,0,0,0,-0.0001327019,7.873903e-05,-0.002534419,0.002892354,0,0,0,0.0002538245,-0.003350836,0.001494384,-0.003298603,0.001740909,0,9.590898e-05,-0.001852343]},{"f":[2,2,21,22,26,-1,-1,22,-1,3,-1,2,-1,-1,16,16,-1,-1,-1,15,-1,-1,2,17,-1,-1,15,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,15.717370311433935,0.5,0,0,15.756821747580634,0,0.5187999999999999,0,4.0437314206251695,0,0,11.98370619265935,8.321928094887362,0,0,0,5.672425341971495,0,0,5.538407798525416,1.292481250360578,0,0,1.0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,12,0,0,15,16,0,0,0,20,0,0,23,24,0,0,27,0,0],"r":[22,19,14,7,6,0,0,9,0,11,0,13,0,0,18,17,0,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,-5.315927e-06,0.002223832,0,-0.006322245,0,-0.00427046,0,-7.023735e-05,-0.002317307,0,0,0.000647073,0.002959595,-0.00266012,0,-0.0001582303,-0.005130643,0,0,0.00341067,-0.0004645526,0,-0.002504274,0.0001461899]},{"f":[1,1,33,32,1,-1,17,2,-1,-1,16,-1,-1,-1,-1,16,-1,-1,17,21,-1,15,-1,-1,34,20,-1,-1,-1],"t":[9.107199999999999,8.102912599421547,37.98858650139573,10.41931383093672,5.417948717948718,0,2.160964047443681,1.292481250360578,0,0,10.826548487290914,0,0,0,0,6.539158811108031,0,0,2.160964047443681,4.5,0,8.361943773735241,0,0,3.5,1.5,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,11,0,0,0,0,16,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[18,15,14,13,6,0,10,9,0,0,12,0,0,0,0,17,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,4.297021e-05,0,0,-0.001826854,-0.0002726229,0,0.0001708888,-0.004266228,0.001452962,-0.001717808,0,-0.0004320023,0.005570151,0,0,0.00357933,0,0.0002077861,-0.003824659,0,0,-0.000356735,-0.005042263,0.0002174225]},{"f":[3,22,-1,-1,3,22,15,-1,-1,-1,23,18,15,15,17,17,18,18,-1,-1,17,-1,-1,-1,-1,-1,-1,-1,-1],"t":[0.007093253968253968,16.725912049439096,0,0,0.034940494354592615,16.73920708776685,5.700439718141092,0,0,0,0.5,9.845435504646158,10.113742166049189,9.756556322524087,9.267956609922159,7.2941523604036345,3.8571227588330617,2.903677461028802,0,0,5.281121212110536,0,0,0,0,0,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,0,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,13,14,15,16,17,18,0,0,21,0,0,0,0,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,27,26,25,24,23,20,19,0,0,22,0,0,0,0,0,0,0,0],"v":[0,0,0.002010181,-0.0007007415,0,0,0,-0.0008078068,-0.003190581,0.00202823,0,0,0,0,0,0,0,0,-8.728396e-05,-0.006083099,0,0.003183532,0.0003283167,-0.002040793,0.002320903,-0.002264961,0.001450276,-0.0007076312,9.536087e-05]},{"f":[1,1,33,32,1,-1,-1,-1,-1,16,18,-1,36,-1,33,-1,-1,-1,17,21,-1,15,-1,-1,34,20,-1,-1,-1],"t":[9.107199999999999,8.102912599421547,37.98858650139573,10.41931383093672,5.417948717948718,0,0,0,0,6.539158811108031,1.792481250360578,0,2.5,0,15.895530557358477,0,0,0,2.160964047443681,4.5,0,8.361943773735241,0,0,3.5,1.5,0,0,0],"m":[1,1,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,0,0,10,11,0,13,0,15,0,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[18,9,8,7,6,0,0,0,0,17,12,0,14,0,16,0,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,4.21271e-05,-0.0002058462,0.001383255,-0.001622772,0,0,-0.001136124,0,0.00216843,0,-0.001793616,0.0009711755,0.005295917,0,0,0.003384492,0,0.000193415,-0.003643244,0,0,-0.0003437731,-0.004793476,0.0001749777]},{"f":[33,33,33,33,12,3,15,22,-1,-1,-1,-1,-1,36,-1,-1,-1,33,-1,-1,34,-1,17,36,-1,1,-1,-1,-1],"t":[29.807475161786478,27.27007187651612,26.622428940176302,23.76434662895582,0.5,0.05277777777777778,10.113742166049189,16.765402642760364,0,0,0,0,0,3.5,0,0,0,27.807475161786478,0,0,2.5,0,9.26326872279839,3.5,0,9.107199999999999,0,0,0],"m":[1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,0,14,0,0,0,18,0,0,21,0,23,24,0,26,0,0,0],"r":[20,17,16,13,12,11,10,9,0,0,0,0,0,15,0,0,0,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,0,0,0.0001294717,-0.004129104,-0.003740739,2.363393e-05,-0.002525047,0,-0.001325146,0.003595616,-0.002249899,0,0.004334032,8.357521e-05,0,-0.003464471,0,0,0.002862551,0,0.0002476847,-0.001668891,-0.002612866]},{"f":[3,22,-1,-1,3,22,15,-1,-1,-1,22,23,-1,18,16,18,-1,-1,18,1,-1,-1,-1,16,2,-1,-1,-1,-1],"t":[0.008766718459616732,16.725912049439096,0,0,0.034940494354592615,16.73920708776685,5.20945336562895,0,0,0,17.105450904090162,0.5,0,10.009557117932596,7.924812503605781,8.645081439392136,0,0,1.292481250360578,9.107199999999999,0,0,0,10.462502272597265,4.128693921346326,0,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,0,0,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,0,14,15,16,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,13,0,23,18,17,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0.001902584,-0.000618959,0,0,0,-0.0007203709,-0.002962289,0.002011953,0,0,-8.39777e-05,0,0,0,0.0002785796,-0.002618957,0,0,0.0006245305,-0.003127962,-0.000668681,0,0,0.001440703,0.004430964,-0.0006344552,-0.001242767]},{"f":[22,-1,21,-1,20,22,2,21,16,17,-1,-1,15,-1,-1,22,-1,21,-1,-1,16,-1,-1,-1,16,-1,16,-1,-1],"t":[2.084962500721156,0,7.5,0,0.5,16.98920034885663,1.292481250360578,8.5,3.169925001442312,1.292481250360578,0,0,10.066089190457772,0,0,10.224917391507956,0,9.5,0,0,1.0,0,0,0,6.044394119358453,0,7.330916878114617,0,0],"m":[0,0,1,0,0,1,0,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,10,0,0,13,0,0,16,0,18,0,0,21,0,0,0,25,0,27,0,0],"r":[2,0,4,0,24,23,20,15,12,11,0,0,14,0,0,17,0,19,0,0,22,0,0,0,26,0,28,0,0],"v":[0,0.0009380822,0,3.599063e-05,0,0,0,0,0,0,-0.003513158,-0.0001490788,0,-0.008735808,-0.0015121,0,0.003621623,0,0.0002065552,-0.003598943,0,-0.003403331,-0.000138371,0.0007111375,0,0.001120367,0,-0.001227711,0.0001943527]},{"f":[23,22,16,4,17,4,22,22,33,22,1,-1,-1,-1,-1,17,-1,-1,-1,-1,-1,-1,22,-1,-1,-1,15,-1,-1],"t":[8.5,17.105450904090162,1.584962500721156,7.950787011501063,8.148458103439644,4.128693921346326,16.98920034885663,16.98413870862295,23.925614337747046,16.975010474608837,6.196924603174603,0,0,0,0,1.292481250360578,0,0,0,0,0,0,5.247427792245599,0,0,0,6.539158811108031,0,0],"m":[1,1,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,9,10,11,0,0,0,0,16,0,0,0,0,0,0,23,0,0,0,27,0,0],"r":[26,25,22,21,20,19,18,15,14,13,12,0,0,0,0,17,0,0,0,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0,0.0002996164,-0.002556968,0.002713615,0.003100286,0,-0.004852153,-0.001295333,0.001248061,-0.002097814,-0.002061569,0.002432094,0,0.00123395,-4.362446e-05,-0.001258116,0,0.002409768,-0.0001491651]},{"f":[33,33,33,33,12,19,20,-1,-1,23,-1,-1,-1,33,-1,-1,-1,33,-1,-1,34,-1,17,36,-1,34,-1,-1,-1],"t":[29.807475161786478,27.27007187651612,26.622428940176302,23.58371303411409,0.5,1.5,1.5,0,0,1.5,0,0,0,24.145246550704165,0,0,0,27.807475161786478,0,0,2.5,0,9.26326872279839,3.5,0,3.5,0,0,0],"m":[1,1,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,0,0,0,14,0,0,0,18,0,0,21,0,23,24,0,26,0,0,0],"r":[20,17,16,13,12,9,8,0,0,11,0,0,0,15,0,0,0,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,0,-2.01323e-05,-0.004442101,0,-0.0001524279,0.001092607,-0.002338185,0,0.003555767,-0.0002923288,-0.002109673,0,0.004130243,0.0001036654,0,-0.003254503,0,0,0.002722517,0,-0.00187298,0.00014145,-0.002456667]},{"f":[33,33,33,33,1,1,36,-1,1,-1,-1,4,-1,-1,-1,33,-1,-1,0,22,-1,-1,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,35.98606694718995,29.807475161786478,27.27007187651612,12.18570183086312,10.402747650036153,3.5,0,8.046875,0,0,4.0437314206251695,0,0,0,27.807475161786478,0,0,8.5,16.978341770814396,0,0,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,0,9,0,0,12,0,0,0,16,0,0,19,20,0,0,0,0,25,0,27,0,0],"r":[24,23,18,15,14,11,8,0,10,0,0,13,0,0,0,17,0,0,22,21,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0,0,0,0,-2.236905e-05,0,0.0002112417,0.005245944,0,-0.004523497,4.456945e-05,0.001865114,0,0.003926375,9.850556e-05,0,0,0.001877912,-0.000861626,-0.002298059,0.001642455,0,0.000563252,0,0.0003216953,-0.002857752]},{"f":[2,2,3,-1,2,-1,-1,16,22,4,19,-1,-1,22,-1,-1,-1,23,16,18,3,-1,-1,3,-1,-1,-1,-1,-1],"t":[5.40929108874043,4.0437314206251695,0.5187999999999999,0,3.903677461028802,0,0,7.330916878114617,16.819976309430963,4.951187557243013,0.5,0,0,8.643851681338631,0,0,0,2.5,10.681238411777805,4.128693921346326,0.33771929824561403,0,0,0.41565040650406504,0,0,0,0,0],"m":[1,1,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,0,0,14,0,0,0,18,19,20,21,0,0,24,0,0,0,0,0],"r":[28,7,4,0,6,0,0,17,16,13,12,0,0,15,0,0,0,27,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,-0.002599443,0,3.744863e-05,0.002386558,0,0,0,0,0.0002696812,0.006214275,0,-0.003036742,-5.755344e-05,-0.002621101,0,0,0,0,0.0001854384,-0.006075585,0,-0.002501005,0.001251536,0.0006595234,-0.005147438,0.0001620232]},{"f":[15,-1,15,21,-1,-1,21,-1,17,1,-1,22,24,-1,22,-1,-1,37,-1,-1,17,-1,2,2,21,-1,-1,-1,-1],"t":[9.756556322524087,0,10.066089190457772,6.5,0,0,3.5,0,2.160964047443681,0.01933482600988898,0,16.725912049439096,9.5,0,15.439474430023438,0,0,6.890000000000001,0,0,3.245926548164837,0,5.065928480304397,1.792481250360578,5.5,0,0,0,0],"m":[1,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,0,12,13,0,15,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[2,0,6,5,0,0,8,0,20,11,0,17,14,0,16,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,4.053846e-05,0,0,-0.0007636693,-0.005058632,0,0.00148321,0,0,-0.001881966,0,0,0.006157233,0,0.0006386893,0.003928121,0,0.000538298,-0.00221681,0,-0.004005609,0,0,0,0.001542942,-0.0006678246,-0.002052594,0.0004312116]},{"f":[23,1,22,22,-1,16,-1,-1,-1,2,19,-1,4,25,-1,24,-1,-1,2,22,-1,-1,2,-1,-1,-1,15,-1,-1],"t":[8.5,5.417948717948718,17.083799521744545,17.074218326240924,0,7.741466986401147,0,0,0,8.049190845056867,1.5,0,2.453445297804259,0.5,0,7.5,0,0,3.903677461028802,14.158358850608169,0,0,4.2849278041654735,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,0,10,11,0,13,14,0,16,0,0,19,20,0,0,23,0,0,0,27,0,0],"r":[26,9,8,5,0,7,0,0,0,25,12,0,18,15,0,17,0,0,22,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,5.15555e-05,0,-0.00528663,-0.000537238,0.00342264,0,0,-0.0003501315,0,0,-0.0008858694,0,0.0008927227,0.004554065,0,0,-0.00406089,9.465915e-06,0,0.005613958,-0.0002177147,-0.002974077,0,0.002326904,-0.000138359]},{"f":[23,1,16,15,-1,-1,-1,17,18,4,19,-1,17,-1,-1,-1,22,-1,-1,1,17,-1,-1,33,-1,33,-1,-1,-1],"t":[0.5,3.354166666666667,11.868050853745158,11.683433292835403,0,0,0,6.734566509914796,5.108372929097653,1.292481250360578,0.5,0,1.792481250360578,0,0,0,11.36027577861459,0,0,6.646978021978022,7.344289078557851,0,0,21.011466140970732,0,21.16864017939033,0,0,0],"m":[0,1,1,1,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,13,0,0,0,17,0,0,20,21,0,0,24,0,26,0,0,0],"r":[28,7,6,5,0,0,0,19,16,15,12,0,14,0,0,0,18,0,0,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,4.665429e-05,-0.004447261,0.002847583,0,0,0,0,-0.003836855,0,0.001008895,-0.003880797,9.550321e-05,0,0.0002539539,0.005111669,0,0,-0.006783125,-0.001222005,0,0.003059342,0,-0.003139286,0.0006396841,7.255589e-05]},{"f":[22,22,4,16,-1,15,-1,22,-1,-1,-1,-1,22,1,-1,-1,15,-1,32,-1,15,-1,15,22,22,-1,-1,-1,-1],"t":[6.671801068872866,6.228305549787459,4.614409345247941,2.807354922057604,0,3.169925001442312,0,2.903677461028802,0,0,0,0,7.082728314749225,1.000936329588015,0,0,5.169925001442312,0,3.753897320099348,0,5.491853096329675,0,5.614709844115208,13.689894219688643,13.129761635363703,0,0,0,0],"m":[0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,0,0,13,14,0,0,17,0,19,0,21,0,23,24,25,0,0,0,0],"r":[12,11,10,5,0,7,0,9,0,0,0,0,16,15,0,0,18,0,20,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0.0001748357,0,0.003856175,0,-0.002136453,0.001247996,-0.001181963,0.001447152,0,0,-0.001824515,-0.0001809701,0,-0.0001148762,0,0.004156895,0,0.001030172,0,0,0,-0.002964868,0.00125604,-0.002461064,3.992807e-05]},{"f":[33,32,15,15,15,-1,-1,4,-1,2,-1,-1,15,18,33,-1,-1,-1,36,-1,-1,24,-1,-1,2,-1,11,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,8.005624549193879,7.924812503605781,0,0,3.6427011094311244,0,6.139515068224824,0,0,10.066089190457772,8.03615005796417,29.807475161786478,0,0,0,3.5,0,0,9.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,0,13,14,15,0,0,0,19,0,0,22,0,0,25,0,27,0,0],"r":[24,21,12,7,6,0,0,9,0,11,0,0,18,17,16,0,0,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,1.833335e-05,-0.001288839,0,0.001074107,0,-0.002581587,0.0007635017,0,0,0,-0.0004135182,-0.003301587,0.000587599,0,-0.003603936,1.29789e-06,0,0.002396917,-0.000511807,0,0.0005317938,0,-0.002660718,0.0002884979]},{"f":[2,2,3,-1,2,-1,-1,16,22,4,19,-1,-1,22,-1,-1,-1,23,16,18,3,-1,-1,3,-1,-1,-1,-1,-1],"t":[5.40929108874043,4.0437314206251695,0.5187999999999999,0,3.903677461028802,0,0,7.330916878114617,16.819976309430963,4.951187557243013,0.5,0,0,8.643851681338631,0,0,0,2.5,10.681238411777805,4.128693921346326,0.33771929824561403,0,0,0.41565040650406504,0,0,0,0,0],"m":[1,1,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,0,0,14,0,0,0,18,19,20,21,0,0,24,0,0,0,0,0],"r":[28,7,4,0,6,0,0,17,16,13,12,0,0,15,0,0,0,27,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,-0.002436959,0,3.45659e-05,0.002246299,0,0,0,0,0.0002695934,0.005657506,0,-0.00288125,-5.888126e-05,-0.002434113,0,0,0,0,0.000114365,-0.005747671,0,-0.002373634,0.00125603,0.0006192777,-0.004877933,0.0001547705]},{"f":[23,1,16,15,-1,-1,-1,17,18,4,19,-1,17,-1,-1,-1,22,-1,-1,1,17,-1,-1,33,-1,33,-1,-1,-1],"t":[0.5,3.354166666666667,11.868050853745158,11.683433292835403,0,0,0,6.734566509914796,5.108372929097653,1.292481250360578,0.5,0,1.792481250360578,0,0,0,11.36027577861459,0,0,6.646978021978022,7.344289078557851,0,0,21.011466140970732,0,21.16864017939033,0,0,0],"m":[0,1,1,1,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,13,0,0,0,17,0,0,20,21,0,0,24,0,26,0,0,0],"r":[28,7,6,5,0,0,0,19,16,15,12,0,14,0,0,0,18,0,0,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,4.312619e-05,-0.004192155,0.002724354,0,0,0,0,-0.003627221,0,0.0009554082,-0.003678735,9.510937e-05,0,0.0002347807,0.004814898,0,0,-0.006408491,-0.001181223,0,0.00286422,0,-0.002989306,0.0005928982,7.086195e-05]},{"f":[23,1,-1,18,19,-1,22,4,22,-1,25,-1,23,-1,-1,37,22,-1,36,-1,-1,-1,22,-1,-1,2,-1,-1,-1],"t":[8.5,5.417948717948718,0,9.755304827902876,1.5,0,15.717370311433935,2.453445297804259,11.775312925643524,0,0.5,0,0.5,0,0,11.26,11.775312925643524,0,3.5,0,0,0,16.759160379664443,0,0,1.792481250360578,0,0,0],"m":[1,1,0,1,1,0,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,7,8,9,0,11,0,13,0,0,16,17,0,19,0,0,0,23,0,0,26,0,0,0],"r":[28,3,0,25,6,0,22,15,10,0,12,0,14,0,0,21,18,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,4.240011e-05,0,0,-0.0003399727,0,0,0,0.003327886,0,-0.001046529,0,0.0003839998,0.003792318,0,0,0.003465999,0,-0.002113518,0.001319155,-0.004510246,0,-0.0104009,0.001623688,0,0.001178703,-0.004267208,0.001082709]},{"f":[2,2,-1,21,-1,-1,2,-1,18,2,4,15,22,-1,4,-1,-1,-1,-1,-1,15,2,4,-1,-1,-1,2,-1,-1],"t":[5.40929108874043,5.2849278041654735,0,7.5,0,0,5.538407798525416,0,9.273793716589307,8.9274737487839,7.997154617983022,2.0,7.5621616439616925,0,6.16978583919235,0,0,0,0,0,9.862637357558794,9.645601091186107,6.044228501743113,0,0,0,10.41668595978746,0,0],"m":[1,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,0,9,10,11,12,13,0,15,0,0,0,0,0,21,22,23,0,0,0,27,0,0],"r":[6,3,0,5,0,0,8,0,20,19,18,17,14,0,16,0,0,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,-1.838432e-05,0,-0.004677318,-7.308613e-05,0,0.001912573,0,0,0,0,0,0.000692915,0,-0.001794664,-0.00619789,0.0001818501,0.002779106,-0.0008095989,0,0,0,-0.0004992548,0.002506361,-0.002597052,0,0.005416312,-0.0002441138]},{"f":[4,4,2,2,3,-1,2,-1,-1,-1,18,22,16,-1,-1,-1,15,-1,-1,2,-1,-1,16,15,17,-1,-1,-1,-1],"t":[9.266718101290426,9.05390992190079,5.40929108874043,4.0437314206251695,0.5187999999999999,0,3.903677461028802,0,0,0,10.090748425878532,8.32532955673354,7.651051691178929,0,0,0,8.618385502258606,0,0,9.20088558729989,0,0,10.696098170955864,4.247927513443585,9.293471190846834,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,12,13,0,0,0,17,0,0,20,0,0,23,24,25,0,0,0,0],"r":[22,19,10,9,6,0,8,0,0,0,16,15,14,0,0,0,18,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.002326614,0,3.23055e-05,0.002145682,-0.0003816402,0,0,0,0.003984154,-0.002818845,-2.215655e-05,0,1.281569e-05,0.005241677,0,-0.002419263,-0.0003903762,0,0,0,0.002757849,-0.0003250171,-0.002229412,0.00292931]},{"f":[2,2,4,4,4,22,-1,23,24,-1,-1,-1,22,-1,-1,16,-1,-1,23,16,-1,-1,-1,4,-1,16,-1,-1,-1],"t":[10.680359255641626,9.266718101290426,8.950330235806282,7.950787011501063,7.408266855279292,14.03140051979143,0,2.5,6.5,0,0,0,8.658207063149977,0,0,4.087462841250339,0,0,0.5,2.321928094887362,0,0,0,9.328332883600808,0,4.247927513443585,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,0,13,0,0,16,0,0,19,20,0,0,0,24,0,26,0,0,0],"r":[28,23,18,15,12,7,0,11,10,0,0,0,14,0,0,17,0,0,22,21,0,0,0,25,0,27,0,0,0],"v":[0,0,0,0,0,0,6.382708e-05,0,0,-0.0053878,-0.0004587916,3.746395e-05,0,-0.003927722,0.0002876398,0,0.007970478,-0.002981898,0,0,-0.001633402,0.000322287,-0.002810532,0,0.004035511,0,0.0006566655,-0.001842715,-0.001098238]},{"f":[22,-1,21,-1,20,22,2,21,16,22,22,-1,-1,-1,15,-1,-1,22,-1,21,-1,-1,16,-1,-1,-1,16,-1,-1],"t":[2.084962500721156,0,7.5,0,0.5,16.98920034885663,1.292481250360578,8.5,3.169925001442312,16.978341770814396,16.975010474608837,0,0,0,10.066089190457772,0,0,10.224917391507956,0,9.5,0,0,1.0,0,0,0,6.044394119358453,0,0],"m":[0,0,1,0,0,1,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,10,11,0,0,0,15,0,0,18,0,20,0,0,23,0,0,0,27,0,0],"r":[2,0,4,0,26,25,22,17,14,13,12,0,0,0,16,0,0,19,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0.0008381778,0,3.507826e-05,0,0,0,0,0,0,0,-0.00194298,0.002541671,-0.003227874,0,-0.00827699,-0.001467008,0,0.00340747,0,0.0001909591,-0.003431059,0,-0.003105776,-0.0001114047,0.0006840911,0,0.001050062,-7.81902e-05]},{"f":[23,-1,1,-1,1,2,-1,-1,18,16,17,-1,18,-1,1,-1,-1,17,22,24,-1,-1,-1,-1,16,4,-1,-1,-1],"t":[0.5,0,0.01933482600988898,0,0.3808851224105462,5.14960400919364,0,0,10.009557117932596,7.924812503605781,8.571721503073015,0,8.729619742794132,0,2.0015923566878984,0,0,1.292481250360578,16.725912049439096,7.5,0,0,0,0,10.462502272597265,4.128693921346326,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,0,0,9,10,11,0,13,0,15,0,0,18,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,4,0,8,7,0,0,24,17,12,0,14,0,16,0,0,23,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-7.686503e-05,0,-0.0012729,0,0,0.0002960431,0.004442453,0,0,0,0.0002279005,0,-0.006407772,0,-0.004240404,0.001151778,0,0,0,0.004764506,0.0009115996,-0.003520044,-0.0005418304,0,0,0.001300601,0.004096608,-0.0006332122]},{"f":[22,23,22,-1,15,22,-1,-1,-1,4,1,18,18,-1,-1,17,-1,-1,2,22,-1,-1,-1,16,-1,-1,25,-1,-1],"t":[17.105450904090162,3.5,16.978341770814396,0,6.0,17.069460217922263,0,0,0,6.451187557243013,8.046875,8.496347487943302,7.467398296089201,0,0,9.523560624409399,0,0,4.128693921346326,15.212004812197344,0,0,0,6.569855608330948,0,0,0.5,0,0],"m":[1,1,1,0,1,0,0,0,0,1,1,1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,0,0,10,11,12,13,0,0,16,0,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,9,4,0,8,7,0,0,0,23,18,15,14,0,0,17,0,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,-1.47207e-05,0,0,-0.005988447,-0.001435857,0.001493757,0,0,0,0,-0.0001868329,0.0007150649,0,-0.003119494,0.0009949551,0,0,0.0007205396,0.005812916,-0.0004397406,0,0.004026567,0.0003949163,0,1.855839e-05,-0.002730191]},{"f":[3,22,-1,-1,3,22,15,-1,-1,-1,23,18,15,18,15,18,18,-1,-1,-1,-1,-1,2,-1,4,-1,-1,-1,-1],"t":[0.007093253968253968,16.752707246237605,0,0,0.034940494354592615,16.73920708776685,4.247927513443585,0,0,0,0.5,9.845435504646158,10.342074667999139,9.267956609922159,9.756556322524087,7.298618922788638,3.6427011094311244,0,0,0,0,0,1.292481250360578,0,2.160964047443681,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,13,14,15,16,17,0,0,0,0,0,23,0,25,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,27,22,21,20,19,18,0,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0.001543184,-0.001384043,0,0,0,-0.0006819662,-0.002661056,0.001858178,0,0,0,0,0,0,0,-0.0001374177,0.0004715885,-0.00193807,-0.001840134,0.002604949,0,0.003311182,0,-0.001952484,0.001724845,-0.0006390429,8.659154e-05]},{"f":[15,17,2,2,2,-1,-1,-1,16,-1,-1,-1,23,21,-1,33,-1,22,1,-1,19,2,-1,-1,-1,17,-1,-1,-1],"t":[1.584962500721156,9.293471190846834,7.894792293346235,5.065928480304397,4.128693921346326,0,0,0,1.584962500721156,0,0,0,0.5,1.5,0,37.33732815674957,0,14.03140051979143,3.354166666666667,0,0.5,1.292481250360578,0,0,0,2.160964047443681,0,0,0],"m":[0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,0,0,0,13,14,0,16,0,18,19,0,21,22,0,0,0,26,0,0,0],"r":[12,11,8,7,6,0,0,0,10,0,0,0,28,15,0,17,0,25,20,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0,0.0002740867,0.002284059,-0.003260396,0,0.00477731,-5.891899e-05,-0.002234908,0,0,0.0001147015,0,0.0004915625,0,0,-2.838205e-05,0,0,-0.007402583,-0.000641399,-0.0001684462,0,0.001475448,-0.00779603,6.558655e-05]},{"f":[22,22,4,16,-1,15,-1,22,-1,-1,-1,-1,22,1,-1,-1,21,23,22,-1,15,-1,-1,17,-1,4,-1,-1,-1],"t":[6.671801068872866,6.228305549787459,4.614409345247941,2.807354922057604,0,3.169925001442312,0,2.903677461028802,0,0,0,0,7.082728314749225,1.000936329588015,0,0,7.5,2.5,16.819976309430963,0,5.044394119358453,0,0,3.245926548164837,0,6.451187557243013,0,0,0],"m":[0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,0,0,13,14,0,0,17,18,19,0,21,0,0,24,0,26,0,0,0],"r":[12,11,10,5,0,7,0,9,0,0,0,0,16,15,0,0,28,23,20,0,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,0.0001421931,0,0.003724013,0,-0.001976003,0.001295521,-0.001070436,0.001380165,0,0,-0.001664304,-9.275003e-05,0,0,0,-1.745877e-06,0,-0.00450495,0.000601354,0,0.001129113,0,-0.0005056882,0.002430005,-0.0001588322]},{"f":[22,22,4,16,-1,15,-1,22,-1,-1,-1,-1,22,1,-1,-1,22,21,4,4,4,-1,-1,-1,22,-1,-1,-1,-1],"t":[6.671801068872866,6.228305549787459,4.614409345247941,2.807354922057604,0,3.169925001442312,0,2.903677461028802,0,0,0,0,7.082728314749225,1.000936329588015,0,0,11.406459975416917,9.5,6.451187557243013,6.118903736861568,5.169368191286958,0,0,0,10.029977181607594,0,0,0,0],"m":[0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,0,0,13,14,0,0,17,18,19,20,21,0,0,0,25,0,0,0,0],"r":[12,11,10,5,0,7,0,9,0,0,0,0,16,15,0,0,28,27,24,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0,0.0001351045,0,0.003541131,0,-0.001879664,0.001231545,-0.001017256,0.001311741,0,0,-0.001581684,-8.814013e-05,0,0,0,0,0,0.0001345128,0.002311601,-0.004396169,0,0.004906134,9.191799e-05,-0.001546391,-3.83021e-05]},{"f":[1,1,33,32,1,-1,17,-1,16,-1,-1,-1,-1,16,18,-1,-1,-1,17,22,33,-1,-1,-1,34,20,-1,-1,-1],"t":[9.107199999999999,8.102912599421547,37.98858650139573,10.41931383093672,5.417948717948718,0,2.160964047443681,0,10.826548487290914,0,0,0,0,6.539158811108031,1.792481250360578,0,0,0,2.160964047443681,15.439474430023438,25.701296736393502,0,0,0,3.5,1.5,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,7,0,9,0,0,0,0,14,15,0,0,0,19,20,21,0,0,0,25,26,0,0,0],"r":[18,13,12,11,6,0,8,0,10,0,0,0,0,17,16,0,0,0,24,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0,3.79439e-05,0,-0.0007529528,0,0.000147932,-0.003873806,0.001254217,-0.001377039,0,0,-0.001041091,0.0004313479,0.004696546,0,0,0,0.005360682,-0.0002945195,-0.0028101,0,0,-0.0003009543,-0.004304437,0.0002054138]},{"f":[23,22,16,17,16,17,-1,2,22,-1,-1,2,-1,-1,-1,-1,1,-1,1,15,-1,-1,-1,0,-1,-1,16,-1,-1],"t":[8.5,17.105450904090162,6.409390936137702,11.78143940991899,6.392317422778761,7.6181566135214025,0,7.128362343532883,16.978341770814396,0,0,9.427971070948235,0,0,0,0,0.012999172356714255,0,0.3808851224105462,10.462502272597265,0,0,0,7.5,0,0,0.0,0,0],"m":[1,1,0,1,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,0,0,17,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,16,15,14,7,0,11,10,0,0,13,0,0,0,0,18,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,-1.044766e-05,0,0,0.0006770313,-0.001758846,0,0.005545325,-0.001097066,0.00144893,-0.003285362,0,-0.002729147,0,0,0.002858411,0.000108037,-0.0001001002,0,-0.0001478776,-0.002591172,0,-3.71773e-05,0.002394061]},{"f":[4,4,2,2,-1,15,-1,-1,18,22,16,-1,-1,15,-1,-1,15,-1,-1,2,-1,-1,16,15,17,-1,-1,-1,-1],"t":[9.266718101290426,9.05390992190079,5.40929108874043,5.2849278041654735,0,5.672425341971495,0,0,10.090748425878532,8.32532955673354,7.651051691178929,0,0,2.0,0,0,8.618385502258606,0,0,9.20088558729989,0,0,10.696098170955864,4.247927513443585,9.293471190846834,0,0,0,0],"m":[1,1,1,1,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,0,14,0,0,17,0,0,20,0,0,23,24,25,0,0,0,0],"r":[22,19,8,5,0,7,0,0,16,13,12,0,0,15,0,0,18,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,-1.480689e-05,0,-9.378831e-05,-0.004437224,0,0,0,0.003643262,-0.002501573,0,-0.001517064,5.20129e-05,0,2.914181e-05,0.004893838,0,-0.002309288,-0.0003880981,0,0,0,0.002527737,-0.000294979,-0.002091427,0.00279867]}]},"p_near":{"link":"logistic","base":-1.7083696958261418,"trees":[{"f":[2,37,0,-1,23,-1,-1,22,25,-1,-1,23,-1,18,-1,-1,19,37,2,20,16,-1,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,7.05,6.5,0,0.5,0,0,12.613886328427084,0.5,0,0,1.5,0,6.434604125275413,0,0,0.5,6.970000000000001,8.9274737487839,0.5,11.407798850221974,0,0,0,0,0,1.5,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,1,0,1,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,12,0,14,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,7,4,0,6,0,0,11,10,0,0,13,0,15,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,-0.05134991,0,0.02850011,-0.03064597,0,0,0.03896673,0.1367045,0,0.2659932,0,0.1553289,0.03896673,0,0,0,0,0,-0.02919536,0.06537082,0.1157193,0.1211529,0.1970584,0,0.1136987,0.2807844]},{"f":[2,34,22,-1,23,21,-1,-1,37,-1,-1,37,-1,-1,19,37,2,20,22,-1,22,-1,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,2.5,12.613886328427084,0,1.5,3.5,0,0,7.92,0,0,6.890000000000001,0,0,0.5,6.970000000000001,8.9274737487839,0.5,10.735831902194036,0,14.247293309538719,0,0,0,0,0,1.5,0,0],"m":[1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,0,0,9,0,0,12,0,0,15,16,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[14,11,4,0,8,7,0,0,10,0,0,13,0,0,26,25,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0.03933735,0,0,0.1317519,0.2439659,0,-0.04342557,0.1180851,0,-0.05040726,-0.006948983,0,0,0,0,0,0.05438295,0,-0.04454419,0.0734199,0.1018529,0.1032284,0.1639021,0,0.1004456,0.2212872]},{"f":[2,37,0,-1,23,-1,-1,22,25,-1,22,-1,-1,23,-1,-1,19,37,2,20,-1,-1,-1,33,-1,-1,23,-1,-1],"t":[5.228690439536267,7.05,6.5,0,0.5,0,0,12.613886328427084,0.5,0,11.876328901062104,0,0,4.5,0,0,0.5,6.970000000000001,6.692122061121539,0.5,0,0,0,26.99635432609064,0,0,1.5,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,1,1,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,11,0,0,14,0,0,17,18,19,20,0,0,0,24,0,0,27,0,0],"r":[16,7,4,0,6,0,0,13,10,0,12,0,0,15,0,0,26,23,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,-0.04990029,0,0.02537912,-0.02975497,0,0,0.02542094,0,0.1973522,0.05091504,0,0.1675345,0.02542094,0,0,0,0,-0.03252746,0.090436,0.08535718,0,0.1165523,0.2076601,0,0.08974753,0.1846235]},{"f":[2,34,22,-1,23,-1,37,-1,-1,37,-1,2,-1,15,-1,-1,21,-1,19,2,0,-1,-1,2,18,-1,-1,-1,-1],"t":[5.228690439536267,2.5,12.613886328427084,0,1.5,0,7.92,0,0,6.890000000000001,0,2.903677461028802,0,7.118941072723508,0,0,3.5,0,0.5,6.85792738526389,6.5,0,0,10.510264838663748,9.229707804263573,0,0,0,0],"m":[1,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,1,1,1,0,0,1,1,0,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,0,12,0,14,0,0,17,0,19,20,21,0,0,24,25,0,0,0,0],"r":[16,9,4,0,6,0,8,0,0,11,0,13,0,15,0,0,18,0,28,23,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0.03268055,0,0.1432333,0,-0.04315071,0.09192994,0,-0.04897868,0,-0.02256726,0,-0.02210076,0.09475449,0,-0.03384799,0,0,0,-0.01155765,0.09359161,0,0,0.121633,0.02142336,0.2204496,0.1539528]},{"f":[2,37,0,21,-1,-1,-1,33,-1,25,-1,-1,21,-1,9,22,2,1,21,-1,-1,-1,-1,17,5,-1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,0,23.168926553414675,0,0.5,0,0,3.5,0,0.5,12.15291175877056,9.883406784235138,8.046875,7.5,0,0,0,0,6.864066550153108,0.5,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,10,0,0,13,0,15,16,17,18,19,0,0,0,0,24,25,0,0,0,0],"r":[12,7,6,5,0,0,0,9,0,11,0,0,14,0,28,23,22,21,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.05334797,-0.02888053,-0.009250229,0,0.1181088,0,0.01730229,0.1059986,0,-0.03293861,0,0,0,0,0,-0.004513102,0.06153231,0.1268148,0.170874,0,0,-0.0365463,0.09638796,0.1425742,-0.05553876]},{"f":[2,32,23,32,-1,22,-1,-1,-1,37,-1,4,-1,15,-1,-1,21,-1,9,22,23,-1,-1,1,-1,17,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,3.9474088816539714,0,14.158358850608169,0,0,0,6.890000000000001,0,2.903677461028802,0,7.118941072723508,0,0,3.5,0,0.5,12.15291175877056,2.5,0,0,0.512280701754386,0,6.864066550153108,0,0,0],"m":[1,0,1,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,0,10,0,12,0,14,0,0,17,0,19,20,21,0,0,24,0,26,0,0,0],"r":[16,9,8,5,0,7,0,0,0,11,0,13,0,15,0,0,18,0,28,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,0.01098206,0,0.07999277,0.1516447,-0.005184722,0,-0.04760259,0,-0.02242395,0,-0.02105073,0.08940546,0,-0.03204336,0,0,0,0.06671569,-0.01711626,0,-0.02781527,0,0.07818029,0.1311688,-0.0550242]},{"f":[2,34,22,-1,23,-1,37,-1,-1,37,21,-1,-1,2,-1,-1,21,-1,19,2,0,-1,-1,15,-1,22,-1,-1,-1],"t":[5.228690439536267,2.5,12.613886328427084,0,1.5,0,7.92,0,0,6.890000000000001,6.5,0,0,2.903677461028802,0,0,3.5,0,0.5,6.85792738526389,5.5,0,0,11.696967526234287,0,16.725912049439096,0,0,0],"m":[1,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,0,0,14,0,0,17,0,19,20,21,0,0,24,0,26,0,0,0],"r":[16,9,4,0,6,0,8,0,0,13,12,0,0,15,0,0,18,0,28,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0.024374,0,0.1072201,0,-0.04385857,0.07284668,0,0,-0.0523185,-0.02657467,0,-0.02090259,0.03661775,0,-0.03116183,0,0,0,-0.0217389,0.06285095,0,0.1006363,0,0.01197789,0.1238998,0.1128529]},{"f":[2,37,0,21,-1,-1,-1,33,-1,25,-1,-1,21,-1,9,22,2,1,21,-1,-1,-1,-1,1,-1,17,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,0,23.168926553414675,0,0.5,0,0,3.5,0,0.5,12.15291175877056,9.883406784235138,8.046875,7.5,0,0,0,0,0.512280701754386,0,6.864066550153108,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,10,0,0,13,0,15,16,17,18,19,0,0,0,0,24,0,26,0,0,0],"r":[12,7,6,5,0,0,0,9,0,11,0,0,14,0,28,23,22,21,20,0,0,0,0,25,0,27,0,0,0],"v":[0,0,0,0,-0.05179555,-0.02754394,-0.008589689,0,0.09122144,0,0.0101004,0.08629266,0,-0.0302937,0,0,0,0,0,-0.007914038,0.0470884,0.1010863,0.1319321,0,-0.02933038,0,0.06636024,0.1097159,-0.054343]},{"f":[2,32,23,32,-1,-1,-1,37,21,-1,-1,4,-1,15,-1,-1,21,17,-1,-1,9,22,23,-1,-1,1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,3.9474088816539714,0,0,0,6.890000000000001,6.5,0,0,2.903677461028802,0,7.118941072723508,0,0,3.5,1.292481250360578,0,0,0.5,12.15291175877056,2.5,0,0,0.512280701754386,0,0,0],"m":[1,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,14,0,0,17,18,0,0,21,22,23,0,0,26,0,0,0],"r":[16,7,6,5,0,0,0,11,10,0,0,13,0,15,0,0,20,19,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.007352915,0.08435759,-0.008120347,0,0,-0.05128953,-0.02548231,0,-0.02102126,0,-0.02128574,0.07526447,0,0,0.05232471,-0.05455402,0,0,0,0.05231447,-0.01824147,0,-0.02838343,0.09101017,-0.05387992]},{"f":[2,37,0,21,-1,-1,16,-1,-1,22,25,-1,-1,23,-1,-1,21,17,-1,-1,19,2,37,1,-1,-1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,6.044394119358453,0,0,12.613886328427084,0.5,0,0,4.5,0,0,3.5,1.292481250360578,0,0,0.5,8.9274737487839,6.05,0.6824793747286149,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,0,14,0,0,17,18,0,0,21,22,23,24,0,0,0,0,0],"r":[16,9,6,5,0,0,8,0,0,13,12,0,0,15,0,0,20,19,0,0,28,27,26,25,0,0,0,0,0],"v":[0,0,0,0,-0.05078965,-0.02663234,0,-0.02663234,0.006679134,0,0,0.006679134,0.06715705,0,0.08983186,0.006679134,0,0,0.04824232,-0.05400376,0,0,0,0,-0.05275006,0.01385535,0.07156054,0.08616012,0.09100117]},{"f":[2,32,23,22,-1,-1,-1,37,21,-1,-1,4,22,-1,-1,15,-1,-1,21,-1,19,2,37,-1,-1,1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,14.158358850608169,0,0,0,6.890000000000001,6.5,0,0,2.903677461028802,17.094543568094064,0,0,7.118941072723508,0,0,2.5,0,0.5,8.9274737487839,6.775,0,0,2.01685992705101,0,0,0],"m":[1,0,1,1,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,1,1,1,0,0,1,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,0,16,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[18,7,6,5,0,0,0,11,10,0,0,15,14,0,0,17,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.04265729,0.09956031,-0.009130187,0,0,-0.05030417,-0.02468826,0,0,-0.02468826,0.08997507,0,-0.0204007,0.06688811,0,-0.04428007,0,0,0,0.00318417,0.06634833,0,0.05205172,0.1134234,0.08448426]},{"f":[2,37,0,21,-1,-1,-1,22,25,-1,-1,23,-1,-1,21,20,2,-1,-1,-1,9,22,2,-1,-1,1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,0,12.613886328427084,0.5,0,0,4.5,0,0,4.5,1.5,8.612669628062957,0,0,0,0.5,12.15291175877056,9.138550601137563,0,0,0.512280701754386,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,0,15,16,17,0,0,0,21,22,23,0,0,26,0,0,0],"r":[14,7,6,5,0,0,0,11,10,0,0,13,0,0,20,19,18,0,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.04982098,-0.02574493,-0.007612804,0,0,0.004189322,0.05897685,0,0.07956892,0.004189322,0,0,0,-0.04984219,0.02885224,0.07324884,0,0,0,0.02625169,0.07944841,0,-0.03038773,0.07947236,-0.05303717]},{"f":[2,32,23,32,-1,-1,-1,37,21,-1,-1,4,22,-1,-1,-1,19,21,-1,2,37,22,-1,17,-1,-1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,3.9474088816539714,0,0,0,6.890000000000001,6.5,0,0,2.903677461028802,17.094543568094064,0,0,0,0.5,2.5,0,8.9274737487839,6.775,10.735831902194036,0,6.686432530056294,0,0,0,0,0],"m":[1,0,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,0,0,17,18,0,20,21,22,0,24,0,0,0,0,0],"r":[16,7,6,5,0,0,0,11,10,0,0,15,14,0,0,0,28,19,0,27,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0.003079002,0.0653839,-0.00994937,0,0,-0.04935017,-0.02397531,0,0,-0.02397531,0.08162515,0.03232094,0,0,-0.05502644,0,0,0,0.02965982,0,-0.04723008,0.01395375,0.05799928,0.07048127,0.07531179]},{"f":[2,37,0,21,-1,-1,16,-1,-1,22,25,-1,22,-1,-1,-1,19,21,-1,1,2,21,-1,-1,1,-1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,6.044394119358453,0,0,12.613886328427084,0.5,0,11.876328901062104,0,0,0,0.5,2.5,0,6.147321428571429,8.9274737487839,7.5,0,0,2.01685992705101,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,13,0,0,0,17,18,0,20,21,22,0,0,25,0,0,0,0],"r":[16,9,6,5,0,0,8,0,0,15,12,0,14,0,0,0,28,19,0,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.04887828,-0.02495121,0,-0.02495121,0.002258802,0,0,0.002258802,0,0.1010497,0.01253451,0.06378338,0,0,-0.05460514,0,0,0,-0.01766827,0.03432182,0,0.03994211,0.09355377,0.07824164,0.07124798]},{"f":[2,32,23,22,-1,-1,24,-1,-1,37,21,-1,-1,23,-1,22,-1,-1,19,21,-1,2,1,21,-1,-1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,14.158358850608169,0,0,9.5,0,0,6.890000000000001,6.5,0,0,0.5,0,17.094543568094064,0,0,0.5,2.5,0,10.510264838663748,6.147321428571429,7.5,0,0,0,0,0],"m":[1,0,1,1,0,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,0,14,0,16,0,0,19,20,0,22,23,24,0,0,0,0,0],"r":[18,9,6,5,0,0,8,0,0,13,12,0,0,15,0,17,0,0,28,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0.03254898,0.08262798,0,-0.04053136,0.05733405,0,0,-0.04841712,-0.02320139,0,0.02090364,0,-0.02320139,0.080959,0,0,-0.05420203,0,0,0,-0.003615538,0.03817374,0.07340287,0.09828975,0.0675898]},{"f":[2,37,0,21,-1,-1,16,-1,-1,22,25,-1,-1,-1,19,21,-1,2,1,21,2,-1,-1,2,-1,-1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,6.044394119358453,0,0,12.613886328427084,0.5,0,0,0,0.5,2.5,0,10.510264838663748,6.147321428571429,7.5,8.767235038274581,0,0,6.4675825248018475,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,0,0,15,16,0,18,19,20,21,0,0,24,0,0,0,0,0],"r":[14,9,6,5,0,0,8,0,0,13,12,0,0,0,28,17,0,27,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,-0.04795167,-0.02415789,0,-0.02415789,0.0008313582,0,0,0.0008313582,0.04731106,0.05767906,0,0,-0.05381553,0,0,0,0,-0.01684763,0.03015068,0,0.01007363,0.0501089,0.06910251,0.09257225,0.06427027]},{"f":[2,32,23,22,-1,-1,-1,37,21,-1,-1,4,22,-1,-1,-1,21,20,2,-1,-1,-1,1,21,-1,-1,22,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,14.158358850608169,0,0,0,6.890000000000001,6.5,0,0,2.903677461028802,17.094543568094064,0,0,0,4.5,1.5,8.612669628062957,0,0,0,0.6824793747286149,7.5,0,0,12.121442107629743,0,0],"m":[1,0,1,1,0,0,0,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,0,0,17,18,19,0,0,0,23,24,0,0,27,0,0],"r":[16,7,6,5,0,0,0,11,10,0,0,15,14,0,0,0,22,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0.02906729,0.07601426,-0.01245716,0,0,-0.04749573,-0.02249502,0,0,-0.02249502,0.06858353,0.02841536,0,0,0,-0.04812345,0.02159476,0.05082435,0,0,-0.0550579,0.05747865,0,0.02587045,0.0627123]},{"f":[2,0,21,-1,19,-1,-1,15,15,35,-1,-1,19,15,-1,-1,-1,-1,19,21,-1,2,1,21,-1,-1,-1,-1,-1],"t":[5.228690439536267,6.5,6.5,0,0.5,0,0,9.92481250360578,6.169925001442312,1.5,0,0,2.5,7.118941072723508,0,0,0,0,0.5,2.5,0,10.510264838663748,6.147321428571429,7.5,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,0,13,14,0,0,0,0,19,20,0,22,23,24,0,0,0,0,0],"r":[18,7,4,0,6,0,0,17,12,11,0,0,16,15,0,0,0,0,28,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,-0.04703249,0,-0.01259492,-0.04356861,0,0,0,0.0632249,-0.01259492,0,0,0.08957084,0.04209667,-0.01259492,-0.01259492,0,0,-0.05327294,0,0,0,-0.003818762,0.0314443,0.06287891,0.08533607,0.05880398]},{"f":[2,0,21,-1,-1,15,23,22,15,-1,25,-1,-1,-1,-1,-1,19,21,-1,2,22,20,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,6.5,6.5,0,0,9.92481250360578,3.5,14.158358850608169,6.0,0,0.5,0,0,0,0,0,0.5,2.5,0,8.9274737487839,16.725912049439096,0.5,0,0,0,0,1.5,0,0],"m":[1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,11,0,0,0,0,0,17,18,0,20,21,22,0,0,0,0,27,0,0],"r":[16,5,4,0,0,15,14,13,10,0,12,0,0,0,0,0,26,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,-0.04657328,-0.01484132,0,0,0,0,0.002813312,0,-0.01274856,0.05220433,0.06986092,-0.008226951,-0.01274856,0,0,-0.05292087,0,0,0,-0.007739295,0.03141063,0.05582555,0.05229711,0,0.01389942,0.06341743]},{"f":[2,0,21,-1,19,-1,-1,15,23,22,-1,-1,-1,-1,32,21,20,2,-1,-1,-1,9,2,18,-1,-1,-1,-1,-1],"t":[5.228690439536267,6.5,6.5,0,0.5,0,0,9.92481250360578,3.5,14.158358850608169,0,0,0,0,6.474913355379556,5.5,1.5,8.612669628062957,0,0,0,0.5,9.138550601137563,8.03615005796417,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,0,0,0,0,1,0,1,1,0,0,0,1,1,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,0,0,0,15,16,17,18,0,0,0,22,23,24,0,0,0,0,0],"r":[14,7,4,0,6,0,0,13,12,11,0,0,0,0,28,21,20,19,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,-0.04611315,0,-0.01283797,-0.04271988,0,0,0,0.02233582,0.06681988,-0.007861901,-0.01283797,0,0,0,0,-0.03767678,0.0198463,0.04844246,0,0,0,0.03390737,-0.008316357,0.06128327,-0.05148846,0.06797863]},{"f":[2,37,21,-1,1,-1,15,-1,-1,22,25,-1,-1,-1,22,2,24,15,5,-1,23,-1,-1,-1,-1,-1,1,-1,-1],"t":[5.228690439536267,7.05,2.5,0,4.208333333333334,0,9.92481250360578,0,0,12.613886328427084,0.5,0,0,0,13.87915215758784,10.510264838663748,8.5,10.457380879072534,0.5,0,2.5,0,0,0,0,0,0.6824793747286149,0,0],"m":[1,1,0,0,1,0,1,0,0,0,0,0,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,0,0,0,15,16,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[14,9,4,0,6,0,8,0,0,13,12,0,0,0,26,25,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,-0.05276047,0,-0.03525455,0,-0.001089294,-0.03602129,0,0,-0.001089294,0.04007175,0.04734464,0,0,0,0,0,-0.04191718,0,0.02412223,-0.03027988,-0.03890672,0.06477139,0.07493377,0,-0.04498801,0.05252825]},{"f":[2,0,21,-1,-1,15,23,22,-1,-1,37,-1,-1,-1,21,-1,32,9,22,2,1,21,-1,-1,-1,-1,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,9.92481250360578,1.5,14.158358850608169,0,0,9.34,0,0,0,2.5,0,6.474913355379556,0.5,13.64082212194147,10.510264838663748,6.610817307692308,7.5,0,0,0,0,0,0,0],"m":[1,1,1,0,0,1,1,1,0,0,1,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,11,0,0,0,15,0,17,18,19,20,21,22,0,0,0,0,0,0,0],"r":[14,5,4,0,0,13,10,9,0,0,12,0,0,0,16,0,28,27,26,25,24,23,0,0,0,0,0,0,0],"v":[0,0,0,-0.04556921,-0.01538865,0,0,0,0.0223368,0.07318662,0,-0.01451039,0.07066788,-0.01451039,0,-0.04387378,0,0,0,0,0,0,-0.01631393,0.01887548,0.0361375,0.07320277,0.04513764,-0.05214398,0.06655788]},{"f":[2,0,21,-1,-1,15,19,37,15,-1,-1,-1,-1,-1,21,-1,32,2,1,-1,16,-1,22,-1,-1,3,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,9.92481250360578,2.5,8.05,6.044394119358453,0,0,0,0,0,2.5,0,6.474913355379556,8.9274737487839,0.8611111111111112,0,3.584962500721156,0,13.87915215758784,0,0,0.9640400250156348,0,0,0],"m":[1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,0,0,0,0,15,0,17,18,19,0,21,0,23,0,0,26,0,0,0],"r":[14,5,4,0,0,13,12,11,10,0,0,0,0,0,16,0,28,25,20,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0,-0.04511189,-0.01481516,0,0,0,0,-0.01453377,0.03593709,0.06755506,-0.01453377,-0.01453377,0,-0.0432549,0,0,0,-0.05111178,0,-0.02900328,0,0.01070052,0.04107791,0,0.07094377,0.02969833,0.06400766]},{"f":[2,0,21,-1,-1,23,22,-1,-1,17,-1,-1,21,-1,32,2,1,-1,16,-1,22,3,-1,-1,-1,3,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,1.5,15.212004812197344,0,0,7.6181566135214025,0,0,2.5,0,6.474913355379556,8.9274737487839,0.8611111111111112,0,3.584962500721156,0,13.87915215758784,0.023705703211517166,0,0,0,0.9640400250156348,0,0,0],"m":[1,1,1,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,0,13,0,15,16,17,0,19,0,21,22,0,0,0,26,0,0,0],"r":[12,5,4,0,0,9,8,0,0,11,0,0,14,0,28,25,18,0,20,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,-0.04465106,-0.01454237,0,0,0.01293482,0.06945275,0,-0.01454237,0.05480059,0,-0.04263479,0,0,0,-0.05055317,0,-0.02806512,0,0,-0.05852134,0.01383713,0.03908987,0,0.06799596,0.02814566,0.06164277]},{"f":[2,0,21,-1,-1,15,23,22,-1,-1,37,-1,-1,-1,21,-1,32,2,1,-1,16,-1,22,1,-1,-1,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,8.618385502258606,1.5,14.158358850608169,0,0,9.34,0,0,0,2.5,0,6.474913355379556,8.9274737487839,0.8611111111111112,0,3.584962500721156,0,13.87915215758784,6.610817307692308,0,0,0,0,0],"m":[1,1,1,0,0,1,1,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,11,0,0,0,15,0,17,18,19,0,21,0,23,24,0,0,0,0,0],"r":[14,5,4,0,0,13,10,9,0,0,12,0,0,0,16,0,28,27,20,0,22,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.04418606,-0.01449264,0,0,0,0.02158059,0.06753908,0,-0.01449264,0.06384168,-0.01449264,0,-0.042013,0,0,0,-0.04999921,0,-0.02714755,0,0,-0.0003798591,0.03384598,0.03721916,0.04321895,0.05943802]},{"f":[2,0,21,-1,-1,15,-1,-1,21,-1,1,21,-1,-1,22,33,21,22,-1,22,-1,-1,15,23,-1,-1,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,9.92481250360578,0,0,2.5,0,0.6824793747286149,7.5,0,0,13.64082212194147,34.58978958281367,4.5,12.199665042528586,0,12.637102544927284,0,0,11.696967526234287,2.5,0,0,0,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,0,11,12,0,0,15,16,17,18,0,20,0,0,23,24,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,10,0,14,13,0,0,28,27,22,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.04371632,-0.01440975,0,0.01971736,-0.01440975,0,-0.04138908,0,0,-0.04938147,0.04136651,0,0,0,0,-0.03738437,0,0.03851791,-0.04357449,0,0,0.02984819,-0.01322046,-0.03759537,0.07082867,0.04363319]},{"f":[2,0,21,-1,-1,23,22,-1,-1,17,-1,-1,21,-1,32,2,1,-1,16,-1,22,-1,2,-1,-1,3,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,1.5,15.212004812197344,0,0,7.6181566135214025,0,0,2.5,0,6.474913355379556,8.9274737487839,0.8611111111111112,0,3.584962500721156,0,13.87915215758784,0,6.139515068224824,0,0,0.9640400250156348,0,0,0],"m":[1,1,1,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,0,13,0,15,16,17,0,19,0,21,0,23,0,0,26,0,0,0],"r":[12,5,4,0,0,9,8,0,0,11,0,0,14,0,28,25,18,0,20,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0,-0.04324129,-0.0143338,0,0,0.01096677,0.06527487,0,-0.0143338,0.05075265,0,-0.0407627,0,0,0,-0.04925367,0,-0.0264701,0,0.008459378,0,0.05214475,0.01718813,0,0.06282177,0.0233736,0.05593669]},{"f":[21,-1,2,0,16,-1,23,-1,-1,-1,1,2,-1,-1,22,1,21,-1,2,-1,-1,-1,18,33,-1,-1,15,-1,-1],"t":[2.5,0,2.453445297804259,8.5,2.0,0,0.5,0,0,0,1.000936329588015,6.8391900325545585,0,0,12.121442107629743,12.028571428571428,4.5,0,9.299206200930918,0,0,0,7.252657678068118,19.167172160526107,0,0,2.0,0,0],"m":[0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,0,0,15,16,17,0,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,10,9,6,0,8,0,0,0,14,13,0,0,22,21,18,0,20,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.04920136,0,0,0,-0.04326393,0,-0.002404876,-0.02615403,0.0210241,0,0,-0.04093877,0.02401701,0,0,0,-0.04815723,0,0.007642388,0.0671335,0.05389912,0,0,0.02022347,0.04703901,0,0.06250508,-0.01189632]},{"f":[21,-1,2,0,16,-1,-1,-1,1,21,-1,-1,32,2,16,-1,18,3,22,-1,-1,16,-1,-1,-1,3,-1,-1,-1],"t":[2.5,0,2.453445297804259,8.5,2.0,0,0,0,0.9444444444444444,7.5,0,0,6.474913355379556,8.9274737487839,3.584962500721156,0,7.252657678068118,0.40731707317073174,16.725912049439096,0,0,6.930737337562887,0,0,0,0.9640400250156348,0,0,0],"m":[0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,10,0,0,13,14,15,0,17,18,19,0,0,22,0,0,0,26,0,0,0],"r":[2,0,8,7,6,0,0,0,12,11,0,0,28,25,16,0,24,21,20,0,0,23,0,0,0,27,0,0,0],"v":[0,-0.04889217,0,0,0,-0.04268026,-0.01582361,0.01988705,0,0,-0.04756305,0.03886991,0,0,0,-0.02312053,0,0,0,-0.01464019,0.03094376,0,0.01424678,0.04244139,-0.009668511,0,0.0699555,0.01919585,0.04943752]},{"f":[21,-1,2,1,16,-1,-1,15,23,22,-1,1,-1,-1,17,-1,-1,-1,32,2,1,-1,22,15,-1,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,9.92481250360578,0,0,9.92481250360578,3.5,14.03140051979143,0,5.015625,0,0,7.723832212141186,0,0,0,6.474913355379556,10.510264838663748,0.8611111111111112,0,13.64082212194147,10.457380879072534,0,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,0,12,0,0,15,0,0,0,19,20,21,0,23,24,0,0,0,0,0],"r":[2,0,18,7,6,0,0,17,14,11,0,13,0,0,16,0,0,0,28,27,22,0,26,25,0,0,0,0,0],"v":[0,-0.04858154,0,0,0,-0.03907538,-0.004557052,0,0,0,0.01218236,0,0.2859055,0.03979016,0,-0.03248448,0.05093697,-0.02404769,0,0,0,-0.0417707,0,0,0.01107016,-0.04122675,0.03098393,0.06204166,0.05156832]},{"f":[21,-1,2,0,16,-1,-1,-1,1,21,-1,-1,32,2,16,-1,18,3,22,-1,-1,16,-1,-1,-1,3,-1,-1,-1],"t":[2.5,0,2.453445297804259,8.5,2.0,0,0,0,0.9444444444444444,7.5,0,0,6.474913355379556,8.9274737487839,3.584962500721156,0,7.252657678068118,0.40731707317073174,16.725912049439096,0,0,6.930737337562887,0,0,0,0.9640400250156348,0,0,0],"m":[0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,10,0,0,13,14,15,0,17,18,19,0,0,22,0,0,0,26,0,0,0],"r":[2,0,8,7,6,0,0,0,12,11,0,0,28,25,16,0,24,21,20,0,0,23,0,0,0,27,0,0,0],"v":[0,-0.04826885,0,0,0,-0.04166863,-0.0148518,0.01849157,0,0,-0.04651086,0.03662241,0,0,0,-0.02245313,0,0,0,-0.01475288,0.02819657,0,0.01276393,0.03942293,-0.009015289,0,0.06700576,0.01746681,0.0462802]},{"f":[21,-1,2,1,-1,15,23,2,4,22,-1,-1,-1,-1,17,-1,-1,-1,32,2,1,-1,22,15,-1,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,0,9.92481250360578,3.5,3.39067985676233,2.453445297804259,14.03140051979143,0,0,0,0,7.723832212141186,0,0,0,6.474913355379556,10.510264838663748,0.8611111111111112,0,13.64082212194147,10.457380879072534,0,0,0,0,0],"m":[0,0,1,1,0,1,1,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,0,0,0,15,0,0,0,19,20,21,0,23,24,0,0,0,0,0],"r":[2,0,18,5,0,17,14,13,12,11,0,0,0,0,16,0,0,0,28,27,22,0,26,25,0,0,0,0,0],"v":[0,-0.04795351,0,0,-0.03081808,0,0,0,0,0,0.004400261,0.04639005,0.06592963,-0.05208316,0,-0.03149088,0.04742329,-0.02308424,0,0,0,-0.04014378,0,0,0.01014232,-0.04010457,0.02833,0.05930186,0.04859849]},{"f":[21,-1,2,0,-1,-1,1,-1,33,2,16,18,-1,15,-1,-1,18,3,-1,-1,-1,3,-1,-1,25,22,-1,-1,-1],"t":[2.5,0,2.453445297804259,8.5,0,0,0.9444444444444444,0,20.752541988539424,8.9274737487839,7.118941072723508,3.6427011094311244,0,5.78135971352466,0,0,8.03615005796417,0.33771929824561403,0,0,0,0.9640400250156348,0,0,0.5,12.121442107629743,0,0,0],"m":[0,0,1,1,0,0,0,0,1,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,11,12,0,14,0,0,17,18,0,0,0,22,0,0,25,26,0,0,0],"r":[2,0,6,5,0,0,8,0,24,21,16,13,0,15,0,0,20,19,0,0,0,23,0,0,28,27,0,0,0],"v":[0,-0.04763494,0,0,-0.01909251,0.01722565,0,-0.03247331,0,0,0,0,-0.02932899,0,0.02323193,-0.03170618,0,0,0.0005019516,0.03863599,-0.01375857,0,0.06437264,0.01589839,0,0,-0.03414619,0.03560709,0.05658904]},{"f":[21,-1,2,1,16,-1,-1,15,23,2,4,22,-1,-1,-1,-1,17,-1,-1,-1,32,2,1,-1,2,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,9.92481250360578,0,0,9.92481250360578,3.5,3.39067985676233,2.453445297804259,14.03140051979143,0,0,0,0,7.723832212141186,0,0,0,6.337096134072842,8.9274737487839,0.8611111111111112,0,7.754129516000594,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,11,12,0,0,0,0,17,0,0,0,21,22,23,0,25,0,0,0,0],"r":[2,0,20,7,6,0,0,19,16,15,14,13,0,0,0,0,18,0,0,0,28,27,24,0,26,0,0,0,0],"v":[0,-0.04731258,0,0,0,-0.03738272,-0.001975644,0,0,0,0,0,0.004700065,0.04450977,0.06103039,-0.05044497,0,-0.03050973,0.04511656,-0.0223045,0,0,0,-0.04683938,0,0.01234415,-0.04777094,0.03164205,0.04435139]},{"f":[21,-1,2,1,-1,15,2,19,-1,-1,-1,-1,1,-1,33,2,16,18,-1,-1,18,3,-1,-1,-1,3,-1,-1,-1],"t":[2.5,0,2.453445297804259,4.54375,0,9.92481250360578,1.792481250360578,2.5,0,0,0,0,0.9444444444444444,0,20.752541988539424,8.9274737487839,7.118941072723508,3.6427011094311244,0,0,8.03615005796417,0.33771929824561403,0,0,0,0.9640400250156348,0,0,0],"m":[0,0,1,1,0,1,1,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,0,0,13,0,15,16,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[2,0,12,5,0,11,10,9,0,0,0,0,14,0,28,25,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,-0.04698592,0,0,-0.02924094,0,0,0,0.02662478,-0.03210394,-0.0240471,-0.02950968,0,-0.0313425,0,0,0,0,-0.02842189,0.006977665,0,0,-0.0001296144,0.03595058,-0.01357567,0,0.06185342,0.01354981,0.03848262]},{"f":[21,-1,2,1,16,-1,21,-1,-1,15,23,2,4,22,-1,-1,-1,-1,-1,-1,32,2,15,22,-1,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,9.92481250360578,0,7.5,0,0,9.92481250360578,3.5,3.39067985676233,2.453445297804259,14.03140051979143,0,0,0,0,0,0,6.474913355379556,10.510264838663748,9.92481250360578,13.64082212194147,0,0,0,0,0],"m":[0,0,1,1,1,0,1,0,0,1,1,1,1,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,11,12,13,14,0,0,0,0,0,0,21,22,23,24,0,0,0,0,0],"r":[2,0,20,9,6,0,8,0,0,19,18,17,16,15,0,0,0,0,0,0,28,27,26,25,0,0,0,0,0],"v":[0,-0.04665443,0,0,0,-0.03634106,0,-0.02485282,0.05085381,0,0,0,0,0,0.003855855,0.04203111,0.05666671,-0.04962,-0.01546787,-0.02106922,0,0,0,0,0.006719832,0.03069862,-0.01349788,0.05591427,0.04342491]},{"f":[21,-1,2,1,-1,15,23,2,4,22,-1,25,-1,-1,-1,-1,17,-1,-1,-1,32,2,15,22,-1,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,0,9.92481250360578,3.5,3.39067985676233,2.453445297804259,14.03140051979143,0,0.5,0,0,0,0,7.723832212141186,0,0,0,6.474913355379556,10.510264838663748,10.043027283594547,13.64082212194147,0,0,0,0,0],"m":[0,0,1,1,0,1,1,1,1,1,0,1,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,0,0,17,0,0,0,21,22,23,24,0,0,0,0,0],"r":[2,0,20,5,0,19,16,15,14,11,0,13,0,0,0,0,18,0,0,0,28,27,26,25,0,0,0,0,0],"v":[0,-0.04631764,0,0,-0.02777808,0,0,0,0,0,0.003655738,0,0.07466978,0.01109733,0.0538774,-0.04845789,0,-0.02898236,0.04262988,-0.0203451,0,0,0,0,0.006355353,0.02938993,-0.01301426,0.05445227,0.04209795]},{"f":[21,-1,2,37,16,-1,23,-1,-1,-1,1,21,-1,-1,37,22,21,32,-1,17,-1,-1,23,-1,-1,18,-1,-1,-1],"t":[2.5,0,2.453445297804259,8.05,2.0,0,0.5,0,0,0,0.9444444444444444,7.5,0,0,9.075,13.64082212194147,4.5,4.167695177346962,0,11.207174616271544,0,0,3.5,0,0,7.252657678068118,0,0,0],"m":[0,0,1,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,1,0,0,1,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,0,0,15,16,17,18,0,20,0,0,23,0,0,26,0,0,0],"r":[2,0,10,9,6,0,8,0,0,0,14,13,0,0,28,25,22,19,0,21,0,0,24,0,0,27,0,0,0],"v":[0,-0.04597508,0,0,0,-0.03876072,0,0.002182574,-0.02193025,0.02006405,0,0,-0.04453599,0.03415506,0,0,0,0,0.02791156,0,-0.0378967,0.03107904,0,0.01510551,-0.02715591,0,0.02842011,-0.02093516,0.0495966]},{"f":[21,-1,2,1,21,-1,-1,15,2,19,37,-1,-1,-1,-1,-1,1,-1,33,22,21,16,-1,-1,-1,18,-1,-1,-1],"t":[2.5,0,2.453445297804259,4.54375,6.5,0,0,9.92481250360578,1.792481250360578,2.5,8.05,0,0,0,0,0,0.9444444444444444,0,34.58978958281367,13.415167820124083,4.5,5.78135971352466,0,0,0,7.519373778108656,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,1,0,0,0,0,0,0,0,1,1,0,1,0,0,0,1,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,11,0,0,0,0,0,17,0,19,20,21,22,0,0,0,26,0,0,0],"r":[2,0,16,7,6,0,0,15,14,13,12,0,0,0,0,0,18,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,-0.0456263,0,0,0,-0.04352707,-0.007658946,0,0,0,0,0.0144468,0.05367531,-0.03055248,-0.02329862,-0.02716471,0,-0.03018577,0,0,0,0,0.005269386,-0.03518827,0.01191654,0,0.02718519,-0.01751452,0.0539777]},{"f":[21,-1,2,1,16,4,-1,-1,21,-1,-1,15,16,16,-1,-1,16,22,-1,15,-1,-1,23,-1,-1,-1,33,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,9.92481250360578,4.0437314206251695,0,0,7.5,0,0,9.92481250360578,6.044394119358453,5.044394119358453,0,0,7.118941072723508,12.121442107629743,0,6.584962500721156,0,0,3.5,0,0,0,29.297756302860655,0,0],"m":[0,0,1,1,1,1,0,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,6,0,0,9,0,0,12,13,14,0,0,17,18,0,20,0,0,23,0,0,0,27,0,0],"r":[2,0,26,11,8,7,0,0,10,0,0,25,16,15,0,0,22,19,0,21,0,0,24,0,0,0,28,0,0],"v":[0,-0.04527088,0,0,0,0,-0.03795573,0.05443006,0,-0.02240816,0.05033973,0,0,0,0.01165059,-0.05502919,0,0,-0.03418476,0,0.05688752,0.1828984,0,0.01769152,-0.05311149,-0.01854299,0,0.01127519,0.05012653]},{"f":[21,-1,37,2,16,-1,23,32,-1,-1,21,-1,17,-1,-1,1,2,-1,-1,15,-1,21,-1,-1,21,4,-1,-1,-1],"t":[2.5,0,8.05,3.084962500721156,2.0,0,0.5,7.014643613484123,0,0,6.5,0,1.792481250360578,0,0,0.6824793747286149,6.8391900325545585,0,0,10.376125389276174,0,4.5,0,0,9.5,1.292481250360578,0,0,0],"m":[0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,0,11,0,13,0,0,16,17,0,0,20,0,22,0,0,25,26,0,0,0],"r":[2,0,24,15,6,0,10,9,0,0,12,0,14,0,0,19,18,0,0,21,0,23,0,0,28,27,0,0,0],"v":[0,-0.04490843,0,0,0,-0.03737126,0,0,0.007712535,-0.04573087,0,-0.03647454,0,0.01384044,-0.0199533,0,0,-0.05361513,0.01188741,0,0.01384044,0,-0.05306828,0.004451765,0,0,0.01384044,0.0348549,0.08400629]},{"f":[21,-1,2,22,1,16,-1,23,15,22,-1,-1,-1,-1,25,-1,-1,-1,1,-1,33,22,21,-1,-1,18,-1,-1,-1],"t":[2.5,0,2.453445297804259,17.094543568094064,8.375,2.0,0,0.5,10.501837184902296,14.03140051979143,0,0,0,0,2.5,0,0,0,0.9444444444444444,0,34.58978958281367,13.415167820124083,4.5,0,0,7.519373778108656,0,0,0],"m":[0,0,1,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,0,0,0,0,15,0,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[2,0,18,17,14,7,0,13,12,11,0,0,0,0,16,0,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,-0.04453855,0,0,0,0,-0.04285045,0,0,0,-0.0124953,0.05135107,0.08373111,-0.02359198,0,0.02722831,-0.02621211,0.05897048,0,-0.02875853,0,0,0,-0.01558627,0.009967963,0,0.02480846,-0.01703299,0.05086243]},{"f":[21,-1,2,1,21,-1,-1,15,16,16,-1,-1,16,22,-1,15,-1,-1,23,-1,-1,-1,32,2,15,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,6.5,0,0,9.92481250360578,6.044394119358453,5.044394119358453,0,0,7.118941072723508,12.121442107629743,0,6.584962500721156,0,0,3.5,0,0,0,6.474913355379556,10.510264838663748,9.92481250360578,0,0,0,0],"m":[0,0,1,1,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,0,0,13,14,0,16,0,0,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,7,6,0,0,21,12,11,0,0,18,15,0,17,0,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.0441609,0,0,0,-0.03835686,-0.007375393,0,0,0,0.01066815,-0.05425878,0,0,-0.0329907,0,0.05275929,0.1615261,0,0.01679704,-0.05269746,-0.01720838,0,0,0,0.0108168,-0.01361279,0.05181233,0.03676989]},{"f":[21,-1,37,4,16,0,16,-1,-1,23,-1,22,-1,-1,-1,1,21,17,-1,-1,-1,-1,21,4,19,-1,-1,-1,-1],"t":[2.5,0,8.05,3.084962500721156,10.501837184902296,4.5,5.169925001442312,0,0,0.5,0,17.094543568094064,0,0,0,2.01685992705101,7.5,10.269270673170688,0,0,0,0,9.5,1.292481250360578,0.5,0,0,0,0],"m":[0,0,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,12,0,0,0,16,17,18,0,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,15,14,9,8,0,0,11,0,13,0,0,0,21,20,19,0,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.04377516,0,0,0,0,0,-0.02059419,-0.0431916,0,0.00839848,0,-0.02059419,0.01230817,0.01230817,0,0,0,-0.03625751,0.01230817,0.01060766,0.01230817,0,0,0,0.05215809,0.01230817,0.03183008,0.07773629]},{"f":[21,-1,2,16,-1,16,2,-1,-1,15,-1,16,15,-1,-1,18,21,-1,-1,-1,37,1,-1,22,21,-1,-1,-1,-1],"t":[2.5,0,2.453445297804259,2.0,0,5.169925001442312,1.792481250360578,0,0,6.108524456778169,0,6.832890014164741,6.569855608330948,0,0,2.160964047443681,7.5,0,0,0,9.075,0.9444444444444444,0,12.199665042528586,4.5,0,0,0,0],"m":[0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,0,0,10,0,12,13,0,0,16,17,0,0,0,21,22,0,24,25,0,0,0,0],"r":[2,0,20,5,0,9,8,0,0,11,0,15,14,0,0,19,18,0,0,0,28,23,0,27,26,0,0,0,0],"v":[0,-0.043381,0,0,-0.0324052,0,0,0.03908272,-0.03865623,0,-0.05145791,0,0,0.01360851,0.09799413,0,0,-0.004173689,0.06160731,-0.02526535,0,0,-0.02741526,0,0,-0.03178637,0.006311698,0.01654803,0.04187049]},{"f":[21,-1,37,2,16,-1,23,32,-1,-1,21,-1,17,-1,17,-1,-1,15,-1,-1,21,4,19,-1,-1,24,-1,-1,-1],"t":[2.5,0,8.05,3.084962500721156,2.0,0,0.5,7.014643613484123,0,0,6.5,0,1.792481250360578,0,8.496347487943302,0,0,8.618385502258606,0,0,9.5,1.292481250360578,0.5,0,0,9.5,0,0,0],"m":[0,0,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,0,11,0,13,0,15,0,0,18,0,0,21,22,23,0,0,26,0,0,0],"r":[2,0,20,17,6,0,10,9,0,0,12,0,14,0,16,0,0,19,0,0,28,25,24,0,0,27,0,0,0],"v":[0,-0.04297815,0,0,0,-0.03491544,0,0,0.007463702,-0.04469251,0,-0.03391638,0,0.01141863,0,-0.039468,0.01141863,0,0.01078732,-0.009093951,0,0,0,0.05048926,0.01141863,0,0.01981933,0.05550001,0.07445053]},{"f":[21,-1,2,37,21,15,15,-1,-1,-1,-1,37,22,-1,16,-1,-1,21,4,-1,-1,-1,3,18,-1,-1,17,-1,-1],"t":[2.5,0,8.9274737487839,4.470000000000001,6.5,5.087462841250339,3.169925001442312,0,0,0,0,8.05,12.199665042528586,0,6.044394119358453,0,0,9.5,1.292481250360578,0,0,0,0.9640400250156348,9.011202871982467,0,0,10.460454442916898,0,0],"m":[0,0,1,1,1,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,0,0,0,12,13,0,15,0,0,18,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,22,11,10,9,8,0,0,0,0,17,14,0,16,0,0,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.04256635,0,0,0,0,0,-0.05131789,-0.004097161,-0.04841785,-0.004097161,0,0,-0.004097161,0,-0.004097161,0.01167757,0,0,0.01167757,0.02786755,0.07381561,0,0,0.05948354,-0.01135936,0,-0.001675911,0.03707276]},{"f":[21,-1,4,37,21,-1,20,-1,-1,18,16,-1,21,-1,-1,-1,1,22,-1,-1,1,-1,34,22,21,-1,-1,-1,-1],"t":[2.5,0,3.084962500721156,6.890000000000001,6.5,0,0.5,0,0,2.160964047443681,2.807354922057604,0,3.5,0,0,0,2.01685992705101,12.457649623491124,0,0,2.969223484848485,0,3.5,12.121442107629743,5.5,0,0,0,0],"m":[0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,11,0,13,0,0,0,17,18,0,0,21,0,23,24,25,0,0,0,0],"r":[2,0,16,9,6,0,8,0,0,15,12,0,14,0,0,0,20,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,-0.04214538,0,0,0,-0.0296257,0,-0.005772898,-0.03525213,0,0,-0.005772898,0,-0.005772898,0.03645334,-0.005772898,0,0,0.002012647,-0.03589173,0,0.07415715,0,0,0,-0.02932495,0.00852195,0.01846851,0.05320863]},{"f":[21,-1,2,37,21,-1,-1,16,35,22,-1,-1,-1,22,25,-1,17,-1,-1,15,3,-1,-1,22,-1,-1,3,-1,-1],"t":[2.5,0,8.9274737487839,4.470000000000001,6.5,0,0,6.044394119358453,1.5,12.613886328427084,0,0,0,12.199665042528586,0.5,0,9.26326872279839,0,0,7.118941072723508,0.5803571428571428,0,0,12.738408632106918,0,0,0.9640400250156348,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,0,0,0,14,15,0,17,0,0,20,21,0,0,24,0,0,27,0,0],"r":[2,0,26,7,6,0,0,13,12,11,0,0,0,19,16,0,18,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,-0.04171503,0,0,0,-0.03356613,-0.00385435,0,0,0,-0.00385435,0.03885311,-0.00385435,0,0,-0.00385435,0,0.05082708,0.0047808,0,0,0.005784174,0.06458847,0,0.05119786,0.004693344,0,0.04619199,0.01033541]},{"f":[21,-1,4,16,0,-1,-1,16,22,-1,-1,-1,1,21,-1,-1,1,-1,34,22,21,1,-1,-1,-1,2,-1,-1,-1],"t":[2.5,0,3.084962500721156,11.620219825507487,4.5,0,0,11.868050853745158,11.683871801979876,0,0,0,2.01685992705101,7.5,0,0,2.969223484848485,0,3.5,12.121442107629743,5.5,5.546070460704607,0,0,0,6.483793263648036,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,0,13,14,0,0,17,0,19,20,21,22,0,0,0,26,0,0,0],"r":[2,0,12,7,6,0,0,11,10,0,0,0,16,15,0,0,18,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,-0.04127515,0,0,0,-0.02706231,-0.002621483,0,0,0.1443441,0.03665415,0.000915704,0,0,-0.02502298,0.009438208,0,0.0706893,0,0,0,0,-0.009956547,-0.06767161,0.008048767,0,0.02466861,0.001073646,0.0509581]},{"f":[21,-1,2,37,21,15,-1,-1,-1,16,35,22,-1,-1,-1,22,25,-1,17,-1,-1,15,3,-1,-1,-1,22,-1,-1],"t":[2.5,0,8.9274737487839,4.470000000000001,6.5,5.087462841250339,0,0,0,6.044394119358453,1.5,12.613886328427084,0,0,0,12.199665042528586,0.5,0,9.26326872279839,0,0,7.118941072723508,0.5803571428571428,0,0,0,11.681676585676911,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,11,12,0,0,0,16,17,0,19,0,0,22,23,0,0,0,27,0,0],"r":[2,0,26,9,8,7,0,0,0,15,14,13,0,0,0,21,18,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,-0.04082558,0,0,0,0,-0.01443879,-0.04763503,-0.003833449,0,0,0,-0.003833449,0.03675042,-0.003833449,0,0,-0.003833449,0,0.04852314,0.004139135,0,0,0.005120311,0.06168576,0.008175539,0,0.001021283,0.03529796]},{"f":[21,-1,2,37,16,-1,1,-1,4,4,23,-1,21,-1,-1,-1,15,-1,22,-1,-1,-1,20,2,4,-1,-1,-1,-1],"t":[1.5,0,8.9274737487839,8.05,2.321928094887362,0,0.4794055810397554,0,3.084962500721156,1.792481250360578,0.5,0,6.5,0,0,0,10.457380879072534,0,16.725912049439096,0,0,0,1.5,10.510264838663748,9.266718101290426,0,0,0,0],"m":[0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,1,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,9,10,11,0,13,0,0,0,17,0,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,21,6,0,8,0,16,15,12,0,14,0,0,0,18,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.04993984,0,0,0,-0.0320502,0,-0.05364995,0,0,0,0.006711889,0,-0.03269655,0.0005045657,-0.0288941,0,0.006711889,0,-0.02810975,0.006711889,0.01801191,0,0,0,0.02271667,-0.04546493,0.04741198,0.07439243]},{"f":[21,-1,2,37,21,22,-1,-1,-1,37,22,-1,16,-1,-1,21,33,-1,27,-1,-1,-1,20,2,4,-1,-1,-1,-1],"t":[1.5,0,8.9274737487839,4.470000000000001,6.5,16.819976309430963,0,0,0,8.05,12.199665042528586,0,6.044394119358453,0,0,9.5,13.895530557358477,0,0.5,0,0,0,1.5,10.510264838663748,9.266718101290426,0,0,0,0],"m":[0,0,1,1,1,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,11,0,13,0,0,16,17,0,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,9,8,7,0,0,0,15,12,0,14,0,0,21,18,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.04985466,0,0,0,0,-0.03749304,-0.004439844,-0.004439844,0,0,-0.004439844,0,-0.004439844,0.008773839,0,0,0.05255991,0,0.008773839,0.04488811,0.06799054,0,0,0,0.02162829,-0.0435164,0.04626432,0.07398263]},{"f":[21,-1,2,1,21,-1,22,22,-1,-1,-1,4,37,2,-1,-1,-1,34,22,22,-1,-1,2,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,2.532051282051282,6.5,0,7.082728314749225,6.671801068872866,0,0,0,3.084962500721156,8.05,1.792481250360578,0,0,0,3.5,13.747003318603246,12.637102544927284,0,0,6.483793263648036,0,0,0,1.5,0,0],"m":[0,0,1,0,1,0,0,1,0,0,0,1,1,1,0,0,0,1,1,1,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,0,7,8,0,0,0,12,13,14,0,0,0,18,19,20,0,0,23,0,0,0,27,0,0],"r":[2,0,26,11,6,0,10,9,0,0,0,17,16,15,0,0,0,25,22,21,0,0,24,0,0,0,28,0,0],"v":[0,-0.04976949,0,0,0,-0.04299488,0,0,-0.03097717,0.08727394,-0.007417513,0,0,0,-0.004850413,-0.03354984,0.0128789,0,0,0,0.003078168,-0.04929808,0,0.02815026,-0.007554823,0.04817462,0,0.01526761,0.07361517]},{"f":[21,-1,2,16,26,-1,-1,1,-1,15,22,21,-1,-1,-1,15,-1,1,21,-1,-1,15,16,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,2.584962500721156,0.5,0,0,0.4794055810397554,0,5.78135971352466,16.819976309430963,4.5,0,0,0,6.169925001442312,0,4.92309205350118,7.5,0,0,6.857980995127572,6.6865005271832185,0,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,12,0,0,0,16,0,18,19,0,0,22,23,0,0,0,27,0,0],"r":[2,0,26,7,6,0,0,9,0,15,14,13,0,0,0,17,0,21,20,0,0,25,24,0,0,0,28,0,0],"v":[0,-0.04968413,0,0,0,0.01309162,-0.03546466,0,-0.05299355,0,0,0,-0.006853767,0.02121278,0.1357665,0,-0.04971195,0,0,-0.02951811,0.002683889,0,0,0.02933642,0.1377346,0.005932405,0,0.01454233,0.07328184]},{"f":[21,-1,2,1,-1,16,26,-1,-1,15,22,-1,-1,15,-1,1,-1,18,15,19,-1,-1,2,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,0,0,5.78135971352466,16.819976309430963,0,0,6.169925001442312,0,4.92309205350118,0,8.729619742794132,6.857980995127572,1.5,0,0,1.292481250360578,0,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,0,10,11,0,0,14,0,16,0,18,19,20,0,0,23,0,0,0,27,0,0],"r":[2,0,26,5,0,9,8,0,0,13,12,0,0,15,0,17,0,25,22,21,0,0,24,0,0,0,28,0,0],"v":[0,-0.04959839,0,0,-0.05228592,0,0,0.01262366,-0.03453981,0,0,0.009208582,0.1212337,0,-0.04848066,0,-0.01171547,0,0,0,0.1280567,0.02248596,0,-0.0116267,0.01806142,-0.0412347,0,0.0138511,0.07297461]},{"f":[21,-1,2,1,-1,16,26,2,-1,-1,21,-1,-1,37,4,4,-1,-1,15,-1,22,-1,-1,16,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,1.792481250360578,0,0,8.5,0,0,8.05,3.084962500721156,1.792481250360578,0,0,10.457380879072534,0,16.725912049439096,0,0,5.044394119358453,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,0,0,1,1,1,0,0,1,0,1,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,0,0,14,15,16,0,0,19,0,21,0,0,24,0,0,27,0,0],"r":[2,0,26,5,0,13,10,9,0,0,12,0,0,23,18,17,0,0,20,0,22,0,0,25,0,0,28,0,0],"v":[0,-0.04951208,0,0,-0.05196165,0,0,0,-0.01487707,0.04028857,0,-0.03830456,0.03716279,0,0,0,-0.002612464,-0.0267502,0,0.008896182,0,-0.02690315,0.008896182,0,0.06076272,0.01241319,0,0.01319227,0.07268553]},{"f":[21,-1,2,1,-1,16,26,2,-1,-1,-1,37,4,4,-1,-1,15,-1,22,-1,-1,-1,20,2,4,-1,-1,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,1.792481250360578,0,0,0,8.05,3.084962500721156,1.792481250360578,0,0,10.457380879072534,0,16.725912049439096,0,0,0,1.5,10.510264838663748,9.266718101290426,0,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,0,1,1,1,0,0,1,0,1,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,0,12,13,14,0,0,17,0,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,5,0,11,10,9,0,0,0,21,16,15,0,0,18,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.04942501,0,0,-0.05163247,0,0,0,-0.01428203,0.03879967,-0.03290268,0,0,0,-0.002486539,-0.02592857,0,0.008544841,0,-0.02580445,0.008544841,0.01684772,0,0,0,0.01788594,-0.04408516,0.0437903,0.07240675]},{"f":[21,-1,2,1,-1,2,22,16,-1,16,22,-1,21,-1,-1,15,-1,22,-1,32,-1,-1,-1,34,-1,-1,1,-1,-1],"t":[1.5,0,9.341519573083566,0.6824793747286149,0,2.453445297804259,17.094543568094064,2.0,0,5.169925001442312,13.747003318603246,0,4.5,0,0,6.108524456778169,0,6.930301023985015,0,6.337096134072842,0,0,0,3.5,0,0,3.015625,0,0],"m":[0,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,0,6,7,8,0,10,11,0,13,0,0,16,0,18,0,20,0,0,0,24,0,0,27,0,0],"r":[2,0,26,5,0,23,22,9,0,15,12,0,14,0,0,17,0,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,-0.04933698,0,0,-0.04004954,0,0,0,-0.03300616,0,0,0.003495727,0,0.1795929,0.04468963,0,-0.04984272,0,0.07056401,0,0.0346409,-0.009048814,0.04844247,0,0.004168946,0.044918,0,0.01347157,0.08675308]},{"f":[21,-1,2,1,-1,2,22,23,21,1,-1,15,-1,-1,15,-1,1,-1,-1,-1,-1,37,22,21,-1,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,2.453445297804259,17.094543568094064,4.5,7.5,4.54375,0,10.457380879072534,0,0,11.683433292835403,0,2.0015923566878984,0,0,0,0,12.52,12.121442107629743,4.5,0,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,1,0,1,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,15,0,17,0,0,0,0,22,23,24,0,0,0,0,0],"r":[2,0,28,5,0,21,20,19,14,11,0,13,0,0,16,0,18,0,0,0,0,27,26,25,0,0,0,0,0],"v":[0,-0.04924782,0,0,-0.04218048,0,0,0,0,0,-0.0311681,0,0.006909913,-0.02777852,0,0.007931393,0,0.1439446,-0.00551275,-0.03277736,0.04510326,0,0,0,-0.03886386,0.002805203,0.0100585,0.05958475,0.04233455]},{"f":[21,-1,2,1,-1,16,26,-1,-1,15,22,-1,-1,15,-1,1,21,-1,-1,18,15,16,-1,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,0,0,5.78135971352466,16.819976309430963,0,0,6.169925001442312,0,4.92309205350118,6.5,0,0,8.729619742794132,6.857980995127572,6.554588851677638,0,0,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,0,10,11,0,0,14,0,16,17,0,0,20,21,22,0,0,0,0,27,0,0],"r":[2,0,26,5,0,9,8,0,0,13,12,0,0,15,0,19,18,0,0,25,24,23,0,0,0,0,28,0,0],"v":[0,-0.04915733,0,0,-0.05075186,0,0,0.0114666,-0.03167116,0,0,0.008024315,0.1027502,0,-0.04759378,0,0,-0.03882388,-0.002491575,0,0,0,0.0197086,0.1134931,0.007650019,-0.04020552,0,0.01129196,0.07135309]},{"f":[21,-1,2,1,-1,37,18,2,2,15,-1,-1,18,-1,-1,-1,17,-1,21,-1,-1,21,33,-1,27,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,8.05,7.519373778108656,3.084962500721156,1.792481250360578,2.0,0,0,4.357122758833062,0,0,0,8.571721503073015,0,5.5,0,0,9.5,13.895530557358477,0,0.5,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,0,13,0,0,0,17,0,19,0,0,22,23,0,25,0,0,0,0],"r":[2,0,28,5,0,21,16,15,12,11,0,0,14,0,0,0,18,0,20,0,0,27,24,0,26,0,0,0,0],"v":[0,-0.04906534,0,0,-0.04091493,0,0,0,0,0,-0.04492389,-5.010009e-05,0,-0.03565973,0.006471874,0.006471874,0,-0.03856837,0,-0.02334742,0.001892959,0,0,0.04753227,0,0.006471874,0.04082209,0.06384489,0.041051]},{"f":[21,-1,2,1,-1,23,22,4,-1,24,21,2,-1,-1,2,-1,-1,-1,15,1,-1,-1,15,-1,20,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,7.5,14.03140051979143,1.292481250360578,0,9.5,5.5,8.612669628062957,0,0,5.40929108874043,0,0,0,5.700439718141092,2.532051282051282,0,0,6.169925001442312,0,0.5,0,0,0,0],"m":[0,0,1,0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,10,11,12,0,0,15,0,0,0,19,20,0,0,23,0,25,0,0,0,0],"r":[2,0,28,5,0,27,18,9,0,17,14,13,0,0,16,0,0,0,22,21,0,0,24,0,26,0,0,0,0],"v":[0,-0.04897165,0,0,-0.04005608,0,0,0,-0.01500493,0,0,0,-0.01449538,0.03366804,0,0.02270486,-0.0008874588,0.04713837,0,0,0.004658667,0.05422136,0,-0.04333924,0,-0.005988206,0.02611942,-0.02902094,0.0400781]},{"f":[21,-1,2,1,-1,18,17,2,37,21,18,-1,22,-1,-1,-1,-1,-1,33,-1,17,22,-1,-1,-1,0,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,7.430398005367313,7.198837316474133,9.034782574103549,6.970000000000001,6.5,1.792481250360578,0,12.199665042528586,0,0,0,0,0,31.279747787828097,0,7.392087364565857,9.998977010081727,0,0,0,10.5,0,0,0],"m":[0,0,1,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,11,0,13,0,0,0,0,0,19,0,21,22,0,0,0,26,0,0,0],"r":[2,0,28,5,0,25,18,17,16,15,12,0,14,0,0,0,0,0,20,0,24,23,0,0,0,27,0,0,0],"v":[0,-0.04887609,0,0,-0.03920008,0,0,0,0,0,0,-0.03412893,0,-0.03893961,-0.001668505,-0.001668505,0.006719302,0.02656833,0,0.00160113,0,0,-0.008771029,0.1731208,0.02888569,0,-0.01185115,0.03325099,0.0391244]},{"f":[21,-1,2,1,-1,23,22,4,23,-1,21,-1,-1,24,-1,-1,15,1,-1,-1,15,-1,20,22,-1,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,7.5,14.03140051979143,1.292481250360578,0.5,0,9.5,0,0,9.5,0,0,5.700439718141092,2.532051282051282,0,0,6.169925001442312,0,0.5,16.844368517741394,0,0,0,0,0],"m":[0,0,1,0,0,1,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,0,11,0,0,14,0,0,17,18,0,0,21,0,23,24,0,0,0,0,0],"r":[2,0,28,5,0,27,16,13,10,0,12,0,0,15,0,0,20,19,0,0,22,0,26,25,0,0,0,0,0],"v":[0,-0.04877848,0,0,-0.03834743,0,0,0,0,-0.0007415783,0,-0.0367286,0.03408687,0,0.0002519168,0.04472044,0,0,0.004238806,0.05138736,0,-0.04189957,0,0,-0.01406468,0.02643565,0.02459002,-0.02811762,0.03818848]},{"f":[21,-1,2,1,-1,16,26,-1,21,-1,-1,16,22,-1,-1,16,4,-1,-1,1,21,-1,22,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,0,8.5,0,0,5.169925001442312,16.819976309430963,0,0,6.108524456778169,4.951187557243013,0,0,4.92309205350118,7.5,0,8.32532955673354,0,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,9,0,0,12,13,0,0,16,17,0,0,20,21,0,23,0,0,0,27,0,0],"r":[2,0,26,5,0,11,8,0,10,0,0,15,14,0,0,19,18,0,0,25,22,0,24,0,0,0,28,0,0],"v":[0,-0.04867862,0,0,-0.04927712,0,0,0.009803803,0,-0.03497796,0.04020465,0,0,0.01038503,0.1138281,0,0,-0.05221024,0.007619351,0,0,-0.02399654,0,0.06939004,0.001728215,0.008708279,0,0.009344453,0.0708777]},{"f":[21,-1,2,1,-1,18,17,16,16,16,-1,22,-1,-1,-1,15,16,-1,-1,-1,33,-1,17,22,-1,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,7.430398005367313,7.198837316474133,6.930737337562887,5.78135971352466,2.584962500721156,0,15.756821747580634,0,0,0,10.457380879072534,7.924812503605781,0,0,0,31.279747787828097,0,7.392087364565857,9.998977010081727,0,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,0,16,17,0,0,0,21,0,23,24,0,0,0,0,0],"r":[2,0,28,5,0,27,20,15,14,11,0,13,0,0,0,19,18,0,0,0,22,0,26,25,0,0,0,0,0],"v":[0,-0.04857634,0,0,-0.03701495,0,0,0,0,0,-0.01451733,0,0.007137679,0.04689465,-0.0300675,0,0,0.001465424,0.02380207,-0.009367041,0,0.001123717,0,0,-0.008267491,0.1491712,0.02719548,-0.008604041,0.03702496]},{"f":[21,-1,4,16,0,21,22,-1,-1,-1,-1,16,22,-1,-1,-1,1,21,-1,-1,1,-1,2,2,-1,22,-1,-1,-1],"t":[1.5,0,3.084962500721156,11.620219825507487,5.5,6.5,14.03140051979143,0,0,0,0,11.868050853745158,11.683871801979876,0,0,0,2.01685992705101,7.5,0,0,2.969223484848485,0,9.299206200930918,3.245926548164837,0,16.819976309430963,0,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,0,12,13,0,0,0,17,18,0,0,21,0,23,24,0,26,0,0,0],"r":[2,0,16,11,10,9,8,0,0,0,0,15,14,0,0,0,20,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,-0.04847146,0,0,0,0,0,-0.04272313,-0.01004036,-0.01004036,-0.001066744,0,0,0.1128854,0.02220971,0.001119293,0,0,-0.02465371,0.008690264,0,0.06157739,0,0,0.03410772,0,-0.000170169,0.02720208,0.05347866]},{"f":[21,-1,22,23,21,1,-1,22,25,22,-1,-1,-1,2,23,-1,27,-1,-1,-1,37,-1,-1,-1,18,32,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,6.5,2.0015923566878984,0,11.775312925643524,0.5,7.25627765006036,0,0,0,9.012591414317754,2.5,0,0.5,0,0,0,9.075,0,0,0,8.729619742794132,8.341935934311302,0,0,0],"m":[0,0,1,1,1,0,0,0,1,1,0,0,0,1,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,0,0,0,14,15,0,17,0,0,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,23,20,7,0,13,12,11,0,0,0,19,16,0,18,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,-0.0483638,0,0,0,0,-0.03197678,0,0,0,-0.001837247,-0.04632334,0.02483216,0,0,0.006511403,0,-0.03851128,0.03396099,0.06753089,0,0.004626959,0.0442567,-0.04067866,0,0,-0.01064013,0.07598303,-0.03523382]},{"f":[21,-1,2,1,-1,16,26,-1,21,-1,-1,15,-1,15,-1,1,-1,22,25,-1,-1,15,-1,2,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.39811320754716983,0,2.321928094887362,0.5,0,8.5,0,0,5.78135971352466,0,6.169925001442312,0,4.92309205350118,0,12.199665042528586,0.5,0,0,7.118941072723508,0,1.292481250360578,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,9,0,0,12,0,14,0,16,0,18,19,0,0,22,0,24,0,0,27,0,0],"r":[2,0,26,5,0,11,8,0,10,0,0,13,0,15,0,17,0,21,20,0,0,23,0,25,0,0,28,0,0],"v":[0,-0.04825318,0,0,-0.0506809,0,0,0.009075243,0,-0.03346517,0.03873662,0,0.008781625,0,-0.04434036,0,-0.01051462,0,0,-0.02129964,0.03662667,0,0.04646605,0,-0.01880435,0.01871891,0,0.008388415,0.06931155]},{"f":[21,-1,2,1,-1,22,23,21,22,22,1,-1,25,-1,-1,-1,0,-1,23,-1,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.4794055810397554,0,17.094543568094064,7.5,6.5,11.775312925643524,10.558733410935197,1.000936329588015,0,0.5,0,0,0,1.5,0,2.5,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,11,0,13,0,0,0,17,0,19,0,0,0,0,24,25,0,0,0,0],"r":[2,0,28,5,0,23,22,21,16,15,12,0,14,0,0,0,18,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,-0.04813942,0,0,-0.03468243,0,0,0,0,0,0,0.122574,0,-0.05685315,-0.004615148,-0.05216004,0,-0.04552942,0,0.007554467,-0.02042955,0.005470228,-0.03986527,0,0,-0.01127363,0.07149181,-0.03502767,0.04864151]},{"f":[21,-1,4,37,21,-1,-1,22,-1,22,24,-1,16,-1,-1,-1,15,16,16,16,-1,-1,-1,-1,22,25,-1,-1,-1],"t":[1.5,0,3.9534452978042594,4.470000000000001,6.5,0,0,12.199665042528586,0,16.73920708776685,7.5,0,6.044394119358453,0,0,0,10.376125389276174,7.266786540694901,5.78135971352466,5.491853096329675,0,0,0,0,16.725912049439096,0.5,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,0,13,0,0,0,17,18,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,16,7,6,0,0,9,0,15,12,0,14,0,0,0,24,23,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04802234,0,0,0,-0.0354209,-0.007442192,0,-0.007442192,0,0,-0.006477223,0,-0.004008228,0.02756081,-0.007442192,0,0,0,0,0.002898538,0.04779405,-0.01691177,0.02251224,0,0,-0.06015028,-0.008365658,0.03199672]},{"f":[21,-1,2,1,-1,27,11,4,2,2,16,-1,-1,-1,-1,15,16,3,-1,-1,-1,22,25,-1,-1,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.4794055810397554,0,0.5,0.5,4.951187557243013,8.612669628062957,6.994342343386083,2.0,0,0,0,0,10.376125389276174,6.882643049361842,0.5803571428571428,0,0,0,16.725912049439096,0.5,0,0,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,11,0,0,0,0,16,17,18,0,0,0,22,23,0,0,0,0,0,0],"r":[2,0,28,5,0,27,26,15,14,13,12,0,0,0,0,21,20,19,0,0,0,25,24,0,0,0,0,0,0],"v":[0,-0.04790176,0,0,-0.03337599,0,0,0,0,0,0,-0.02823457,-0.001001615,-0.0545662,0.05364832,0,0,0,-0.04866518,0.005421228,0.02437115,0,0,-0.05713768,-0.01045506,0.03168308,-0.02858791,0.02654387,0.04792086]},{"f":[21,-1,2,37,15,-1,15,-1,22,16,18,16,-1,17,-1,-1,-1,-1,0,-1,-1,21,33,-1,27,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,8.05,5.78135971352466,0,6.169925001442312,0,12.199665042528586,10.501837184902296,10.37503779525057,7.651051691178929,0,1.292481250360578,0,0,0,0,3.5,0,0,9.5,13.895530557358477,0,0.5,0,0,0,0],"m":[0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,9,10,11,12,0,14,0,0,0,0,19,0,0,22,23,0,25,0,0,0,0],"r":[2,0,28,21,6,0,8,0,18,17,16,13,0,15,0,0,0,0,20,0,0,27,24,0,26,0,0,0,0],"v":[0,-0.04777751,0,0,0,0.004186739,0,-0.03728867,0,0,0,0,-0.0424349,0,0.004825962,-0.01841777,-0.0527865,0.004825962,0,-0.02140017,0.004825962,0,0,0.04403683,0,0.004825962,0.03603353,0.06005426,0.04725857]},{"f":[21,-1,2,22,23,1,-1,37,15,-1,15,-1,22,21,-1,-1,0,-1,-1,15,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,17.094543568094064,7.5,0.4794055810397554,0,9.34,5.78135971352466,0,6.169925001442312,0,12.199665042528586,5.5,0,0,3.5,0,0,6.108524456778169,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,0,11,0,13,14,0,0,17,0,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,7,0,19,10,0,12,0,16,15,0,0,18,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.04764942,0,0,0,0,-0.03228648,0,0,0.006048445,0,-0.03599126,0,0,-0.03099685,-0.003728494,0,-0.01776544,0.007138946,0,0.007305945,0.02808422,-0.03863075,0,0,-0.0110596,0.06802305,-0.03379511,0.03298558]},{"f":[21,-1,2,22,23,1,-1,25,2,32,15,-1,23,21,-1,-1,-1,-1,37,-1,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,17.094543568094064,7.5,0.4794055810397554,0,6.5,2.453445297804259,10.463013322934223,2.0,0,3.5,7.5,0,0,0,0,11.57,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,11,0,13,14,0,0,0,0,19,0,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,7,0,21,18,17,12,0,16,15,0,0,0,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,-0.0475173,0,0,0,0,-0.03146479,0,0,0,0,-0.04430363,0,0,-0.000898385,0.02602013,-0.01751566,-0.0441493,0,0.003163307,0.05615568,-0.05006505,-0.03786854,0,0,-0.01059428,0.064088,-0.03277483,0.04615613]},{"f":[21,-1,2,1,-1,21,-1,2,4,-1,15,16,-1,-1,22,-1,-1,-1,20,3,-1,17,2,-1,-1,22,-1,-1,-1],"t":[1.5,0,8.9274737487839,0.39811320754716983,0,2.5,0,7.754129516000594,4.951187557243013,0,10.113742166049189,7.118941072723508,0,0,16.725912049439096,0,0,0,1.5,0.9640400250156348,0,10.460454442916898,9.266718101290426,0,0,12.457649623491124,0,0,0],"m":[0,0,1,0,0,0,0,1,1,0,1,1,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0],"l":[1,0,3,4,0,6,0,8,9,0,11,12,0,0,15,0,0,0,19,20,0,22,23,0,0,26,0,0,0],"r":[2,0,18,5,0,7,0,17,10,0,14,13,0,0,16,0,0,0,28,21,0,25,24,0,0,27,0,0,0],"v":[0,-0.047381,0,0,-0.0493066,0,-0.03416496,0,0,-0.001896749,0,0,-0.002479757,0.028629,0,-0.0205434,0.03636186,-0.03807373,0,0,0.03004712,0,0,0.006748572,-0.05723914,0,0.04717709,-0.02008235,0.06842665]},{"f":[21,-1,2,27,11,1,15,-1,-1,25,22,4,15,-1,17,-1,-1,24,21,-1,-1,-1,17,-1,-1,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.5,0.5,0.4794055810397554,10.462502272597265,0,0,6.5,12.199665042528586,1.292481250360578,10.501837184902296,0,10.269270673170688,0,0,9.5,4.5,0,0,0,0.5,0,0,0,0,0,0],"m":[0,0,1,1,1,0,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,11,12,13,0,15,0,0,18,19,0,0,0,23,0,0,0,0,0,0],"r":[2,0,28,27,26,9,8,0,0,25,22,17,14,0,16,0,0,21,20,0,0,0,24,0,0,0,0,0,0],"v":[0,-0.04724033,0,0,0,0,0,-0.0513696,0.01210377,0,0,0,0,-0.02390886,0,-0.02567579,0.1044394,0,0,-0.02750984,0.003003729,0.04093375,0,-0.03689284,0.006436161,-0.04921654,-0.02667362,0.0242645,0.04541875]},{"f":[21,-1,2,21,-1,25,18,17,-1,33,33,-1,-1,2,-1,-1,18,18,17,23,-1,-1,-1,-1,-1,-1,1,-1,-1],"t":[1.5,0,9.341519573083566,2.5,0,6.5,7.383290334558488,7.198837316474133,0,31.279747787828097,10.073602462471115,0,0,3.39067985676233,0,0,10.620215166896223,9.273793716589307,8.571721503073015,2.5,0,0,0,0,0,0,3.015625,0,0],"m":[0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,8,0,10,11,0,0,14,0,0,17,18,19,20,0,0,0,0,0,0,27,0,0],"r":[2,0,26,5,0,25,16,9,0,13,12,0,0,15,0,0,24,23,22,21,0,0,0,0,0,0,28,0,0],"v":[0,-0.04709515,0,0,-0.03360134,0,0,0,-0.0002862654,0,0,0.04170918,-0.004472949,0,0.1089442,0.02745428,0,0,0,0,0.01415397,-0.0431679,0.01125457,-0.03154277,0.01080276,-0.05689081,0,0.002429439,0.08094656]},{"f":[21,-1,2,37,32,16,17,22,4,-1,-1,-1,-1,15,-1,15,-1,1,-1,-1,-1,21,33,-1,27,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,8.05,10.41931383093672,2.584962500721156,9.135888799148791,17.105450904090162,5.65757478112815,0,0,0,0,5.78135971352466,0,6.169925001442312,0,4.643065068493151,0,0,0,9.5,13.895530557358477,0,0.5,0,0,0,0],"m":[0,0,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,0,0,0,14,0,16,0,18,0,0,0,22,23,0,25,0,0,0,0],"r":[2,0,28,21,20,13,12,11,10,0,0,0,0,15,0,17,0,19,0,0,0,27,24,0,26,0,0,0,0],"v":[0,-0.04694528,0,0,0,0,0,0,0,-0.03485982,-0.006402535,0.004194797,0.004194797,0,0.004194797,0,-0.03419248,0,-0.008795806,0.004194797,-0.03366648,0,0,0.04317016,0,0.004194797,0.03352374,0.05855175,0.04446806]},{"f":[21,-1,22,23,2,1,-1,21,-1,25,-1,-1,20,3,17,-1,-1,2,-1,18,-1,-1,-1,-1,18,32,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,9.051177541662383,0.39811320754716983,0,2.5,0,6.5,0,0,1.5,0.9640400250156348,9.44286754736589,0,0,9.266718101290426,0,10.457893746154532,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,10,0,0,13,14,15,0,0,18,0,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,12,7,0,9,0,11,0,0,22,17,16,0,0,19,0,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04679056,0,0,0,0,-0.04889546,0,-0.03368229,0,0.0008906734,-0.05598086,0,0,0,0.04895621,-0.01033175,0,0.01272672,0,-0.05821599,0.02459681,0.07352587,-0.03598477,0,0,-0.01069441,0.06109605,-0.03230747]},{"f":[21,-1,2,33,15,22,4,21,-1,-1,-1,1,-1,21,-1,-1,15,-1,1,21,-1,-1,-1,25,36,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,29.297756302860655,5.78135971352466,14.03140051979143,3.8571227588330617,3.5,0,0,0,2.532051282051282,0,3.5,0,0,6.169925001442312,0,4.92309205350118,6.5,0,0,0,0.5,3.5,0,0,0,0],"m":[0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,0,12,0,14,0,0,17,0,19,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,16,11,10,9,0,0,0,13,0,15,0,0,18,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.04663085,0,0,0,0,0,0,-0.04727671,-0.004257542,0.008204,0,0.002253581,0,0.1308596,0.03311993,0,-0.03873337,0,0,-0.03635847,-0.001119192,0.004309659,0,0,-0.04994089,0.01296594,0.03135386,0.04377937]},{"f":[21,-1,2,1,-1,21,-1,25,4,4,32,15,-1,-1,-1,-1,-1,-1,20,3,17,-1,-1,2,-1,18,-1,-1,-1],"t":[1.5,0,9.051177541662383,0.39811320754716983,0,2.5,0,6.5,3.084962500721156,2.69615871138938,10.463013322934223,2.0,0,0,0,0,0,0,1.5,0.9640400250156348,9.44286754736589,0,0,9.266718101290426,0,10.457893746154532,0,0,0],"m":[0,0,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,9,10,11,12,0,0,0,0,0,0,19,20,21,0,0,24,0,26,0,0,0],"r":[2,0,18,5,0,7,0,17,16,15,14,13,0,0,0,0,0,0,28,23,22,0,0,25,0,27,0,0,0],"v":[0,-0.04646598,0,0,-0.04832233,0,-0.03239399,0,0,0,0,0,-0.04189867,0.0002409953,-0.03865575,-0.04846429,0.00399874,-0.05483847,0,0,0,0.04778974,-0.01017131,0,0.01109382,0,-0.05598712,0.02398447,0.06591128]},{"f":[21,-1,2,22,23,37,15,-1,15,-1,22,16,18,-1,-1,-1,0,-1,-1,15,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,17.094543568094064,7.5,9.34,5.78135971352466,0,6.169925001442312,0,12.199665042528586,10.501837184902296,10.37503779525057,0,0,0,3.5,0,0,6.108524456778169,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,0,11,12,13,0,0,0,17,0,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,19,8,0,10,0,16,15,14,0,0,0,18,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.04629583,0,0,0,0,0,0.005002243,0,-0.03335805,0,0,0,-0.01585172,-0.05135809,0.006004001,0,-0.01843826,0.005908812,0,0.006004001,0.02574206,-0.03512231,0,0,-0.01048606,0.0572009,-0.031323,0.04312844]},{"f":[21,-1,2,21,-1,25,18,-1,18,18,17,2,1,-1,-1,-1,-1,-1,23,22,-1,16,-1,-1,-1,-1,1,-1,-1],"t":[1.5,0,9.341519573083566,2.5,0,6.5,7.383290334558488,0,10.620215166896223,9.273793716589307,9.265614570460752,8.049190845056867,4.145833333333334,0,0,0,0,0,0.5,11.228801658478353,0,5.044394119358453,0,0,0,0,3.015625,0,0],"m":[0,0,1,0,0,1,1,0,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,9,10,11,12,13,0,0,0,0,0,19,20,0,22,0,0,0,0,27,0,0],"r":[2,0,26,5,0,25,8,0,18,17,16,15,14,0,0,0,0,0,24,21,0,23,0,0,0,0,28,0,0],"v":[0,-0.04612025,0,0,-0.03201163,0,0,0.002326566,0,0,0,0,0,-0.04225896,-0.004834584,0.02993111,0.03145768,-0.02948948,0,0,-0.007581322,0,0.01726435,0.1520849,-0.001625747,-0.05400262,0,0.001435862,0.07788863]},{"f":[21,-1,2,33,15,22,4,16,-1,-1,20,-1,-1,23,-1,-1,15,-1,22,-1,22,-1,-1,25,36,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,29.297756302860655,5.78135971352466,14.03140051979143,2.453445297804259,5.087462841250339,0,0,1.5,0,0,2.5,0,0,6.169925001442312,0,6.930301023985015,0,12.199665042528586,0,0,0.5,3.5,0,0,0,0],"m":[0,0,1,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,14,0,0,17,0,19,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,23,16,13,10,9,0,0,12,0,0,15,0,0,18,0,20,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.0459391,0,0,0,0,0,0,-0.007404275,-0.0555345,0,0.002213119,0.04520265,0,0.05246906,0.01045252,0,-0.03662119,0,0.05297465,0,-0.01338931,0.002638506,0,0,-0.04867302,0.01171209,0.02950625,0.04222579]},{"f":[21,-1,22,23,2,1,-1,21,-1,25,-1,-1,20,3,17,-1,-1,2,-1,18,-1,-1,-1,-1,18,1,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,9.051177541662383,0.39811320754716983,0,2.5,0,6.5,0,0,1.5,0.9640400250156348,9.44286754736589,0,0,9.266718101290426,0,10.457893746154532,0,0,0,0,8.729619742794132,7.9752100840336135,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,8,0,10,0,0,13,14,15,0,0,18,0,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,12,7,0,9,0,11,0,0,22,17,16,0,0,19,0,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04575228,0,0,0,0,-0.04776658,0,-0.03216827,0,0.0006915878,-0.05293562,0,0,0,0.04619806,-0.01121868,0,0.01107355,0,-0.05392078,0.02276939,0.0704734,-0.03414641,0,0,0.05484038,-0.009619258,-0.0302163]},{"f":[21,-1,2,18,17,26,32,22,16,-1,1,-1,-1,0,-1,-1,-1,-1,33,-1,2,1,-1,-1,-1,-1,17,-1,-1],"t":[1.5,0,10.510264838663748,7.383290334558488,7.198837316474133,1.5,3.9474088816539714,10.27595502635321,10.462502272597265,0,1.0491071428571428,0,0,2.5,0,0,0,0,31.279747787828097,0,3.753897320099348,2.532051282051282,0,0,0,0,1.792481250360578,0,0],"m":[0,0,1,1,1,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,11,0,0,14,0,0,0,0,19,0,21,22,0,0,0,0,27,0,0],"r":[2,0,26,25,18,17,16,13,10,0,12,0,0,15,0,0,0,0,20,0,24,23,0,0,0,0,28,0,0],"v":[0,-0.04555965,0,0,0,0,0,0,0,0.000737844,0,0.08367819,-0.002749327,0,-0.03104963,-0.003448997,0.008426521,-0.04119531,0,-0.002168649,0,0,-0.02578424,0.1123493,0.02434705,-0.006941192,0,-0.01313254,0.05084248]},{"f":[21,-1,22,23,21,1,-1,2,23,22,-1,15,-1,33,-1,16,-1,-1,24,-1,-1,-1,-1,-1,18,1,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,6.5,2.0015923566878984,0,9.025136904799169,2.5,14.03140051979143,0,5.044394119358453,0,19.167172160526107,0,8.618385502258606,0,0,12.5,0,0,0,0,0,8.729619742794132,7.9752100840336135,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,0,12,0,14,0,16,0,0,19,0,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,22,7,0,21,18,11,0,13,0,15,0,17,0,0,20,0,0,0,0,0,28,27,0,0,0],"v":[0,-0.04536112,0,0,0,0,-0.02725008,0,0,0,-0.003150558,0,0.06041647,0,-0.04690705,0,0.03665075,-0.009247334,0,0.01001764,-0.04910839,0.05583604,0.00379833,-0.03336142,0,0,0.05254067,-0.009303683,-0.02908685]},{"f":[21,-1,2,27,16,17,-1,-1,15,22,3,-1,-1,1,-1,21,-1,-1,15,-1,1,-1,15,-1,18,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.5,2.584962500721156,9.070120320271018,0,0,5.78135971352466,14.03140051979143,0.023705703211517166,0,0,2.532051282051282,0,3.5,0,0,6.169925001442312,0,4.92309205350118,0,6.857980995127572,0,8.03615005796417,0,0,0,0],"m":[0,0,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,11,0,0,14,0,16,0,0,19,0,21,0,23,0,25,0,0,0,0],"r":[2,0,28,27,8,7,0,0,18,13,12,0,0,15,0,17,0,0,20,0,22,0,24,0,26,0,0,0,0],"v":[0,-0.04515658,0,0,0,0,-0.01905173,0.0186765,0,0,0,-0.05130367,0.004104435,0,0.003810395,0,0.120572,0.03341529,0,-0.03478252,0,-0.007918015,0,0.02653102,0,0.003824314,-0.04146723,0.02019011,0.04136698]},{"f":[21,-1,4,15,37,21,-1,-1,-1,22,-1,-1,15,16,16,16,-1,-1,-1,17,3,-1,-1,-1,22,25,-1,-1,-1],"t":[1.5,0,3.9534452978042594,11.683433292835403,4.470000000000001,6.5,0,0,0,11.683871801979876,0,0,10.457380879072534,7.118941072723508,5.78135971352466,5.491853096329675,0,0,0,9.011202871982467,0.3033788641265277,0,0,0,16.725912049439096,0.5,0,0,0],"m":[0,0,1,1,0,1,0,0,0,0,0,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,0,0,13,14,15,16,0,0,0,20,21,0,0,0,25,26,0,0,0],"r":[2,0,12,9,8,7,0,0,0,11,0,0,24,19,18,17,0,0,0,23,22,0,0,0,28,27,0,0,0],"v":[0,-0.04494596,0,0,0,0,-0.03333517,-0.009986201,-0.0005210401,0,0.07262877,0.0005740016,0,0,0,0,0.000459813,0.03733974,-0.01563177,0,0,-0.01478715,0.02934673,0.003006156,0,0,-0.0535174,-0.008191478,0.02782182]},{"f":[21,-1,8,4,15,-1,23,16,22,-1,24,-1,-1,-1,-1,15,16,16,16,-1,-1,-1,-1,22,25,-1,-1,-1,-1],"t":[1.5,0,0.5,3.9534452978042594,1.584962500721156,0,0.5,10.501837184902296,14.03140051979143,0,7.5,0,0,0,0,10.457380879072534,6.882643049361842,5.78135971352466,5.491853096329675,0,0,0,0,16.725912049439096,0.5,0,0,0,0],"m":[0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,9,0,11,0,0,0,0,16,17,18,19,0,0,0,0,24,25,0,0,0,0],"r":[2,0,28,15,6,0,14,13,10,0,12,0,0,0,0,23,22,21,20,0,0,0,0,27,26,0,0,0,0],"v":[0,-0.04472917,0,0,0,-0.04128825,0,0,0,-0.004301209,0,0.009477323,0.1079229,0.04615746,-0.00613413,0,0,0,0,0.0004590438,0.03511031,-0.01647268,0.01755462,0,0,-0.05141409,-0.007803508,0.02741095,0.02633406]},{"f":[21,-1,37,32,15,22,-1,-1,15,-1,1,21,-1,-1,22,2,17,-1,-1,-1,33,-1,-1,-1,4,-1,35,-1,-1],"t":[1.5,0,9.34,10.41931383093672,5.78135971352466,4.4828921423310435,0,0,6.169925001442312,0,4.92309205350118,6.5,0,0,12.199665042528586,5.228690439536267,4.128693921346326,0,0,0,18.128588230264874,0,0,0,1.292481250360578,0,2.5,0,0],"m":[0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,0,11,12,0,0,15,16,17,0,0,0,21,0,0,0,25,0,27,0,0],"r":[2,0,24,23,8,7,0,0,10,0,14,13,0,0,20,19,18,0,0,0,22,0,0,0,26,0,28,0,0],"v":[0,-0.04450615,0,0,0,0,-0.03336007,0.005301483,0,-0.03034243,0,0,-0.03325438,-0.0007562919,0,0,0,-0.002801849,-0.04448712,0.005393992,0,-0.02876742,0.005393992,-0.03048752,0,0.005393992,0,0.01088568,0.07411001]},{"f":[21,-1,25,2,37,32,16,17,-1,-1,15,-1,16,-1,1,21,-1,-1,22,-1,-1,-1,4,-1,35,-1,-1,-1,-1],"t":[1.5,0,6.5,10.680359255641626,9.34,10.41931383093672,2.584962500721156,9.135888799148791,0,0,5.78135971352466,0,6.108524456778169,0,4.92309205350118,6.5,0,0,12.199665042528586,0,0,0,1.292481250360578,0,2.5,0,0,0,0],"m":[0,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,15,16,0,0,19,0,0,0,23,0,25,0,0,0,0],"r":[2,0,28,27,22,21,10,9,0,0,12,0,14,0,18,17,0,0,20,0,0,0,24,0,26,0,0,0,0],"v":[0,-0.04427684,0,0,0,0,0,0,-0.01701763,0.005361211,0,0.005361211,0,-0.03390987,0,0,-0.03084639,-0.00191216,0,-0.01200581,0.005361211,-0.02938185,0,0.005361211,0,0.01203113,0.07248287,0.04045006,-0.04519658]},{"f":[21,-1,22,23,2,1,-1,21,-1,21,-1,-1,20,2,-1,2,4,-1,-1,17,-1,-1,-1,-1,18,32,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,9.051177541662383,0.39811320754716983,0,2.5,0,9.5,0,0,1.5,9.193524834446507,0,10.510264838663748,9.266718101290426,0,0,1.792481250360578,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,10,0,0,13,14,0,16,17,0,0,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,12,7,0,9,0,11,0,0,22,15,0,19,18,0,0,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04404121,0,0,0,0,-0.04689441,0,-0.03174546,0,-0.000439466,0.01719139,0,0,0.03074281,0,0,0.005180268,-0.03862947,0,-0.01563769,0.04997608,0.06751223,-0.03205908,0,0,-0.01130819,0.04983677,-0.02871723]},{"f":[21,-1,8,22,21,33,33,-1,-1,-1,-1,17,22,-1,-1,22,-1,22,-1,15,-1,15,-1,15,-1,2,-1,-1,-1],"t":[1.5,0,0.5,12.199665042528586,5.5,44.79706586676442,15.103049306997898,0,0,0,0,0.5,15.676331898425197,0,0,12.921158001713904,0,13.415167820124083,0,5.700439718141092,0,6.108524456778169,0,7.118941072723508,0,2.453445297804259,0,0,0],"m":[0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,0,12,13,0,0,16,0,18,0,20,0,22,0,24,0,26,0,0,0],"r":[2,0,28,11,10,9,8,0,0,0,0,15,14,0,0,17,0,19,0,21,0,23,0,25,0,27,0,0,0],"v":[0,-0.04379923,0,0,0,0,0,0.03142173,-0.02352737,0.02339872,0.002154243,0,0,-0.0441131,0.07472403,0,0.0169751,0,-0.03399584,0,0.01826115,0,-0.03469834,0,0.02046308,0,-0.0157063,0.006375886,0.02634362]},{"f":[21,-1,27,11,4,2,16,17,-1,-1,16,-1,-1,-1,20,22,-1,24,-1,-1,2,16,-1,22,-1,-1,-1,-1,-1],"t":[1.5,0,0.5,0.5,4.951187557243013,9.075473573191719,2.0,2.160964047443681,0,0,5.169925001442312,0,0,0,0.5,6.930301023985015,0,2.5,0,0,9.28539480853161,5.906890595608519,0,10.316277538115674,0,0,0,0,0],"m":[0,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,0,15,16,0,18,0,0,21,22,0,24,0,0,0,0,0],"r":[2,0,28,27,14,13,10,9,0,0,12,0,0,0,20,17,0,19,0,0,26,23,0,25,0,0,0,0,0],"v":[0,-0.0435509,0,0,0,0,0,0,-0.04940988,0.02385243,0,0.01378884,-0.004566912,0.04270859,0,0,0.0576218,0,-0.0008057243,-0.04217749,0,0,-0.01578926,0,-0.0018102,0.02614339,0.07704892,-0.02177644,0.02013569]},{"f":[21,-1,25,33,11,22,22,1,-1,-1,18,-1,-1,17,22,-1,-1,-1,22,17,-1,-1,-1,25,36,-1,-1,-1,-1],"t":[1.5,0,6.5,29.297756302860655,0.5,12.199665042528586,10.27595502635321,1.13125,0,0,10.609627990977145,0,0,0.5,15.717370311433935,0,0,0,17.094543568094064,9.135888799148791,0,0,0,0.5,3.5,0,0,0,0],"m":[0,0,1,1,1,0,1,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,14,15,0,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,18,13,10,9,0,0,12,0,0,17,16,0,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.0432962,0,0,0,0,0,0,0.01970439,-0.002737218,0,-0.0186567,0.06141893,0,0,-0.04382727,0.0789862,0.004999762,0,0,-0.03450104,0.04000203,0.02326237,0,0,-0.04629939,0.01091659,0.02651226,-0.04465514]},{"f":[21,-1,25,22,-1,22,1,-1,1,-1,17,-1,-1,22,-1,22,18,4,-1,-1,23,-1,-1,17,22,-1,-1,-1,-1],"t":[1.5,0,6.5,4.926654777701837,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,4.583709072915869,0,0,8.32532955673354,0,12.199665042528586,10.609627990977145,1.292481250360578,0,0,0.5,0,0,0.5,15.717370311433935,0,0,0,0],"m":[0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,11,0,0,14,0,16,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,5,0,13,8,0,10,0,12,0,0,15,0,23,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.04303516,0,0,-0.02877073,0,0,0.0481367,0,-0.03650455,0,-0.0123332,0.02737821,0,-0.04346265,0,0,0,-0.02179873,-0.001909484,0,0.06288032,-0.001714988,0,0,-0.04348262,0.07406342,0.004145708,-0.04365731]},{"f":[21,-1,33,11,4,2,-1,-1,15,16,3,-1,-1,-1,22,18,-1,-1,-1,22,17,-1,-1,-1,25,36,-1,-1,-1],"t":[1.5,0,29.297756302860655,0.5,4.951187557243013,9.075473573191719,0,0,10.376125389276174,7.266786540694901,0.5803571428571428,0,0,0,16.725912049439096,1.292481250360578,0,0,0,17.094543568094064,9.135888799148791,0,0,0,0.5,3.5,0,0,0],"m":[0,0,1,1,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,11,0,0,0,15,16,0,0,0,20,21,0,0,0,25,26,0,0,0],"r":[2,0,24,19,8,7,0,0,14,13,12,0,0,0,18,17,0,0,0,23,22,0,0,0,28,27,0,0,0],"v":[0,-0.04276782,0,0,0,0,-0.003244542,0.04136913,0,0,0,-0.04011176,0.003969951,0.018208,0,0,-0.04945447,-0.005916833,0.0251378,0,0,-0.03367199,0.03804658,0.02170831,0,0,-0.04544727,0.01115122,0.02541649]},{"f":[21,-1,2,25,37,32,16,17,-1,-1,15,-1,15,-1,1,21,-1,-1,-1,-1,4,35,-1,-1,35,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,6.5,9.34,10.41931383093672,2.584962500721156,9.135888799148791,0,0,5.78135971352466,0,6.169925001442312,0,4.92309205350118,6.5,0,0,0,0,1.292481250360578,2.5,0,0,2.5,0,0,0,0],"m":[0,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,15,16,0,0,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,28,27,20,19,10,9,0,0,12,0,14,0,18,17,0,0,0,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.04249422,0,0,0,0,0,0,-0.01540029,0.005132975,0,0.005132975,0,-0.02749546,0,0,-0.03054733,-0.00171548,0.005132975,-0.02803948,0,0,0.0321742,0.005132975,0,0.01040127,0.07168238,-0.04282359,0.03903024]},{"f":[21,-1,27,11,4,2,-1,-1,20,22,2,22,22,-1,-1,-1,-1,-1,2,22,16,-1,23,-1,-1,-1,-1,-1,-1],"t":[1.5,0,0.5,0.5,4.951187557243013,9.075473573191719,0,0,0.5,10.27595502635321,7.661668144640085,8.133841411710264,7.5621616439616925,0,0,0,0,0,9.28539480853161,14.03140051979143,7.118941072723508,0,1.5,0,0,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,0,1,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,11,12,13,0,0,0,0,0,19,20,21,0,23,0,0,0,0,0,0],"r":[2,0,28,27,8,7,0,0,18,17,16,15,14,0,0,0,0,0,26,25,22,0,24,0,0,0,0,0,0],"v":[0,-0.04221442,0,0,0,0,-0.002624558,0.0406923,0,0,0,0,0,0.02380553,-0.04679734,0.06314779,-0.006816104,-0.007912248,0,0,0,-0.01833373,0,0.001581558,0.04182561,0.03225132,0.0746659,-0.01958289,0.01831579]},{"f":[21,-1,25,22,-1,22,1,-1,1,-1,17,-1,21,16,-1,-1,-1,22,-1,22,18,-1,-1,17,22,-1,-1,-1,-1],"t":[1.5,0,6.5,4.926654777701837,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,4.583709072915869,0,7.5,10.372865060112588,0,0,0,8.32532955673354,0,12.199665042528586,10.609627990977145,0,0,0.5,15.717370311433935,0,0,0,0],"m":[0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,11,0,13,14,0,0,0,18,0,20,21,0,0,24,25,0,0,0,0],"r":[2,0,28,5,0,17,8,0,10,0,12,0,16,15,0,0,0,19,0,23,22,0,0,27,26,0,0,0,0],"v":[0,-0.04192852,0,0,-0.02784962,0,0,0.04438512,0,-0.03583653,0,-0.01183342,0,0,0.09617002,0.01973489,0.006666726,0,-0.04213585,0,0,-0.006458389,0.01838362,0,0,-0.04295816,0.07020756,0.004006022,-0.04210723]},{"f":[21,-1,2,33,22,22,15,-1,-1,18,-1,-1,17,22,-1,-1,23,2,-1,-1,18,-1,-1,25,36,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,29.297756302860655,12.199665042528586,10.27595502635321,11.054604317960672,0,0,10.609627990977145,0,0,0.5,15.717370311433935,0,0,3.5,9.012591414317754,0,0,6.499823868264185,0,0,0.5,3.5,0,0,0,0],"m":[0,0,1,1,0,1,1,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,14,0,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,23,12,9,8,0,0,11,0,0,16,15,0,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.04163659,0,0,0,0,0,0.0007908027,0.0451383,0,-0.01899406,0.05603306,0,0,-0.0425941,0.06590589,0,0,0.005107633,0.04624051,0,0.0043691,-0.02441895,0,0,-0.04453443,0.01048237,0.02414156,0.0382888]},{"f":[21,-1,22,23,21,2,1,-1,21,-1,-1,20,1,4,-1,-1,-1,-1,0,1,-1,-1,-1,-1,18,1,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,9.5,9.051177541662383,0.6824793747286149,0,2.5,0,0,1.5,2.01685992705101,9.266718101290426,0,0,0,0,7.5,6.015625,0,0,0,0,8.729619742794132,7.9752100840336135,0,0,0],"m":[0,0,1,1,1,1,0,0,0,0,0,1,1,1,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,13,14,0,0,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,18,11,8,0,10,0,0,17,16,15,0,0,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04133877,0,0,0,0,0,-0.0441684,0,-0.03057852,-0.0004097421,0,0,0,0.007245895,-0.03038825,0.02615113,0.06391196,0,0,0.01264546,-0.03005297,0.0554922,-0.02970498,0,0,0.04889499,-0.01019332,-0.02786179]},{"f":[21,-1,22,-1,22,1,-1,1,-1,17,-1,21,16,-1,-1,-1,22,-1,22,-1,17,22,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,4.926654777701837,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,4.583709072915869,0,7.5,10.372865060112588,0,0,0,8.32532955673354,0,12.199665042528586,0,0.5,15.717370311433935,0,0,12.921158001713904,0,13.415167820124083,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,0,8,0,10,0,12,13,0,0,0,17,0,19,0,21,22,0,0,25,0,27,0,0],"r":[2,0,4,0,16,7,0,9,0,11,0,15,14,0,0,0,18,0,20,0,24,23,0,0,26,0,28,0,0],"v":[0,-0.04103517,0,-0.02748127,0,0,0.04090813,0,-0.03524016,0,-0.01133647,0,0,0.09153008,0.01857239,0.006178016,0,-0.04132925,0,-0.003993234,0,0,-0.04214797,0.06238161,0,0.0144967,0,-0.03227029,0.002567573]},{"f":[21,-1,2,8,1,-1,22,23,21,22,-1,23,33,-1,-1,-1,0,1,-1,-1,-1,-1,18,1,-1,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.5,0.3126984126984127,0,17.094543568094064,7.5,9.5,14.116600814324018,0,1.5,16.993854061608424,0,0,0,7.5,6.015625,0,0,0,0,8.729619742794132,7.9752100840336135,0,0,0,0,0],"m":[0,0,1,1,0,0,1,1,1,1,0,0,1,0,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,9,10,0,12,13,0,0,0,17,18,0,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,27,6,0,22,21,16,11,0,15,14,0,0,0,20,19,0,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.04072594,0,0,0,-0.05055082,0,0,0,0,-0.002814897,0,0,0.04748269,0.006466959,-0.0008773881,0,0,0.01230905,-0.02898986,0.05481169,-0.02901288,0,0,0.0465744,-0.009905904,-0.02708573,0.02443185,0.03770173]},{"f":[21,-1,25,37,32,23,1,-1,-1,4,2,2,22,19,23,-1,-1,-1,-1,-1,-1,-1,-1,4,-1,35,-1,-1,-1],"t":[1.5,0,6.5,9.34,10.41931383093672,3.5,2.01685992705101,0,0,6.451187557243013,4.0437314206251695,1.792481250360578,17.094543568094064,0.5,7.5,0,0,0,0,0,0,0,0,1.292481250360578,0,2.5,0,0,0],"m":[0,0,1,1,1,1,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,11,12,13,14,15,0,0,0,0,0,0,0,0,24,0,26,0,0,0],"r":[2,0,28,23,22,9,8,0,0,21,20,19,18,17,16,0,0,0,0,0,0,0,0,25,0,27,0,0,0],"v":[0,-0.04041125,0,0,0,0,0,-0.004797754,0.004197415,0,0,0,0,0,0,0.004817313,-0.03168209,-0.03438584,0.004817313,-0.05133093,0.004817313,-0.04170566,-0.02704965,0,0.004817313,0,0.009675346,0.07083378,-0.04150261]},{"f":[21,4,22,-1,-1,16,-1,-1,25,37,15,-1,16,-1,22,16,-1,-1,-1,4,25,-1,-1,22,-1,-1,17,-1,-1],"t":[2.5,4.951187557243013,13.87915215758784,0,0,5.78135971352466,0,0,2.5,9.205,5.169925001442312,0,6.108524456778169,0,12.199665042528586,10.501837184902296,0,0,0,1.292481250360578,0.5,0,0,11.876328901062104,0,0,1.792481250360578,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,0,15,16,0,0,0,20,21,0,0,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,26,19,12,0,14,0,18,17,0,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,-0.04635256,0.02629075,0,0.05758389,-0.03982065,0,0,0,0.005909675,0,-0.01455159,0,0,-0.01511334,0.007675853,0.003623878,0,0,0.007675853,0.05174116,0,0.008595503,0.0498464,0,-0.05933105,-0.001125528]},{"f":[21,-1,22,-1,22,1,-1,1,-1,17,-1,21,16,-1,-1,-1,22,-1,22,18,-1,18,-1,-1,17,22,-1,-1,-1],"t":[1.5,0,4.926654777701837,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,4.583709072915869,0,7.5,10.372865060112588,0,0,0,8.32532955673354,0,12.199665042528586,10.609627990977145,0,10.682994316786338,0,0,0.5,15.717370311433935,0,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,0,8,0,10,0,12,13,0,0,0,17,0,19,20,0,22,0,0,25,26,0,0,0],"r":[2,0,4,0,16,7,0,9,0,11,0,15,14,0,0,0,18,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,-0.03981271,0,-0.02690138,0,0,0.03843373,0,-0.03444583,0,-0.01101633,0,0,0.0872553,0.01711465,0.006302162,0,-0.04049558,0,0,-0.005889104,0,0.07673547,0.003012865,0,0,-0.04175453,0.05837514,0.003158565]},{"f":[8,4,15,-1,-1,15,16,16,16,-1,0,-1,-1,-1,21,-1,16,2,18,-1,-1,-1,-1,22,25,-1,-1,-1,-1],"t":[0.5,3.9534452978042594,1.584962500721156,0,0,10.457380879072534,7.118941072723508,5.78135971352466,5.491853096329675,0,3.5,0,0,0,3.5,0,11.620219825507487,6.033380966193453,6.531360383582522,0,0,0,0,16.725912049439096,0.5,0,0,0,0],"m":[1,1,0,0,0,1,1,1,1,0,1,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,11,0,0,0,15,0,17,18,19,0,0,0,0,24,25,0,0,0,0],"r":[28,5,4,0,0,23,14,13,10,0,12,0,0,0,16,0,22,21,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03899899,-0.002298874,0,0,0,0,0.0006494024,0,0.003678795,0.05515755,-0.01364798,0,-0.01781177,0,0,0,0.05129896,-0.00280212,0.01403365,-0.0009492856,0,0,-0.04672495,-0.006810343,0.02479433,0.02292819]},{"f":[21,4,22,-1,-1,16,-1,-1,25,37,15,4,-1,17,-1,-1,16,-1,-1,4,25,-1,-1,35,-1,-1,18,-1,-1],"t":[2.5,4.951187557243013,13.87915215758784,0,0,5.78135971352466,0,0,2.5,9.205,5.169925001442312,9.266718101290426,0,9.845435504646158,0,0,6.108524456778169,0,0,1.292481250360578,0.5,0,0,2.5,0,0,1.292481250360578,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,0,14,0,0,17,0,0,20,21,0,0,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,26,19,16,13,0,15,0,0,18,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,-0.04609355,0.02545893,0,0.0519442,-0.03931514,0,0,0,0,0.007281549,0,-0.04083408,0.007281549,0,-0.01405551,-4.769928e-05,0,0,0.007281549,0.05103897,0,0.01723915,0.06565749,0,-0.05920916,-0.001254031]},{"f":[2,22,22,27,11,21,-1,4,2,-1,-1,18,-1,-1,-1,-1,22,18,-1,-1,2,-1,-1,18,32,-1,-1,-1,-1],"t":[10.680359255641626,17.094543568094064,17.069460217922263,0.5,0.5,2.5,0,9.266718101290426,9.051177541662383,0,0,10.457893746154532,0,0,0,0,17.078739781790915,6.076776015854056,0,0,4.0437314206251695,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,0,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,16,15,14,7,0,11,10,0,0,13,0,0,0,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.01988046,0,0,6.293282e-05,0.01693904,0,-0.05649365,0.02197476,-0.02393363,0.02389892,0,0,-0.0723663,-0.01157007,0,-0.02839885,0.02769368,0,0,-0.01350405,0.0432997,-0.02630726,0.03728505]},{"f":[2,27,8,1,-1,32,33,11,4,2,-1,-1,20,22,2,-1,-1,-1,-1,17,-1,-1,-1,33,-1,-1,-1,-1,-1],"t":[10.680359255641626,0.5,0.5,0.3126984126984127,0,10.463013322934223,29.297756302860655,0.5,4.951187557243013,8.612669628062957,0,0,0.5,10.27595502635321,7.661668144640085,0,0,0,0,9.135888799148791,0,0,0,41.64100553123272,0,0,0,0,0],"m":[1,1,1,0,0,1,1,1,1,1,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,0,0,13,14,15,0,0,0,0,20,0,0,0,24,0,0,0,0,0],"r":[28,27,26,5,0,23,22,19,12,11,0,0,18,17,16,0,0,0,0,21,0,0,0,25,0,0,0,0,0],"v":[0,0,0,0,-0.05001371,0,0,0,0,0,-0.003193292,0.0514572,0,0,0,0.02700366,-0.006611672,-0.00806361,0.01331996,0,-0.02566665,0.03819604,0.01865378,0,-0.04484214,0.002602119,0.02246143,0.01653092,0.03669659]},{"f":[22,22,22,22,22,-1,-1,4,-1,22,-1,-1,23,-1,-1,22,18,-1,-1,2,-1,-1,18,1,17,-1,-1,-1,-1],"t":[17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,3.753897320099348,0,16.819976309430963,0,0,7.5,0,0,17.078739781790915,6.076776015854056,0,0,4.0437314206251695,0,0,8.729619742794132,7.9752100840336135,7.588709768994619,0,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,0,13,0,0,16,17,0,0,20,0,0,23,24,25,0,0,0,0],"r":[22,15,12,7,6,0,0,9,0,11,0,0,14,0,0,19,18,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,6.61179e-06,0.05759227,0,-0.03096453,0,-0.02248856,0.02572651,0,0.03168952,-0.02299863,0,0,-0.06922217,-0.01128777,0,-0.02765806,0.02659547,0,0,0,0.008723347,0.06358994,-0.009342047,-0.02546829]},{"f":[21,-1,2,22,-1,22,-1,22,16,-1,-1,22,-1,22,18,2,4,-1,-1,-1,23,-1,-1,17,22,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,4.926654777701837,0,5.9525054311951475,0,7.770639075561187,10.462502272597265,0,0,8.32532955673354,0,12.199665042528586,10.609627990977145,9.266718101290426,1.292481250360578,0,0,0,0.5,0,0,0.5,15.717370311433935,0,0,0,0],"m":[0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,0,8,9,0,0,12,0,14,15,16,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,5,0,7,0,11,10,0,0,13,0,23,20,19,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.03894383,0,0,-0.02591774,0,0.07389122,0,0,0.0005112473,0.03779606,0,-0.04018153,0,0,0,0,-0.0194922,9.134581e-05,-0.04471223,0,0.05623916,-0.00377298,0,0,-0.04125589,0.05493933,0.002987771,0.03611181]},{"f":[25,21,-1,15,37,36,23,-1,2,21,-1,-1,18,-1,-1,17,-1,-1,33,15,-1,-1,21,22,-1,-1,-1,-1,-1],"t":[6.5,1.5,0,13.56045225766691,7.92,3.5,0.5,0,4.0437314206251695,7.5,0,0,9.229707804263573,0,0,7.198837316474133,0,0,15.81450217830399,5.169925001442312,0,0,9.5,16.838461650489297,0,0,0,0,0],"m":[1,0,0,1,1,1,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,0,4,5,6,7,0,9,10,0,0,13,0,0,16,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,3,0,27,18,15,8,0,12,11,0,0,14,0,0,17,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,-0.0386027,0,0,0,0,0.003043284,0,0,-0.02336767,0.001949692,0,0.003043284,-0.01638389,0,-0.006704644,-0.05539274,0,0,0.05456151,0.0090468,0,0,0.003043284,0.02206681,0.04491113,-0.02113175,-0.03906335]},{"f":[22,22,22,22,22,-1,-1,-1,23,16,22,-1,-1,2,-1,-1,-1,22,18,-1,-1,2,-1,-1,18,32,-1,-1,-1],"t":[17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,0,7.5,6.0,16.98920034885663,0,0,5.538407798525416,0,0,0,17.078739781790915,6.076776015854056,0,0,4.0437314206251695,0,0,8.729619742794132,8.341935934311302,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,11,0,0,14,0,0,0,18,19,0,0,22,0,0,25,26,0,0,0],"r":[24,17,8,7,6,0,0,0,16,13,12,0,0,15,0,0,0,21,20,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,2.879763e-05,0.05604016,-0.013476,0,0,0,0.08787637,0.009391198,0,-0.02395913,0.03445103,-0.02257724,0,0,-0.06627058,-0.01105752,0,-0.02712769,0.02545373,0,0,-0.01371785,0.04020514,-0.02465916]},{"f":[2,8,1,-1,33,32,33,33,37,-1,-1,4,15,16,-1,-1,-1,-1,21,2,-1,1,-1,-1,-1,-1,-1,-1,-1],"t":[10.680359255641626,0.5,0.3126984126984127,0,44.79706586676442,10.482680779086738,20.752541988539424,13.895530557358477,8.05,0,0,4.80643424864552,7.266786540694901,5.044394119358453,0,0,0,0,9.5,1.292481250360578,0,9.107199999999999,0,0,0,0,0,0,0],"m":[1,1,0,0,1,1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,9,0,0,12,13,14,0,0,0,0,19,20,0,22,0,0,0,0,0,0,0],"r":[28,27,4,0,26,25,18,11,10,0,0,17,16,15,0,0,0,0,24,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0,-0.04914958,0,0,0,0,0,-0.0003484918,0.03864565,0,0,0,-0.00420483,-0.07289036,-0.01140236,0.02850373,0,0,-0.008816227,0,0.005908955,0.04515286,0.05212909,-0.05412855,0.02036192,0.0219938,0.03549929]},{"f":[21,-1,22,-1,22,-1,22,16,-1,-1,22,-1,22,-1,17,22,-1,-1,22,36,-1,-1,22,-1,15,23,-1,-1,-1],"t":[1.5,0,4.926654777701837,0,5.9525054311951475,0,7.770639075561187,10.462502272597265,0,0,8.133841411710264,0,12.199665042528586,0,0.5,15.717370311433935,0,0,12.921158001713904,2.5,0,0,13.415167820124083,0,5.169925001442312,3.5,0,0,0],"m":[0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,0,7,8,0,0,11,0,13,0,15,16,0,0,19,20,0,0,23,0,25,26,0,0,0],"r":[2,0,4,0,6,0,10,9,0,0,12,0,14,0,18,17,0,0,22,21,0,0,24,0,28,27,0,0,0],"v":[0,-0.03825098,0,-0.02536491,0,0.06905526,0,0,0.0002927825,0.03531875,0,-0.04425602,0,-0.003689371,0,0,-0.04081468,0.05212172,0,0,-0.007164225,0.02117701,0,-0.03106824,0,0,0.03410449,-0.0001712189,-0.0006686058]},{"f":[21,4,22,-1,-1,16,-1,-1,25,37,-1,4,25,-1,-1,35,-1,-1,17,-1,22,36,16,-1,-1,-1,22,-1,-1],"t":[2.5,4.951187557243013,13.87915215758784,0,0,5.78135971352466,0,0,2.5,9.205,0,1.292481250360578,0.5,0,0,2.5,0,0,1.792481250360578,0,12.613886328427084,2.5,4.247927513443585,0,0,0,12.637102544927284,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,13,0,0,16,0,0,19,0,21,22,23,0,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,18,11,0,15,14,0,0,17,0,0,20,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,-0.04564456,0.02393552,0,0.04905142,-0.03890582,0,0,0.0002782962,0,0,0.006534391,0.04935628,0,0.01528042,0.06359917,0,-0.05591899,0,0,0,-0.03365172,-0.003175018,0.02347732,0,0.05093611,-0.02491219]},{"f":[2,22,22,22,22,22,-1,-1,22,-1,-1,23,-1,-1,22,18,-1,-1,2,-1,-1,18,1,17,-1,-1,-1,-1,-1],"t":[10.680359255641626,17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,16.819976309430963,0,0,7.5,0,0,17.078739781790915,6.076776015854056,0,0,4.0437314206251695,0,0,8.729619742794132,7.9752100840336135,7.588709768994619,0,0,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,9,0,0,12,0,0,15,16,0,0,19,0,0,22,23,24,0,0,0,0,0],"r":[28,21,14,11,8,7,0,0,10,0,0,13,0,0,18,17,0,0,20,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.000137097,0.054663,0,-0.03506131,0.003082815,0,0.02884207,-0.02215727,0,0,-0.06312083,-0.01061445,0,-0.02667727,0.02456631,0,0,0,0.0063784,0.06140676,-0.009093028,-0.02365658,0.03489929]},{"f":[22,-1,22,16,-1,-1,1,17,21,22,17,-1,-1,-1,16,-1,22,-1,16,-1,-1,-1,1,1,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.082728314749225,10.681238411777805,0,0,2.01685992705101,11.651826559171445,6.5,10.092754721228424,1.792481250360578,0,0,0,1.0,0,16.25454052700782,0,7.924812503605781,0,0,0,2.969223484848485,2.532051282051282,0,0,9.299206200930918,0,0],"m":[0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,0,0,0,15,0,17,0,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,6,5,0,0,22,21,14,13,12,0,0,0,16,0,18,0,20,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.03877638,0,0,0.002128306,0.04237145,0,0,0,0,0,0.0663754,-0.03566395,-0.03770366,0,0.0426125,0,-0.01107746,0,-0.0003484879,0.05806144,0.02698556,0,0,0.004212369,0.06687305,0,-0.0004285725,0.03520718]},{"f":[15,15,22,22,21,4,-1,16,-1,-1,25,-1,-1,16,22,-1,2,-1,-1,-1,18,32,-1,17,-1,-1,-1,-1,-1],"t":[13.56045225766691,13.323758479706308,17.094543568094064,17.069460217922263,2.5,4.951187557243013,0,5.78135971352466,0,0,2.5,0,0,7.774787059601174,17.078739781790915,0,4.0437314206251695,0,0,0,8.729619742794132,8.341935934311302,0,8.571721503073015,0,0,0,0,0],"m":[1,1,1,1,0,1,0,1,0,0,1,0,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,11,0,0,14,15,0,17,0,0,0,21,22,0,24,0,0,0,0,0],"r":[28,27,20,13,10,7,0,9,0,0,12,0,0,19,16,0,18,0,0,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.03741092,0,0.04601529,-0.03849768,0,0.001328845,-0.007745085,0,0,-0.05638241,0,-0.03092265,0.01915006,0.01062877,0,0,-0.01443564,0,0.01798947,0.0633233,-0.02247111,0.02680592,-0.02021767]},{"f":[22,-1,22,1,-1,1,-1,18,-1,-1,22,-1,21,17,37,-1,-1,-1,15,-1,18,3,-1,22,-1,-1,22,-1,-1],"t":[2.69615871138938,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,3.084962500721156,0,0,8.133841411710264,0,2.5,1.292481250360578,3.97,0,0,0,10.457380879072534,0,10.620215166896223,0.26136363636363635,0,16.725912049439096,0,0,11.722778233171695,0,0],"m":[0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,0,13,14,15,0,0,0,19,0,21,22,0,24,0,0,27,0,0],"r":[2,0,10,5,0,7,0,9,0,0,12,0,18,17,16,0,0,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,-0.03838517,0,0,0.03163942,0,-0.02815111,0,-0.01202527,0.01782893,0,-0.04341897,0,0,0,-0.03761462,0.01906616,-0.04380175,0,0.00130031,0,0,0.03565261,0,-0.01698605,0.01783731,0,0.06921919,-0.0034232]},{"f":[33,32,33,23,22,-1,15,-1,17,25,-1,-1,-1,18,21,-1,-1,1,-1,22,-1,-1,33,37,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,29.297756302860655,3.5,14.116600814324018,0,5.044394119358453,0,6.199638962254484,2.5,0,0,0,6.400854179458231,6.5,0,0,1.981615336266499,0,17.083799521744545,0,0,39.9785486130134,9.629999999999999,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,9,10,0,0,0,14,15,0,0,18,0,20,0,0,23,24,0,0,0,0,0],"r":[28,27,22,13,6,0,8,0,12,11,0,0,0,17,16,0,0,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.001039466,0,0.0387713,0,0,0.03186536,-0.04233575,0.0113977,0,0,-0.04612823,0.006035849,0,0.06917963,0,-0.03337164,0.008957684,0,0,0.01008027,0.04995612,-0.02450837,-0.05303169,0.01959925]},{"f":[2,21,-1,1,-1,21,2,-1,17,15,-1,22,-1,-1,-1,22,-1,-1,3,22,-1,-1,16,15,2,-1,-1,-1,-1],"t":[9.051177541662383,2.5,0,0.39811320754716983,0,9.5,7.005613627711627,0,11.364132993990529,10.113742166049189,0,16.725912049439096,0,0,0,15.756821747580634,0,0,0.9640400250156348,13.005894632618489,0,0,11.696967526234287,4.247927513443585,9.266718101290426,0,0,0,0],"m":[1,0,0,0,0,1,1,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,0,4,0,6,7,0,9,10,0,12,0,0,0,16,0,0,19,20,0,0,23,24,25,0,0,0,0],"r":[18,3,0,5,0,15,8,0,14,11,0,13,0,0,0,17,0,0,22,21,0,0,28,27,26,0,0,0,0],"v":[0,0,-0.03115147,0,-0.04547573,0,0,0.0004238226,0,0,-0.03344622,0,-0.0170306,0.02108246,0.01772352,0,0.0185142,-0.03842968,0,0,0.004159251,0.05352754,0,0,0,0.00730741,-0.02474082,0.02932175,-0.04087207]},{"f":[2,27,23,-1,18,21,-1,18,18,15,-1,-1,-1,22,-1,22,-1,-1,1,-1,22,15,-1,-1,18,-1,-1,-1,-1],"t":[10.680359255641626,0.5,3.5,0,6.400854179458231,6.5,0,5.303665156874805,1.292481250360578,13.56045225766691,0,0,0,17.069460217922263,0,17.074218326240924,0,0,1.981615336266499,0,17.094543568094064,3.4594316186372973,0,0,8.729619742794132,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,9,10,0,0,0,14,0,16,0,0,19,0,21,22,0,0,25,0,0,0,0],"r":[28,27,4,0,18,7,0,13,12,11,0,0,0,15,0,17,0,0,20,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,0.0008688135,0,0,-0.04530512,0,0,0,0.01267497,-0.01909423,-0.01047629,0,0.04587989,0,-0.03299907,0.03870555,0,0.06557318,0,0,0.006409351,-0.03748134,0,0.0233713,-0.02952561,0.01483354,0.03406799]},{"f":[22,-1,22,15,-1,-1,22,-1,22,-1,17,22,-1,-1,22,36,22,-1,-1,-1,22,-1,15,23,2,-1,-1,-1,-1],"t":[2.69615871138938,0,7.770639075561187,10.066089190457772,0,0,8.133841411710264,0,12.199665042528586,0,0.5,15.717370311433935,0,0,12.921158001713904,2.5,12.613886328427084,0,0,0,13.415167820124083,0,5.169925001442312,2.5,1.292481250360578,0,0,0,0],"m":[0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,0,11,12,0,0,15,16,17,0,0,0,21,0,23,24,25,0,0,0,0],"r":[2,0,6,5,0,0,8,0,10,0,14,13,0,0,20,19,18,0,0,0,22,0,28,27,26,0,0,0,0],"v":[0,-0.03798241,0,0,0.00438222,0.04970522,0,-0.04254784,0,-0.003646481,0,0,-0.04092188,0.0497635,0,0,0,-0.0308517,0.04979545,0.01966191,0,-0.02916723,0,0,0,0.07359063,0.01449303,0.002055427,-0.0006468962]},{"f":[4,32,16,17,2,19,15,17,15,-1,-1,-1,15,-1,15,-1,-1,-1,37,-1,33,-1,-1,-1,22,-1,-1,-1,-1],"t":[3.084962500721156,10.463013322934223,10.501837184902296,10.37503779525057,1.792481250360578,2.5,5.169925001442312,1.792481250360578,4.087462841250339,0,0,0,6.108524456778169,0,7.118941072723508,0,0,0,6.890000000000001,0,18.188502736593104,0,0,0,11.683871801979876,0,0,0,0],"m":[1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,13,0,15,0,0,0,19,0,21,0,0,0,25,0,0,0,0],"r":[28,27,24,23,18,17,12,11,10,0,0,0,14,0,16,0,0,0,20,0,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,-0.02327791,0.03037042,0.05166419,0,-0.04850989,0,0.02297538,-0.001783744,-0.02128623,0,-0.03892199,0,-0.005302947,0.04110297,-0.03265456,0,0.04364554,0.001078311,-0.03696068,0.001923163]},{"f":[33,32,33,33,37,21,-1,-1,-1,4,15,16,-1,-1,-1,-1,21,33,-1,2,33,-1,-1,24,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,20.752541988539424,13.895530557358477,8.05,2.5,0,0,0,4.80643424864552,7.266786540694901,5.044394119358453,0,0,0,0,9.5,21.16864017939033,0,1.292481250360578,27.893589438525005,0,0,9.5,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,12,0,0,0,0,17,18,0,20,21,0,0,24,0,0,0,0,0],"r":[28,27,16,9,8,7,0,0,0,15,14,13,0,0,0,0,26,19,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.02179544,0.0001554081,0.0370031,0,0,0,-0.003090679,-0.07086655,-0.01082045,0.02677882,0,0,0.03708092,0,0,-0.006006292,-0.03985529,0,0.004339499,0.05840513,0.05044537,-0.05214598,0.01916364]},{"f":[22,-1,22,-1,21,17,4,-1,-1,-1,30,23,-1,4,2,17,17,21,-1,18,-1,-1,-1,-1,-1,-1,21,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,2.5,1.292481250360578,4.951187557243013,0,0,0,1.5,3.5,0,6.451187557243013,4.0437314206251695,8.496347487943302,6.434604125275413,7.5,0,1.292481250360578,0,0,0,0,0,0,3.5,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,0,0,11,12,0,14,15,16,17,18,0,20,0,0,0,0,0,0,27,0,0],"r":[2,0,4,0,10,9,8,0,0,0,26,13,0,25,24,23,22,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,-0.02416019,0,0.0636792,0,0,0,-0.02405238,0.02371367,-0.04329052,0,0,0.002772439,0,0,0,0,0,-0.02974168,0,0.02807088,-0.02682687,-0.04017566,0.01919947,0.006469301,-0.03581546,0,-0.05222685,-0.00501312]},{"f":[2,21,22,-1,-1,1,-1,21,2,4,-1,22,-1,-1,-1,22,-1,-1,22,4,-1,-1,18,1,-1,-1,4,-1,-1],"t":[9.051177541662383,2.5,13.827286379004683,0,0,0.39811320754716983,0,9.5,6.959990297524481,4.80643424864552,0,14.116600814324018,0,0,0,15.756821747580634,0,0,11.681676585676911,9.266718101290426,0,0,9.273793716589307,4.835403151065801,0,0,10.510264838663748,0,0],"m":[1,0,1,0,0,0,0,1,1,1,0,1,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,9,10,0,12,0,0,0,16,0,0,19,20,0,0,23,24,0,0,27,0,0],"r":[18,5,4,0,0,7,0,15,14,11,0,13,0,0,0,17,0,0,22,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,-0.03832385,0.02393021,0,-0.04490813,0,0,0,-0.001329259,0,0.002892213,0.02317814,-0.008625237,0,0.01707988,-0.03763073,0,0,0.006467514,-0.03953748,0,0,0.04863647,-0.007007478,0,-0.02456756,0.04456657]},{"f":[25,22,-1,23,22,-1,-1,4,2,16,17,3,-1,17,16,-1,-1,-1,-1,21,17,-1,-1,-1,21,-1,-1,-1,-1],"t":[6.5,2.69615871138938,0,3.5,14.116600814324018,0,0,6.451187557243013,4.0437314206251695,7.330916878114617,7.723832212141186,0.80625,0,1.292481250360578,2.807354922057604,0,0,0,0,9.5,0.5,0,0,0,7.5,0,0,0,0],"m":[1,0,0,1,1,0,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,10,11,12,0,14,15,0,0,0,0,20,21,0,0,0,25,0,0,0,0],"r":[28,3,0,7,6,0,0,27,24,19,18,13,0,17,16,0,0,0,0,23,22,0,0,0,26,0,0,0,0],"v":[0,0,-0.03733523,0,0,-8.63422e-05,0.008762544,0,0,0,0,0,0.04189127,0,0,-0.0119619,0.03718052,-0.02608683,0.0287739,0,0,0.02879585,-0.04921855,0.03637455,0,0.02945256,0.001609076,-0.03200583,-0.03620637]},{"f":[2,21,22,-1,-1,1,-1,21,-1,0,22,-1,16,2,-1,-1,-1,-1,22,4,-1,-1,18,1,-1,-1,4,-1,-1],"t":[9.051177541662383,2.5,13.827286379004683,0,0,0.39811320754716983,0,9.5,0,7.5,10.735831902194036,0,11.98370619265935,5.065928480304397,0,0,0,0,11.681676585676911,9.266718101290426,0,0,9.273793716589307,4.835403151065801,0,0,10.510264838663748,0,0],"m":[1,0,1,0,0,0,0,1,0,1,0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,0,10,11,0,13,14,0,0,0,0,19,20,0,0,23,24,0,0,27,0,0],"r":[18,5,4,0,0,7,0,9,0,17,12,0,16,15,0,0,0,0,22,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,-0.037856,0.0231667,0,-0.0445055,0,-0.0005473865,0,0,0.02649672,0,0,-0.04126792,0.002072523,0.02755521,0.04942722,0,0,0.006229683,-0.03773779,0,0,0.04747722,-0.006457718,0,-0.02275057,0.04408524]},{"f":[8,1,15,-1,-1,22,21,1,22,-1,-1,33,19,33,23,-1,-1,-1,-1,-1,-1,17,23,-1,-1,2,-1,-1,-1],"t":[0.5,0.3126984126984127,1.584962500721156,0,0,12.199665042528586,5.5,1.000936329588015,9.505613627711627,0,0,44.79706586676442,2.5,32.199935296219856,1.5,0,0,0,0,0,0,0.5,1.5,0,0,9.012591414317754,0,0,0],"m":[1,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,0,12,13,14,15,0,0,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,5,4,0,0,21,20,11,10,0,0,19,18,17,16,0,0,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0.02199107,-0.04797761,0,0,0,0,0.1185201,-0.006499067,0,0,0,0,-0.01103942,-0.04410049,-0.05355518,0.01082353,0.01819656,0.0009895513,0,0,-0.0410158,0.03106066,0,0.001505373,0.02136791,0.01939352]},{"f":[22,-1,22,16,-1,-1,1,17,21,15,22,21,-1,-1,-1,22,-1,17,-1,-1,-1,-1,1,1,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.082728314749225,10.681238411777805,0,0,2.01685992705101,11.651826559171445,7.5,7.417852514885898,10.194756329264862,5.5,0,0,0,14.247293309538719,0,2.453445297804259,0,0,0,0,2.969223484848485,2.532051282051282,0,0,9.299206200930918,0,0],"m":[0,0,0,1,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,12,0,0,0,16,0,18,0,0,0,0,23,24,0,0,27,0,0],"r":[2,0,6,5,0,0,22,21,20,15,14,13,0,0,0,17,0,19,0,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.0369412,0,0,0.001639016,0.03571176,0,0,0,0,0,0,0.08520961,-0.02146925,-0.04369329,0,-0.01923165,0,-0.03895147,0.0549225,0.0003756083,0.02367122,0,0,0.002402603,0.06378397,0,-0.0002677128,0.03301319]},{"f":[33,33,37,-1,-1,4,15,16,-1,-1,-1,-1,21,33,-1,2,-1,35,2,2,-1,-1,22,-1,-1,0,-1,-1,-1],"t":[20.752541988539424,13.895530557358477,8.05,0,0,4.80643424864552,7.266786540694901,5.044394119358453,0,0,0,0,9.5,21.16864017939033,0,1.292481250360578,0,2.5,3.753897320099348,2.903677461028802,0,0,16.832741648970774,0,0,9.5,0,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,0,0,13,14,0,16,0,18,19,20,0,0,23,0,0,26,0,0,0],"r":[12,5,4,0,0,11,10,9,0,0,0,0,28,15,0,17,0,25,22,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,-0.0002714585,0.03621296,0,0,0,-0.002837578,-0.06947134,-0.01032719,0.02512787,0,0,0.03577378,0,-0.0113362,0,0,0,-0.001377679,0.03135811,0,-0.01695878,0.0153108,0,0.01301329,0.06334502,0.04845068]},{"f":[15,37,36,23,-1,2,21,22,23,-1,-1,-1,-1,18,-1,22,-1,-1,17,-1,-1,33,15,-1,-1,21,-1,-1,-1],"t":[13.56045225766691,7.92,3.5,0.5,0,3.5799356683891945,7.5,16.964783007086943,2.5,0,0,0,0,9.229707804263573,0,12.457649623491124,0,0,7.198837316474133,0,0,13.895530557358477,5.169925001442312,0,0,9.5,0,0,0],"m":[1,1,1,0,0,1,1,1,1,0,0,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,0,0,14,0,16,0,0,19,0,0,22,23,0,0,26,0,0,0],"r":[28,21,18,5,0,13,12,11,10,0,0,0,0,15,0,17,0,0,20,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.002374337,0,0,0,0,-0.01831063,-0.04897854,0.002374337,0.002297135,0,0.002374337,0,-0.004599243,-0.0413623,0,-0.00531649,-0.05323033,0,0,0.05180018,0.00721344,0,0.002374337,0.03907194,-0.01816584]},{"f":[22,-1,22,-1,21,-1,18,18,18,18,18,4,2,-1,-1,-1,-1,-1,-1,15,15,-1,-1,21,-1,-1,18,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,2.5,0,10.651680794759452,9.273793716589307,9.265614570460752,9.26091905453048,9.135888799148791,9.266718101290426,9.051177541662383,0,0,0,0,0,0,8.618385502258606,5.044394119358453,0,0,7.5,0,0,10.682994316786338,0,0],"m":[0,0,0,0,0,0,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,0,3,0,5,0,7,8,9,10,11,12,13,0,0,0,0,0,0,20,21,0,0,24,0,0,27,0,0],"r":[2,0,4,0,6,0,26,19,18,17,16,15,14,0,0,0,0,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,-0.02314795,0,0.0605996,0,-0.02031821,0,0,0,0,0,0,0,0.0001227477,0.01649975,-0.02945901,0.04058596,-0.03882634,0.02707376,0,0,-0.006317172,0.05819963,0,-0.04570328,0.004292899,0,0.07613289,0.004180338]},{"f":[25,27,23,-1,21,-1,18,22,-1,18,23,16,-1,0,-1,-1,-1,22,-1,-1,17,-1,2,-1,-1,33,-1,-1,-1],"t":[6.5,0.5,3.5,0,6.5,0,6.499823868264185,13.179405754316653,0,5.303665156874805,4.5,10.501837184902296,0,4.5,0,0,0,17.069460217922263,0,0,8.496347487943302,0,1.292481250360578,0,0,32.199935296219856,0,0,0],"m":[1,1,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,0,10,11,12,0,14,0,0,0,18,0,0,21,0,23,0,0,26,0,0,0],"r":[28,25,4,0,6,0,20,9,0,17,16,13,0,15,0,0,0,19,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0,0.001134893,0,-0.03385489,0,0,-0.02273701,0,0,0,0.04682833,0,-0.02662347,0.03881443,-0.003071736,0,0.04098085,0.004223899,0,-0.03833507,0,0.02962506,-0.02383783,0,0.03083834,-0.001440019,-0.03546117]},{"f":[8,2,1,-1,15,15,32,15,-1,-1,19,-1,-1,22,-1,16,-1,17,-1,-1,18,17,-1,-1,0,-1,-1,-1,-1],"t":[0.5,10.680359255641626,0.3126984126984127,0,10.457380879072534,9.756556322524087,7.491051153478379,8.076815597050832,0,0,1.5,0,0,10.12257207114862,0,9.92481250360578,0,10.415240582199587,0,0,10.620215166896223,9.27958770372691,0,0,3.5,0,0,0,0],"m":[1,1,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,0,11,0,0,14,0,16,0,18,0,0,21,22,0,0,25,0,0,0,0],"r":[28,27,4,0,20,13,10,9,0,0,12,0,0,15,0,17,0,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.04729026,0,0,0,0,-0.0001991041,-0.01622991,0,-0.01703066,0.03232403,0,0.0726373,0,0.002465464,0,0.08133166,-0.003246547,0,0,-0.005307552,-0.03497255,0,0.007487656,0.0792107,0.03928104,0.01863938]},{"f":[33,32,33,33,37,23,-1,-1,-1,4,15,16,-1,-1,-1,-1,21,33,-1,2,33,-1,-1,24,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,20.752541988539424,13.895530557358477,8.05,0.5,0,0,0,4.80643424864552,7.266786540694901,5.044394119358453,0,0,0,0,9.5,21.16864017939033,0,1.292481250360578,27.893589438525005,0,0,9.5,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,12,0,0,0,0,17,18,0,20,21,0,0,24,0,0,0,0,0],"r":[28,27,16,9,8,7,0,0,0,15,14,13,0,0,0,0,26,19,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0.003577248,-0.002376671,0.03438303,0,0,0,-0.00214203,-0.06805924,-0.00969207,0.02384082,0,0,0.03473951,0,0,-0.005273238,-0.03741992,0,0.003833323,0.054813,0.04690393,-0.05073755,0.01757133]},{"f":[22,-1,22,-1,21,17,4,-1,-1,-1,30,23,-1,4,2,17,17,-1,-1,-1,22,22,-1,-1,-1,-1,21,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,2.5,1.292481250360578,4.951187557243013,0,0,0,1.5,3.5,0,6.451187557243013,4.0437314206251695,8.496347487943302,6.434604125275413,0,0,0,17.078739781790915,17.069460217922263,0,0,0,0,3.5,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,0,0,11,12,0,14,15,16,17,0,0,0,21,22,0,0,0,0,27,0,0],"r":[2,0,4,0,10,9,8,0,0,0,26,13,0,25,20,19,18,0,0,0,24,23,0,0,0,0,28,0,0],"v":[0,-0.02272625,0,0.05668928,0,0,0,-0.02254475,0.02224281,-0.04233469,0,0,0.002482434,0,0,0,0,-0.005700716,-0.03821915,0.01737904,0,0,0.005565345,-0.04110728,0.04160162,-0.03058842,0,-0.05182257,-0.004855111]},{"f":[22,-1,15,15,22,22,22,22,22,-1,15,-1,-1,4,-1,-1,-1,16,22,-1,-1,-1,18,32,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,13.56045225766691,13.323758479706308,17.094543568094064,17.069460217922263,16.978341770814396,16.73920708776685,14.116600814324018,0,10.376125389276174,0,0,3.753897320099348,0,0,0,7.774787059601174,17.078739781790915,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0,0],"m":[0,0,1,1,1,1,1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,11,0,0,14,0,0,0,18,19,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,27,22,17,16,13,10,0,12,0,0,15,0,0,0,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.03611249,0,0,0,0,0,0,0,-0.001037258,0,0.01519008,-0.007656955,0,-0.0237626,0.007648706,0.01774712,0,0,-0.05406541,-0.009209838,0.008465644,0,0,-0.01592934,0.03762167,-0.02386461,0.02318288,-0.01717751]},{"f":[25,22,-1,22,16,-1,-1,1,17,21,15,22,21,-1,-1,-1,22,-1,17,-1,-1,-1,-1,1,1,-1,-1,-1,-1],"t":[6.5,2.69615871138938,0,7.082728314749225,10.681238411777805,0,0,2.01685992705101,11.651826559171445,7.5,7.417852514885898,10.194756329264862,5.5,0,0,0,14.247293309538719,0,2.453445297804259,0,0,0,0,2.969223484848485,2.532051282051282,0,0,0,0],"m":[1,0,0,0,1,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,10,11,12,13,0,0,0,17,0,19,0,0,0,0,24,25,0,0,0,0],"r":[28,3,0,7,6,0,0,23,22,21,16,15,14,0,0,0,18,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,-0.03570673,0,0,0.00137873,0.03366261,0,0,0,0,0,0,0.07896909,-0.02110503,-0.04260297,0,-0.0178005,0,-0.03810966,0.05149144,1.244076e-05,0.02250619,0,0,0.00149073,0.06162453,0.0004161218,-0.03397166]},{"f":[33,32,33,22,22,1,16,-1,-1,1,-1,-1,18,-1,-1,17,22,-1,-1,-1,33,22,37,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,29.297756302860655,12.199665042528586,10.27595502635321,1.13125,10.46352437327118,0,0,2.01685992705101,0,0,10.609627990977145,0,0,0.5,15.717370311433935,0,0,0,39.9785486130134,16.98920034885663,9.629999999999999,0,0,0,0,0,0],"m":[1,1,1,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,0,0,13,0,0,16,17,0,0,0,21,22,23,0,0,0,0,0,0],"r":[28,27,20,15,12,9,8,0,0,11,0,0,14,0,0,19,18,0,0,0,26,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0.004270737,0.04692356,0,-0.03020038,0.001639145,0,-0.01461203,0.04533336,0,0,-0.03929696,0.04715554,0.002448613,0,0,0,0.01650566,0.04842568,-0.01015498,-0.02285446,-0.04992887,0.01664914]},{"f":[22,-1,25,15,37,36,23,-1,2,21,22,-1,-1,19,-1,-1,-1,17,-1,-1,33,15,-1,-1,36,-1,-1,-1,-1],"t":[2.69615871138938,0,6.5,13.56045225766691,7.92,3.5,0.5,0,3.5799356683891945,6.5,16.964783007086943,0,0,0.5,0,0,0,7.198837316474133,0,0,15.81450217830399,5.169925001442312,0,0,5.5,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,11,0,0,14,0,0,0,18,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,28,27,20,17,8,0,16,13,12,0,0,15,0,0,0,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.0352937,0,0,0,0,0,0.002413458,0,0,0,-0.03520479,0.002413458,0,0.002413458,-0.016439,0.0004723459,0,-0.005069178,-0.05217482,0,0,0.05170425,0.004999724,0,0.002413458,0.03809208,-0.01652327,-0.03319]},{"f":[2,8,1,-1,22,21,1,-1,1,-1,-1,-1,22,18,1,-1,-1,36,22,-1,-1,15,-1,-1,22,-1,-1,-1,-1],"t":[10.680359255641626,0.5,0.3126984126984127,0,12.199665042528586,5.5,1.000936329588015,0,3.354166666666667,0,0,0,12.921158001713904,1.292481250360578,6.806765389082463,0,0,2.5,12.613886328427084,0,0,10.457380879072534,0,0,13.415167820124083,0,0,0,0],"m":[1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,0,13,14,15,0,0,18,19,0,0,22,0,0,25,0,0,0,0],"r":[28,27,4,0,12,11,8,0,10,0,0,0,24,17,16,0,0,21,20,0,0,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.04658577,0,0,0,0.05527156,0,-0.04178605,-0.008136349,0.0007364046,0,0,0,0.009859745,-0.04234059,0,0,-0.02946419,0.04828823,0,0.03026439,-0.02794717,0,-0.02978623,0.001300487,0.01919271,0.03072546]},{"f":[2,21,22,22,-1,-1,-1,1,-1,4,4,1,-1,-1,-1,4,-1,-1,3,3,-1,17,-1,-1,16,16,-1,-1,-1],"t":[9.051177541662383,2.5,13.827286379004683,6.106096185954236,0,0,0,0.39811320754716983,0,7.950787011501063,7.408266855279292,4.92309205350118,0,0,0,8.950330235806282,0,0,0.9640400250156348,0.00554530545305453,0,9.44286754736589,0,0,11.031356596255709,4.247927513443585,0,0,0],"m":[1,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,0,8,0,10,11,12,0,0,0,16,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[18,7,6,5,0,0,0,9,0,15,14,13,0,0,0,17,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0.01606443,-0.04576444,0.02207317,0,-0.04322778,0,0,0,-0.002904912,0.002317431,0.02181977,0,-0.04028141,0.002935224,0,0,-0.0120038,0,0.06067694,0.003668424,0,0,-0.003980463,0.02937165,-0.03344942]},{"f":[22,-1,22,15,4,21,-1,-1,22,-1,22,0,-1,-1,15,-1,-1,-1,22,-1,22,-1,17,22,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.770639075561187,10.066089190457772,1.292481250360578,6.5,0,0,5.9525054311951475,0,6.671801068872866,6.5,0,0,8.361943773735241,0,0,0,8.133841411710264,0,12.199665042528586,0,0.5,15.717370311433935,0,0,9.012591414317754,0,0],"m":[0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,5,6,0,0,9,0,11,12,0,0,15,0,0,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,18,17,8,7,0,0,10,0,14,13,0,0,16,0,0,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.0348474,0,0,0,0,-0.03992865,0.004595526,0,0.04870009,0,0,-0.04307294,-0.0003253425,0,0.01270228,-0.03122984,0.03873067,0,-0.04189229,0,-0.00295863,0,0,-0.03859241,0.0452805,0,0.001277438,0.01955989]},{"f":[33,32,33,3,2,-1,2,21,-1,-1,-1,23,-1,21,-1,18,-1,-1,33,22,37,-1,-1,22,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,29.297756302860655,0.32902480033627574,6.055241654908112,0,9.075473573191719,7.5,0,0,0,3.5,0,6.5,0,6.499823868264185,0,0,39.9785486130134,16.98920034885663,9.629999999999999,0,0,17.078739781790915,0,0,0,0,0],"m":[1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,12,0,14,0,16,0,0,19,20,21,0,0,24,0,0,0,0,0],"r":[28,27,18,11,6,0,10,9,0,0,0,13,0,15,0,17,0,0,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.04444242,0,0,-0.03354237,-0.002899533,0.03339488,0,0.002084142,0,-0.02795478,0,0.006215366,-0.0176704,0,0,0,0.0155726,0.04674452,0,-0.03162319,0.01221448,-0.02201819,-0.04932598,0.01616105]},{"f":[22,23,21,4,32,15,15,22,-1,-1,-1,-1,-1,2,22,-1,-1,24,-1,-1,-1,-1,18,32,-1,17,-1,-1,-1],"t":[17.094543568094064,7.5,10.5,3.084962500721156,10.463013322934223,7.924812503605781,7.721099188707185,14.03140051979143,0,0,0,0,0,3.245926548164837,13.415167820124083,0,0,9.5,0,0,0,0,8.729619742794132,8.341935934311302,0,7.588709768994619,0,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,0,14,15,0,0,18,0,0,0,0,23,24,0,26,0,0,0],"r":[22,21,20,13,12,11,10,9,0,0,0,0,0,17,16,0,0,19,0,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,0,0,-0.01061683,0.007784138,-0.04411773,0.005209784,-0.03398798,0,0,0.0008644373,0.0406258,0,-0.0001781357,0.03884771,0.04138301,-0.02440777,0,0,-0.01488895,0,0.01316324,0.05681217,-0.02277068]},{"f":[27,15,22,-1,-1,15,-1,1,18,-1,22,22,-1,-1,-1,15,25,-1,-1,18,19,-1,-1,24,-1,-1,33,-1,-1],"t":[0.5,5.78135971352466,16.819976309430963,0,0,6.108524456778169,0,4.92309205350118,10.620215166896223,0,12.457649623491124,11.228801658478353,0,0,0,6.857980995127572,1.5,0,0,6.864066550153108,2.5,0,0,9.5,0,0,32.199935296219856,0,0],"m":[1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,9,0,11,12,0,0,0,16,17,0,0,20,21,0,0,24,0,0,27,0,0],"r":[26,5,4,0,0,7,0,15,10,0,14,13,0,0,0,19,18,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0.001035242,0.04265881,0,-0.03188543,0,0,-0.009290596,0,0,-0.005644319,0.06624457,-0.006614333,0,0,0.003197538,0.03481111,0,0,0.002501974,0.05039567,0,-0.05072342,-0.00585167,0,0.02879898,-0.001896365]},{"f":[2,8,1,-1,15,15,-1,22,-1,16,-1,17,-1,-1,21,33,15,15,-1,-1,-1,-1,23,-1,22,-1,-1,-1,-1],"t":[10.680359255641626,0.5,0.3126984126984127,0,10.457380879072534,9.756556322524087,0,10.12257207114862,0,9.92481250360578,0,10.415240582199587,0,0,5.5,44.79706586676442,10.46352437327118,10.462502272597265,0,0,0,0,0.5,0,16.73920708776685,0,0,0,0],"m":[1,1,0,0,1,1,0,0,0,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,10,0,12,0,0,15,16,17,18,0,0,0,0,23,0,25,0,0,0,0],"r":[28,27,4,0,14,7,0,9,0,11,0,13,0,0,22,21,20,19,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0,-0.04593672,0,0,-0.0004167475,0,0.06683724,0,0.002277786,0,0.07594551,-0.00400826,0,0,0,0,-0.03010192,0.0142536,-0.05115349,0.01516818,0,0.03467046,0,-0.008392041,0.02457925,0.01800276,0.03007879]},{"f":[22,-1,15,15,22,22,22,22,22,-1,-1,4,-1,22,-1,-1,-1,18,-1,18,-1,-1,18,1,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,13.56045225766691,13.323758479706308,17.094543568094064,17.069460217922263,16.978341770814396,16.73920708776685,16.25454052700782,0,0,3.753897320099348,0,16.819976309430963,0,0,0,6.076776015854056,0,7.383290334558488,0,0,8.729619742794132,7.9752100840336135,0,0,0,0,0],"m":[0,0,1,1,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,0,12,0,14,0,0,0,18,0,20,0,0,23,24,0,0,0,0,0],"r":[2,0,28,27,22,17,16,11,10,0,0,13,0,15,0,0,0,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.03450026,0,0,0,0,0,0,0,0.0002936113,0.02296362,0,-0.02263418,0,-0.01793057,0.02168535,0.01732284,0,-0.03491052,0,0.03177174,-0.02586941,0,0,0.03693904,-0.008089789,-0.02241954,0.02159721,-0.01561999]},{"f":[22,-1,25,2,8,1,-1,15,15,-1,22,-1,16,-1,-1,18,17,-1,-1,0,15,-1,-1,22,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,6.5,10.680359255641626,0.5,0.3126984126984127,0,10.457380879072534,9.756556322524087,0,10.12257207114862,0,9.92481250360578,0,0,10.620215166896223,9.27958770372691,0,0,2.5,11.683433292835403,0,0,13.589275375459419,0,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,0,11,0,13,0,0,16,17,0,0,20,21,0,0,24,0,0,0,0,0],"r":[2,0,28,27,26,7,0,15,10,0,12,0,14,0,0,19,18,0,0,23,22,0,0,25,0,0,0,0,0],"v":[0,-0.03409807,0,0,0,0,-0.04539693,0,0,-0.0001328235,0,0.06194904,0,0.002144281,0.0266071,0,0,-0.004803607,-0.0339822,0,0,-0.0429613,0.01509952,0,0.08312032,0.02578596,0.01715231,0.02952665,-0.03189176]},{"f":[4,32,4,15,-1,16,17,17,16,15,18,-1,-1,-1,-1,-1,-1,22,-1,-1,-1,-1,33,2,15,-1,-1,-1,-1],"t":[3.084962500721156,10.463013322934223,2.903677461028802,2.0,0,10.501837184902296,10.37503779525057,9.075473573191719,7.930737337562887,7.118941072723508,1.792481250360578,0,0,0,0,0,0,11.683871801979876,0,0,0,0,41.64100553123272,3.245926548164837,8.005624549193879,0,0,0,0],"m":[1,1,1,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,11,0,0,0,0,0,0,18,0,0,0,0,23,24,25,0,0,0,0],"r":[22,21,20,5,0,17,16,15,14,13,12,0,0,0,0,0,0,19,0,0,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,-0.03642264,0,0,0,0,0,0,-0.008782032,0.01208001,-0.01539432,0.01115627,0.03854479,-0.0304719,0,0.03617102,-0.0002818038,-0.04304733,-0.03282471,0,0,0,0.02929073,-0.04577661,0.0001435927,0.03697725]},{"f":[27,15,22,-1,-1,15,-1,1,21,-1,-1,15,33,17,-1,-1,-1,18,-1,24,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,5.78135971352466,16.819976309430963,0,0,6.108524456778169,0,4.92309205350118,6.5,0,0,6.857980995127572,21.257266042509173,8.729619742794132,0,0,0,6.864066550153108,0,9.5,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,0,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,9,0,0,12,13,14,0,0,0,18,0,20,0,0,23,0,25,0,27,0,0],"r":[22,5,4,0,0,7,0,11,10,0,0,17,16,15,0,0,0,19,0,21,0,0,24,0,26,0,28,0,0],"v":[0,0,0,0.001030086,0.03907527,0,-0.0308897,0,0,-0.02259926,-0.001792785,0,0,0,0.0416159,-0.01868563,0.0006853423,0,0.003672269,0,-0.04876935,-0.005876308,0,0.02794371,0,-0.02058915,0,0.02694352,-0.01717436]},{"f":[4,2,2,-1,-1,3,-1,-1,15,16,3,-1,23,22,15,-1,-1,-1,-1,26,-1,-1,22,18,-1,18,-1,-1,-1],"t":[4.951187557243013,8.612669628062957,7.005613627711627,0,0,0.00554530545305453,0,0,10.376125389276174,6.882643049361842,0.5803571428571428,0,4.5,14.03140051979143,5.78135971352466,0,0,0,0,0.5,0,0,16.725912049439096,1.292481250360578,0,9.273793716589307,0,0,0],"m":[1,1,1,0,0,1,0,0,1,1,0,0,1,1,1,0,0,0,0,0,0,0,1,1,0,1,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,14,15,0,0,0,0,20,0,0,23,24,0,26,0,0,0],"r":[8,5,4,0,0,7,0,0,22,19,12,0,18,17,16,0,0,0,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0,0,-0.00146533,-0.03427049,0,-0.01129313,0.05030356,0,0,0,-0.03543203,0,0,0,0.006005043,-0.01800662,0.04110833,-0.02266304,0,0.0411355,0.009645151,0,0,-0.03721798,0,0.002133525,-0.03131865,0.01883526]},{"f":[22,-1,22,-1,21,17,4,-1,-1,-1,30,37,15,-1,16,-1,22,16,-1,-1,0,-1,-1,24,-1,-1,21,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,2.5,1.292481250360578,4.951187557243013,0,0,0,1.5,11.26,5.169925001442312,0,6.108524456778169,0,12.199665042528586,7.6653359171851765,0,0,4.5,0,0,12.5,0,0,3.5,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,0,0,11,12,13,0,15,0,17,18,0,0,21,0,0,24,0,0,27,0,0],"r":[2,0,4,0,10,9,8,0,0,0,26,23,14,0,16,0,20,19,0,0,22,0,0,25,0,0,28,0,0],"v":[0,-0.0203116,0,0.05136133,0,0,0,-0.02133435,0.02094251,-0.04151091,0,0,0,0.003956935,0,-0.01284733,0,0,-0.02160912,-0.000993997,0,-0.008085439,0.00685698,0,0.03807201,0.007949522,0,-0.04891911,-0.004511346]},{"f":[21,17,32,-1,16,-1,22,-1,-1,4,22,16,21,-1,-1,16,22,2,-1,-1,-1,-1,-1,-1,22,0,-1,-1,-1],"t":[9.5,1.292481250360578,3.753897320099348,0,3.169925001442312,0,16.786010507999382,0,0,10.41668595978746,12.121442107629743,10.46352437327118,4.5,0,0,11.868050853745158,11.67903991803059,2.453445297804259,0,0,0,0,0,0,15.756821747580634,7.5,0,0,0],"m":[1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,12,13,0,0,16,17,18,0,0,0,0,0,0,25,26,0,0,0],"r":[24,9,4,0,6,0,8,0,0,23,22,15,14,0,0,21,20,19,0,0,0,0,0,0,28,27,0,0,0],"v":[0,0,0,-0.03969274,0,-0.01581501,0,0.002800823,-0.04287218,0,0,0,0,-0.04186954,-0.007481976,0,0,0,-0.03025258,0.0302263,0.0651291,-0.004238073,0.003730774,0.04607086,0,0,0.007644117,0.04667794,-0.03674767]},{"f":[27,21,16,17,15,-1,-1,-1,4,2,2,-1,16,-1,-1,-1,-1,22,0,-1,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,9.5,1.584962500721156,2.160964047443681,3.4594316186372973,0,0,0,9.328332883600808,9.293467526905953,9.075473573191719,0,10.826548487290914,0,0,0,0,15.756821747580634,7.5,0,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,1,0,1,1,0,0,0,1,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,0,9,10,11,0,13,0,0,0,0,18,19,0,0,0,23,0,25,0,27,0,0],"r":[22,17,8,7,6,0,0,0,16,15,12,0,14,0,0,0,0,21,20,0,0,0,24,0,26,0,28,0,0],"v":[0,0,0,0,0,-0.03887765,-0.002588995,0.009320053,0,0,0,-0.0008234847,0,0.003289547,0.05154509,-0.04013943,0.02040004,0,0,0.007252843,0.04612885,-0.03596093,0,0.02704894,0,-0.0194308,0,0.026615,-0.01599409]},{"f":[27,15,22,15,21,18,-1,-1,-1,16,17,-1,22,22,16,-1,-1,-1,-1,18,-1,-1,-1,15,-1,-1,33,-1,-1],"t":[0.5,5.78135971352466,16.819976309430963,4.087462841250339,7.5,1.792481250360578,0,0,0,6.409390936137702,1.292481250360578,0,13.054613167076562,12.613886328427084,4.247927513443585,0,0,0,0,3.084962500721156,0,0,0,6.108524456778169,0,0,32.199935296219856,0,0],"m":[1,0,1,1,1,1,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,0,13,14,15,0,0,0,0,20,0,0,0,24,0,0,27,0,0],"r":[26,23,22,9,8,7,0,0,0,19,12,0,18,17,16,0,0,0,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.02381931,0.005381404,0.003122376,0,0,0.02190813,0,0,0,-0.02637937,0.005978022,0.04486072,-0.01902183,0,0.009360407,0.05838593,0.03693146,0,-0.0294905,-0.0006045397,0,0.02576842,-0.001151966]},{"f":[22,23,21,4,32,15,-1,17,15,22,-1,-1,-1,-1,-1,24,3,-1,-1,-1,-1,-1,18,32,-1,17,-1,-1,-1],"t":[17.094543568094064,7.5,10.5,2.453445297804259,10.463013322934223,2.0,0,0.5,9.862637357558794,8.643851681338631,0,0,0,0,0,9.5,0.0339080459770115,0,0,0,0,0,8.729619742794132,8.341935934311302,0,7.588709768994619,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,6,0,8,9,10,0,0,0,0,0,16,17,0,0,0,0,0,23,24,0,26,0,0,0],"r":[22,21,20,15,14,7,0,13,12,11,0,0,0,0,0,19,18,0,0,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,-0.03771356,0,0,0,0.01735183,-0.0390538,0.04532453,-0.002345566,-0.03041858,0,0,0.03684941,0.0003128534,0.03673299,0.03830928,-0.02334418,0,0,-0.01258525,0,0.01218302,0.05320149,-0.02233006]},{"f":[25,4,4,32,15,-1,4,2,18,17,16,-1,-1,-1,-1,37,-1,-1,-1,-1,-1,33,2,15,-1,-1,-1,-1,-1],"t":[6.5,3.084962500721156,2.903677461028802,10.463013322934223,2.0,0,2.453445297804259,1.792481250360578,3.245926548164837,1.792481250360578,7.930737337562887,0,0,0,0,6.335,0,0,0,0,0,41.64100553123272,3.245926548164837,8.005624549193879,0,0,0,0,0],"m":[1,1,1,1,0,0,1,1,0,1,1,0,0,0,0,1,0,0,0,0,0,1,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,9,10,11,0,0,0,0,16,0,0,0,0,0,22,23,24,0,0,0,0,0],"r":[28,21,20,19,6,0,18,15,14,13,12,0,0,0,0,17,0,0,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03501134,0,0,0,0,0,-0.007854834,0.020726,0.02703403,-0.005615446,0,-0.03310545,0.001009192,0.02404281,-0.03046935,-0.04253979,0,0,0,0.02753548,-0.045023,0.0002251664,0.03502533,-0.03084417]},{"f":[22,23,21,15,-1,3,-1,21,33,-1,-1,23,-1,22,-1,-1,-1,22,33,-1,-1,-1,18,32,-1,17,-1,-1,-1],"t":[17.094543568094064,7.5,10.5,10.457380879072534,0,0.26136363636363635,0,5.5,37.46769166150554,0,0,0.5,0,16.759160379664443,0,0,0,16.98920034885663,26.622428940176302,0,0,0,8.729619742794132,8.341935934311302,0,8.571721503073015,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,9,0,0,12,0,14,0,0,0,18,19,0,0,0,23,24,0,26,0,0,0],"r":[22,17,16,5,0,7,0,11,10,0,0,13,0,15,0,0,0,21,20,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0.0007866337,0,0.02917926,0,0,-0.03165145,0.0002664779,0,0.02907745,0,-0.00933823,0.03652526,0.03578635,0,0,-0.02635081,0.02512382,-0.03843597,0,0,-0.0114683,0,0.01295004,0.0541117,-0.02146951]},{"f":[4,2,2,-1,-1,3,-1,-1,15,16,-1,18,2,1,-1,-1,1,-1,-1,18,-1,-1,22,18,-1,18,-1,-1,-1],"t":[4.951187557243013,8.612669628062957,7.005613627711627,0,0,0.00554530545305453,0,0,10.376125389276174,7.651051691178929,0,6.076776015854056,6.033380966193453,8.102912599421547,0,0,1.987429289754871,0,0,8.03615005796417,0,0,16.725912049439096,1.292481250360578,0,9.273793716589307,0,0,0],"m":[1,1,1,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,13,14,0,0,17,0,0,20,0,0,23,24,0,26,0,0,0],"r":[8,5,4,0,0,7,0,0,22,11,0,19,16,15,0,0,18,0,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0,0,-0.001348156,-0.03259894,0,-0.01118427,0.04722752,0,0,0.0002006886,0,0,0,0.06392461,0.0005316531,0,0.01392467,-0.02292956,0,0.0406101,0.003673919,0,0,-0.03586346,0,0.001891895,-0.03084241,0.01771647]},{"f":[27,22,22,-1,-1,22,-1,17,16,-1,-1,23,22,-1,-1,1,-1,1,-1,33,-1,16,22,-1,-1,-1,33,-1,-1],"t":[0.5,16.786010507999382,16.765402642760364,0,0,16.807008825964132,0,3.39067985676233,3.321928094887362,0,0,2.5,16.838461650489297,0,0,1.282051282051282,0,3.015625,0,23.450447293378375,0,7.924812503605781,17.078739781790915,0,0,0,32.199935296219856,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,9,0,0,12,13,0,0,16,0,18,0,20,0,22,23,0,0,0,27,0,0],"r":[26,5,4,0,0,7,0,11,10,0,0,15,14,0,0,17,0,19,0,21,0,25,24,0,0,0,28,0,0],"v":[0,0,0,-4.580222e-05,0.05116745,0,-0.03499084,0,0,-0.003094783,-0.04449653,0,0,0.01057429,0.05996108,0,0.04495647,0,-0.03503925,0,-0.02139206,0,0,-0.02403295,0.01145945,0.02192115,0,0.0251988,-0.001252562]},{"f":[25,22,4,15,-1,23,15,-1,-1,32,-1,-1,35,22,22,-1,22,-1,23,-1,-1,-1,-1,17,22,-1,-1,-1,-1],"t":[6.5,12.199665042528586,1.292481250360578,9.862637357558794,0,0.5,10.681238411777805,0,0,10.463013322934223,0,0,2.5,9.40390385294334,8.61260371847175,0,8.643851681338631,0,0.5,0,0,0,0,0.5,15.717370311433935,0,0,0,0],"m":[1,0,0,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,15,0,17,0,19,0,0,0,0,24,25,0,0,0,0],"r":[28,23,12,5,0,9,8,0,0,11,0,0,22,21,16,0,18,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02845477,0,0,0.07609298,0.005413449,0,-0.001088143,-0.0434042,0,0,0,-0.0002853763,0,0.05841484,0,-0.02240015,0.03698431,-0.007016873,0.02880697,0,0,-0.03750356,0.03998551,0.002322317,-0.03046169]},{"f":[2,37,32,23,36,-1,-1,2,21,22,23,21,-1,-1,-1,-1,19,2,-1,-1,-1,-1,18,-1,-1,21,-1,-1,-1],"t":[10.680359255641626,7.92,6.6765734127490415,0.5,2.5,0,0,4.0437314206251695,7.5,16.978341770814396,2.5,5.5,0,0,0,0,1.5,1.792481250360578,0,0,0,0,2.160964047443681,0,0,9.5,0,0,0],"m":[1,1,1,0,0,0,0,1,1,1,1,1,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,8,9,10,11,12,0,0,0,0,17,18,0,0,0,0,23,0,0,26,0,0,0],"r":[28,25,22,7,6,0,0,21,16,15,14,13,0,0,0,0,20,19,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.02812402,0.00132493,0,0,0,0,0,-0.03188234,-0.002490756,-0.04846759,0.00132493,0,0,0.00132493,-0.04103839,-0.03618565,-0.0003603635,0,-0.004077653,-0.04622951,0,0.002575392,0.03504305,0.02787434]},{"f":[22,-1,22,1,-1,15,-1,16,-1,-1,1,17,18,17,21,15,-1,-1,-1,-1,-1,-1,1,1,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.082728314749225,1.000936329588015,0,3.169925001442312,0,5.614709844115208,0,0,2.01685992705101,11.651826559171445,11.26283925556662,11.207174616271544,7.5,9.92481250360578,0,0,0,0,0,0,2.9611191860465116,2.532051282051282,0,0,9.299206200930918,0,0],"m":[0,0,0,0,0,1,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,12,13,14,15,16,0,0,0,0,0,0,23,24,0,0,27,0,0],"r":[2,0,10,5,0,7,0,9,0,0,22,21,20,19,18,17,0,0,0,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.03239367,0,0,0.02018731,0,-0.03587803,0,0.0634071,0.00104425,0,0,0,0,0,0,-0.02581246,-0.0005323232,-0.0001515717,0.0443586,-0.05209633,0.01835752,0,0,-0.0005926011,0.06243899,0,-0.0001253904,0.03017675]},{"f":[25,22,4,15,-1,23,15,-1,-1,33,-1,-1,35,1,-1,22,22,-1,-1,-1,36,-1,-1,17,22,-1,-1,-1,-1],"t":[6.5,12.199665042528586,1.292481250360578,9.862637357558794,0,0.5,10.681238411777805,0,0,44.805779984159756,0,0,2.5,0.01933482600988898,0,9.40390385294334,8.61260371847175,0,0,0,3.5,0,0,0.5,15.717370311433935,0,0,0,0],"m":[1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,0,16,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,12,5,0,9,8,0,0,11,0,0,20,15,0,19,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02755856,0,0,0.06891497,0.004914856,0,-0.001197858,-0.04355167,0,0,0.04063529,0,0,-0.0003675541,0.02587335,-0.008045766,0,0.008725815,0.04978249,0,0,-0.03675614,0.03842542,0.002311892,-0.02998506]},{"f":[2,1,-1,21,22,22,-1,-1,-1,4,4,-1,-1,4,-1,-1,20,2,16,-1,-1,18,21,-1,-1,15,-1,-1,-1],"t":[9.051177541662383,0.39811320754716983,0,2.5,13.827286379004683,6.106096185954236,0,0,0,7.950787011501063,7.408266855279292,0,0,8.950330235806282,0,0,1.5,9.241970852671834,10.372865060112588,0,0,10.457893746154532,7.5,0,0,9.862637357558794,0,0,0],"m":[1,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,0,0,0,10,11,0,0,14,0,0,17,18,19,0,0,22,23,0,0,26,0,0,0],"r":[16,3,0,9,8,7,0,0,0,13,12,0,0,15,0,0,28,21,20,0,0,25,24,0,0,27,0,0,0],"v":[0,0,-0.04186884,0,0,0,0.01769125,-0.04515822,0.02227478,0,0,0.0001868565,0.01840334,0,-0.03962944,0.003375661,0,0,0,0.005754393,0.0399687,0,0,-0.0407683,0.002770523,0,0.04105563,-0.01852386,0.04236688]},{"f":[22,-1,33,21,36,4,-1,-1,-1,24,18,-1,-1,33,-1,18,-1,-1,33,-1,37,32,-1,-1,2,-1,35,-1,-1],"t":[2.69615871138938,0,20.752541988539424,3.5,3.5,4.128693921346326,0,0,0,7.5,6.531360383582522,0,0,13.895530557358477,0,4.245926548164837,0,0,21.16864017939033,0,9.205,10.41931383093672,0,0,1.292481250360578,0,2.5,0,0],"m":[0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,11,0,0,14,0,16,0,0,19,0,21,22,0,0,25,0,27,0,0],"r":[2,0,18,9,8,7,0,0,0,13,12,0,0,15,0,17,0,0,20,0,24,23,0,0,26,0,28,0,0],"v":[0,-0.03195651,0,0,0,0,-0.06372255,0.006175873,-0.0077718,0,0,-0.00163461,-0.02773952,0,0.04538962,0,-0.01592748,0.02601571,0,0.03076367,0,0,0.0004107926,-0.01870973,0,0.00528633,0,0.01051736,0.05750603]},{"f":[27,23,-1,21,-1,18,22,-1,18,23,16,-1,-1,-1,22,-1,22,-1,-1,1,-1,22,16,-1,-1,-1,33,-1,-1],"t":[0.5,3.5,0,6.5,0,6.499823868264185,13.179405754316653,0,5.303665156874805,4.5,10.501837184902296,0,0,0,17.069460217922263,0,17.074218326240924,0,0,1.99834912080135,0,17.094543568094064,8.618385502258606,0,0,0,32.199935296219856,0,0],"m":[1,1,0,0,0,1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,0,4,0,6,7,0,9,10,11,0,0,0,15,0,17,0,0,20,0,22,23,0,0,0,27,0,0],"r":[26,3,0,5,0,19,8,0,14,13,12,0,0,0,16,0,18,0,0,21,0,25,24,0,0,0,28,0,0],"v":[0,0,0.000824726,0,-0.02963807,0,0,-0.02040781,0,0,0,0.04244725,-0.00792386,-0.003179793,0,0.03775435,0,-0.02945221,0.03636092,0,0.03644093,0,0,-0.04708406,-0.003223382,0.003048554,0,0.02440747,-0.001498673]},{"f":[4,4,21,22,22,-1,-1,-1,2,1,-1,4,4,30,-1,21,-1,-1,-1,4,-1,-1,22,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,13.87915215758784,6.106096185954236,0,0,0,9.051177541662383,0.39811320754716983,0,7.754129516000594,7.408266855279292,1.5,0,3.5,0,0,0,8.950330235806282,0,0,13.005894632618489,0,0,0,2.160964047443681,0,0],"m":[1,1,0,1,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,0,12,13,14,0,16,0,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,8,7,6,0,0,0,22,11,0,19,18,15,0,17,0,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0.01694725,-0.04493482,0.02167449,0,0,-0.04129783,0,0,0,0.0007716419,0,-0.04556525,-0.005025756,0.01990549,0,-0.03398605,0.003076303,0,0.00313436,0.03812477,-0.01562184,0,-0.007925638,0.04221917]},{"f":[22,-1,22,1,-1,15,-1,16,-1,-1,1,22,-1,22,21,-1,-1,21,-1,17,-1,-1,1,1,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.082728314749225,1.000936329588015,0,3.169925001442312,0,5.614709844115208,0,0,2.01685992705101,8.133841411710264,0,9.087343393278285,7.5,0,0,3.5,0,11.364132993990529,0,0,2.969223484848485,2.532051282051282,0,0,9.299206200930918,0,0],"m":[0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,12,0,14,15,0,0,18,0,20,0,0,23,24,0,0,27,0,0],"r":[2,0,10,5,0,7,0,9,0,0,22,13,0,17,16,0,0,19,0,21,0,0,26,25,0,0,28,0,0],"v":[0,-0.03158437,0,0,0.01914193,0,-0.03533647,0,0.06007071,0.0009043419,0,0,-0.03723484,0,0,0.09265711,0.008415869,0,-0.04255008,0,-0.005447855,0.01721488,0,0,-0.0004495164,0.05519666,0,-0.00016906,0.02901499]},{"f":[22,-1,22,18,1,-1,-1,36,22,-1,-1,15,16,2,1,-1,-1,22,17,-1,-1,-1,-1,-1,22,-1,15,-1,-1],"t":[12.199665042528586,0,12.921158001713904,1.292481250360578,6.806765389082463,0,0,2.5,12.613886328427084,0,0,10.457380879072534,8.321928094887362,1.292481250360578,7.0046728971962615,0,0,12.679695715883515,11.364132993990529,0,0,0,0,0,13.415167820124083,0,7.118941072723508,0,0],"m":[0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,14,15,0,0,18,19,0,0,0,0,0,25,0,27,0,0],"r":[2,0,24,7,6,0,0,11,10,0,0,23,22,17,16,0,0,21,20,0,0,0,0,0,26,0,28,0,0],"v":[0,-0.001813116,0,0,0,0.007316444,-0.04011865,0,0,-0.02735308,0.04557229,0,0,0,0,0.07023691,0.01833549,0,0,-0.01525961,0.0266005,0.03245897,0.06241558,-0.02431888,0,-0.02783277,0,0.006726407,-0.002320807]},{"f":[27,23,-1,21,-1,18,22,-1,18,-1,-1,1,-1,22,16,-1,-1,17,-1,-1,22,22,0,-1,-1,-1,22,-1,-1],"t":[0.5,3.5,0,6.5,0,6.499823868264185,13.179405754316653,0,5.303665156874805,0,0,1.99834912080135,0,17.094543568094064,8.618385502258606,0,0,8.729619742794132,0,0,16.98920034885663,16.978341770814396,8.5,0,0,0,17.078739781790915,0,0],"m":[1,1,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,0,4,0,6,7,0,9,0,0,12,0,14,15,0,0,18,0,0,21,22,23,0,0,0,27,0,0],"r":[20,3,0,5,0,11,8,0,10,0,0,13,0,17,16,0,0,19,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0.0007819701,0,-0.02857849,0,0,-0.01943657,0,0.001000318,0.02340619,0,0.03517011,0,0,-0.04644577,-0.002915101,0,0.01959187,-0.0281862,0,0,0,-0.02775744,0.0186644,0.07011168,0,-0.02898187,0.01858902]},{"f":[25,1,15,22,22,-1,-1,-1,15,-1,18,1,-1,2,16,23,-1,-1,-1,2,-1,-1,1,-1,-1,1,-1,-1,-1],"t":[6.5,4.92309205350118,5.78135971352466,16.819976309430963,16.73920708776685,0,0,0,7.266786540694901,0,10.620215166896223,1.987429289754871,0,3.245926548164837,7.741466986401147,1.5,0,0,0,6.139515068224824,0,0,3.015625,0,0,4.949128919860627,0,0,0],"m":[1,1,0,1,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,0,11,12,0,14,15,16,0,0,0,20,0,0,23,0,0,26,0,0,0],"r":[28,25,8,7,6,0,0,0,10,0,22,13,0,19,18,17,0,0,0,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0,0.002199906,-0.02030799,0.03018442,0,-0.02550686,0,0,0.01059011,0,0,0,-0.02162832,0.05591538,-0.04292744,0,0.01165886,-0.008210649,0,0.004111009,0.06448319,0,0.04286975,0.001379564,-0.02920992]},{"f":[22,-1,22,1,-1,1,-1,18,0,-1,-1,22,-1,16,-1,-1,22,-1,15,-1,3,-1,18,22,-1,-1,0,-1,-1],"t":[2.69615871138938,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,3.084962500721156,6.5,0,0,6.870153907126159,0,6.6865005271832185,0,0,8.133841411710264,0,10.457380879072534,0,0.26136363636363635,0,10.620215166896223,16.725912049439096,0,0,2.5,0,0],"m":[0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,9,0,0,12,0,14,0,0,17,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,16,5,0,7,0,11,10,0,0,13,0,15,0,0,18,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.03115883,0,0,0.01786936,0,-0.02449956,0,0,-0.03491359,0.0003025789,0,0.05139884,0,-0.002607587,0.0210302,0,-0.04041813,0,0.0008830353,0,0.02683725,0,0,-0.01340833,0.01318489,0,-0.004594597,0.04559774]},{"f":[33,32,33,33,3,2,-1,2,21,-1,-1,3,-1,-1,18,21,-1,-1,-1,22,37,-1,-1,22,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,29.297756302860655,0.32902480033627574,6.055241654908112,0,8.612669628062957,7.5,0,0,0.00554530545305453,0,0,1.792481250360578,7.5,0,0,0,16.98920034885663,9.629999999999999,0,0,17.078739781790915,0,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,15,16,0,0,0,20,21,0,0,24,0,0,0,0,0],"r":[28,27,26,19,14,7,0,11,10,0,0,13,0,0,18,17,0,0,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.04155677,0,0,-0.04578689,-0.002924064,0,-0.01310025,0.04458219,0,0,-0.007510666,0.005314759,0.002786034,0,0,0.01424358,0.04329861,0,-0.0296961,0.01285596,-0.02273425,-0.04610887,0.01356457]},{"f":[22,-1,22,16,22,36,-1,18,22,-1,-1,2,24,-1,-1,17,-1,-1,15,-1,-1,15,-1,-1,22,-1,15,-1,-1],"t":[12.199665042528586,0,12.921158001713904,7.930737337562887,12.613886328427084,2.5,0,1.792481250360578,12.278990886161445,0,0,1.292481250360578,4.5,0,0,11.364132993990529,0,0,4.247927513443585,0,0,10.457380879072534,0,0,13.415167820124083,0,7.118941072723508,0,0],"m":[0,0,0,1,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,0,0,12,13,0,0,16,0,0,19,0,0,22,0,0,25,0,27,0,0],"r":[2,0,24,21,18,7,0,11,10,0,0,15,14,0,0,17,0,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,-0.00177763,0,0,0,0,-0.02647477,0,0,-0.03421813,0.02023357,0,0,0.02128815,0.06526815,0,-0.03923757,0.02538865,0,0.04554109,-0.0008853601,0,0.03822801,-0.02403069,0,-0.02658656,0,0.006403141,-0.002166651]},{"f":[1,-1,1,-1,16,1,-1,16,16,-1,-1,4,-1,-1,15,33,4,17,-1,-1,26,-1,1,-1,-1,23,-1,-1,-1],"t":[4.92309205350118,0,4.949128919860627,0,6.108524456778169,5.417948717948718,0,5.044394119358453,4.523561956057013,0,0,4.80643424864552,0,0,10.457380879072534,20.752541988539424,4.0437314206251695,6.636169092079312,0,0,0.5,0,6.147321428571429,0,0,3.5,0,0,0],"m":[1,0,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,0,8,9,0,0,12,0,0,15,16,17,18,0,0,21,0,23,0,0,26,0,0,0],"r":[2,0,4,0,14,7,0,11,10,0,0,13,0,0,28,25,20,19,0,0,22,0,24,0,0,27,0,0,0],"v":[0,-0.001578708,0,0.04227986,0,0,-0.04340054,0,0,0.0009605669,0.04145586,0,-0.0425108,-0.0009419996,0,0,0,0,-0.007750911,-0.06940017,0,0.04069515,0,-0.03090671,0.006581522,0,0.01221195,-0.001705005,-0.005287951]},{"f":[33,32,33,33,3,2,-1,2,21,-1,-1,3,-1,-1,37,-1,33,-1,21,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,0.32902480033627574,6.055241654908112,0,8.612669628062957,7.5,0,0,0.00554530545305453,0,0,8.05,0,18.39964081076096,0,9.5,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,15,0,17,0,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,14,7,0,11,10,0,0,13,0,0,16,0,18,0,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.04052649,0,0,-0.03510759,-0.002695856,0,-0.01241086,0.04347556,0,-3.426364e-05,0,0.02765346,0,0.004394954,0.04323707,0,0,0.01386919,0.05107421,-0.0261251,-0.02171692,-0.04553307,0.01325531]},{"f":[22,-1,22,1,-1,1,-1,18,0,-1,-1,22,-1,16,-1,-1,22,-1,15,-1,3,-1,18,18,-1,-1,0,-1,-1],"t":[2.69615871138938,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,3.084962500721156,6.5,0,0,6.870153907126159,0,6.6865005271832185,0,0,8.133841411710264,0,10.457380879072534,0,0.26136363636363635,0,10.620215166896223,9.273793716589307,0,0,2.5,0,0],"m":[0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,9,0,0,12,0,14,0,0,17,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,16,5,0,7,0,11,10,0,0,13,0,15,0,0,18,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.03076566,0,0,0.01711783,0,-0.02383297,0,0,-0.03415549,0.000101996,0,0.04784915,0,-0.002492942,0.01969234,0,-0.03972442,0,0.0007864858,0,0.0247203,0,0,-0.007160826,-0.04125801,0,-0.004229439,0.04274598]},{"f":[27,22,22,-1,-1,22,17,-1,-1,15,-1,17,-1,15,18,-1,22,-1,17,-1,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,16.786010507999382,16.765402642760364,0,0,16.819976309430963,8.378590816408359,0,0,5.044394119358453,0,3.39067985676233,0,10.113742166049189,6.4178278454199,0,17.083799521744545,0,8.729619742794132,0,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,12,0,14,15,0,17,0,19,0,0,0,23,0,25,0,27,0,0],"r":[22,5,4,0,0,9,8,0,0,11,0,13,0,21,16,0,18,0,20,0,0,0,24,0,26,0,28,0,0],"v":[0,0,0,-2.660957e-05,0.04997001,0,0,-0.04619802,-0.01273923,0,0.03178635,0,-0.03118849,0,0,0.008800678,0,-0.03303198,0,0.01807895,-0.02746346,0.04874634,0,0.02290535,0,-0.01848068,0,0.0245664,-0.01491572]},{"f":[33,32,33,33,33,33,22,22,-1,-1,22,-1,15,-1,18,-1,-1,-1,17,-1,-1,37,33,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,29.297756302860655,27.27007187651612,24.56142327888218,16.786010507999382,16.765402642760364,0,0,16.819976309430963,0,5.044394119358453,0,3.39067985676233,0,0,0,2.903677461028802,0,0,9.629999999999999,31.22997243441001,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,11,0,13,0,15,0,0,0,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,18,17,10,9,0,0,12,0,14,0,16,0,0,0,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,-2.141054e-05,0.04876181,0,-0.03306207,0,0.02990965,0,-0.03366025,0.002082608,0.02399528,0,0.001728813,-0.03549486,0,0,0.02187664,-0.001662582,0.03706144,-0.0210734,-0.0448693,0.01281724]},{"f":[22,22,22,22,22,-1,-1,22,17,-1,-1,2,-1,-1,23,16,22,-1,-1,-1,-1,18,-1,18,-1,-1,2,-1,-1],"t":[17.083799521744545,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,16.819976309430963,8.378590816408359,0,0,2.453445297804259,0,0,7.5,6.0,16.98920034885663,0,0,0,0,6.076776015854056,0,7.383290334558488,0,0,4.0437314206251695,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,12,0,0,15,16,17,0,0,0,0,22,0,24,0,0,27,0,0],"r":[26,21,14,7,6,0,0,11,10,0,0,13,0,0,20,19,18,0,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0,4.392446e-05,0.04757847,0,0,-0.04490171,-0.01103917,0,-0.02467231,0.01469652,0,0,0,0.07095646,0.005795883,0.01031241,-0.01924533,0,-0.03550463,0,0.0283056,-0.02079809,0,0.002422244,0.03877211]},{"f":[22,-1,2,1,16,18,21,-1,-1,15,18,-1,-1,-1,21,15,-1,4,-1,-1,-1,25,16,-1,-1,-1,18,-1,-1],"t":[12.199665042528586,0,9.012591414317754,4.92309205350118,5.169925001442312,2.160964047443681,7.5,0,0,4.857980995127572,4.128693921346326,0,0,0,7.5,8.361943773735241,0,2.453445297804259,0,0,0,0.5,6.409390936137702,0,0,0,9.229707804263573,0,0],"m":[0,0,1,1,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,0,10,11,0,0,0,15,16,0,18,0,0,0,22,23,0,0,0,27,0,0],"r":[2,0,26,21,14,9,8,0,0,13,12,0,0,0,20,17,0,19,0,0,0,25,24,0,0,0,28,0,0],"v":[0,-0.001752997,0,0,0,0,0,-0.03641152,0.01504698,0,0,0.07218105,0.0005244503,0.08311255,0,0,-0.04334181,0,-0.03998563,0.005120862,-0.00194471,0,0,-0.005389022,0.01200471,-0.00289504,0,0.03760807,-0.01055258]},{"f":[22,-1,22,16,-1,-1,1,17,2,22,-1,2,1,-1,-1,-1,2,2,-1,-1,-1,-1,1,-1,2,-1,0,-1,-1],"t":[2.69615871138938,0,7.082728314749225,10.681238411777805,0,0,2.532051282051282,11.651826559171445,4.0437314206251695,15.782972563647261,0,1.292481250360578,1.000936329588015,0,0,0,9.266718101290426,9.075473573191719,0,0,0,0,2.9611191860465116,0,9.299206200930918,0,3.5,0,0],"m":[0,0,0,1,0,0,0,1,1,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,12,13,0,0,0,17,18,0,0,0,0,23,0,25,0,27,0,0],"r":[2,0,6,5,0,0,22,21,16,11,0,15,14,0,0,0,20,19,0,0,0,0,24,0,26,0,28,0,0],"v":[0,-0.03036152,0,0,0.001600217,0.02160278,0,0,0,0,-0.03467802,0,0,0.07536121,-0.004443392,-0.03775268,0,0,-0.005489362,0.01248007,-0.02118905,0.01663449,0,0.05815262,0,-0.0001906665,0,0.0006184862,0.05224698]},{"f":[22,16,22,22,16,36,-1,18,22,-1,-1,18,-1,-1,-1,-1,-1,-1,23,33,1,-1,-1,22,15,-1,-1,-1,-1],"t":[14.116600814324018,6.894817763307944,12.921158001713904,12.613886328427084,6.569855608330948,2.5,0,1.792481250360578,10.27595502635321,0,0,5.108372929097653,0,0,0,0,0,0,1.5,16.993854061608424,2.0015923566878984,0,0,15.676331898425197,6.247927513443585,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,0,0,0,0,19,20,21,0,0,24,25,0,0,0,0],"r":[18,17,16,15,14,7,0,11,10,0,0,13,0,0,0,0,0,0,28,23,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.02619252,0,0,0.003911938,-0.01855909,0,0.02714492,-0.004212059,-0.03410418,0.0385494,-0.02583539,0.001574441,0,0,0,-0.007791871,0.06246263,0,0,0.007744395,-0.0312847,0.04225412,-0.001058465]},{"f":[21,22,23,-1,18,18,0,-1,-1,-1,-1,23,33,1,-1,-1,22,15,-1,-1,-1,18,-1,-1,22,0,-1,-1,-1],"t":[9.5,14.116600814324018,3.5,0,5.374934713698423,5.303665156874805,6.5,0,0,0,0,1.5,16.993854061608424,2.0015923566878984,0,0,15.676331898425197,6.247927513443585,0,0,0,9.229707804263573,0,0,15.756821747580634,7.5,0,0,0],"m":[1,1,1,0,0,1,1,0,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,1,1,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,0,12,13,14,0,0,17,18,0,0,0,22,0,0,25,26,0,0,0],"r":[24,11,4,0,10,9,8,0,0,0,0,21,16,15,0,0,20,19,0,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,-0.0008115927,0,0,0,-0.02406884,0.01047196,0.03843496,-0.03364889,0,0,0,-0.00749714,0.06001579,0,0,0.007407136,-0.02958025,0.04175003,0,0.0005027684,-0.02152111,0,0,0.0062299,0.04418973,-0.03477343]},{"f":[33,21,36,4,-1,-1,-1,24,18,22,-1,-1,-1,3,-1,33,-1,-1,33,-1,37,32,-1,-1,2,21,-1,-1,-1],"t":[20.752541988539424,3.5,3.5,4.128693921346326,0,0,0,7.5,6.531360383582522,16.73920708776685,0,0,0,0.4330891330891331,0,18.39964081076096,0,0,21.16864017939033,0,9.205,10.41931383093672,0,0,2.453445297804259,5.5,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,0,0,0,14,0,16,0,0,19,0,21,22,0,0,25,26,0,0,0],"r":[18,7,6,5,0,0,0,13,12,11,0,0,0,15,0,17,0,0,20,0,24,23,0,0,28,27,0,0,0],"v":[0,0,0,0,-0.0597028,0.003906886,-0.007406011,0,0,0,-0.003151613,0.02633197,-0.02418927,0,-0.02090006,0,0.04088733,-0.008182004,0,0.02869483,0,0,0.0002344734,-0.0166037,0,0,0.004504352,0.02322543,0.02435416]},{"f":[4,32,4,-1,-1,-1,2,16,-1,17,-1,-1,24,22,15,15,-1,16,1,-1,16,-1,-1,18,-1,-1,-1,-1,-1],"t":[3.084962500721156,10.463013322934223,2.903677461028802,0,0,0,3.245926548164837,8.005624549193879,0,9.265614570460752,0,0,9.5,17.083799521744545,5.78135971352466,4.087462841250339,0,6.569855608330948,1.5994791666666668,0,5.78135971352466,0,0,1.292481250360578,0,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,1,1,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,10,0,0,13,14,15,16,0,18,19,0,21,0,0,24,0,0,0,0,0],"r":[6,5,4,0,0,0,12,9,0,11,0,0,28,27,26,17,0,23,20,0,22,0,0,25,0,0,0,0,0],"v":[0,0,0,-0.0006461299,-0.04051701,-0.02890446,0,0,0.03464158,0,-0.01852052,0.02423861,0,0,0,0,-0.0004624868,0,0,0.05812915,0,0.007691341,-0.01549938,0,0.008451238,0.05645556,-0.003720987,0.0395611,0.03116522]},{"f":[22,-1,2,4,21,-1,2,1,-1,4,1,22,22,-1,-1,23,-1,-1,-1,4,-1,-1,22,-1,-1,1,-1,-1,-1],"t":[2.69615871138938,0,10.680359255641626,9.266718101290426,2.5,0,9.051177541662383,0.39811320754716983,0,7.950787011501063,1.13125,9.087343393278285,8.133841411710264,0,0,4.5,0,0,0,8.950330235806282,0,0,13.005894632618489,0,0,2.997611464968153,0,0,0],"m":[0,0,1,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,10,11,12,13,0,0,16,0,0,0,20,0,0,23,0,0,26,0,0,0],"r":[2,0,28,25,6,0,22,9,0,19,18,15,14,0,0,17,0,0,0,21,0,0,24,0,0,27,0,0,0],"v":[0,-0.0300226,0,0,0,-0.02509467,0,0,-0.04026436,0,0,0,0,0.007613368,0.04327431,0,-0.01799437,0.0293027,-9.672435e-05,0,-0.03777843,0.001849039,0,0.003260246,0.03586905,0,-0.02931022,0.01837179,0.02535304]},{"f":[25,4,32,15,-1,17,-1,22,-1,19,23,1,-1,2,21,-1,-1,-1,36,-1,-1,-1,-1,1,3,-1,-1,-1,-1],"t":[6.5,2.453445297804259,10.463013322934223,2.0,0,0.5,0,9.40390385294334,0,1.5,0.5,2.68018018018018,0,1.292481250360578,3.5,0,0,0,3.5,0,0,0,0,12.18570183086312,0.0339080459770115,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,10,11,12,0,14,15,0,0,0,19,0,0,0,0,24,25,0,0,0,0],"r":[28,23,22,5,0,7,0,9,0,21,18,13,0,17,16,0,0,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.03446682,0,0.01583025,0,-0.02335122,0,0,0,-0.01118463,0,0,0.02344095,0.1070385,-0.003384102,0,0.03944749,-0.002020414,-0.005994974,-0.02727702,0,0,0.03599253,0.0004344727,0.03939422,-0.02818043]},{"f":[22,-1,17,22,-1,1,22,-1,15,-1,-1,15,-1,-1,22,36,22,-1,-1,-1,22,-1,23,1,23,-1,-1,-1,-1],"t":[12.199665042528586,0,1.292481250360578,12.278990886161445,0,2.532051282051282,15.782972563647261,0,10.066089190457772,0,0,5.78135971352466,0,0,12.921158001713904,2.5,12.613886328427084,0,0,0,13.415167820124083,0,1.5,5.015625,0.5,0,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,0,12,0,0,15,16,17,0,0,0,21,0,23,24,25,0,0,0,0],"r":[2,0,14,5,0,11,8,0,10,0,0,13,0,0,20,19,18,0,0,0,22,0,28,27,26,0,0,0,0],"v":[0,-0.001686009,0,0,-0.03301135,0,0,-0.03490858,0,0.04807254,-0.02909895,0,0.02451571,-0.002959219,0,0,0,-0.02428826,0.04225951,0.01936212,0,-0.02678958,0,0,0,0.05905133,-0.001730838,0.005120272,-0.0006524599]},{"f":[22,22,22,22,22,22,15,-1,-1,17,-1,-1,-1,22,-1,2,15,-1,-1,-1,23,-1,-1,18,-1,-1,18,-1,-1],"t":[17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,16.752707246237605,13.56045225766691,0,0,2.453445297804259,0,0,0,16.807008825964132,0,3.903677461028802,6.169925001442312,0,0,0,7.5,0,0,6.076776015854056,0,0,8.729619742794132,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,0,0,10,0,0,0,14,0,16,17,0,0,0,21,0,0,24,0,0,27,0,0],"r":[26,23,20,13,12,9,8,0,0,11,0,0,0,15,0,19,18,0,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0.0006829172,-0.02608594,0,0.02050267,-0.04245286,0.04631768,0,-0.02921383,0,0,0.02048107,-0.03918855,0.01157325,0,0.02759683,-0.01871247,0,-0.03177584,0.008538379,0,0.01935821,-0.02065751]},{"f":[27,15,15,-1,15,-1,16,22,22,22,16,-1,-1,-1,-1,-1,18,-1,-1,15,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,5.78135971352466,4.087462841250339,0,4.169925001442312,0,6.409390936137702,15.756821747580634,13.054613167076562,12.613886328427084,4.247927513443585,0,0,0,0,0,3.084962500721156,0,0,6.108524456778169,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,8,9,10,11,0,0,0,0,0,17,0,0,20,0,0,23,0,25,0,27,0,0],"r":[22,19,4,0,6,0,16,15,14,13,12,0,0,0,0,0,18,0,0,21,0,0,24,0,26,0,28,0,0],"v":[0,0,0,-0.001238172,0,0.06780113,0,0,0,0,0,-0.02270872,0.007770407,0.03863289,-0.01697488,0.0403669,0,0.007864233,0.05566339,0,-0.02736884,-0.0004334135,0,0.0222906,0,-0.01789004,0,0.02328286,-0.01365444]},{"f":[4,4,21,22,-1,22,-1,-1,2,1,-1,4,4,30,-1,21,-1,-1,-1,4,-1,-1,22,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,6.106096185954236,0,13.87915215758784,0,0,9.051177541662383,0.39811320754716983,0,7.950787011501063,7.408266855279292,1.5,0,3.5,0,0,0,8.950330235806282,0,0,13.005894632618489,0,0,0,2.160964047443681,0,0],"m":[1,1,0,0,0,1,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,12,13,14,0,16,0,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,22,11,0,19,18,15,0,17,0,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0.01890487,0,-0.04426829,0.01818911,0,0,-0.03972482,0,0,0,0.0007249247,0,-0.04166343,-0.004630165,0.01647553,0,-0.03674189,0.001444212,0,0.002758468,0.03484661,-0.0152927,0,-0.006973507,0.04078479]},{"f":[22,4,15,16,15,-1,-1,-1,23,15,-1,-1,33,-1,-1,35,1,-1,21,17,-1,4,-1,16,-1,-1,-1,-1,-1],"t":[12.199665042528586,1.292481250360578,9.862637357558794,7.741466986401147,7.266786540694901,0,0,0,0.5,10.681238411777805,0,0,44.805779984159756,0,0,2.5,0.01933482600988898,0,4.5,0.5,0,2.903677461028802,0,5.491853096329675,0,0,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,0,0,13,0,0,16,17,0,19,20,0,22,0,24,0,0,0,0,0],"r":[28,15,8,7,6,0,0,0,12,11,0,0,14,0,0,27,18,0,26,21,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.02483366,0.03713004,-0.04729705,0,0,0.06087465,0.005373672,0,-0.001119741,-0.04281668,0,0,0.03771278,0,0,0.05233549,0,-0.05088953,0,-0.04809056,-0.001642014,0.0002699809,0.02506441,0.001325468]},{"f":[36,-1,33,33,21,2,-1,33,-1,-1,2,-1,23,-1,4,-1,20,22,17,-1,22,-1,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.891308976303876,3.5,2.453445297804259,0,15.856946627097248,0,0,1.292481250360578,0,0.5,0,1.792481250360578,0,1.5,16.725912049439096,2.69615871138938,0,12.613886328427084,0,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,0,13,0,15,0,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,10,7,0,9,0,0,12,0,14,0,16,0,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0.02529036,0,0,0,0,-0.0284073,0,-0.04444668,0.01742876,0,0.01613508,0,-0.03248316,0,-0.04495773,0,0,0,-0.02565915,0,-0.005284666,0.02752015,0.02094187,0.02532219,-0.03402766,0,0.02781116,0.0004082212]},{"f":[33,32,33,33,18,21,15,-1,22,-1,2,1,-1,25,-1,-1,-1,-1,37,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,1.792481250360578,7.5,3.584962500721156,0,10.12257207114862,0,8.767235038274581,5.985044642857143,0,1.5,0,0,0,0,11.57,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,1,1,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,11,12,0,14,0,0,0,0,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,18,17,8,0,10,0,16,13,0,15,0,0,0,0,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.0201183,0,0.02392245,0,0,-0.03568305,0,0.006096749,-0.01804667,0.03441589,0.002825529,0,0.0006935193,0.03130436,0,0,0.01299401,0.04836138,-0.02280869,-0.02143972,-0.0436607,0.01208763]},{"f":[22,23,21,15,15,32,-1,-1,33,-1,18,3,-1,-1,-1,-1,-1,22,33,-1,-1,-1,18,32,-1,17,-1,-1,-1],"t":[17.094543568094064,7.5,10.5,10.457380879072534,9.756556322524087,7.98281324063098,0,0,37.46769166150554,0,8.03615005796417,0.11826564442602569,0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,0,8.729619742794132,8.341935934311302,0,7.588709768994619,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,6,0,0,9,0,11,12,0,0,0,0,0,18,19,0,0,0,23,24,0,26,0,0,0],"r":[22,17,16,15,8,7,0,0,10,0,14,13,0,0,0,0,0,21,20,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,-0.000834941,0.01938192,0,0.0007856535,0,0,-0.001288467,0.06259131,-0.001634458,-0.00392608,0.03296644,0,0,-0.02434379,0.02153605,-0.03790291,0,0,-0.01218666,0,0.009104323,0.04948617,-0.02000539]},{"f":[4,4,21,22,-1,37,-1,-1,2,1,-1,4,2,15,-1,-1,-1,4,-1,-1,16,15,-1,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,9.075473573191719,0.39811320754716983,0,7.950787011501063,7.894792293346235,5.169925001442312,0,0,0,8.950330235806282,0,0,10.826548487290914,2.0,0,0,0,0,2.160964047443681,0,0],"m":[1,1,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,12,13,14,0,0,0,18,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,20,11,0,17,16,15,0,0,0,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0.02013509,0,-0.0437893,0.01871182,0,0,-0.04319745,0,0,0,0.003375057,-0.001080397,0.03315488,0,-0.03607525,0.002658986,0,0,0.006490786,-0.02571648,0.03915046,-0.01479455,0,-0.006471147,0.04034967]},{"f":[22,23,-1,21,-1,2,-1,-1,22,16,2,17,-1,1,-1,-1,22,2,-1,-1,15,-1,-1,15,-1,-1,22,-1,-1],"t":[12.199665042528586,2.5,0,7.5,0,5.459058925515887,0,0,12.921158001713904,7.930737337562887,1.292481250360578,1.792481250360578,0,7.0046728971962615,0,0,12.613886328427084,8.049190845056867,0,0,4.247927513443585,0,0,10.457380879072534,0,0,13.415167820124083,0,0],"m":[0,1,0,0,0,1,0,0,0,1,0,1,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,0,14,0,0,17,18,0,0,21,0,0,24,0,0,27,0,0],"r":[8,3,0,5,0,7,0,0,26,23,16,13,0,15,0,0,20,19,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,-0.0006528354,0,-0.04593677,0,-0.01976725,0.02243094,0,0,0,0,-0.02405458,0,0.05767292,0.01747531,0,0,-0.02515652,0.02113054,0,0.04130097,-0.00257966,0,0.03345486,-0.02218937,0,-0.02351257,0.0008620913]},{"f":[36,-1,22,23,-1,21,-1,2,-1,-1,2,1,16,18,-1,15,18,-1,-1,-1,21,15,-1,-1,-1,-1,18,-1,-1],"t":[1.5,0,12.199665042528586,2.5,0,7.5,0,5.459058925515887,0,0,9.012591414317754,4.92309205350118,5.169925001442312,2.160964047443681,0,4.857980995127572,4.128693921346326,0,0,0,7.5,8.361943773735241,0,0,0,0,9.229707804263573,0,0],"m":[0,0,0,1,0,0,0,1,0,0,1,1,0,1,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,12,13,14,0,16,17,0,0,0,21,22,0,0,0,0,27,0,0],"r":[2,0,10,5,0,7,0,9,0,0,26,25,20,15,0,19,18,0,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0.02476074,0,0,-0.0008353313,0,-0.04558936,0,-0.01894191,0.02122613,0,0,0,0,-0.01438286,0,0,0.06300734,8.286513e-05,0.0712922,0,0,-0.04187867,-0.00997961,-0.001467274,0.003327807,0,0.03436003,-0.008900435]},{"f":[4,15,-1,21,-1,15,-1,-1,2,21,21,22,-1,-1,-1,-1,20,4,-1,23,-1,16,-1,22,-1,-1,22,-1,-1],"t":[4.951187557243013,11.683433292835403,0,7.5,0,12.27029532647204,0,0,5.40929108874043,7.5,3.5,13.129761635363703,0,0,0,0,0.5,5.505613627711627,0,3.5,0,8.321928094887362,0,16.759160379664443,0,0,10.316277538115674,0,0],"m":[1,1,0,1,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,0,0,0,0,17,18,0,20,0,22,0,24,0,0,27,0,0],"r":[8,3,0,5,0,7,0,0,16,15,14,13,0,0,0,0,26,19,0,21,0,23,0,25,0,0,28,0,0],"v":[0,0,-0.001797476,0,-0.005139253,0,0.03744125,-0.01748509,0,0,0,0,0.07575251,-0.003019467,0.06641454,0.0006618821,0,0,-0.03592881,0,0.0009868638,0,-0.03500572,0,-0.00962263,0.0301016,0,-0.0260699,0.01568162]},{"f":[3,22,-1,-1,2,4,21,22,-1,37,-1,-1,2,1,-1,4,-1,4,-1,-1,16,-1,-1,16,21,-1,-1,-1,-1],"t":[0.007093253968253968,16.725912049439096,0,0,10.680359255641626,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,9.034782574103549,0.39811320754716983,0,7.950787011501063,0,8.950330235806282,0,0,4.087462841250339,0,0,10.462502272597265,6.5,0,0,0,0],"m":[0,0,0,0,1,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,10,0,0,13,14,0,16,0,18,0,0,21,0,0,24,25,0,0,0,0],"r":[4,3,0,0,28,23,12,9,0,11,0,0,20,15,0,17,0,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,-0.02394067,0.02091734,0,0,0,0,0.01887879,0,-0.04336437,0.0185444,0,0,-0.03752422,0,0.0006043857,0,-0.03485586,0.0007657266,0,0.000833088,0.02398852,0,0,0.02351993,-0.01488787,-0.04070603,0.03398669]},{"f":[22,-1,25,4,4,32,15,18,-1,-1,21,-1,-1,-1,-1,33,2,15,19,-1,-1,-1,22,15,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,6.5,3.084962500721156,2.903677461028802,10.463013322934223,11.683433292835403,10.69085411329249,0,0,7.5,0,0,0,0,41.64100553123272,3.245926548164837,8.005624549193879,0.5,0,0,0,17.083799521744545,5.78135971352466,0,0,0,0,0],"m":[0,0,1,1,1,1,1,1,0,0,1,0,0,0,0,1,0,1,1,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,0,0,16,17,18,19,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,15,14,13,10,9,0,0,12,0,0,0,0,27,22,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.02914437,0,0,0,0,0,0,-0.0008054098,-0.04709604,0,-0.004998145,0.02858695,-0.02663439,-0.03996095,0,0,0,0,0.002941973,0.03489168,-0.0419578,0,0,0.002896637,-0.003213114,0.03371058,0.03234353,-0.02749096]},{"f":[36,-1,33,33,21,2,-1,33,-1,-1,2,-1,23,-1,4,-1,20,22,33,-1,22,-1,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.891308976303876,3.5,2.453445297804259,0,15.856946627097248,0,0,1.292481250360578,0,0.5,0,1.792481250360578,0,1.5,16.725912049439096,8.065928480304397,0,12.613886328427084,0,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,0,13,0,15,0,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,10,7,0,9,0,0,12,0,14,0,16,0,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0.02438049,0,0,0,0,-0.02692898,0,-0.04324688,0.01710071,0,0.01467326,0,-0.03117858,0,-0.04383846,0,0,0,-0.02240988,0,-0.004995817,0.03780037,0.02021388,0.02337153,-0.03292273,0,0.02697186,0.0003744369]},{"f":[33,23,21,15,-1,15,-1,15,23,-1,-1,22,22,-1,17,-1,-1,-1,-1,-1,3,-1,33,-1,33,18,-1,-1,-1],"t":[29.297756302860655,8.5,10.5,10.457380879072534,0,10.462502272597265,0,10.681238411777805,0.5,0,0,16.73920708776685,11.722778233171695,0,5.597993649014255,0,0,0,0,0,0.33771929824561403,0,32.199935296219856,0,44.79706586676442,2.160964047443681,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,9,0,0,12,13,0,15,0,0,0,0,0,21,0,23,0,25,26,0,0,0],"r":[20,19,18,5,0,7,0,11,10,0,0,17,14,0,16,0,0,0,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0.0005003071,0,-0.04315276,0,0,0.06633598,0.009673874,0,0,0.008603091,0,-0.02751981,-0.003949825,0.01668754,0.02864369,-0.02368592,0,0.02752875,0,0.0216597,0,0,-0.0007092938,-0.0391465,0.007496616]},{"f":[4,4,15,-1,-1,16,-1,-1,2,21,21,22,-1,-1,-1,-1,20,4,-1,23,-1,16,-1,22,-1,-1,22,-1,-1],"t":[4.951187557243013,4.554262228389085,1.584962500721156,0,0,10.462502272597265,0,0,5.40929108874043,7.5,3.5,13.129761635363703,0,0,0,0,0.5,5.505613627711627,0,3.5,0,8.321928094887362,0,16.759160379664443,0,0,10.316277538115674,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,0,0,0,0,17,18,0,20,0,22,0,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,16,15,14,13,0,0,0,0,26,19,0,21,0,23,0,25,0,0,28,0,0],"v":[0,0,0,-0.02750557,-0.0002960292,0,-0.03990733,-0.003064654,0,0,0,0,0.0700733,-0.002983254,0.06527874,0.0005435246,0,0,-0.03496801,0,0.000936177,0,-0.03367515,0,-0.008198478,0.02851581,0,-0.02540437,0.01497287]},{"f":[22,22,22,22,22,22,-1,17,-1,-1,-1,22,17,-1,-1,2,-1,-1,23,-1,-1,18,22,-1,-1,-1,18,-1,-1],"t":[17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,16.752707246237605,0,2.453445297804259,0,0,0,16.819976309430963,8.378590816408359,0,0,2.453445297804259,0,0,7.5,0,0,6.076776015854056,17.078739781790915,0,0,0,8.729619742794132,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,8,0,0,0,12,13,0,0,16,0,0,19,0,0,22,23,0,0,0,27,0,0],"r":[26,21,18,11,10,7,0,9,0,0,0,15,14,0,0,17,0,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0.0003650238,0,0.0172546,-0.04069183,0.04438824,0,0,-0.04364489,-0.009579366,0,-0.02370568,0.01369038,0,0.02639753,-0.01828363,0,0,-0.04791408,-0.01062507,0.007771705,0,0.01814341,-0.01950769]},{"f":[22,-1,25,4,4,21,22,22,-1,-1,-1,2,1,-1,4,-1,4,-1,-1,16,15,-1,-1,-1,-1,17,-1,-1,-1],"t":[2.69615871138938,0,6.5,10.510264838663748,9.266718101290426,2.5,13.827286379004683,6.106096185954236,0,0,0,9.075473573191719,0.39811320754716983,0,7.950787011501063,0,8.950330235806282,0,0,10.826548487290914,2.0,0,0,0,0,2.160964047443681,0,0,0],"m":[0,0,1,1,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,0,12,13,0,15,0,17,0,0,20,21,0,0,0,0,26,0,0,0],"r":[2,0,28,25,24,11,10,9,0,0,0,19,14,0,16,0,18,0,0,23,22,0,0,0,0,27,0,0,0],"v":[0,-0.028622,0,0,0,0,0,0,0.01832962,-0.04297113,0.02121797,0,0,-0.04223473,0,0.0006578646,0,-0.03358727,0.001731551,0,0,0.005971122,-0.02420703,0.03755015,-0.01442419,0,-0.006386911,0.0397233,-0.02686291]},{"f":[36,-1,22,-1,22,16,2,17,-1,24,-1,-1,22,2,-1,-1,15,-1,-1,15,17,-1,-1,-1,22,-1,15,-1,-1],"t":[1.5,0,12.199665042528586,0,12.921158001713904,7.930737337562887,1.292481250360578,1.792481250360578,0,4.5,0,0,12.613886328427084,8.049190845056867,0,0,4.247927513443585,0,0,10.457380879072534,3.084962500721156,0,0,0,13.415167820124083,0,7.118941072723508,0,0],"m":[0,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,0,10,0,0,13,14,0,0,17,0,0,20,21,0,0,0,25,0,27,0,0],"r":[2,0,4,0,24,19,12,9,0,11,0,0,16,15,0,0,18,0,0,23,22,0,0,0,26,0,28,0,0],"v":[0,0.0239651,0,-0.00175032,0,0,0,0,-0.02348125,0,0.01566499,0.05391111,0,0,-0.02384504,0.01917566,0,0.03946719,-0.00246655,0,0,0.01573934,0.05490786,-0.02088897,0,-0.02144794,0,0.005356133,-0.002048346]},{"f":[4,-1,2,22,1,22,-1,-1,-1,-1,1,17,2,22,22,-1,-1,-1,-1,16,16,15,-1,-1,-1,-1,22,-1,-1],"t":[4.951187557243013,0,5.40929108874043,14.03140051979143,2.6153846153846154,13.129761635363703,0,0,0,0,7.40625,6.734566509914796,6.793888758164105,9.087343393278285,8.133841411710264,0,0,0,0,11.756139298372927,4.247927513443585,2.0,0,0,0,0,12.121442107629743,0,0],"m":[1,0,0,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,0,11,12,13,14,15,0,0,0,0,20,21,22,0,0,0,0,27,0,0],"r":[2,0,10,9,8,7,0,0,0,0,26,19,18,17,16,0,0,0,0,25,24,23,0,0,0,0,28,0,0],"v":[0,-0.001042679,0,0,0,0,0.07394285,-0.0113962,-0.0004553394,0.03943833,0,0,0,0,0,-0.0222494,0.02154863,-0.03320664,-0.0001373231,0,0,0,0.01184813,-0.03197164,0.01193328,-0.008677325,0,0.0006820325,0.03395331]},{"f":[8,3,22,-1,-1,2,15,15,-1,1,-1,21,-1,-1,21,33,15,15,-1,-1,-1,-1,23,22,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,10.457380879072534,9.756556322524087,0,1.987429289754871,0,5.5,0,0,5.5,44.79706586676442,10.46352437327118,10.462502272597265,0,0,0,0,0.5,11.406459975416917,0,0,0,0,0],"m":[1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,10,0,12,0,0,15,16,17,18,0,0,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,14,9,0,11,0,13,0,0,22,21,20,19,0,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.03569901,0.02035782,0,0,0,-0.0003756614,0,0.06504405,0,0.02548947,0.002341145,0,0,0,0,-0.02753561,0.02011086,-0.0477024,0.0112779,0,0,0.004604285,0.04721073,-0.002865922,0.03336892,0.01290231]},{"f":[21,4,2,2,1,-1,2,-1,-1,22,-1,-1,-1,0,-1,-1,22,0,22,-1,23,2,-1,0,-1,-1,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0.6824793747286149,0,7.005613627711627,0,0,13.005894632618489,0,0,0,3.5,0,0,15.756821747580634,7.5,10.735831902194036,0,4.5,5.065928480304397,0,1.5,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,0,0,0,14,0,0,17,18,19,0,21,22,0,24,0,0,0,0,0],"r":[16,13,12,9,6,0,8,0,0,11,0,0,0,15,0,0,28,27,20,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03429811,0,0.0001403677,-0.008758438,0,0.002900409,0.03926632,-0.04038251,0,0.0002093894,0.04874062,0,0,0,0.01590879,0,0,-0.02967046,0,-0.01735912,0.01227165,0.0245434,0.04333462,-0.03536052]},{"f":[4,32,15,-1,19,-1,32,21,17,15,-1,-1,16,2,25,21,-1,-1,-1,-1,1,-1,-1,-1,-1,-1,37,-1,-1],"t":[2.453445297804259,10.463013322934223,2.0,0,1.5,0,7.491051153478379,7.5,1.792481250360578,6.523561956057013,0,0,6.539158811108031,1.292481250360578,0.5,3.5,0,0,0,0,6.015625,0,0,0,0,0,12.52,0,0],"m":[1,1,0,0,1,0,1,1,0,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,0,13,14,15,16,0,0,0,0,21,0,0,0,0,0,27,0,0],"r":[26,25,4,0,6,0,24,23,12,11,0,0,20,19,18,17,0,0,0,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,-0.03293005,0,0.002191644,0,0,0,0,-0.03254394,-0.004242374,0,0,0,0,-0.02619942,0.04539413,0.006165644,-0.01786685,0,0.008691074,-0.04285326,-0.03327073,0.01489119,-0.02510917,0,0.0006700538,0.03656825]},{"f":[25,4,32,15,-1,17,-1,22,18,-1,-1,19,23,1,-1,15,-1,-1,36,-1,-1,-1,-1,1,3,-1,-1,-1,-1],"t":[6.5,2.453445297804259,10.463013322934223,2.0,0,0.5,0,9.40390385294334,3.084962500721156,0,0,1.5,0.5,4.008411214953271,0,7.118941072723508,0,0,3.5,0,0,0,0,12.18570183086312,0.0339080459770115,0,0,0,0],"m":[1,1,1,0,0,0,0,0,1,0,0,1,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,9,0,0,12,13,14,0,16,0,0,19,0,0,0,0,24,25,0,0,0,0],"r":[28,23,22,5,0,7,0,11,10,0,0,21,18,15,0,17,0,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.03246613,0,0.01357387,0,0,-0.04266774,-6.309872e-05,0,0,0,0.005990131,0,0.08702033,0.01705688,0,0.03738086,-0.001913596,-0.005479172,-0.02432844,0,0,0.03271053,0.0004103088,0.03662969,-0.02648385]},{"f":[21,4,2,2,1,-1,2,-1,-1,22,-1,-1,-1,0,-1,-1,22,0,22,-1,23,2,-1,0,-1,-1,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0.6824793747286149,0,7.005613627711627,0,0,13.005894632618489,0,0,0,3.5,0,0,15.756821747580634,7.5,10.735831902194036,0,4.5,5.065928480304397,0,1.5,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,0,0,0,14,0,0,17,18,19,0,21,22,0,24,0,0,0,0,0],"r":[16,13,12,9,6,0,8,0,0,11,0,0,0,15,0,0,28,27,20,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03377754,0,0.0001383288,-0.008384941,0,0.002711103,0.03838246,-0.0393618,0,0.0001497505,0.04795249,0,0,0,0.01530604,0,0,-0.02868962,0,-0.01682956,0.01172211,0.02419863,0.04271399,-0.03457889]},{"f":[8,3,22,-1,-1,2,15,15,-1,-1,21,33,15,15,-1,-1,-1,-1,18,18,-1,-1,1,15,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,10.457380879072534,9.756556322524087,0,0,5.5,44.79706586676442,10.46352437327118,10.462502272597265,0,0,0,0,9.265614570460752,9.229707804263573,0,0,2.9055555555555554,11.683433292835403,0,0,0,0,0],"m":[1,0,0,0,0,1,1,1,0,0,0,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,11,12,13,14,0,0,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,10,9,0,0,18,17,16,15,0,0,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.03449238,0.01914682,0,0,0,-0.0003517633,0.009682495,0,0,0,0,-0.02678963,0.02004963,-0.04666453,0.009976811,0,0,-0.0002666043,-0.04659687,0,0,-0.04563641,0.0123061,0.03246251,0.03266598,0.01258721]},{"f":[22,-1,15,22,22,22,18,4,15,-1,-1,-1,18,-1,-1,1,21,-1,-1,-1,-1,-1,15,-1,37,-1,34,-1,-1],"t":[2.69615871138938,0,5.78135971352466,16.819976309430963,16.73920708776685,15.439474430023438,1.792481250360578,1.292481250360578,2.584962500721156,0,0,0,6.459338463407563,0,0,4.008411214953271,7.5,0,0,0,0,0,6.108524456778169,0,9.205,0,3.5,0,0],"m":[0,0,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,0,0,13,0,0,16,17,0,0,0,0,0,23,0,25,0,27,0,0],"r":[2,0,22,21,20,15,12,11,10,0,0,0,14,0,0,19,18,0,0,0,0,0,24,0,26,0,28,0,0],"v":[0,-0.02783737,0,0,0,0,0,0,0,0.02484232,-0.03488583,4.039765e-05,0,0.01005282,-0.002200887,0,0,-0.03658613,0.0165868,0.05256053,-0.01853958,0.02462665,0,-0.02632126,0,-0.0009494897,0,0.02156601,0.005128151]},{"f":[23,21,22,-1,23,33,1,-1,-1,22,15,-1,-1,-1,17,-1,15,-1,-1,-1,22,22,33,-1,-1,1,-1,-1,-1],"t":[7.5,10.5,14.116600814324018,0,1.5,16.993854061608424,2.0015923566878984,0,0,15.676331898425197,6.247927513443585,0,0,0,5.9770981551934375,0,7.366322214245816,0,0,0,17.105450904090162,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,0,0,0,15,0,17,0,0,0,21,22,23,0,0,26,0,0,0],"r":[20,19,4,0,14,9,8,0,0,13,12,0,0,0,16,0,18,0,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,-0.0009708463,0,0,0,-0.004988053,0.05440767,0,0,0.005884021,-0.02897938,0.04008004,0,-0.005981048,0,0.02399114,-0.001627852,0.02476031,0,0,0,-0.02340315,0.02058087,0,0.01249541,-0.04666244,0.01048642]},{"f":[16,-1,21,-1,17,15,3,-1,16,0,-1,22,-1,-1,22,-1,-1,-1,17,-1,17,-1,17,23,-1,-1,17,-1,-1],"t":[7.924812503605781,0,3.5,0,9.26091905453048,10.457380879072534,0.3033788641265277,0,9.92481250360578,2.5,0,9.945253865134848,0,0,11.681676585676911,0,0,0,9.265614570460752,0,9.27958770372691,0,10.620215166896223,1.5,0,0,10.68167758836394,0,0],"m":[1,0,0,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,9,10,0,12,0,0,15,0,0,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,4,0,18,17,8,0,14,11,0,13,0,0,16,0,0,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.001259405,0,-0.02948889,0,0,0,-0.0136455,0,0,-0.02607614,0,0.04960621,0.008238833,0,0.01344415,0.05283981,-0.001083745,0,-0.03060703,0,0.02640547,0,0,-0.0433397,-0.003500445,0,0.04556726,0.0003782666]},{"f":[4,4,32,-1,-1,-1,2,16,-1,17,-1,-1,22,24,22,22,22,22,15,-1,-1,-1,-1,-1,18,-1,-1,-1,-1],"t":[3.084962500721156,2.903677461028802,10.463013322934223,0,0,0,3.245926548164837,8.005624549193879,0,9.265614570460752,0,0,17.083799521744545,9.5,17.069460217922263,16.819976309430963,16.786010507999382,16.73920708776685,5.78135971352466,0,0,0,0,0,6.076776015854056,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,1,1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,10,0,0,13,14,15,16,17,18,19,0,0,0,0,0,25,0,0,0,0],"r":[6,5,4,0,0,0,12,9,0,11,0,0,28,27,24,23,22,21,20,0,0,0,0,0,26,0,0,0,0],"v":[0,0,0,-0.0006138507,-0.02425995,-0.03962292,0,0,0.03199152,0,-0.01758537,0.02164423,0,0,0,0,0,0,0,0.002411892,-0.004072813,0.02219211,-0.01757543,0.01508234,0,-0.03540113,0.01049434,0.02951154,0.03376621]},{"f":[8,2,1,-1,4,21,22,-1,22,-1,-1,2,21,-1,0,-1,-1,16,-1,3,-1,-1,16,21,-1,-1,-1,-1,-1],"t":[0.5,10.680359255641626,0.3126984126984127,0,9.266718101290426,2.5,6.106096185954236,0,13.827286379004683,0,0,9.034782574103549,9.5,0,7.5,0,0,4.087462841250339,0,0.8775684931506849,0,0,10.462502272597265,6.5,0,0,0,0,0],"m":[1,1,0,0,1,0,0,0,1,0,0,1,1,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,12,13,0,15,0,0,18,0,20,0,0,23,24,0,0,0,0,0],"r":[28,27,4,0,22,11,8,0,10,0,0,17,14,0,16,0,0,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.04094138,0,0,0,0.01756101,0,-0.04256792,0.01792846,0,0,-0.0004901151,0,0.004788067,0.0415163,0,0.0003354334,0,0.04325274,0.007762232,0,0,0.02098427,-0.01461658,-0.0389493,0.03176173,0.01249203]},{"f":[25,4,15,-1,17,-1,22,1,18,-1,17,-1,-1,17,-1,-1,22,17,-1,-1,18,-1,-1,1,3,-1,-1,-1,-1],"t":[6.5,2.453445297804259,2.0,0,0.5,0,11.67903991803059,6.015625,9.938949842906478,0,10.37503779525057,0,0,6.584884225259038,0,0,11.722778233171695,1.792481250360578,0,0,9.229707804263573,0,0,12.18570183086312,0.0339080459770115,0,0,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,9,0,11,0,0,14,0,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,4,0,6,0,16,13,10,0,12,0,0,15,0,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03198243,0,0.01308648,0,0,0,-0.03460838,0,0.05912069,-0.01481275,0,0.04288732,-0.03850048,0,0,-0.01825382,0.04499281,0,-0.0001959819,-0.01919032,0,0,0.03163215,0.0004356065,0.03533668,-0.02632281]},{"f":[16,16,-1,2,-1,-1,21,-1,17,15,3,-1,16,0,-1,22,-1,-1,-1,-1,17,-1,17,-1,17,23,-1,-1,-1],"t":[7.924812503605781,7.741466986401147,0,6.055241654908112,0,0,3.5,0,9.26091905453048,10.457380879072534,0.3033788641265277,0,9.92481250360578,2.5,0,9.945253865134848,0,0,0,0,9.265614570460752,0,9.27958770372691,0,10.620215166896223,1.5,0,0,0],"m":[1,1,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,0,9,10,11,0,13,14,0,16,0,0,0,0,21,0,23,0,25,26,0,0,0],"r":[6,3,0,5,0,0,8,0,20,19,12,0,18,15,0,17,0,0,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,-0.0004032148,0,-0.04433486,0.003336594,0,-0.02862102,0,0,0,-0.01278204,0,0,-0.02516528,0,0.04691202,0.007705458,0.03390429,-0.001079247,0,-0.02979412,0,0.02430044,0,0,-0.04220717,-0.003209401,0.00208029]},{"f":[22,21,1,22,-1,-1,15,-1,25,-1,37,15,-1,-1,15,-1,-1,-1,37,17,21,22,-1,-1,15,-1,-1,-1,-1],"t":[12.121442107629743,5.5,1.000936329588015,10.092754721228424,0,0,3.169925001442312,0,0.5,0,6.775,5.614709844115208,0,0,5.906890595608519,0,0,0,4.875,3.39067985676233,7.5,13.054613167076562,0,0,10.066089190457772,0,0,0,0],"m":[0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,1,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,0,7,0,9,0,11,12,0,0,15,0,0,0,19,20,21,22,0,0,25,0,0,0,0],"r":[18,17,6,5,0,0,8,0,10,0,14,13,0,0,16,0,0,0,28,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0,0,0.04831649,-0.0265408,0,-0.04476444,0,-0.0369535,0,0,-0.003097653,-0.03915964,0,-0.003097653,0.03178391,6.678472e-05,0,0,0,0,-0.0006865153,-0.03468089,0,-0.0006865153,-0.03959899,-0.0006865153,0.002627289]},{"f":[25,22,23,-1,21,-1,2,-1,-1,37,17,21,4,-1,-1,15,-1,-1,-1,16,-1,15,3,-1,16,-1,-1,-1,-1],"t":[6.5,12.121442107629743,2.5,0,7.5,0,5.459058925515887,0,0,4.875,3.39067985676233,7.5,6.209321624687949,0,0,10.066089190457772,0,0,0,6.044394119358453,0,7.118941072723508,0.3033788641265277,0,6.584962500721156,0,0,0,0],"m":[1,0,1,0,0,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,12,13,0,0,16,0,0,0,20,0,22,23,0,25,0,0,0,0],"r":[28,9,4,0,6,0,8,0,0,19,18,15,14,0,0,17,0,0,0,21,0,27,24,0,26,0,0,0,0],"v":[0,0,0,-0.0006370621,0,-0.04437701,0,-0.01786025,0.01900329,0,0,0,0,-0.03339679,-0.0005356923,0,-0.0005356923,-0.03913497,-0.0005356923,0,-0.0005356923,0,0,-0.0005356923,0,0.01347476,0.04558825,0.0007340236,-0.0258347]},{"f":[23,21,22,-1,23,33,1,-1,-1,22,15,-1,-1,-1,-1,0,1,-1,-1,-1,22,22,33,-1,-1,1,-1,-1,-1],"t":[7.5,9.5,14.116600814324018,0,1.5,16.993854061608424,2.0015923566878984,0,0,15.676331898425197,6.247927513443585,0,0,0,0,7.5,6.015625,0,0,0,17.105450904090162,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,0,0,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,0,0,0,0,16,17,0,0,0,21,22,23,0,0,26,0,0,0],"r":[20,15,4,0,14,9,8,0,0,13,12,0,0,0,0,19,18,0,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,-0.001379787,0,0,0,-0.004185043,0.05156556,0,0,0.005056465,-0.02928403,0.03898462,-0.0001120287,0,0,0.008283445,-0.02311807,0.04114284,0,0,0,-0.02335391,0.01966944,0,0.01206375,-0.04620541,0.0100656]},{"f":[16,17,18,17,17,-1,2,-1,-1,1,21,-1,-1,33,-1,-1,-1,18,-1,15,21,-1,-1,-1,16,-1,21,-1,-1],"t":[7.930737337562887,8.571721503073015,7.430398005367313,7.198837316474133,6.434604125275413,0,6.4675825248018475,0,0,6.9941860465116275,3.5,0,0,21.011466140970732,0,0,0,8.729619742794132,0,7.6653359171851765,7.5,0,0,0,8.060695931687555,0,3.5,0,0],"m":[1,1,1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,0,0,0,18,0,20,21,0,0,0,25,0,27,0,0],"r":[24,17,16,9,6,0,8,0,0,13,12,0,0,15,0,0,0,19,0,23,22,0,0,0,26,0,28,0,0],"v":[0,0,0,0,0,-0.001418591,0,-0.03599597,0.01940146,0,0,0.057784,0.003426188,0,-0.02894766,0.006610314,-0.02763173,0,0.03280716,0,0,0.006588754,-0.02345091,0.02966597,0,0.018686,0,-0.02760588,0.001050784]},{"f":[8,3,22,-1,-1,2,1,-1,4,21,22,-1,37,-1,-1,15,15,-1,21,-1,-1,16,-1,-1,1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,0.3126984126984127,0,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,5.169925001442312,4.087462841250339,0,3.5,0,0,6.108524456778169,0,0,2.997611464968153,0,0,0,0],"m":[1,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,11,0,13,0,0,16,17,0,19,0,0,22,0,0,25,0,0,0,0],"r":[28,5,4,0,0,27,8,0,24,15,12,0,14,0,0,21,18,0,20,0,0,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.03272105,0.01841225,0,0,-0.03056911,0,0,0,0.0182483,0,-0.04205205,0.01742445,0,0,0.0004192113,0,0.04974772,0.009153413,0,-0.009879473,0.0005216367,0,-0.02518899,0.01526023,0.03105576,0.01211619]},{"f":[25,22,-1,22,22,21,-1,-1,-1,22,16,15,-1,-1,1,16,1,-1,21,-1,-1,-1,21,-1,18,-1,-1,-1,-1],"t":[6.5,2.69615871138938,0,5.9525054311951475,4.926654777701837,6.5,0,0,0,12.121442107629743,1.584962500721156,1.0,0,0,1.13125,10.696098170955864,0.9444444444444444,0,6.5,0,0,0,3.5,0,10.609627990977145,0,0,0,0],"m":[1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,5,6,0,0,0,10,11,12,0,0,15,16,17,0,19,0,0,0,23,0,25,0,0,0,0],"r":[28,3,0,9,8,7,0,0,0,27,14,13,0,0,22,21,18,0,20,0,0,0,24,0,26,0,0,0,0],"v":[0,0,-0.02704887,0,0,0,-0.02930824,0.01334777,0.04521775,0,0,0,0.01902397,-0.03613128,0,0,0,-0.0431467,0,0.05180143,0.0002334168,0.02854442,0,-0.04222893,0,-0.0032152,0.00927358,0.001095341,-0.02559633]},{"f":[4,15,15,15,-1,24,-1,-1,-1,21,-1,15,-1,-1,2,21,21,22,-1,-1,-1,-1,1,2,-1,-1,22,-1,-1],"t":[4.951187557243013,11.683433292835403,10.690871009292314,10.462502272597265,0,7.5,0,0,0,7.5,0,12.27029532647204,0,0,5.40929108874043,7.5,3.5,13.129761635363703,0,0,0,0,7.40625,5.459058925515887,0,0,12.121442107629743,0,0],"m":[1,1,1,1,0,1,0,0,0,1,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,0,10,0,12,0,0,15,16,17,18,0,0,0,0,23,24,0,0,27,0,0],"r":[14,9,8,5,0,7,0,0,0,11,0,13,0,0,22,21,20,19,0,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,-0.001817789,0,0.03803736,-0.008559667,-0.02164495,0,-0.004515661,0,0.03054417,-0.01638557,0,0,0,0,0.06059782,-0.003225312,0.06082254,0.0003011362,0,0,-0.03844007,-0.001004055,0,0.0004700966,0.03247342]},{"f":[27,15,22,22,22,18,-1,-1,1,21,-1,-1,-1,-1,-1,15,4,-1,-1,-1,22,22,0,-1,-1,-1,22,-1,-1],"t":[0.5,5.78135971352466,16.819976309430963,16.73920708776685,15.439474430023438,1.792481250360578,0,0,4.008411214953271,7.5,0,0,0,0,0,6.108524456778169,4.80643424864552,0,0,0,16.98920034885663,16.978341770814396,8.5,0,0,0,17.078739781790915,0,0],"m":[1,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,9,10,0,0,0,0,0,16,17,0,0,0,21,22,23,0,0,0,27,0,0],"r":[20,15,14,13,8,7,0,0,12,11,0,0,0,0,0,19,18,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.003235987,0.00413684,0,0,-0.03597985,0.01601078,0.04796945,-0.0170894,0.02259043,0,0,-0.03603506,-0.00288004,-0.0003639394,0,0,0,-0.02823613,0.01662817,0.05993406,0,-0.0285014,0.0168168]},{"f":[4,-1,4,15,22,-1,33,-1,-1,-1,21,23,22,21,-1,15,-1,-1,2,17,-1,-1,17,15,-1,-1,-1,-1,-1],"t":[4.951187557243013,0,5.89476682248518,7.417852514885898,14.03140051979143,0,15.968216440900115,0,0,0,7.5,3.5,11.681676585676911,4.5,0,1.584962500721156,0,0,7.255412984064776,9.265614570460752,0,0,11.26283925556662,10.457380879072534,0,0,0,0,0],"m":[1,0,0,1,1,0,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,13,14,0,16,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,10,9,6,0,8,0,0,0,28,27,18,15,0,17,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.0009132264,0,0,0,-0.0001751799,0,0.04808792,-0.001581857,0.05279631,0,0,0,0,-0.04291609,0,-0.02080327,0.009811948,0,0,-0.01052853,0.008834626,0,0,0.05281529,0.004345018,-0.007790232,-0.02689716,0.004052124]},{"f":[22,-1,22,22,21,-1,-1,-1,21,17,22,-1,-1,-1,4,15,-1,2,-1,22,-1,-1,18,2,-1,-1,15,-1,-1],"t":[2.69615871138938,0,5.9525054311951475,4.926654777701837,6.5,0,0,0,2.5,1.292481250360578,12.027747761539269,0,0,0,9.266718101290426,1.0,0,9.051177541662383,0,13.005894632618489,0,0,10.457893746154532,10.680359255641626,0,0,9.862637357558794,0,0],"m":[0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,0,9,10,11,0,0,0,15,16,0,18,0,20,0,0,23,24,0,0,27,0,0],"r":[2,0,8,7,6,0,0,0,14,13,12,0,0,0,22,17,0,19,0,21,0,0,26,25,0,0,28,0,0],"v":[0,-0.02657932,0,0,0,-0.02889629,0.01252947,0.042177,0,0,0,-0.02814726,0.01226751,-0.03642527,0,0,0.02015033,0,-0.0001881156,0,0.002778319,0.02767714,0,0,-0.03615105,0.02904504,0,0.0371524,-0.01354331]},{"f":[33,33,23,-1,-1,-1,33,22,-1,-1,3,33,22,-1,3,-1,-1,-1,17,33,33,33,33,-1,-1,-1,-1,-1,-1],"t":[24.416594167368267,23.58371303411409,8.5,0,0,0,26.99635432609064,12.121442107629743,0,0,0.9218559218559219,38.67649696721479,16.838461650489297,0,0.08766192532385064,0,0,0,7.980000966034041,37.56709924177028,35.37421412138842,32.199935296219856,28.819877645289402,0,0,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,0,11,12,13,0,15,0,0,0,19,20,21,22,23,0,0,0,0,0,0],"r":[6,5,4,0,0,0,10,9,0,0,18,17,14,0,16,0,0,0,28,27,26,25,24,0,0,0,0,0,0],"v":[0,0,0,5.598831e-05,-0.02696004,-0.01840146,0,0,0.00508871,0.03684558,0,0,0,-0.004811488,0,-0.005468393,0.03706093,0.04711068,0,0,0,0,0,-0.003725716,0.02228911,-0.02331346,0.02495639,-0.01304203,-0.03588906]},{"f":[25,22,-1,22,1,4,21,-1,-1,22,-1,22,0,-1,-1,-1,-1,22,-1,22,18,22,-1,-1,18,-1,-1,-1,-1],"t":[6.5,2.69615871138938,0,7.770639075561187,5.489664082687339,1.292481250360578,7.5,0,0,5.9525054311951475,0,6.671801068872866,6.5,0,0,0,0,8.133841411710264,0,12.121442107629743,10.609627990977145,10.27595502635321,0,0,10.682994316786338,0,0,0,0],"m":[1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,7,0,0,10,0,12,13,0,0,0,0,18,0,20,21,22,0,0,25,0,0,0,0],"r":[28,3,0,17,16,9,8,0,0,11,0,15,14,0,0,0,0,19,0,27,24,23,0,0,26,0,0,0,0],"v":[0,0,-0.0262417,0,0,0,0,-0.03768972,0.01197017,0,0.03824131,0,0,-0.04050405,0.0005897113,0.006449357,0.02782908,0,-0.03658672,0,0,0,0.001107241,-0.009685326,0,0.04904049,0.00055213,0.001018606,-0.02499145]},{"f":[33,33,23,-1,-1,-1,33,22,-1,-1,3,33,22,-1,3,-1,-1,-1,17,33,22,33,-1,-1,22,-1,-1,-1,-1],"t":[24.416594167368267,23.58371303411409,8.5,0,0,0,26.99635432609064,12.121442107629743,0,0,0.9218559218559219,38.67649696721479,16.838461650489297,0,0.08766192532385064,0,0,0,7.980000966034041,37.56709924177028,16.98920034885663,29.807475161786478,0,0,17.083799521744545,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,0,11,12,13,0,15,0,0,0,19,20,21,22,0,0,25,0,0,0,0],"r":[6,5,4,0,0,0,10,9,0,0,18,17,14,0,16,0,0,0,28,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0,3.970423e-05,-0.02625631,-0.01753681,0,0,0.005134633,0.03587205,0,0,0,-0.00463181,0,-0.005306734,0.0360213,0.04575587,0,0,0,0,-0.0007939868,0.02214049,0,-0.03402329,0.003480547,-0.01252444,-0.03477807]},{"f":[16,17,18,-1,-1,16,17,-1,17,-1,-1,22,-1,18,-1,-1,16,22,17,-1,-1,-1,21,-1,1,22,-1,-1,-1],"t":[7.930737337562887,8.496347487943302,7.6181566135214025,0,0,5.044394119358453,9.229707804263573,0,10.269270673170688,0,0,15.676331898425197,0,8.729619742794132,0,0,8.060695931687555,15.676331898425197,6.219136028062415,0,0,0,3.5,0,1.4859550561797752,15.756821747580634,0,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,0,9,0,0,12,0,14,0,0,17,18,19,0,0,0,23,0,25,26,0,0,0],"r":[16,5,4,0,0,11,8,0,10,0,0,13,0,15,0,0,22,21,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,-0.001378356,-0.04167064,0,0,0.01960716,0,-0.04588864,0.001237215,0,0.06547288,0,0.04692287,-0.02603725,0,0,0,0.02064676,-0.008731698,0.03856093,0,-0.02623483,0,0,0.005461226,0.04140703,2.372956e-05]},{"f":[4,4,15,-1,-1,16,-1,-1,2,21,21,22,-1,-1,-1,-1,1,17,2,22,15,2,-1,-1,-1,-1,-1,-1,-1],"t":[4.951187557243013,4.554262228389085,1.584962500721156,0,0,10.462502272597265,0,0,5.40929108874043,7.5,3.5,13.129761635363703,0,0,0,0,7.40625,6.734566509914796,6.793888758164105,13.054613167076562,5.044394119358453,5.459058925515887,0,0,0,0,0,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,0,0,0,0,17,18,19,20,21,22,0,0,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,16,15,14,13,0,0,0,0,28,27,26,25,24,23,0,0,0,0,0,0,0],"v":[0,0,0,-0.02494754,-0.0002139714,0,-0.03807139,-0.002140533,0,0,0,0,0.05606312,-0.003243222,0.05835315,0.0001669682,0,0,0,0,0,0,-0.0374641,0.01527579,-0.02687666,-0.04115644,-0.0002324589,0.002772271,0.01661506]},{"f":[25,4,19,15,-1,-1,36,-1,16,17,-1,-1,18,0,-1,-1,32,-1,33,-1,21,-1,-1,1,3,-1,-1,-1,-1],"t":[6.5,2.453445297804259,0.5,3.169925001442312,0,0,2.5,0,7.118941072723508,1.792481250360578,0,0,2.160964047443681,9.5,0,0,7.491051153478379,0,44.805779984159756,0,3.5,0,0,12.18570183086312,0.0339080459770115,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,9,10,0,0,13,14,0,0,17,0,19,0,21,0,0,24,25,0,0,0,0],"r":[28,23,6,5,0,0,8,0,12,11,0,0,16,15,0,0,18,0,20,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02697864,0.003248511,0,-0.03079984,0,0,-0.02387205,0.01201437,0,0,-0.002865558,0.03631147,0,-0.05237024,0,0.003880128,0,0.003349352,-0.03890548,0,0,0.03030723,0.0004141825,0.03408797,-0.02450605]},{"f":[21,4,2,2,1,-1,2,22,-1,23,-1,-1,15,17,-1,-1,-1,22,-1,-1,-1,0,-1,-1,22,0,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0.6824793747286149,0,7.005613627711627,14.116600814324018,0,1.5,0,0,10.113742166049189,10.620215166896223,0,0,0,13.005894632618489,0,0,0,3.5,0,0,15.756821747580634,7.5,0,0,0],"m":[1,1,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0],"l":[1,2,3,4,5,0,7,8,0,10,0,0,13,14,0,0,0,18,0,0,0,22,0,0,25,26,0,0,0],"r":[24,21,20,17,6,0,12,9,0,11,0,0,16,15,0,0,0,19,0,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,-0.03230585,0,0,-0.001304598,0,0.01527706,0.0001979062,0,0,-0.02298694,0.006158863,0.003690036,0,0.00307307,0.03642986,-0.03650131,0,0.0005006791,0.04558712,0,0,0.005482793,0.04064822,-0.03301573]},{"f":[8,3,22,-1,-1,2,1,-1,4,21,22,-1,37,-1,-1,15,15,-1,-1,16,-1,-1,16,21,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,0.3126984126984127,0,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,5.169925001442312,4.087462841250339,0,0,6.108524456778169,0,0,10.462502272597265,6.5,0,0,0,0,0],"m":[1,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,11,0,13,0,0,16,17,0,0,20,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,8,0,22,15,12,0,14,0,0,19,18,0,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.03173148,0.01704593,0,0,-0.0299514,0,0,0,0.01760053,0,-0.0414468,0.01733499,0,0,0.0004933107,0.01126183,0,-0.009502414,0.0004843427,0,0,0.02019691,-0.01312088,-0.03649686,0.02988037,0.01116644]},{"f":[33,33,21,2,-1,33,-1,-1,2,22,-1,-1,30,4,-1,20,22,17,-1,22,-1,-1,-1,-1,-1,-1,33,-1,-1],"t":[20.752541988539424,18.891308976303876,3.5,2.453445297804259,0,15.856946627097248,0,0,1.292481250360578,15.212004812197344,0,0,2.5,1.792481250360578,0,1.5,16.725912049439096,2.69615871138938,0,12.613886328427084,0,0,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,13,14,0,16,17,18,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,12,11,0,0,24,15,0,23,22,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0,-0.02452999,0,-0.04127794,0.01576108,0,0,0.005046867,0.03760152,0,0,-0.03820311,0,0,0,-0.02002746,0,-0.004205467,0.02378399,0.01871024,0.02319121,-0.02687431,-0.03383524,0,0.02456691,0.000360695]},{"f":[16,-1,16,-1,21,-1,16,4,-1,-1,20,15,22,15,17,-1,22,-1,-1,-1,4,-1,-1,-1,22,22,-1,-1,-1],"t":[7.930737337562887,0,8.618385502258606,0,3.5,0,9.271463027904375,3.245926548164837,0,0,0.5,10.681238411777805,12.35227706330556,10.066089190457772,2.903677461028802,0,8.658207063149977,0,0,0,5.89476682248518,0,0,0,16.73920708776685,16.25454052700782,0,0,0],"m":[1,0,0,0,0,0,0,1,0,0,1,0,1,1,1,0,1,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,0,3,0,5,0,7,8,0,0,11,12,13,14,15,0,17,0,0,0,21,0,0,0,25,26,0,0,0],"r":[2,0,4,0,6,0,10,9,0,0,24,23,20,19,16,0,18,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,-0.001063955,0,0.01333103,0,-0.02688735,0,0,-0.04511819,0.0001526749,0,0,0,0,0,0.010377,0,0.004737869,-0.03250155,0.04380135,0,0.04847562,0.01819511,-0.001277052,0,0,-0.002613306,0.02752653,-0.02741783]},{"f":[21,4,2,2,1,-1,-1,22,-1,-1,-1,16,-1,16,-1,-1,22,0,22,-1,16,4,-1,16,-1,-1,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0.6824793747286149,0,0,13.005894632618489,0,0,0,4.247927513443585,0,10.696098170955864,0,0,15.756821747580634,7.5,10.735831902194036,0,11.98370619265935,5.065928480304397,0,7.266786540694901,0,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,0,12,0,14,0,0,17,18,19,0,21,22,0,24,0,0,0,0,0],"r":[16,11,10,7,6,0,0,9,0,0,0,13,0,15,0,0,28,27,20,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03171008,-0.0004766024,0,0.002888615,0.03560998,-0.03450842,0,-0.009820952,0,0.04111999,-0.000496728,0,0,0,0.01374746,0,0,-0.03303478,0,-0.01365763,0.01660294,0.02269609,0.04013746,-0.03237197]},{"f":[4,4,21,18,22,-1,-1,2,-1,-1,2,1,-1,4,15,-1,25,-1,-1,4,-1,-1,22,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,5.065928480304397,13.827286379004683,0,0,3.903677461028802,0,0,9.051177541662383,0.39811320754716983,0,7.950787011501063,5.169925001442312,0,2.5,0,0,8.950330235806282,0,0,13.005894632618489,0,0,0,2.160964047443681,0,0],"m":[1,1,0,1,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,12,0,14,15,0,17,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,10,7,6,0,0,9,0,0,22,13,0,19,16,0,18,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,-0.04113697,0.01756147,0,-0.02222392,0.05127317,0,0,-0.03508693,0,0,0.003001537,0,-0.0004957098,-0.0305543,0,-0.03321926,0.001836429,0,0.00251609,0.02686963,-0.01371772,0,-0.005371275,0.03747273]},{"f":[22,23,21,4,-1,22,22,23,2,16,15,-1,-1,22,-1,-1,-1,-1,-1,-1,-1,22,33,-1,-1,1,-1,-1,-1],"t":[17.105450904090162,7.5,10.5,4.951187557243013,0,17.074218326240924,17.069460217922263,6.5,5.40929108874043,5.700439718141092,3.584962500721156,0,0,14.03140051979143,0,0,0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,1,0,1,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,11,0,0,14,0,0,0,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,21,20,5,0,19,18,17,16,13,12,0,0,15,0,0,0,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.0009997352,0,0,0,0,0,0,0.007620789,0.07040175,0,0.0003360312,0.03682267,-0.0004932758,0.03414001,-0.03060701,0.02619212,0.02167428,0,0,-0.0215016,0.01894759,0,0.01481926,-0.04532985,0.0117433]},{"f":[23,22,22,-1,-1,17,-1,-1,21,24,-1,-1,18,22,-1,22,-1,18,23,-1,-1,-1,17,19,-1,-1,18,-1,-1],"t":[2.5,16.819976309430963,16.25454052700782,0,0,2.453445297804259,0,0,6.5,12.5,0,0,6.499823868264185,8.133841411710264,0,15.676331898425197,0,6.076776015854056,3.5,0,0,0,8.496347487943302,0.5,0,0,9.229707804263573,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,0,13,14,0,16,0,18,19,0,0,0,23,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,12,11,0,0,22,15,0,17,0,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0.0006735762,-0.03532002,0,-0.007554074,0.03523631,0,0,0.006886978,-0.0390611,0,0,-0.03701584,0,0.01029627,0,0,-0.04855926,-5.871044e-05,0.03292958,0,0,-0.004715374,-0.04341279,0,0.0144666,-0.00938543]},{"f":[8,3,22,-1,-1,2,1,-1,23,22,35,32,22,-1,33,0,-1,-1,-1,36,33,-1,-1,-1,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,0.3126984126984127,0,1.5,15.676331898425197,2.5,7.014643613484123,14.03140051979143,0,21.257266042509173,2.5,0,0,0,3.5,37.63181656149436,0,0,0,0,0,0,0,0],"m":[1,0,0,0,0,1,0,0,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,11,12,13,0,15,16,0,0,0,20,21,0,0,0,0,0,0,0,0],"r":[28,5,4,0,0,27,8,0,26,25,24,19,14,0,18,17,0,0,0,23,22,0,0,0,0,0,0,0,0],"v":[0,0,0,-0.03118631,0.01630555,0,0,-0.02888304,0,0,0,0,0,-0.0002720803,0,0,-0.0006481759,0.03508697,-0.008311087,0,0,-0.05644862,4.340041e-05,-0.00086682,0.01809704,0.03783801,-0.001507857,0.02931823,0.01105729]},{"f":[16,17,17,17,-1,-1,-1,-1,21,-1,16,-1,16,4,-1,-1,20,15,22,15,17,-1,22,-1,-1,-1,-1,-1,-1],"t":[7.930737337562887,11.78143940991899,10.682994316786338,8.496347487943302,0,0,0,0,3.5,0,8.618385502258606,0,9.271463027904375,3.245926548164837,0,0,0.5,10.681238411777805,12.35227706330556,10.066089190457772,2.903677461028802,0,8.658207063149977,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,1,1,1,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,0,9,0,11,0,13,14,0,0,17,18,19,20,21,0,23,0,0,0,0,0,0],"r":[8,7,6,5,0,0,0,0,10,0,12,0,16,15,0,0,28,27,26,25,22,0,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.001975277,0.008311174,-0.04418727,0.02003739,0,-0.02478347,0,0.01357791,0,0,-0.04474785,0.0002253555,0,0,0,0,0,0.009894216,0,0.004440165,-0.03099192,0.04150549,0.03165619,-0.001204267,-0.002186118]},{"f":[22,-1,33,22,33,-1,-1,-1,33,-1,22,22,22,22,-1,4,16,-1,-1,-1,17,22,-1,-1,-1,18,-1,-1,-1],"t":[2.69615871138938,0,24.416594167368267,16.98920034885663,23.58371303411409,0,0,0,26.99635432609064,0,17.078739781790915,17.069460217922263,16.978341770814396,16.73920708776685,0,3.753897320099348,9.92481250360578,0,0,0,6.781300095563516,16.98920034885663,0,0,0,6.076776015854056,0,0,0],"m":[0,0,0,1,0,0,0,0,0,0,1,1,1,1,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,0,11,12,13,14,0,16,17,0,0,0,21,22,0,0,0,26,0,0,0],"r":[2,0,8,7,6,0,0,0,10,0,28,25,20,15,0,19,18,0,0,0,24,23,0,0,0,27,0,0,0],"v":[0,-0.02547825,0,0,0,-0.0002476696,-0.01604304,-0.02717801,0,0.01936865,0,0,0,0,0.0007040368,0,0,-0.02797233,0.02074528,0.003387655,0,0,0.02338323,-0.03301097,0.03359794,0,-0.04653524,-0.003239492,0.01124988]},{"f":[22,-1,23,22,22,-1,-1,17,-1,-1,21,24,-1,-1,18,22,-1,22,-1,18,-1,-1,17,19,-1,-1,18,-1,-1],"t":[2.69615871138938,0,2.5,16.819976309430963,16.25454052700782,0,0,2.453445297804259,0,0,6.5,12.5,0,0,6.499823868264185,8.133841411710264,0,15.676331898425197,0,6.076776015854056,0,0,8.496347487943302,0.5,0,0,9.229707804263573,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,0,11,12,0,0,15,16,0,18,0,20,0,0,23,24,0,0,27,0,0],"r":[2,0,10,7,6,0,0,9,0,0,14,13,0,0,22,17,0,19,0,21,0,0,26,25,0,0,28,0,0],"v":[0,-0.02515423,0,0,0,0.000704717,-0.03488848,0,-0.006973633,0.03300325,0,0,0.006510812,-0.03826711,0,0,-0.03652593,0,0.009660491,0,-0.001743891,0.03179637,0,0,-0.004727726,-0.04293511,0,0.0133122,-0.009550406]},{"f":[25,23,22,22,-1,-1,-1,21,24,-1,-1,18,18,22,-1,22,-1,-1,-1,1,-1,16,22,-1,18,-1,-1,-1,-1],"t":[6.5,2.5,16.819976309430963,16.25454052700782,0,0,0,6.5,12.5,0,0,6.400854179458231,6.076776015854056,8.133841411710264,0,15.10428213920758,0,0,0,1.99834912080135,0,10.681238411777805,17.094543568094064,0,8.729619742794132,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,14,0,16,0,0,0,20,0,22,23,0,25,0,0,0,0],"r":[28,7,6,5,0,0,0,11,10,0,0,19,18,15,0,17,0,0,0,21,0,27,24,0,26,0,0,0,0],"v":[0,0,0,0,0.0006823822,-0.03417564,0.02017578,0,0,0.006218864,-0.03751466,0,0,0,-0.03607472,0,0.01203601,-0.001452403,0.03581099,0,0.02141946,0,0,-0.02716145,0,0.01481245,-0.02622359,0.01039911,-0.0232857]},{"f":[36,-1,33,33,15,22,17,22,-1,-1,22,-1,1,-1,-1,-1,15,4,-1,-1,22,20,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.891308976303876,5.169925001442312,15.212004812197344,2.69615871138938,13.054613167076562,0,0,12.613886328427084,0,6.276785714285714,0,0,0,7.266786540694901,1.792481250360578,0,0,16.725912049439096,0.5,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,0,0,17,18,0,0,21,22,0,0,0,0,27,0,0],"r":[2,0,26,25,16,15,10,9,0,0,12,0,14,0,0,0,20,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0.02050397,0,0,0,0,0,0,0.002142837,-0.04767578,0,-0.002264973,0,-0.003538711,0.03637511,0.03925254,0,0,-0.05528219,-0.01087847,0,0,-0.01976376,-0.0008804921,0.01868639,-0.03369838,0,0.02288101,0.000318746]},{"f":[25,37,23,15,-1,-1,-1,4,25,-1,-1,35,2,-1,18,-1,-1,-1,17,-1,22,36,16,-1,-1,-1,22,-1,-1],"t":[2.5,9.205,8.5,5.78135971352466,0,0,0,1.292481250360578,0.5,0,0,2.5,2.453445297804259,0,1.292481250360578,0,0,0,1.292481250360578,0,12.613886328427084,2.5,4.247927513443585,0,0,0,12.637102544927284,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,15,0,0,0,19,0,21,22,23,0,0,0,27,0,0],"r":[18,7,6,5,0,0,0,11,10,0,0,17,14,0,16,0,0,0,20,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0.001951522,-0.001060956,-0.02363016,0,0,0.003567369,0.03744002,0,0,0.003567369,0,0.003567369,0.028041,0.0456299,0,-0.03416587,0,0,0,-0.02033631,-0.0008586856,0.008761099,0,0.03451626,-0.0121327]},{"f":[1,16,-1,-1,1,-1,22,21,22,15,-1,21,-1,-1,21,-1,23,24,-1,18,-1,2,-1,-1,-1,33,-1,-1,-1],"t":[0.01933482600988898,11.031356596255709,0,0,0.3126984126984127,0,12.121442107629743,5.5,8.723206471664755,4.087462841250339,0,3.5,0,0,3.5,0,0.5,12.5,0,5.065928480304397,0,2.903677461028802,0,0,0,29.807475161786478,0,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,0,5,0,7,8,9,10,0,12,0,0,15,0,17,18,0,20,0,22,0,0,0,26,0,0,0],"r":[4,3,0,0,6,0,28,25,14,11,0,13,0,0,16,0,24,19,0,21,0,23,0,0,0,27,0,0,0],"v":[0,0,-0.03400401,0.04537052,0,-0.03789964,0,0,0,0,-0.007184911,0,0.06628803,-0.0007320487,0,-0.04653108,0,0,-0.04477798,0,0.04281073,0,-0.04643455,0.004750472,-0.04433968,0,-0.0002987713,0.03029624,0.0009458898]},{"f":[36,-1,33,33,21,2,-1,33,-1,-1,2,-1,23,-1,4,-1,20,22,33,-1,22,-1,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.99840809449714,3.5,2.453445297804259,0,15.856946627097248,0,0,1.292481250360578,0,0.5,0,1.792481250360578,0,1.5,16.725912049439096,8.065928480304397,0,12.613886328427084,0,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,0,13,0,15,0,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,10,7,0,9,0,0,12,0,14,0,16,0,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0.02044277,0,0,0,0,-0.02257426,0,-0.0394709,0.01284038,0,0.01162218,0,-0.02722273,0,-0.03707243,0,0,0,-0.01786116,0,-0.004003238,0.03405199,0.01764761,0.01851457,-0.03492638,0,0.02221299,0.00027235]},{"f":[22,-1,21,4,2,2,-1,-1,-1,16,-1,16,-1,-1,22,0,17,2,17,-1,-1,15,22,-1,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,9.5,9.328332883600808,9.293467526905953,9.051177541662383,0,0,0,4.247927513443585,0,10.696098170955864,0,0,15.756821747580634,7.5,6.781300095563516,1.292481250360578,1.792481250360578,0,0,7.679480099505446,10.735831902194036,0,0,0,0,0,0],"m":[0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,0,12,0,0,15,16,17,18,19,0,0,22,23,0,0,0,0,0,0],"r":[2,0,14,9,8,7,0,0,0,11,0,13,0,0,28,27,26,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,-0.02485101,0,0,0,0,-0.0005769298,0.007877116,-0.03295867,0,-0.009814891,0,0.04109197,-0.001516849,0,0,0,0,0,-0.001697724,-0.03655869,0,0,0.01019393,-0.01758902,0.02174561,0.02202161,0.03911782,-0.03137057]},{"f":[22,23,21,22,-1,23,33,1,-1,-1,-1,-1,24,-1,1,16,15,-1,-1,-1,-1,22,33,-1,-1,1,-1,-1,-1],"t":[17.105450904090162,7.5,9.5,14.116600814324018,0,1.5,16.993854061608424,2.0015923566878984,0,0,0,0,13.5,0,6.015625,7.6653359171851765,6.0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,1,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,0,0,0,0,13,0,15,16,17,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,21,12,5,0,11,10,9,0,0,0,0,14,0,20,19,18,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.001213727,0,0,0,-0.00379562,0.04623889,-0.001725897,-0.0003100634,0,0.03864936,0,0,0,0.01213902,-0.01607928,0.01427906,-0.02032466,0,0,-0.02003348,0.01851451,0,0.01572385,-0.04412042,0.01089214]},{"f":[16,17,17,17,18,-1,-1,-1,-1,-1,21,-1,16,-1,16,3,4,-1,-1,-1,1,15,22,-1,-1,16,-1,-1,-1],"t":[7.930737337562887,11.78143940991899,10.682994316786338,8.496347487943302,7.6181566135214025,0,0,0,0,0,3.5,0,8.618385502258606,0,9.271463027904375,0.9816932380428807,3.6427011094311244,0,0,0,1.4859550561797752,10.066089190457772,9.178103463238745,0,0,11.221587121264806,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,0,0,0,11,0,13,0,15,16,17,0,0,0,21,22,23,0,0,26,0,0,0],"r":[10,9,8,7,6,0,0,0,0,0,12,0,14,0,20,19,18,0,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.001339186,-0.04041629,0.007801152,-0.04355901,0.01889013,0,-0.02376542,0,0.01244373,0,0,0,-0.008579983,0.0212306,-0.03469697,0,0,0,0.01037786,0.05008463,0,-0.02308971,0.01988139,0.0004778028]},{"f":[8,3,22,-1,-1,2,4,21,18,22,-1,-1,22,-1,-1,15,-1,16,21,-1,-1,-1,16,21,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,9.266718101290426,2.5,5.065928480304397,13.827286379004683,0,0,11.36027577861459,0,0,5.169925001442312,0,6.108524456778169,4.5,0,0,0,10.462502272597265,6.5,0,0,0,0,0],"m":[1,0,0,0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,0,13,0,0,16,0,18,19,0,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,22,15,12,11,0,0,14,0,0,17,0,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.03014898,0.01538133,0,0,0,0,0,-0.04066202,0.01613425,0,0.04676049,-0.02137157,0,0.00243548,0,0,-0.02427941,-0.003852762,0.0003589115,0,0,0.01893244,-0.01295922,-0.03506403,0.02797847,0.01049968]},{"f":[4,4,21,18,22,-1,-1,2,-1,-1,25,2,4,-1,-1,-1,18,-1,22,36,18,-1,-1,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,5.065928480304397,13.827286379004683,0,0,3.903677461028802,0,0,2.5,9.051177541662383,7.754129516000594,0,0,0,1.292481250360578,0,12.613886328427084,2.5,1.792481250360578,0,0,0,0,0,2.160964047443681,0,0],"m":[1,1,0,1,1,0,0,1,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,12,13,0,0,0,17,0,19,20,21,0,0,0,0,0,27,0,0],"r":[26,25,10,7,6,0,0,9,0,0,16,15,14,0,0,0,18,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,-0.04031961,0.01569653,0,-0.02149531,0.04596397,0,0,0,0.0006965278,-0.01188369,0.00810894,0,-0.04416768,0,0,0,-0.001686983,-0.01925433,0.007898811,0.01319782,-0.01340446,0,-0.005718443,0.03690042]},{"f":[22,-1,22,1,21,1,-1,17,2,-1,-1,-1,-1,-1,22,-1,22,18,22,20,-1,-1,21,-1,-1,18,-1,-1,-1],"t":[2.69615871138938,0,7.770639075561187,5.489664082687339,6.5,3.015625,0,9.265614570460752,4.882435795368045,0,0,0,0,0,8.133841411710264,0,12.121442107629743,10.609627990977145,10.27595502635321,0.5,0,0,4.5,0,0,10.682994316786338,0,0,0],"m":[0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,0,0,0,0,0,15,0,17,18,19,20,0,0,23,0,0,26,0,0,0],"r":[2,0,14,13,12,7,0,11,10,0,0,0,0,0,16,0,28,25,22,21,0,0,24,0,0,27,0,0,0],"v":[0,-0.02453138,0,0,0,0,-0.04049141,0,0,-0.01873646,0.007323555,0.01894908,0.006562314,0.0246449,0,-0.03416198,0,0,0,0,0.00876268,-0.004773854,0,-0.04713527,-0.005165242,0,0.04511654,0.0006395994,0.0007737751]},{"f":[22,-1,22,22,21,-1,-1,-1,16,17,18,-1,-1,16,22,22,-1,-1,-1,22,-1,18,-1,-1,21,-1,16,-1,-1],"t":[2.69615871138938,0,5.9525054311951475,4.926654777701837,6.5,0,0,0,7.930737337562887,8.496347487943302,7.6181566135214025,0,0,5.044394119358453,16.978341770814396,12.35227706330556,0,0,0,15.676331898425197,0,8.729619742794132,0,0,3.5,0,8.618385502258606,0,0],"m":[0,0,0,1,1,0,0,0,1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,10,11,0,0,14,15,16,0,0,0,20,0,22,0,0,25,0,27,0,0],"r":[2,0,8,7,6,0,0,0,24,13,12,0,0,19,18,17,0,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.02422063,0,0,0,-0.02753878,0.01134654,0.03485806,0,0,0,-0.001513643,-0.04000898,0,0,0,0.004981419,-0.02006701,0.03306218,0,0.06030667,0,0.04459808,-0.02604242,0,-0.02295241,0,0.01175113,0.0006843663]},{"f":[22,23,21,4,2,-1,-1,0,22,-1,-1,-1,0,1,16,4,-1,-1,-1,-1,-1,22,33,-1,-1,1,-1,-1,-1],"t":[17.105450904090162,7.5,9.5,9.328332883600808,9.293467526905953,0,0,3.5,12.457649623491124,0,0,0,7.5,6.015625,7.6653359171851765,3.084962500721156,0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,0,13,14,15,16,0,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,21,12,7,6,0,0,11,10,0,0,0,20,19,18,17,0,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.000185369,-0.03137401,0,0,0.008606427,-0.0206178,0.043221,0,0,0,0,-0.02775815,0.005387595,0.01383997,-0.01944011,0.03823548,0,0,-0.01941925,0.01791244,0,0.01508148,-0.04342878,0.01049646]},{"f":[22,-1,23,22,15,15,-1,-1,15,-1,4,1,15,-1,-1,-1,-1,-1,18,2,-1,4,-1,-1,17,-1,17,-1,-1],"t":[2.69615871138938,0,1.5,15.676331898425197,10.462502272597265,10.043027283594547,0,0,10.46352437327118,0,5.065928480304397,4.008411214953271,10.794415866350105,0,0,0,0,0,9.229707804263573,9.012591414317754,0,9.037514954341955,0,0,9.265614570460752,0,10.009557117932596,0,0],"m":[0,0,1,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,0,11,12,13,0,0,0,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,18,17,8,7,0,0,10,0,16,15,14,0,0,0,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.02391201,0,0,0,0,0.0004602681,-0.02114284,0,0.04489037,0,0,0,0.04586316,0.01055539,-0.008378484,-0.02838705,0.03640799,0,0,-0.00117253,0,0.04571736,-0.004295459,0,-0.04516115,0,0.01892066,-0.01268308]},{"f":[33,23,21,15,15,-1,-1,15,-1,15,23,-1,-1,-1,-1,-1,3,22,-1,-1,33,-1,33,-1,33,22,-1,-1,-1],"t":[29.807475161786478,8.5,10.5,10.457380879072534,9.756556322524087,0,0,10.462502272597265,0,10.681238411777805,0.5,0,0,0,0,0,0.33771929824561403,17.02942582914998,0,0,32.199935296219856,0,34.58978958281367,0,37.56709924177028,16.98920034885663,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,11,0,0,0,0,0,17,18,0,0,21,0,23,0,25,26,0,0,0],"r":[16,15,14,7,6,0,0,9,0,13,12,0,0,0,0,0,20,19,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,-0.0003790543,0.008291678,0,-0.040128,0,0,0.05803811,0.009232172,-0.002909215,0.0190022,-0.01801611,0,0,0.03093176,-0.001897351,0,0.02445234,0,-0.0322784,0,0,0.0266269,-0.01597224,-0.01197799]},{"f":[33,32,33,33,33,33,1,24,33,33,-1,-1,15,-1,4,-1,-1,-1,-1,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,32.199935296219856,29.807475161786478,12.64516129032258,9.5,13.895530557358477,11.716303571902237,0,0,6.169925001442312,0,4.128693921346326,0,0,0,0,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,10,0,0,13,0,15,0,0,0,0,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,20,19,18,17,12,11,0,0,14,0,16,0,0,0,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0.0001607452,0.02579381,0,-0.02174558,0,-0.00410909,0.008030233,0.01722699,-0.01199041,0.02344105,-0.01712159,0,0,0.009329272,0.04356282,-0.01929627,-0.02319546,-0.03854973,0.009643183]},{"f":[1,17,18,16,21,22,22,-1,-1,-1,18,18,-1,-1,22,-1,-1,16,-1,22,-1,2,-1,-1,-1,-1,1,-1,-1],"t":[2.532051282051282,11.651826559171445,11.26283925556662,9.756556322524087,6.5,8.723206471664755,7.5621616439616925,0,0,0,6.4178278454199,1.292481250360578,0,0,10.35974204013634,0,0,10.060695931687555,0,9.024458927812285,0,2.453445297804259,0,0,0,0,2.9611191860465116,0,0],"m":[0,1,1,1,1,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,15,0,0,18,0,20,0,22,0,0,0,0,27,0,0],"r":[26,25,24,17,10,9,8,0,0,0,14,13,0,0,16,0,0,19,0,21,0,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,-0.03509058,0.04877611,-0.03626321,0,0,-0.003711011,-0.03929201,0,-0.006199358,0.04543044,0,0.0484611,0,0.01275903,0,-0.03512803,-0.0005994074,-0.0465203,0.0128422,0,0.05052738,0.0001371739]},{"f":[33,32,33,33,33,33,23,21,15,-1,15,-1,15,23,-1,-1,-1,-1,-1,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,32.199935296219856,29.807475161786478,8.5,10.5,10.457380879072534,0,10.462502272597265,0,10.681238411777805,0.5,0,0,0,0,0,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,11,0,13,14,0,0,0,0,0,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,20,19,18,17,10,0,12,0,16,15,0,0,0,0,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0.0003446176,0,-0.03950454,0,0,0.05447682,0.008849646,-0.002559884,0.01820507,-0.01766443,0.0224661,-0.01647238,0,0,0.008887926,0.04202946,-0.01854242,-0.02235574,-0.03810222,0.009178049]},{"f":[27,23,22,22,-1,-1,-1,21,0,-1,-1,18,22,-1,17,-1,-1,17,19,-1,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,2.5,16.819976309430963,16.25454052700782,0,0,0,6.5,6.5,0,0,6.499823868264185,8.882387211598338,0,0.5,0,0,8.496347487943302,0.5,0,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,15,0,0,18,19,0,0,0,23,0,25,0,27,0,0],"r":[22,7,6,5,0,0,0,11,10,0,0,17,14,0,16,0,0,21,20,0,0,0,24,0,26,0,28,0,0],"v":[0,0,0,0,0.0004583834,-0.03360559,0.01921091,0,0,-0.04184666,-0.006392555,0,0,-0.02195227,0,0.04321053,0.002300021,0,0,-0.004263506,-0.04158171,0.001277264,0,0.01728573,0,-0.0147515,0,0.0176754,-0.01021025]},{"f":[1,16,-1,-1,1,-1,22,22,21,1,-1,-1,4,16,-1,-1,2,-1,-1,21,-1,17,17,-1,-1,18,-1,-1,-1],"t":[0.01933482600988898,11.031356596255709,0,0,0.3126984126984127,0,12.121442107629743,9.087343393278285,3.5,3.4375,0,0,6.434604125275413,3.169925001442312,0,0,6.894613978429,0,0,3.5,0,10.609627990977145,3.245926548164837,0,0,10.682994316786338,0,0,0],"m":[0,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,0,5,0,7,8,9,10,0,0,13,14,0,0,17,0,0,20,0,22,23,0,0,26,0,0,0],"r":[4,3,0,0,6,0,28,19,12,11,0,0,16,15,0,0,18,0,0,21,0,25,24,0,0,27,0,0,0],"v":[0,0,-0.03262876,0.04355081,0,-0.03658632,0,0,0,0,0.05832828,-0.02110503,0,0,-0.02163607,0.0001817776,0,0.0282215,-0.0002435664,0,-0.04566864,0,0,0.0002466937,-0.009592551,0,0.0404941,0.004142735,0.0009244092]},{"f":[1,17,18,1,16,1,-1,16,22,-1,-1,15,-1,-1,-1,17,4,-1,-1,23,22,-1,-1,-1,-1,-1,1,-1,-1],"t":[2.532051282051282,11.651826559171445,11.26283925556662,1.4859550561797752,11.221587121264806,0.39811320754716983,0,9.862637357558794,10.896650035079205,0,0,10.066089190457772,0,0,0,7.2941523604036345,3.753897320099348,0,0,1.5,10.224917391507956,0,0,0,0,0,2.9611191860465116,0,0],"m":[0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,0,16,17,0,0,20,21,0,0,0,0,0,27,0,0],"r":[26,25,24,15,14,7,0,11,10,0,0,13,0,0,0,19,18,0,0,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.0428421,0,0,0.002349578,-0.009495544,0,0.03904236,-0.0120723,0.01950438,0,0,-0.03403728,-0.01021274,0,0,0.00432344,-0.032232,0.02828478,-0.04577896,0.01215068,0,0.04902383,0.0001374588]},{"f":[22,22,22,22,22,-1,-1,32,-1,2,-1,22,-1,-1,23,17,22,-1,-1,-1,-1,18,-1,18,-1,-1,2,-1,-1],"t":[17.07907323966382,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,6.208926257442949,0,3.903677461028802,0,16.819976309430963,0,0,7.5,6.781300095563516,16.98920034885663,0,0,0,0,6.076776015854056,0,7.383290334558488,0,0,4.0437314206251695,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,12,0,0,15,16,17,0,0,0,0,22,0,24,0,0,27,0,0],"r":[26,21,14,7,6,0,0,9,0,11,0,13,0,0,20,19,18,0,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-5.862762e-05,0.04289036,0,0.0140831,0,-0.03705638,0,-0.02073352,0.01322906,0,0,0,0.02111441,-0.02308533,0.03847408,-0.01464109,0,-0.0329202,0,0.02074971,-0.01973395,0,0.0008027369,0.02722976]},{"f":[33,32,33,33,33,33,1,24,33,37,-1,-1,32,-1,4,-1,-1,-1,-1,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,32.199935296219856,29.807475161786478,12.64516129032258,9.5,13.895530557358477,8.05,0,0,5.845435504646156,0,4.80643424864552,0,0,0,0,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,10,0,0,13,0,15,0,0,0,0,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,20,19,18,17,12,11,0,0,14,0,16,0,0,0,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0.0001954926,0.02753784,0,-0.03135453,0,-0.004277736,0.008928019,0.0166291,-0.01156325,0.02131871,-0.01547853,0,0,0.008351983,0.04044206,-0.01713424,-0.02216533,-0.03763772,0.008883194]},{"f":[27,22,22,-1,-1,24,-1,22,-1,22,-1,18,-1,22,-1,23,-1,-1,22,0,-1,-1,22,-1,22,-1,17,-1,-1],"t":[0.5,16.73920708776685,16.725912049439096,0,0,6.5,0,16.765402642760364,0,16.786010507999382,0,3.39067985676233,0,16.819976309430963,0,2.5,0,0,16.978341770814396,8.5,0,0,16.98920034885663,0,17.078739781790915,0,4.128693921346326,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,0,10,0,12,0,14,0,16,0,0,19,20,0,0,23,0,25,0,27,0,0],"r":[18,5,4,0,0,7,0,9,0,11,0,13,0,15,0,17,0,0,22,21,0,0,24,0,26,0,28,0,0],"v":[0,0,0,3.248992e-06,0.02236146,0,0.02307242,0,-0.03285547,0,0.05056606,0,-0.02511892,0,-0.01335278,0,0.02781915,0.0004492083,0,0,-0.02858647,0.0130073,0,0.05153679,0,-0.02638865,0,0.00437282,0.03353279]},{"f":[22,23,21,4,2,-1,-1,16,-1,16,-1,-1,0,1,16,15,-1,-1,-1,-1,-1,22,33,-1,-1,1,-1,-1,-1],"t":[17.105450904090162,7.5,9.5,9.328332883600808,9.293467526905953,0,0,4.247927513443585,0,10.696098170955864,0,0,7.5,6.015625,7.6653359171851765,6.0,0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,0,13,14,15,16,0,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,21,12,7,6,0,0,9,0,11,0,0,20,19,18,17,0,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.0001841701,-0.03015335,0,-0.009023451,0,0.04011197,-0.001152659,0,0,0,0,0.01078874,-0.01486718,0.01308399,-0.01831711,0.0375592,0,0,-0.01784648,0.01689643,0,0.01574567,-0.04233949,0.01021732]},{"f":[1,16,-1,-1,1,-1,22,-1,37,16,-1,21,-1,17,-1,-1,25,23,1,-1,15,19,-1,-1,-1,-1,33,-1,-1],"t":[0.01933482600988898,11.031356596255709,0,0,0.3126984126984127,0,12.121442107629743,0,4.875,10.372865060112588,0,6.5,0,3.245926548164837,0,0,0.5,4.5,4.949128919860627,0,7.118941072723508,1.5,0,0,0,0,38.67649696721479,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,0,0,5,0,7,0,9,10,0,12,0,14,0,0,17,18,19,0,21,22,0,0,0,0,27,0,0],"r":[4,3,0,0,6,0,8,0,16,11,0,13,0,15,0,0,26,25,20,0,24,23,0,0,0,0,28,0,0],"v":[0,0,-0.03068195,0.04247626,0,-0.0356296,0,-0.001420239,0,0,-0.000690105,0,-0.04008498,0,-0.03850944,-0.000690105,0,0,0,0.03783063,0,0,0.04405953,0.002963187,0.002522635,-0.000690105,0,-0.000690105,0.04045227]},{"f":[22,-1,22,22,21,-1,-1,-1,21,17,22,-1,-1,-1,4,15,-1,15,22,-1,-1,-1,18,2,-1,-1,15,-1,-1],"t":[2.69615871138938,0,5.9525054311951475,4.926654777701837,6.5,0,0,0,2.5,1.292481250360578,12.027747761539269,0,0,0,9.266718101290426,1.0,0,1.584962500721156,10.194756329264862,0,0,0,10.457893746154532,10.680359255641626,0,0,9.862637357558794,0,0],"m":[0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,0,9,10,11,0,0,0,15,16,0,18,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,8,7,6,0,0,0,14,13,12,0,0,0,22,17,0,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.0234169,0,0,0,-0.02706728,0.01061899,0.03262572,0,0,0,-0.0245002,0.01103377,-0.03368796,0,0,0.01756277,0,0,-0.02728704,0.006751426,0.0003492933,0,0,-0.03092196,0.02498004,0,0.03214731,-0.01101499]},{"f":[4,4,21,18,22,-1,-1,15,-1,-1,25,15,15,-1,-1,-1,18,-1,22,36,18,-1,-1,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,5.065928480304397,13.827286379004683,0,0,5.614709844115208,0,0,2.5,5.169925001442312,4.087462841250339,0,0,0,1.292481250360578,0,12.613886328427084,2.5,1.792481250360578,0,0,0,0,0,2.160964047443681,0,0],"m":[1,1,0,1,1,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,12,13,0,0,0,17,0,19,20,21,0,0,0,0,0,27,0,0],"r":[26,25,10,7,6,0,0,9,0,0,16,15,14,0,0,0,18,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,-0.0397427,0.01358932,0,0.04231364,-0.02036749,0,0,0,0.0006554825,0.01238794,-0.0004639606,0,-0.04330753,0,0,0,-0.001427181,-0.0184904,0.00780679,0.0128989,-0.0121621,0,-0.005054549,0.03565654]},{"f":[36,-1,33,33,15,22,17,22,-1,-1,22,-1,1,-1,-1,-1,15,4,-1,-1,22,20,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.99840809449714,5.169925001442312,15.212004812197344,2.69615871138938,13.054613167076562,0,0,12.613886328427084,0,6.276785714285714,0,0,0,7.266786540694901,1.792481250360578,0,0,16.725912049439096,0.5,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,0,0,17,18,0,0,21,22,0,0,0,0,27,0,0],"r":[2,0,26,25,16,15,10,9,0,0,12,0,14,0,0,0,20,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0.01926051,0,0,0,0,0,0,0.002176211,-0.04699047,0,-0.001525107,0,-0.003768597,0.03342055,0.03662304,0,0,-0.05354741,-0.008933978,0,0,-0.01827848,-0.000178523,0.01637283,-0.03455966,0,0.02182082,0.0002470965]},{"f":[22,22,22,22,22,-1,-1,32,-1,2,-1,22,-1,-1,23,17,22,-1,-1,-1,-1,18,-1,18,-1,-1,2,-1,-1],"t":[17.07907323966382,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,6.208926257442949,0,3.903677461028802,0,16.819976309430963,0,0,7.5,6.781300095563516,16.98920034885663,0,0,0,0,6.076776015854056,0,7.383290334558488,0,0,4.0437314206251695,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,12,0,0,15,16,17,0,0,0,0,22,0,24,0,0,27,0,0],"r":[26,21,14,7,6,0,0,9,0,11,0,13,0,0,20,19,18,0,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-8.274457e-05,0.04065332,0,0.01380531,0,-0.03618138,0,-0.01884859,0.01191364,0,0,0,0.01931848,-0.02192679,0.03689126,-0.01378302,0,-0.0305744,0,0.02004292,-0.01903441,0,0.001066688,0.02610127]},{"f":[22,-1,4,4,21,18,22,-1,-1,22,-1,-1,15,-1,25,-1,18,-1,22,36,18,-1,-1,-1,-1,-1,17,-1,-1],"t":[2.69615871138938,0,10.510264838663748,9.283086494496427,2.5,5.065928480304397,13.827286379004683,0,0,11.36027577861459,0,0,1.0,0,2.5,0,1.292481250360578,0,12.613886328427084,2.5,1.792481250360578,0,0,0,0,0,2.160964047443681,0,0],"m":[0,0,1,1,0,1,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,0,15,0,17,0,19,20,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,12,9,8,0,0,11,0,0,14,0,16,0,18,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,-0.0231273,0,0,0,0,0,-0.03937696,0.01280615,0,0.04117672,-0.02027805,0,0.01489144,0,0.0004949228,0,-0.04241776,0,0,0,-0.001286757,-0.01768719,0.007447166,0.01169727,-0.01228884,0,-0.004851012,0.03523205]},{"f":[36,-1,33,33,-1,-1,33,-1,22,22,22,22,22,-1,-1,22,-1,22,-1,24,15,-1,-1,22,-1,-1,-1,-1,-1],"t":[1.5,0,20.752541988539424,18.99840809449714,0,0,21.16864017939033,0,17.078739781790915,17.069460217922263,16.978341770814396,16.73920708776685,16.25454052700782,0,0,16.765402642760364,0,16.786010507999382,0,6.5,6.169925001442312,0,0,16.838461650489297,0,0,0,0,0],"m":[0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,11,12,13,0,0,16,0,18,0,20,21,0,0,24,0,0,0,0,0],"r":[2,0,6,5,0,0,8,0,28,27,26,15,14,0,0,17,0,19,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0.01904812,0,0,-0.001386145,-0.0338623,0,0.02115922,0,0,0,0,0,0.0001416421,0.01828222,0,-0.03150567,0,0.04845059,0,0,0.0009769029,-0.03671669,0,-0.008649636,0.02874892,0.01836524,-0.02889569,0.008258235]},{"f":[2,4,18,17,15,-1,22,-1,15,-1,-1,27,-1,15,-1,-1,15,16,-1,-1,-1,17,-1,22,-1,-1,33,-1,-1],"t":[2.453445297804259,1.792481250360578,3.245926548164837,1.792481250360578,7.6653359171851765,0,10.735831902194036,0,10.113742166049189,0,0,0.5,0,8.618385502258606,0,0,10.501837184902296,6.832890014164741,0,0,0,2.69615871138938,0,11.775312925643524,0,0,38.67649696721479,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,7,0,9,0,0,12,0,14,0,0,17,18,0,0,0,22,0,24,0,0,27,0,0],"r":[26,21,16,11,6,0,8,0,10,0,0,13,0,15,0,0,20,19,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.009714011,0,0.03074234,0,0.007601305,-0.01988504,0,0.005662974,0,0.0561421,-0.02759225,0,0,0.001642888,-0.02223094,0.01108274,0,-0.0232225,0,-0.04498874,0.03277145,0,0.0002996279,0.03187001]},{"f":[33,32,33,33,33,33,33,33,23,-1,-1,19,-1,-1,19,22,-1,-1,-1,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,32.199935296219856,29.807475161786478,27.27007187651612,24.56142327888218,1.5,0,0,0.5,0,0,0.5,16.978341770814396,0,0,0,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,12,0,0,15,16,0,0,0,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,20,19,14,11,10,0,0,13,0,0,18,17,0,0,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0.001053212,-0.001994669,0,0.002408304,0.0368105,0,0,-0.004734427,0.01843955,-0.03337248,0.02041653,-0.01548377,0,0,0.007549821,0.03852867,-0.0156064,-0.02379339,-0.03698385,0.008782827]},{"f":[1,16,-1,-1,1,-1,22,21,15,-1,22,21,-1,-1,21,-1,24,-1,23,17,-1,-1,-1,-1,37,16,-1,-1,-1],"t":[0.01933482600988898,11.031356596255709,0,0,0.3126984126984127,0,12.121442107629743,5.5,3.169925001442312,0,8.723206471664755,3.5,0,0,3.5,0,12.5,0,0.5,7.980000966034041,0,0,0,0,4.875,10.372865060112588,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,0,7,8,9,0,11,12,0,0,15,0,17,0,19,20,0,0,0,0,25,26,0,0,0],"r":[4,3,0,0,6,0,24,23,10,0,14,13,0,0,16,0,18,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,0,-0.03005443,0.04154321,0,-0.03509368,0,0,0,-0.04281656,0,0,0.05290852,-0.0003390853,0,-0.0445974,0,-0.04502883,0,0,0.02607024,-0.002710098,-0.03981304,-4.747419e-05,0,0,-0.0005959057,-0.01383349,0.002150883]},{"f":[21,4,2,2,2,22,-1,22,-1,-1,15,17,-1,-1,-1,22,-1,-1,-1,16,-1,16,-1,-1,22,0,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,7.005613627711627,14.116600814324018,0,16.73920708776685,0,0,10.113742166049189,10.620215166896223,0,0,0,13.005894632618489,0,0,0,4.247927513443585,0,10.696098170955864,0,0,15.756821747580634,7.5,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,11,12,0,0,0,16,0,0,0,20,0,22,0,0,25,26,0,0,0],"r":[24,19,18,15,10,7,0,9,0,0,14,13,0,0,0,17,0,0,0,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.00137234,0,0.006864906,-0.002328207,0,0,-0.02088969,0.004742376,0.003412793,0,0.001778179,0.03235362,-0.02802522,0,-0.00838907,0,0.03976938,-0.001021682,0,0,0.004647844,0.03688829,-0.02771817]},{"f":[4,19,15,-1,-1,36,-1,16,17,-1,33,4,-1,0,-1,-1,-1,18,37,-1,-1,32,-1,33,-1,-1,37,-1,-1],"t":[2.453445297804259,0.5,3.169925001442312,0,0,2.5,0,7.118941072723508,1.792481250360578,0,21.257266042509173,1.292481250360578,0,7.5,0,0,0,2.160964047443681,6.890000000000001,0,0,7.491051153478379,0,44.805779984159756,0,0,12.52,0,0],"m":[1,1,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,9,0,11,12,0,14,0,0,0,18,19,0,0,22,0,24,0,0,27,0,0],"r":[26,5,4,0,0,7,0,17,10,0,16,13,0,15,0,0,0,21,20,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,-0.02378909,0.002966342,0,-0.02705645,0,0,-0.02253216,0,0,0.02094642,0,-0.02305267,0.01930283,-0.004778912,0,0,-0.01891737,0.007030573,0,-0.04785072,0,0.003618846,-0.02354262,0,0.0004568836,0.03109942]},{"f":[22,-1,36,-1,33,22,-1,-1,33,22,-1,-1,22,22,22,-1,17,22,15,-1,-1,-1,-1,-1,18,17,-1,-1,-1],"t":[2.69615871138938,0,1.5,0,24.416594167368267,16.98920034885663,0,0,26.99635432609064,12.121442107629743,0,0,17.078739781790915,17.069460217922263,16.978341770814396,0,6.781300095563516,16.98920034885663,8.618385502258606,0,0,0,0,0,8.729619742794132,7.6181566135214025,0,0,0],"m":[0,0,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,0,5,6,0,0,9,10,0,0,13,14,15,0,17,18,19,0,0,0,0,0,25,26,0,0,0],"r":[2,0,4,0,8,7,0,0,12,11,0,0,24,23,16,0,22,21,20,0,0,0,0,0,28,27,0,0,0],"v":[0,-0.02258769,0,0.01870917,0,0,-0.001163067,-0.02515026,0,0,0.00219386,0.03294481,0,0,0,-0.0001417385,0,0,0,-0.011951,0.0349717,-0.03003484,0.03119447,-0.02653055,0,0,0.0008072368,0.03555081,-0.01353254]},{"f":[21,4,2,2,-1,22,-1,-1,-1,16,-1,16,-1,-1,22,0,17,2,17,-1,-1,15,22,-1,-1,-1,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0,13.005894632618489,0,0,0,4.247927513443585,0,10.696098170955864,0,0,15.756821747580634,7.5,6.781300095563516,1.292481250360578,1.792481250360578,0,0,7.679480099505446,10.735831902194036,0,0,0,0,0,0],"m":[1,1,1,1,0,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,0,10,0,12,0,0,15,16,17,18,19,0,0,22,23,0,0,0,0,0,0],"r":[14,9,8,5,0,7,0,0,0,11,0,13,0,0,28,27,26,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0005573471,0,0.001674208,0.03163256,-0.02700685,0,-0.008037009,0,0.03859088,-0.0009962494,0,0,0,0,0,-5.211721e-05,-0.0355644,0,0,0.007644839,-0.01557479,0.02041412,0.01979599,0.03645675,-0.02683742]},{"f":[4,4,21,22,-1,37,-1,-1,15,15,-1,-1,16,22,2,33,-1,-1,-1,22,-1,-1,15,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,5.169925001442312,4.087462841250339,0,0,6.108524456778169,16.98413870862295,4.80643424864552,26.622428940176302,0,0,0,16.98920034885663,0,0,5.700439718141092,0,0,0,2.160964047443681,0,0],"m":[1,1,0,0,0,1,0,0,0,1,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,13,14,15,16,0,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,12,11,0,0,22,19,18,17,0,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0.01525134,0,-0.03902519,0.01520265,0,0,-0.0001732564,0.009336089,0,0,0,0,-0.03819535,0.007397454,-0.004184137,0,0.03345825,-0.004526526,0,0.03207897,0.000147787,-0.01179276,0,-0.00450298,0.03475647]}]},"p_fail":{"link":"logistic","base":-2.9214699960499186,"trees":[{"f":[37,2,4,22,-1,-1,20,-1,-1,21,-1,-1,22,36,-1,23,-1,-1,4,32,-1,32,-1,19,36,-1,-1,-1,-1],"t":[7.05,6.247802615595391,5.065928480304397,17.094543568094064,0,0,0.5,0,0,3.5,0,0,12.613886328427084,4.5,0,1.5,0,0,6.16978583919235,5.450056031488728,0,6.208926257442949,0,2.5,2.5,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,0,0,13,14,0,16,0,0,19,20,0,22,0,24,25,0,0,0,0],"r":[12,9,6,5,0,0,8,0,0,11,0,0,18,15,0,17,0,0,28,21,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.04769787,0.0934937,0,-0.02507065,0.09083067,0,-0.04844851,0.0934937,0,0,0.0934937,0,0.0934937,0.3504119,0,0,0.4894688,0,0.0934937,0,0,0.0934937,0.3642448,0.0934937,0.6431695]},{"f":[37,2,22,-1,-1,21,-1,-1,22,21,-1,37,4,-1,-1,-1,4,19,22,22,36,-1,-1,-1,32,-1,-1,-1,-1],"t":[7.05,5.228690439536267,17.094543568094064,0,0,3.5,0,0,12.613886328427084,5.5,0,9.075,3.245926548164837,0,0,0,6.16978583919235,2.5,16.25454052700782,15.439474430023438,3.5,0,0,0,9.271599002932193,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,13,0,0,0,17,18,19,20,21,0,0,0,25,0,0,0,0],"r":[8,5,4,0,0,7,0,0,16,11,0,15,14,0,0,0,28,27,24,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0,-0.04745094,0.0681254,0,-0.05016321,0.0681254,0,0,0.0681254,0,0,0.0681254,0.139018,0.3786697,0,0,0,0,0,0.2480394,0.0681254,0.3715553,0,0.08642291,0.2383047,0.0681254,0.3876677]},{"f":[37,2,22,-1,-1,21,-1,-1,22,21,-1,33,-1,-1,19,32,-1,15,-1,22,22,33,-1,-1,-1,-1,2,-1,-1],"t":[7.05,5.228690439536267,17.094543568094064,0,0,3.5,0,0,12.613886328427084,5.5,0,27.893589438525005,0,0,2.5,5.450056031488728,0,6.169925001442312,0,17.094543568094064,16.98920034885663,17.460839126440824,0,0,0,0,1.292481250360578,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,0,0,15,16,0,18,0,20,21,22,0,0,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,14,11,0,13,0,0,26,17,0,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,-0.0471081,0.05273543,0,-0.04993683,0.05273543,0,0,0.05273543,0,0.06704428,0.3058953,0,0,0.2611051,0,0.05273543,0,0,0,0.06439798,0.1988733,0.05273543,0.3195874,0,0.05273543,0.1830133]},{"f":[37,2,22,-1,-1,21,-1,9,-1,-1,22,21,-1,1,-1,3,-1,-1,4,19,22,15,-1,-1,22,-1,-1,-1,-1],"t":[7.05,5.228690439536267,17.094543568094064,0,0,3.5,0,0.5,0,0,12.613886328427084,5.5,0,8.980632551528075,0,0.9197769197769198,0,0,6.16978583919235,2.5,16.98920034885663,13.56045225766691,0,0,17.094543568094064,0,0,0,0],"m":[1,1,1,0,0,0,0,1,0,0,0,0,0,1,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0],"l":[1,2,3,0,0,6,0,8,0,0,11,12,0,14,0,16,0,0,19,20,21,22,0,0,25,0,0,0,0],"r":[10,5,4,0,0,7,0,9,0,0,18,13,0,15,0,17,0,0,28,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.0467624,0.04229328,0,-0.04971282,0,0.04229328,-0.04846064,0,0,0.04229328,0,0.04229328,0,0.09199477,0.259104,0,0,0,0,0.1681462,0.04229328,0,0.04229328,0.1356608,0.04229328,0.2279098]},{"f":[37,2,-1,21,-1,-1,32,19,21,22,-1,15,-1,15,-1,22,22,-1,-1,-1,-1,-1,3,-1,17,-1,17,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0,10.41931383093672,2.5,9.5,14.03140051979143,0,5.044394119358453,0,6.169925001442312,0,17.094543568094064,16.98920034885663,0,0,0,0,0,0.01140250391999155,0,7.980000966034041,0,8.729619742794132,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,0,12,0,14,0,16,17,0,0,0,0,0,23,0,25,0,27,0,0],"r":[6,3,0,5,0,0,22,21,20,11,0,13,0,15,0,19,18,0,0,0,0,0,24,0,26,0,28,0,0],"v":[0,0,-0.04758055,0,-0.04961903,0.0231401,0,0,0,0,0.06343669,0,0.2028083,0,0.0231401,0,0,0.1305835,0.02508399,0.211893,0.2532368,0.0231401,0,0.2139762,0,0.0231401,0,0.3284481,0.0231401]},{"f":[37,2,-1,21,-1,9,-1,-1,32,19,21,22,-1,-1,-1,4,-1,-1,2,22,-1,23,-1,-1,18,2,-1,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0.5,0,0,10.41931383093672,2.5,9.5,12.613886328427084,0,0,0,2.69615871138938,0,0,6.118903736861568,17.094543568094064,0,7.5,0,0,8.03615005796417,6.8391900325545585,0,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,1,0,0,1,1,0,1,0,0,1,1,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,0,0,0,16,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[8,3,0,5,0,7,0,0,18,15,14,13,0,0,0,17,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,-0.0472698,0,-0.04940234,0,0.01861817,-0.04935216,0,0,0,0,0.05051997,0.1155308,0.2080956,0,0.01861817,0.09211612,0,0,0.01861817,0,0.2278524,0.01861817,0,0,0.1058997,0.2163142,0.01861817]},{"f":[37,2,-1,21,-1,-1,32,19,21,22,16,-1,-1,1,-1,-1,-1,33,-1,-1,2,22,-1,23,-1,-1,18,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0,10.41931383093672,2.5,9.5,15.439474430023438,6.0,0,0,7.6141732283464565,0,0,0,24.04238488811619,0,0,6.118903736861568,17.094543568094064,0,7.5,0,0,8.03615005796417,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0],"l":[1,2,0,4,0,0,7,8,9,10,11,0,0,14,0,0,0,18,0,0,21,22,0,24,0,0,27,0,0],"r":[6,3,0,5,0,0,20,17,16,13,12,0,0,15,0,0,0,19,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,-0.04695588,0,-0.04918672,0.01489223,0,0,0,0,0,0.01489223,0.08088957,0,0.07379098,0.1299393,0.1775134,0,0.01489223,0.0841136,0,0,0.01489223,0,0.1894964,0.01489223,0,0.1321402,0.01489223]},{"f":[37,2,-1,21,-1,9,-1,-1,32,19,21,22,-1,15,-1,-1,-1,4,-1,-1,2,22,-1,23,-1,-1,22,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0.5,0,0,10.41931383093672,2.5,9.5,12.613886328427084,0,5.044394119358453,0,0,0,2.69615871138938,0,0,6.118903736861568,17.094543568094064,0,7.5,0,0,16.765402642760364,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,1,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,0,14,0,0,0,18,0,0,21,22,0,24,0,0,27,0,0],"r":[8,3,0,5,0,7,0,0,20,17,16,13,0,15,0,0,0,19,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,-0.04663823,0,-0.04897172,0,0.01189007,-0.04915927,0,0,0,0,0.04010872,0,0.128136,0.08068234,0.1553895,0,0.01189007,0.07539508,0,0,0.01189007,0,0.1613723,0.01189007,0,0.08239529,0.2135406]},{"f":[32,37,-1,19,-1,-1,2,2,22,-1,16,-1,-1,0,-1,22,2,-1,-1,2,-1,-1,1,19,-1,-1,20,-1,-1],"t":[10.41931383093672,6.970000000000001,0,2.5,0,0,9.138550601137563,5.228690439536267,17.094543568094064,0,6.539158811108031,0,0,4.5,0,14.03140051979143,6.942466823974881,0,0,5.40929108874043,0,0,2.0761752136752136,1.5,0,0,0.5,0,0],"m":[0,0,0,1,0,0,1,1,1,0,1,0,0,1,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0],"l":[1,2,0,4,0,0,7,8,9,0,11,0,0,14,0,16,17,0,0,20,0,0,23,24,0,0,27,0,0],"r":[6,3,0,5,0,0,22,13,10,0,12,0,0,15,0,19,18,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,-0.009080677,0,0.07817652,0.03068926,0,0,0,-0.04601527,0,-0.0302457,0.1278346,0,-0.01969078,0,0,0.01363114,0.118908,0,0.308648,0.05017494,0,0,0.01796037,0.1290202,0,0.3238385,0.0726058]},{"f":[37,2,-1,21,-1,9,-1,-1,32,19,4,-1,-1,33,-1,24,-1,-1,2,22,-1,23,-1,-1,18,22,-1,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0.5,0,0,10.41931383093672,2.5,6.08730682361797,0,0,24.04238488811619,0,8.5,0,0,6.118903736861568,17.094543568094064,0,7.5,0,0,8.03615005796417,16.759160379664443,0,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,1,0,0,1,1,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,0,0,14,0,16,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[8,3,0,5,0,7,0,0,18,13,12,0,0,15,0,17,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,-0.04600386,0,-0.04871792,0,0.007965274,-0.04873968,0,0,0,0.06515144,0.128155,0,0.007965274,0,0.007965274,0.1174494,0,0,0.007965274,0,0.1249281,0.007965274,0,0,0.0732324,0.1747689,0.007965274]},{"f":[32,37,17,-1,-1,19,-1,-1,2,2,22,-1,16,-1,-1,0,-1,22,-1,23,-1,-1,1,22,-1,-1,2,-1,-1],"t":[10.41931383093672,7.125,4.2849278041654735,0,0,2.5,0,0,9.138550601137563,5.228690439536267,17.094543568094064,0,6.539158811108031,0,0,4.5,0,11.775312925643524,0,4.5,0,0,2.01685992705101,16.725912049439096,0,0,10.510264838663748,0,0],"m":[0,0,1,0,0,1,0,0,1,1,1,0,1,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,0,0,16,0,18,0,20,0,0,23,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,22,15,12,0,14,0,0,17,0,19,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,-0.03423149,0.02865827,0,0.06654153,0.03003279,0,0,0,-0.04538788,0,-0.02954708,0.1011357,0,-0.01893109,0,0.0106163,0,0.09251107,0.02714556,0,0,0.02421608,0.1225957,0,0.09327025,0.2530409]},{"f":[32,37,17,-1,-1,19,4,-1,-1,-1,2,22,-1,17,-1,-1,4,4,3,18,-1,-1,-1,17,3,-1,-1,-1,-1],"t":[10.41931383093672,7.205,4.2849278041654735,0,0,2.5,6.08730682361797,0,0,0,5.228690439536267,17.094543568094064,0,7.588709768994619,0,0,10.510264838663748,5.40929108874043,0.9300813008130082,1.292481250360578,0,0,0,11.78143940991899,0.9148166023166023,0,0,0,0],"m":[0,0,1,0,0,1,1,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,0,0,11,12,0,14,0,0,17,18,19,20,0,0,0,24,25,0,0,0,0],"r":[10,5,4,0,0,9,8,0,0,0,16,13,0,15,0,0,28,23,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03362229,0.0267103,0,0,0.0559467,0.1115538,0.02782049,0,0,-0.04502868,0,-0.0291198,0.09162809,0,0,0,0,0.0830128,-0.009944685,0.225705,0,0,0.09089829,-0.01522781,0.117968,0.1732332]},{"f":[37,2,-1,21,-1,-1,22,21,-1,-1,23,22,18,22,-1,4,22,22,-1,-1,-1,-1,33,-1,-1,18,-1,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0,14.116600814324018,8.5,0,0,7.5,17.094543568094064,2.160964047443681,15.782972563647261,0,2.453445297804259,16.765402642760364,16.73920708776685,0,0,0,0,20.752541988539424,0,0,8.729619742794132,0,0,0],"m":[1,1,0,0,0,0,0,1,0,0,1,1,0,0,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,0,16,17,18,0,0,0,0,23,0,0,26,0,0,0],"r":[6,3,0,5,0,0,10,9,0,0,28,25,22,15,0,21,20,19,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0,-0.0450083,0,-0.04848159,0.004115474,0,0,0.004115474,0.09244651,0,0,0,0,0.1137074,0,0,0,0.07352224,0.004115474,0.1430886,0.01691213,0,0.004115474,0.0478023,0,0.1306,0.004115474,0.004115474]},{"f":[2,32,19,23,37,-1,-1,-1,-1,22,-1,-1,2,37,-1,22,-1,22,-1,22,3,-1,32,-1,-1,-1,0,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,8.05,0,0,0,0,17.094543568094064,0,0,10.100798745059642,4.875,0,14.116600814324018,0,14.158358850608169,0,17.094543568094064,0.026802550058364013,0,8.599793042091783,0,0,0,2.5,0,0],"m":[1,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,0,10,0,0,13,14,0,16,0,18,0,20,21,0,23,0,0,0,27,0,0],"r":[12,9,8,7,6,0,0,0,0,11,0,0,26,15,0,17,0,19,0,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0.03467511,0.07517031,-0.05267765,-0.05050093,0,-0.04429315,0.03881418,0,0,-0.01054975,0,0.02064976,0,0.196239,0,0,0.07833555,0,0.01853533,0.05589292,0.1184446,0,0.007721112,0.152295]},{"f":[2,32,19,23,37,22,-1,-1,-1,-1,-1,22,-1,-1,2,37,-1,22,18,-1,-1,22,-1,22,2,-1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,8.05,12.613886328427084,0,0,0,0,0,17.094543568094064,0,0,10.510264838663748,4.875,0,14.116600814324018,7.980000966034041,0,0,14.158358850608169,0,17.094543568094064,8.049190845056867,0,0,0,0],"m":[1,0,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,12,0,0,15,16,0,18,19,0,0,22,0,24,25,0,0,0,0],"r":[14,11,10,9,8,7,0,0,0,0,0,13,0,0,28,17,0,21,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.01037496,0.04517335,0.06944228,-0.05197606,-0.05014537,0,-0.04390751,0.03608557,0,0,-0.00982346,0,0,0.01749402,0.06173516,0,0.1673379,0,0,0.03146814,0.08225331,0.1105438,0.1242269]},{"f":[2,32,19,23,-1,-1,-1,22,-1,-1,2,37,-1,22,-1,22,-1,22,3,22,-1,-1,-1,36,-1,-1,0,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,0,0,0,17.094543568094064,0,0,10.100798745059642,4.875,0,14.116600814324018,0,14.158358850608169,0,16.759160379664443,0.12256006006006007,16.73920708776685,0,0,0,2.5,0,0,2.5,0,0],"m":[1,0,1,1,0,0,0,1,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,0,14,0,16,0,18,19,20,0,0,0,24,0,0,27,0,0],"r":[10,7,6,5,0,0,0,9,0,0,26,13,0,15,0,17,0,23,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.04323287,-0.05129791,-0.04979607,0,-0.04351432,0.03361381,0,0,-0.009722018,0,0.01774984,0,0.1459777,0,0,0,0.06562953,0.01546931,0.01546931,0,0.01546931,0.08406263,0,0.002805423,0.1184653]},{"f":[2,32,19,23,-1,-1,-1,22,-1,-1,2,1,21,-1,-1,17,32,-1,-1,17,-1,21,-1,22,-1,-1,1,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,0,0,0,17.094543568094064,0,0,8.9274737487839,4.92309205350118,7.5,0,0,7.198837316474133,6.337096134072842,0,0,7.781300095563516,0,4.5,0,17.083799521744545,0,0,2.01685992705101,0,0],"m":[1,0,1,1,0,0,0,1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,13,0,0,16,17,0,0,20,0,22,0,24,0,0,27,0,0],"r":[10,7,6,5,0,0,0,9,0,0,26,15,14,0,0,19,18,0,0,21,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0.04019397,-0.05064109,-0.04945217,0,-0.04311322,0.03136577,0,0,0,-0.03874625,0.01260896,0,0,-0.02822746,0.06900363,0,0.1011399,0,-0.04572707,0,0.03976827,0.101486,0,0.03259705,0.09209488]},{"f":[37,2,-1,21,-1,-1,22,21,-1,-1,23,22,22,18,22,22,-1,-1,-1,22,23,-1,-1,-1,-1,18,-1,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0,14.116600814324018,8.5,0,0,7.5,17.094543568094064,16.98920034885663,2.160964047443681,16.765402642760364,15.782972563647261,0,0,0,16.844368517741394,1.5,0,0,0,0,8.729619742794132,0,0,0],"m":[1,1,0,0,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,15,16,0,0,0,20,21,0,0,0,0,26,0,0,0],"r":[6,3,0,5,0,0,10,9,0,0,28,25,24,19,18,17,0,0,0,23,22,0,0,0,0,27,0,0,0],"v":[0,0,-0.04329869,0,-0.04809694,0.00108665,0,0,0.00108665,0.07404429,0,0,0,0,0,0,0.08492338,0.02684222,0.1149836,0,0,0.0408302,0.00108665,0.08200073,0.00108665,0,0.1011915,0.00108665,0.00108665]},{"f":[2,32,19,23,-1,-1,-1,22,-1,-1,21,-1,2,1,21,-1,-1,17,32,22,-1,-1,-1,24,-1,-1,1,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,0,0,0,17.094543568094064,0,0,3.5,0,8.9274737487839,4.92309205350118,7.5,0,0,7.198837316474133,6.337096134072842,15.782972563647261,0,0,0,5.5,0,0,2.01685992705101,0,0],"m":[1,0,1,1,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,0,13,14,15,0,0,18,19,20,0,0,0,24,0,0,27,0,0],"r":[10,7,6,5,0,0,0,9,0,0,12,0,26,17,16,0,0,23,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.03544255,-0.05079942,-0.04916817,0,-0.04238859,0.02788346,0,-0.04983604,0,0,0,-0.03669358,0.01109883,0,0,0,-0.04248968,0.05992567,0.06142792,0,0.1085325,0.04500142,0,0.03381769,0.08928647]},{"f":[21,24,16,22,-1,-1,-1,17,-1,-1,2,24,-1,35,-1,-1,1,-1,2,32,4,21,-1,-1,-1,36,-1,-1,-1],"t":[4.5,7.5,6.392317422778761,12.613886328427084,0,0,0,11.578476357671452,0,0,4.882435795368045,7.5,0,2.5,0,0,1.282051282051282,0,9.068775776507199,6.337096134072842,5.714180086352146,7.5,0,0,0,4.5,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,0,14,0,0,17,0,19,20,21,22,0,0,0,26,0,0,0],"r":[10,7,6,5,0,0,0,9,0,0,16,13,0,15,0,0,18,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,-0.02120774,0.04750176,-0.02658132,0,-0.04620745,0.09207463,0,0,-0.03297489,0,0.0432284,-0.04898179,0,-0.01578352,0,0,0,0,0.1064438,0.01990581,-0.007052136,0,0.08881526,0.04026965,0.07190423]},{"f":[21,24,20,-1,-1,17,-1,-1,2,24,22,-1,15,-1,-1,35,-1,-1,1,-1,2,32,4,21,-1,-1,-1,-1,-1],"t":[4.5,7.5,1.5,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,6.584962500721156,0,0,2.5,0,0,1.282051282051282,0,9.068775776507199,6.337096134072842,5.714180086352146,7.5,0,0,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,0,0,16,0,0,19,0,21,22,23,24,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,18,15,12,0,14,0,0,17,0,0,20,0,28,27,26,25,0,0,0,0,0],"v":[0,0,0,-0.01489534,0.03607625,0,-0.04587588,0.08355575,0,0,0,-0.03804654,0,0.05311217,-0.04043443,0,0.04045386,-0.04866499,0,-0.01522208,0,0,0,0,0.09582695,0.01864698,-0.006744542,0.05059549,0.06568794]},{"f":[21,24,16,22,-1,-1,-1,17,-1,-1,2,24,22,-1,15,-1,-1,35,-1,-1,0,-1,2,1,-1,22,-1,-1,-1],"t":[4.5,7.5,6.392317422778761,12.613886328427084,0,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,6.584962500721156,0,0,2.5,0,0,2.5,0,7.661668144640085,4.92309205350118,0,17.094543568094064,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,13,0,15,0,0,18,0,0,21,0,23,24,0,26,0,0,0],"r":[10,7,6,5,0,0,0,9,0,0,20,17,14,0,16,0,0,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,0,0,0,-0.02069481,0.04262807,-0.02611898,0,-0.04553799,0.07617299,0,0,0,-0.03750723,0,0.04942457,-0.0397771,0,0.03791869,-0.04835057,0,-0.01324508,0,0,0.00989946,0,0.03356092,0.08526907,0.0618625]},{"f":[21,24,1,-1,15,-1,-1,17,-1,-1,2,24,22,-1,-1,35,-1,-1,1,-1,2,1,3,-1,-1,-1,1,-1,-1],"t":[4.5,7.5,6.241071428571429,0,6.247927513443585,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,0,2.5,0,0,0.7457446808510639,0,9.068775776507199,4.92309205350118,0.02996265189107678,0,0,0,2.01685992705101,0,0],"m":[0,0,0,0,1,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,0,0,11,12,13,0,0,16,0,0,19,0,21,22,23,0,0,0,27,0,0],"r":[10,7,4,0,6,0,0,9,0,0,18,15,14,0,0,17,0,0,20,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,-0.03749565,0,0.03327541,-0.04072238,0,-0.0451934,0.06972815,0,0,0,-0.03695925,0.01076815,0,0.03559294,-0.04803796,0,-0.03453626,0,0,0,0.04762514,-0.0022136,0.03765599,0,0.02828295,0.07306647]},{"f":[21,24,20,-1,-1,17,-1,-1,2,24,22,-1,-1,35,15,-1,-1,-1,0,-1,2,22,2,21,-1,-1,-1,-1,-1],"t":[4.5,7.5,0.5,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,0,2.5,8.618385502258606,0,0,0,2.5,0,7.640240405159187,17.094543568094064,6.011183906514227,7.5,0,0,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,1,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,0,14,15,0,0,0,19,0,21,22,23,24,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,18,13,12,0,0,17,16,0,0,0,20,0,28,27,26,25,0,0,0,0,0],"v":[0,0,0,-0.01566103,0.03023375,0,-0.04484172,0.06406368,0,0,0,-0.03640272,0.01019892,0,0,0.04343813,-0.003260204,-0.0477266,0,-0.01264021,0,0,0,0,0.09407714,0.008180154,0.0106975,0.08078468,0.05320676]},{"f":[21,24,1,-1,-1,17,-1,-1,2,24,-1,35,-1,-1,1,-1,22,2,4,21,4,-1,16,-1,-1,-1,-1,-1,-1],"t":[4.5,7.5,6.241071428571429,0,0,11.578476357671452,0,0,4.882435795368045,7.5,0,2.5,0,0,0.7457446808510639,0,17.094543568094064,9.068775776507199,5.89476682248518,7.5,4.0437314206251695,0,6.022367813028454,0,0,0,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,0,0,15,0,17,18,19,20,21,0,23,0,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,14,11,0,13,0,0,16,0,28,27,26,25,22,0,24,0,0,0,0,0,0],"v":[0,0,0,-0.03685333,0.02180261,0,-0.0444826,0.05905361,0,0,-0.02985524,0,0.03159317,-0.04741601,0,-0.03399038,0,0,0,0,0,0.01851626,0,0.02402876,0.1257916,0.01502488,-0.001678763,0.04544176,0.07722198]},{"f":[21,24,-1,17,-1,-1,2,24,-1,35,-1,-1,1,-1,22,2,4,21,22,4,-1,15,-1,-1,-1,-1,-1,-1,-1],"t":[4.5,7.5,0,11.578476357671452,0,0,4.882435795368045,7.5,0,2.5,0,0,0.7457446808510639,0,17.094543568094064,9.068775776507199,5.89476682248518,7.5,16.98920034885663,4.0437314206251695,0,6.0,0,0,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,0,10,0,0,13,0,15,16,17,18,19,20,0,22,0,0,0,0,0,0,0],"r":[6,3,0,5,0,0,12,9,0,11,0,0,14,0,28,27,26,25,24,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0.01204038,0,-0.04411569,0.05459654,0,0,-0.02917017,0,0.02975699,-0.0471057,0,-0.03334568,0,0,0,0,0,0,0.03599023,0,0.02607497,0.1228093,-0.02388121,0.01414081,-0.001597897,0.04221126,0.07469412]},{"f":[21,24,16,22,-1,-1,-1,17,-1,-1,2,24,22,-1,15,-1,-1,35,-1,-1,0,-1,2,22,4,-1,-1,-1,-1],"t":[4.5,7.5,6.392317422778761,12.613886328427084,0,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,6.584962500721156,0,0,2.5,0,0,2.5,0,7.640240405159187,17.094543568094064,5.89476682248518,0,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,13,0,15,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[10,7,6,5,0,0,0,9,0,0,20,17,14,0,16,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,-0.02161302,0.03444387,-0.0249776,0,-0.04374068,0.0506102,0,0,0,-0.03488475,0,0.04865143,-0.03849569,0,0.0280519,-0.04679523,0,-0.01251703,0,0,0,0.02208245,0.00594982,0.07343997,0.0444141]},{"f":[21,24,-1,17,-1,-1,2,24,22,-1,-1,35,-1,-1,1,-1,22,2,4,21,22,-1,-1,-1,25,-1,-1,-1,-1],"t":[4.5,7.5,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,0,2.5,0,0,0.7457446808510639,0,17.094543568094064,9.068775776507199,5.89476682248518,7.5,16.98920034885663,0,0,0,0.5,0,0,0,0],"m":[0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,1,1,1,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,0,0,12,0,0,15,0,17,18,19,20,21,0,0,0,25,0,0,0,0],"r":[6,3,0,5,0,0,14,11,10,0,0,13,0,0,16,0,28,27,24,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0.01077333,0,-0.04335725,0.04702737,0,0,0,-0.03429881,0.01213996,0,0.02646501,-0.0464842,0,-0.03273892,0,0,0,0,0,0.05840486,-0.02381983,0.0120509,0,-0.008741902,0.0477524,0.03715148,0.07030573]},{"f":[21,24,20,-1,-1,17,-1,-1,2,24,22,16,-1,16,-1,-1,-1,35,15,-1,-1,-1,1,-1,22,2,-1,-1,-1],"t":[4.5,7.5,0.5,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,7.930737337562887,0,7.948367231584678,0,0,0,2.5,8.618385502258606,0,0,0,0.7457446808510639,0,17.094543568094064,10.246900367172017,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,0,14,0,0,0,18,19,0,0,0,23,0,25,26,0,0,0],"r":[8,5,4,0,0,7,0,0,22,17,16,13,0,15,0,0,0,21,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,-0.01620981,0.02469631,0,-0.04296512,0.04379272,0,0,0,0,-0.04605363,0,0.1118448,-0.01814951,0.01148982,0,0,0.03450058,-0.008189892,-0.04617224,0,-0.03209099,0,0,0.01515762,0.04674731,0.06834029]},{"f":[21,22,-1,-1,2,32,23,-1,-1,22,-1,17,-1,-1,1,-1,22,23,-1,-1,15,-1,22,20,-1,36,-1,-1,-1],"t":[3.5,14.03140051979143,0,0,4.95344529780426,9.271599002932193,3.5,0,0,17.094543568094064,0,7.588709768994619,0,0,0.7457446808510639,0,12.199665042528586,2.5,0,0,2.0,0,17.094543568094064,1.5,0,5.5,0,0,0],"m":[0,1,0,0,1,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,0,12,0,0,15,0,17,18,0,0,21,0,23,24,0,26,0,0,0],"r":[4,3,0,0,14,9,8,0,0,11,0,13,0,0,16,0,20,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,0,-0.0492693,0.01850451,0,0,0,0.0297388,-0.03108552,0,-0.0300383,0,-0.03732463,0.05620476,0,-0.03248162,0,0,0.01040253,-0.03359049,0,0.07708978,0,0,0.01205445,0,0.07895045,0.01385546,0.0675537]},{"f":[21,-1,2,32,23,37,-1,-1,-1,22,-1,23,-1,-1,1,-1,22,2,22,23,-1,-1,2,16,-1,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,3.5,8.05,0,0,0,17.094543568094064,0,7.5,0,0,1.282051282051282,0,17.094543568094064,7.997154617983022,14.116600814324018,2.5,0,0,6.011183906514227,6.247927513443585,0,0,0,0,0],"m":[0,0,1,0,1,1,0,0,0,1,0,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,0,12,0,0,15,0,17,18,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,14,9,8,7,0,0,0,11,0,13,0,0,16,0,28,27,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.04718663,0,0,0,0,0.006771033,0.03678638,-0.03012814,0,-0.0351069,0,0.04960633,-0.04001062,0,-0.01580857,0,0,0,0,0.007391575,-0.03172959,0,0,0.0191489,0.1123809,0.01037857,0.03187703,0.06580984]},{"f":[21,-1,2,32,23,-1,-1,22,-1,-1,1,21,-1,-1,22,2,4,18,-1,-1,22,1,-1,-1,16,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,3.5,0,0,17.094543568094064,0,0,1.282051282051282,7.5,0,0,17.094543568094064,7.997154617983022,5.89476682248518,5.303665156874805,0,0,16.759160379664443,8.985394456289978,0,0,8.060695931687555,0,0,0,0],"m":[0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,1,1,1,0,0,1,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,0,11,12,0,0,15,16,17,18,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,10,7,6,0,0,9,0,0,14,13,0,0,28,27,20,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.04694051,0,0,0,0.01720874,-0.02918956,0,-0.03453166,0.02455382,0,0,-0.03967469,0.008688512,0,0,0,0,0.01007621,0.04274037,0,0,-0.02485978,0.03548785,0,-0.01045169,0.09564767,0.02987968,0.06417699]},{"f":[21,-1,2,32,23,-1,-1,22,-1,-1,22,2,1,15,-1,-1,22,-1,-1,-1,22,-1,22,-1,22,-1,22,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,4.5,0,0,17.094543568094064,0,0,16.765402642760364,10.510264838663748,4.008411214953271,10.043027283594547,0,0,16.73920708776685,0,0,0,16.786010507999382,0,16.838461650489297,0,16.975010474608837,0,17.094543568094064,0,0],"m":[0,0,1,0,1,0,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,8,0,0,11,12,13,14,0,0,17,0,0,0,21,0,23,0,25,0,27,0,0],"r":[2,0,10,7,6,0,0,9,0,0,20,19,16,15,0,0,18,0,0,0,22,0,24,0,26,0,28,0,0],"v":[0,-0.0466876,0,0,0,0.01441173,-0.05029048,0,-0.03394833,0.02309867,0,0,0,0,0.0001123449,-0.03173058,0,0.02017585,-0.03624753,0.04229864,0,0.122649,0,-0.02831197,0,0.08257443,0,0.0003875773,0.06264172]},{"f":[21,-1,2,32,10,-1,37,-1,-1,22,-1,-1,1,-1,22,2,23,-1,-1,-1,2,16,-1,-1,22,-1,22,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,0.5,0,8.05,0,0,17.094543568094064,0,0,0.8611111111111112,0,14.116600814324018,10.510264838663748,2.5,0,0,0,6.011183906514227,6.247927513443585,0,0,16.765402642760364,0,16.786010507999382,0,0],"m":[0,0,1,0,0,0,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,0,15,16,17,0,0,0,21,22,0,0,25,0,27,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,14,0,20,19,18,0,0,0,24,23,0,0,26,0,28,0,0],"v":[0,-0.04642763,0,0,0,-0.0483602,0,0.0004197282,0.03627079,0,-0.03335729,0.02174877,0,-0.03121126,0,0,0,0.006779979,-0.02569572,0.04693949,0,0,0.01533142,0.1008213,0,0.005871671,0,0.1136719,0.01681201]},{"f":[21,-1,2,32,23,19,37,-1,-1,-1,-1,22,-1,-1,22,1,-1,2,16,17,-1,-1,1,2,-1,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,4.5,2.5,8.05,0,0,0,0,17.094543568094064,0,0,17.094543568094064,0.8611111111111112,0,8.9274737487839,6.0,7.6181566135214025,0,0,6.241071428571429,5.40929108874043,0,0,0,0,0],"m":[0,0,1,0,1,1,1,0,0,0,0,1,0,0,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,0,12,0,0,15,16,0,18,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,14,11,10,9,8,0,0,0,0,13,0,0,28,17,0,27,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.04616034,0,0,0,0,0,0.004349323,0.03509822,-0.04451945,-0.04941121,0,-0.03275894,0.02049418,0,0,-0.03056908,0,0,0,0.005214205,-0.03269304,0,0,0.05043723,0.002658791,0.02975996,0.0245357,0.0607468]},{"f":[21,-1,2,32,23,-1,-1,22,-1,23,-1,-1,22,9,2,22,23,-1,-1,2,16,-1,-1,0,-1,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,9.271599002932193,3.5,0,0,17.094543568094064,0,7.5,0,0,17.094543568094064,0.5,10.510264838663748,14.116600814324018,2.5,0,0,5.8698903048866296,6.247927513443585,0,0,10.5,0,0,0,0,0],"m":[0,0,1,0,1,0,0,1,0,1,0,0,1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,0,0,13,14,15,16,17,0,0,20,21,0,0,24,0,0,0,0,0],"r":[2,0,12,7,6,0,0,9,0,11,0,0,28,27,26,19,18,0,0,23,22,0,0,25,0,0,0,0,0],"v":[0,-0.04588547,0,0,0,0.01812057,-0.02792353,0,-0.02898088,0,0.04048717,-0.04056214,0,0,0,0,0,0.004344982,-0.0255992,0,0,0.01477525,0.0990032,0,0.009703885,0.05599894,0.03638467,-0.04495106,0.05939642]},{"f":[21,-1,2,32,10,-1,37,-1,-1,22,-1,-1,22,2,1,-1,20,-1,6,-1,-1,-1,22,-1,22,-1,17,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,0.5,0,8.05,0,0,17.094543568094064,0,0,16.765402642760364,10.042900245509953,0.8611111111111112,0,0.5,0,0.5,0,0,0,16.786010507999382,0,16.844368517741394,0,8.571721503073015,0,0],"m":[0,0,1,0,0,0,1,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,14,15,0,17,0,19,0,0,0,23,0,25,0,27,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,22,21,16,0,18,0,20,0,0,0,24,0,26,0,28,0,0],"v":[0,-0.04560277,0,0,0,-0.04729053,0,-1.370328e-05,0.03239601,0,-0.03160328,0.01823889,0,0,0,-0.04492289,0,-0.003970515,0,0.01107503,0.08406212,0.03206478,0,0.1042065,0,-0.02511207,0,0.01125656,0.06441634]},{"f":[21,-1,2,1,-1,15,19,17,22,22,37,15,-1,16,-1,16,-1,-1,-1,-1,-1,-1,-1,-1,22,9,-1,-1,-1],"t":[2.5,0,4.95344529780426,4.208333333333334,0,9.92481250360578,2.5,8.729619742794132,17.094543568094064,16.25454052700782,10.26,5.044394119358453,0,6.044394119358453,0,6.832890014164741,0,0,0,0,0,0,0,0,17.094543568094064,0.5,0,0,0],"m":[0,0,1,1,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,11,12,0,14,0,16,0,0,0,0,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,22,21,20,19,18,13,0,15,0,17,0,0,0,0,0,0,0,0,28,27,0,0,0],"v":[0,-0.045312,0,0,-0.03704434,0,0,0,0,0,0,0,0.03114977,0,-0.04344409,0,0.03093049,-0.0009472042,0.06598908,-0.0446415,0.07566763,-0.03917708,-0.04533979,-0.03784864,0,0,0.009485297,-0.04435298,0.0572878]},{"f":[21,22,-1,-1,0,2,-1,-1,2,32,25,-1,-1,-1,22,2,22,-1,2,16,-1,-1,22,-1,1,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,2.5,6.692122061121539,0,0,5.022197059679227,8.041220073122355,1.5,0,0,0,17.094543568094064,9.341519573083566,14.116600814324018,0,5.8698903048866296,6.247927513443585,0,0,16.759160379664443,0,3.354166666666667,0,0,0,0],"m":[0,1,0,0,0,1,0,0,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,0,0,0,15,16,17,0,19,20,0,0,23,0,25,0,0,0,0],"r":[4,3,0,0,8,7,0,0,14,13,12,0,0,0,28,27,18,0,22,21,0,0,24,0,26,0,0,0,0],"v":[0,0,-0.04941795,0.02040811,0,0,-0.03471049,-0.007214486,0,0,0,0.0179928,-0.005743234,-0.007214486,0,0,0,-0.002035699,0,0,0.01385001,0.09164148,0,0.001010537,0,0.0697626,0.006305775,0.03437896,0.05677849]},{"f":[21,-1,2,1,-1,10,-1,15,37,16,33,-1,-1,16,-1,32,-1,-1,-1,-1,22,9,2,1,-1,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,4.208333333333334,0,0.5,0,10.457380879072534,10.26,6.832890014164741,21.257266042509173,0,0,7.930737337562887,0,8.230739723643078,0,0,0,0,17.094543568094064,0.5,8.9274737487839,4.92309205350118,0,0,0,0,0],"m":[0,0,1,1,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,1,1,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,9,10,11,0,0,14,0,16,0,0,0,0,21,22,23,24,0,0,0,0,0],"r":[2,0,20,5,0,7,0,19,18,13,12,0,0,15,0,17,0,0,0,0,28,27,26,25,0,0,0,0,0],"v":[0,-0.04476815,0,0,-0.0362323,0,-0.0458471,0,0,0,0,0.02693624,-0.01336106,0,-0.03647401,0,0.02646223,-0.01483124,0.05658854,-0.03682662,0,0,0,0,-0.004279183,0.01234558,0.01975654,-0.04380781,0.05499583]},{"f":[21,22,-1,-1,2,1,-1,10,-1,15,37,25,15,17,-1,-1,4,-1,22,-1,-1,-1,-1,-1,22,9,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,4.95344529780426,4.208333333333334,0,0.5,0,10.457380879072534,10.26,1.5,6.857980995127572,8.729619742794132,0,0,1.292481250360578,0,15.676331898425197,0,0,0,0,0,17.094543568094064,0.5,0,0,0],"m":[0,1,0,0,1,1,0,0,0,1,1,1,0,1,0,0,1,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,0,0,5,6,0,8,0,10,11,12,13,14,0,0,17,0,19,0,0,0,0,0,25,26,0,0,0],"r":[4,3,0,0,24,7,0,9,0,23,22,21,16,15,0,0,18,0,20,0,0,0,0,0,28,27,0,0,0],"v":[0,0,-0.04927137,0.02153649,0,0,-0.03570202,0,-0.04543541,0,0,0,0,0,0.03003618,-0.03446454,0,-0.02258577,0,0.03079846,-0.02124878,-0.02108944,0.0541399,-0.03631026,0,0,0.008151014,-0.04349749,0.05389784]},{"f":[21,-1,23,0,21,-1,-1,22,21,-1,-1,22,15,-1,-1,22,-1,15,-1,22,2,-1,22,-1,-1,18,-1,-1,-1],"t":[2.5,0,7.5,2.5,7.5,0,0,12.199665042528586,8.5,0,0,12.921158001713904,7.930737337562887,0,0,14.116600814324018,0,5.044394119358453,0,17.094543568094064,3.084962500721156,0,14.158358850608169,0,0,8.729619742794132,0,0,0],"m":[0,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,0,0,16,0,18,0,20,21,0,23,0,0,26,0,0,0],"r":[2,0,28,7,6,0,0,11,10,0,0,15,14,0,0,17,0,19,0,25,22,0,24,0,0,27,0,0,0],"v":[0,-0.04419066,0,0,0,-0.03259207,-0.006138191,0,0,-0.006138191,0.04357473,0,0,0.0212662,0.125602,0,-0.006138191,0,0.0339114,0,0,-0.006138191,0,0.03899336,0.006318712,0,0.06261998,-0.006138191,-0.04853153]},{"f":[21,-1,2,1,-1,10,-1,17,22,-1,-1,-1,22,2,1,-1,22,-1,-1,-1,22,-1,22,-1,22,-1,22,-1,-1],"t":[2.5,0,4.95344529780426,4.208333333333334,0,0.5,0,8.729619742794132,17.07907323966382,0,0,0,16.765402642760364,10.510264838663748,4.008411214953271,0,16.73920708776685,0,0,0,16.786010507999382,0,16.838461650489297,0,16.975010474608837,0,17.094543568094064,0,0],"m":[0,0,1,1,0,0,0,1,1,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,9,0,0,0,13,14,15,0,17,0,0,0,21,0,23,0,25,0,27,0,0],"r":[2,0,12,5,0,7,0,11,10,0,0,0,20,19,16,0,18,0,0,0,22,0,24,0,26,0,28,0,0],"v":[0,-0.04385387,0,0,-0.0349297,0,-0.04484473,0,0,0.00600041,0.06816357,-0.04190301,0,0,0,-0.007255128,0,0.01373254,-0.03810842,0.02839899,0,0.09229408,0,-0.02894215,0,0.07035562,0,-0.003676729,0.05189646]},{"f":[21,-1,23,21,17,18,-1,-1,15,16,-1,-1,-1,37,1,2,16,-1,-1,22,2,4,-1,-1,-1,-1,-1,-1,-1],"t":[2.5,0,7.5,4.5,7.198837316474133,1.792481250360578,0,0,6.857980995127572,6.247927513443585,0,0,0,11.73,8.992857142857144,4.0437314206251695,6.569855608330948,0,0,16.765402642760364,10.510264838663748,5.714180086352146,0,0,0,0,0,0,0],"m":[0,0,1,0,1,1,0,0,1,1,0,0,0,1,1,1,0,0,0,1,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,0,0,0,14,15,16,17,0,0,20,21,22,0,0,0,0,0,0,0],"r":[2,0,28,13,8,7,0,0,12,11,0,0,0,27,26,19,18,0,0,25,24,23,0,0,0,0,0,0,0],"v":[0,-0.0435077,0,0,0,0,-0.04187091,-0.007272224,0,0,0.01563269,0.07788661,-0.03591708,0,0,0,0,-0.03691492,-0.001781169,0,0,0,0.008281345,-0.0131824,0.02680986,0.02282309,0.02093767,0.05843752,-0.04805242]},{"f":[21,22,-1,-1,23,21,17,18,-1,-1,15,16,-1,3,-1,-1,-1,37,2,1,21,-1,-1,22,-1,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,4.5,7.198837316474133,1.792481250360578,0,0,6.857980995127572,6.247927513443585,0,0.5803571428571428,0,0,0,11.73,7.640240405159187,4.92309205350118,7.5,0,0,12.199665042528586,0,0,0,0,0],"m":[0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,0,11,12,0,14,0,0,0,18,19,20,21,0,0,24,0,0,0,0,0],"r":[4,3,0,0,28,17,10,9,0,0,16,13,0,15,0,0,0,27,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,-0.04897354,0.02675818,0,0,0,0,-0.04141658,-0.006956498,0,0,0.01479017,0,0.01785389,0.1227712,-0.03521335,0,0,0,0,-0.03332217,0.001615612,0,-0.00205188,0.01489583,0.01739247,0.05616515,-0.04767729]},{"f":[21,22,-1,-1,23,21,22,-1,17,-1,22,-1,22,-1,-1,37,15,22,-1,-1,1,2,-1,-1,17,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,4.5,12.199665042528586,0,1.292481250360578,0,12.278990886161445,0,12.613886328427084,0,0,11.73,5.044394119358453,12.583618810773686,0,0,4.92309205350118,7.005613627711627,0,0,7.430398005367313,0,0,0,0],"m":[0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,0,13,0,0,16,17,18,0,0,21,22,0,0,25,0,0,0,0],"r":[4,3,0,0,28,15,8,0,10,0,12,0,14,0,0,27,20,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,0,-0.04888828,0.02546888,0,0,0,-0.04170173,0,-0.03974655,0,0.04629398,0,-0.04470534,0.009147239,0,0,0,0.001289558,0.02951269,0,0,-0.01946889,0.01071366,0,-0.002258804,0.0173456,0.05401979,-0.04729785]},{"f":[21,22,-1,-1,23,21,22,-1,17,-1,22,36,-1,-1,22,-1,4,-1,-1,37,1,2,16,-1,-1,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,4.5,12.199665042528586,0,1.292481250360578,0,12.278990886161445,2.5,0,0,12.613886328427084,0,3.084962500721156,0,0,11.73,8.992857142857144,4.0437314206251695,7.930737337562887,0,0,0,0,0,0],"m":[0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,12,0,0,15,0,17,0,0,20,21,22,23,0,0,0,0,0,0],"r":[4,3,0,0,28,19,8,0,10,0,14,13,0,0,16,0,18,0,0,27,26,25,24,0,0,0,0,0,0],"v":[0,0,-0.04880087,0.02425331,0,0,0,-0.04126047,0,-0.03924992,0,0,-0.01910794,0.06144132,0,-0.04415405,0,-0.007312026,0.0228358,0,0,0,0,-0.02553072,0.003437068,0.006317643,0.01894109,0.05198711,-0.04691395]},{"f":[21,-1,23,0,21,-1,-1,22,21,-1,-1,22,15,-1,-1,22,16,-1,-1,15,16,-1,4,-1,-1,0,-1,-1,-1],"t":[2.5,0,7.5,2.5,7.5,0,0,12.199665042528586,8.5,0,0,12.921158001713904,7.930737337562887,0,0,14.116600814324018,10.462502272597265,0,0,7.266786540694901,6.247927513443585,0,3.245926548164837,0,0,10.5,0,0,0],"m":[0,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,1,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,0,0,16,17,0,0,20,21,0,23,0,0,26,0,0,0],"r":[2,0,28,7,6,0,0,11,10,0,0,15,14,0,0,19,18,0,0,25,22,0,24,0,0,27,0,0,0],"v":[0,-0.04224213,0,0,0,-0.03050701,-0.005608156,0,0,-0.005608156,0.03809337,0,0,0.01822794,0.1093571,0,0,-0.005608156,0.07042198,0,0,0.007620166,0,0.01794224,0.06913722,0,-0.0002947462,0.05159234,-0.04652544]},{"f":[21,-1,23,21,17,18,-1,-1,15,-1,-1,1,16,2,22,-1,17,-1,-1,-1,16,-1,22,-1,-1,2,-1,-1,-1],"t":[2.5,0,7.5,4.5,7.198837316474133,1.792481250360578,0,0,6.857980995127572,0,0,8.992857142857144,7.930737337562887,4.80643424864552,17.094543568094064,0,8.729619742794132,0,0,0,7.948367231584678,0,16.759160379664443,0,0,5.2849278041654735,0,0,0],"m":[0,0,1,0,1,1,0,0,1,0,0,1,1,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,0,3,4,5,6,0,0,9,0,0,12,13,14,15,0,17,0,0,0,21,0,23,0,0,26,0,0,0],"r":[2,0,28,11,8,7,0,0,10,0,0,25,20,19,16,0,18,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,-0.04184212,0,0,0,0,-0.0400542,-0.007071741,0,0.02153407,-0.03367574,0,0,0,0,-0.04198294,0,0.08426537,-0.04637398,0.002076236,0,0.09779169,0,0.002774478,0.02704839,0,0.01441048,0.06510857,-0.0461322]},{"f":[21,22,-1,-1,23,21,22,-1,17,-1,22,-1,22,-1,-1,37,15,22,-1,-1,22,22,2,-1,-1,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,4.5,12.199665042528586,0,1.292481250360578,0,12.278990886161445,0,12.613886328427084,0,0,11.73,5.044394119358453,12.583618810773686,0,0,17.078739781790915,17.02942582914998,1.292481250360578,0,0,0,0,0,0],"m":[0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,0,13,0,0,16,17,18,0,0,21,22,23,0,0,0,0,0,0],"r":[4,3,0,0,28,15,8,0,10,0,12,0,14,0,0,27,20,19,0,0,26,25,24,0,0,0,0,0,0],"v":[0,0,-0.04855028,0.02712687,0,0,0,-0.0403746,0,-0.03806783,0,0.03874685,0,-0.04367935,0.00748707,0,0,0,0.001227466,0.02607162,0,0,0,-0.02032093,0.003960392,-0.047245,0.02348072,0.04782709,-0.04573414]},{"f":[21,22,-1,-1,23,1,21,-1,-1,21,22,-1,17,-1,22,36,-1,-1,22,-1,2,-1,17,-1,-1,37,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,0.9444444444444444,7.5,0,0,4.5,12.199665042528586,0,1.292481250360578,0,12.278990886161445,2.5,0,0,12.613886328427084,0,3.084962500721156,0,4.208926257442949,0,0,11.73,0,0,0],"m":[0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,0,13,0,15,16,0,0,19,0,21,0,23,0,0,26,0,0,0],"r":[4,3,0,0,28,9,8,0,0,25,12,0,14,0,18,17,0,0,20,0,22,0,24,0,0,27,0,0,0],"v":[0,0,-0.04845345,0.02581625,0,0,0,-0.0475481,0.02935341,0,0,-0.03883403,0,-0.03746568,0,0,-0.01974357,0.05223118,0,-0.04303301,0,-0.006108139,0,-0.02547879,0.02754156,0,0.004929074,0.04609694,-0.04533118]},{"f":[21,22,-1,-1,23,0,21,-1,-1,21,22,-1,22,15,-1,-1,22,15,-1,-1,18,32,-1,-1,-1,0,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,2.5,7.5,0,0,8.5,12.199665042528586,0,12.921158001713904,7.930737337562887,0,0,17.094543568094064,5.044394119358453,0,0,8.729619742794132,8.341935934311302,0,0,0,10.5,0,0,0],"m":[0,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,0,13,14,0,0,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[4,3,0,0,28,9,8,0,0,25,12,0,16,15,0,0,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0,-0.0483537,0.02458094,0,0,0,-0.02917053,-0.005283171,0,0,-0.005283171,0,0,0.01639215,0.09511894,0,0,0.01999178,-0.001952071,0,0,0.0005619419,0.07348316,-0.005283171,0,0.008752542,0.06373888,-0.04492329]},{"f":[21,22,-1,-1,2,16,-1,1,21,-1,1,-1,-1,17,22,35,22,18,-1,33,-1,-1,-1,-1,-1,-1,22,-1,-1],"t":[2.5,13.87915215758784,0,0,4.95344529780426,2.0,0,4.208333333333334,6.5,0,1.13125,0,0,8.729619742794132,17.07907323966382,2.5,16.25454052700782,2.160964047443681,0,21.011466140970732,0,0,0,0,0,0,17.094543568094064,0,0],"m":[0,1,0,0,1,0,0,1,1,0,0,0,0,1,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,0,0,5,6,0,8,9,0,11,0,0,14,15,16,17,18,0,20,0,0,0,0,0,0,27,0,0],"r":[4,3,0,0,26,7,0,13,10,0,12,0,0,25,24,23,22,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,-0.04825086,0.02341546,0,0,-0.04576753,0,0,-0.04319346,0,0.1066285,-0.02206816,0,0,0,0,0,0.01974611,0,-0.02803953,0.006045121,-0.04009057,-0.04314444,0.05530916,-0.04066525,0,0.004515912,0.04894231]},{"f":[21,-1,2,37,21,-1,-1,16,-1,22,36,-1,-1,22,-1,15,18,22,4,-1,22,-1,33,-1,-1,-1,-1,-1,-1],"t":[2.5,0,8.9274737487839,4.470000000000001,6.5,0,0,6.044394119358453,0,12.199665042528586,4.5,0,0,12.738408632106918,0,7.266786540694901,8.729619742794132,16.765402642760364,1.292481250360578,0,14.03140051979143,0,21.257266042509173,0,0,0,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,0,0,14,0,16,17,18,19,0,21,0,23,0,0,0,0,0,0],"r":[2,0,28,7,6,0,0,9,0,13,12,0,0,15,0,27,26,25,20,0,22,0,24,0,0,0,0,0,0],"v":[0,-0.04002015,0,0,0,-0.04276778,-0.008689074,0,-0.008471613,0,0,-0.008689074,0.02910244,0,0.07308604,0,0,0,0,-0.0008009462,0,-0.0006805364,0,0.0770557,0.007129929,0.08080158,-0.008689074,0.001663858,0.0141975]},{"f":[21,24,-1,-1,23,1,21,-1,-1,21,22,22,22,21,5,-1,-1,-1,1,1,-1,-1,-1,-1,-1,0,-1,-1,-1],"t":[2.5,6.5,0,0,7.5,0.9444444444444444,7.5,0,0,8.5,14.116600814324018,12.921158001713904,12.199665042528586,5.5,0.5,0,0,0,7.772727272727273,2.969223484848485,0,0,0,0,0,10.5,0,0,0],"m":[0,0,0,0,1,0,1,0,0,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,12,13,14,15,0,0,0,19,20,0,0,0,0,0,26,0,0,0],"r":[4,3,0,0,28,9,8,0,0,25,24,23,18,17,16,0,0,0,22,21,0,0,0,0,0,27,0,0,0],"v":[0,0,0.02483755,-0.04828474,0,0,0,-0.04716233,0.02722272,0,0,0,0,0,0,0.01679259,-0.04112182,0.001563527,0,0,0.01306856,0.06467286,-0.0007962808,-0.03095008,0.006670292,0,0.007506407,0.0615206,-0.04432304]},{"f":[21,22,-1,-1,23,1,21,-1,-1,21,2,33,-1,22,17,16,18,-1,-1,21,-1,-1,-1,-1,-1,0,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,0.9444444444444444,7.5,0,0,8.5,3.084962500721156,21.16864017939033,0,17.094543568094064,2.160964047443681,7.930737337562887,1.792481250360578,0,0,5.5,0,0,0,0,0,10.5,0,0,0],"m":[0,1,0,0,1,0,1,0,0,1,1,0,0,1,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,12,0,14,15,16,17,0,0,20,0,0,0,0,0,26,0,0,0],"r":[4,3,0,0,28,9,8,0,0,25,24,13,0,23,22,19,18,0,0,21,0,0,0,0,0,27,0,0,0],"v":[0,0,-0.04795446,0.02502213,0,0,0,-0.04694082,0.02577395,0,0,0,0.009318887,0,0,0,0,-0.03690457,0.06536049,0,-0.0356707,0.03524162,-0.03065157,0.02379027,0.00454146,0,0.007110988,0.06012012,-0.04389734]},{"f":[21,24,-1,-1,23,2,37,21,-1,-1,22,16,-1,22,-1,22,-1,-1,18,32,-1,-1,-1,22,18,-1,-1,-1,-1],"t":[2.5,6.5,0,0,7.5,8.9274737487839,4.470000000000001,6.5,0,0,17.094543568094064,6.044394119358453,0,12.199665042528586,0,15.676331898425197,0,0,8.729619742794132,8.341935934311302,0,0,0,16.759160379664443,10.460454442916898,0,0,0,0],"m":[0,0,0,0,1,1,1,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,0,11,12,0,14,0,16,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[4,3,0,0,28,23,10,9,0,0,18,13,0,15,0,17,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0.02452211,-0.0480799,0,0,0,0,-0.04201388,-0.007746937,0,0,-0.00718756,0,-0.003774955,0,0.02277743,-0.002076101,0,0,-0.003835715,0.06897731,-0.007746937,0,0,0.001869323,0.02938925,0.05522759,-0.0434666]},{"f":[21,22,-1,-1,23,1,21,-1,-1,21,2,1,-1,15,19,-1,-1,-1,22,-1,22,-1,22,-1,33,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,0.9444444444444444,7.5,0,0,10.5,4.882435795368045,4.208333333333334,0,10.457380879072534,2.5,0,0,0,16.765402642760364,0,16.786010507999382,0,16.844368517741394,0,28.295013863458657,0,0,0,0],"m":[0,1,0,0,1,0,1,0,0,1,1,1,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,12,0,14,15,0,0,0,19,0,21,0,23,0,25,0,0,0,0],"r":[4,3,0,0,28,9,8,0,0,27,18,13,0,17,16,0,0,0,20,0,22,0,24,0,26,0,0,0,0],"v":[0,0,-0.04773352,0.02462824,0,0,0,-0.04663599,0.0244132,0,0,0,-0.0287097,0,0,0.005634083,-0.04359395,-0.04076002,0,0.002659729,0,0.07997755,0,-0.02762028,0,0.07424978,0.006695558,0.08385683,-0.04303091]},{"f":[21,24,-1,-1,23,21,22,-1,17,-1,-1,1,16,2,17,17,-1,-1,-1,-1,22,-1,24,-1,-1,2,-1,-1,-1],"t":[2.5,6.5,0,0,7.5,4.5,12.199665042528586,0,1.292481250360578,0,0,8.992857142857144,7.930737337562887,4.80643424864552,8.496347487943302,0.5,0,0,0,0,16.844368517741394,0,7.5,0,0,5.2849278041654735,0,0,0],"m":[0,0,0,0,1,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,0,12,13,14,15,16,0,0,0,0,21,0,23,0,0,26,0,0,0],"r":[4,3,0,0,28,11,8,0,10,0,0,25,20,19,18,17,0,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0.02419855,-0.04786217,0,0,0,-0.03808526,0,-0.03572993,0.005870632,0,0,0,0,0,0.03862579,-0.04792835,0.01637885,0.0003100423,0,0.004470237,0,0.007539149,0.08173934,0,0.01125712,0.05923616,-0.04259034]},{"f":[21,22,-1,-1,16,27,-1,22,-1,-1,33,24,22,22,22,22,22,-1,-1,-1,2,-1,-1,-1,18,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,1.0,0.5,0,16.975010474608837,0,0,35.98606694718995,9.5,17.078739781790915,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,3.903677461028802,0,0,0,8.729619742794132,0,0,0,0],"m":[0,1,0,0,0,1,0,0,0,0,1,1,1,1,1,1,1,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,0,8,0,0,11,12,13,14,15,16,17,0,0,0,21,0,0,0,25,0,0,0,0],"r":[4,3,0,0,10,7,0,9,0,0,28,27,24,23,20,19,18,0,0,0,22,0,0,0,26,0,0,0,0],"v":[0,0,-0.04749917,0.02423347,0,0,-0.04686202,0,0.05621368,-0.005436559,0,0,0,0,0,0,0,0.0004997464,0.08364889,-0.03217078,0,-0.03242364,0.07578061,-0.0460498,0,0.04698117,-0.04408739,0.03197118,-0.03342913]},{"f":[21,24,-1,-1,23,1,21,-1,-1,21,2,1,21,-1,17,15,-1,-1,-1,15,19,24,-1,-1,-1,-1,-1,-1,-1],"t":[2.5,6.5,0,0,7.5,0.9444444444444444,7.5,0,0,10.5,4.882435795368045,4.208333333333334,6.5,0,0.5,1.0,0,0,0,10.457380879072534,2.5,7.5,0,0,0,0,0,0,0],"m":[0,0,0,0,1,0,1,0,0,1,1,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,12,13,0,15,16,0,0,0,20,21,22,0,0,0,0,0,0,0],"r":[4,3,0,0,28,9,8,0,0,27,26,19,14,0,18,17,0,0,0,25,24,23,0,0,0,0,0,0,0],"v":[0,0,0.02386832,-0.04763062,0,0,0,-0.04636445,0.02292783,0,0,0,0,-0.0424868,0,0,-0.02165982,0.1055114,-0.02353047,0,0,0,-0.008981635,0.01098317,-0.04289208,-0.04013795,0.00496509,0.08177919,-0.04192196]},{"f":[21,22,-1,-1,2,37,21,-1,-1,16,-1,22,36,-1,-1,22,1,-1,-1,15,18,22,-1,-1,-1,-1,22,-1,-1],"t":[2.5,13.87915215758784,0,0,8.9274737487839,4.470000000000001,6.5,0,0,6.044394119358453,0,12.199665042528586,4.5,0,0,12.738408632106918,6.806765389082463,0,0,7.266786540694901,8.729619742794132,16.765402642760364,0,0,0,0,16.765402642760364,0,0],"m":[0,1,0,0,1,1,1,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,0,0,5,6,7,0,0,10,0,12,13,0,0,16,17,0,0,20,21,22,0,0,0,0,27,0,0],"r":[4,3,0,0,26,9,8,0,0,11,0,15,14,0,0,19,18,0,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,-0.04725055,0.02383873,0,0,0,-0.04093104,-0.007546297,0,-0.007546297,0,0,-0.007546297,0.02396932,0,0,0.01910986,0.09846133,0,0,0,0.01396939,0.0720513,-0.007546297,7.156398e-05,0,0.00795188,0.05227254]},{"f":[21,-1,23,21,22,21,-1,16,-1,-1,33,20,-1,-1,33,-1,-1,1,16,2,17,-1,-1,-1,-1,2,-1,-1,-1],"t":[1.5,0,7.5,4.5,14.03140051979143,3.5,0,5.78135971352466,0,0,12.738373101969731,0.5,0,0,19.167172160526107,0,0,8.992857142857144,7.930737337562887,4.80643424864552,8.496347487943302,0,0,0,0,5.2849278041654735,0,0,0],"m":[0,0,1,0,1,1,0,0,0,0,1,1,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,12,0,0,15,0,0,18,19,20,21,0,0,0,0,26,0,0,0],"r":[2,0,28,17,10,7,0,9,0,0,14,13,0,0,16,0,0,25,24,23,22,0,0,0,0,27,0,0,0],"v":[0,-0.046071,0,0,0,0,-0.0435097,0,0.006826579,-0.02956261,0,0,-0.005971999,0.1224262,0,-0.04402448,0.009464642,0,0,0,0,-0.04075631,0.01413626,5.431809e-07,0.005686057,0,0.01053436,0.05721801,-0.04159016]},{"f":[21,-1,23,1,21,-1,-1,21,22,21,-1,23,-1,-1,15,-1,15,17,-1,-1,15,4,-1,-1,0,-1,-1,-1,-1],"t":[1.5,0,7.5,0.9444444444444444,7.5,0,0,10.5,14.116600814324018,3.5,0,3.5,0,0,5.044394119358453,0,6.108524456778169,7.67595272073847,0,0,7.266786540694901,3.084962500721156,0,0,10.5,0,0,0,0],"m":[0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,0,12,0,0,15,0,17,18,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,28,7,6,0,0,27,14,11,0,13,0,0,16,0,20,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.04588038,0,0,0,-0.04607412,0.02125406,0,0,0,-0.04249296,0,0.003742941,-0.02438879,0,0.02290859,0,0,-0.04536661,0.02687842,0,0,0.003454589,0.04807964,0,-0.002520004,0.0412374,0.07973202,-0.041128]},{"f":[21,-1,2,37,21,-1,-1,16,-1,22,36,-1,-1,22,1,-1,-1,22,22,-1,1,-1,-1,-1,22,17,-1,-1,-1],"t":[1.5,0,8.9274737487839,4.470000000000001,6.5,0,0,6.108524456778169,0,12.199665042528586,4.5,0,0,12.738408632106918,6.806765389082463,0,0,15.676331898425197,14.116600814324018,0,6.431742323097463,0,0,0,16.765402642760364,12.18494601563336,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,0,0,14,15,0,0,18,19,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,7,6,0,0,9,0,13,12,0,0,17,16,0,0,23,20,0,22,0,0,0,28,27,0,0,0],"v":[0,-0.04568415,0,0,0,-0.04086517,-0.007595534,0,-0.006430627,0,0,-0.007595534,0.0220775,0,0,0.01827047,0.09160768,0,0,-0.007595534,0,0.08459528,0.01453342,-0.0005234241,0,0,0.003334381,0.03665177,0.04993822]},{"f":[21,-1,23,21,22,22,4,-1,22,22,22,21,2,-1,-1,-1,-1,-1,16,-1,-1,-1,-1,36,15,-1,-1,-1,-1],"t":[1.5,0,7.5,8.5,15.439474430023438,14.158358850608169,1.292481250360578,0,14.116600814324018,13.054613167076562,12.613886328427084,5.5,7.038658222940513,0,0,0,0,0,6.247927513443585,0,0,0,0,3.5,12.000704269011246,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,11,12,13,0,0,0,0,0,19,0,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,21,8,0,18,17,16,15,14,0,0,0,0,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,-0.0454822,0,0,0,0,0,-0.02043748,0,0,0,0,0,-0.02337914,0.01722297,0.006105582,0.02885044,-0.02568061,0,0.002582041,0.0639365,-0.04359709,0.005296599,0,0,-0.0006533681,0.07682648,0.05743497,-0.04079216]},{"f":[21,-1,23,21,0,22,-1,-1,4,22,-1,15,25,23,-1,4,-1,-1,-1,-1,23,-1,-1,36,15,-1,-1,-1,-1],"t":[1.5,0,7.5,8.5,2.5,12.278990886161445,0,0,9.427971070948235,14.116600814324018,0,7.266786540694901,0.5,0.5,0,4.951187557243013,0,0,0,0,0.5,0,0,3.5,12.000704269011246,0,0,0,0],"m":[0,0,1,1,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,0,12,13,14,0,16,0,0,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,23,8,7,0,0,20,11,0,19,18,15,0,17,0,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.04527443,0,0,0,0,-0.007049605,-0.03442445,0,0,-0.007049605,0,0,0,0.1660129,0,0.01137514,0.04552523,-0.001928244,-0.0007938151,0,0.04430979,-0.002838154,0,0,-0.0006220144,0.0743123,0.05622959,-0.04031933]},{"f":[21,-1,23,21,15,22,21,-1,23,4,-1,-1,-1,-1,22,18,3,-1,-1,22,-1,-1,-1,36,15,-1,-1,-1,-1],"t":[1.5,0,7.5,8.5,10.457380879072534,14.116600814324018,3.5,0,3.5,4.0437314206251695,0,0,0,0,16.759160379664443,10.680359255641626,0.23857586512866016,0,0,11.722778233171695,0,0,0,3.5,12.000704269011246,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,0,0,0,0,15,16,17,0,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,14,13,8,0,12,11,0,0,0,0,22,19,18,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.04506075,0,0,0,0,0,-0.04211861,0,0,-0.004151898,0.01314008,-0.02995186,0.006288763,0,0,0,0.007543494,-0.03079766,0,0.04910407,-0.03419864,0.05698289,0,0,-0.0005921548,0.07185534,0.05507834,-0.03984266]},{"f":[21,-1,23,1,21,-1,-1,21,22,22,22,16,-1,22,22,22,-1,-1,-1,-1,-1,-1,18,32,-1,-1,-1,-1,-1],"t":[1.5,0,7.5,0.9444444444444444,7.5,0,0,10.5,17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0,0],"m":[0,0,1,0,1,0,0,1,1,1,1,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,11,12,0,14,15,16,0,0,0,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,7,6,0,0,27,22,21,20,13,0,19,18,17,0,0,0,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.04484107,0,0,0,-0.04542495,0.01968913,0,0,0,0,0,-0.05351182,0,0,0,0.002015116,-0.02453143,0.07684184,-0.03141265,0.03571493,-0.02150517,0,0,-0.008228402,0.05958407,-0.04216529,0.07462045,-0.03936237]},{"f":[21,-1,16,27,-1,22,-1,-1,33,33,22,22,22,22,22,22,-1,-1,-1,-1,22,-1,-1,-1,18,-1,-1,-1,-1],"t":[1.5,0,1.0,0.5,0,16.975010474608837,0,0,35.98606694718995,29.297756302860655,17.074218326240924,16.98413870862295,16.844368517741394,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,16.903290731167747,0,0,0,8.729619742794132,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,10,11,12,13,14,15,16,0,0,0,0,21,0,0,0,25,0,0,0,0],"r":[2,0,8,5,0,7,0,0,28,27,24,23,20,19,18,17,0,0,0,0,22,0,0,0,26,0,0,0,0],"v":[0,-0.04461533,0,0,-0.0450234,0,0.05262886,-0.005716507,0,0,0,0,0,0,0,0,0.00119906,-0.02361084,0.07351184,-0.03079958,0,0.09085205,-0.001532166,-0.04142191,0,0.04371251,-0.05037161,0.02569157,-0.0331999]},{"f":[21,-1,1,21,-1,-1,23,21,22,15,-1,-1,22,21,-1,-1,22,-1,22,16,-1,-1,15,-1,15,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.5,10.5,7.95646805166752,3.584962500721156,0,0,12.199665042528586,5.5,0,0,12.921158001713904,0,14.116600814324018,6.832890014164741,0,0,5.044394119358453,0,6.108524456778169,0,0,0,0],"m":[0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,0,13,14,0,0,17,0,19,20,0,0,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,28,27,12,11,0,0,16,15,0,0,18,0,22,21,0,0,24,0,26,0,0,0,0],"v":[0,-0.04438347,0,0,-0.04515581,0.01865035,0,0,0,0,-0.02031184,0.05099774,0,0,-0.02694096,0.0002096226,0,0.02066217,0,0,-0.04717985,-0.009431105,0,0.02047234,0,-0.03165113,0.003890518,0.07214702,-0.03855637]},{"f":[21,-1,16,27,-1,22,-1,-1,33,33,22,22,22,22,22,22,-1,-1,-1,-1,22,-1,-1,-1,18,-1,-1,-1,-1],"t":[1.5,0,1.0,0.5,0,16.975010474608837,0,0,35.98606694718995,29.297756302860655,17.074218326240924,16.98413870862295,16.844368517741394,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,16.903290731167747,0,0,0,8.729619742794132,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,10,11,12,13,14,15,16,0,0,0,0,21,0,0,0,25,0,0,0,0],"r":[2,0,8,5,0,7,0,0,28,27,24,23,20,19,18,17,0,0,0,0,22,0,0,0,26,0,0,0,0],"v":[0,-0.04414543,0,0,-0.04437951,0,0.0502634,-0.005598849,0,0,0,0,0,0,0,0,0.001080079,-0.02296416,0.07040045,-0.03005782,0,0.08488696,-0.001347097,-0.04086561,0,0.04141813,-0.04961409,0.02494974,-0.03183053]},{"f":[21,-1,2,37,21,-1,-1,22,21,-1,-1,22,18,-1,1,21,-1,-1,-1,-1,22,15,1,-1,21,-1,-1,-1,-1],"t":[1.5,0,7.128362343532883,4.21,7.5,0,0,14.116600814324018,8.5,0,0,16.73920708776685,2.160964047443681,0,6.431742323097463,4.5,0,0,0,0,16.759160379664443,11.683433292835403,2.01685992705101,0,6.5,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,0,15,16,0,0,0,0,21,22,23,0,25,0,0,0,0],"r":[2,0,20,7,6,0,0,11,10,0,0,19,14,0,18,17,0,0,0,0,28,27,24,0,26,0,0,0,0],"v":[0,-0.0439012,0,0,0,-0.03199786,-0.007626234,0,0,-0.00609195,0.02598802,0,0,0.0238801,0,0,0.1116578,0.01334135,-0.00652092,-0.004581889,0,0,0,-0.002878923,0,0.0422152,0.0102295,-0.03061198,0.04311101]},{"f":[21,-1,23,21,15,16,22,21,-1,-1,-1,17,15,-1,-1,-1,22,18,3,-1,-1,22,-1,-1,-1,0,-1,-1,-1],"t":[1.5,0,7.5,8.5,10.457380879072534,9.92481250360578,14.116600814324018,3.5,0,0,0,10.415240582199587,6.169925001442312,0,0,0,16.759160379664443,10.680359255641626,0.23857586512866016,0,0,11.722778233171695,0,0,0,10.5,0,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,0,12,13,0,0,0,17,18,19,0,0,22,0,0,0,26,0,0,0],"r":[2,0,28,25,16,11,10,9,0,0,0,15,14,0,0,0,24,21,20,0,0,23,0,0,0,27,0,0,0],"v":[0,-0.04365075,0,0,0,0,0,0,-0.0411592,-0.004603287,0.004460966,0,0,0.007009178,0.06733153,0.001853978,0,0,0,0.008107928,-0.02943402,0,0.0460579,-0.03354471,0.0487388,0,0.004866312,0.05174681,-0.03787428]},{"f":[21,-1,16,27,-1,22,-1,-1,33,33,22,22,-1,-1,22,21,-1,-1,22,-1,22,16,-1,-1,15,-1,-1,-1,-1],"t":[1.5,0,1.0,0.5,0,16.975010474608837,0,0,35.98606694718995,29.297756302860655,7.95646805166752,6.870153907126159,0,0,12.199665042528586,6.5,0,0,12.921158001713904,0,14.116600814324018,6.832890014164741,0,0,5.044394119358453,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,10,11,12,0,0,15,16,0,0,19,0,21,22,0,0,25,0,0,0,0],"r":[2,0,8,5,0,7,0,0,28,27,14,13,0,0,18,17,0,0,20,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.04339407,0,0,-0.04378851,0,0.04819539,-0.005406306,0,0,0,0,-0.02389523,0.04210003,0,0,-0.03065252,0.0002217809,0,0.01886576,0,0,-0.04687731,-0.00558849,0,0.01937173,0.0006606143,0.02320266,-0.03121608]},{"f":[21,-1,1,21,-1,-1,23,21,22,15,-1,-1,22,21,3,-1,-1,-1,22,1,-1,-1,22,16,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.5,10.5,7.95646805166752,3.584962500721156,0,0,12.199665042528586,5.5,0.15991874474642756,0,0,0,12.921158001713904,7.772727272727273,0,0,14.116600814324018,6.832890014164741,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,0,13,14,15,0,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,28,27,12,11,0,0,18,17,16,0,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.04313119,0,0,-0.04459568,0.01791357,0,0,0,0,-0.01911264,0.04536785,0,0,0,0.0001918975,-0.03945259,0.0003018653,0,0,0.03004077,-0.002107936,0,0,-0.04646361,-0.008638938,0.003875568,0.06930752,-0.03704108]},{"f":[21,-1,4,16,-1,16,2,-1,-1,16,-1,21,-1,-1,22,15,-1,-1,22,-1,22,-1,22,-1,22,3,-1,-1,-1],"t":[1.5,0,3.753897320099348,2.0,0,5.044394119358453,1.792481250360578,0,0,6.108524456778169,0,8.5,0,0,16.759160379664443,11.683433292835403,0,0,16.807008825964132,0,16.844368517741394,0,16.975010474608837,0,17.078739781790915,0.9876439904368615,0,0,0],"m":[0,0,1,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,0,0,10,0,12,0,0,15,16,0,0,19,0,21,0,23,0,25,26,0,0,0],"r":[2,0,14,5,0,9,8,0,0,11,0,13,0,0,18,17,0,0,20,0,22,0,24,0,28,27,0,0,0],"v":[0,-0.04286212,0,0,-0.0348447,0,0,0.02531216,-0.02378263,0,-0.0451174,0,-0.003173148,0.0281381,0,0,0.003821338,-0.02271719,0,0.06354274,0,-0.03527543,0,0.06521234,0,0,0.01636762,-0.03980294,0.02923775]},{"f":[21,-1,1,21,-1,-1,16,27,-1,22,-1,-1,33,24,22,22,22,22,22,-1,-1,-1,-1,-1,18,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,0.0,0.5,0,16.975010474608837,0,0,35.98606694718995,9.5,17.078739781790915,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,0,0,8.729619742794132,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,0,1,1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,10,0,0,13,14,15,16,17,18,19,0,0,0,0,0,25,0,0,0,0],"r":[2,0,6,5,0,0,12,9,0,11,0,0,28,27,24,23,22,21,20,0,0,0,0,0,26,0,0,0,0],"v":[0,-0.04258691,0,0,-0.04431565,0.01697249,0,0,-0.05040285,0,0.04385041,-0.004967678,0,0,0,0,0,0,0,0.0002371222,0.06538128,-0.02883743,0.03477109,-0.04185014,0,0.03867437,-0.04186403,0.02395807,-0.02946598]},{"f":[21,-1,1,21,-1,-1,23,21,22,22,22,16,-1,22,22,22,-1,-1,-1,-1,-1,-1,18,32,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.5,10.5,17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,1,1,1,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,12,0,14,15,16,0,0,0,0,0,0,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,28,27,22,21,20,13,0,19,18,17,0,0,0,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.04230561,0,0,-0.04403862,0.01614924,0,0,0,0,0,0,-0.05099021,0,0,0,0.001699628,-0.02265345,0.06315547,-0.02799189,0.02845052,-0.02020206,0,0,-0.01347737,0.05533891,-0.03881414,0.06573984,-0.03602711]},{"f":[21,-1,4,16,-1,16,-1,16,-1,21,35,16,-1,-1,-1,-1,22,15,21,-1,-1,-1,22,-1,22,-1,22,-1,-1],"t":[1.5,0,3.753897320099348,2.0,0,5.044394119358453,0,6.108524456778169,0,8.5,2.5,6.832890014164741,0,0,0,0,16.759160379664443,11.683433292835403,3.5,0,0,0,16.807008825964132,0,16.844368517741394,0,16.975010474608837,0,0],"m":[0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,0,10,11,12,0,0,0,0,17,18,19,0,0,0,23,0,25,0,27,0,0],"r":[2,0,16,5,0,7,0,9,0,15,14,13,0,0,0,0,22,21,20,0,0,0,24,0,26,0,28,0,0],"v":[0,-0.04201831,0,0,-0.03356758,0,0.01781394,0,-0.04488894,0,0,0,0.0133181,-0.005131029,-0.04254985,0.02616599,0,0,0,-0.02346069,0.005221255,-0.02194071,0,0.05834848,0,-0.03289196,0,0.0591334,0.006902691]},{"f":[21,-1,2,16,-1,17,2,-1,-1,32,1,17,-1,-1,17,17,22,22,-1,2,-1,-1,-1,-1,-1,-1,22,-1,-1],"t":[1.5,0,5.022197059679227,2.0,0,2.160964047443681,1.792481250360578,0,0,8.230739723643078,4.208333333333334,10.620215166896223,0,0,8.729619742794132,7.467398296089201,15.676331898425197,12.613886328427084,0,2.453445297804259,0,0,0,0,0,0,17.094543568094064,0,0],"m":[0,0,1,0,0,0,1,0,0,1,1,1,0,0,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,0,10,11,12,0,0,15,16,17,18,0,20,0,0,0,0,0,0,27,0,0],"r":[2,0,26,5,0,9,8,0,0,25,14,13,0,0,24,23,22,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,-0.04172508,0,0,-0.04152036,0,0,0.01495722,-0.01907947,0,0,0,-0.03524169,0.01970894,0,0,0,0,-0.0286238,0,-0.007675352,0.02086252,-0.0270904,0.05987334,-0.03318593,-0.0273981,0,0.001951025,0.04283957]},{"f":[21,-1,22,22,-1,-1,0,2,-1,-1,15,22,-1,23,-1,-1,22,18,-1,-1,22,1,21,-1,17,-1,-1,-1,-1],"t":[1.5,0,7.95646805166752,6.870153907126159,0,0,2.5,6.209321624687949,0,0,3.321928094887362,12.278990886161445,0,1.5,0,0,14.116600814324018,10.680359255641626,0,0,14.247293309538719,6.431742323097463,3.5,0,7.67595272073847,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,0,11,12,0,14,0,0,17,18,0,0,21,22,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,10,9,0,0,16,13,0,15,0,0,20,19,0,0,28,27,24,0,26,0,0,0,0],"v":[0,-0.04142605,0,0,-0.02680442,0.03720654,0,0,-0.03620511,-0.005810674,0,0,0.002106607,0,0.05187278,0.01047778,0,0,-0.005810674,0.03292529,0,0,0,0.1443347,0,0.003374921,0.05566415,0.001708498,-9.389661e-05]},{"f":[21,-1,1,21,-1,-1,16,27,-1,22,-1,22,-1,-1,33,33,22,22,-1,-1,18,36,18,-1,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,0.0,0.5,0,16.975010474608837,0,17.094543568094064,0,0,35.98606694718995,29.297756302860655,17.078739781790915,16.98413870862295,0,0,8.729619742794132,4.5,6.076776015854056,0,0,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,1,0,0,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,10,0,12,0,0,15,16,17,18,0,0,21,22,23,0,0,0,0,0,0],"r":[2,0,6,5,0,0,14,9,0,11,0,13,0,0,28,27,20,19,0,0,26,25,24,0,0,0,0,0,0],"v":[0,-0.04112132,0,0,-0.04362534,0.01553306,0,0,-0.04957439,0,0.03938465,0,-0.01187725,0.03417886,0,0,0,0,0.0009343161,-0.03924989,0,0,0,0.005950492,0.06892507,-0.01285721,-0.04734982,0.02094184,-0.02831571]},{"f":[21,-1,2,37,21,-1,-1,16,-1,22,22,-1,2,-1,-1,-1,22,24,-1,1,21,2,-1,-1,-1,21,-1,-1,-1],"t":[1.5,0,7.128362343532883,4.21,6.5,0,0,6.108524456778169,0,16.73920708776685,12.199665042528586,0,1.292481250360578,0,0,0,16.759160379664443,3.5,0,2.997611464968153,7.5,9.221514256038809,0,0,0,6.5,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,0,13,0,0,0,17,18,0,20,21,22,0,0,0,26,0,0,0],"r":[2,0,16,7,6,0,0,9,0,15,12,0,14,0,0,0,28,19,0,25,24,23,0,0,0,27,0,0,0],"v":[0,-0.04081105,0,0,0,-0.03721462,-0.006585851,0,-0.006059933,0,0,-0.003568792,0,-0.005748575,0.01747117,-0.006585851,0,0,-0.03273532,0,0,0,-0.0009418384,-0.03870391,0.01235918,0,0.0485052,0.006571797,0.03660347]},{"f":[21,-1,22,22,-1,15,-1,-1,22,21,8,-1,-1,-1,22,18,-1,-1,22,16,-1,-1,15,-1,15,-1,15,-1,-1],"t":[1.5,0,7.95646805166752,6.870153907126159,0,2.584962500721156,0,0,12.199665042528586,5.5,0.5,0,0,0,12.921158001713904,5.022197059679227,0,0,14.116600814324018,6.832890014164741,0,0,5.044394119358453,0,6.169925001442312,0,7.266786540694901,0,0],"m":[0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,10,11,0,0,0,15,16,0,0,19,20,0,0,23,0,25,0,27,0,0],"r":[2,0,8,5,0,7,0,0,14,13,12,0,0,0,18,17,0,0,22,21,0,0,24,0,26,0,28,0,0],"v":[0,-0.04049537,0,0,-0.02597528,0,-0.004696531,0.04611787,0,0,0,-0.03694939,0.001057964,0.0008961766,0,0,0.006140275,0.03742557,0,0,-0.0460469,-0.005878225,0,0.01572396,0,-0.02693714,0,0.01747372,-0.002085951]},{"f":[21,-1,1,21,-1,2,16,-1,16,22,2,-1,-1,-1,21,25,-1,-1,15,22,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,3.084962500721156,2.0,0,5.044394119358453,15.782972563647261,1.792481250360578,0,0,0,7.5,1.5,0,0,7.930737337562887,17.07907323966382,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,0,9,10,11,0,0,0,15,16,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,8,0,14,13,12,0,0,0,18,17,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04017447,0,0,-0.03922691,0,0,-0.04462307,0,0,0,0.0334882,-0.01707655,-0.009494072,0,0,-0.004410805,-0.02888808,0,0,-0.04237766,0.01583686,0.06561047,0.001422746,0,0,0.006760758,-0.03675581,0.0511419]},{"f":[21,-1,22,22,-1,-1,0,2,22,-1,-1,-1,18,22,-1,18,16,-1,22,-1,15,15,4,-1,-1,-1,-1,-1,-1],"t":[1.5,0,7.95646805166752,6.870153907126159,0,0,2.5,6.209321624687949,16.25454052700782,0,0,0,10.680359255641626,12.199665042528586,0,10.326824911906375,6.108524456778169,0,13.054613167076562,0,6.857980995127572,6.820178962415188,2.453445297804259,0,0,0,0,0,0],"m":[0,0,0,1,0,0,0,1,1,0,0,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,0,0,0,13,14,0,16,17,0,19,0,21,22,23,0,0,0,0,0,0],"r":[2,0,6,5,0,0,12,11,10,0,0,0,28,15,0,27,18,0,20,0,26,25,24,0,0,0,0,0,0],"v":[0,-0.03984853,0,0,-0.02514894,0.03237616,0,0,0,-0.04287186,-0.005324636,-0.005324636,0,0,-0.005324636,0,0,-0.005324636,0,0.03212414,0,0,0,1.81118e-05,0.02190174,0.1113206,-0.001228513,0.03197323,0.01902973]},{"f":[21,-1,1,21,-1,2,16,-1,16,-1,15,-1,21,16,33,-1,-1,-1,15,22,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,3.084962500721156,2.0,0,5.044394119358453,0,6.169925001442312,0,7.5,6.832890014164741,21.257266042509173,0,0,0,7.930737337562887,17.07907323966382,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,0,0,0,1,0,1,0,0,0,1,1,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,11,0,13,14,15,0,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,8,0,10,0,12,0,18,17,16,0,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.03951774,0,0,-0.03881549,0,0,-0.04428515,0,0.01423833,0,-0.04066374,0,0,0,0.02291585,-0.01859955,-0.02507965,0,0,-0.03841224,0.0151285,0.05897641,0.00127933,0,0,0.006624721,-0.0363166,0.04985753]},{"f":[21,-1,23,21,15,16,16,-1,-1,17,-1,-1,22,22,16,-1,-1,1,-1,22,3,-1,-1,-1,-1,0,-1,-1,-1],"t":[1.5,0,7.5,8.5,10.457380879072534,7.930737337562887,7.679480099505446,0,0,1.792481250360578,0,0,16.759160379664443,11.722778233171695,10.501837184902296,0,0,4.87351462522852,0,15.782972563647261,0.9367573126142597,0,0,0,0,10.5,0,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,14,15,0,0,18,0,20,21,0,0,0,0,26,0,0,0],"r":[2,0,28,25,12,9,8,0,0,11,0,0,24,17,16,0,0,19,0,23,22,0,0,0,0,27,0,0,0],"v":[0,-0.03918232,0,0,0,0,0,-0.001366369,-0.03538726,0,0.02999083,0.002842506,0,0,0,-0.03953442,0.0304412,0,-0.03393047,0,0,0.001194571,-0.04739987,0.03898419,0.04005036,0,0.002693387,0.04864266,-0.03419142]},{"f":[21,-1,1,21,-1,-1,22,15,-1,-1,22,21,3,-1,-1,-1,22,18,-1,-1,22,16,16,-1,3,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.95646805166752,3.584962500721156,0,0,12.199665042528586,5.5,0.15991874474642756,0,0,0,12.921158001713904,5.022197059679227,0,0,14.116600814324018,11.98370619265935,6.832890014164741,0,0.9300813008130082,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,0,11,12,13,0,0,0,17,18,0,0,21,22,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,10,9,0,0,16,15,14,0,0,0,20,19,0,0,28,27,24,0,26,0,0,0,0],"v":[0,-0.03884249,0,0,-0.04292969,0.01353767,0,0,-0.01714962,0.03484716,0,0,0,-0.0003613278,-0.03737172,3.735105e-05,0,0,0.005750901,0.03482473,0,0,0,-0.04546503,0,0.009922641,-0.02616948,0.06059164,0.002679587]},{"f":[21,-1,23,22,22,22,16,-1,22,22,22,-1,2,-1,-1,-1,-1,22,-1,32,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,7.5,17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,0,6.515505953718852,0,0,0,0,16.903290731167747,0,2.69615871138938,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,11,0,13,0,0,0,0,18,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,17,8,0,16,15,12,0,14,0,0,0,0,19,0,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.03849849,0,0,0,0,0,-0.049562,0,0,0,0.001178517,0,-0.04966474,0.005874608,0.05880673,-0.02701998,0,0.07267422,0,-0.03866225,0.02365846,-0.01854536,0,0,-0.01484886,0.05234519,-0.03759737,-0.03370726]},{"f":[21,-1,1,21,-1,2,16,-1,16,22,15,-1,-1,-1,1,15,-1,16,-1,15,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,3.084962500721156,2.0,0,5.044394119358453,15.782972563647261,3.169925001442312,0,0,0,8.046875,6.169925001442312,0,6.832890014164741,0,7.930737337562887,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,0,9,10,11,0,0,0,15,16,0,18,0,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,8,0,14,13,12,0,0,0,22,17,0,19,0,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.03815056,0,0,-0.03827172,0,0,-0.04384255,0,0,0,0.0005435638,0.03826474,-0.009580161,0,0,-0.039842,0,0.01300223,0,-0.04751183,0.002614804,-0.03753913,0.001173686,0,0,0.005904612,-0.0351082,0.04823457]},{"f":[21,-1,1,21,-1,-1,23,21,22,15,-1,-1,22,21,-1,23,-1,-1,15,21,-1,-1,15,17,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.5,10.5,7.95646805166752,3.584962500721156,0,0,14.116600814324018,3.5,0,3.5,0,0,5.044394119358453,4.5,0,0,6.108524456778169,7.67595272073847,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,0,13,14,0,16,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,28,27,12,11,0,0,18,15,0,17,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.03779896,0,0,-0.04262449,0.01279318,0,0,0,0,-0.01602039,0.03201368,0,0,-0.03762463,0,0.001230399,-0.02089674,0,0,0.05675212,0.008726838,0,0,-0.0435526,0.0276931,0.002808998,0.05620512,-0.03288507]},{"f":[21,-1,4,16,-1,16,-1,16,-1,21,-1,-1,22,15,-1,-1,22,-1,22,-1,22,-1,22,3,-1,-1,18,-1,-1],"t":[1.5,0,3.753897320099348,2.0,0,5.044394119358453,0,6.108524456778169,0,9.5,0,0,16.759160379664443,11.683433292835403,0,0,16.807008825964132,0,16.844368517741394,0,16.975010474608837,0,17.078739781790915,0.9876439904368615,0,0,6.076776015854056,0,0],"m":[0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,0,10,0,0,13,14,0,0,17,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,12,5,0,7,0,9,0,11,0,0,16,15,0,0,18,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.03744396,0,0,-0.03045687,0,0.01377087,0,-0.04319898,0,-0.002704366,0.02789751,0,0,0.002808301,-0.02100878,0,0.04992055,0,-0.03180911,0,0.05401418,0,0,0.01384821,-0.03774676,0,-0.007790685,0.0528454]},{"f":[21,-1,1,21,-1,-1,16,27,-1,22,-1,22,-1,-1,33,33,22,22,22,-1,-1,-1,18,36,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,0.0,0.5,0,16.975010474608837,0,17.094543568094064,0,0,35.98606694718995,29.297756302860655,17.078739781790915,16.98413870862295,16.844368517741394,0,0,0,8.729619742794132,4.5,0,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,1,0,0,1,1,1,1,1,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,10,0,12,0,0,15,16,17,18,19,0,0,0,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,14,9,0,11,0,13,0,0,28,27,22,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.03708583,0,0,-0.04231704,0.0121534,0,0,-0.04856811,0,0.035951,0,-0.01112444,0.03447137,0,0,0,0,0,2.717095e-05,0.03182792,-0.03782661,0,0,0.04872084,-0.01389883,-0.04687394,0.01867221,-0.02695284]},{"f":[21,-1,1,21,-1,2,16,-1,17,21,-1,1,-1,22,22,17,-1,37,-1,-1,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,3.084962500721156,2.0,0,2.160964047443681,3.5,0,1.000936329588015,0,15.782972563647261,12.679695715883515,1.792481250360578,0,6.335,0,0,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,0,9,10,0,12,0,14,15,16,0,18,0,0,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,8,0,22,11,0,13,0,21,20,17,0,19,0,0,0,0,0,0,28,27,0,0,0],"v":[0,-0.03672484,0,0,-0.03760351,0,0,-0.04313957,0,0,-0.03837817,0,0.06688621,0,0,0,-0.03342332,0,-0.005676927,0.05641691,0.03497029,-0.01645368,-0.01225245,0.001093049,0,0,0.005374607,-0.03449066,0.04695946]},{"f":[21,-1,22,22,22,16,-1,22,22,22,15,-1,-1,2,-1,-1,-1,-1,2,15,-1,-1,-1,-1,18,25,-1,-1,-1],"t":[1.5,0,17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,13.56045225766691,0,0,6.515505953718852,0,0,0,0,4.128693921346326,6.0,0,0,0,0,8.729619742794132,1.5,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,1,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,11,0,0,14,0,0,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,18,7,0,17,16,13,12,0,0,15,0,0,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.03636128,0,0,0,0,-0.04837939,0,0,0,0,0.001609291,-0.03994322,0,-0.04851931,0.005053669,0.0562097,-0.02591685,0,0,0.03638483,-0.03839813,0.0334195,-0.01920342,0,0,0.05081489,-0.03454055,-0.03633062]},{"f":[21,-1,1,21,-1,-1,16,16,23,-1,21,15,-1,-1,18,22,2,-1,-1,17,-1,-1,-1,-1,21,-1,33,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.930737337562887,7.679480099505446,2.5,0,6.5,9.92481250360578,0,0,8.729619742794132,17.094543568094064,2.453445297804259,0,0,7.65814076587311,0,0,0,0,4.5,0,35.98606694718995,0,0],"m":[0,0,0,1,0,0,1,1,1,0,0,1,0,0,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,0,7,8,9,0,11,12,0,0,15,16,17,0,0,20,0,0,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,24,23,10,0,14,13,0,0,22,19,18,0,0,21,0,0,0,0,26,0,28,0,0],"v":[0,-0.03599544,0,0,-0.04200938,0.01146663,0,0,0,0.003823118,0,0,-0.04964319,0.005125839,0,0,0,-0.03314527,-0.0005861092,0,-0.02612062,0.05682441,-0.0383852,-0.03368746,0,-0.02718161,0,0.005421687,-0.03514925]},{"f":[1,21,17,-1,-1,22,22,-1,-1,-1,2,21,15,-1,22,36,-1,22,18,-1,-1,-1,-1,-1,4,-1,22,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,0,0,7.640240405159187,10.5,10.457380879072534,0,16.759160379664443,4.5,0,15.782972563647261,1.792481250360578,0,0,0,0,0,7.950787011501063,0,12.921158001713904,0,0],"m":[0,1,1,0,0,1,0,0,0,0,1,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,0,11,12,13,0,15,16,0,18,19,0,0,0,0,0,25,0,27,0,0],"r":[10,5,4,0,0,9,8,0,0,0,24,23,14,0,22,17,0,21,20,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,-0.04528932,0.01756331,0,0,0.03143234,-0.006623206,0.04526575,0,0,0,0.0008692184,0,0,-0.03860163,0,0,0.0002389873,-0.03456944,0.03461777,0.04594592,0.05394909,0,0.03810514,0,0.01485232,-0.01298895]},{"f":[21,-1,1,21,-1,16,23,16,-1,-1,21,-1,2,17,-1,-1,-1,1,22,21,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,7.930737337562887,1.5,6.832890014164741,0,0,6.5,0,5.065928480304397,8.496347487943302,0,0,0,8.046875,17.078739781790915,4.5,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,1,1,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,0,13,14,0,0,0,18,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,17,10,9,0,0,12,0,16,15,0,0,0,23,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.03528415,0,0,-0.03709063,0,0,0,0.006460179,-0.01776892,0,-0.03630059,0,0,-0.04648903,0.00888371,-0.001897841,0,0,0,-0.03582028,0.003567248,0.05102468,-0.04818414,0,0,0.005150059,-0.03339474,0.04575383]},{"f":[21,-1,22,22,22,22,22,10,-1,22,15,-1,-1,2,-1,-1,-1,-1,22,-1,32,-1,-1,-1,18,25,-1,-1,-1],"t":[1.5,0,17.094543568094064,16.98920034885663,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,16.73920708776685,13.56045225766691,0,0,6.515505953718852,0,0,0,0,16.903290731167747,0,2.69615871138938,0,0,0,8.729619742794132,1.5,0,0,0],"m":[0,0,1,1,1,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,7,8,0,10,11,0,0,14,0,0,0,0,19,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,23,18,17,16,9,0,13,12,0,0,15,0,0,0,0,20,0,22,0,0,0,28,27,0,0,0],"v":[0,-0.03491332,0,0,0,0,0,0,-0.04757049,0,0,0.001557939,-0.0388918,0,-0.04737802,0.004484664,0.05331158,-0.02499392,0,0.06610248,0,-0.03955096,0.01904964,-0.01843418,0,0,0.04902665,-0.03298333,-0.03517368]},{"f":[1,21,17,-1,-1,22,22,-1,2,22,-1,-1,22,21,-1,-1,-1,-1,2,21,-1,36,-1,-1,4,-1,22,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,6.247802615595391,15.782972563647261,0,0,16.73920708776685,7.5,0,0,0,0,7.640240405159187,9.5,0,3.5,0,0,7.950787011501063,0,12.921158001713904,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,9,10,0,0,13,14,0,0,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[18,5,4,0,0,17,8,0,12,11,0,0,16,15,0,0,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,0,0,-0.04494275,0.01654138,0,0,0.02905408,0,0,-0.03748118,0.03376526,0,0,-0.01394865,0.01366294,-0.02382422,0.04282743,0,0,-0.0008830709,0,0.0002338056,0.04609576,0,0.03603105,0,0.01396907,-0.01254279]},{"f":[21,-1,22,-1,22,-1,22,21,18,15,-1,-1,-1,-1,22,18,-1,-1,22,16,-1,16,3,-1,-1,-1,15,-1,-1],"t":[1.5,0,6.870153907126159,0,7.95646805166752,0,12.199665042528586,5.5,1.292481250360578,10.462502272597265,0,0,0,0,12.921158001713904,5.022197059679227,0,0,14.116600814324018,6.832890014164741,0,11.98370619265935,0.9300813008130082,0,0,0,5.044394119358453,0,0],"m":[0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,0,7,8,9,10,0,0,0,0,15,16,0,0,19,20,0,22,23,0,0,0,27,0,0],"r":[2,0,4,0,6,0,14,13,12,11,0,0,0,0,18,17,0,0,26,21,0,25,24,0,0,0,28,0,0],"v":[0,-0.0341861,0,-0.02351237,0,0.02462828,0,0,0,0,-0.01773833,0.03118704,-0.03899074,-0.0002496866,0,0,0.003995168,0.03164629,0,0,-0.04490937,0,0,0.008538227,-0.02407242,0.04324245,0,0.01298894,-0.0001194084]},{"f":[1,21,17,-1,-1,22,22,-1,2,22,-1,-1,22,21,22,-1,-1,-1,-1,-1,2,21,-1,-1,4,-1,22,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,6.247802615595391,15.782972563647261,0,0,16.73920708776685,7.5,15.212004812197344,0,0,0,0,0,7.640240405159187,10.5,0,0,7.950787011501063,0,12.921158001713904,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,9,10,0,0,13,14,15,0,0,0,0,0,21,22,0,0,25,0,27,0,0],"r":[20,5,4,0,0,19,8,0,12,11,0,0,18,17,16,0,0,0,0,0,24,23,0,0,26,0,28,0,0],"v":[0,0,0,-0.04460296,0.01437523,0,0,0.02658723,0,0,-0.03699775,0.03288056,0,0,0,-0.0289936,-0.001027945,0.01262732,-0.02310523,0.04112329,0,0,-0.0003411152,0.04991078,0,0.03411911,0,0.01281603,-0.01197847]},{"f":[16,16,22,21,-1,23,-1,-1,16,16,21,-1,-1,-1,4,17,22,22,-1,-1,-1,-1,-1,-1,21,-1,33,-1,-1],"t":[7.930737337562887,7.679480099505446,14.116600814324018,3.5,0,2.5,0,0,6.108524456778169,5.700439718141092,3.5,0,0,0,3.084962500721156,8.729619742794132,16.765402642760364,16.725912049439096,0,0,0,0,0,0,4.5,0,35.98606694718995,0,0],"m":[1,1,1,1,0,1,0,0,1,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,0,0,15,16,17,18,0,0,0,0,0,0,25,0,27,0,0],"r":[24,23,8,5,0,7,0,0,14,13,12,0,0,0,22,21,20,19,0,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0,-0.03741457,0,0.003553897,-0.02065787,0,0,0,0.05755425,-0.002051408,-0.03739278,0,0,0,0,0.003816632,-0.0369634,0.04237707,-0.04461457,0.03660426,-0.03312169,0,-0.02630323,0,0.004572917,-0.03472151]},{"f":[1,21,-1,25,22,22,-1,-1,17,-1,-1,1,37,-1,-1,21,-1,-1,2,21,15,-1,15,-1,15,-1,-1,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,16.98920034885663,0,0,8.729619742794132,0,0,6.431742323097463,5.539999999999999,0,0,4.5,0,0,6.118903736861568,9.5,3.0,0,8.076815597050832,0,9.92481250360578,0,0,0,0],"m":[1,0,0,1,1,1,0,0,1,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,0,0,12,13,0,0,16,0,0,19,20,21,0,23,0,25,0,0,0,0],"r":[18,3,0,11,8,7,0,0,10,0,0,15,14,0,0,17,0,0,28,27,22,0,24,0,26,0,0,0,0],"v":[0,0,-0.040988,0,0,0,0.000550166,-0.01916023,0,0.04655631,-0.01094139,0,0,-0.003065214,0.0512748,0,-0.007546844,-0.04293696,0,0,0,0.01074388,0,-0.03818886,0,0.009734258,-0.03237232,0.02280608,0.04362413]},{"f":[1,21,17,-1,-1,22,22,-1,2,22,-1,-1,22,21,22,-1,-1,-1,-1,-1,2,21,-1,15,-1,-1,21,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,6.247802615595391,15.782972563647261,0,0,16.73920708776685,7.5,15.212004812197344,0,0,0,0,0,9.025136904799169,8.5,0,7.774787059601174,0,0,6.5,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,0,0,13,14,15,0,0,0,0,0,21,22,0,24,0,0,27,0,0],"r":[20,5,4,0,0,19,8,0,12,11,0,0,18,17,16,0,0,0,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0,-0.0442258,0.01364621,0,0,0.02466354,0,0,-0.03642663,0.031885,0,0,0,-0.02839138,-0.001217117,0.01187416,-0.0218931,0.03938221,0,0,-0.0006539971,0,0.005666048,0.06171031,0,0.03685784,0.003633558]},{"f":[23,22,16,33,-1,-1,33,24,22,22,22,22,22,-1,-1,-1,22,-1,-1,-1,18,32,-1,-1,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,41.975696368331896,0,0,35.98606694718995,9.5,17.078739781790915,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,16.903290731167747,0,0,0,8.729619742794132,6.6765734127490415,0,0,0,0,0,0,0],"m":[1,1,0,1,0,0,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,10,11,12,13,0,0,0,17,0,0,0,21,22,0,0,0,0,0,0,0],"r":[28,27,6,5,0,0,26,25,20,19,16,15,14,0,0,0,18,0,0,0,24,23,0,0,0,0,0,0,0],"v":[0,0,0,0,-0.03043149,0.01091065,0,0,0,0,0,0,0,-0.0008118916,0.0500112,-0.02424359,0,0.06320059,0.003731958,-0.0377676,0,0,0.04668406,-0.007788845,-0.03854966,0.01903378,-0.02537439,0.03074929,-0.03088151]},{"f":[16,16,22,21,-1,23,-1,-1,16,16,21,-1,-1,-1,4,17,22,22,-1,-1,-1,-1,-1,-1,21,-1,33,-1,-1],"t":[7.930737337562887,7.679480099505446,14.116600814324018,3.5,0,2.5,0,0,6.108524456778169,5.700439718141092,3.5,0,0,0,3.084962500721156,8.729619742794132,16.765402642760364,16.725912049439096,0,0,0,0,0,0,4.5,0,35.98606694718995,0,0],"m":[1,1,1,1,0,1,0,0,1,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,0,0,15,16,17,18,0,0,0,0,0,0,25,0,27,0,0],"r":[24,23,8,5,0,7,0,0,14,13,12,0,0,0,22,21,20,19,0,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0,-0.03657603,0,0.003325339,-0.0201145,0,0,0,0.05470296,-0.001652724,-0.03654839,0,0,0,0,0.00360352,-0.03577841,0.03907122,-0.04327172,0.03496011,-0.03241937,0,-0.02546191,0,0.004310161,-0.03382166]},{"f":[23,22,1,21,-1,33,11,22,22,-1,-1,16,-1,-1,-1,1,4,-1,22,35,-1,-1,-1,-1,2,-1,-1,-1,-1],"t":[7.5,17.105450904090162,9.107199999999999,2.5,0,21.80651201897376,0.5,16.759160379664443,16.73920708776685,0,0,8.321928094887362,0,0,0,8.980632551528075,3.6427011094311244,0,16.838461650489297,2.5,0,0,0,0,6.118903736861568,0,0,0,0],"m":[1,1,1,0,0,1,1,1,1,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,12,0,0,0,16,17,0,19,20,0,0,0,0,25,0,0,0,0],"r":[28,27,24,5,0,15,14,11,10,0,0,13,0,0,0,23,18,0,22,21,0,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.04023759,0,0,0,0,0.001358292,-0.03618354,0,-0.009776591,0.0355936,-0.03692126,0,0,-0.01468595,0,0,-0.03988866,0.00531139,0.02147941,-0.04617929,0,0.002510107,0.04279764,0.02976307,-0.0303814]},{"f":[22,-1,22,-1,22,21,18,15,-1,-1,-1,-1,1,22,-1,22,-1,-1,22,15,-1,-1,22,16,-1,16,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,5.5,1.292481250360578,10.462502272597265,0,0,0,0,1.282051282051282,15.717370311433935,0,16.786010507999382,0,0,12.921158001713904,7.930737337562887,0,0,14.116600814324018,6.832890014164741,0,11.98370619265935,0,0,0],"m":[0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,7,8,0,0,0,0,13,14,0,16,0,0,19,20,0,0,23,24,0,26,0,0,0],"r":[2,0,4,0,12,11,10,9,0,0,0,0,18,15,0,17,0,0,22,21,0,0,28,25,0,27,0,0,0],"v":[0,-0.02276697,0,0.02221951,0,0,0,0,-0.016648,0.02865262,-0.03818693,-0.0004490657,0,0,-0.04458753,0,0.08471516,-0.01926575,0,0,0.005400982,0.03671182,0,0,-0.04329496,0,-0.006526605,0.04715056,0.002032241]},{"f":[23,1,21,2,22,-1,-1,-1,18,22,15,-1,22,-1,-1,16,22,-1,15,-1,-1,-1,0,-1,-1,18,-1,-1,-1],"t":[7.5,4.92309205350118,5.5,8.612669628062957,16.978341770814396,0,0,0,10.460454442916898,16.759160379664443,11.054604317960672,0,11.722778233171695,0,0,8.96289600533726,17.078739781790915,0,7.774787059601174,0,0,0,2.5,0,0,10.415240582199587,0,0,0],"m":[1,1,0,1,1,0,0,0,1,1,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,11,0,13,0,0,16,17,0,19,0,0,0,23,0,0,26,0,0,0],"r":[28,25,8,7,6,0,0,0,22,15,12,0,14,0,0,21,18,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.03767375,0.01594472,0.007147679,0,0,0,-0.001518122,0,0.01289474,-0.03476661,0,0,-0.01853461,0,-0.004875199,0.05593625,0.04626422,0,-0.01660668,0.01803495,0,0.002552575,-0.03136239,-0.02989691]},{"f":[23,22,16,33,-1,-1,37,33,1,21,2,-1,-1,-1,18,33,-1,22,-1,-1,36,-1,-1,-1,21,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,41.975696368331896,0,0,11.73,35.98606694718995,4.92309205350118,5.5,8.612669628062957,0,0,0,8.729619742794132,29.297756302860655,0,11.775312925643524,0,0,3.5,0,0,0,5.5,0,0,0,0],"m":[1,1,0,1,0,0,1,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,10,11,0,0,0,15,16,0,18,0,0,21,0,0,0,25,0,0,0,0],"r":[28,27,6,5,0,0,24,23,14,13,12,0,0,0,20,17,0,19,0,0,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.02914945,0.009270846,0,0,0,0,0,-0.03159743,0.007054867,-0.0003126234,0,0,0.0002963171,0,-0.02988421,0.01102159,0,0.00659777,-0.04445348,-0.04488339,0,0.01102159,0.04055271,0.02851139,-0.02938429]},{"f":[4,16,-1,18,2,-1,21,-1,22,22,-1,-1,-1,33,-1,18,-1,17,22,22,25,-1,-1,-1,-1,-1,22,-1,-1],"t":[3.245926548164837,2.0,0,2.160964047443681,1.792481250360578,0,6.5,0,16.765402642760364,16.73920708776685,0,0,0,20.752541988539424,0,6.531360383582522,0,8.729619742794132,17.074218326240924,15.676331898425197,1.5,0,0,0,0,0,17.094543568094064,0,0],"m":[1,0,0,0,1,0,1,0,1,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,5,0,7,0,9,10,0,0,0,14,0,16,0,18,19,20,21,0,0,0,0,0,27,0,0],"r":[26,3,0,13,6,0,8,0,12,11,0,0,0,15,0,17,0,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,-0.04367882,0,0,0.01110924,0,-0.02880601,0,0,0.007411336,-0.03054416,0.04028828,0,-0.04550984,0,-0.02372651,0,0,0,0,0.01622035,-0.01738184,-0.0253585,0.0460922,-0.01962101,0,0.001104154,0.04361174]},{"f":[22,15,-1,-1,22,21,4,-1,18,-1,-1,-1,1,22,-1,22,-1,-1,15,18,-1,-1,0,-1,16,-1,22,-1,-1],"t":[7.95646805166752,3.584962500721156,0,0,12.199665042528586,5.5,3.753897320099348,0,4.128693921346326,0,0,0,1.282051282051282,15.717370311433935,0,16.786010507999382,0,0,2.0,6.515678298127854,0,0,3.5,0,6.108524456778169,0,15.676331898425197,0,0],"m":[0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,0,0,5,6,7,0,9,0,0,0,13,14,0,16,0,0,19,20,0,0,23,0,25,0,27,0,0],"r":[4,3,0,0,12,11,8,0,10,0,0,0,18,15,0,17,0,0,22,21,0,0,24,0,26,0,28,0,0],"v":[0,0,-0.01527696,0.02382192,0,0,0,-0.04556211,0,0.007736276,-0.03174542,-0.0004746289,0,0,-0.04404701,0,0.081333,-0.0189889,0,0,-0.002649186,0.03303319,0,-0.02078143,0,-0.005306992,0,0.01282279,-0.002211129]},{"f":[22,22,22,16,-1,22,22,22,15,-1,-1,2,-1,-1,-1,-1,22,-1,32,-1,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,13.56045225766691,0,0,6.515505953718852,0,0,0,0,16.903290731167747,0,2.69615871138938,0,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,12,0,0,0,0,17,0,19,0,0,0,23,0,25,26,0,0,0],"r":[22,21,16,5,0,15,14,11,10,0,0,13,0,0,0,0,18,0,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,-0.04476454,0,0,0,0,0.001080863,-0.03809541,0,-0.0441257,0.006368085,0.04626344,-0.02279396,0,0.06074829,0,-0.0377506,0.01751778,-0.01600387,0,0.05086283,0,0,0.04200197,-0.0343183,-0.04092094]},{"f":[16,16,23,3,-1,-1,21,22,-1,-1,0,25,22,-1,2,17,-1,-1,-1,-1,-1,-1,21,22,-1,-1,33,-1,-1],"t":[7.930737337562887,7.679480099505446,1.5,0.46415770609318996,0,0,6.5,16.98413870862295,0,0,10.5,1.5,8.882387211598338,0,5.065928480304397,7.723832212141186,0,0,0,0,0,0,4.5,16.964783007086943,0,0,35.98606694718995,0,0],"m":[1,1,1,0,0,0,0,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,12,13,0,15,16,0,0,0,0,0,0,23,24,0,0,27,0,0],"r":[22,21,6,5,0,0,10,9,0,0,20,19,14,0,18,17,0,0,0,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,-0.03177998,0.003675534,0,0,-0.03967718,0.008399244,0,0,0,-0.03784429,0,0,-0.03253222,0.01074341,0.003486562,-0.04012532,0.03481599,-0.0311672,0,0,-0.03489667,0.009353348,0,0.003875825,-0.0320983]},{"f":[37,4,16,-1,18,-1,23,22,22,21,-1,-1,-1,-1,22,21,-1,-1,17,-1,-1,18,-1,32,-1,-1,21,-1,-1],"t":[11.73,3.245926548164837,2.0,0,2.160964047443681,0,1.5,15.676331898425197,14.03140051979143,5.5,0,0,0,0,17.07907323966382,8.5,0,0,8.729619742794132,0,0,5.303665156874805,0,6.208926257442949,0,0,6.5,0,0],"m":[1,1,0,0,0,0,0,1,1,1,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,0,0,0,15,16,0,0,19,0,0,22,0,24,0,0,27,0,0],"r":[26,21,4,0,6,0,14,13,12,11,0,0,0,0,18,17,0,0,20,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,-0.04211611,0,0.002510851,0,0,0,0,-0.04140454,-0.001043257,0.007682738,-0.02352092,0,0,-0.04447206,0.005765152,0,0.007682738,-0.0411484,0,-0.001431787,0,-0.02643618,0.007682738,0,0.007682738,0.04430404]},{"f":[22,22,22,16,-1,22,22,22,22,-1,-1,2,-1,-1,-1,-1,22,-1,32,-1,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,16.725912049439096,0,0,6.515505953718852,0,0,0,0,16.903290731167747,0,2.69615871138938,0,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,12,0,0,0,0,17,0,19,0,0,0,23,0,25,26,0,0,0],"r":[22,21,16,5,0,15,14,11,10,0,0,13,0,0,0,0,18,0,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,-0.04380095,0,0,0,0,-0.0001549932,0.02517291,0,-0.04320023,0.0058056,0.04487524,-0.02193033,0,0.05881776,0,-0.03687298,0.01685398,-0.01530467,0,0.05004955,0,0,0.04061515,-0.03358007,-0.03986542]},{"f":[1,21,17,-1,-1,22,22,-1,21,22,-1,-1,22,22,21,-1,-1,21,2,-1,-1,-1,-1,-1,3,-1,15,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,7.5,16.725912049439096,0,0,16.725912049439096,9.024458927812285,8.5,0,0,8.5,6.451187557243013,0,0,0,0,0,0.0339080459770115,0,5.044394119358453,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,0,0,13,14,15,0,0,18,19,0,0,0,0,0,25,0,27,0,0],"r":[24,5,4,0,0,23,8,0,12,11,0,0,22,17,16,0,0,21,20,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,-0.04303826,0.01113399,0,0,0.0220408,0,0,-0.02425201,0.00917943,0,0,0,-0.03410327,0.02014267,0,0,0.07436227,0.01887429,-0.001948031,-0.03128686,0.03579711,0,0.0184512,0,0.005072573,-0.001178746]},{"f":[22,-1,22,1,22,-1,-1,-1,22,21,4,-1,18,16,-1,-1,-1,-1,1,22,-1,22,-1,-1,15,-1,0,-1,-1],"t":[6.870153907126159,0,7.95646805166752,4.008411214953271,7.082728314749225,0,0,0,12.199665042528586,5.5,3.753897320099348,0,4.128693921346326,5.78135971352466,0,0,0,0,1.282051282051282,15.717370311433935,0,16.786010507999382,0,0,1.584962500721156,0,2.5,0,0],"m":[0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,10,11,0,13,14,0,0,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,8,7,6,0,0,0,18,17,12,0,16,15,0,0,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.02116289,0,0,0,0.02297743,-0.02048075,0.04364325,0,0,0,-0.04510033,0,0,0.05777522,0.001559113,-0.03086705,-0.0006664709,0,0,-0.0435371,0,0.07724595,-0.01786724,0,0.02649964,0,-0.02387062,0.001588241]},{"f":[1,21,-1,25,22,22,22,-1,2,-1,-1,22,-1,4,-1,-1,17,-1,-1,1,37,-1,-1,21,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,16.98920034885663,16.844368517741394,0,4.128693921346326,0,0,17.078739781790915,0,5.83200138144325,0,0,8.729619742794132,0,0,6.431742323097463,5.539999999999999,0,0,4.5,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,1,0,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0],"l":[1,2,0,4,5,6,7,0,9,0,0,12,0,14,0,0,17,0,0,20,21,0,0,24,0,0,27,0,0],"r":[26,3,0,19,16,11,8,0,10,0,0,13,0,15,0,0,18,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,-0.0386659,0,0,0,0,-0.0003573252,0,-0.02966444,0.02904536,0,-0.03488498,0,-0.02546538,0.04227216,0,0.04159356,-0.008051253,0,0,-0.003009605,0.04804979,0,-0.007907642,-0.04089777,0,0.001718148,0.04139564]},{"f":[23,21,15,16,-1,2,22,-1,-1,22,-1,-1,22,23,18,-1,-1,3,-1,4,-1,22,15,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,11.775312925643524,0,0,16.765402642760364,0,0,16.759160379664443,0.5,10.680359255641626,0,0,0.15991874474642756,0,6.749860669831508,0,15.782972563647261,11.054604317960672,0,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,15,0,0,18,0,20,0,22,23,0,0,0,0,0,0],"r":[28,27,12,5,0,9,8,0,0,11,0,0,26,17,16,0,0,19,0,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0009200338,0,0,0.008159861,0.1231972,0,0.001272801,0.03394909,0,0,0,-0.01198311,0.04409055,0,0.00455494,0,-0.03746531,0,0,-0.004066924,-0.03726454,0.0364301,0.02711662,0.03326431,-0.02804092]},{"f":[22,-1,22,-1,22,21,4,-1,18,-1,-1,-1,22,15,-1,-1,22,16,-1,-1,15,21,-1,-1,15,17,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,5.5,3.753897320099348,0,4.128693921346326,0,0,0,12.921158001713904,7.930737337562887,0,0,14.116600814324018,6.832890014164741,0,0,5.044394119358453,4.5,0,0,6.169925001442312,7.67595272073847,0,0,0],"m":[0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,7,0,9,0,0,0,13,14,0,0,17,18,0,0,21,22,0,0,25,26,0,0,0],"r":[2,0,4,0,12,11,8,0,10,0,0,0,16,15,0,0,20,19,0,0,24,23,0,0,28,27,0,0,0],"v":[0,-0.02062812,0,0.01833599,0,0,0,-0.04480601,0,0.007366949,-0.03020402,-0.0006440062,0,0,0.004048895,0.03323765,0,0,-0.04389083,-0.00314322,0,0,0.05044376,0.006581648,0,0,-0.03496036,0.02819683,0.001320179]},{"f":[37,25,22,22,22,22,33,10,-1,21,-1,22,22,15,37,-1,-1,-1,-1,-1,-1,18,-1,-1,-1,-1,-1,-1,-1],"t":[12.52,6.5,17.078739781790915,17.02942582914998,16.838461650489297,16.786010507999382,35.98606694718995,0.5,0,2.5,0,16.759160379664443,16.73920708776685,13.56045225766691,7.92,0,0,0,0,0,0,5.065928480304397,0,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,10,0,12,13,14,15,0,0,0,0,0,0,22,0,0,0,0,0,0,0],"r":[28,27,26,25,24,21,20,9,0,11,0,19,18,17,16,0,0,0,0,0,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,-0.04357399,0,-0.03755064,0,0,0,0,-2.248409e-05,0.006643442,-0.03723074,-0.03202099,0.01044043,-0.04380336,0,-0.03155387,0.01044043,0.01044043,-0.04047122,0.01000917,-0.03271139,0.0211646]},{"f":[4,19,37,21,-1,-1,-1,-1,22,3,-1,2,16,16,-1,-1,22,16,-1,17,-1,-1,-1,2,33,-1,-1,-1,-1],"t":[4.0437314206251695,2.5,4.470000000000001,6.5,0,0,0,0,17.078739781790915,0.13721264367816094,0,5.8698903048866296,6.0,5.78135971352466,0,0,14.03140051979143,7.266786540694901,0,6.531360383582522,0,0,0,6.793888758164105,29.297756302860655,0,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,0,0,0,9,10,0,12,13,14,0,0,17,18,0,20,0,0,0,24,25,0,0,0,0],"r":[8,7,6,5,0,0,0,0,28,11,0,23,16,15,0,0,22,19,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.0380358,-0.007650061,-0.0001614822,-0.03841138,0,0,0.03346782,0,0,0,-0.0008485986,-0.03832504,0,0,-0.0235098,0,0.05852162,0.004494898,0.04696023,0,0,-0.02474213,0.01527734,0.0004130082,0.03833289]},{"f":[23,21,15,16,-1,2,22,-1,-1,22,-1,-1,22,22,16,-1,22,-1,-1,1,-1,22,33,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,11.775312925643524,0,0,16.765402642760364,0,0,16.759160379664443,11.722778233171695,10.501837184902296,0,11.683871801979876,0,0,4.87351462522852,0,15.782972563647261,35.98606694718995,0,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,15,0,17,0,0,20,0,22,23,0,0,0,0,0,0],"r":[28,27,12,5,0,9,8,0,0,11,0,0,26,19,16,0,18,0,0,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0008511883,0,0,0.007479311,0.1078587,0,0.001225443,0.0323811,0,0,0,-0.03458501,0,0.04082287,0.002329716,0,-0.02985581,0,0,0.01404621,-0.03889483,0.03340085,0.02550476,0.03178215,-0.02772391]},{"f":[4,19,37,21,-1,-1,-1,-1,22,22,22,22,22,15,21,-1,-1,22,-1,1,-1,-1,16,-1,-1,-1,-1,-1,-1],"t":[4.0437314206251695,2.5,4.470000000000001,6.5,0,0,0,0,17.078739781790915,17.02942582914998,16.844368517741394,16.819976309430963,16.759160379664443,11.683433292835403,3.5,0,0,15.782972563647261,0,3.354166666666667,0,0,9.047123912114026,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,1,1,1,1,1,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,0,9,10,11,12,13,14,15,0,0,18,0,20,0,0,23,0,0,0,0,0,0],"r":[8,7,6,5,0,0,0,0,28,27,26,25,22,17,16,0,0,19,0,21,0,0,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.03757822,-0.007338955,-0.0001679753,-0.03801239,0,0,0,0,0,0,0,-0.01731596,0.004275734,0,-0.03406439,0,-0.02891757,0.03534956,0,-0.01282505,0.04227748,-0.03985752,0.03319051,-0.03371939,0.03642016]},{"f":[23,21,15,16,-1,2,22,-1,-1,-1,23,18,-1,-1,22,3,-1,4,-1,22,15,-1,18,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,11.775312925643524,0,0,0,0.5,10.680359255641626,0,0,16.759160379664443,0.15991874474642756,0,6.749860669831508,0,15.782972563647261,11.054604317960672,0,9.265614570460752,0,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,1,0,0,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,0,11,12,0,0,15,16,0,18,0,20,21,0,23,0,0,0,0,0,0],"r":[28,27,10,5,0,9,8,0,0,0,14,13,0,0,26,17,0,19,0,25,22,0,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0008137362,0,0,0.007287805,0.09818403,0.003022284,0,0,-0.01129833,0.04118405,0,0,0.004279094,0,-0.03575305,0,0,-0.002947206,0,-0.04610209,-0.00293657,0.03055063,0.02284039,0.0304822,-0.02741312]},{"f":[37,25,4,16,-1,18,2,-1,21,-1,22,-1,22,-1,-1,23,22,-1,-1,22,21,-1,-1,17,-1,-1,-1,-1,-1],"t":[12.52,6.5,3.245926548164837,2.0,0,2.160964047443681,1.792481250360578,0,6.5,0,16.73920708776685,0,16.765402642760364,0,0,1.5,15.676331898425197,0,0,17.07907323966382,8.5,0,0,8.729619742794132,0,0,0,0,0],"m":[1,1,1,0,0,0,1,0,1,0,1,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,9,0,11,0,13,0,0,16,17,0,0,20,21,0,0,24,0,0,0,0,0],"r":[28,27,26,5,0,15,8,0,10,0,12,0,14,0,0,19,18,0,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,-0.04151075,0,0,0.009953302,0,-0.02590938,0,0.00598079,0,-0.02769539,0.009953302,0,0,0.001489379,-0.02270373,0,0,-0.04360947,0.004622311,0,0.009953302,-0.04004547,0.001737412,-0.0325325,0.02011708]},{"f":[16,27,-1,22,-1,22,-1,-1,22,22,22,22,22,37,33,-1,-1,-1,-1,-1,2,-1,-1,-1,18,33,-1,-1,-1],"t":[0.0,0.5,0,16.975010474608837,0,17.094543568094064,0,0,17.078739781790915,16.98920034885663,16.844368517741394,16.786010507999382,16.765402642760364,11.73,35.98606694718995,0,0,0,0,0,3.903677461028802,0,0,0,8.729619742794132,20.091102901552432,0,0,0],"m":[0,1,0,0,0,1,0,0,1,1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,13,14,15,0,0,0,0,0,21,0,0,0,25,26,0,0,0],"r":[8,3,0,5,0,7,0,0,24,23,20,19,18,17,16,0,0,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,0,-0.04429889,0,0.03156798,0,-0.01058458,0.03037179,0,0,0,0,0,0,0,-0.0002285236,-0.04265765,0.02819952,0.04045087,-0.02026709,0,-0.007706993,0.04584722,-0.03390756,0,0,0.03885594,-0.01162431,-0.0311866]},{"f":[22,-1,22,-1,22,21,22,0,-1,-1,33,18,33,-1,-1,-1,-1,15,21,-1,-1,16,-1,-1,22,-1,22,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,6.5,11.775312925643524,15.5,0,0,35.98606694718995,4.128693921346326,35.98439522565276,0,0,0,0,7.774787059601174,7.5,0,0,11.756139298372927,0,0,12.921158001713904,0,14.116600814324018,0,0],"m":[0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,0,0,11,12,13,0,0,0,0,18,19,0,0,22,0,0,25,0,27,0,0],"r":[2,0,4,0,24,17,10,9,0,0,16,15,14,0,0,0,0,21,20,0,0,23,0,0,26,0,28,0,0],"v":[0,-0.01936995,0,0.01718378,0,0,0,0,-0.0418689,0.01613514,0,0,0,-0.002151339,0.03980001,-0.02614875,-0.04150753,0,0,-0.02486643,9.604249e-05,0,0.02612519,-0.01506922,0,0.009020859,0,-0.01422753,0.001590904]},{"f":[23,21,15,4,37,-1,-1,22,16,-1,-1,-1,22,22,33,-1,22,-1,-1,1,-1,22,33,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,4.0437314206251695,4.470000000000001,0,0,17.078739781790915,9.047123912114026,0,0,0,16.759160379664443,11.722778233171695,73.23848001446603,0,11.683871801979876,0,0,4.87351462522852,0,15.782972563647261,35.98606694718995,0,0,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,0,13,14,15,0,17,0,0,20,0,22,23,0,0,0,0,0,0],"r":[28,27,12,7,6,0,0,11,10,0,0,0,26,19,16,0,18,0,0,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,-0.01972546,-0.0002675474,0,0,-0.0006489014,0.01220754,0.03416047,0,0,0,-0.03458928,0,0.03713988,0.001487132,0,-0.02772609,0,0,0.01260747,-0.03719497,0.02834835,0.02143715,0.02936902,-0.02640155]},{"f":[21,4,16,-1,16,36,22,2,-1,-1,-1,-1,16,-1,-1,22,18,-1,32,-1,20,-1,-1,-1,36,18,-1,-1,-1],"t":[9.5,3.245926548164837,2.0,0,5.044394119358453,3.5,15.782972563647261,1.292481250360578,0,0,0,0,6.108524456778169,0,0,17.094543568094064,5.303665156874805,0,6.208926257442949,0,0.5,0,0,0,3.5,1.792481250360578,0,0,0],"m":[1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,0,0,0,13,0,0,16,17,0,19,0,21,0,0,0,25,26,0,0,0],"r":[24,15,4,0,12,11,10,9,0,0,0,0,14,0,0,23,18,0,20,0,22,0,0,0,28,27,0,0,0],"v":[0,0,0,-0.04162805,0,0,0,0,0.02214346,-0.02123699,-0.01447639,0.03134874,0,-0.03748368,-0.003091021,0,0,-0.002335299,0,-0.02587865,0,0.001476428,0.02051819,0.0377323,0,0,0.01253233,-0.02183775,0.04059007]},{"f":[9,16,16,-1,-1,21,-1,17,4,22,-1,-1,22,1,-1,-1,-1,17,-1,17,-1,22,18,-1,-1,18,-1,-1,-1],"t":[0.5,7.930737337562887,7.679480099505446,0,0,4.5,0,1.792481250360578,5.108372929097653,12.199665042528586,0,0,16.725912049439096,1.987429289754871,0,0,0,5.065928480304397,0,6.219136028062415,0,16.838461650489297,7.519373778108656,0,0,6.400854179458231,0,0,0],"m":[1,1,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,0,8,9,10,0,0,13,14,0,0,0,18,0,20,0,22,23,0,0,26,0,0,0],"r":[28,5,4,0,0,7,0,17,12,11,0,0,16,15,0,0,0,19,0,21,0,25,24,0,0,27,0,0,0],"v":[0,0,0,-0.000579045,-0.029974,0,-0.02252365,0,0,0,0.01185023,0.04560286,0,0,0.01400594,-0.03960625,0.01487129,0,-0.01468594,0,0.04292927,0,0,-0.002576344,0.008704066,0,0.03685414,-0.01384072,-0.02850907]},{"f":[23,21,15,22,21,-1,23,-1,-1,-1,22,22,-1,22,-1,1,-1,-1,-1,0,22,-1,22,1,-1,-1,-1,-1,-1],"t":[7.5,8.5,11.054604317960672,14.116600814324018,3.5,0,3.5,0,0,0,16.759160379664443,11.683871801979876,0,15.782972563647261,0,4.008411214953271,0,0,0,10.5,10.224917391507956,0,13.589275375459419,1.99834912080135,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,12,0,14,0,16,0,0,0,20,21,0,23,24,0,0,0,0,0],"r":[28,19,10,9,6,0,8,0,0,0,18,13,0,15,0,17,0,0,0,27,22,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03420563,0,0.0006637714,-0.02101308,0.002952378,0,0,0.02603522,0,-0.03269707,0,-0.02747008,0.02436748,0.02184548,0,0,0.02157967,0,0,0.001063255,-0.02295418,0.01402991,0.04204404,-0.025924]},{"f":[16,27,-1,22,-1,22,-1,22,-1,22,-1,-1,24,33,22,22,22,22,-1,-1,-1,-1,18,-1,-1,-1,21,-1,-1],"t":[0.0,0.5,0,16.975010474608837,0,16.978341770814396,0,16.98920034885663,0,17.094543568094064,0,0,10.5,35.98606694718995,17.078739781790915,16.98920034885663,16.844368517741394,16.786010507999382,0,0,0,0,8.729619742794132,0,0,0,5.5,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,0,1,1,1,1,1,1,0,0,0,0,1,0,0,0,1,0,0],"l":[1,2,0,4,0,6,0,8,0,10,0,0,13,14,15,16,17,18,0,0,0,0,23,0,0,0,27,0,0],"r":[12,3,0,5,0,7,0,9,0,11,0,0,26,25,22,21,20,19,0,0,0,0,24,0,0,0,28,0,0],"v":[0,0,-0.04377156,0,0.03053834,0,-0.02920295,0,0.03069325,0,-0.01955002,0.02907301,0,0,0,0,0,0,0.0005554394,-0.01967355,0.02611749,-0.03348163,0,0.02371023,-0.03055981,-0.04003184,0,-0.01000152,0.03326369]},{"f":[4,19,25,0,-1,-1,-1,-1,22,3,-1,22,22,33,32,22,36,-1,-1,21,-1,-1,-1,1,-1,-1,-1,-1,-1],"t":[4.0437314206251695,2.5,5.5,10.5,0,0,0,0,17.078739781790915,0.13721264367816094,0,17.02942582914998,16.844368517741394,25.24368171001516,6.474913355379556,16.759160379664443,3.5,0,0,4.5,0,0,0,8.992857142857144,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,1,1,1,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,0,9,10,0,12,13,14,15,16,17,0,0,20,0,0,0,24,0,0,0,0,0],"r":[8,7,6,5,0,0,0,0,28,11,0,27,26,23,22,19,18,0,0,21,0,0,0,25,0,0,0,0,0],"v":[0,0,0,0,-0.001815253,0.01304727,-0.02896619,-0.03684765,0,0,0.0297223,0,0,0,0,0,0,0.0002567637,-0.03104826,0,-0.02157197,0.03593005,0.04592138,0,-0.03830531,0.01434913,0.03024525,-0.03241794,0.02917006]},{"f":[9,4,-1,22,22,22,22,4,15,19,17,-1,-1,-1,17,19,-1,-1,-1,18,-1,1,-1,-1,-1,-1,-1,-1,-1],"t":[0.5,4.0437314206251695,0,17.078739781790915,17.02942582914998,16.844368517741394,16.819976309430963,5.108372929097653,6.0,0.5,2.69615871138938,0,0,0,7.5507416557826375,0.5,0,0,0,7.980000966034041,0,6.015625,0,0,0,0,0,0,0],"m":[1,1,0,1,1,1,1,0,1,1,1,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,7,8,9,10,11,0,0,0,15,16,0,0,0,20,0,22,0,0,0,0,0,0,0],"r":[28,3,0,27,26,25,24,19,14,13,12,0,0,0,18,17,0,0,0,21,0,23,0,0,0,0,0,0,0],"v":[0,0,-0.001503967,0,0,0,0,0,0,0,0,-0.03121455,0.01661443,-0.03221111,0,0,0.06524631,0.01974229,-0.006954385,0,-0.003771089,0,0.002669158,0.03777952,-0.03805878,0.0288783,-0.03182133,0.02766429,-0.02796035]},{"f":[23,21,15,16,-1,2,-1,-1,23,18,18,-1,-1,-1,2,-1,22,17,22,-1,-1,22,15,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,0,0,0.5,10.680359255641626,1.292481250360578,0,0,0,6.247802615595391,0,16.759160379664443,1.292481250360578,16.725912049439096,0,0,15.782972563647261,11.683433292835403,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,0,0,15,0,17,18,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,8,5,0,7,0,0,14,13,12,0,0,0,16,0,26,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0007520889,0,0.05558977,0.002451594,0,0,0,0.01486154,-0.0278513,0.03671718,0,-0.03396625,0,0,0,-0.04229276,-0.001433588,0,0,0.001545216,-0.02844528,0.02556977,0.02046043,0.02680156,-0.02552626]},{"f":[22,-1,22,-1,22,21,4,-1,18,16,-1,-1,-1,-1,25,22,1,-1,17,-1,-1,22,16,18,-1,-1,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,5.5,3.753897320099348,0,4.128693921346326,5.78135971352466,0,0,0,0,1.5,12.921158001713904,6.806765389082463,0,1.292481250360578,0,0,17.094543568094064,6.108524456778169,5.303665156874805,0,0,0,0,0],"m":[0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,9,10,0,0,0,0,15,16,17,0,19,0,0,22,23,24,0,0,0,0,0],"r":[2,0,4,0,14,13,8,0,12,11,0,0,0,0,28,21,18,0,20,0,0,27,26,25,0,0,0,0,0],"v":[0,-0.01841266,0,0.01459139,0,0,0,-0.04367686,0,0,0.05738918,0.000560006,-0.02817846,-0.0009530716,0,0,0,0.007338378,0,-0.02402383,0.05596067,0,0,0,-0.004180405,-0.0427599,0.002307904,0.02380197,-0.007199142]},{"f":[1,21,15,-1,-1,22,1,4,-1,2,-1,-1,22,21,-1,-1,22,-1,-1,-1,3,-1,23,-1,21,22,-1,-1,-1],"t":[2.01685992705101,6.5,1.584962500721156,0,0,16.765402642760364,1.13125,4.128693921346326,0,6.434604125275413,0,0,15.212004812197344,7.5,0,0,15.782972563647261,0,0,0,0.0339080459770115,0,2.5,0,6.5,16.975010474608837,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,10,0,0,13,14,0,0,17,0,0,0,21,0,23,0,25,26,0,0,0],"r":[20,5,4,0,0,19,12,9,0,11,0,0,16,15,0,0,18,0,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0,0.008588529,-0.04193537,0,0,0,0.03205036,0,-0.0350176,0.006684227,0,0,-0.03962562,0.01110695,0,0.03484633,-0.006362182,0.03373683,0,0.01576775,0,0.00230263,0,0,-0.04627764,0.004661067,-0.0006622719]},{"f":[23,21,15,22,-1,-1,23,18,18,-1,-1,-1,2,-1,22,17,22,-1,-1,22,15,-1,18,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,14.116600814324018,0,0,0.5,10.680359255641626,1.292481250360578,0,0,0,6.247802615595391,0,16.759160379664443,1.292481250360578,16.725912049439096,0,0,15.782972563647261,11.696967526234287,0,9.265614570460752,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,1,0,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,0,15,16,17,0,0,20,21,0,23,0,0,0,0,0,0],"r":[28,27,6,5,0,0,12,11,10,0,0,0,14,0,26,19,18,0,0,25,22,0,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.001886748,0.003116849,0,0,0,0.01387099,-0.02690405,0.03527857,0,-0.03328413,0,0,0,-0.04186564,-0.001427556,0,0,-1.031038e-06,0,-0.04459475,0.001154378,0.02398799,0.01957031,0.02561695,-0.02505729]},{"f":[1,21,-1,25,22,22,22,-1,-1,22,-1,4,-1,-1,17,-1,-1,1,37,-1,-1,21,22,-1,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,16.98920034885663,16.978341770814396,0,0,17.078739781790915,0,5.83200138144325,0,0,8.729619742794132,0,0,6.431742323097463,5.539999999999999,0,0,4.5,12.613886328427084,0,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,0,0,10,0,12,0,0,15,0,0,18,19,0,0,22,23,0,0,0,27,0,0],"r":[26,3,0,17,14,9,8,0,0,11,0,13,0,0,16,0,0,21,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,-0.03641567,0,0,0,0,0.0003281725,0.02919956,0,-0.03197466,0,-0.02475006,0.02386182,0,0.03859508,-0.009820765,0,0,-0.002454756,0.04179329,0,0,-0.02178736,0.0008623797,-0.03993215,0,0.001344532,0.03696772]},{"f":[22,-1,22,15,-1,-1,22,16,17,4,15,-1,-1,2,16,-1,-1,-1,-1,16,17,-1,22,-1,1,-1,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,2.584962500721156,0,0,12.199665042528586,10.462502272597265,4.128693921346326,6.16978583919235,5.491853096329675,0,0,7.661668144640085,6.554588851677638,0,0,0,0,11.756139298372927,9.075473573191719,0,11.67903991803059,0,3.892276422764228,0,0,0,0],"m":[0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,0,0,14,15,0,0,0,0,20,21,0,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,28,19,18,13,12,0,0,17,16,0,0,0,0,27,22,0,24,0,26,0,0,0,0],"v":[0,-0.01779532,0,0,-0.008206224,0.01869656,0,0,0,0,0,-0.0402469,0.000275312,0,0,0.07768203,0.002324341,-0.00286437,-0.01895322,0,0,-0.008548105,0,-0.02602594,0,0.002616477,0.04744119,-0.002304271,0.001001513]},{"f":[1,21,-1,25,22,22,22,23,-1,1,-1,-1,2,-1,-1,22,-1,-1,17,-1,-1,2,-1,32,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,16.98920034885663,16.844368517741394,5.5,0,4.92309205350118,0,0,4.128693921346326,0,0,17.078739781790915,0,0,8.729619742794132,0,0,3.245926548164837,0,5.954008630531401,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,8,0,10,0,0,13,0,0,16,0,0,19,0,0,22,0,24,0,0,27,0,0],"r":[26,3,0,21,18,15,12,9,0,11,0,0,14,0,0,17,0,0,20,0,0,23,0,25,0,0,28,0,0],"v":[0,0,-0.0359931,0,0,0,0,0,0.0008008813,0,-0.0009490432,-0.038977,0,-0.02876386,0.02426914,0,-0.03122566,-0.001167428,0,0.03733335,-0.009503803,0,-0.02298416,0,-0.005793546,0.02086369,0,0.001407464,0.03598735]},{"f":[16,27,-1,22,-1,22,-1,22,-1,22,-1,-1,25,37,33,1,18,21,-1,22,15,-1,-1,-1,-1,-1,-1,-1,-1],"t":[0.0,0.5,0,16.975010474608837,0,16.978341770814396,0,16.98920034885663,0,17.094543568094064,0,0,5.5,11.26,35.98606694718995,4.92309205350118,10.482680779086738,6.5,0,16.759160379664443,11.054604317960672,0,0,0,0,0,0,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,0,1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,6,0,8,0,10,0,0,13,14,15,16,17,18,0,20,21,0,0,0,0,0,0,0,0],"r":[12,3,0,5,0,7,0,9,0,11,0,0,28,27,26,25,24,19,0,23,22,0,0,0,0,0,0,0,0],"v":[0,0,-0.04258086,0,0.02917803,0,-0.02908089,0,0.02741263,0,-0.01765123,0.02726299,0,0,0,0,0,0,-0.01992403,0,0,-0.002557998,-0.02161626,0.01069455,0.008184686,0.002707297,-0.0403115,0.02111565,-0.01934854]},{"f":[23,21,4,16,-1,0,25,16,17,-1,-1,21,-1,22,-1,-1,-1,-1,22,2,-1,21,15,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,3.245926548164837,2.0,0,10.5,1.5,6.832890014164741,8.729619742794132,0,0,4.5,0,16.786010507999382,0,0,0,0,17.094543568094064,3.39067985676233,0,3.5,6.108524456778169,0,0,0,0,0,0],"m":[1,1,1,0,0,1,1,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,12,0,14,0,0,0,0,19,20,0,22,23,0,0,0,0,0,0],"r":[28,27,18,5,0,17,16,11,10,0,0,13,0,15,0,0,0,0,26,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0386289,0,0,0,0,0.008781062,-0.032185,0,-0.03881174,0,-0.0009756952,-0.03293237,-0.02158317,0.02042736,0,0,0.02358997,0,0,-0.03254938,0.007182342,0.001180854,0.03511917,0.02463128,-0.02427073]},{"f":[4,19,22,22,7,3,-1,-1,-1,-1,22,-1,1,-1,22,-1,15,-1,-1,-1,32,35,2,-1,-1,-1,33,-1,-1],"t":[4.0437314206251695,2.5,16.73920708776685,16.725912049439096,0.5,0.013290990355210539,0,0,0,0,16.752707246237605,0,1.3897752394988947,0,16.765402642760364,0,5.044394119358453,0,0,0,6.208926257442949,1.5,4.128693921346326,0,0,0,20.091102901552432,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,11,0,13,0,15,0,17,0,0,0,21,22,23,0,0,0,27,0,0],"r":[20,19,10,9,8,7,0,0,0,0,12,0,14,0,16,0,18,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0.01915891,-0.0004800527,-0.02061663,0.02235007,0,-0.04101381,0,0.01770038,0,-0.03612755,0,0.04305543,-0.007149387,-0.0360606,0,0,0,0.04771961,0.000543631,-0.02792749,0,0.0488383,0.006023567]},{"f":[9,4,19,22,-1,22,-1,-1,-1,22,22,22,22,22,15,-1,22,18,-1,-1,1,-1,-1,-1,-1,-1,-1,-1,-1],"t":[0.5,4.0437314206251695,2.5,16.73920708776685,0,16.752707246237605,0,0,0,17.078739781790915,17.02942582914998,16.844368517741394,16.819976309430963,16.759160379664443,11.683433292835403,0,15.782972563647261,9.265614570460752,0,0,3.354166666666667,0,0,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,1,1,1,1,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,0,10,11,12,13,14,15,0,17,18,0,0,21,0,0,0,0,0,0,0,0],"r":[28,9,8,5,0,7,0,0,0,27,26,25,24,23,16,0,20,19,0,0,22,0,0,0,0,0,0,0,0],"v":[0,0,0,0,0.0007239242,0,-0.04001536,-0.004028935,-0.03567426,0,0,0,0,0,0,0.002721333,0,0,-0.04405957,-0.003748194,0,-0.02339982,0.02501184,0.02286986,-0.03704645,0.02779309,-0.03041822,0.02432198,-0.02635812]},{"f":[22,-1,22,-1,22,16,17,4,15,-1,-1,2,16,-1,-1,-1,36,-1,-1,22,3,-1,-1,2,17,-1,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,10.462502272597265,4.128693921346326,6.16978583919235,5.491853096329675,0,0,7.661668144640085,6.554588851677638,0,0,0,4.5,0,0,11.67903991803059,0.9197769197769198,0,0,6.959990297524481,9.075473573191719,0,0,0,0],"m":[0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,0,0,12,13,0,0,0,17,0,0,20,21,0,0,24,25,0,0,0,0],"r":[2,0,4,0,28,19,16,11,10,0,0,15,14,0,0,0,18,0,0,23,22,0,0,27,26,0,0,0,0],"v":[0,-0.01704002,0,0.01241154,0,0,0,0,0,-0.03979355,0.0002104061,0,0,0.07158735,0.002002327,-0.002840142,0,-0.02403978,-0.0009302347,0,0,0.0007774116,-0.03592212,0,0,-0.02350598,0.05231824,0.001323554,0.0009511319]},{"f":[37,25,22,21,16,-1,-1,33,23,-1,-1,-1,22,-1,22,16,-1,-1,15,33,-1,-1,22,15,-1,-1,-1,-1,-1],"t":[12.52,6.5,12.199665042528586,4.5,5.78135971352466,0,0,35.98606694718995,3.5,0,0,0,12.921158001713904,0,14.116600814324018,6.832890014164741,0,0,8.076815597050832,21.225056427472516,0,0,15.782972563647261,11.054604317960672,0,0,0,0,0],"m":[1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,0,13,0,15,16,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,27,12,7,6,0,0,11,10,0,0,0,14,0,18,17,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.008487619,-0.03903273,0,0,0.0006991581,-0.02184652,-0.03923225,0,0.008266468,0,0,-0.04261478,-0.001457769,0,0,0.007622474,-0.003871938,0,0,-0.007779817,-0.0404934,0.0005805188,-0.03073089,0.01718633]},{"f":[1,21,15,-1,-1,22,1,2,-1,2,-1,-1,22,21,-1,-1,22,-1,-1,-1,3,-1,23,-1,21,32,-1,-1,-1],"t":[2.01685992705101,6.5,1.584962500721156,0,0,16.765402642760364,1.13125,4.128693921346326,0,6.434604125275413,0,0,15.212004812197344,7.5,0,0,15.782972563647261,0,0,0,0.0339080459770115,0,2.5,0,6.5,8.599793042091783,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,10,0,0,13,14,0,0,17,0,0,0,21,0,23,0,25,26,0,0,0],"r":[20,5,4,0,0,19,12,9,0,11,0,0,16,15,0,0,18,0,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0,0.00792157,-0.04131482,0,0,0,0.03507159,0,-0.03373966,0.006709944,0,0,-0.03863128,0.01174717,0,0.03275735,-0.006369885,0.03204121,0,0.01527975,0,0.002081461,0,0,-0.04578793,0.003438653,-0.0006419702]},{"f":[23,21,4,19,37,21,-1,-1,22,-1,-1,-1,32,35,4,20,16,-1,-1,-1,-1,-1,33,-1,37,-1,-1,-1,-1],"t":[7.5,10.5,4.0437314206251695,2.5,4.470000000000001,6.5,0,0,16.73920708776685,0,0,0,6.208926257442949,1.5,4.2849278041654735,0.5,5.169925001442312,0,0,0,0,0,20.091102901552432,0,8.2,0,0,0,0],"m":[1,1,1,1,1,1,0,0,1,0,0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,9,0,0,0,13,14,15,16,17,0,0,0,0,0,23,0,25,0,0,0,0],"r":[28,27,12,11,8,7,0,0,10,0,0,0,22,21,20,19,18,0,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.03485482,-0.006660176,0,0.001313949,-0.006194045,-0.03498043,0,0,0,0,0,-0.003048922,0.06191166,-0.0198233,0.0002484178,-0.02984766,0,0.04666257,0,-0.006764908,0.01602912,0.02366009,-0.02375298]},{"f":[23,22,16,37,-1,22,-1,-1,24,33,25,0,21,-1,25,-1,36,2,-1,-1,-1,-1,-1,-1,21,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,8.2,0,17.02942582914998,0,0,10.5,35.98606694718995,5.5,10.5,2.5,0,1.5,0,3.5,2.453445297804259,0,0,0,0,0,0,5.5,0,0,0,0],"m":[1,1,0,1,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,12,13,0,15,0,17,18,0,0,0,0,0,0,25,0,0,0,0],"r":[28,27,8,5,0,7,0,0,24,23,22,21,14,0,16,0,20,19,0,0,0,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.04288964,0,0.009648156,-0.02152431,0,0,0,0,0,-0.03454881,0,0.0009965807,0,0,-0.01686766,0.003565301,-0.03923718,0.008033139,-0.01783608,-0.03821387,0,-0.01095233,0.03096692,0.02194829,-0.02330577]},{"f":[0,32,9,23,22,21,-1,23,18,17,22,-1,-1,25,19,-1,-1,-1,-1,16,-1,1,-1,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,0.5,7.5,14.116600814324018,3.5,0,3.5,7.519373778108656,2.160964047443681,13.129761635363703,0,0,0.5,0.5,0,0,0,0,6.882643049361842,0,3.015625,0,0,0,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,1,1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,10,11,0,0,14,15,0,0,0,0,20,0,22,0,0,0,0,0,0,0],"r":[28,27,26,25,24,7,0,19,18,13,12,0,0,17,16,0,0,0,0,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.03150349,0,0,0,0,0.006522176,-0.03361463,0,0,-0.006741272,-0.03121529,0.0008433752,0.008393098,0,-0.03047496,0,-0.03273121,0.008122041,0.001713299,-0.02286527,-0.02551937,-0.02891637,0.02210169]},{"f":[1,21,15,-1,-1,22,1,22,22,4,-1,2,-1,-1,-1,-1,22,21,-1,-1,22,-1,-1,-1,4,-1,22,-1,-1],"t":[2.01685992705101,6.5,1.584962500721156,0,0,16.765402642760364,1.13125,15.676331898425197,11.722778233171695,5.169368191286958,0,6.434604125275413,0,0,0,0,15.212004812197344,7.5,0,0,15.782972563647261,0,0,0,4.0437314206251695,0,17.083799521744545,0,0],"m":[0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,1,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,12,0,0,0,0,17,18,0,0,21,0,0,0,25,0,27,0,0],"r":[24,5,4,0,0,23,16,15,14,11,0,13,0,0,0,0,20,19,0,0,22,0,0,0,26,0,28,0,0],"v":[0,0,0,0.00756629,-0.04069803,0,0,0,0,0,0.02737264,0,-0.03770863,0.00626781,-0.03556869,0.0558781,0,0,-0.03805363,0.01126003,0,0.0310247,-0.006160298,0.03109916,0,-0.001095643,0,0.002450967,0.03863816]},{"f":[23,21,15,4,-1,32,-1,-1,22,33,-1,-1,2,16,-1,-1,22,15,33,-1,22,21,-1,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,4.0437314206251695,0,6.208926257442949,0,0,11.683871801979876,73.23848001446603,0,0,6.247802615595391,11.98370619265935,0,0,16.759160379664443,13.56045225766691,35.98606694718995,0,15.782972563647261,6.5,0,0,0,0,0,0,0],"m":[1,1,1,1,0,1,0,0,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,13,14,0,0,17,18,19,0,21,22,0,0,0,0,0,0,0],"r":[28,27,8,5,0,7,0,0,12,11,0,0,16,15,0,0,26,25,20,0,24,23,0,0,0,0,0,0,0],"v":[0,0,0,0,-0.001197142,0,0.001433055,0.01543327,0,0,-0.0336332,0.03119191,0,0,-0.0416607,0.000948247,0,0,0,0.009266374,0,0,-0.04592799,-0.00639504,0.02064763,-0.03296686,0.01719661,0.02276484,-0.02259417]},{"f":[37,25,22,21,16,15,-1,-1,-1,33,23,-1,-1,-1,1,-1,22,-1,22,16,-1,0,-1,-1,15,-1,-1,-1,-1],"t":[12.52,6.5,12.199665042528586,4.5,5.78135971352466,5.357552004618084,0,0,0,35.98606694718995,3.5,0,0,0,0.4906446161274747,0,12.921158001713904,0,14.116600814324018,6.832890014164741,0,3.5,0,0,7.266786540694901,0,0,0,0],"m":[1,1,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,0,0,0,15,0,17,0,19,20,0,22,0,0,25,0,0,0,0],"r":[28,27,14,9,8,7,0,0,0,13,12,0,0,0,16,0,18,0,24,21,0,23,0,0,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.0300412,0.00803332,-0.03831544,0,0,0.0003267036,-0.02060435,-0.03760148,0,-0.03252169,0,0.007678009,0,0,-0.04183475,0,-0.02896185,0.004066075,0,0.006057357,-0.001391861,-0.02950608,0.01629964]},{"f":[23,21,22,16,37,-1,22,-1,-1,25,37,33,4,18,-1,23,22,-1,-1,22,-1,-1,-1,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,17.105450904090162,0.0,8.2,0,17.02942582914998,0,0,5.5,11.26,35.98606694718995,3.245926548164837,2.160964047443681,0,1.5,15.676331898425197,0,0,17.07907323966382,0,0,0,0,0,0,0,0,0],"m":[1,1,1,0,1,0,1,0,0,1,1,1,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,12,13,14,0,16,17,0,0,20,0,0,0,0,0,0,0,0,0],"r":[28,27,26,9,6,0,8,0,0,25,24,23,22,15,0,19,18,0,0,21,0,0,0,0,0,0,0,0,0],"v":[0,0,0,0,0,-0.04249957,0,0.009164167,-0.02137413,0,0,0,0,0,0.001237015,0,0,0.002077751,-0.02193729,0,-0.03611581,-0.001112154,0.002213939,-0.03769628,0.01861395,-0.01851134,0.02086601,0.02186465,-0.02222705]},{"f":[1,18,21,2,1,-1,-1,-1,22,15,4,4,4,-1,-1,-1,-1,-1,16,22,-1,-1,-1,-1,18,36,-1,-1,-1],"t":[4.92309205350118,10.482680779086738,6.5,8.767235038274581,4.208333333333334,0,0,0,16.759160379664443,13.56045225766691,9.266718101290426,9.037514954341955,8.049190845056867,0,0,0,0,0,8.96289600533726,17.078739781790915,0,0,0,0,10.415240582199587,1.5,0,0,0],"m":[1,1,1,1,1,0,0,0,1,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,11,12,13,0,0,0,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[24,23,8,7,6,0,0,0,18,17,16,15,14,0,0,0,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0,-0.03450696,0.005165675,0.005018082,0,0,0,0,0,-0.001807297,-0.02985526,0.01412054,-0.01711252,-0.03887372,0,0,-0.01390294,0.01595285,0.034365,0.007370148,0,0,0.008056233,-6.709972e-06,-0.02956845]},{"f":[0,32,22,-1,22,-1,22,-1,25,21,-1,23,22,21,-1,-1,-1,16,-1,-1,0,36,-1,-1,22,-1,-1,-1,-1],"t":[18.5,10.482680779086738,6.870153907126159,0,7.95646805166752,0,8.133841411710264,0,1.5,2.5,0,0.5,11.612768260520431,8.5,0,0,0,7.930737337562887,0,0,9.5,3.5,0,0,12.613886328427084,0,0,0,0],"m":[1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,0,7,0,9,10,0,12,13,14,0,0,0,18,0,0,21,22,0,0,25,0,0,0,0],"r":[28,27,4,0,6,0,8,0,20,11,0,17,16,15,0,0,0,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.01570901,0,0.01035319,0,-0.02821645,0,0,-0.03239921,0,0,0,-0.02438695,0.00623479,0.01035319,0,-0.004199505,0.001930141,0,0,-0.00782354,-0.03934819,0,-0.006457531,0.01035319,-0.02808919,0.02091893]},{"f":[1,21,2,22,16,1,-1,-1,-1,-1,-1,-1,18,36,18,18,2,-1,-1,16,-1,19,-1,-1,-1,21,-1,-1,-1],"t":[4.92309205350118,5.5,8.612669628062957,16.978341770814396,5.044394119358453,2.9861172120945896,0,0,0,0,0,0,10.415240582199587,1.5,8.729619742794132,7.198837316474133,5.40929108874043,0,0,5.906890595608519,0,1.5,0,0,0,9.5,0,0,0],"m":[1,0,1,1,0,1,0,0,0,0,0,0,1,1,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,0,13,14,15,16,17,0,0,20,0,22,0,0,0,26,0,0,0],"r":[12,11,10,9,8,7,0,0,0,0,0,0,28,25,24,19,18,0,0,21,0,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0,0,-0.03549645,0.007193742,-0.04507015,0.01794875,0.004350499,-0.0005569191,0,0,0,0,0,0.01018342,-0.009695928,0,-0.0280299,0,0.06885403,-0.01830041,-0.04010735,0,-0.0008610118,0.02381008,-0.02885752]},{"f":[23,22,22,22,22,22,22,22,22,-1,-1,-1,-1,4,-1,-1,-1,-1,22,-1,25,17,18,-1,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,3.9534452978042594,0,0,0,0,17.078739781790915,0,0.5,8.729619742794132,6.076776015854056,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,0,14,0,0,0,0,19,0,21,22,23,0,0,0,0,0,0],"r":[28,27,18,17,16,13,12,11,10,0,0,0,0,15,0,0,0,0,20,0,26,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0.0003543207,-0.01598054,0.03555436,-0.01776992,0,-0.02670447,0.04565068,-0.02232468,0.0279993,0,-0.02612518,0,0,0,-0.005342238,0.04186882,-0.02262198,-0.01986869,0.02062469,-0.02170787]},{"f":[16,16,-1,-1,21,22,-1,-1,17,4,22,22,-1,-1,-1,22,22,-1,-1,-1,17,-1,17,-1,22,-1,18,-1,-1],"t":[7.930737337562887,7.679480099505446,0,0,4.5,16.964783007086943,0,0,1.792481250360578,5.108372929097653,12.199665042528586,9.40390385294334,0,0,0,16.725912049439096,12.606173770495205,0,0,0,5.065928480304397,0,6.219136028062415,0,16.838461650489297,0,6.400854179458231,0,0],"m":[1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,12,0,0,0,16,17,0,0,0,21,0,23,0,25,0,27,0,0],"r":[4,3,0,0,8,7,0,0,20,15,14,13,0,0,0,19,18,0,0,0,22,0,24,0,26,0,28,0,0],"v":[0,0,-0.0004320748,-0.02834351,0,0,-0.03112944,0.007461188,0,0,0,0,0.03855456,0.000242866,0.03845999,0,0,0.00230292,-0.04334327,0.01143947,0,-0.01354915,0,0.03238742,0,-0.0003485959,0,0.03154377,-0.01343238]},{"f":[0,32,23,1,18,21,1,-1,-1,22,15,4,22,20,-1,16,-1,-1,-1,-1,-1,16,-1,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,7.5,4.92309205350118,10.482680779086738,6.5,2.0015923566878984,0,0,16.759160379664443,13.56045225766691,9.266718101290426,16.73920708776685,0.5,0,6.247927513443585,0,0,0,0,0,7.924812503605781,0,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,0,0,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,11,12,13,14,0,16,0,0,0,0,0,22,0,0,0,0,0,0,0],"r":[28,27,26,25,24,9,8,0,0,21,20,19,18,15,0,17,0,0,0,0,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.03768246,-0.007392102,0,0,0,0,0,0.004503932,0,-0.03340966,-0.002889244,-0.01826908,-0.01632148,-0.03791676,0,-0.0156111,0.01038487,0.006770207,0.001294866,-0.0214632,-0.02727234,0.02097505]},{"f":[37,25,22,33,21,16,15,-1,-1,-1,23,-1,-1,-1,1,-1,25,-1,25,2,-1,-1,22,2,-1,-1,-1,-1,-1],"t":[12.52,6.5,12.199665042528586,35.98606694718995,4.5,5.78135971352466,5.357552004618084,0,0,0,3.5,0,0,0,0.4906446161274747,0,1.5,0,3.5,3.245926548164837,0,0,12.613886328427084,1.292481250360578,0,0,0,0,0],"m":[1,1,0,1,0,1,1,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,0,0,0,15,0,17,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,27,14,13,10,9,8,0,0,0,12,0,0,0,16,0,18,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.02927179,0.007505689,-0.03694912,0,0.0002151429,-0.01976116,-0.03723897,0,-0.03155182,0,0.002171845,0,0,-0.02736368,0.003736286,0,0,0.007505689,-0.03515427,0.007505689,-0.02853124,0.01524558]},{"f":[1,21,-1,25,22,22,-1,22,-1,4,-1,-1,17,17,-1,-1,-1,1,37,-1,-1,21,22,-1,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,17.02942582914998,0,17.078739781790915,0,5.83200138144325,0,0,8.729619742794132,6.8199529586472405,0,0,0,6.431742323097463,5.539999999999999,0,0,4.5,12.613886328427084,0,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,0,1,0,1,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,8,0,10,0,0,13,14,0,0,0,18,19,0,0,22,23,0,0,0,27,0,0],"r":[26,3,0,17,12,7,0,9,0,11,0,0,16,15,0,0,0,21,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,-0.03269254,0,0,0,0.0005384438,0,-0.04033622,0,-0.02165308,0.01790456,0,0,-0.007525335,0.04604144,-0.009022261,0,0,-0.001766978,0.03560762,0,0,-0.02014275,0.001126372,-0.03694107,0,0.0007174215,0.03238949]},{"f":[1,18,21,1,-1,15,-1,-1,22,15,4,4,4,4,-1,-1,-1,-1,-1,-1,16,22,-1,-1,-1,-1,18,-1,-1],"t":[4.92309205350118,10.482680779086738,6.5,2.0015923566878984,0,6.022367813028454,0,0,16.759160379664443,13.56045225766691,9.266718101290426,9.037514954341955,8.049190845056867,7.005613627711627,0,0,0,0,0,0,8.96289600533726,17.078739781790915,0,0,0,0,10.415240582199587,0,0],"m":[1,1,1,1,0,1,0,0,1,1,1,1,1,1,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,12,13,14,0,0,0,0,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,20,19,18,17,16,15,0,0,0,0,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,-0.03749273,0,0.003683369,-0.02439378,0,0,0,0,0,0,-0.003096391,0.01841876,-0.02901573,0.01330212,-0.01571817,-0.03749282,0,0,-0.01281117,0.01299959,0.03211413,0.006339923,0,0.001489835,-0.0279138]},{"f":[23,22,16,37,-1,22,-1,-1,24,33,8,22,21,-1,22,-1,16,-1,-1,16,-1,-1,-1,-1,21,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,8.2,0,17.02942582914998,0,0,10.5,35.98606694718995,0.5,14.116600814324018,3.5,0,13.129761635363703,0,6.832890014164741,0,0,3.4594316186372973,0,0,0,0,5.5,0,0,0,0],"m":[1,1,0,1,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,12,13,0,15,0,17,0,0,20,0,0,0,0,25,0,0,0,0],"r":[28,27,8,5,0,7,0,0,24,23,22,19,14,0,16,0,18,0,0,21,0,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.04186564,0,0.008329338,-0.01989297,0,0,0,0,0,-0.02970353,0,0.0003803036,0,-0.04187925,-0.004100064,0,0.0389723,0.001321569,0.01145128,-0.03548271,0,-0.01053596,0.02873495,0.01977488,-0.02099798]},{"f":[9,4,16,-1,17,-1,0,22,22,16,-1,-1,18,-1,23,-1,-1,15,17,-1,18,-1,-1,-1,-1,22,-1,-1,-1],"t":[0.5,3.245926548164837,2.0,0,0.5,0,10.5,16.73920708776685,11.67903991803059,12.225508720236002,0,0,2.160964047443681,0,1.5,0,0,7.6653359171851765,6.8199529586472405,0,8.729619742794132,0,0,0,0,17.094543568094064,0,0,0],"m":[1,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,0,13,0,15,0,0,18,19,0,21,0,0,0,0,26,0,0,0],"r":[28,25,4,0,6,0,24,17,12,11,0,0,14,0,16,0,0,23,20,0,22,0,0,0,0,27,0,0,0],"v":[0,0,0,-0.03900973,0,0.02836884,0,0,0,0,-0.03795652,-0.001703966,0,0.005269366,0,-0.0006369295,-0.02834131,0,0,-0.02331221,0,0.007903221,-0.0372977,-0.02931036,0.01849179,0,0.0008018865,0.03370499,-0.02348772]},{"f":[0,32,9,17,16,2,16,-1,-1,-1,22,21,-1,-1,24,-1,-1,17,16,-1,-1,2,21,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,0.5,2.160964047443681,2.584962500721156,6.434604125275413,2.0,0,0,0,15.782972563647261,3.5,0,0,7.5,0,0,2.69615871138938,5.672425341971495,0,0,1.792481250360578,8.5,0,0,0,0,0,0],"m":[1,1,1,0,0,1,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,15,0,0,18,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,17,10,9,8,0,0,0,14,13,0,0,16,0,0,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.03818005,0.009599584,0.001160176,0,0,-0.01767583,0.009599584,0,0.008443298,-0.01841312,0,0,0.005619404,-0.03998831,0,0,-0.01019473,0.009599584,0.0005668617,-0.02319468,-0.02674319,0.01939121]},{"f":[1,4,21,-1,22,4,22,-1,2,22,-1,-1,16,-1,18,22,2,-1,-1,-1,-1,-1,22,-1,-1,-1,3,-1,-1],"t":[2.01685992705101,9.266718101290426,3.5,0,16.765402642760364,9.037514954341955,7.082728314749225,0,6.16978583919235,15.782972563647261,0,0,2.0,0,1.792481250360578,16.73920708776685,6.515505953718852,0,0,0,0,0,16.807008825964132,0,0,0,0.0339080459770115,0,0],"m":[0,1,0,0,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,10,0,0,13,0,15,16,17,0,0,0,0,0,23,0,0,0,27,0,0],"r":[26,25,4,0,22,21,8,0,12,11,0,0,14,0,20,19,18,0,0,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.03270746,0,0,0,0.01174115,0,0,-0.03505356,0.02998449,0,0.01641878,0,0,0,0.01073702,-0.004477631,-0.01530491,-0.02335988,0.01200775,0,0.03661017,-0.01697719,-0.02348055,0,0.01388563,0.0002607881]},{"f":[36,-1,22,21,22,16,-1,-1,16,17,-1,-1,33,-1,-1,15,2,-1,-1,15,-1,-1,25,22,-1,22,-1,-1,-1],"t":[1.5,0,12.199665042528586,6.5,11.775312925643524,10.681238411777805,0,0,5.78135971352466,4.128693921346326,0,0,35.98606694718995,0,0,9.269126679149418,9.266718101290426,0,0,10.681238411777805,0,0,1.5,13.054613167076562,0,13.589275375459419,0,0,0],"m":[0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,0,0,13,0,0,16,17,0,0,20,0,0,23,24,0,26,0,0,0],"r":[2,0,22,15,8,7,0,0,12,11,0,0,14,0,0,19,18,0,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0.01731195,0,0,0,0,-0.03784749,0.009337127,0,0,0.06113893,0.00208247,0,-0.001135055,-0.04093453,0,0,0.0006372041,-0.02105346,0,0.03746657,0.001760726,0,0,0.01187996,0,-0.02643743,0.00122301,-0.006109866]},{"f":[1,18,21,1,-1,15,15,-1,17,-1,-1,22,-1,-1,22,15,15,-1,-1,-1,16,22,-1,-1,-1,-1,18,-1,-1],"t":[4.92309205350118,10.482680779086738,6.5,2.0015923566878984,0,6.022367813028454,3.321928094887362,0,3.5221970596792267,0,0,16.844368517741394,0,0,16.759160379664443,11.054604317960672,8.361943773735241,0,0,0,8.96289600533726,17.078739781790915,0,0,0,0,10.415240582199587,0,0],"m":[1,1,1,1,0,1,0,0,0,0,0,1,0,0,1,1,1,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,7,0,9,0,0,12,0,0,15,16,17,0,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,14,5,0,11,8,0,10,0,0,13,0,0,20,19,18,0,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,-0.0363659,0,0,-0.01925466,0,0.04742671,0.001625,0,-0.03816592,0.01154999,0,0,0,-0.003439671,0.011905,-0.01926756,0,0,-0.01235609,0.01243254,0.03035654,0.005938423,0,0.001456313,-0.02718615]},{"f":[16,27,-1,22,-1,22,-1,22,-1,22,33,-1,-1,-1,24,33,8,22,21,-1,-1,16,-1,-1,-1,-1,21,-1,-1],"t":[0.0,0.5,0,16.975010474608837,0,16.978341770814396,0,16.98920034885663,0,17.094543568094064,30.6384348520704,0,0,0,10.5,35.98606694718995,0.5,14.116600814324018,3.5,0,0,3.4594316186372973,0,0,0,0,5.5,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,0,6,0,8,0,10,11,0,0,0,15,16,17,18,19,0,0,22,0,0,0,0,27,0,0],"r":[14,3,0,5,0,7,0,9,0,13,12,0,0,0,26,25,24,21,20,0,0,23,0,0,0,0,28,0,0],"v":[0,0,-0.04065578,0,0.02660504,0,-0.027938,0,0.02466142,0,0,0.01366091,-0.01946498,0.02483497,0,0,0,0,0,-0.0284631,-0.0009673824,0,0.0371967,0.001257417,0.01080124,-0.03444043,0,-0.01016611,0.02744654]},{"f":[0,32,23,-1,21,36,-1,-1,22,-1,24,16,-1,-1,4,3,22,22,16,-1,-1,2,-1,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,2.5,0,6.5,3.5,0,0,8.882387211598338,0,7.5,8.321928094887362,0,0,9.221342183932917,0.9973346627775462,16.765402642760364,16.73920708776685,6.247927513443585,0,0,9.012591414317754,0,0,0,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,1,0,0,1,0,1,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,0,0,9,0,11,12,0,0,15,16,17,18,19,0,0,22,0,0,0,0,0,0,0],"r":[28,27,4,0,8,7,0,0,10,0,14,13,0,0,26,25,24,21,20,0,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0.001008796,0,0,0.007734436,-0.03733756,0,-0.03371288,0,0,-0.02804208,-0.001402643,0,0,0,0,0,-0.0369591,0.00412716,0,-0.04195115,0.003241869,0.006791216,0.009016185,-0.01645094,-0.02584637,0.01899083]},{"f":[36,-1,24,37,32,33,33,-1,-1,-1,1,-1,-1,-1,25,36,22,22,-1,16,-1,-1,17,17,-1,-1,-1,-1,-1],"t":[1.5,0,7.5,8.05,6.337096134072842,21.16864017939033,19.167172160526107,0,0,0,7.992857142857143,0,0,0,2.5,5.5,17.094543568094064,17.02942582914998,0,7.774787059601174,0,0,8.729619742794132,6.8199529586472405,0,0,0,0,0],"m":[0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,0,0,0,15,16,17,18,0,20,0,0,23,24,0,0,0,0,0],"r":[2,0,14,13,10,9,8,0,0,0,12,0,0,0,28,27,22,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0.01709646,0,0,0,0,0,-0.003657788,0.00903672,-0.01527097,0,0.0006681056,-0.03995246,0.02224554,0,0,0,0,0.0008774713,0,-0.02764402,0.006217401,0,0,-0.005967939,0.04481716,-0.01709764,0.03368201,-0.02593326]},{"f":[9,4,16,-1,17,-1,0,25,0,-1,22,22,-1,33,-1,-1,22,-1,-1,15,-1,-1,-1,22,2,-1,-1,-1,-1],"t":[0.5,3.245926548164837,2.0,0,0.5,0,10.5,1.5,2.5,0,16.73920708776685,12.199665042528586,0,21.80651201897376,0,0,16.765402642760364,0,0,2.807354922057604,0,0,0,17.094543568094064,3.39067985676233,0,0,0,0],"m":[1,1,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,9,0,11,12,0,14,0,0,17,0,0,20,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,6,0,22,19,10,0,16,13,0,15,0,0,18,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03793159,0,0.02656791,0,0,0,-0.02725224,0,0,-0.007877083,0,0.004830291,-0.01395248,0,-0.01395248,0.007264611,0,-0.0008221642,-0.02808961,0.01727183,0,0,0.02346592,0.0002589914,0.03273515,-0.02265144]},{"f":[0,32,23,-1,16,16,27,-1,22,-1,-1,-1,18,4,18,22,-1,-1,-1,15,-1,22,18,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,1.5,0,6.523561956057013,5.700439718141092,0.5,0,8.882387211598338,0,0,0,10.45532575654158,4.0437314206251695,2.160964047443681,16.73920708776685,0,0,0,11.054604317960672,0,15.782972563647261,9.265614570460752,0,0,0,0,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,1,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,0,13,14,15,16,0,0,0,20,0,22,23,0,0,0,0,0,0],"r":[28,27,4,0,12,11,8,0,10,0,0,0,26,19,18,17,0,0,0,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0.001314337,0,0,0,-0.03683438,0,-0.03332304,-0.0004380951,-0.03663239,0,0,0,0,0.005932363,-0.008496944,-0.01519748,0,0.008806047,0,0,-0.04249566,0.009029194,0.005896337,-0.02125673,-0.02544133,0.01824841]},{"f":[22,22,22,22,22,10,-1,22,-1,2,-1,-1,-1,-1,22,-1,-1,22,-1,4,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,16.73920708776685,0,6.515505953718852,0,0,0,0,16.903290731167747,0,0,17.078739781790915,0,5.83200138144325,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,8,0,10,0,0,0,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,14,13,12,7,0,9,0,11,0,0,0,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.03716428,0,0.0005899204,0,-0.03705176,0.004892647,0.03329184,-0.01729782,0,0.04992859,0.00295869,0,-0.03563697,0,-0.01896676,0.01634258,0,0.04366789,0,0,0.03543847,-0.03006035,-0.03616866]},{"f":[1,22,18,1,21,-1,17,22,3,-1,-1,-1,22,-1,-1,22,-1,-1,16,-1,2,-1,-1,16,-1,-1,18,-1,-1],"t":[4.031939779270633,16.759160379664443,10.482680779086738,2.9733542319749215,4.5,0,5.83200138144325,16.725912049439096,0.8516483516483516,0,0,0,10.029977181607594,0,0,8.658207063149977,0,0,2.0,0,1.292481250360578,0,0,7.924812503605781,0,0,10.415240582199587,0,0],"m":[1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,0,7,8,9,0,0,0,13,0,0,16,0,0,19,0,21,0,0,24,0,0,27,0,0],"r":[26,23,18,15,6,0,12,11,10,0,0,0,14,0,0,17,0,0,20,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.02496745,0,0,0,0.02276873,0.0006980129,-0.01957281,0,-0.03273084,-0.003985078,0,0.001181061,-0.02268705,0,0.01780436,0,0.02586372,-0.003349068,0,-0.01616854,0.02322389,0,0.001226166,-0.02681659]},{"f":[36,-1,24,37,32,33,33,-1,-1,-1,1,-1,-1,-1,25,36,22,22,-1,22,-1,1,-1,-1,17,-1,-1,-1,-1],"t":[1.5,0,7.5,8.05,6.337096134072842,21.16864017939033,19.167172160526107,0,0,0,7.992857142857143,0,0,0,2.5,5.5,17.094543568094064,17.02942582914998,0,17.078739781790915,0,3.966632962588473,0,0,8.729619742794132,0,0,0,0],"m":[0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,1,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,0,0,0,15,16,17,18,0,20,0,22,0,0,25,0,0,0,0],"r":[2,0,14,13,10,9,8,0,0,0,12,0,0,0,28,27,24,19,0,21,0,23,0,0,26,0,0,0,0],"v":[0,0.01661246,0,0,0,0,0,-0.003599391,0.008643721,-0.01477682,0,0.0004175491,-0.03906081,0.02130811,0,0,0,0,0.0008321283,0,-0.03653109,0,0.01288627,-0.02185493,0,0.03364766,-0.01678479,0.03303076,-0.02503573]},{"f":[1,21,16,-1,-1,22,1,22,22,4,-1,2,-1,-1,-1,-1,22,21,-1,-1,22,-1,0,-1,-1,-1,3,-1,-1],"t":[2.01685992705101,6.5,1.584962500721156,0,0,16.765402642760364,1.13125,15.676331898425197,11.722778233171695,5.169368191286958,0,6.434604125275413,0,0,0,0,15.212004812197344,7.5,0,0,15.782972563647261,0,4.5,0,0,0,0.0339080459770115,0,0],"m":[0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,12,0,0,0,0,17,18,0,0,21,0,23,0,0,0,27,0,0],"r":[26,5,4,0,0,25,16,15,14,11,0,13,0,0,0,0,20,19,0,0,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0.01329827,-0.03444596,0,0,0,0,0,0.02182062,0,-0.03592064,0.004812239,-0.03425778,0.0523529,0,0,-0.0366883,0.009704955,0,0.02773332,0,-0.03141702,-0.004063269,0.03000068,0,0.01348583,0.0002424909]},{"f":[23,21,15,16,23,-1,21,-1,-1,2,22,-1,-1,22,21,-1,-1,-1,22,33,-1,-1,2,16,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,1.5,0,6.5,0,0,5.065928480304397,12.613886328427084,0,0,16.765402642760364,6.5,0,0,0,11.683871801979876,73.23848001446603,0,0,6.247802615595391,11.98370619265935,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,15,0,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,27,18,9,6,0,8,0,0,13,12,0,0,17,16,0,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.002125205,0,-0.02203343,-0.0007829523,0,0,0.007681896,0.07611699,0,0,0.02733753,-0.001467127,0.02761282,0,0,-0.03208947,0.0284722,0,0,-0.04059994,0.002950779,-0.003783727,0.01911772,-0.01981439]},{"f":[4,16,-1,0,25,23,-1,21,-1,18,22,22,-1,-1,-1,21,-1,-1,15,-1,-1,-1,22,2,15,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,10.5,1.5,0.5,0,4.5,0,8.729619742794132,16.765402642760364,16.73920708776685,0,0,0,8.5,0,0,2.807354922057604,0,0,0,17.094543568094064,3.39067985676233,6.820178962415188,0,0,0,0],"m":[1,0,0,1,1,0,0,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0],"l":[1,2,0,4,5,6,0,8,0,10,11,12,0,0,0,16,0,0,19,0,0,0,23,24,25,0,0,0,0],"r":[22,3,0,21,18,7,0,9,0,15,14,13,0,0,0,17,0,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,0,-0.03727764,0,0,0,0.007426856,0,-0.02991588,0,0,0,0.0003004174,-0.02058601,0.007519704,0,-0.04265443,0.007519704,0,-0.0008667786,-0.02675799,0.01688877,0,0,0,0.008609003,0.05568028,0.0001737734,0.03153949]},{"f":[22,22,22,22,22,10,-1,-1,-1,-1,22,-1,32,-1,22,-1,-1,22,-1,4,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,0,0,0,16.903290731167747,0,2.69615871138938,0,16.978341770814396,0,0,17.078739781790915,0,5.83200138144325,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,11,0,13,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,10,9,8,7,0,0,0,0,12,0,14,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.03646398,-5.729218e-05,0.03130665,-0.01676685,0,0.04844019,0,-0.03382156,0,-0.005869855,0.02223841,0,-0.03460722,0,-0.01789892,0.01464121,0,0.04265708,0,0,0.03413462,-0.02906111,-0.03490866]},{"f":[0,32,23,1,22,15,18,17,22,3,-1,22,-1,18,-1,-1,-1,17,-1,-1,-1,-1,16,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,7.5,4.031939779270633,16.759160379664443,13.56045225766691,10.482680779086738,6.199638962254484,16.73920708776685,0.8516483516483516,0,12.921158001713904,0,3.39067985676233,0,0,0,7.335383440499621,0,0,0,0,7.924812503605781,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,10,0,12,0,14,0,0,0,18,0,0,0,0,23,0,0,0,0,0,0],"r":[28,27,26,25,22,21,20,17,16,11,0,13,0,15,0,0,0,19,0,0,0,0,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0.008735198,0,-2.308258e-05,0,-0.03493561,0.008735198,-0.02711705,0,-0.03183461,-0.008406527,0.005657192,-0.03662844,0,-0.01564824,0.008735198,0.001027786,-0.01937589,-0.02454708,0.01762184]},{"f":[4,16,-1,17,-1,0,25,0,-1,22,22,-1,33,-1,-1,22,-1,0,-1,-1,-1,-1,22,2,15,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,0,10.5,1.5,2.5,0,16.73920708776685,12.199665042528586,0,21.80651201897376,0,0,16.765402642760364,0,6.5,0,0,0,0,17.094543568094064,3.39067985676233,6.820178962415188,0,0,0,0],"m":[1,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0],"l":[1,2,0,4,0,6,7,8,0,10,11,0,13,0,0,16,0,18,0,0,0,0,23,24,25,0,0,0,0],"r":[22,3,0,5,0,21,20,9,0,15,12,0,14,0,0,17,0,19,0,0,0,0,28,27,26,0,0,0,0],"v":[0,0,-0.03670403,0,0.02349386,0,0,0,-0.02638664,0,0,-0.007642003,0,0.004293565,-0.01355315,0,-0.01355315,0,-0.01355315,0.006912449,-0.01652606,0.0162454,0,0,0,0.008259998,0.05349254,0.0002081394,0.03038432]},{"f":[36,-1,22,21,22,16,-1,-1,16,17,-1,-1,33,-1,-1,15,22,21,-1,-1,0,1,-1,-1,-1,15,-1,-1,-1],"t":[1.5,0,12.199665042528586,6.5,11.775312925643524,10.681238411777805,0,0,5.78135971352466,4.128693921346326,0,0,35.98606694718995,0,0,9.269126679149418,10.27595502635321,9.5,0,0,8.5,1.99834912080135,0,0,0,10.681238411777805,0,0,0],"m":[0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,0,0,13,0,0,16,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[2,0,28,15,8,7,0,0,12,11,0,0,14,0,0,25,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0.01576173,0,0,0,0,-0.03595106,0.008220654,0,0,0.06180165,0.001541605,0,-0.00115046,-0.04011966,0,0,0,0.0005735295,0.02754618,0,0,-0.005525349,-0.03227311,0.008919784,0,0.03452866,0.001671597,0.0007395262]},{"f":[25,0,21,-1,33,11,36,-1,-1,-1,25,22,2,22,36,15,-1,-1,-1,-1,-1,-1,-1,21,16,-1,-1,-1,-1],"t":[6.5,10.5,2.5,0,21.16864017939033,0.5,4.5,0,0,0,1.5,16.838461650489297,6.977234485717126,16.73920708776685,3.5,7.118941072723508,0,0,0,0,0,0,0,8.5,2.807354922057604,0,0,0,0],"m":[1,1,0,0,1,1,1,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,11,12,13,14,15,16,0,0,0,0,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,10,9,8,0,0,0,22,21,20,19,18,17,0,0,0,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03035081,0,0,0,0.001537552,-0.02012441,-0.03121628,0,0,0,0,0,0,-0.005252742,-0.03704239,0.00215211,-0.01553366,-0.03344279,0.00215211,-0.02881779,0,0,0.03126858,0.00215211,0.03325288,-0.01752567]},{"f":[22,22,22,22,16,-1,-1,-1,22,2,-1,-1,32,-1,22,-1,-1,22,-1,4,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,16.903290731167747,3.245926548164837,0,0,2.69615871138938,0,16.978341770814396,0,0,17.078739781790915,0,5.83200138144325,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,0,0,13,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,8,7,6,0,0,0,12,11,0,0,14,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,-0.03342047,0.0001751879,-0.03575353,0,0,-0.01602722,0.04106375,0,-0.03312424,0,-0.005891778,0.02088217,0,-0.03411187,0,-0.01741116,0.01383589,0,0.04156113,0,0,0.03342582,-0.02811128,-0.03452368]},{"f":[9,4,16,-1,17,-1,0,22,22,16,-1,-1,18,-1,-1,15,18,17,-1,-1,-1,-1,-1,22,2,-1,-1,-1,-1],"t":[0.5,3.245926548164837,2.0,0,0.5,0,10.5,16.73920708776685,11.67903991803059,12.225508720236002,0,0,2.160964047443681,0,0,7.451211111832329,8.729619742794132,6.781300095563516,0,0,0,0,0,17.094543568094064,3.39067985676233,0,0,0,0],"m":[1,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,0,13,0,0,16,17,18,0,0,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,6,0,22,15,12,11,0,0,14,0,0,21,20,19,0,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03612379,0,0.02204615,0,0,0,0,-0.03653742,-0.00106871,0,0.004026285,-0.00432336,0,0,0,-0.02031299,0.006390081,-0.03437425,-0.02537835,0.01511048,0,0,0.02125922,0.0003183325,0.02926431,-0.02132668]},{"f":[1,4,21,-1,22,4,-1,15,-1,-1,22,-1,-1,-1,3,-1,23,33,33,33,-1,-1,-1,22,1,-1,-1,-1,-1],"t":[2.01685992705101,9.266718101290426,3.5,0,16.765402642760364,9.037514954341955,0,1.584962500721156,0,0,16.807008825964132,0,0,0,0.0339080459770115,0,0.5,73.23848001446603,21.16864017939033,19.167172160526107,0,0,0,14.116600814324018,5.489664082687339,0,0,0,0],"m":[0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,0,11,0,0,0,15,0,17,18,19,20,0,0,0,24,25,0,0,0,0],"r":[14,13,4,0,10,7,0,9,0,0,12,0,0,0,16,0,28,23,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.0307465,0,0,-0.003405211,0,-0.001765521,0.02048834,0,0.03373604,-0.01420917,-0.02148532,0,0.01300618,0,0,0,0,-0.01576698,0.01834916,-0.01550049,0,0,0.003353618,0.02570128,0.1075703,-0.0007620793]},{"f":[36,-1,22,33,-1,-1,15,18,20,-1,-1,-1,15,2,-1,-1,15,15,33,-1,-1,18,-1,-1,18,-1,15,-1,-1],"t":[1.5,0,12.613886328427084,8.532371382375128,0,0,5.044394119358453,3.39067985676233,1.5,0,0,0,6.169925001442312,4.80643424864552,0,0,6.857980995127572,6.820178962415188,21.257266042509173,0,0,1.292481250360578,0,0,6.864066550153108,0,9.756556322524087,0,0],"m":[0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,1,0,1,0,0],"l":[1,0,3,4,0,0,7,8,9,0,0,0,13,14,0,0,17,18,19,0,0,22,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,12,11,10,0,0,0,16,15,0,0,24,21,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0.01541521,0,0,-0.03884941,-0.0007086158,0,0,0,-0.002280015,0.02577748,0.02119771,0,0,-0.04271015,-0.005538247,0,0,0,0.01500467,-0.00729789,0,-0.008228481,0.122688,0,0.0006314057,0,-0.04561954,-0.001444666]},{"f":[22,22,22,22,22,22,22,-1,-1,-1,4,-1,-1,-1,32,-1,-1,4,-1,22,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,3.9534452978042594,0,0,0,2.69615871138938,0,0,5.83200138144325,0,17.074218326240924,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,0,0,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,14,13,10,9,8,0,0,0,12,0,0,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0002766423,0.03031757,-0.01562612,0,-0.02706208,0.04061381,-0.02174086,0,-0.02019812,0.03073861,0,-0.02140525,0,-0.02297876,0.01378555,0,0.04054511,0,0,0.03262991,-0.02693838,-0.03372325]},{"f":[23,21,15,16,23,-1,21,-1,-1,2,22,-1,-1,22,21,-1,-1,-1,22,33,-1,-1,2,16,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,1.5,0,6.5,0,0,5.065928480304397,12.613886328427084,0,0,16.765402642760364,6.5,0,0,0,11.683871801979876,73.23848001446603,0,0,6.247802615595391,11.98370619265935,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,15,0,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,27,18,9,6,0,8,0,0,13,12,0,0,17,16,0,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.00187755,0,-0.02094084,-0.000621669,0,0,0.007334077,0.06921221,0,0,0.02629596,-0.00122017,0.02579276,0,0,-0.03107945,0.02620881,0,0,-0.03989516,0.003599346,-0.003719058,0.01815634,-0.01883694]},{"f":[32,32,37,25,22,22,22,22,16,-1,21,-1,-1,-1,2,-1,-1,25,-1,22,-1,-1,-1,-1,-1,-1,18,-1,-1],"t":[10.463013322934223,10.41931383093672,12.52,6.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,2.5,0,0,0,4.128693921346326,0,0,3.5,0,17.069460217922263,0,0,0,0,0,0,3.5221970596792267,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,11,0,0,0,15,0,0,18,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,24,23,22,17,14,13,10,0,12,0,0,0,16,0,0,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,-0.03045751,0,-0.02942563,0.0003410311,-0.03481996,0,-0.0153244,0.009290385,0,-0.03816857,0,-0.03081084,0.008635513,0.009256029,-0.02682189,0.01865143,-0.0108869,0,0.01800078,-0.02030181]},{"f":[1,22,15,18,17,22,3,17,-1,-1,15,16,-1,-1,1,-1,-1,-1,17,-1,-1,-1,-1,16,-1,-1,18,-1,-1],"t":[4.031939779270633,16.759160379664443,13.56045225766691,10.482680779086738,6.199638962254484,16.73920708776685,0.8516483516483516,1.792481250360578,0,0,5.78135971352466,3.321928094887362,0,0,1.987429289754871,0,0,0,7.335383440499621,0,0,0,0,7.924812503605781,0,0,10.415240582199587,0,0],"m":[1,1,1,1,1,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,0,11,12,0,0,15,0,0,0,19,0,0,0,0,24,0,0,27,0,0],"r":[26,23,22,21,18,17,10,9,0,0,14,13,0,0,16,0,0,0,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0.03332434,0.002419703,0,0,-0.00562872,0.008798187,0,-0.004553145,-0.03949348,-0.02612626,0,-0.03090538,-0.008088934,0.005194101,-0.03617331,0,-0.01499009,0.02115338,0,0.001198715,-0.02556]},{"f":[36,-1,24,37,32,33,20,-1,-1,-1,1,-1,-1,-1,25,36,22,16,18,-1,-1,-1,23,22,-1,-1,-1,-1,-1],"t":[1.5,0,7.5,8.05,6.337096134072842,21.16864017939033,1.5,0,0,0,7.992857142857143,0,0,0,2.5,5.5,12.199665042528586,10.462502272597265,5.491496787347155,0,0,0,0.5,14.116600814324018,0,0,0,0,0],"m":[0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,1,0,1,1,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,0,0,0,15,16,17,18,19,0,0,0,23,24,0,0,0,0,0],"r":[2,0,14,13,10,9,8,0,0,0,12,0,0,0,28,27,22,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,0.0151101,0,0,0,0,0,-0.003926233,0.008031238,-0.01297366,0,4.835463e-05,-0.03719461,0.01974195,0,0,0,0,0,-0.001984422,-0.0229716,0.005120954,0,0,0.008744671,0.08830057,0.0006895943,0.03172311,-0.02289858]},{"f":[1,21,-1,24,36,4,33,33,21,-1,-1,-1,-1,1,-1,-1,1,-1,-1,21,22,-1,17,-1,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,7.5,3.5,4.128693921346326,21.16864017939033,19.167172160526107,3.5,0,0,0,0,6.241071428571429,0,0,7.992857142857143,0,0,3.5,14.03140051979143,0,2.453445297804259,0,0,0,6.118903736861568,0,0],"m":[1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,8,9,0,0,0,0,14,0,0,17,0,0,20,21,0,23,0,0,0,27,0,0],"r":[26,3,0,19,16,13,12,11,10,0,0,0,0,15,0,0,18,0,0,25,22,0,24,0,0,0,28,0,0],"v":[0,0,-0.02959937,0,0,0,0,0,0,-0.04328795,-0.002594844,0.01766977,-0.01330622,0,-0.01123939,0.0192136,0,-0.006471855,-0.03025158,0,0,-0.02058904,0,8.650688e-05,0.08909283,0.0002672712,0,0.0004468031,0.02934524]},{"f":[22,22,22,22,22,10,-1,22,-1,-1,-1,-1,22,-1,24,-1,-1,22,-1,4,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,16.73920708776685,0,0,0,0,16.903290731167747,0,8.5,0,0,17.078739781790915,0,5.83200138144325,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,0,0,13,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,12,11,10,7,0,9,0,0,0,0,14,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.03459883,0,0.0005389407,-0.01168945,0.02892919,-0.01486639,0,0.04579073,0,0.007121963,-0.02775931,0,-0.03275164,0,-0.01552009,0.01169231,0,0.0400122,0,0,0.03171197,-0.02567927,-0.03339427]},{"f":[9,16,16,-1,-1,21,22,-1,-1,17,4,22,22,-1,-1,-1,1,-1,22,-1,-1,17,19,-1,-1,17,-1,-1,-1],"t":[0.5,7.930737337562887,7.679480099505446,0,0,4.5,16.844368517741394,0,0,1.792481250360578,5.108372929097653,12.199665042528586,9.40390385294334,0,0,0,2.782861472075366,0,16.725912049439096,0,0,5.065928480304397,0.5,0,0,6.219136028062415,0,0,0],"m":[1,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,0,10,11,12,13,0,0,0,17,0,19,0,0,22,23,0,0,26,0,0,0],"r":[28,5,4,0,0,9,8,0,0,21,16,15,14,0,0,0,18,0,20,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,-0.0002254485,-0.02604705,0,0,-0.0297874,0.004244366,0,0,0,0,0.03017014,-0.0008129859,0.03137929,0,0.01396942,0,-0.03410699,0.006646086,0,0,-0.002872328,-0.02871541,0,0.0276251,0.0006187469,-0.02074276]},{"f":[1,21,-1,32,33,33,33,21,22,-1,0,-1,-1,-1,15,-1,-1,37,33,-1,-1,-1,1,-1,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,10.463013322934223,21.225056427472516,17.737700742794445,13.660441030622824,3.5,14.03140051979143,0,2.5,0,0,0,7.266786540694901,0,0,7.92,20.752541988539424,0,0,0,8.980632551528075,0,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,8,9,0,11,0,0,0,15,0,0,18,19,0,0,0,23,0,0,0,27,0,0],"r":[26,3,0,25,22,17,14,13,10,0,12,0,0,0,16,0,0,21,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,-0.02914395,0,0,0,0,0,0,-0.02014119,0,-0.008700306,0.07864425,-4.540569e-05,0,-0.03124685,-0.002390379,0,0,-0.03399046,0.01579079,0.04109544,0,-0.001328528,-0.03205171,0.01555589,0,0.0004704381,0.02890218]},{"f":[23,21,15,16,-1,2,-1,-1,22,33,-1,-1,2,16,-1,-1,22,16,-1,18,4,-1,-1,-1,17,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,0,0,11.683871801979876,73.23848001446603,0,0,6.247802615595391,11.98370619265935,0,0,15.782972563647261,11.031356596255709,0,9.265614570460752,6.749860669831508,0,0,0,2.453445297804259,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,13,14,0,0,17,18,0,20,21,0,0,0,25,0,0,0,0],"r":[28,27,8,5,0,7,0,0,12,11,0,0,16,15,0,0,24,19,0,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.0005990482,0,0.03877404,0.001554693,0,0,-0.03079736,0.02424184,0,0,-0.03947643,0.003284262,0,0,0.002016877,0,0,0.0004564276,-0.04186025,-0.003473469,0,-0.006637059,0.02020296,0.01677374,-0.01872448]},{"f":[22,22,22,22,22,22,22,10,-1,-1,-1,-1,4,15,-1,-1,-1,-1,32,-1,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,0,0,0,3.9534452978042594,7.6653359171851765,0,0,0,0,2.69615871138938,0,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,13,14,0,0,0,0,19,0,0,0,23,0,25,26,0,0,0],"r":[22,21,18,17,12,11,10,9,0,0,0,0,16,15,0,0,0,0,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,0,-0.03384561,-6.219846e-05,0.02796024,-0.01420096,0,0,0.01173548,-0.04562418,0.0386632,-0.02157325,0,-0.01938285,0.02931987,-0.01040311,0,0.03933645,0,0,0.03064014,-0.02489054,-0.03289275]},{"f":[1,4,21,-1,15,2,22,-1,16,-1,-1,16,22,-1,-1,18,-1,-1,15,-1,22,-1,16,-1,-1,-1,3,-1,-1],"t":[2.01685992705101,9.266718101290426,3.5,0,9.269126679149418,6.434604125275413,7.082728314749225,0,9.250298417906333,0,0,2.0,9.024458927812285,0,0,1.292481250360578,0,0,10.342074667999139,0,11.722778233171695,0,11.031356596255709,0,0,0,0.0339080459770115,0,0],"m":[0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,12,13,0,0,16,0,0,19,0,21,0,23,0,0,0,27,0,0],"r":[26,25,4,0,18,11,8,0,10,0,0,15,14,0,0,17,0,0,20,0,22,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.0297471,0,0,0,0.005446147,0,-0.03597124,0.01151041,0,0,-0.009590379,0.02101905,0,-0.0005372139,-0.0194222,0,0.05442632,0,0.0142029,0,0.004948389,-0.03325814,-0.02087975,0,0.01219149,0.0002699354]},{"f":[36,-1,22,33,-1,-1,15,18,1,21,-1,-1,22,-1,-1,-1,16,27,17,-1,23,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,12.613886328427084,8.532371382375128,0,0,5.044394119358453,3.39067985676233,6.241071428571429,7.5,0,0,15.782972563647261,0,0,0,6.108524456778169,0.5,7.67595272073847,0,4.5,0,0,0,15.676331898425197,0,15.717370311433935,0,0],"m":[0,0,0,0,0,0,0,1,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,0,13,0,0,0,17,18,19,0,21,0,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,16,15,12,11,0,0,14,0,0,0,24,23,20,0,22,0,0,0,26,0,28,0,0],"v":[0,0.01488951,0,0,-0.03820586,-0.0006625095,0,0,0,0,-0.03913077,-0.001805941,0,0.02311472,-0.008063909,0.01970324,0,0,0,-0.03744182,0,0.0338147,-0.03602174,0.001631269,0,0.005500774,0,-0.04863694,0.000902261]},{"f":[22,22,22,22,16,-1,37,-1,-1,-1,32,-1,2,-1,-1,25,-1,22,-1,-1,36,-1,18,1,17,-1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,11.73,0,0,0,6.337096134072842,0,4.128693921346326,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,7.6141732283464565,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,10,9,6,0,8,0,0,0,12,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.03035862,0,-0.0001485749,0.01899811,-0.03375386,0,0.03242157,0,-0.02567146,0.007893181,0,-0.03673554,0,-0.02998947,0.008397111,0,0.04289446,0,0,0,-0.01223014,0.03694283,-0.01767824,-0.03306512]},{"f":[23,21,15,4,-1,-1,22,33,-1,-1,2,-1,22,16,15,-1,-1,18,4,-1,-1,-1,17,22,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,4.0437314206251695,0,0,11.683871801979876,73.23848001446603,0,0,3.903677461028802,0,15.782972563647261,11.031356596255709,10.462502272597265,0,0,9.265614570460752,5.169368191286958,0,0,0,4.208926257442949,16.725912049439096,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,13,14,15,0,0,18,19,0,0,0,23,24,0,0,0,0,0],"r":[28,27,6,5,0,0,10,9,0,0,12,0,22,17,16,0,0,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,-0.000954109,0.002807875,0,0,-0.0305722,0.0227498,0,-0.02905386,0,0,0,-0.01173313,0.01044533,0,0,0.005051833,-0.04199783,-0.003314509,0,0,-0.03508307,0.004414403,0.02159654,0.01599363,-0.0184517]},{"f":[1,22,15,18,17,22,3,4,-1,-1,15,15,-1,-1,1,-1,-1,-1,2,-1,-1,-1,-1,16,-1,-1,18,-1,-1],"t":[4.031939779270633,16.759160379664443,13.56045225766691,10.482680779086738,5.83200138144325,16.73920708776685,0.8516483516483516,3.753897320099348,0,0,5.78135971352466,2.584962500721156,0,0,1.987429289754871,0,0,0,5.065928480304397,0,0,0,0,7.924812503605781,0,0,10.415240582199587,0,0],"m":[1,1,1,1,1,1,0,1,0,0,1,1,0,0,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,0,11,12,0,0,15,0,0,0,19,0,0,0,0,24,0,0,27,0,0],"r":[26,23,22,21,18,17,10,9,0,0,14,13,0,0,16,0,0,0,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0.00454647,0.04033168,0,0,-0.003402182,0.01209681,0,-0.004514952,-0.03867778,-0.02527137,0,-0.03548722,-0.007789007,0.004439769,-0.03470454,0,-0.01421519,0.01986672,0,0.001108231,-0.02520512]},{"f":[36,-1,22,21,-1,-1,22,2,25,21,-1,15,16,17,-1,-1,-1,-1,33,-1,33,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,0,15.676331898425197,6.321815375530864,0.5,4.5,0,8.005624549193879,7.651051691178929,7.67595272073847,0,0,0,0,19.167172160526107,0,21.16864017939033,0,0,0,15.717370311433935,0,15.756821747580634,0,0],"m":[0,0,1,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,12,13,14,0,0,0,0,19,0,21,0,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,24,23,18,11,0,17,16,15,0,0,0,0,20,0,22,0,0,0,26,0,28,0,0],"v":[0,0.0144912,0,0,-0.02526506,-0.0007067007,0,0,0,0,0.05785902,0,0,0,-0.01237391,0.02727261,0.02820652,-0.01798242,0,-0.03044807,0,0.01480105,-0.009015611,-0.01362962,0,-0.04744306,0,0.02700996,-0.000275287]},{"f":[1,21,-1,24,36,-1,22,22,-1,-1,-1,21,22,1,-1,-1,0,-1,-1,-1,2,16,-1,21,-1,33,-1,-1,-1],"t":[9.107199999999999,2.5,0,7.5,3.5,0,16.765402642760364,16.725912049439096,0,0,0,3.5,14.03140051979143,6.069196428571429,0,0,2.5,0,0,0,6.118903736861568,2.807354922057604,0,5.5,0,22.152182868153965,0,0,0],"m":[1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,7,8,0,0,0,12,13,14,0,0,17,0,0,0,21,22,0,24,0,26,0,0,0],"r":[20,3,0,11,6,0,10,9,0,0,0,19,16,15,0,0,18,0,0,0,28,23,0,25,0,27,0,0,0],"v":[0,0,-0.02818292,0,0,-0.001124767,0,0,-0.02180734,0.01282015,-0.03191553,0,0,0,-0.0331883,0.005449223,0,-0.008514059,0.06932958,0.0001982928,0,0,0.02081909,0,-0.02956655,0,-0.002693192,0.01096304,0.0276481]},{"f":[22,22,22,22,22,22,22,-1,-1,-1,4,15,-1,-1,-1,-1,-1,25,-1,22,-1,-1,36,-1,18,2,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,3.9534452978042594,7.6653359171851765,0,0,0,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,0,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,16,15,10,9,8,0,0,0,14,13,0,0,0,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0002942564,0.02709838,-0.01374291,0,0,0.01176798,-0.04466282,0.03702947,-0.02104984,0.01341006,0,-0.03593011,0,-0.02975049,0.007513909,0,0.04190738,0,0,0.0309028,-0.01202722,-0.03258933]},{"f":[23,22,22,22,22,-1,-1,22,17,-1,22,-1,-1,22,-1,-1,25,-1,22,-1,-1,36,-1,18,2,-1,-1,-1,-1],"t":[7.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0,0,16.903290731167747,3.39067985676233,0,16.844368517741394,0,0,16.978341770814396,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,11,0,0,14,0,0,17,0,19,0,0,22,0,24,25,0,0,0,0],"r":[28,21,16,7,6,0,0,13,10,0,12,0,0,15,0,0,18,0,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,-3.52445e-05,-0.03300605,0,0,-0.01457058,0,0.005093953,0.05099494,0,-0.009183819,0.01375261,0,-0.03414116,0,-0.02933781,0.007421387,0,0.0422119,0,0,0.03518917,-0.01151126,-0.03208034,-0.01827722]},{"f":[4,16,-1,0,25,-1,-1,-1,32,19,20,-1,-1,-1,33,-1,2,16,-1,22,17,21,-1,-1,-1,-1,1,-1,-1],"t":[3.245926548164837,2.0,0,10.5,1.5,0,0,0,6.208926257442949,0.5,1.5,0,0,0,21.16864017939033,0,5.357122758833062,6.820178962415188,0,13.747003318603246,5.918419679874358,4.5,0,0,0,0,8.992857142857144,0,0],"m":[1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,1,1,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,0,0,0,9,10,11,0,0,0,15,0,17,18,0,20,21,22,0,0,0,0,27,0,0],"r":[8,3,0,7,6,0,0,0,14,13,12,0,0,0,16,0,26,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,-0.0345645,0,0,-0.0007873226,-0.01430889,0.01534646,0,0,0,-0.003207907,0.01700523,-0.02698362,0,0.02531228,0,0,-2.235858e-05,0,0,0,-0.008603364,0.03685534,-0.02775145,0.06462431,0,-0.0004173437,0.0217643]},{"f":[17,19,21,-1,22,22,24,-1,4,2,22,33,-1,22,-1,-1,-1,-1,-1,-1,-1,4,-1,-1,17,22,-1,-1,-1],"t":[2.160964047443681,1.5,4.5,0,16.765402642760364,16.725912049439096,4.5,0,7.856894618075133,7.543917866187989,15.782972563647261,13.249860669831508,0,13.64082212194147,0,0,0,0,0,0,0,4.128693921346326,0,0,2.69615871138938,16.752707246237605,0,0,0],"m":[0,1,0,0,1,1,0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,0,5,6,7,0,9,10,11,12,0,14,0,0,0,0,0,0,0,22,0,0,25,26,0,0,0],"r":[24,21,4,0,20,19,8,0,18,17,16,13,0,15,0,0,0,0,0,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,-0.02612836,0,0,0,-0.03591819,0,0,0,0,0.02135707,0,0.003933521,-0.03595072,-0.01079651,0.02500678,-0.007310479,0.0209034,-0.02511985,0,0.007800726,0.06144135,0,0,-0.04062761,0.002623101,-0.0002987259]},{"f":[4,16,-1,0,25,23,15,-1,-1,21,22,-1,-1,18,-1,21,-1,-1,15,-1,-1,-1,22,2,15,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,10.5,1.5,0.5,6.857980995127572,0,0,4.5,17.069460217922263,0,0,8.729619742794132,0,8.5,0,0,2.807354922057604,0,0,0,17.094543568094064,3.39067985676233,6.820178962415188,0,0,0,0],"m":[1,0,0,1,1,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0],"l":[1,2,0,4,5,6,7,0,0,10,11,0,0,14,0,16,0,0,19,0,0,0,23,24,25,0,0,0,0],"r":[22,3,0,21,18,9,8,0,0,13,12,0,0,15,0,17,0,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,0,-0.03407333,0,0,0,0,0.006434462,-0.01134249,0,0,-0.03729104,0.006434462,0,-0.0005879263,0,-0.04057551,0.006434462,0,0.0002733105,-0.02273548,0.01450832,0,0,0,0.005917388,0.04730994,0.0002249151,0.02664937]},{"f":[22,22,22,22,22,22,22,-1,-1,-1,23,16,-1,-1,-1,-1,32,-1,-1,25,-1,-1,36,-1,18,2,-1,-1,-1],"t":[17.083799521744545,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,3.5,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,0,0,17,0,0,20,0,0,23,0,25,26,0,0,0],"r":[22,19,16,15,10,9,8,0,0,0,14,13,0,0,0,0,18,0,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0002689262,0.02652874,-0.01289692,0,0,0.01255175,-0.03653024,0.03988442,-0.01983948,0,-0.01911446,0.02693024,0,-0.02439305,-0.0004113575,0,0.03957781,0,0,0.02952421,-0.01125484,-0.03117568]},{"f":[25,0,21,-1,33,11,22,22,-1,-1,-1,-1,25,2,22,36,32,-1,-1,-1,-1,-1,-1,21,16,-1,-1,-1,-1],"t":[6.5,10.5,2.5,0,21.16864017939033,0.5,11.67903991803059,10.27595502635321,0,0,0,0,1.5,4.699371845969097,15.676331898425197,3.5,6.337096134072842,0,0,0,0,0,0,8.5,2.807354922057604,0,0,0,0],"m":[1,1,0,0,1,1,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,0,0,0,13,14,15,16,17,0,0,0,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,12,11,10,9,0,0,0,0,22,21,20,19,18,0,0,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.02738359,0,0,0,0,0.0009149088,-0.02071366,0.001871156,-0.02812344,0,0,0,0,0,-0.007981284,-0.04038152,0.001871156,-0.0209522,0.001679905,-0.0241815,0,0,0.02876611,0.001871156,0.03091676,-0.01585394]},{"f":[22,22,22,22,22,22,-1,-1,23,16,-1,-1,-1,-1,-1,25,-1,22,-1,-1,36,-1,18,1,17,-1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,0,0,5.5,6.0,0,0,0,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,7.6141732283464565,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,9,10,0,0,0,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,14,13,8,7,0,0,12,11,0,0,0,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,0,0.0001397589,-0.01247359,0,0,0.01216983,-0.03579127,0.03879286,-0.01911321,0.01186519,0,-0.03404331,0,-0.02917847,0.006650592,0,0.03857237,0,0,0,-0.01275578,0.03537248,-0.01576702,-0.03077856]},{"f":[32,15,22,20,-1,-1,1,16,-1,-1,22,-1,-1,15,-1,-1,33,20,-1,2,25,-1,-1,-1,0,25,-1,-1,-1],"t":[6.208926257442949,5.044394119358453,12.613886328427084,0.5,0,0,6.241071428571429,7.417852514885898,0,0,15.782972563647261,0,0,6.169925001442312,0,0,21.16864017939033,1.5,0,2.453445297804259,1.5,0,0,0,11.5,1.5,0,0,0],"m":[0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,0,14,0,0,17,18,0,20,21,0,0,0,25,26,0,0,0],"r":[16,13,6,5,0,0,10,9,0,0,12,0,0,15,0,0,24,19,0,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0,-0.03800288,0.0001975778,0,0,-0.03606196,0.003321249,0,0.01774582,-0.008016322,0,-0.03009095,-0.00198731,0,0,-0.006672178,0,0,0.00967076,-0.01403249,0.03567468,0,0,-0.0002387131,-0.01990208,0.01798672]},{"f":[4,18,16,-1,-1,23,1,22,-1,15,17,-1,-1,-1,25,-1,-1,21,22,35,-1,-1,17,-1,-1,2,-1,-1,-1],"t":[3.084962500721156,2.160964047443681,2.0,0,0,0.5,5.015625,11.67903991803059,0,6.820178962415188,3.5221970596792267,0,0,0,1.5,0,0,8.5,17.094543568094064,1.5,0,0,8.729619742794132,0,0,1.292481250360578,0,0,0],"m":[1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,10,11,0,0,0,15,0,0,18,19,20,0,0,23,0,0,26,0,0,0],"r":[28,5,4,0,0,17,14,9,0,13,12,0,0,0,16,0,0,25,22,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,-0.02965453,0.001710067,0,0,0,-0.01496299,0,0,0.0374878,-0.02034454,0.07531467,0,0.0004620555,-0.02440106,0,0,0,-0.004668929,-0.04032762,0,0.03136271,-0.03021892,0,0.02509902,-0.02314448,0.0007884743]},{"f":[32,22,33,22,22,18,-1,-1,-1,22,-1,-1,1,-1,-1,-1,33,20,-1,2,25,-1,-1,-1,0,25,-1,-1,-1],"t":[6.208926257442949,16.838461650489297,13.660441030622824,15.782972563647261,14.03140051979143,1.292481250360578,0,0,0,16.725912049439096,0,0,8.245481927710843,0,0,0,21.16864017939033,1.5,0,2.453445297804259,1.5,0,0,0,11.5,1.5,0,0,0],"m":[0,1,0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,0,0,13,0,0,0,17,18,0,20,21,0,0,0,25,26,0,0,0],"r":[16,15,12,9,8,7,0,0,0,11,0,0,14,0,0,0,24,19,0,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.01497974,-0.001061518,0.01936158,0,-0.01711024,0.004103512,0,-0.03231887,-7.980924e-05,0.01958767,0,0,-0.006422545,0,0,0.009201105,-0.01279095,0.03420512,0,0,-0.0002290338,-0.01876179,0.01713414]},{"f":[23,21,15,16,-1,-1,22,33,-1,-1,2,-1,22,16,15,-1,-1,18,4,-1,-1,-1,17,22,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,0,11.683871801979876,73.23848001446603,0,0,3.903677461028802,0,15.782972563647261,11.031356596255709,10.462502272597265,0,0,9.265614570460752,5.169368191286958,0,0,0,4.208926257442949,16.725912049439096,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,13,14,15,0,0,18,19,0,0,0,23,24,0,0,0,0,0],"r":[28,27,6,5,0,0,10,9,0,0,12,0,22,17,16,0,0,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,-0.0006087073,0.003411418,0,0,-0.03076971,0.02150358,0,-0.02745154,0,0,0,-0.01127498,0.01007587,0,0,0.004301979,-0.04139095,-0.003193003,0,0,-0.03325891,0.002609342,0.02016668,0.01511057,-0.0176329]},{"f":[4,16,-1,18,18,21,24,-1,-1,-1,1,-1,-1,16,16,-1,-1,15,-1,17,1,-1,1,-1,-1,2,-1,-1,-1],"t":[3.084962500721156,2.0,0,2.160964047443681,1.792481250360578,4.5,6.5,0,0,0,8.046875,0,0,5.044394119358453,3.321928094887362,0,0,6.169925001442312,0,10.620215166896223,4.54375,0,5.015625,0,0,1.292481250360578,0,0,0],"m":[1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,7,0,0,0,11,0,0,14,15,0,0,18,0,20,21,0,23,0,0,26,0,0,0],"r":[28,3,0,13,10,9,8,0,0,0,12,0,0,17,16,0,0,19,0,25,22,0,24,0,0,27,0,0,0],"v":[0,0,-0.03320279,0,0,0,0,0.01760705,-0.03602775,0.001729622,0,0.02838924,-0.002256033,0,0,-0.01611914,0.03850315,0,-0.03690258,0,0,-0.0378286,0,0.03244263,-0.007398219,0,0.03264725,-0.02160085,0.0007683509]},{"f":[22,22,22,22,22,-1,23,15,-1,-1,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,1,17,-1,-1,-1,-1],"t":[17.083799521744545,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,7.6141732283464565,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,12,11,6,0,10,9,0,0,0,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0002440042,0,0,0.01807521,-0.03117506,0.03770076,-0.01839873,0,-0.01860192,0.0252875,0,-0.02328365,0,-0.02625701,0.006448386,0,0.03765662,0,0,0,-0.01223394,0.0343222,-0.01510918,-0.02981399]},{"f":[32,22,33,22,22,-1,-1,22,-1,-1,1,-1,-1,-1,33,20,-1,2,25,-1,-1,-1,0,25,4,-1,-1,-1,-1],"t":[6.208926257442949,16.838461650489297,13.660441030622824,15.782972563647261,14.03140051979143,0,0,16.725912049439096,0,0,8.245481927710843,0,0,0,21.16864017939033,1.5,0,2.453445297804259,1.5,0,0,0,11.5,1.5,3.084962500721156,0,0,0,0],"m":[0,1,0,1,1,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,0,0,0,15,16,0,18,19,0,0,0,23,24,25,0,0,0,0],"r":[14,13,10,7,6,0,0,9,0,0,12,0,0,0,22,17,0,21,20,0,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.002888478,0.01852643,0,-0.01618932,0.003759426,0,-0.03154887,-3.582095e-05,0.01857665,0,0,-0.006275131,0,0,0.009095738,-0.01214284,0.03281602,0,0,0,-0.002590961,0.001066631,-0.01786807,0.01649808]},{"f":[1,22,25,-1,-1,16,-1,-1,18,22,22,1,22,-1,23,2,-1,-1,18,-1,16,-1,20,-1,-1,-1,-1,-1,-1],"t":[4.031939779270633,16.759160379664443,1.5,0,0,7.924812503605781,0,0,10.415240582199587,15.676331898425197,15.439474430023438,7.772727272727273,11.775312925643524,0,0.5,2.160964047443681,0,0,5.339740049752724,0,7.442943495848728,0,0.5,0,0,0,0,0,0],"m":[1,1,0,0,0,1,0,0,1,1,1,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,13,0,15,16,0,0,19,0,21,0,23,0,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,27,26,25,14,0,18,17,0,0,20,0,22,0,24,0,0,0,0,0,0],"v":[0,0,0,-0.02225745,-0.001783778,0,-0.01416877,0.01830887,0,0,0,0,0,-0.007821517,0,0,0.009316643,0.07134654,0,-0.0003091048,0,-0.0238768,0,0.01152414,0.05313715,-0.002465099,0.01802416,-0.002181439,-0.02429624]},{"f":[22,22,22,22,22,-1,4,15,-1,-1,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,2,17,-1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,16.975010474608837,16.844368517741394,0,3.9534452978042594,7.6653359171851765,0,0,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,12,11,6,0,10,9,0,0,0,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0002095625,0,0,0.01113905,-0.0422334,0.0326599,-0.01765068,0,-0.02058615,0.01557893,0,-0.03286851,0,-0.02861173,0.005806409,0,0.03646748,0,0,0,-0.006789821,0.03728963,-0.0101863,-0.02937043]},{"f":[23,22,22,22,22,16,-1,-1,-1,22,-1,32,-1,22,-1,-1,22,-1,18,-1,-1,36,-1,18,2,-1,-1,-1,-1],"t":[7.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,16.903290731167747,0,2.69615871138938,0,16.978341770814396,0,0,17.078739781790915,0,6.118903736861568,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,0,12,0,14,0,0,17,0,19,0,0,22,0,24,25,0,0,0,0],"r":[28,21,16,9,8,7,0,0,0,11,0,13,0,15,0,0,18,0,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.02935135,0.0001593727,-0.03105675,0,0.02826367,0,-0.02837601,0,-0.006207359,0.01523199,0,-0.02958067,0,0.006355457,-0.02579155,0,0.03699422,0,0,0.03228821,-0.009744283,-0.02890147,-0.01753425]},{"f":[23,21,15,16,-1,2,22,-1,-1,4,-1,-1,33,0,16,-1,22,-1,22,21,-1,16,-1,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,12.613886328427084,0,0,8.049190845056867,0,0,35.98606694718995,9.5,10.501837184902296,0,11.683871801979876,0,15.782972563647261,6.5,0,11.031356596255709,0,0,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,15,0,17,0,19,20,0,22,0,0,0,0,0,0,0],"r":[28,27,12,5,0,9,8,0,0,11,0,0,26,25,16,0,18,0,24,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,-0.0005924223,0,0,0.004955928,0.0555374,0,0.002679129,-0.02121981,0,0,0,-0.0358792,0,0.002420917,0,0,-0.03996886,0,0.002420917,-0.01410616,0.002420917,0.01065069,-0.02238214,0.0144151,-0.01720937]},{"f":[25,17,19,21,-1,22,4,-1,-1,-1,2,2,17,32,-1,-1,1,-1,-1,-1,24,-1,-1,17,16,-1,-1,-1,-1],"t":[6.5,2.160964047443681,1.5,4.5,0,16.765402642760364,7.856894618075133,0,0,0,2.69615871138938,1.792481250360578,1.292481250360578,2.69615871138938,0,0,8.046875,0,0,0,11.5,0,0,2.69615871138938,5.672425341971495,0,0,0,0],"m":[1,0,1,0,0,1,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,0,11,12,13,14,0,0,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,10,5,0,9,8,0,0,0,20,19,16,15,0,0,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.0248033,0,0,0.005262183,-0.006870847,-0.02500953,0,0,0,0,-0.03779576,0.02527195,0,0.0500981,0.00290769,-0.01647936,0,-0.0002554258,0.06434647,0,0,0.004598065,-0.03823436,-0.0002876117,-0.01515024]},{"f":[22,22,22,22,22,-1,23,15,-1,-1,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,1,17,-1,-1,-1,-1],"t":[17.083799521744545,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,7.6141732283464565,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,12,11,6,0,10,9,0,0,0,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0001988777,0,0,0.01686162,-0.03011173,0.03576517,-0.01668208,0,-0.01769169,0.02298584,0,-0.02219739,0,-0.02528363,0.005642732,0,0.03485524,0,0,0,-0.01180912,0.03301565,-0.01388192,-0.02858068]},{"f":[23,22,22,22,22,16,-1,-1,-1,22,-1,32,-1,22,-1,-1,22,-1,18,-1,-1,36,-1,18,2,-1,-1,-1,-1],"t":[7.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,16.903290731167747,0,2.69615871138938,0,16.978341770814396,0,0,17.078739781790915,0,6.118903736861568,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,0,12,0,14,0,0,17,0,19,0,0,22,0,24,25,0,0,0,0],"r":[28,21,16,9,8,7,0,0,0,11,0,13,0,15,0,0,18,0,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.02897547,0.0001624062,-0.03044849,0,0.02682469,0,-0.02757954,0,-0.005928615,0.01427496,0,-0.02873263,0,0.006076068,-0.02516378,0,0.03526522,0,0,0.03122327,-0.009199561,-0.02812298,-0.0170479]},{"f":[23,22,21,15,4,-1,-1,18,22,3,-1,21,-1,22,-1,-1,17,22,-1,22,-1,-1,-1,23,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,10.5,10.457380879072534,4.128693921346326,0,0,10.680359255641626,15.782972563647261,0.15991874474642756,0,6.5,0,15.212004812197344,0,0,4.491496787347155,16.725912049439096,0,16.765402642760364,0,0,0,0.5,0,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,10,0,12,0,14,0,0,17,18,0,20,0,0,0,24,0,0,0,0,0],"r":[28,27,26,7,6,0,0,23,16,11,0,13,0,15,0,0,22,19,0,21,0,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.0009888626,0.002588299,0,0,0,0.003848945,0,-0.04326574,0,-0.01326948,0.01495182,0,0,-0.03343784,0,0.01086603,-0.02286115,0.01973809,0,0.02481359,-0.01719959,0.01373303,0.01639651,-0.01673274]},{"f":[36,22,33,-1,-1,22,-1,22,3,-1,-1,25,16,16,-1,-1,22,2,25,-1,-1,-1,22,-1,-1,-1,1,-1,-1],"t":[5.5,12.613886328427084,8.532371382375128,0,0,12.921158001713904,0,13.589275375459419,0.05277777777777778,0,0,3.5,6.108524456778169,5.700439718141092,0,0,15.676331898425197,5.8698903048866296,0.5,0,0,0,15.717370311433935,0,0,0,8.992857142857144,0,0],"m":[1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,9,0,0,12,13,14,0,0,17,18,19,0,0,0,23,0,0,0,27,0,0],"r":[26,5,4,0,0,7,0,11,10,0,0,25,16,15,0,0,22,21,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.03365481,-0.0008517176,0,0.009899635,0,0,0.03172948,-0.02746437,0,0,0,-0.00222406,-0.03203401,0,0,0,0.01932599,-0.001266322,-0.0114306,0,-0.04593816,0.001437757,0.02613119,0,-0.007694907,0.03033718]},{"f":[25,17,19,21,-1,22,4,-1,-1,-1,2,2,17,32,-1,-1,33,-1,-1,-1,24,-1,-1,17,16,-1,-1,-1,-1],"t":[6.5,2.160964047443681,1.5,4.5,0,16.765402642760364,7.856894618075133,0,0,0,2.69615871138938,1.792481250360578,1.292481250360578,2.69615871138938,0,0,9.184798673139337,0,0,0,9.5,0,0,2.69615871138938,5.672425341971495,0,0,0,0],"m":[1,0,1,0,0,1,1,0,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,0,11,12,13,14,0,0,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,10,5,0,9,8,0,0,0,20,19,16,15,0,0,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02413015,0,0,0.004997364,-0.00677334,-0.02353784,0,0,0,0,-0.03727882,0.0241627,0,0.04977472,0.004352348,-0.01598046,0,-0.001347914,0.05937283,0,0,0.004449632,-0.03759374,-0.0002935407,-0.01526205]},{"f":[36,-1,18,-1,18,-1,15,22,-1,-1,16,16,-1,22,-1,-1,2,22,21,-1,-1,-1,33,-1,17,17,-1,-1,-1],"t":[1.5,0,7.723832212141186,0,8.03615005796417,0,2.0,11.722778233171695,0,0,9.92481250360578,6.584962500721156,0,11.424679641461715,0,0,1.292481250360578,14.247293309538719,6.5,0,0,0,34.303028789183955,0,9.270294853526895,9.265614570460752,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,0,7,8,0,0,11,12,0,14,0,0,17,18,19,0,0,0,23,0,25,26,0,0,0],"r":[2,0,4,0,6,0,10,9,0,0,16,13,0,15,0,0,22,21,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0.01303046,0,-0.0005658297,0,0.0413735,0,0,-0.002869389,0.01978187,0,0,-0.005232505,0,0.01085328,-0.03648865,0,0,0,-0.01069603,0.05452268,-0.01665529,0,0.01951536,0,0,-0.008433096,0.03822161,-0.01375019]},{"f":[25,17,19,21,-1,22,22,22,4,15,-1,22,-1,-1,-1,25,-1,-1,-1,-1,2,2,-1,-1,-1,17,-1,-1,-1],"t":[6.5,2.160964047443681,1.5,4.5,0,16.765402642760364,16.725912049439096,15.782972563647261,7.856894618075133,6.022367813028454,0,13.64082212194147,0,0,0,0.5,0,0,0,0,2.69615871138938,1.792481250360578,0,0,0,2.69615871138938,0,0,0],"m":[1,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,0,12,0,0,0,16,0,0,0,0,21,22,0,0,0,26,0,0,0],"r":[28,25,20,5,0,19,18,15,14,11,0,13,0,0,0,17,0,0,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,-0.02361886,0,0,0,0,0,0.01633615,0,0.003789626,-0.03399917,-0.00644698,0,-0.01879727,0.02903499,0.0182728,-0.0226865,0,0,0.01126917,-0.01551672,0.04500507,0,-0.02469458,-0.0002707668,-0.01491889]},{"f":[22,22,22,22,22,-1,4,-1,-1,-1,32,-1,-1,25,33,-1,-1,22,-1,-1,36,-1,18,2,17,-1,-1,-1,-1],"t":[17.083799521744545,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,0,3.9534452978042594,0,0,0,2.69615871138938,0,0,3.5,34.128137685572966,0,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,0,0,14,15,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,13,10,9,6,0,8,0,0,0,12,0,0,17,16,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0001952333,0,-0.02350132,0.03068452,-0.01569596,0,-0.01694218,0.02149412,0,0,-0.03554215,-0.006445065,0,-0.02472433,0.005319961,0,0.03330506,0,0,0,-0.006927613,0.0355758,-0.008751248,-0.0277367]},{"f":[1,22,25,-1,18,18,-1,22,21,15,-1,17,-1,-1,-1,-1,-1,16,-1,1,5,-1,-1,-1,18,1,-1,-1,-1],"t":[4.031939779270633,16.759160379664443,1.5,0,10.482680779086738,5.491496787347155,0,15.212004812197344,7.5,12.711021616791836,0,9.265614570460752,0,0,0,0,0,7.924812503605781,0,3.172945205479452,0.5,0,0,0,10.415240582199587,4.145833333333334,0,0,0],"m":[1,1,0,0,1,1,0,1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,9,10,0,12,0,0,0,0,0,18,0,20,21,0,0,0,25,26,0,0,0],"r":[24,17,4,0,16,7,0,15,14,11,0,13,0,0,0,0,0,19,0,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,-0.0217266,0,0,-0.001840272,0,0,0,-0.03843723,0,-0.02837423,0.01957583,0.0002296663,0.01793702,0.004021918,0,-0.01360556,0,0,0.007667737,0.04612837,0.00746323,0,0,0.02217102,0.0007627109,-0.02319038]},{"f":[36,-1,32,32,37,22,22,18,4,-1,21,22,2,-1,-1,-1,-1,-1,25,-1,22,-1,-1,-1,-1,-1,18,-1,-1],"t":[1.5,0,10.463013322934223,10.41931383093672,12.52,17.083799521744545,17.02942582914998,7.723832212141186,5.505613627711627,0,7.5,15.782972563647261,7.128362343532883,0,0,0,0,0,3.5,0,17.069460217922263,0,0,0,0,0,3.5221970596792267,0,0],"m":[0,0,1,1,1,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,11,12,13,0,0,0,0,0,19,0,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,24,23,18,17,10,0,16,15,14,0,0,0,0,0,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0.01285817,0,0,0,0,0,0,0,0.0004645854,0,0,0,-0.03973472,-0.002708061,0.008167785,-0.001001435,0.00374324,0,-0.03042655,0,-0.02708706,0.005065974,0.008167785,0.01650307,-0.009464405,0,0.01482556,-0.01859576]},{"f":[4,16,-1,18,17,-1,0,15,-1,-1,-1,23,1,22,-1,-1,33,-1,-1,21,22,35,-1,-1,-1,2,-1,-1,-1],"t":[3.084962500721156,2.0,0,2.160964047443681,0.5,0,5.5,7.930737337562887,0,0,0,0.5,5.015625,11.67903991803059,0,0,17.938048609286376,0,0,8.5,17.07907323966382,1.5,0,0,0,1.292481250360578,0,0,0],"m":[1,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,1,0,0,0],"l":[1,2,0,4,5,0,7,8,0,0,0,12,13,14,0,0,17,0,0,20,21,22,0,0,0,26,0,0,0],"r":[28,3,0,11,6,0,10,9,0,0,0,19,16,15,0,0,18,0,0,25,24,23,0,0,0,27,0,0,0],"v":[0,0,-0.03224973,0,0,0.01857331,0,0,-0.03967426,-0.007577726,0.002390075,0,0,0,-0.01324479,0.05386171,0,-0.0359599,-0.001833894,0,0,0,-0.003690879,-0.03904348,0.009167576,0,0.02151365,-0.02134234,0.0007247974]},{"f":[25,17,19,21,-1,22,22,22,4,15,-1,-1,-1,25,-1,-1,-1,-1,2,2,-1,-1,-1,17,16,-1,-1,-1,-1],"t":[6.5,2.160964047443681,1.5,4.5,0,16.765402642760364,16.725912049439096,15.782972563647261,7.856894618075133,6.022367813028454,0,0,0,0.5,0,0,0,0,2.69615871138938,1.792481250360578,0,0,0,2.69615871138938,5.672425341971495,0,0,0,0],"m":[1,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,0,0,0,14,0,0,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[28,23,18,5,0,17,16,13,12,11,0,0,0,15,0,0,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02304816,0,0,0,0,0,0.01546698,-6.83899e-05,-0.006040092,0,-0.0179384,0.0281367,0.01774904,-0.02181204,0,0,0.01067945,-0.01517634,0.04229225,0,0,0.00524275,-0.03684096,-0.0002902153,-0.0147579]},{"f":[1,22,25,-1,-1,16,-1,-1,18,22,22,1,22,22,-1,-1,18,23,2,-1,-1,-1,15,-1,-1,-1,-1,-1,-1],"t":[4.031939779270633,16.759160379664443,1.5,0,0,7.924812503605781,0,0,10.415240582199587,15.676331898425197,15.439474430023438,7.772727272727273,11.775312925643524,8.530347965843777,0,0,5.339740049752724,0.5,2.453445297804259,0,0,0,6.820178962415188,0,0,0,0,0,0],"m":[1,1,0,0,0,1,0,0,1,1,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,13,14,0,0,17,18,19,0,0,0,23,0,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,27,26,25,16,15,0,0,22,21,20,0,0,0,24,0,0,0,0,0,0],"v":[0,0,0,-0.02118453,-0.001659795,0,-0.01302399,0.01645274,0,0,0,0,0,0,0.001446704,-0.03127928,0,0,0,0.006392764,0.0578416,-0.0001175846,0,-0.01194249,0.04073813,-0.002241079,0.01611131,-0.002102261,-0.02272082]},{"f":[36,-1,22,21,-1,-1,22,2,25,15,21,-1,17,1,-1,-1,-1,-1,33,-1,33,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,0,15.676331898425197,6.321815375530864,0.5,8.005624549193879,4.5,0,7.67595272073847,4.949128919860627,0,0,0,0,19.167172160526107,0,21.16864017939033,0,0,0,15.717370311433935,0,15.756821747580634,0,0],"m":[0,0,1,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,0,13,14,0,0,0,0,19,0,21,0,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,24,23,18,17,12,0,16,15,0,0,0,0,20,0,22,0,0,0,26,0,28,0,0],"v":[0,0.01261551,0,0,-0.0248341,-0.0004877547,0,0,0,0,0,0.04680855,0,0,-0.02774555,0.01956807,0.03765899,-0.01748867,0,-0.02682827,0,0.01328253,-0.007032233,-0.0122124,0,-0.04446373,0,0.0261126,-0.0003434513]},{"f":[4,16,-1,15,17,22,-1,-1,37,18,-1,23,-1,-1,18,22,22,18,-1,-1,-1,-1,-1,22,-1,-1,22,-1,-1],"t":[3.245926548164837,2.0,0,12.000704269011246,0.5,15.717370311433935,0,0,5.05,1.292481250360578,0,3.5,0,0,2.160964047443681,16.765402642760364,16.73920708776685,1.792481250360578,0,0,0,0,0,13.930814526604713,0,0,17.094543568094064,0,0],"m":[1,0,0,1,0,1,0,0,1,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,0,9,10,0,12,0,0,15,16,17,18,0,0,0,0,0,24,0,0,27,0,0],"r":[26,3,0,23,8,7,0,0,14,11,0,13,0,0,22,21,20,19,0,0,0,0,0,25,0,0,28,0,0],"v":[0,0,-0.03188048,0,0,0,0.005489266,0.04767831,0,0,-0.03808363,0,-0.006245923,-0.03046873,0,0,0,0,0.002037554,0.01483946,-0.006245923,0.01997592,-0.005020823,0,0.04281369,-0.01350057,0,0.0005774775,0.02346037]},{"f":[25,17,3,-1,1,21,2,-1,-1,22,0,-1,-1,21,-1,-1,1,-1,22,22,-1,-1,24,-1,-1,17,-1,-1,-1],"t":[6.5,2.160964047443681,0.12256006006006007,0,5.3560606060606055,6.5,6.209321624687949,0,0,10.27595502635321,2.5,0,0,7.5,0,0,6.015625,0,15.782972563647261,15.439474430023438,0,0,6.5,0,0,2.69615871138938,0,0,0],"m":[1,0,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,0,0,14,0,0,17,0,19,20,0,0,23,0,0,26,0,0,0],"r":[28,25,4,0,16,9,8,0,0,13,12,0,0,15,0,0,18,0,22,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0.02018299,0,0,0,-0.0386854,0.004698478,0,0,0.002300318,0.03185257,0,-0.02725084,-0.002151736,0,0.03151299,0,0,0.0009147154,0.01617029,0,0.0261104,-0.01387582,0,-0.02352998,-0.0002845665,-0.01441598]},{"f":[4,16,-1,15,17,22,-1,-1,0,-1,22,22,7,3,-1,23,-1,-1,-1,-1,16,-1,-1,22,-1,-1,22,-1,-1],"t":[3.245926548164837,2.0,0,12.000704269011246,0.5,15.717370311433935,0,0,2.5,0,16.73920708776685,16.725912049439096,0.5,0.013290990355210539,0,3.5,0,0,0,0,5.169925001442312,0,0,13.930814526604713,0,0,17.094543568094064,0,0],"m":[1,0,0,1,0,1,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,0,9,0,11,12,13,14,0,16,0,0,0,0,21,0,0,24,0,0,27,0,0],"r":[26,3,0,23,8,7,0,0,10,0,20,19,18,15,0,17,0,0,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,-0.03143667,0,0,0,0.005285641,0.04639147,0,-0.02653058,0,0,0,0,0.01213223,0,-0.0003657774,-0.01070984,-0.0139461,0.02048465,0,0.02962974,-0.01197582,0,0.04022961,-0.01317249,0,0.0005694225,0.02282868]},{"f":[32,32,37,25,22,22,22,22,16,-1,-1,-1,2,15,-1,-1,-1,25,-1,22,-1,-1,-1,-1,-1,-1,18,-1,-1],"t":[10.463013322934223,10.41931383093672,12.52,6.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,4.128693921346326,4.087462841250339,0,0,0,3.5,0,17.069460217922263,0,0,0,0,0,0,3.5221970596792267,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,13,14,0,0,0,18,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,24,23,22,17,12,11,10,0,0,0,16,15,0,0,0,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,-0.02589777,0.0001678156,-0.03004744,0,0,0.008212787,-0.03194411,0.008212787,0,-0.0297227,0,-0.02693562,0.004827883,0.008212787,-0.02376206,0.01650675,-0.009061229,0,0.01421445,-0.01823891]},{"f":[36,-1,22,21,-1,22,22,33,-1,23,-1,-1,21,-1,-1,16,-1,22,-1,2,22,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,13.129761635363703,12.613886328427084,8.532371382375128,0,2.5,0,0,7.5,0,0,6.832890014164741,0,13.589275375459419,0,2.69615871138938,13.930814526604713,0,0,0,15.676331898425197,0,15.717370311433935,0,0],"m":[0,0,1,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,10,0,0,13,0,0,16,0,18,0,20,21,0,0,0,25,0,27,0,0],"r":[2,0,24,5,0,15,12,9,0,11,0,0,14,0,0,17,0,19,0,23,22,0,0,0,26,0,28,0,0],"v":[0,0.01220917,0,0,-0.02409089,0,0,0,-0.03618505,0,0.0006346911,-0.01362974,0,0.01533413,-0.002257378,0,-0.03969738,0,-0.0229445,0,0,0.03573857,-0.01733003,-0.003465642,0,0.004971389,0,-0.04288695,0.001041905]},{"f":[32,15,22,20,-1,-1,1,16,-1,-1,22,-1,-1,15,-1,-1,33,20,-1,2,-1,-1,0,1,-1,37,-1,-1,-1],"t":[6.208926257442949,5.044394119358453,12.613886328427084,0.5,0,0,6.241071428571429,7.417852514885898,0,0,15.782972563647261,0,0,6.169925001442312,0,0,21.16864017939033,1.5,0,2.453445297804259,0,0,11.5,8.980632551528075,0,9.205,0,0,0],"m":[0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,0,14,0,0,17,18,0,20,0,0,23,24,0,26,0,0,0],"r":[16,13,6,5,0,0,10,9,0,0,12,0,0,15,0,0,22,19,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,-0.0362492,0.0001357379,0,0,-0.03398597,0.002689093,0,0.01479019,-0.005806546,0,-0.02727202,-0.001577577,0,0,-0.005802042,0,0.002663439,0.03022486,0,0,-7.066511e-06,0,-0.02770138,0.005836702,0.01512042]},{"f":[36,22,33,-1,-1,22,-1,22,-1,25,16,16,-1,-1,22,2,25,-1,-1,-1,22,-1,22,-1,-1,-1,1,-1,-1],"t":[5.5,12.613886328427084,8.532371382375128,0,0,13.129761635363703,0,13.589275375459419,0,3.5,6.108524456778169,5.700439718141092,0,0,15.676331898425197,5.8698903048866296,0.5,0,0,0,15.717370311433935,0,15.756821747580634,0,0,0,8.992857142857144,0,0],"m":[1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,0,10,11,12,0,0,15,16,17,0,0,0,21,0,23,0,0,0,27,0,0],"r":[26,5,4,0,0,7,0,9,0,25,14,13,0,0,20,19,18,0,0,0,22,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.03182236,-0.0007828596,0,0.008368893,0,-0.02736078,0,0,0,-0.002160545,-0.03025718,0,0,0,0.01682621,-0.001269217,-0.01057397,0,-0.04136889,0,0.02518297,-0.0004440288,0.02486866,0,-0.007279599,0.0293479]},{"f":[16,-1,22,22,22,19,16,4,-1,-1,1,25,-1,-1,16,-1,-1,33,-1,2,-1,-1,-1,25,-1,-1,2,-1,-1],"t":[9.92481250360578,0,16.759160379664443,16.73920708776685,16.725912049439096,0.5,10.681238411777805,6.266299719336492,0,0,4.030980086372361,0.5,0,0,12.225508720236002,0,0,35.98606694718995,0,6.942466823974881,0,0,0,1.5,0,0,6.011183906514227,0,0],"m":[1,0,1,1,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,12,0,0,15,0,0,18,0,20,0,0,0,24,0,0,27,0,0],"r":[2,0,26,23,22,17,10,9,0,0,14,13,0,0,16,0,0,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,-0.0008160621,0,0,0,0,0,0,0.04764705,-0.002022995,0,0,-0.03386935,-0.00233711,0,0.0379485,-0.0005453786,0,0.003127101,0,-0.04038036,-0.01183358,0.03146734,0,-0.03823973,0.0001627594,0,0.06018883,0.007404572]},{"f":[4,16,-1,17,22,-1,-1,0,-1,15,-1,-1,32,19,-1,-1,33,-1,2,16,21,-1,-1,22,17,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,15.717370311433935,0,0,2.5,0,12.000704269011246,0,0,6.208926257442949,0.5,0,0,21.16864017939033,0,5.357122758833062,6.820178962415188,3.5,0,0,14.03140051979143,5.918419679874358,0,0,0,0],"m":[1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0],"l":[1,2,0,4,5,0,0,8,0,10,0,0,13,14,0,0,17,0,19,20,21,0,0,24,25,0,0,0,0],"r":[12,3,0,7,6,0,0,9,0,11,0,0,16,15,0,0,18,0,28,23,22,0,0,27,26,0,0,0,0],"v":[0,0,-0.03098857,0,0,0.00506188,0.04363966,0,-0.02341522,0,-0.001336491,0.02620222,0,0,-0.001033395,-0.02291685,0,0.01871044,0,0,0,-0.01406246,0.007581998,0,0,0.02065052,-0.026709,0.05568416,0.0001094698]},{"f":[22,-1,22,15,-1,-1,22,-1,15,23,18,18,-1,-1,21,-1,-1,-1,15,2,-1,-1,18,16,18,-1,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,2.584962500721156,0,0,8.133841411710264,0,2.0,2.5,10.269270673170688,2.453445297804259,0,0,6.5,0,0,0,2.584962500721156,1.292481250360578,0,0,11.694748780085186,9.92481250360578,7.383290334558488,0,0,0,0],"m":[0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,11,12,0,0,15,0,0,0,19,20,0,0,23,24,25,0,0,0,0],"r":[2,0,6,5,0,0,8,0,18,17,14,13,0,0,16,0,0,0,22,21,0,0,28,27,26,0,0,0,0],"v":[0,-0.01048992,0,0,-0.008350996,0.009516204,0,-0.02540252,0,0,0,0,0.0002736101,-0.02411812,0,0.01510917,0.0005031756,0.0257099,0,0,0.02439665,-0.02427584,0,0,0,-0.0004306658,-0.01468085,0.00174774,-0.02108148]},{"f":[22,22,22,22,22,22,22,22,-1,-1,-1,-1,23,16,-1,-1,-1,-1,32,-1,-1,22,-1,-1,25,-1,27,-1,-1],"t":[17.094543568094064,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,17.078739781790915,0,0,1.5,0,1.5,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,13,14,0,0,0,0,19,0,0,22,0,0,25,0,27,0,0],"r":[24,21,18,17,12,11,10,9,0,0,0,0,16,15,0,0,0,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0.0002952252,-0.01181247,0.02571841,-0.01107796,0,0,0.01034804,-0.03309592,0.03414202,-0.01536872,0,-0.01614956,0.02062544,0,-0.01907575,0.0003986176,0,0.03382643,0,-0.02742342,0.006084136]},{"f":[9,22,21,-1,22,22,33,-1,-1,-1,16,-1,22,-1,15,16,-1,25,-1,-1,-1,22,-1,22,-1,22,-1,-1,-1],"t":[0.5,14.116600814324018,3.5,0,13.129761635363703,12.613886328427084,8.532371382375128,0,0,0,6.832890014164741,0,13.589275375459419,0,10.457380879072534,7.330916878114617,0,1.5,0,0,0,15.676331898425197,0,15.717370311433935,0,15.756821747580634,0,0,0],"m":[1,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,11,0,13,0,15,16,0,18,0,0,0,22,0,24,0,26,0,0,0],"r":[28,21,4,0,10,9,8,0,0,0,12,0,14,0,20,17,0,19,0,0,0,23,0,25,0,27,0,0,0],"v":[0,0,0,-0.02420477,0,0,0,-0.0309049,0.000112628,0.008335214,0,-0.03891524,0,-0.02096234,0,0,0.01538258,0,0.00301665,-0.0222228,0.02796338,0,0.005047686,0,-0.03997375,0,0.02366556,-0.0002778811,-0.01735849]},{"f":[36,22,22,22,22,22,-1,23,16,-1,-1,-1,-1,32,-1,-1,22,-1,-1,25,-1,17,2,-1,-1,-1,1,-1,-1],"t":[5.5,17.094543568094064,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,17.078739781790915,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0,8.992857142857144,0,0],"m":[1,1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,0,0,14,0,0,17,0,0,20,0,22,23,0,0,0,27,0,0],"r":[26,19,16,13,12,7,0,11,10,0,0,0,0,15,0,0,18,0,0,21,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.0003551799,0,0,0.01005279,-0.03240906,0.03324267,-0.01476409,0,-0.01597074,0.01981775,0,-0.01837746,0.0004733124,0,0.03325039,0,0,0.0238726,-0.01811125,-0.02684913,0,-0.006804508,0.02892849]},{"f":[23,22,22,22,22,16,-1,-1,-1,22,22,-1,-1,24,-1,-1,22,-1,18,-1,-1,32,-1,33,-1,18,-1,-1,-1],"t":[7.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,16.903290731167747,16.844368517741394,0,0,8.5,0,0,17.078739781790915,0,6.118903736861568,0,0,8.341935934311302,0,37.46769166150554,0,5.491496787347155,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,0,0,14,0,0,17,0,19,0,0,22,0,24,0,26,0,0,0],"r":[28,21,16,9,8,7,0,0,0,13,12,0,0,15,0,0,18,0,20,0,0,23,0,25,0,27,0,0,0],"v":[0,0,0,0,0,0,-0.02731352,0.0001382179,-0.02931118,0,0,0.002317593,0.03775648,0,0.004102936,-0.02275056,0,-0.02682704,0,0.005481593,-0.02327126,0,-0.02149697,0,0.04152171,0,-0.01528678,0.01024062,-0.01575288]},{"f":[36,22,22,4,-1,-1,-1,23,33,15,18,-1,2,-1,-1,15,-1,18,-1,-1,-1,18,16,-1,-1,-1,1,-1,-1],"t":[5.5,11.67903991803059,10.27595502635321,1.292481250360578,0,0,0,0.5,9.184798673139337,5.672425341971495,1.292481250360578,0,1.292481250360578,0,0,6.857980995127572,0,10.680359255641626,0,0,0,10.45532575654158,6.523561956057013,0,0,0,8.992857142857144,0,0],"m":[1,0,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,13,0,0,16,0,18,0,0,0,22,23,0,0,0,27,0,0],"r":[26,7,6,5,0,0,0,21,20,15,12,0,14,0,0,17,0,19,0,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,-0.0256417,0.001478922,-0.0189827,0,0,0,0,-0.02067456,0,0.03733521,0.003650071,0,0.05576668,0,-0.008857605,0.0240074,-0.002760774,0,0,-0.004792323,0.001468052,-0.01964785,0,-0.00657326,0.02843799]},{"f":[4,16,-1,19,0,22,23,16,-1,21,-1,22,-1,22,-1,-1,-1,22,-1,1,-1,22,-1,0,-1,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,2.5,10.5,16.73920708776685,4.5,6.832890014164741,0,4.5,0,15.676331898425197,0,15.717370311433935,0,0,0,16.752707246237605,0,1.3897752394988947,0,16.765402642760364,0,6.5,0,0,0,0,0],"m":[1,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,4,5,6,7,8,0,10,0,12,0,14,0,0,0,18,0,20,0,22,0,24,0,0,0,0,0],"r":[28,3,0,27,26,17,16,9,0,11,0,13,0,15,0,0,0,19,0,21,0,23,0,25,0,0,0,0,0],"v":[0,0,-0.03038959,0,0,0,0,0,0.002713899,0,-0.03660111,0,0.001212375,0,-0.03944028,-0.001942143,0.00571377,0,-0.03324837,0,0.00571377,0,-0.02701678,0,-0.02382006,0.00571377,0.01252382,-0.02823429,0.0008039568]},{"f":[36,-1,22,21,-1,23,22,22,33,-1,-1,-1,17,-1,-1,16,-1,1,-1,16,-1,-1,22,17,18,-1,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,3.5,13.129761635363703,12.613886328427084,8.532371382375128,0,0,0,6.9827117829050644,0,0,6.882643049361842,0,3.015625,0,9.047123912114026,0,0,14.158358850608169,7.67595272073847,3.5221970596792267,0,0,0,0],"m":[0,0,1,0,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,0,0,0,13,0,0,16,0,18,0,20,0,0,23,24,25,0,0,0,0],"r":[2,0,22,5,0,15,12,11,10,0,0,0,14,0,0,17,0,19,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0.01194332,0,0,-0.02269799,0,0,0,0,-0.03396925,0.0003604977,0.009013224,0,-0.03511601,-0.002739511,0,-0.02483694,0,-0.02673286,0,-0.00130045,0.02491257,0,0,0,0.03003735,-0.002905559,0.03995877,-0.0004125795]},{"f":[36,22,22,22,16,0,-1,-1,33,-1,-1,32,-1,-1,22,-1,-1,25,-1,17,24,-1,16,-1,-1,-1,1,-1,-1],"t":[5.5,17.094543568094064,17.02942582914998,16.978341770814396,0.0,9.5,0,0,35.98606694718995,0,0,2.69615871138938,0,0,17.078739781790915,0,0,1.5,0,8.729619742794132,8.5,0,3.321928094887362,0,0,0,8.992857142857144,0,0],"m":[1,1,1,1,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,0,9,0,0,12,0,0,15,0,0,18,0,20,21,0,23,0,0,0,27,0,0],"r":[26,17,14,11,8,7,0,0,10,0,0,13,0,0,16,0,0,19,0,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.02856119,-0.002359248,0,0.0001545208,-0.02120544,0,-0.01758649,0.01251943,0,-0.02654596,0.0003993916,0,0.03229379,0,0,-0.02228186,0,-0.01118246,0.02984381,-0.02668656,0,-0.006006715,0.02777579]},{"f":[0,32,18,4,18,4,19,-1,-1,22,16,22,-1,33,-1,-1,-1,-1,-1,21,22,2,-1,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,7.723832212141186,5.505613627711627,7.383290334558488,4.951187557243013,2.5,0,0,14.03140051979143,7.266786540694901,7.082728314749225,0,15.910824717275187,0,0,0,0,0,7.5,15.782972563647261,7.128362343532883,0,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,11,12,0,14,0,0,0,0,0,20,21,22,0,0,0,0,0,0,0],"r":[28,27,26,19,18,9,8,0,0,17,16,13,0,15,0,0,0,0,0,25,24,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.0001258966,-0.026812,0,0,0,0.00635591,0,-0.04119804,-0.007854937,0.003298861,0.00635591,-0.02563621,0,0,0,-0.03878438,-0.002032912,0.00635591,-6.868587e-06,0.002483203,-0.01990555,0.01284458]},{"f":[32,32,37,25,22,22,22,22,22,7,-1,-1,-1,21,-1,-1,-1,25,-1,22,-1,-1,-1,-1,-1,-1,18,-1,-1],"t":[10.463013322934223,10.41931383093672,12.52,6.5,17.083799521744545,17.02942582914998,16.759160379664443,16.73920708776685,16.725912049439096,0.5,0,0,0,7.5,0,0,0,3.5,0,17.069460217922263,0,0,0,0,0,0,3.5221970596792267,0,0],"m":[1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,10,0,0,0,14,0,0,0,18,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,24,23,22,17,16,13,12,11,0,0,0,15,0,0,0,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0.0001385361,-0.01379701,0.007405428,0,0.001339232,-0.03356032,0.00394992,0,-0.02819549,0,-0.02547243,0.005162231,0.007405428,-0.02265052,0.01488203,-0.00852197,0,0.01346689,-0.01740567]},{"f":[9,17,3,-1,1,21,2,-1,-1,22,0,-1,-1,22,-1,-1,1,23,-1,22,-1,-1,-1,17,22,-1,-1,-1,-1],"t":[0.5,2.160964047443681,0.23857586512866016,0,5.489664082687339,6.5,5.065928480304397,0,0,10.27595502635321,2.5,0,0,16.759160379664443,0,0,6.015625,0.5,0,12.679695715883515,0,0,0,2.69615871138938,16.752707246237605,0,0,0,0],"m":[1,0,0,0,1,1,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,0,0,14,0,0,17,18,0,20,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,16,9,8,0,0,13,12,0,0,15,0,0,22,19,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0.0197193,0,0,0,-0.03778464,0.002768378,0,0,0.002179115,0.02985171,0,-0.009191311,0.01285769,0,0,0.06122814,0,-0.02238117,0.02633824,0.002300494,0,0,-0.03724582,0.004473968,-0.0002673413,-0.01687313]},{"f":[36,-1,22,21,-1,22,22,22,22,33,-1,23,-1,-1,21,-1,-1,-1,-1,16,-1,-1,22,17,18,-1,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,13.930814526604713,13.589275375459419,13.129761635363703,12.613886328427084,8.532371382375128,0,2.5,0,0,7.5,0,0,0,0,7.266786540694901,0,0,14.158358850608169,7.67595272073847,3.5221970596792267,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,15,0,0,0,0,20,0,0,23,24,25,0,0,0,0],"r":[2,0,22,5,0,19,18,17,14,11,0,13,0,0,16,0,0,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0.01150582,0,0,-0.02193139,0,0,0,0,0,-0.03342411,0,0.0005177747,-0.01198808,0,0.01277901,-0.002932467,-0.0242891,0.01892417,0,-0.03346829,-0.004215131,0,0,0,0.02885655,-0.002810389,0.03838764,-0.0003660619]},{"f":[36,22,22,4,-1,-1,-1,23,33,15,18,-1,2,-1,-1,15,-1,22,-1,4,-1,-1,-1,18,-1,-1,18,-1,-1],"t":[5.5,11.67903991803059,10.27595502635321,1.292481250360578,0,0,0,0.5,9.184798673139337,5.672425341971495,1.292481250360578,0,1.292481250360578,0,0,6.857980995127572,0,11.683871801979876,0,2.69615871138938,0,0,0,10.45532575654158,0,0,4.245926548164837,0,0],"m":[1,0,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,13,0,0,16,0,18,0,20,0,0,0,24,0,0,27,0,0],"r":[26,7,6,5,0,0,0,23,22,15,12,0,14,0,0,17,0,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-0.02502607,0.001249596,-0.01804626,0,0,0,0,-0.01978315,0,0.0346484,0.003056412,0,0.05186416,0,0.02782874,0,-0.02941145,0.009223243,-0.002488165,0,-2.828262e-05,-0.01902705,0,-0.003607992,0.03061127]},{"f":[36,22,22,4,-1,-1,21,-1,-1,23,33,33,33,-1,-1,1,16,-1,15,-1,-1,-1,-1,18,-1,-1,1,-1,-1],"t":[5.5,11.67903991803059,10.27595502635321,1.292481250360578,0,0,7.5,0,0,0.5,21.257266042509173,19.167172160526107,16.11799658182957,0,0,2.969223484848485,1.584962500721156,0,9.756556322524087,0,0,0,0,10.45532575654158,0,0,8.992857142857144,0,0],"m":[1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,12,13,0,0,16,17,0,19,0,0,0,0,24,0,0,27,0,0],"r":[26,9,6,5,0,0,8,0,0,23,22,15,14,0,0,21,18,0,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-0.02458856,0.001186713,0,-0.02883821,-0.006469457,0,0,0,0,0.0003889512,-0.03885435,0,0,0.01584202,0,-0.02486421,0.02039624,0.01608246,-0.009620983,0,-2.687333e-05,-0.01838971,0,-0.005373443,0.02672298]},{"f":[36,-1,16,-1,22,22,21,-1,33,16,-1,22,0,-1,-1,-1,-1,2,-1,-1,22,-1,22,-1,2,-1,2,-1,-1],"t":[1.5,0,9.92481250360578,0,16.725912049439096,15.756821747580634,4.5,0,35.98606694718995,10.681238411777805,0,15.212004812197344,3.5,0,0,0,0,7.071452393754301,0,0,16.73920708776685,0,16.752707246237605,0,6.011183906514227,0,9.068775776507199,0,0],"m":[0,0,1,0,1,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,0,5,6,7,0,9,10,0,12,13,0,0,0,0,18,0,0,21,0,23,0,25,0,27,0,0],"r":[2,0,4,0,20,17,8,0,16,11,0,15,14,0,0,0,0,19,0,0,22,0,24,0,26,0,28,0,0],"v":[0,0.01161462,0,-0.0008253242,0,0,0,-0.02252791,0,0,0.01407398,0,0,-0.01189688,0.001976899,0.01824497,-0.03182736,0,-0.002351529,-0.01818838,0,0.02945295,0,-0.02906311,0,0.05459514,0,-0.003660772,0.02043778]},{"f":[0,32,32,33,-1,15,-1,-1,33,-1,2,33,-1,16,-1,23,-1,15,17,-1,-1,22,32,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,6.208926257442949,13.660441030622824,0,6.169925001442312,0,0,21.16864017939033,0,3.084962500721156,24.416594167368267,0,2.0,0,0.5,0,7.930737337562887,7.344289078557851,0,0,12.613886328427084,10.019277334827486,0,0,0,0,0,0],"m":[1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,0,11,12,0,14,0,16,0,18,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,8,5,0,7,0,0,10,0,26,13,0,15,0,17,0,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-6.000791e-05,0,-0.02859716,0.0002913545,0,0.006210531,0,0,-0.0163167,0,-0.02717173,0,0.006210531,0,0,-0.03449295,0.001884348,0,0,-0.0008964449,-0.03220954,0.006210531,0.0007825591,-0.01923655,0.01252153]},{"f":[36,22,22,4,-1,-1,21,-1,-1,23,33,15,18,-1,17,-1,-1,15,-1,18,-1,-1,-1,18,-1,-1,18,-1,-1],"t":[5.5,11.67903991803059,10.27595502635321,1.292481250360578,0,0,7.5,0,0,0.5,9.184798673139337,5.672425341971495,1.292481250360578,0,10.594763897338481,0,0,6.857980995127572,0,10.680359255641626,0,0,0,10.45532575654158,0,0,4.245926548164837,0,0],"m":[1,0,1,0,0,0,1,0,0,0,1,1,1,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,12,13,0,15,0,0,18,0,20,0,0,0,24,0,0,27,0,0],"r":[26,9,6,5,0,0,8,0,0,23,22,17,14,0,16,0,0,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-0.02405035,0.001054004,0,-0.02818671,-0.006121747,0,0,0,0,-0.01936885,0,0.01949042,-0.001847633,0,0.04673717,0,-0.007678575,0.02161485,-0.002203193,0,-2.937709e-05,-0.01774363,0,-0.00333549,0.02992343]},{"f":[21,15,-1,22,22,21,18,-1,-1,-1,-1,17,22,-1,22,-1,-1,-1,25,18,22,2,-1,-1,-1,18,-1,-1,-1],"t":[9.5,11.054604317960672,0,15.782972563647261,11.876328901062104,7.5,9.265614570460752,0,0,0,0,4.491496787347155,16.725912049439096,0,16.765402642760364,0,0,0,0.5,1.792481250360578,13.589275375459419,6.554507209719489,0,0,0,6.781300095563516,0,0,0],"m":[1,1,0,1,0,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,7,0,0,0,0,12,13,0,15,0,0,0,19,20,21,22,0,0,0,26,0,0,0],"r":[18,3,0,11,10,9,8,0,0,0,0,17,14,0,16,0,0,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0.0001306395,0,0,0,0,-0.03109817,0.003737936,0.01280989,-0.03928888,0,0,-0.03102462,0,0.01023138,-0.02207271,0.01811167,0,0,0,0,-0.01176288,0.01604863,0.01944935,0,-0.02709823,0.003339791,0.03560569]},{"f":[36,-1,22,21,1,-1,2,-1,-1,-1,22,17,18,-1,21,33,-1,-1,-1,-1,22,21,-1,-1,4,22,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,6.069196428571429,0,2.453445297804259,0,0,0,14.158358850608169,7.67595272073847,3.5221970596792267,0,4.5,15.968216440900115,0,0,0,0,15.439474430023438,7.5,0,0,8.049190845056867,15.676331898425197,0,0,0],"m":[0,0,1,0,1,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,13,0,15,16,0,0,0,0,21,22,0,0,25,26,0,0,0],"r":[2,0,10,9,6,0,8,0,0,0,20,19,14,0,18,17,0,0,0,0,24,23,0,0,28,27,0,0,0],"v":[0,0.01127949,0,0,0,-0.03491008,0,-0.02441836,0.01295538,-0.0005645311,0,0,0,0.02758678,0,0,0.03529548,-0.001435192,-0.03072509,0.03703045,0,0,-0.0382676,-0.002350667,0,0,0.01078785,-0.001398656,0.02464549]},{"f":[23,21,15,-1,22,22,21,-1,-1,-1,17,22,-1,22,-1,-1,-1,25,18,22,2,-1,-1,-1,17,-1,-1,-1,-1],"t":[7.5,9.5,11.054604317960672,0,15.782972563647261,11.876328901062104,7.5,0,0,0,4.491496787347155,16.725912049439096,0,16.765402642760364,0,0,0,0.5,1.792481250360578,13.589275375459419,6.554507209719489,0,0,0,6.781300095563516,0,0,0,0],"m":[1,1,1,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,11,12,0,14,0,0,0,18,19,20,21,0,0,0,25,0,0,0,0],"r":[28,17,4,0,10,9,8,0,0,0,16,13,0,15,0,0,0,27,24,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0.0001717044,0,0,0,-0.01119803,0.01226257,-0.03870008,0,0,-0.03061427,0,0.009821786,-0.02115368,0.01720715,0,0,0,0,-0.01129842,0.01528353,0.01951297,0,-0.02385384,0.0005770256,0.03512267,-0.01510579]},{"f":[25,17,15,16,15,1,2,-1,-1,-1,-1,1,16,-1,-1,0,-1,15,-1,-1,21,-1,-1,17,16,-1,-1,-1,-1],"t":[6.5,2.160964047443681,10.066089190457772,7.930737337562887,7.679480099505446,5.489664082687339,6.434604125275413,0,0,0,0,2.9265316718587746,10.681238411777805,0,0,4.5,0,8.361943773735241,0,0,7.5,0,0,2.69615871138938,5.672425341971495,0,0,0,0],"m":[1,0,1,1,1,1,1,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,0,12,13,0,0,16,0,18,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,20,11,10,9,8,0,0,0,0,15,14,0,0,17,0,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.02063876,0.003318765,0.005779536,-0.03163007,0,0,0.05206223,0.009809757,0,-0.01829017,0,0.01952295,0.0001341528,0,-0.03550612,-0.001097156,0,0,0.006273604,-0.03495474,-0.0002904841,-0.01335855]},{"f":[4,16,-1,17,22,-1,-1,0,-1,15,-1,-1,32,19,-1,-1,33,-1,2,16,-1,22,17,-1,-1,-1,1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,15.717370311433935,0,0,2.5,0,12.000704269011246,0,0,6.208926257442949,0.5,0,0,21.16864017939033,0,5.357122758833062,6.820178962415188,0,14.03140051979143,5.918419679874358,0,0,0,8.992857142857144,0,0],"m":[1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,1,0,0],"l":[1,2,0,4,5,0,0,8,0,10,0,0,13,14,0,0,17,0,19,20,0,22,23,0,0,0,27,0,0],"r":[12,3,0,7,6,0,0,9,0,11,0,0,16,15,0,0,18,0,26,21,0,25,24,0,0,0,28,0,0],"v":[0,0,-0.02898804,0,0,0.004734501,0.04041734,0,-0.02164402,0,-0.001253426,0.02184157,0,0,-0.001138326,-0.02161096,0,0.01822622,0,0,7.838059e-05,0,0,0.01908821,-0.02560621,0.04961958,0,-0.0002946053,0.01803736]},{"f":[32,32,18,18,-1,-1,23,33,18,19,15,-1,-1,33,-1,-1,22,18,-1,-1,-1,-1,17,-1,-1,-1,18,-1,-1],"t":[10.463013322934223,10.41931383093672,7.198837316474133,6.864066550153108,0,0,3.5,21.257266042509173,8.03615005796417,1.5,6.820178962415188,0,0,21.011466140970732,0,0,12.664623226706265,9.265614570460752,0,0,0,0,8.571721503073015,0,0,0,3.5221970596792267,0,0],"m":[1,1,1,1,0,0,1,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,10,11,0,0,14,0,0,17,18,0,0,0,0,23,0,0,0,27,0,0],"r":[26,25,6,5,0,0,22,21,16,13,12,0,0,15,0,0,20,19,0,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,-0.0002646311,-0.03273447,0,0,0,0,0,-0.01250424,0.05171404,0,-0.0240908,0.01130802,0,0,-0.01969439,0.0018294,0.01516394,-0.007125205,0,-0.03164153,-0.001045833,-0.008383151,0,0.01258778,-0.01683235]},{"f":[22,21,-1,22,22,33,-1,-1,21,21,15,-1,-1,-1,22,-1,-1,16,-1,22,-1,15,16,-1,25,-1,-1,-1,-1],"t":[14.116600814324018,3.5,0,13.129761635363703,12.613886328427084,8.532371382375128,0,0,6.5,4.5,4.247927513443585,0,0,0,12.679695715883515,0,0,6.832890014164741,0,13.589275375459419,0,10.457380879072534,7.330916878114617,0,1.5,0,0,0,0],"m":[1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,10,11,0,0,0,15,0,0,18,0,20,0,22,23,0,25,0,0,0,0],"r":[28,3,0,17,8,7,0,0,14,13,12,0,0,0,16,0,0,19,0,21,0,27,24,0,26,0,0,0,0],"v":[0,0,-0.02170537,0,0,0,-0.02895092,2.788778e-05,0,0,0,0.008283474,-0.02295847,0.02530636,0,-0.02848512,0.003626055,0,-0.03735856,0,-0.01844812,0,0,0.01410868,0,0.003503246,-0.02135924,0.02433066,0.0008761896]},{"f":[4,4,21,22,16,17,2,-1,2,-1,-1,2,-1,-1,22,-1,-1,-1,25,20,-1,-1,-1,16,-1,-1,2,-1,-1],"t":[9.266718101290426,9.12919100246427,9.5,14.116600814324018,10.066089190457772,5.83200138144325,5.357122758833062,0,6.209321624687949,0,0,5.065928480304397,0,0,12.606173770495205,0,0,0,0.5,0.5,0,0,0,3.321928094887362,0,0,10.510264838663748,0,0],"m":[1,1,1,1,1,1,1,0,1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,0,12,0,0,15,0,0,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,18,17,14,11,8,0,10,0,0,13,0,0,16,0,0,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0.0006240543,0,-0.03967037,0.002036687,0,-0.0407844,-0.006726319,0,0.0008870068,0.02398334,0.0008842878,0,0,0.01100079,-0.008142506,0.0346951,0,0.00238734,0.02084421,0,-0.022301,0.001155364]},{"f":[36,-1,22,21,1,-1,2,-1,-1,-1,22,17,18,-1,21,33,-1,-1,-1,-1,22,21,-1,-1,4,22,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,6.069196428571429,0,2.453445297804259,0,0,0,14.158358850608169,7.67595272073847,3.5221970596792267,0,4.5,15.968216440900115,0,0,0,0,15.439474430023438,7.5,0,0,8.049190845056867,15.676331898425197,0,0,0],"m":[0,0,1,0,1,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,13,0,15,16,0,0,0,0,21,22,0,0,25,26,0,0,0],"r":[2,0,10,9,6,0,8,0,0,0,20,19,14,0,18,17,0,0,0,0,24,23,0,0,28,27,0,0,0],"v":[0,0.01121195,0,0,0,-0.03408446,0,-0.02380469,0.01279845,-0.000541759,0,0,0,0.02648216,0,0,0.03230134,-0.001455192,-0.02950427,0.03577266,0,0,-0.03758023,-0.002185425,0,0,0.01013767,-0.001380137,0.0231694]},{"f":[23,22,33,16,37,-1,-1,33,18,-1,33,18,33,-1,15,-1,1,-1,-1,-1,-1,-1,17,22,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,35.98606694718995,0.0,8.2,0,0,32.32039757950231,7.198837316474133,0,21.257266042509173,8.03615005796417,19.167172160526107,0,6.820178962415188,0,4.757007376185458,0,0,0,0,0,7.859614184977944,16.975010474608837,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,11,12,13,0,15,0,17,0,0,0,0,0,23,24,0,0,0,0,0],"r":[28,27,22,7,6,0,0,21,10,0,20,19,14,0,16,0,18,0,0,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.032848,0.004623132,0,0,-0.0004886829,0,0,0,-0.02078839,0,0.005895375,0,0.0002273617,0.05470741,0.000675328,-0.00702401,0.008440921,0,0,0.01555368,-0.006539068,-0.03257316,0.0149061,-0.01447693]},{"f":[17,3,-1,15,15,15,3,-1,1,22,-1,-1,2,-1,-1,-1,1,-1,15,-1,-1,16,-1,-1,17,22,-1,-1,-1],"t":[2.160964047443681,0.23857586512866016,0,10.066089190457772,7.930737337562887,7.679480099505446,0.8758561643835616,0,5.3560606060606055,12.921158001713904,0,0,2.453445297804259,0,0,0,2.6153846153846154,0,8.005624549193879,0,0,11.696967526234287,0,0,2.69615871138938,16.752707246237605,0,0,0],"m":[0,0,0,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,7,0,9,10,0,0,13,0,0,0,17,0,19,0,0,22,0,0,25,26,0,0,0],"r":[24,3,0,21,16,15,8,0,12,11,0,0,14,0,0,0,18,0,20,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0.01836958,0,0,0,0,-0.02171847,0,0,-4.068974e-05,-0.0330322,0,0.002727684,0.0475376,-0.03045184,0,0.04304723,0,0.01636597,-0.002306708,0,-0.03636079,-0.0008601216,0,0,-0.03577964,0.004556282,-0.000299649]},{"f":[36,-1,22,21,1,-1,2,-1,-1,19,22,22,22,22,33,-1,-1,21,-1,-1,-1,-1,16,-1,15,-1,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,6.069196428571429,0,2.453445297804259,0,0,2.5,13.930814526604713,13.589275375459419,13.129761635363703,12.613886328427084,8.532371382375128,0,0,7.5,0,0,0,0,7.266786540694901,0,5.672425341971495,0,0,0,0],"m":[0,0,1,0,1,0,1,0,0,1,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,11,12,13,14,15,0,0,18,0,0,0,0,23,0,25,0,0,0,0],"r":[2,0,28,9,6,0,8,0,0,27,22,21,20,17,16,0,0,19,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0.01082348,0,0,0,-0.03366455,0,-0.02360236,0.01093684,0,0,0,0,0,0,-0.03235467,0.0002680472,0,0.01111827,-0.00324784,-0.02162411,0.01683858,0,-0.03239394,0,0.002812655,-0.02752625,-0.01208649,0.0008053466]},{"f":[4,4,21,22,16,-1,22,16,-1,22,-1,17,-1,-1,16,-1,-1,-1,25,20,-1,-1,-1,16,-1,-1,2,-1,-1],"t":[9.266718101290426,9.12919100246427,9.5,14.116600814324018,10.066089190457772,0,12.199665042528586,10.462502272597265,0,11.67903991803059,0,9.265614570460752,0,0,10.681238411777805,0,0,0,0.5,0.5,0,0,0,3.321928094887362,0,0,10.510264838663748,0,0],"m":[1,1,1,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,10,0,12,0,0,15,0,0,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,18,17,6,0,14,9,0,11,0,13,0,0,16,0,0,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.005004915,0,0,-0.0131936,0,-0.0005356274,0,-0.001144047,0.04454448,0,0.03158605,-0.003986986,0.0008175038,0,0,0.01036369,-0.007614103,0.03419294,0,0.002318613,0.01946517,0,-0.02165032,0.001080498]},{"f":[4,35,16,-1,17,22,-1,-1,0,15,-1,-1,15,0,22,22,-1,-1,22,-1,2,-1,-1,-1,-1,-1,22,-1,-1],"t":[3.245926548164837,2.5,2.0,0,0.5,15.717370311433935,0,0,2.5,11.683433292835403,0,0,12.000704269011246,10.5,16.73920708776685,15.717370311433935,0,0,16.765402642760364,0,6.055241654908112,0,0,0,0,0,17.094543568094064,0,0],"m":[1,1,0,0,0,1,0,0,0,1,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,0,9,10,0,0,13,14,15,16,0,0,19,0,21,0,0,0,0,0,27,0,0],"r":[26,25,4,0,8,7,0,0,12,11,0,0,24,23,18,17,0,0,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,-0.02626633,0,0,0.004897532,0.0383355,0,0,-0.03231743,-0.01072318,0,0,0,0,-0.002584429,0.003200933,0,-0.01072318,0,-0.01072318,0.005145272,0.01177902,0.01886583,-0.02829571,0,0.0005011922,0.02026176]},{"f":[36,-1,18,1,-1,1,23,-1,22,-1,-1,1,-1,-1,19,18,19,-1,17,22,-1,22,-1,-1,-1,18,-1,-1,-1],"t":[1.5,0,2.160964047443681,5.992857142857143,0,6.015625,0.5,0,12.679695715883515,0,0,6.241071428571429,0,0,2.5,5.303665156874805,0.5,0,7.67595272073847,15.717370311433935,0,15.782972563647261,0,0,0,5.425874520708029,0,0,0],"m":[0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,0,12,0,0,15,16,17,0,19,20,0,22,0,0,0,26,0,0,0],"r":[2,0,14,5,0,11,8,0,10,0,0,13,0,0,28,25,18,0,24,21,0,23,0,0,0,27,0,0,0],"v":[0,0.01073131,0,0,-0.001111509,0,0,0.05844935,0,-0.02074106,0.01821854,0,-0.02834994,0.003399589,0,0,0,-0.001118159,0,0,-0.04656284,0,0.01881109,-0.02877042,0.01343702,0,0.01547718,9.411942e-05,-0.03204478]},{"f":[22,22,22,22,22,-1,23,16,-1,-1,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,2,17,-1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,16.975010474608837,16.844368517741394,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,12,11,6,0,10,9,0,0,0,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0001741651,0,0,0.00989552,-0.03189508,0.0320004,-0.01397096,0,-0.01707441,0.01199941,0,-0.02750653,0,-0.02528663,0.004844778,0,0.03048668,0,0,0,-0.006123794,0.03543431,-0.0102573,-0.02666742]},{"f":[23,22,33,16,37,-1,-1,33,22,22,22,-1,-1,21,-1,-1,16,-1,-1,0,-1,-1,17,22,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,35.98606694718995,0.0,8.2,0,0,29.297756302860655,16.759160379664443,16.73920708776685,16.725912049439096,0,0,7.5,0,0,9.047123912114026,0,0,10.5,0,0,7.859614184977944,16.975010474608837,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,1,0,0,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,10,11,0,0,14,0,0,17,0,0,20,0,0,23,24,0,0,0,0,0],"r":[28,27,22,7,6,0,0,19,16,13,12,0,0,15,0,0,18,0,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03239246,0.004305551,0,0,0,0,-0.0003872956,0.017824,0,0.001909178,-0.03249436,0,-0.00309557,0.01620558,0,-0.0003302304,0.01144512,0,0,0.01442817,-0.006189615,-0.0319793,0.01432902,-0.01436649]},{"f":[22,22,22,16,-1,33,-1,17,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,2,17,-1,-1,1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,0.0,0,35.98606694718995,0,7.5507416557826375,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,7.9752100840336135,0,0,0],"m":[1,1,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,11,0,0,14,0,16,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[18,13,10,5,0,7,0,9,0,0,12,0,0,15,0,17,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.01053498,0,0.0002413101,0,0.004028045,-0.03155643,0,-0.01675781,0.0116292,0,-0.02682762,0,-0.02491908,0.004774713,0,0.02968877,0,0,0,-0.005968699,0.03492938,0,-2.200378e-05,-0.02569682,-0.0262785]},{"f":[23,22,16,37,-1,22,-1,-1,33,33,18,-1,33,18,33,-1,15,-1,-1,-1,-1,0,-1,-1,17,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,8.2,0,16.975010474608837,0,0,35.98606694718995,29.297756302860655,7.198837316474133,0,21.257266042509173,8.03615005796417,19.167172160526107,0,6.820178962415188,0,0,0,0,10.5,0,0,7.5507416557826375,0,0,0,0],"m":[1,1,0,1,0,0,0,0,1,1,1,0,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,13,14,15,0,17,0,0,0,0,22,0,0,25,0,0,0,0],"r":[28,27,8,5,0,7,0,0,24,21,12,0,20,19,16,0,18,0,0,0,0,23,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.03187602,0,0.01414252,-0.003959792,0,0,0,-0.00060122,0,0,0,-0.0202799,0,0.005676263,0.03932839,0.000549893,-0.007307062,0,-0.0003293615,0.01088567,0,0.003897222,-0.0311391,0.01389528,-0.01425653]},{"f":[4,4,1,1,-1,-1,21,22,22,-1,16,-1,2,-1,-1,-1,17,17,23,-1,-1,-1,-1,2,-1,-1,2,-1,-1],"t":[9.266718101290426,9.12919100246427,0.4906446161274747,0.00982698026493647,0,0,9.5,14.116600814324018,13.129761635363703,0,7.118941072723508,0,2.453445297804259,0,0,0,6.781300095563516,6.584884225259038,5.5,0,0,0,0,9.20088558729989,0,0,10.510264838663748,0,0],"m":[1,1,0,0,0,0,1,1,1,0,1,0,1,0,0,0,1,1,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,11,0,13,0,0,0,17,18,19,0,0,0,0,24,0,0,27,0,0],"r":[26,23,6,5,0,0,16,15,10,0,12,0,14,0,0,0,22,21,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.002054828,-0.03081351,0,0,0,-0.0006639095,0,-0.03913285,0,-0.02991954,0.0004621483,0.0009034818,0,0,0,0.01040111,-0.01834043,-0.01957415,0.02989109,0,0.01719827,0.0007603995,0,-0.02110157,0.001049236]},{"f":[23,22,22,22,16,-1,22,22,22,-1,-1,-1,22,-1,-1,-1,22,-1,2,-1,-1,32,-1,33,-1,18,-1,-1,-1],"t":[7.5,17.083799521744545,16.98920034885663,16.978341770814396,0.0,0,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,16.903290731167747,0,0,0,17.078739781790915,0,6.011183906514227,0,0,8.341935934311302,0,37.46769166150554,0,5.491496787347155,0,0,0],"m":[1,1,1,1,0,0,1,1,1,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,9,0,0,0,13,0,0,0,17,0,19,0,0,22,0,24,0,26,0,0,0],"r":[28,21,16,15,6,0,12,11,10,0,0,0,14,0,0,0,18,0,20,0,0,23,0,25,0,27,0,0,0],"v":[0,0,0,0,0,-0.009827781,0,0,0,-0.0001587768,0.02427942,-0.01061054,0,0.03689536,-0.01406602,0.01427867,0,-0.01617316,0,-0.02042728,0.005927456,0,-0.02141124,0,0.0396384,0,-0.01433463,0.009490787,-0.01429643]},{"f":[36,-1,18,1,21,19,-1,-1,-1,1,23,-1,22,-1,-1,1,-1,25,22,22,-1,22,-1,-1,-1,-1,19,-1,-1],"t":[1.5,0,2.160964047443681,5.992857142857143,4.5,1.5,0,0,0,6.015625,0.5,0,12.679695715883515,0,0,6.241071428571429,0,1.5,15.782972563647261,14.03140051979143,0,15.676331898425197,0,0,0,0,2.5,0,0],"m":[0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,0,0,0,10,11,0,13,0,0,16,0,18,19,20,0,22,0,0,0,0,27,0,0],"r":[2,0,26,9,8,7,0,0,0,15,12,0,14,0,0,17,0,25,24,21,0,23,0,0,0,0,28,0,0],"v":[0,0.01059414,0,0,0,0,-0.02948498,0.003453432,-0.0005478812,0,0,0.05639407,0,-0.02046758,0.01741503,0,-0.02779637,0,0,0,0.001934643,0,0.03481605,0.01123725,-0.004170599,-0.001578983,0,-0.0006724874,-0.03158055]},{"f":[4,16,-1,17,22,-1,-1,0,15,-1,-1,15,-1,-1,32,19,-1,-1,33,-1,2,16,-1,22,17,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,15.717370311433935,0,0,2.5,11.683433292835403,0,0,12.000704269011246,0,0,6.208926257442949,0.5,0,0,21.16864017939033,0,5.357122758833062,6.820178962415188,0,14.03140051979143,1.292481250360578,0,0,0,0],"m":[1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,0,0,12,0,0,15,16,0,0,19,0,21,22,0,24,25,0,0,0,0],"r":[14,3,0,7,6,0,0,11,10,0,0,13,0,0,18,17,0,0,20,0,28,23,0,27,26,0,0,0,0],"v":[0,0,-0.02771688,0,0,0.004936615,0.03698473,0,0,-0.03191672,-0.01059708,0,-0.001198192,0.01731528,0,0,-0.001015727,-0.0198412,0,0.01674298,0,0,0.0005543455,0,0,0.0232747,-0.01521563,0.04493106,3.031976e-05]},{"f":[24,22,-1,-1,15,17,3,-1,15,15,15,3,-1,-1,-1,1,-1,15,-1,-1,16,-1,-1,17,22,-1,-1,-1,-1],"t":[2.5,16.725912049439096,0,0,13.323758479706308,2.160964047443681,0.23857586512866016,0,10.066089190457772,7.930737337562887,7.679480099505446,0.8758561643835616,0,0,0,2.6153846153846154,0,8.005624549193879,0,0,11.696967526234287,0,0,2.69615871138938,16.752707246237605,0,0,0,0],"m":[0,1,0,0,1,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,10,11,12,0,0,0,16,0,18,0,0,21,0,0,24,25,0,0,0,0],"r":[4,3,0,0,28,23,8,0,20,15,14,13,0,0,0,17,0,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,-0.02468129,0.001955816,0,0,0,0.01795559,0,0,0,0,-0.02093395,0.001593168,-0.02974698,0,0.039573,0,0.01485571,-0.002378856,0,-0.03567978,0.008174359,0,0,-0.03469858,0.004563038,-0.0003286931,0.02301642]},{"f":[4,16,-1,17,-1,0,-1,22,22,2,3,-1,22,36,15,-1,-1,-1,33,22,-1,-1,-1,-1,-1,16,-1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,0,2.5,0,16.73920708776685,16.725912049439096,6.515505953718852,0.013290990355210539,0,15.676331898425197,4.5,12.000704269011246,0,0,0,13.660441030622824,15.782972563647261,0,0,0,0,0,5.169925001442312,0,0,0],"m":[1,0,0,0,0,0,0,1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,6,0,8,9,10,11,0,13,14,15,0,0,0,19,20,0,0,0,0,0,26,0,0,0],"r":[28,3,0,5,0,7,0,25,24,23,12,0,18,17,16,0,0,0,22,21,0,0,0,0,0,27,0,0,0],"v":[0,0,-0.02730848,0,0.01382737,0,-0.01978536,0,0,0,0,0.01593739,0,0,0,-0.00194808,0.02317662,0.01090724,0,0,0.01430373,-0.008809323,-0.01030266,-0.01030266,0.01603699,0,0.02892904,-0.01022444,0.0006918482]},{"f":[22,22,22,22,-1,-1,32,-1,2,-1,22,-1,-1,22,-1,2,-1,-1,36,-1,18,2,17,-1,-1,25,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0,0,6.337096134072842,0,4.128693921346326,0,16.978341770814396,0,0,17.078739781790915,0,6.011183906514227,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,1.5,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,9,0,11,0,0,14,0,16,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[18,13,6,5,0,0,8,0,10,0,12,0,0,15,0,17,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,3.589778e-06,-0.02867953,0,0.02596594,0,-0.02250964,0,-0.003091871,0.01277347,0,-0.02474156,0,-0.02036325,0.005026073,0,0.02861016,0,0,0,-0.005674536,0.03452174,0,0.002328657,-0.02091042,-0.02588668]}]}} \ No newline at end of file From 1bf37f949e581822e2f9870170c22a55f462465b Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 21:38:38 -0700 Subject: [PATCH 293/336] afu: register the AXI read channel and the memory adapter both ways The U55C images could not close timing or route. In VX_mem_to_axi, m_axi_rready came combinationally from the response crossbar, which depends on every bank's rvalid and rid (image B: WNS -2.0 ns at 300 MHz); in VX_afu_wrap, the memory adapters were built with REQ_OUT_BUF(0) and RSP_OUT_BUF(0), so 1,145 output bits per adapter passed straight through and every SLR cut on image A crossed a combinational path. Each bank's R channel now enters a two-entry skid buffer with a registered output before the response crossbar, so rready comes from registers. The B channel needs none: bready is tied high and bvalid only feeds registers. The adapters use OUT_BUF 3 both ways (same latency and flops as 2, data straight from a flop). Beacon (U55C, VX_afu_wrap, 2 cores / 2 ports): rready 4 -> 1 levels, the adapter outputs reached combinationally from outside 1,145 -> 0 bits; the design's deepest path and top-50 mean are unchanged. xrtsim passes vecadd, sgemm, sgemm2, demo, amo, fence_mp and vm_test on 2 cores / 2 banks, image A and the single-port path; Verilator -Wall is clean on all six xrtsim configs. A read round trip gains three registers on the multi-port path: sgemm moves within 0.2%, sgemm2 on 2 cores +1.15%, small latency-bound kernels about +17 cycles, against xrtsim's ~22-cycle memory (HBM is 100+). Default rtlsim does not include this logic, so SimX parity is unaffected. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/afu/common/VX_afu_wrap.sv | 6 ++++-- hw/rtl/libs/VX_mem_to_axi.sv | 30 ++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index 90c8615e0e..4a38fbff28 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -893,6 +893,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire [FAB_TAG_WIDTH-1:0] fab_rsp_tag [FAB_NUM_PORTS]; wire fab_rsp_ready [FAB_NUM_PORTS]; + // The GPU and the bank adapter can sit in different SLRs: both + // directions leave this adapter fully registered, data included. for (genvar i = 0; i < VX_MEM_PORTS; ++i) begin : g_vx_mem_adapter VX_mem_data_adapter #( .SRC_DATA_WIDTH (VX_MEM_DATA_WIDTH), @@ -901,8 +903,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .DST_ADDR_WIDTH (VX_MEM_ADDR_A_WIDTH), .SRC_TAG_WIDTH (VX_MEM_TAG_WIDTH), .DST_TAG_WIDTH (FAB_TAG_WIDTH), - .REQ_OUT_BUF (0), - .RSP_OUT_BUF (0) + .REQ_OUT_BUF (3), + .RSP_OUT_BUF (3) ) mem_data_adapter ( .clk (clk), .reset (vx_reset), diff --git a/hw/rtl/libs/VX_mem_to_axi.sv b/hw/rtl/libs/VX_mem_to_axi.sv index 4a3f156e59..a58e74c79d 100644 --- a/hw/rtl/libs/VX_mem_to_axi.sv +++ b/hw/rtl/libs/VX_mem_to_axi.sv @@ -410,14 +410,36 @@ module VX_mem_to_axi #( in_burst_r <= ~m_axi_rlast[i]; end end - assign rsp_xbar_valid_in[i] = m_axi_rvalid[i] && ~in_burst_r; - assign rsp_xbar_data_in[i] = {m_axi_rdata[i], m_axi_rid[i][NUM_PORTS_IN_BITS +: READ_TAG_WIDTH]}; + + // The response crossbar's ready depends on every bank's rvalid and + // rid, so a skid buffer keeps it off the AXI R handshake: rready + // comes from registers only. + wire rsp_buf_ready_in; + wire [DATA_WIDTH-1:0] rsp_buf_data; + wire [READ_FULL_TAG_WIDTH-1:0] rsp_buf_id; + + VX_elastic_buffer #( + .DATAW (DATA_WIDTH + READ_FULL_TAG_WIDTH), + .SIZE (`TO_OUT_BUF_SIZE(3)), + .OUT_REG (`TO_OUT_BUF_REG(3)) + ) rsp_buf ( + .clk (clk), + .reset (reset), + .valid_in (m_axi_rvalid[i] && ~in_burst_r), + .ready_in (rsp_buf_ready_in), + .data_in ({m_axi_rdata[i], m_axi_rid[i][READ_FULL_TAG_WIDTH-1:0]}), + .data_out ({rsp_buf_data, rsp_buf_id}), + .valid_out (rsp_xbar_valid_in[i]), + .ready_out (rsp_xbar_ready_in[i]) + ); + + assign rsp_xbar_data_in[i] = {rsp_buf_data, rsp_buf_id[NUM_PORTS_IN_BITS +: READ_TAG_WIDTH]}; if (NUM_PORTS_IN > 1) begin : g_input_sel - assign rsp_xbar_sel_in[i] = m_axi_rid[i][0 +: NUM_PORTS_IN_BITS]; + assign rsp_xbar_sel_in[i] = rsp_buf_id[0 +: NUM_PORTS_IN_BITS]; end else begin : g_no_input_sel assign rsp_xbar_sel_in[i] = 0; end - assign m_axi_rready[i] = in_burst_r ? 1'b1 : rsp_xbar_ready_in[i]; + assign m_axi_rready[i] = in_burst_r || rsp_buf_ready_in; `RUNTIME_ASSERT(~(m_axi_rvalid[i] && m_axi_rlast[i] == 0), ("*** AXI response error")) `RUNTIME_ASSERT(~(m_axi_rvalid[i] && m_axi_rresp[i] != 0), ("*** AXI response error")) end From c9d3f928a562bf18f3cf406bef62c8749dec2cc5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 29 Oct 2026 22:13:46 -0700 Subject: [PATCH 294/336] cache: pipeline the flush sequencer's crossbar in-flight count When a cache's request arbiter is sliced (fan-in above 1.5 x MAX_FANOUT), the flush sequencer counts the requests inside the crossbar. The count was updated in the cycle the requests fired: the crossbar's arbitration, a popcount over every input and every bank, the counter's add and its compares were one path. At the RTX 5080 shape (32 L2 banks) Beacon puts it at 33 levels in the L2 and 32 in the dcache, the deepest cache paths of the GPU. The fire vectors and their counts are now registered ahead of the counter. A request still counts as in flight from the cycle after it fires: until the counter holds it, the registered fire vector or count does, and the sequencer waits on all three. Decrements arrive as late, so the counter can only overstate, and a flush that waits on in-flight requests can wait up to two cycles longer. Verified: rtlsim flush_coherence (1, 3, and NT16 with 1 and 4 cores + L2), fence_mp (1-3, NT16 2 cores + L2), sgemm NT16/NW16 and vecadd NT32 pass with cycles identical to the parent; at NT16 the dcache crossbar is sliced, so the counter is present. Beacon on the cache DUT: cache_init 8 -> 6 levels, nothing deeper. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_init.sv | 41 +++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index eb265e5ca9..b2c19d4c80 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -101,11 +101,36 @@ module VX_cache_init import VX_gpu_pkg::*; #( localparam INCRW = `CLOG2(`MIN(NUM_REQS, XBAR_SIZE)+1); localparam BANKS_W = `CLOG2(NUM_BANKS+1); - wire [REQS_W-1:0] xbar_req_cnt; - wire [BANKS_W-1:0] bank_req_cnt; + // The fires and their counts are registered ahead of the counter, so + // the crossbar's arbitration, the popcounts and the counter's add are + // three cycles apart. A request is in flight from the cycle after it + // fires: until the counter holds it, the registered fire or count + // does. Decrements arrive as late, so the counter only overstates. + reg [NUM_REQS-1:0] xbar_req_fire_r; + reg [NUM_BANKS-1:0] bank_req_fire_r; + reg [REQS_W-1:0] xbar_req_cnt_r; + reg [BANKS_W-1:0] bank_req_cnt_r; + wire [REQS_W-1:0] xbar_req_cnt; + wire [BANKS_W-1:0] bank_req_cnt; - `POP_COUNT(xbar_req_cnt, xbar_req_fire); - `POP_COUNT(bank_req_cnt, bank_req_fire); + `POP_COUNT(xbar_req_cnt, xbar_req_fire_r); + `POP_COUNT(bank_req_cnt, bank_req_fire_r); + + always @(posedge clk) begin + if (reset) begin + xbar_req_fire_r <= '0; + bank_req_fire_r <= '0; + xbar_req_cnt_r <= '0; + bank_req_cnt_r <= '0; + end else begin + xbar_req_fire_r <= xbar_req_fire; + bank_req_fire_r <= bank_req_fire; + xbar_req_cnt_r <= xbar_req_cnt; + bank_req_cnt_r <= bank_req_cnt; + end + end + + wire xbar_empty; VX_pending_size #( .SIZE (XBAR_SIZE), @@ -114,15 +139,17 @@ module VX_cache_init import VX_gpu_pkg::*; #( ) pending_size ( .clk (clk), .reset (reset), - .incr (INCRW'(xbar_req_cnt)), - .decr (bank_req_cnt), - .empty (no_inflight_reqs), + .incr (INCRW'(xbar_req_cnt_r)), + .decr (bank_req_cnt_r), + .empty (xbar_empty), `UNUSED_PIN (alm_empty), `UNUSED_PIN (full), `UNUSED_PIN (alm_full), `UNUSED_PIN (size) ); + assign no_inflight_reqs = xbar_empty && ~(| xbar_req_fire_r) && (xbar_req_cnt_r == '0); + end else begin : g_no_xbar_inflight assign no_inflight_reqs = 0; `UNUSED_VAR ({xbar_req_fire, bank_req_fire}) From b52d93c334131c75e878d1078d9ee24337b29347 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 30 Oct 2026 04:04:31 -0700 Subject: [PATCH 295/336] logic_gate: full-GPU builds and a registered-interface check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unit DUTs miss what only the assembled GPU has: the cluster, socket and L2 crossbars at full width, the kernel launch path into every core, and every extension at once. Vivado needs 48 h and more on those designs; Beacon needs about 30 min. A new opt-in group, gpu, runs the whole processor at 4 cores: gpu32_full (the 32-bit machine with every extension, tensor format, graphics unit, VM, DXA and DMA, 4-wide fetch and issue), gpu_rtx5080 and gpu_h100 (their profiles, XLEN 64). Core count only widens arbiters and crossbars (the deepest path was the same at 1-8 cores) while memory grows with it: ~95 GB at 4 cores. The group runs one build at a time (ci/logic_gate.py -b gpu -j 1), is its own catalog tier (timing_gpu) and a manually dispatched self-hosted job in logic_gate.yml. To get there, a build may pin its own xlen (it runs in that width's build tree) and a hardware profile, and may be opt_in; xlen, profile and interfaces join the config hash only when set, so existing hashes hold. A build's `interfaces` lists modules whose outgoing interfaces must be registered: their instances keep their ports through Beacon's mapping (BEACON_INTERFACES in the DUT flow), beacon_summary.py records, per output port, the bits a combinational path from outside reaches (ports.csv's `through`), and the gate fails on more bits than the baseline. On the core it finds the dcache/icache rsp_ready and kmu ready pass-throughs; on the GPU builds 33-77 cluster/socket/core outputs. Docs: AGENTS.md now states the Beacon-first loop (reference, --against, fix, synthesize once to confirm) and when to run the gpu group; beacon.md §6.3 gives the calibrated diff rule (+4 levels, n=170 of 170) and §8 the through column; continuous_integration.md the tier and the new assertion; synthesis_analysis.md points timing work at Beacon first. Validated: all three gpu builds and core complete (NO-BASELINE; baselines are recorded by a human). Requires foldsim with the ports `through` column (foldsim 6863e69): the foldsim prebuilt must be refreshed before CI runs a build with `interfaces`. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/logic_gate.yml | 76 +++++++++++++++++++++- AGENTS.md | 2 +- ci/synth_gate.py | 87 ++++++++++++++++++++------ ci/testcase.py | 9 +-- ci/testcases/logic_gate.yaml | 74 ++++++++++++++++++++++ docs/designs/beacon.md | 26 +++++--- docs/designs/continuous_integration.md | 48 ++++++++++++-- docs/synthesis_analysis.md | 2 + hw/scripts/beacon_summary.py | 32 ++++++++-- hw/syn/xilinx/dut/common.mk | 15 ++++- 10 files changed, 327 insertions(+), 44 deletions(-) diff --git a/.github/workflows/logic_gate.yml b/.github/workflows/logic_gate.yml index c9ceb4791d..a9d5a6e4eb 100644 --- a/.github/workflows/logic_gate.yml +++ b/.github/workflows/logic_gate.yml @@ -45,6 +45,10 @@ on: builds: description: "build ids/groups, space separated (blank = all)" default: "" + gpu: + description: "run the full-GPU builds (group gpu) on the self-hosted machine instead" + type: boolean + default: false concurrency: group: logic-gate-${{ github.ref }} @@ -55,6 +59,7 @@ env: jobs: beacon: + if: ${{ !inputs.gpu }} runs-on: ubuntu-22.04 timeout-minutes: 90 steps: @@ -77,7 +82,7 @@ jobs: ../configure --tooldir=$TOOLDIR ci/toolchain_install.sh --foldsim - # The command is the one the catalog declares (the single `run:` case of + # The command is the one the catalog declares (the `gate` case of # ci/testcases/logic_gate.yaml), so what CI runs and what the catalog # says cannot drift apart. - name: Gate @@ -86,7 +91,7 @@ jobs: IN_BUILDS: ${{ github.event.inputs.builds }} run: | set -uo pipefail - CMD=$(python3 -c "import yaml; print(yaml.safe_load(open('ci/testcases/logic_gate.yaml'))['tests'][0]['run'])") + CMD=$(python3 -c "import yaml; print(next(t['run'] for t in yaml.safe_load(open('ci/testcases/logic_gate.yaml'))['tests'] if t['id'] == 'gate'))") # Dispatch inputs reach the shell through an env var, never # interpolated into the script. ARGS=() @@ -123,3 +128,70 @@ jobs: 1) echo "::error::logic_gate FAILED — a unit got deeper than its baseline"; exit 1 ;; *) echo "::error::logic_gate build error (see logs)"; exit 1 ;; esac + + # The full-GPU builds: ~30 min and ~95 GB each, run one at a time, which no + # hosted runner has the memory for. Manual dispatch only (gpu: true), on the + # self-hosted machine the FPGA gate uses. + gpu: + if: ${{ inputs.gpu }} + runs-on: [self-hosted, linux, vivado] + timeout-minutes: 480 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Vortex + uses: ./.github/actions/setup-vortex + with: + prepare: true + + - name: Install foldsim + run: | + set -euo pipefail + mkdir -p tc_foldsim && cd tc_foldsim + ../configure --tooldir=$TOOLDIR + ci/toolchain_install.sh --foldsim + + # The catalog's `gpu` case; a dispatch's builds narrow it. + - name: Gate + id: run + env: + IN_BUILDS: ${{ github.event.inputs.builds }} + run: | + set -uo pipefail + CMD=$(python3 -c "import yaml; print(next(t['run'] for t in yaml.safe_load(open('ci/testcases/logic_gate.yaml'))['tests'] if t['id'] == 'gpu'))") + ARGS=() + for b in ${IN_BUILDS:-}; do ARGS+=(-b "$b"); done + rc=0 + $CMD "${ARGS[@]}" --report logic_gate_gpu.json || rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Summarize + if: always() + run: | + set -uo pipefail + python3 ci/synth_report.py --annotate error logic_gate_gpu.json \ + > logic_gate_gpu_report.md || true + cat logic_gate_gpu_report.md >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: logic-gate-gpu + path: | + logic_gate_gpu.json + logic_gate_gpu_report.md + build*_logic_gate/hw/syn/xilinx/dut/logic_gate_gpu*/build.log + build*_logic_gate/hw/syn/xilinx/dut/logic_gate_gpu*/beacon_* + build*_logic_gate/hw/syn/xilinx/dut/logic_gate_gpu*/beacon/report.txt + if-no-files-found: warn + + - name: Verdict + run: | + case "${{ steps.run.outputs.rc }}" in + 0) echo "logic_gate gpu passed" ;; + 1) echo "::error::logic_gate gpu FAILED — a unit got deeper than its baseline"; exit 1 ;; + *) echo "::error::logic_gate gpu build error (see logs)"; exit 1 ;; + esac diff --git a/AGENTS.md b/AGENTS.md index c553eb0a55..7a6753dde9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,7 +87,7 @@ See [docs/testing.md](docs/testing.md) and [docs/debugging.md](docs/debugging.md - **`ci/regression.sh` is the canonical source of tested configurations.** Use it to discover supported parameter combinations before inventing ad hoc ones. - **Perf-regression baselines (`ci/baselines/perf/*.json`) are golden data — never hand-edit them, and never "fix" a red perf gate by bumping the number.** They are regenerated only by `pytest ci -m perf_gate --update-baselines` (a human-run, reviewed step), and CI must never pass that flag. A `perf_gate` failure means real cycles moved: root-cause it, or — if the change is intended — regenerate the baseline so the diff shows the perf delta for review. Same rule as image goldens and `known_issue:`. - **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,beacon}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/logic_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `logic_gate` means a unit got deeper, and its reasons name the unit. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). -- **Check the logic depth of an RTL change with Beacon before committing it.** A change to a pipeline, arbiter, queue or datapath can add logic levels that only the nightly `fpga_gate` would find. Run `make -C hw/syn/xilinx/dut -beacon`, or `ci/logic_gate.py -b `, for the DUTs it touches. A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. See [docs/designs/beacon.md](docs/designs/beacon.md). +- **Beacon first for any RTL change that can move timing; synthesis only to confirm.** A change to a pipeline, arbiter, queue, interface or datapath can add logic levels that only the nightly `fpga_gate` would find, hours later. Beacon finds them in minutes, so iterate on it, not on Vivado or Yosys: run `ci/logic_gate.py -b ` on the parent revision once (its own build tree, e.g. `git worktree add ../ref HEAD` and `--build-dir build32_beacon_ref`), then `ci/logic_gate.py -b --against build32_beacon_ref` after each edit, and fix what `/beacon_diff.txt` reports until nothing fails (`ci/beacon.py paths` shows a deep unit's paths op by op). Run synthesis once per change, on the DUTs whose depth moved, and feed the result back with `ci/beacon.py validate`. A change to the cluster, socket, L2 or anything only the whole GPU assembles also runs the `gpu` group (`ci/logic_gate.py -b gpu -j 1`, ~30 min and ~95 GB per build). A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. Beacon does not see wires, placement or congestion; a change that moves those still needs synthesis. See [docs/designs/beacon.md §6.3](docs/designs/beacon.md). - **SimX is the RTL's timing model — keep them in lockstep.** Any change that moves RTL cycles (pipeline structure, arbitration, queue depths, cache/memory behavior) must land together with the matching SimX timing-model update, and vice-versa. The `model_parity` CI gate enforces this: a `check: model_parity` case (`ci/testcases/core.yaml` + per-extension parity categories) runs the same app/args/configs on simx and rtlsim and asserts exact retired-instruction match plus cycle agreement within the case tolerance (default 5%). Never widen a tolerance to absorb a divergence — model the behavior. When adding a hardware feature, add or extend a parity case that exercises it. - **When RTL debugging stalls, switch to the SimX-as-oracle pattern.** For numerical bugs, deep pipeline races, or any failure mode where rtlsim is "close but wrong": (1) build/extend the SimX C++ model so it mirrors the *new* RTL architecture and gets to PASS; (2) add matching trace dumps to both SimX and RTL (cycle, FU events, SRAM addresses+data, hazards) — same CSV format on both sides; (3) diff trace files — the first divergence is the bug. Don't keep guessing from output values; localize via trace diff. See [docs/debugging.md](docs/debugging.md#simx-as-oracle-for-rtl-debug). diff --git a/ci/synth_gate.py b/ci/synth_gate.py index f99da2b0f5..90e92b982c 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -547,8 +547,9 @@ def build_command(self, dut_root, work, build, env, args): os.makedirs(work, exist_ok=True) shutil.copy2(os.path.join(dut_root, "build.mk"), os.path.join(work, "Makefile")) - return ["make", "-C", work, "beacon"], dict(DUT=build["dut"], - CONFIGS=build["configs"]) + return ["make", "-C", work, "beacon"], dict( + DUT=build["dut"], CONFIGS=build["configs"], + BEACON_INTERFACES=build.get("interfaces", "")) def extras(self, work): try: @@ -557,6 +558,7 @@ def extras(self, work): except (OSError, ValueError): return {} return {"units": doc.get("units", {}), + "interfaces": doc.get("interfaces", {}), "deepest_paths": doc.get("deepest_paths", []), "fix_levels": doc.get("fix_levels"), "watch_levels": doc.get("watch_levels")} @@ -567,7 +569,9 @@ def incomplete(self, metrics): def compare(self, metrics, base): """Per unit: deeper than its baseline fails; a new unit at the fix threshold fails; shallower, or gone from the watch list, is an - improvement to lock in.""" + improvement to lock in. Per interface output: more bits reached + combinationally from outside its instance fails, fewer is an + improvement.""" cur, ref = metrics.get("units"), base.get("units") if cur is None or ref is None: return [] @@ -589,6 +593,16 @@ def compare(self, metrics, base): "REGRESSION" if now > was else "improvement, update the baseline to lock " "it in")) + cur, ref = metrics.get("interfaces") or {}, base.get("interfaces") or {} + for port in sorted(set(cur) | set(ref)): + now, was = cur.get(port, 0), ref.get(port, 0) + if now != was: + reasons.append("interface %s: %d -> %d bits reached " + "combinationally from outside -- %s" + % (port, was, now, + "REGRESSION, register it" if now > was else + "improvement, update the baseline to lock " + "it in")) return reasons def evidence(self, work): @@ -671,7 +685,7 @@ def baseline_path(root, tool, group): return os.path.join(baseline_dir(root, tool), group + ".json") -def load_catalog(root, tool): +def load_catalog(root, tool, xlen=32): """Builds keyed by id: the SPEC from ci/testcases/.yaml, joined to its recorded metrics from ci/baselines/synthesis//.json. @@ -712,6 +726,12 @@ def load_catalog(root, tool): build = dict(entry) build["group"] = group build["configs"] = entry["configs"] or "" + # A build may pin its own ISA width and hardware profile + # (profiles/.json, folded under its configs by config.mk); a + # profile that models a 64-bit machine needs xlen: 64. + build["xlen"] = entry.get("xlen", xlen) + build["profile"] = entry.get("profile", "") + build["opt_in"] = bool(entry.get("opt_in", False)) build["known_issue"] = entry.get("known_issue", defaults.get("known_issue", "")) build["thresholds"] = entry.get("thresholds", @@ -730,9 +750,10 @@ def load_catalog(root, tool): def select(builds, selectors): """Resolve --build selectors: an exact build id wins over a group of the - same name; otherwise a group selects all its builds.""" + same name; otherwise a group selects all its builds. With no selector, every + build but the opt-in ones, which run only when named.""" if not selectors: - return [builds[b] for b in sorted(builds)] + return [builds[b] for b in sorted(builds) if not builds[b]["opt_in"]] picked = {} for sel in selectors: hits = [b for b in builds.values() if b["id"] == sel] @@ -748,11 +769,15 @@ def select(builds, selectors): def config_hash(build, env, tool): """Fingerprint the synthesis inputs; a change invalidates the stored metrics.""" key = "|".join([build["dut"], str(build["clock_mhz"]), build["configs"]] - + tool.hash_key(env)) - # Appended only when set, so builds without a strategy override keep the - # hash their baselines were recorded under. + + tool.hash_key(dict(env, xlen=build["xlen"]))) + # Appended only when set, so builds without a strategy override or a + # profile keep the hash their baselines were recorded under. if build.get("impl_strategy"): key += "|" + build["impl_strategy"] + if build.get("profile"): + key += "|profile=" + build["profile"] + if build.get("interfaces"): + key += "|interfaces=" + build["interfaces"] return hashlib.sha256(key.encode()).hexdigest()[:16] @@ -783,10 +808,12 @@ def save_baselines(root, tool, results, env): "clock_mhz": b["clock_mhz"], "configs": b["configs"], "config_hash": config_hash(b, env, tool), - "result": dict(r["metrics"], env=dict(env)), + "result": dict(r["metrics"], env=dict(env, xlen=b["xlen"])), } if b.get("impl_strategy"): entries[b["id"]]["impl_strategy"] = b["impl_strategy"] + if b.get("profile"): + entries[b["id"]]["profile"] = b["profile"] with open(path, "w") as fh: json.dump(entries, fh, indent=2, sort_keys=True) fh.write("\n") @@ -1060,6 +1087,9 @@ def run_build(build, build_dir, env, args, tool): benv = dict(os.environ) argv, extra = tool.build_command(dut_root, work, build, env, args) benv.update(extra) + # Set even when empty, so a PROFILE in the caller's environment cannot + # leak into a build that names none. + benv["PROFILE"] = build["profile"] if resume: benv["RESUME"] = "1" # make clean is a no-op; keeps project_1 + *.dcp @@ -1271,6 +1301,9 @@ def report_entry(r, env, tool): "reasons": r["reasons"]} if b.get("impl_strategy"): entry["impl_strategy"] = b["impl_strategy"] + entry["xlen"] = b["xlen"] + if b.get("profile"): + entry["profile"] = b["profile"] return entry @@ -1412,7 +1445,8 @@ def main(argv=None, default_tool="xilinx"): "same build there (ci/beacon.py diff), and a register " "that got deeper fails the run") ap.add_argument("--xlen", type=int, default=32, choices=(32, 64), - help="ISA width (default: 32)") + help="ISA width of the builds that do not pin their own " + "(default: 32)") ap.add_argument("--opt-level", type=int, default=3, metavar="N", help="Vivado optimization level, 0=fastest 3=default") ap.add_argument("--timeout", type=int, default=8 * 3600, metavar="SEC", @@ -1453,7 +1487,7 @@ def main(argv=None, default_tool="xilinx"): args.metric_threshold = overrides root = source_root() - builds = load_catalog(root, tool) + builds = load_catalog(root, tool, args.xlen) if args.matrix: # Ordered longest-first so the slowest DUT is dispatched into the fan-out @@ -1461,7 +1495,7 @@ def main(argv=None, default_tool="xilinx"): # argument run_all() makes for its thread pool. print(json.dumps(sorted( [{"id": b["id"], "group": b["group"], "dut": b["dut"], - "clock_mhz": b["clock_mhz"], + "clock_mhz": b["clock_mhz"], "xlen": b["xlen"], "known_issue": bool(b.get("known_issue"))} for b in select(builds, args.build)], key=lambda b: (-weight(builds[b["id"]], tool), b["id"])))) @@ -1470,8 +1504,10 @@ def main(argv=None, default_tool="xilinx"): if args.list: for b in select(builds, args.build): base = b.get("baseline") or {} - print("%-10s [%s] dut=%-8s %4d MHz %s%s" - % (b["id"], b["group"], b["dut"], b["clock_mhz"], + print("%-10s [%s] dut=%-8s %4d MHz xlen %d%s%s %s%s" + % (b["id"], b["group"], b["dut"], b["clock_mhz"], b["xlen"], + " profile " + b["profile"] if b["profile"] else "", + " (opt-in)" if b["opt_in"] else "", "baseline: " + tool.describe_baseline(base) if base.get(tool.key_metric) is not None else "no baseline", @@ -1490,8 +1526,6 @@ def main(argv=None, default_tool="xilinx"): args.tool_jobs = max(MIN_TOOL_JOBS, min(MAX_TOOL_JOBS, os.cpu_count() // args.jobs)) - build_dir = args.build_dir or os.path.join( - root, "build%d_%s" % (args.xlen, tool.gate)) env = tool.env(args) selected = select(builds, args.build) @@ -1500,11 +1534,24 @@ def main(argv=None, default_tool="xilinx"): if warning: print("WARNING: %s %s" % (b["id"], warning)) - if not args.dry_run: - prepare_build_tree(root, build_dir, args.xlen) - results = run_all(selected, build_dir, env, args, tool) + # One configured build tree per ISA width: a build that pins its own xlen + # runs in that width's tree. + xlens = sorted({b["xlen"] for b in selected}) + if len(xlens) > 1 and (args.build_dir or args.against): + sys.exit("ERROR: --build-dir/--against name one build tree, and the " + "selected builds span xlen %s; select one width" + % "/".join(map(str, xlens))) + results = [] + for xlen in xlens: + build_dir = args.build_dir or os.path.join( + root, "build%d_%s" % (xlen, tool.gate)) + if not args.dry_run: + prepare_build_tree(root, build_dir, xlen) + results += run_all([b for b in selected if b["xlen"] == xlen], + build_dir, env, args, tool) if args.dry_run: return 0 + results.sort(key=lambda r: r["build"]["id"]) gated = tuple(m for m in (args.gate.split(",") if args.gate else tool.gated) if m) outcomes = report(results, env, args, gated, tool, args.update_baseline) diff --git a/ci/testcase.py b/ci/testcase.py index 1b501445b5..09d25e362f 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -48,10 +48,11 @@ # nightly), and everything is what ci.yml's matrix can run as a CELL -- these # cannot be. `fpga` needs a licensed Vivado and hours of a whole machine; # `asic` needs 1-2 hours PER DUT, which is a fan-out of standalone jobs, not one -# cell; `timing` needs foldsim, which the base toolchain does not install. Each -# has its own workflow (.github/workflows/{fpga,asic,timing}_gate.yml). -# Opting in is explicit, never implied. -OPT_IN_TIERS = {"fpga", "asic", "timing"} +# cell; `timing` needs foldsim, which the base toolchain does not install, and +# `timing_gpu` also ~95 GB per build. Each has its own workflow +# (.github/workflows/{fpga,asic,logic}_gate.yml). Opting in is explicit, never +# implied. +OPT_IN_TIERS = {"fpga", "asic", "timing", "timing_gpu"} # simx<->rtlsim cycle-parity default: both timing models must agree within 5%. DEFAULT_PARITY_TOLERANCE = 0.05 diff --git a/ci/testcases/logic_gate.yaml b/ci/testcases/logic_gate.yaml index 56b3061da7..011c1806f6 100644 --- a/ci/testcases/logic_gate.yaml +++ b/ci/testcases/logic_gate.yaml @@ -55,6 +55,13 @@ tests: - id: gate run: ci/logic_gate.py -j 2 +# The full-GPU builds (group gpu, below): ~30 min and ~95 GB of memory each, +# so one at a time, on a large self-hosted machine -- never a hosted runner. +# Its own opt-in tier keeps `--tier timing` from sweeping it in. +- id: gpu + tier: timing_gpu + run: ci/logic_gate.py -b gpu -j 1 --timeout 7200 + # --------------------------------------------------------------------------- # builds — read by ci/logic_gate.py, ignored by the pytest catalog loader. # @@ -62,6 +69,15 @@ tests: # clock_mhz the Fmax gate's target for this DUT (recorded, not asserted) # configs EXTRA defines, appended to the catalog's # group which golden file the results land in +# xlen ISA width, when not the gate's default (32); the build runs in +# that width's own build tree +# profile hardware profile (profiles/.json) under the configs; +# configs win over it +# opt_in run only when selected by id or group, never by default +# interfaces modules whose outputs must be registered (space separated): +# an output bit a combinational path from outside the +# instance reaches is recorded, and one more than the +# baseline fails # known_issue tracked expected failure: still measures and reports, does not # fail the run # --------------------------------------------------------------------------- @@ -83,6 +99,7 @@ builds: dut: core clock_mhz: 300 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE + interfaces: VX_core # Warp-wide local memory with two LSU blocks: both pass engines, the # dual-port banks the read engine reads through, and the atomic logic. @@ -180,3 +197,60 @@ builds: dut: tensor clock_mhz: 250 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# --- gpu (opt-in): the whole processor with every feature on ------------------ +# +# The per-unit DUTs above miss what only the assembled GPU has: the +# cluster/socket/L2 crossbars at full width, the kernel launch path into every +# core, and the cross-unit boundaries with every extension on at once. Vivado +# needs 48 h and more on these; Beacon ~30 min. Four cores, not the products' +# 84-132: core count widens only the arbiters and crossbars -- the deepest path +# was the same 43 levels at 1, 2, 4 and 8 cores -- while memory grows with it +# (~95 GB at 4, 121 GB at 8). Run with `ci/logic_gate.py -b gpu -j 1`. + +# Everything the 32-bit machine has: every extension and tensor format, VM, +# DXA and DMA, the graphics pipeline, ray tracing with TLAS, a 4-wide +# frontend and issue. (D is 64-bit only.) +- id: gpu32_full + group: gpu + opt_in: true + dut: vortex + clock_mhz: 300 + interfaces: VX_core VX_socket VX_cluster + configs: >- + -DVX_CFG_NUM_CORES=4 -DVX_CFG_NUM_CLUSTERS=1 + -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 + -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE -DVX_CFG_EXT_ZACAS_ENABLE + -DVX_CFG_EXT_ZFHMIN_ENABLE -DVX_CFG_EXT_MUFU_ENABLE -DVX_CFG_VM_ENABLE + -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TF32_ENABLE -DVX_CFG_TCU_FP16_ENABLE + -DVX_CFG_TCU_FP8_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_INT8_ENABLE + -DVX_CFG_TCU_INT4_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_MXFP4_ENABLE + -DVX_CFG_TCU_NVFP4_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DVX_CFG_TCU_DSM_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_EXT_DMA_ENABLE + -DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_RTU_TLAS_ENABLE + -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + -DVX_CFG_L2_ENABLE + +# RTX 5080's SM: 48 warps x 32 threads, the graphics pipeline, ray tracing, +# MX/FP4 tensor formats, a 32-bank L2 -- at XLEN 64 with D. +- id: gpu_rtx5080 + group: gpu + opt_in: true + dut: vortex + clock_mhz: 300 + interfaces: VX_core VX_socket VX_cluster + xlen: 64 + profile: rtx5080 + configs: -DVX_CFG_NUM_CORES=4 -DVX_CFG_NUM_CLUSTERS=1 + +# H100's SM: 64 warps and WGMMA, which the RTX 5080 profile does not have. +- id: gpu_h100 + group: gpu + opt_in: true + dut: vortex + clock_mhz: 300 + interfaces: VX_core VX_socket VX_cluster + xlen: 64 + profile: h100 + configs: -DVX_CFG_NUM_CORES=4 -DVX_CFG_NUM_CLUSTERS=1 diff --git a/docs/designs/beacon.md b/docs/designs/beacon.md index ba009fd413..f29a70e846 100644 --- a/docs/designs/beacon.md +++ b/docs/designs/beacon.md @@ -381,11 +381,19 @@ fixed while the change is still open, and synthesis runs once, to confirm. ``` Besides the per-unit verdict against the golden baseline, each build - writes `beacon_diff.txt` (`ci/beacon.py diff`). It lists the registers - that got deeper, new deep registers, and the ones that got shallower, then - the units whose deepest register moved. A register that got deeper fails - the run (`DEEPER`), and so does a new one at the fix threshold (12). A new - one below it is listed but passes, since a rename looks the same. + writes `beacon_diff.txt` (`ci/beacon.py diff`). Its headline is the + deepest register, the mean of the 50 deepest and the count at the watch + threshold, before and after; then it lists the registers that got deeper, + new deep registers, and the ones that got shallower, then the units whose + deepest register moved. The run fails (`DEEPER`) on a register 4 or more + levels deeper, one that reaches the fix threshold (12) from below, or a new + one at it. These limits come from 13 routed U55C builds: a register 4 or + more levels deeper lost slack in Vivado in every case (170 of 170), while + 1 to 3 levels is within the placer's noise, so those are listed as + "deeper by less" to watch, not failed. Of the headline numbers, the top-50 + mean follows Vivado's worst slack (Spearman 0.56) and the single deepest + register does not (0.29). A new register below 12 is listed but passes, + since a rename looks the same. 3. **Fix.** `ci/beacon.py paths /beacon --unit ` prints a deep unit's paths op by op. Edit, and repeat step 2 until nothing got deeper, @@ -535,8 +543,12 @@ So the model is a ranking for where to look first, not a gate. `diff` and ## 8. Diagnostics beyond levels **Port depths.** `port-modules=` names modules whose instances' ports -Beacon reports: the levels at which each input arrives and the levels behind -each output. Small modules are inlined by the mapping, so keep them with +Beacon reports: the levels at which each input arrives, the levels behind +each output, and `through`, the bits of each output that a combinational +path from outside the instance reaches. A module that registers its +outgoing interface has `through` 0 on every output; `logic_gate` gates it +for the modules a build lists under `interfaces` (`BEACON_INTERFACES` in +the DUT flow). Small modules are inlined by the mapping, so keep them with `run --keep`: ```bash diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index d988c0520e..9caf98dc5c 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -287,7 +287,7 @@ catches one that makes a unit **deeper**, in seconds and without synthesis. | gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | `max_levels` against the baseline, and the deepest end point of every RTL unit | | also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | end point bits at ≥ 12 and ≥ 8 levels, the deepest paths | | comparable when | same device, optimization level, xlen | same PDK, VT, corner, xlen | same CIRCT, foldsim, xlen | -| runner | self-hosted, licensed Vivado | hosted, one job per design | hosted, one job for the catalog | +| runner | self-hosted, licensed Vivado | hosted, one job per design | hosted, one job for the catalog; the opt-in `gpu` group self-hosted | | default tolerance | Fmax 2 %, LUT 5 % | Fmax 2 %, cell area 5 % | none: Beacon is deterministic | None is a pytest cell. A cell is a build tree plus a driver; a gate build is @@ -312,14 +312,21 @@ entry points that pin the tool. | file | written by | holds | |---|---|---| -| `ci/testcases/{fpga,asic,timing}_gate.yaml` | a person | per build: a design, a target clock, an area budget (synthesis gates), `CONFIGS`, a group, optional `thresholds` and `known_issue` | +| `ci/testcases/{fpga,asic,logic}_gate.yaml` | a person | per build: a design, a target clock, an area budget (synthesis gates), `CONFIGS`, a group, optional `thresholds`, `known_issue`, `xlen`, `profile` and `opt_in` | | `ci/baselines/synthesis/{xilinx,yosys,beacon}/.json` | the gate | measured metrics, the config hash, the tool versions and environment | -Groups are `core`, `tensor`, `graphics`, `dxa` and `vm`. `asic_gate` uses the +Groups are `core`, `tensor`, `graphics`, `dxa` and `vm`, plus `logic_gate`'s +opt-in `gpu`. `asic_gate` uses the same group names and gates a subset of the builds (§4.5), so where the two gates share a design, a divergence between the tools on the same module is a finding, not noise. `logic_gate` measures exactly `fpga_gate`'s builds, so a -unit it flags can be read against the Fmax gate's critical paths. +unit it flags can be read against the Fmax gate's critical paths; its `gpu` +group adds builds no synthesis gate can afford. + +A build may pin its own `xlen`, and runs in that width's build tree +(`build_`); `profile` folds a hardware profile +(`profiles/.json`) under its `CONFIGS`, which win over it. Both are in +the config hash. An `opt_in` build runs only when selected by id or group. The config hash ties the two files together: edit the spec, and the gate refuses to compare against numbers recorded for the old one. It matters only @@ -384,6 +391,12 @@ Beacon has profiles for the V80 and the U55C only, and fails on any other | no unit deeper than its baseline | the change that adds a level is named, by unit | | no new unit at the fix threshold (12 levels at 300 MHz) | a new deep cone is a regression even with no baseline to compare | | a shallower unit, or one that drops below the watch threshold (8), reads `IMPROVED` | the gain must be recorded | +| no output of a build's `interfaces` modules reached combinationally from outside the instance in more bits than its baseline | modules register their outgoing interfaces: a path that runs through a core, socket or cluster boundary is what fails routing across SLRs, and Beacon names it by port | + +`interfaces` lists modules by name; their instances keep their ports through +Beacon's mapping (`BEACON_INTERFACES` in the DUT flow), and `ports.csv` +counts, per output port, the bits a combinational path from outside the +instance reaches. There is no tolerance: the same RTL and tools give the same levels. A new CIRCT or foldsim can map the same RTL differently, so both versions are in the config @@ -391,6 +404,29 @@ hash and a tool change reads `STALE`. `make -C hw/syn/xilinx/dut -beacon` runs Beacon on one DUT by hand; its `beacon_paths.txt` lists every deep path op by op, by unit, with source lines, and a failing build names it. +#### Full-GPU builds + +The `gpu` group runs Beacon on the whole processor (`vortex` DUT) with every +feature on: the 32-bit machine with every extension, tensor format and +graphics unit, and the `rtx5080` and `h100` profiles at XLEN 64. It catches +what the per-unit DUTs cannot: the cluster, socket and L2 crossbars at full +width, the kernel launch path into every core, and every extension at once. +Vivado needs 48 hours and more on these designs; Beacon needs about 30 minutes. + +| cores | time | peak memory | end points | +|---|---|---|---| +| 1 | 37 min | 101 GB | 7.1 M | +| 2 | 24 min | 90 GB | 13.2 M | +| 4 | 32 min | 94 GB | 25.4 M | +| 8 | 49 min | 121 GB | 50.1 M | + +Measured with the `rtx5080` features at XLEN 64 on a loaded machine. The +builds use 4 cores: the core count widens only the arbiters and crossbars, +and the deepest path was the same at every count, while memory grows with it. +The group is opt-in in the gate (`ci/logic_gate.py -b gpu -j 1`) and its own +tier, `timing_gpu`, in the catalog, so neither the default run nor +`--tier timing` sweeps it in. + #### Running a multi-hour sweep | mechanism | what it does | @@ -549,6 +585,10 @@ installs foldsim (`ci/toolchain_install.sh --foldsim`), runs the catalog's artifacts are the report and, per design, the build log, the summary and Beacon's deepest paths. +A manual dispatch with `gpu: true` runs the catalog's `gpu` case instead, on +the self-hosted machine: one build at a time, up to two hours each, since each +needs about 95 GB of memory. + --- ## 5. Running locally and coverage diff --git a/docs/synthesis_analysis.md b/docs/synthesis_analysis.md index d6ee632a22..af1349ccda 100644 --- a/docs/synthesis_analysis.md +++ b/docs/synthesis_analysis.md @@ -2,6 +2,8 @@ This guide explains how to run synthesis, timing analysis, area analysis, and power analysis for Vortex across all supported back-ends: Xilinx (Vivado), Altera (Quartus), Yosys (open-source), and Synopsys Design Compiler. +**Timing work starts in Beacon, not here.** For an RTL change that can move timing, iterate on [Beacon](designs/beacon.md) (§6.3): it counts the logic levels of every register in minutes, compares them with the parent revision's (`ci/logic_gate.py -b --against `), and names the registers a change made deeper. Synthesize once the change passes there, to confirm Fmax and to measure what Beacon does not see: wires, placement, congestion, area and power. + --- ## Table of Contents diff --git a/hw/scripts/beacon_summary.py b/hw/scripts/beacon_summary.py index d92fa92e30..31cc0e1d25 100644 --- a/hw/scripts/beacon_summary.py +++ b/hw/scripts/beacon_summary.py @@ -12,6 +12,10 @@ beacon_paths.txt if Beacon wrote paths.jsonl: every path at or above the watch threshold, op by op, by unit (write_paths) +If Beacon wrote ports.csv (BEACON_INTERFACES), beacon_units.json also holds +`interfaces`: per output port of those modules' instances, folded like +units, the bits a combinational path from outside the instance reaches. + A unit is an end point's instance path with generate indices folded (g_banks_3_bank -> g_banks_*_bank), so one deep bank reads as one unit. @@ -114,11 +118,16 @@ def main(): w = csv.DictWriter(fh, fieldnames=COLUMNS) w.writeheader() w.writerow(summary) + doc = {"fix_levels": FIX_LEVELS, "watch_levels": WATCH_LEVELS, + "units": dict(sorted(units.items())), + "deepest_paths": ["%d %s -> %s" % d for d in deepest]} + ports = os.path.join(args.run_dir, "ports.csv") + if os.path.exists(ports): + doc["interfaces"] = interfaces(ports) + print("beacon: %d interface outputs reached combinationally from " + "outside" % len(doc["interfaces"])) with open(os.path.join(args.output_dir, "beacon_units.json"), "w") as fh: - json.dump({"fix_levels": FIX_LEVELS, "watch_levels": WATCH_LEVELS, - "units": dict(sorted(units.items())), - "deepest_paths": ["%d %s -> %s" % d for d in deepest]}, - fh, indent=2) + json.dump(doc, fh, indent=2) fh.write("\n") print("beacon: %s max %d levels, %d end point bits >= %d, %d units >= %d" % (args.top, summary["max_levels"], summary["deep_endpoints"], @@ -134,6 +143,21 @@ def fold(name): return INDEX.sub("_*", norm(name)) +def interfaces(path): + """'.' -> bits, for every output port of a kept module's + instance that a combinational path from outside reaches; generate copies + folded, keeping the most bits.""" + out = {} + with open(path) as fh: + for row in csv.DictReader(fh): + bits = int(row["through"]) + if row["direction"] != "out" or not bits: + continue + key = "%s.%s" % (fold(row["instance"]), row["port"]) + out[key] = max(out.get(key, 0), bits) + return dict(sorted(out.items())) + + def write_paths(out, run_dir, min_levels, unit_filter=None, paths=None): """Write a run's deep paths (paths.jsonl) by unit, op by op. diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index fac58b65d0..a36655468e 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -96,14 +96,25 @@ power: BEACON_PATH_LEVELS ?= 8 # The Beacon target: the family of the device, as hw/syn/xilinx/platforms names it. BEACON_TARGET = $(shell python3 $(SCRIPT_DIR)/gen_platform.py --device=$(DEVICE) --emit=family) +# Modules whose interfaces must be registered (space separated): their +# instances keep their ports through mapping, and beacon/ports.csv counts, per +# output port, the bits a combinational path from outside the instance +# reaches (`through`). beacon_summary.py records them for logic_gate. +BEACON_INTERFACES ?= +beacon_comma := , +beacon_empty := +beacon_space := $(beacon_empty) $(beacon_empty) +beacon_keep = $(subst $(beacon_space),$(beacon_comma),$(strip $(BEACON_INTERFACES))) +BEACON_MAP = $(if $(beacon_keep),foldsim-keep-modules{modules=$(beacon_keep)}$(beacon_comma))foldsim-beacon-map{target=$(BEACON_TARGET)} +BEACON_PORTS = $(if $(beacon_keep),ports=beacon/ports.csv port-modules=$(beacon_keep)) beacon: mkdir -p beacon $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -Obeacon/sources.f $(RTL_PKGS) $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t $(TOP_LEVEL_ENTITY) -l beacon/sources.f \ -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r beacon/memories.rpt -o beacon/netlist.mlir - $(FOLDSIM_OPT) beacon/netlist.mlir --pass-pipeline='builtin.module(foldsim-beacon-map{target=$(BEACON_TARGET)})' \ + $(FOLDSIM_OPT) beacon/netlist.mlir --pass-pipeline='builtin.module($(BEACON_MAP))' \ --emit-bytecode -o beacon/mapped.mlirbc - $(FOLDSIM_OPT) beacon/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) target=$(BEACON_TARGET) output=beacon/endpoints.csv instances=beacon/instances.csv report=beacon/report.txt paths=beacon/paths.jsonl path-levels=$(BEACON_PATH_LEVELS)})" + $(FOLDSIM_OPT) beacon/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) target=$(BEACON_TARGET) output=beacon/endpoints.csv instances=beacon/instances.csv report=beacon/report.txt paths=beacon/paths.jsonl path-levels=$(BEACON_PATH_LEVELS) $(BEACON_PORTS)})" python3 $(SCRIPT_DIR)/beacon_summary.py beacon --top $(TOP_LEVEL_ENTITY) clean: From 6f08ef66e034d26e0ba8ba0b59c91327754694d9 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 30 Oct 2026 10:41:56 -0700 Subject: [PATCH 296/336] cta_dispatch: shorten the launch-select and slot-base paths The CTA dispatcher was the deepest unit in the core: 17 logic levels in the logic_gate core build, and 43 at the rtx5080 full-GPU shape, where the per-lane warp selection chained a priority encoder over the lanes into the thread-mask and round-robin updates. Lanes now take their order from popcounts of the ready lanes ahead of them in the rotation (NUM_LANES is asserted a power of two), the warps left are compared as thermometers against each launch position, the LMEM base product is formed a cycle after accept from latched operands (its first reader comes two cycles later), and the occupancy bound is a registered thermometer of constant comparisons, so there is neither a multiplier nor a divider on the ready path. Beacon: core 17 -> 11; rtx5080 full GPU 43 -> 36 (cta_dispatcher 43 -> 14, rem_warps_ram 24 -> 10), no unit deeper. Launch cycles are identical on the regression and CTA tests. This also fixes a hang at NUM_WARPS=1 when a block's last warp is partial: the old select never retired the final partial warp. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_cta_dispatch.sv | 343 +++++++++++++++++++-------------- 1 file changed, 198 insertions(+), 145 deletions(-) diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index b33f1241e7..4d0196faff 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -92,7 +92,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // Fixed-stride LMEM partition: resident CTA in slot i gets LMEM base // i * stride, where stride = aligned_lmem_size (uniform within a kernel). - // cur_lmem_base_r is latched at accept and held stable through DISPATCH. + // The product is formed from the slot and stride latched at accept, a cycle + // later; its first reader, the context write of the first launched warp, + // comes at least two cycles after accept. + reg [`VX_CFG_LMEM_LOG_SIZE:0] cur_stride_r; reg [`VX_CFG_LMEM_LOG_SIZE-1:0] cur_lmem_base_r; // Reverse lookup: warp-ID → CTA slot index. A flop array indexed by wid, @@ -124,6 +127,7 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( reg [PC_BITS-1:0] entry_r; reg [2:0][31:0] block_idx_r; reg [2:0][CTA_TID_WIDTH:0] block_dim_r; + reg [1:0][CTA_TID_WIDTH-1:0] block_last_r; reg [2:0][31:0] grid_dim_r; reg [`VX_CFG_MEM_ADDR_WIDTH-1:0] param_r; reg [CTA_TID_WIDTH:0] block_size_r; @@ -234,12 +238,8 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // ------------------------------------------------------------------------- wire [NW_WIDTH:0] kmu_num_warps; wire [CTA_TID_WIDTH:0] block_size_next; - wire [CTA_TID_WIDTH:0] block_size_cur; wire [`VX_CFG_NUM_THREADS-1:0] partial_tmask; - - // block_size_r retires the warps latched last cycle a cycle late, so the count - // left for the warps being latched now excludes them. - assign block_size_cur = fire_any_r ? block_size_next : block_size_r; + wire part_warp_r; if (NT_BITS > 0) begin : g_nt_nonzero // Ceiling division block_size / NUM_THREADS: upper bits + OR of lower bits. @@ -248,52 +248,77 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // Shared block_size decrement: low NT_BITS bits unchanged; upper bits drop by // the warps launched. assign block_size_next = {block_size_r[CTA_TID_WIDTH:NT_BITS] - (CTA_TID_WIDTH+1-NT_BITS)'(nfired_r), block_size_r[NT_BITS-1:0]}; - // Partial-warp mask: (1 << count) - 1 where count = block_size_cur[NT_BITS-1:0] - assign partial_tmask = (`VX_CFG_NUM_THREADS'(1) << block_size_cur[NT_BITS-1:0]) - `VX_CFG_NUM_THREADS'(1); + // Partial-warp mask: (1 << count) - 1 where count = block_size_r[NT_BITS-1:0], + // which launches never change. + assign partial_tmask = (`VX_CFG_NUM_THREADS'(1) << block_size_r[NT_BITS-1:0]) - `VX_CFG_NUM_THREADS'(1); + assign part_warp_r = (| block_size_r[NT_BITS-1:0]); end else begin : g_nt_zero // NT_BITS=0: NUM_THREADS=1, each warp has exactly 1 thread, no partial warps. assign kmu_num_warps = (NW_WIDTH+1)'(eff_block_size); assign block_size_next = (CTA_TID_WIDTH+1)'(block_size_r - (CTA_TID_WIDTH+1)'(nfired_r)); assign partial_tmask = `VX_CFG_NUM_THREADS'(0); + assign part_warp_r = 1'b0; end - // Warps still to launch, after those latched last cycle. - wire [CTA_TID_WIDTH-NT_BITS:0] full_warps_cur = block_size_cur[CTA_TID_WIDTH:NT_BITS]; - wire [NW_WIDTH:0] rem_warps_cur; - if (NT_BITS > 0) begin : g_rem_nt - assign rem_warps_cur = (NW_WIDTH+1)'(full_warps_cur) + (NW_WIDTH+1)'(|block_size_cur[NT_BITS-1:0]); - end else begin : g_rem_nt1 - assign rem_warps_cur = (NW_WIDTH+1)'(full_warps_cur); + // Warps still to launch, after those latched last cycle, compared with each + // launch position k as thermometers: block_size_r retires the warps latched + // last cycle a cycle late, so excluding them shifts the thermometers of its + // own counts by nfired_r. full_gt[k]: more than k full warps are left; + // rem_gt[k]: more than k warps are left. + localparam FULL_W = CTA_TID_WIDTH + 1 - NT_BITS; + wire [FULL_W-1:0] full_warps_r = block_size_r[CTA_TID_WIDTH:NT_BITS]; + wire [2*NUM_LANES-1:0] full_r_gt, rem_r_gt; + for (genvar j = 0; j < 2*NUM_LANES; ++j) begin : g_rem_therm + assign full_r_gt[j] = (full_warps_r > FULL_W'(j)); + if (j == 0) begin : g_first + assign rem_r_gt[j] = full_r_gt[j] | part_warp_r; + end else begin : g_rest + assign rem_r_gt[j] = full_r_gt[j] | (part_warp_r & full_r_gt[j-1]); + end end + wire [LCNT_WIDTH-1:0] nfired_cur = fire_any_r ? nfired_r : '0; + wire [NUM_LANES-1:0] full_gt = NUM_LANES'(full_r_gt >> nfired_cur); + wire [NUM_LANES-1:0] rem_gt = NUM_LANES'(rem_r_gt >> nfired_cur); // Launch selection: lanes with a free warp, taken in round-robin order from - // lane_rr_r, up to the warps left. order[l] is the lane's position among - // the selected lanes. - reg [LANE_WIDTH-1:0] lane_rr_r; - wire launch_en; - reg [NUM_LANES-1:0] lane_sel; - reg [NUM_LANES-1:0][LANE_WIDTH-1:0] lane_order; - reg [LCNT_WIDTH-1:0] lane_cnt; - always @(*) begin - lane_sel = '0; - lane_order = '0; - lane_cnt = '0; - for (integer o = 0; o < NUM_LANES; ++o) begin - if (launch_en && lane_ready[(int'(lane_rr_r) + o) % NUM_LANES] - && ((NW_WIDTH+1)'(lane_cnt) < rem_warps_cur)) begin - lane_sel[(int'(lane_rr_r) + o) % NUM_LANES] = 1'b1; - lane_order[(int'(lane_rr_r) + o) % NUM_LANES] = LANE_WIDTH'(lane_cnt); - lane_cnt = lane_cnt + LCNT_WIDTH'(1); - end + // lane_rr_r, up to the warps left. A lane's order is its position among the + // selected lanes: the ready lanes ahead of it in the rotation, so a ready lane + // is selected while fewer than the warps left are ahead of it. + `STATIC_ASSERT (`IS_POW2(NUM_LANES), ("invalid parameter")) + reg [LANE_WIDTH-1:0] lane_rr_r; + wire launch_en; + wire [NUM_LANES-1:0] lane_sel; + wire [NUM_LANES-1:0][LANE_WIDTH-1:0] lane_order; + wire [LCNT_WIDTH-1:0] lane_cnt; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_sel + wire [NUM_LANES-1:0] ahead; + for (genvar m = 0; m < NUM_LANES; ++m) begin : g_ahead + assign ahead[m] = (LANE_WIDTH'(m) - lane_rr_r) < (LANE_WIDTH'(l) - lane_rr_r); end + wire [NUM_LANES-1:0] ready_ahead = lane_ready & ahead; + wire [LCNT_WIDTH-1:0] order; + VX_popcount #( + .N (NUM_LANES) + ) order_count ( + .data_in (ready_ahead), + .data_out (order) + ); + assign lane_order[l] = LANE_WIDTH'(order); + assign lane_sel[l] = launch_en && lane_ready[l] && 1'(rem_gt >> order); end + VX_popcount #( + .N (NUM_LANES) + ) lane_count ( + .data_in (lane_sel), + .data_out (lane_cnt) + ); // A warp is full unless it is the CTA's last and the block size is not a // multiple of the warp width. wire [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] lane_tmask; wire [NUM_LANES-1:0] lane_full; for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_tmask - assign lane_full[l] = ((CTA_TID_WIDTH+1-NT_BITS)'(lane_order[l]) < full_warps_cur); + assign lane_full[l] = 1'(full_gt >> lane_order[l]); assign lane_tmask[l] = lane_full[l] ? {`VX_CFG_NUM_THREADS{1'b1}} : partial_tmask; end @@ -312,22 +337,30 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( end // One thread-index advance by `step`, with a single wrap per dimension: every - // step is normalized below the block dimensions. + // step is normalized below the block dimensions. Y and Z are formed for both + // carries in parallel, so the lower dimension's wrap only selects. function automatic logic [2:0][CTA_TID_WIDTH-1:0] tid_advance ( input logic [2:0][CTA_TID_WIDTH-1:0] base, input logic [2:0][CTA_TID_WIDTH-1:0] step, input logic [CTA_TID_WIDTH:0] bdim_x, input logic [CTA_TID_WIDTH:0] bdim_y ); - logic [CTA_TID_WIDTH:0] nx, ny; - logic wx, wy; - nx = {1'b0, base[0]} + {1'b0, step[0]}; - wx = (nx >= bdim_x); - ny = ({1'b0, base[1]} + {1'b0, step[1]}) + (CTA_TID_WIDTH+1)'(wx); - wy = (ny >= bdim_y); + logic [CTA_TID_WIDTH:0] nx, ny0, ny1; + logic wx, wy0, wy1; + logic [CTA_TID_WIDTH-1:0] y0, y1, z0, z1; + nx = {1'b0, base[0]} + {1'b0, step[0]}; + wx = (nx >= bdim_x); + ny0 = {1'b0, base[1]} + {1'b0, step[1]}; + ny1 = {1'b0, base[1]} + {1'b0, step[1]} + (CTA_TID_WIDTH+1)'(1); + wy0 = (ny0 >= bdim_y); + wy1 = (ny1 >= bdim_y); + y0 = wy0 ? CTA_TID_WIDTH'(ny0 - bdim_y) : CTA_TID_WIDTH'(ny0); + y1 = wy1 ? CTA_TID_WIDTH'(ny1 - bdim_y) : CTA_TID_WIDTH'(ny1); + z0 = base[2] + step[2]; + z1 = base[2] + step[2] + CTA_TID_WIDTH'(1); tid_advance[0] = wx ? CTA_TID_WIDTH'(nx - bdim_x) : CTA_TID_WIDTH'(nx); - tid_advance[1] = wy ? CTA_TID_WIDTH'(ny - bdim_y) : CTA_TID_WIDTH'(ny); - tid_advance[2] = base[2] + step[2] + CTA_TID_WIDTH'(wy); + tid_advance[1] = wx ? y1 : y0; + tid_advance[2] = (wx ? wy1 : wy0) ? z1 : z0; endfunction // The window after the warps latched last cycle launch: entries shift down by @@ -359,6 +392,29 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( reg [LCNT_WIDTH-1:0] init_cnt_r; reg [2:0][CTA_TID_WIDTH-1:0] init_acc_r; wire init_busy = (init_cnt_r != '0); + wire [2:0][CTA_TID_WIDTH-1:0] init_next = tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + wire shape_new = (NUM_LANES > 1) && !kmu_is_frag + && ((kmu_req.args.compute.warp_step != shape_step_r) + || (kmu_req.args.compute.block_dim != shape_dim_r)); + + // The window: a launch header reloads it (cleared for a new shape), the + // window build writes one entry per cycle, and launches shift it. No warp + // launches while the window is built, so the three writes are exclusive and + // the shift, the deepest input, takes the last select. + `RUNTIME_ASSERT(reset || ~(init_busy && fire_any_r), ("%t: %s window shifted while being built", $time, INSTANCE_ID)) + always @(posedge clk) begin + if (~reset) begin + for (integer k = 0; k < NUM_LANES; ++k) begin + if ((state == DISPATCH) && fire_any_r) begin + thread_idx_r[k] <= thread_idx_adv[k]; + end else if ((state == IDLE) && kmu_bus_if_fire) begin + thread_idx_r[k] <= shape_new ? '0 : thread_idx_init_r[k]; + end else if ((state == DISPATCH) && init_busy && (int'(init_cnt_r) == k)) begin + thread_idx_r[k] <= init_next; + end + end + end + end // ------------------------------------------------------------------------- // Retirement decode — pipeline: @@ -395,13 +451,13 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // Within one kernel every CTA has the same aligned LMEM footprint, so LMEM // is partitioned into equal slots of pitch `stride = aligned_lmem_size`. // Resident CTA in slot i owns LMEM bytes [i*stride, (i+1)*stride). The - // occupancy bound `usable_slots_r = min(NUM_WARPS, floor(LMEM/stride))` is a + // occupancy bound `usable = min(NUM_WARPS, floor(LMEM/stride))` is a // kernel constant and is registered so the per-CTA admission/ready path sees // only a free-slot test, never the divide/encode. // // Cluster co-residency: the first CTA of a cluster (is_first_of_cluster) // reserves K = cluster_size CONSECUTIVE usable slots, pre-wrapping the slot - // window to 0 if it would overrun usable_slots_r. Members 2..K then take the + // window to 0 if it would overrun the usable range. Members 2..K then take the // following slots, so member r lands at issuer_base + r*stride — exactly what // DXA multicast resolves. Slots free immediately (out of order) on retire. @@ -413,35 +469,23 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( wire [LMEM_LOG:0] stride = kmu_req.aligned_lmem_size; wire is_first_of_cluster = eff_is_first; - reg [NW_WIDTH:0] usable_slots_r; - // Cluster member count K, capped at the slot count (a cluster larger than - // co-residency degenerates to a clamp). - wire [NW_WIDTH:0] cluster_k_raw = eff_cluster_size; - wire [NW_WIDTH:0] cluster_k = (cluster_k_raw > usable_slots_r) ? usable_slots_r - : (cluster_k_raw == 0) ? (NW_WIDTH+1)'(1) : cluster_k_raw; - - // Occupancy bound, registered (off the ready path). usable = largest m in - // [1, NUM_WARPS] with m*stride <= LMEM_SIZE; all slots when stride == 0. - // m*stride <= LMEM_SIZE is tested as stride <= floor(LMEM_SIZE/m), a constant - // per m, so the NUM_WARPS-wide comparator tree has neither a multiplier nor - // a divider. + // Occupancy bound as a thermometer, registered (off the ready path): + // usable_therm_r[m-1] is set iff m slots fit, i.e. m*stride <= LMEM_SIZE, + // tested as stride <= floor(LMEM_SIZE/m), a constant per m, so there is + // neither a multiplier nor a divider. All slots fit when stride == 0, and + // slot 0 always does. localparam PROD_W = LMEM_LOG + NW_WIDTH + 2; + reg [NUM_CTA_SLOTS-1:0] usable_therm_r; always_ff @(posedge clk) begin if (reset) begin - usable_slots_r <= (NW_WIDTH+1)'(NUM_CTA_SLOTS); + usable_therm_r <= '1; end else begin - if (stride == 0) begin - usable_slots_r <= (NW_WIDTH+1)'(NUM_CTA_SLOTS); - end else begin + for (integer m = 1; m <= NUM_CTA_SLOTS; m = m + 1) begin // Granting one slot to a CTA that exceeds capacity keeps the // launch making forward progress, at the cost of letting it // write past its slot. - usable_slots_r <= (NW_WIDTH+1)'(1); - for (integer m = 1; m <= NUM_CTA_SLOTS; m = m + 1) begin - if (PROD_W'(stride) <= PROD_W'(LMEM_SIZE / m)) begin - usable_slots_r <= (NW_WIDTH+1)'(m); - end - end + usable_therm_r[m-1] <= (m == 1) || (stride == 0) + || (PROD_W'(stride) <= PROD_W'(LMEM_SIZE / m)); end end end @@ -451,40 +495,47 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( `RUNTIME_ASSERT(~kmu_bus_if.valid || (stride == 0) || (PROD_W'(stride) <= PROD_W'(LMEM_SIZE)), ("%t: %s CTA shared-memory footprint %0d exceeds the %0d-byte capacity; the CTA overruns its slot", $time, INSTANCE_ID, stride, LMEM_SIZE)) - // Normalize the round-robin pointer to the usable range (covers a kernel - // transition that shrank usable_slots_r), then apply cluster pre-wrap. - // Compares are full (NW_WIDTH+2)-bit to admit usable_slots_r == NUM_CTA_SLOTS. - wire [NW_WIDTH+1:0] tail_ext = {2'b0, tail_r}; - wire [NW_WIDTH+1:0] usable_ext = {1'b0, usable_slots_r}; - wire [CS_BITS-1:0] base_tail = (tail_ext >= usable_ext) ? CS_BITS'(0) : tail_r; - wire cluster_prewrap = is_first_of_cluster - && (({2'b0, base_tail} + {1'b0, cluster_k}) > usable_ext); - wire [CS_BITS-1:0] base_slot = cluster_prewrap ? CS_BITS'(0) : base_tail; - - // Round-robin advance: next slot wraps to 0 at usable_slots_r. - wire [NW_WIDTH+1:0] next_tail_raw = {2'b0, base_slot} + (NW_WIDTH+2)'(1); - wire [CS_BITS-1:0] next_tail = (next_tail_raw >= usable_ext) - ? CS_BITS'(0) : next_tail_raw[CS_BITS-1:0]; - - // Slot LMEM base = base_slot * stride (small multiply, latched at accept; - // off the ready path). - wire [LMEM_LOG-1:0] base_lmem = (LMEM_LOG)'((LMEM_LOG+NW_WIDTH+1)'(base_slot) * (LMEM_LOG+NW_WIDTH+1)'(stride)); + // The slot base is either the round-robin tail or slot 0: the tail is + // normalized to 0 when a kernel transition shrank the usable range below it, + // and a cluster window that would overrun the usable range pre-wraps to 0. + // Every test is a bit of the registered maps shifted by the tail: + // tail_room[j] is set iff slot tail+j is usable. + wire [NUM_CTA_SLOTS:0] usable_room = {1'b0, usable_therm_r}; + wire [NUM_CTA_SLOTS:0] tail_room = usable_room >> tail_r; + wire [NUM_CTA_SLOTS-1:0] tail_busy = slot_valid_r >> tail_r; + if (NUM_CTA_SLOTS > 1) begin : g_room_top + `UNUSED_VAR (tail_room[NUM_CTA_SLOTS]) + end + + // Window of K = max(cluster_size, 1) slots. The fit test may take K unclamped: + // a K above the usable count never fits past slot 0, and at tail 0 both bases + // agree. The window mask is clamped to the usable count, as a cluster larger + // than co-residency degenerates to a clamp. + wire [NUM_CTA_SLOTS-1:0] cluster_therm; + for (genvar j = 0; j < NUM_CTA_SLOTS; ++j) begin : g_cluster_therm + assign cluster_therm[j] = (j == 0) || (eff_cluster_size > (NW_WIDTH+1)'(j)); + end + wire [NUM_CTA_SLOTS-1:0] window_ones = cluster_therm & usable_therm_r; + wire window_fits_tail = ((cluster_therm & ~tail_room[NUM_CTA_SLOTS-1:0]) == '0); + wire window_free_tail = ((tail_busy & window_ones) == '0); + wire window_free_zero = ((slot_valid_r & window_ones) == '0); + + wire tail_usable = tail_room[0]; + wire base_zero = is_first_of_cluster ? ~window_fits_tail : ~tail_usable; + wire [CS_BITS-1:0] base_slot = base_zero ? CS_BITS'(0) : tail_r; + + // Round-robin advance: next slot wraps to 0 at the usable count. + wire [CS_BITS-1:0] tail_inc = tail_r + CS_BITS'(1); + wire [CS_BITS-1:0] next_tail = base_zero ? CS_BITS'(usable_room[1]) + : (tail_room[1] ? tail_inc : CS_BITS'(0)); // Admission: a standalone CTA / following cluster member needs its single // slot free; a first-of-cluster needs all K window slots free. The window - // [base_slot, base_slot+K) never overruns NUM_CTA_SLOTS (pre-wrap ensures - // base_slot+K <= usable_slots_r <= NUM_CTA_SLOTS), so the mask is built at - // NUM_CTA_SLOTS width directly. K == NUM_CTA_SLOTS wraps to all-ones, which - // is the intended full mask. - // The window is tested at both candidate bases in parallel: shifting the - // registered slot map by the tail leaves only the pre-wrap select after the - // window size. - wire [NUM_CTA_SLOTS-1:0] window_ones = (NUM_CTA_SLOTS'(1) << cluster_k) - NUM_CTA_SLOTS'(1); - wire [NUM_CTA_SLOTS-1:0] slot_valid_tail = slot_valid_r >> base_tail; - wire window_free_tail = ((slot_valid_tail & window_ones) == '0); - wire window_free_zero = ((slot_valid_r & window_ones) == '0); - wire cluster_window_free = cluster_prewrap ? window_free_zero : window_free_tail; - wire admit_ok = is_first_of_cluster ? cluster_window_free : ~slot_valid_r[base_tail]; + // [base_slot, base_slot+K) never overruns the usable range, so the mask is + // built at NUM_CTA_SLOTS width directly. + wire cluster_window_free = window_fits_tail ? window_free_tail : window_free_zero; + wire single_slot_free = tail_usable ? ~tail_busy[0] : ~slot_valid_r[0]; + wire admit_ok = is_first_of_cluster ? cluster_window_free : single_slot_free; // A fragment launch's stamps land in the warp record TID_STAGES cycles after the // warp fires (the thread-index ripple), but frag_stamps_r is a single bank. So a // new launch must not overwrite it while an earlier fragment warp is still in that @@ -531,7 +582,7 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( warp_init_lanes_r <= '0; warp_skip_init_r <= '0; tail_r <= '0; - cur_lmem_base_r <= '0; + cur_stride_r <= '0; slot_valid_r <= '0; dispatched_warps<= '0; warp_done_r <= 0; @@ -596,33 +647,32 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( entry_r <= kmu_req.entry; block_idx_r <= kmu_block_idx; block_dim_r <= kmu_req.args.compute.block_dim; + for (integer d = 0; d < 2; ++d) begin + block_last_r[d] <= CTA_TID_WIDTH'(kmu_req.args.compute.block_dim[d] - (CTA_TID_WIDTH+1)'(1)); + end grid_dim_r <= kmu_grid_dim; param_r <= kmu_req.param; block_size_r <= eff_block_size; warp_step_r <= kmu_req.args.compute.warp_step; cluster_size_r <= eff_cluster_size; cta_rank_r <= '0; - thread_idx_r <= thread_idx_init_r; group_step_r <= (NUM_LANES == 1) ? kmu_req.args.compute.warp_step : group_step_r; // A new block shape rebuilds the window constants before launching. - if ((NUM_LANES > 1) && !kmu_is_frag - && ((kmu_req.args.compute.warp_step != shape_step_r) - || (kmu_req.args.compute.block_dim != shape_dim_r))) begin + if (shape_new) begin shape_step_r <= kmu_req.args.compute.warp_step; shape_dim_r <= kmu_req.args.compute.block_dim; init_cnt_r <= LCNT_WIDTH'(1); init_acc_r <= '0; - thread_idx_r <= '0; end // Fixed-stride placement: this CTA lands in slot base_slot // (the round-robin tail, pre-wrapped for a cluster window) // at LMEM base base_slot * stride. The round-robin pointer - // advances to next_tail (wraps at usable_slots_r). Slots + // advances to next_tail (wraps at the usable count). Slots // free out of order on retirement, so a cluster's window is // checked free up front (admit_ok) rather than reclaimed // in order. - cur_lmem_base_r <= base_lmem; + cur_stride_r <= stride; slot_valid_r[base_slot] <= 1'b1; tail_r <= (`VX_CFG_NUM_WARPS > 1) ? next_tail : CS_BITS'(0); cur_slot_r <= base_slot; @@ -643,18 +693,17 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( end end nfired_r <= lane_cnt; - lane_rr_r <= LANE_WIDTH'((int'(lane_rr_r) + int'(lane_cnt)) % NUM_LANES); + lane_rr_r <= (NUM_LANES > 1) ? (lane_rr_r + LANE_WIDTH'(lane_cnt)) : '0; if (fire_any_r) begin cta_rank_r <= cta_rank_r + NW_WIDTH'(nfired_r); block_size_r <= block_size_next; - thread_idx_r <= thread_idx_adv; for (integer l = 0; l < NUM_LANES; ++l) begin if (warp_fire_r[l]) begin warp_init_lanes_r[warp_id_r[l]] <= warp_init_lanes_r[warp_id_r[l]] | warp_tmask_r[l]; end end - if (rem_warps_cur == '0) begin + if (~rem_gt[0]) begin state <= IDLE; end end @@ -662,14 +711,13 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // window constants: step warp_step_r once per cycle if (init_busy) begin if (int'(init_cnt_r) < NUM_LANES) begin - thread_idx_r[init_cnt_r] <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); - thread_idx_init_r[init_cnt_r] <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + thread_idx_init_r[init_cnt_r] <= init_next; init_cnt_r <= init_cnt_r + LCNT_WIDTH'(1); end else begin - group_step_r <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + group_step_r <= init_next; init_cnt_r <= '0; end - init_acc_r <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + init_acc_r <= init_next; end end default:; @@ -677,6 +725,14 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( end end + always_ff @(posedge clk) begin + if (reset) begin + cur_lmem_base_r <= '0; + end else begin + cur_lmem_base_r <= (LMEM_LOG)'((LMEM_LOG+NW_WIDTH+1)'(cur_slot_r) * (LMEM_LOG+NW_WIDTH+1)'(cur_stride_r)); + end + end + // ------------------------------------------------------------------------- // Outputs — all driven combinationally from registered state // ------------------------------------------------------------------------- @@ -745,21 +801,19 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( function automatic logic [2:0][CTA_TID_WIDTH-1:0] tid_next ( input logic [2:0][CTA_TID_WIDTH-1:0] prev, - input logic [CTA_TID_WIDTH:0] bd_x, - input logic [CTA_TID_WIDTH:0] bd_y + input logic [CTA_TID_WIDTH-1:0] last_x, + input logic [CTA_TID_WIDTH-1:0] last_y ); - // prev is a reduced coordinate (prev[0] < bd_x, prev[1] < bd_y: the - // warp bases come from tid_advance), so a +1 step wraps exactly when - // it reaches the dimension, and wraps to 0. - logic [CTA_TID_WIDTH:0] nx, ny; + // prev is a reduced coordinate (prev[0] <= last_x, prev[1] <= last_y: + // the warp bases come from tid_advance), so a +1 step wraps exactly + // when it leaves the last index, and wraps to 0. The wraps are equality + // tests on prev itself, beside the increments rather than after them. logic wx, wy; - nx = {1'b0, prev[0]} + (CTA_TID_WIDTH+1)'(1); - wx = (nx == bd_x); - ny = {1'b0, prev[1]} + (CTA_TID_WIDTH+1)'(wx); - wy = wx && (ny == bd_y); - tid_next[0] = wx ? '0 : CTA_TID_WIDTH'(nx); - tid_next[1] = wy ? '0 : CTA_TID_WIDTH'(ny); - tid_next[2] = prev[2] + CTA_TID_WIDTH'(wy); + wx = (prev[0] == last_x); + wy = wx && (prev[1] == last_y); + tid_next[0] = wx ? '0 : (prev[0] + CTA_TID_WIDTH'(1)); + tid_next[1] = wy ? '0 : (wx ? (prev[1] + CTA_TID_WIDTH'(1)) : prev[1]); + tid_next[2] = wy ? (prev[2] + CTA_TID_WIDTH'(1)) : prev[2]; endfunction typedef logic [`VX_CFG_NUM_THREADS-1:0][2:0][CTA_TID_WIDTH-1:0] cta_tid_arr_t; @@ -771,11 +825,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( wire [TID_STAGES:0] tidp_valid; wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_wid; wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_rank; - // block_dim spans 1..NUM_WARPS*NUM_THREADS inclusive, one bit more than a - // thread index -- truncating it to CTA_TID_WIDTH turns a full-width dimension - // into 0 and makes every lane wrap. - wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdx; - wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdy; + // The last index of each dimension, block_dim - 1: block_dim spans + // 1..NUM_WARPS*NUM_THREADS, so its last index is a thread index. + wire [TID_STAGES:0][CTA_TID_WIDTH-1:0] tidp_last_x; + wire [TID_STAGES:0][CTA_TID_WIDTH-1:0] tidp_last_y; // stage 0: combinational inputs; seed lane 0 with the warp base. cta_tid_arr_t tidp_tid0; @@ -787,8 +840,8 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( assign tidp_valid[0] = warp_fire_r[l]; assign tidp_wid[0] = warp_id_r[l]; assign tidp_rank[0] = cta_csrs.cta_rank + NW_WIDTH'(warp_order_r[l]); - assign tidp_bdx[0] = cta_csrs.block_dim[0]; - assign tidp_bdy[0] = cta_csrs.block_dim[1]; + assign tidp_last_x[0] = block_last_r[0]; + assign tidp_last_y[0] = block_last_r[1]; for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_tid_pipe cta_tid_arr_t nxt; @@ -796,13 +849,13 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( nxt = tidp_tid[s-1]; for (integer k = 0; k < TID_STEP; k = k + 1) begin if (((s-1)*TID_STEP + k + 1) < `VX_CFG_NUM_THREADS) begin - nxt[(s-1)*TID_STEP + k + 1] = tid_next(nxt[(s-1)*TID_STEP + k], tidp_bdx[s-1], tidp_bdy[s-1]); + nxt[(s-1)*TID_STEP + k + 1] = tid_next(nxt[(s-1)*TID_STEP + k], tidp_last_x[s-1], tidp_last_y[s-1]); end end end reg v_r; reg [NW_WIDTH-1:0] wid_r, rank_r; - reg [CTA_TID_WIDTH:0] bdx_r, bdy_r; + reg [CTA_TID_WIDTH-1:0] last_x_r, last_y_r; cta_tid_arr_t tid_r; always @(posedge clk) begin if (reset) begin @@ -813,15 +866,15 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( tid_r <= nxt; wid_r <= tidp_wid[s-1]; rank_r <= tidp_rank[s-1]; - bdx_r <= tidp_bdx[s-1]; - bdy_r <= tidp_bdy[s-1]; + last_x_r <= tidp_last_x[s-1]; + last_y_r <= tidp_last_y[s-1]; end assign tidp_tid[s] = tid_r; assign tidp_valid[s] = v_r; assign tidp_wid[s] = wid_r; assign tidp_rank[s] = rank_r; - assign tidp_bdx[s] = bdx_r; - assign tidp_bdy[s] = bdy_r; + assign tidp_last_x[s] = last_x_r; + assign tidp_last_y[s] = last_y_r; end wire is_frag_warp; @@ -854,10 +907,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( `UNUSED_VAR (is_frag_warp) `endif - // block_dim is consumed only by stages still rippling; the final stage's - // forwarded copy is unused. - `UNUSED_VAR (tidp_bdx[TID_STAGES]) - `UNUSED_VAR (tidp_bdy[TID_STAGES]) + // last_x/last_y are consumed only by stages still rippling; the final + // stage's forwarded copies are unused. + `UNUSED_VAR (tidp_last_x[TID_STAGES]) + `UNUSED_VAR (tidp_last_y[TID_STAGES]) cta_warp_t cta_warp_wdata; assign cta_warp_wdata.cta_rank = tidp_rank[TID_STAGES]; @@ -931,7 +984,7 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( `TRACE(1, ("%t: %s kmu-accept: cta_id=%0d, PC=0x%0h, param=0x%0h, stride=%0d, num_warps=%0d, usable_slots=%0d\n", $time, INSTANCE_ID, base_slot, to_fullPC(kmu_req.PC), kmu_req.param, - stride, kmu_num_warps, usable_slots_r)) + stride, kmu_num_warps, $countones(usable_therm_r))) end // Warps dispatched to the schedulers for (integer l = 0; l < NUM_LANES; ++l) begin @@ -952,7 +1005,7 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // Admission gate status when KMU presents a CTA but is stalled if (kmu_bus_if.valid && !kmu_bus_if.ready && state == IDLE) begin `TRACE(4, ("%t: %s stall: admit_ok=%b, base_slot=%0d, stride=%0d, usable_slots=%0d\n", - $time, INSTANCE_ID, admit_ok, base_slot, stride, usable_slots_r)) + $time, INSTANCE_ID, admit_ok, base_slot, stride, $countones(usable_therm_r))) end end `endif From c98b2c9d77a971ce14d87e67e4008adc7a292a28 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 30 Oct 2026 10:42:49 -0700 Subject: [PATCH 297/336] tcu: decode the WGMMA A-buffer stripe writes per word The full-GPU logic_gate build at the h100 shape (NT32, NW64, 32 LMEM banks, XLEN 64) put the deepest path of the whole GPU, 47 logic levels, into the A-buffer stripe RAM's write data. The write loop placed each of the 64 response words at a position computed from rsp_ctr_r, so every iteration stacked another mux on the last. Each stripe word has only one possible source per mode: response word w % BANK_ROW_WORDS in block-major order, or row word w % k_words in row major order. The write data is now fixed wiring plus one small mux per row-major source word, and rsp_ctr_r only drives the write enables, which are compares against constants. Beacon: A-buffer RAM 47 -> 8 levels; h100 full GPU 47 -> 36 (the new deepest is the fp64 FMA); tensor and tcu gates unchanged at 25, none deeper. circt-lec proves the RAM next-state equivalent on six geometries including h100. Cycles are identical on the WGMMA, sparse and DXA-fed rtlsim cases at XLEN 32 and 64; model_parity passes. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/tcu/VX_tcu_abuf.sv | 70 ++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/hw/rtl/tcu/VX_tcu_abuf.sv b/hw/rtl/tcu/VX_tcu_abuf.sv index ceca424587..30efc1cc92 100644 --- a/hw/rtl/tcu/VX_tcu_abuf.sv +++ b/hw/rtl/tcu/VX_tcu_abuf.sv @@ -238,7 +238,6 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // and each request/response accumulates the row stride, keeping the // multiplier off the LMEM request-address and response-lane paths. localparam ROW_OFF_W = LDM_W + FETCH_CTR_W + 4; - wire [3:0] row_k_words = slot_is_sparse_r ? 4'(TCU_TC_K) : 4'(TCU_WG_FEDP_K); wire [3:0] row_k_words_alloc = req_is_sparse ? 4'(TCU_TC_K) : 4'(TCU_WG_FEDP_K); wire [ROW_OFF_W-1:0] row_word_off_init = @@ -357,8 +356,11 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Each LMEM response writes one bank-row at offset rsp_ctr_r. // ----------------------------------------------------------------------- - logic [A_STRIPE_WORDS*32-1:0] storage_wdata; - logic [A_STRIPE_WORDS-1:0] storage_wren; + wire [A_STRIPE_WORDS-1:0][31:0] storage_wdata; + wire [A_STRIPE_WORDS-1:0] storage_wren; + + wire storage_write = in_fetch && tcu_lmem_if.rsp_valid; + wire [BANK_ROW_WORDS-1:0][31:0] rsp_words = tcu_lmem_if.rsp_data.data; // Row-major per-response decoding: the row being written is rsp_ctr_r, // and its tcK source words start at lane = row_word_off & (BANK_ROW_WORDS-1) @@ -366,36 +368,36 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [BANK_ROW_WORDS_LOG2:0] row_lane_rsp = (BANK_ROW_WORDS_LOG2+1)'( row_word_off_rsp_r & ROW_OFF_W'(BANK_ROW_WORDS - 1)); - always_comb begin - storage_wdata = '0; - storage_wren = '0; - if (in_fetch && tcu_lmem_if.rsp_valid) begin - if (slot_row_major_r) begin - // Row-major: write k_words of row rsp_ctr_r into - // storage[rsp_ctr_r * k_words .. + k_words). Source words start - // at row_lane_rsp inside the LMEM response. - for (int k = 0; k < TCU_WG_FEDP_K; ++k) begin - automatic int dst = int'(rsp_ctr_r) * int'(row_k_words) + k; - automatic int src = int'(row_lane_rsp) + k; - if (k < int'(row_k_words) && dst < A_STRIPE_WORDS && src < BANK_ROW_WORDS) begin - storage_wren[dst] = 1'b1; - storage_wdata[dst * 32 +: 32] = - tcu_lmem_if.rsp_data.data[src * 32 +: 32]; - end - end - end else begin - // Block-major: one LMEM response carries BANK_ROW_WORDS - // 32-bit words (= NUM_BANKS * XLEN_RATIO). Store all of them - // starting at the current LMEM-row offset within the stripe. - for (int b = 0; b < BANK_ROW_WORDS; ++b) begin - if (int'(rsp_ctr_r) * BANK_ROW_WORDS + b < A_STRIPE_WORDS) begin - storage_wren[int'(rsp_ctr_r) * BANK_ROW_WORDS + b] = 1'b1; - storage_wdata[(int'(rsp_ctr_r) * BANK_ROW_WORDS + b) * 32 +: 32] = - tcu_lmem_if.rsp_data.data[b * 32 +: 32]; - end - end - end - end + // Source word k of the current row. A lane past the response is never + // written, so its index may wrap. + wire [TCU_WG_FEDP_K-1:0][31:0] row_src_words; + for (genvar k = 0; k < TCU_WG_FEDP_K; ++k) begin : g_row_src + wire [`UP(BANK_ROW_WORDS_LOG2)-1:0] src_idx = `UP(BANK_ROW_WORDS_LOG2)'(row_lane_rsp) + + `UP(BANK_ROW_WORDS_LOG2)'(k); + assign row_src_words[k] = rsp_words[src_idx]; + end + + // Stripe word w has one source per mode: response word w % BANK_ROW_WORDS + // (block-major) or row word w % k_words (row-major). The write data is + // therefore fixed wiring, and only the enables decode rsp_ctr_r. + for (genvar w = 0; w < A_STRIPE_WORDS; ++w) begin : g_storage_wr + localparam DN_ROW = w / TCU_WG_FEDP_K; + localparam DN_K = w % TCU_WG_FEDP_K; + localparam SP_ROW = w / TCU_TC_K; + localparam SP_K = w % TCU_TC_K; + localparam BM_ROW = w / BANK_ROW_WORDS; + localparam BM_B = w % BANK_ROW_WORDS; + + wire dn_wren = (int'(rsp_ctr_r) == DN_ROW) && (int'(row_lane_rsp) + DN_K < BANK_ROW_WORDS); + wire sp_wren = (int'(rsp_ctr_r) == SP_ROW) && (int'(row_lane_rsp) + SP_K < BANK_ROW_WORDS); + wire bm_wren = (int'(rsp_ctr_r) == BM_ROW); + + assign storage_wren[w] = storage_write + && (slot_row_major_r ? (slot_is_sparse_r ? sp_wren : dn_wren) + : bm_wren); + assign storage_wdata[w] = slot_row_major_r + ? (slot_is_sparse_r ? row_src_words[SP_K] : row_src_words[DN_K]) + : rsp_words[BM_B]; end wire [A_STRIPE_WORDS-1:0][31:0] storage_rdata; @@ -411,7 +413,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .clk (clk), .reset (reset), .read (1'b1), - .write (in_fetch && tcu_lmem_if.rsp_valid), + .write (storage_write), .wren (storage_wren), .waddr (1'b0), .wdata (storage_wdata), From 0ae111601bd65664d644cbf72ea8c7d1c5a41255 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 30 Oct 2026 15:01:49 -0700 Subject: [PATCH 298/336] scheduler: count retired instructions per issue slice instret_r added a popcount of every warp's commit bit into its 44-bit counter, which made it the deepest path of the logic_gate core build (18 levels). An issue slice commits at most one warp per cycle, so the count is the popcount of one OR per slice: ISSUE_WIDTH bits instead of NUM_WARPS. The counter's value is unchanged every cycle, so SimX needs no change. Beacon: core 18 -> 15 levels, scheduler 18 -> 10, nothing deeper. Registering the count instead was tried and rejected: it moves the add into the CSR read paths (+6 levels). Retired-instruction counts match SimX exactly on rtlsim sgemm at ISSUE_WIDTH 1 and 4. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_scheduler.sv | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/hw/rtl/core/VX_scheduler.sv b/hw/rtl/core/VX_scheduler.sv index 2e92beac9e..c16017a9d9 100644 --- a/hw/rtl/core/VX_scheduler.sv +++ b/hw/rtl/core/VX_scheduler.sv @@ -145,10 +145,21 @@ module VX_scheduler import VX_gpu_pkg::*; #( // a CTA counts from its accept cycle, which busy leaves to the issuer wire kmu_fire = kmu_bus_if.valid && kmu_bus_if.ready; + // An issue slice commits at most one warp per cycle (warp w belongs to slice + // w % ISSUE_WIDTH), so the commits are counted per slice, not per warp. + wire [`VX_CFG_ISSUE_WIDTH-1:0] committed_slices; + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_committed_slices + wire [PER_ISSUE_WARPS-1:0] slice_warps; + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_warps + assign slice_warps[w] = committed_warps[w * `VX_CFG_ISSUE_WIDTH + i]; + end + assign committed_slices[i] = (| slice_warps); + end + reg [PERF_CTR_BITS-1:0] cycles_r; reg [PERF_CTR_BITS-1:0] instret_r; - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] committed_warps_cnt; - `POP_COUNT(committed_warps_cnt, committed_warps); + wire [`CLOG2(`VX_CFG_ISSUE_WIDTH+1)-1:0] committed_cnt; + `POP_COUNT(committed_cnt, committed_slices); always @(posedge clk) begin if (reset) begin cycles_r <= '0; @@ -157,7 +168,7 @@ module VX_scheduler import VX_gpu_pkg::*; #( if (busy || kmu_fire) begin cycles_r <= cycles_r + 1; end - instret_r <= instret_r + PERF_CTR_BITS'(committed_warps_cnt); + instret_r <= instret_r + PERF_CTR_BITS'(committed_cnt); end end assign cycles = cycles_r; From 926658724d3b80505492145d3544ff49e47ad20d Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 30 Oct 2026 20:55:15 -0700 Subject: [PATCH 299/336] config: keep a Makefile's own CONFIGS under a hardware profile config.mk folded the profile into CONFIGS with `override`, and GNU make ignores every later plain assignment to an overridden variable. So under any --profile, the -D flags a Makefile appends after including config.mk were silently dropped: the 80 test Makefiles that add their own CONFIGS (rt_smoke_tlas's -DVX_CFG_RTU_BVH_WIDTH=0, the graphics and TCU tests' extension and shape defaults), and the hw/unittest, synopsys, opae and sandbox flows. rt_smoke_tlas on rtx5080 then ran the CW-BVH4 walker over a flat scene and missed. CONFIGS reaches make through the environment (blackbox.sh, RUNTIME_ARGS), where a plain assignment already wins, so the override is now applied only when CONFIGS comes from the make command line. The guarded per-app defaults (NUM_WARPS, NUM_CORES, ISSUE_WIDTH) see the profile's values and no longer conflict; merging an already merged list is a no-op, so sub-makes resolve the same set. Verified on SimX XLEN 64: rt_smoke_tlas and rt_smoke_deep_tlas on rtx5080 (VM on and off) now pass (rt_smoke_tlas failed before); gfx_tex on rtx5080, sgemm on a100, and rt_smoke_tlas without a profile pass. make resolves the expected CONFIGS with an environment CONFIGS, a command-line CONFIGS (profile still applied) and no profile. Co-Authored-By: Claude Opus 5.5 --- config.mk.in | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/config.mk.in b/config.mk.in index 5c26a2b871..b7de36b210 100644 --- a/config.mk.in +++ b/config.mk.in @@ -69,8 +69,15 @@ VRT_HOME ?= /usr/local # CONFIGS win over the profile. PROFILE ?= @PROFILE@ ifneq ($(strip $(PROFILE)),) -override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --xlen=$(XLEN) --merge-configs --cflags='$(CONFIGS)') +PROFILE_CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --xlen=$(XLEN) --merge-configs --cflags='$(CONFIGS)') ifneq ($(.SHELLSTATUS),0) $(error invalid hardware profile '$(PROFILE)') endif +# A Makefile that includes this one appends its own -D flags to CONFIGS, and an +# override would silently drop them; only a command-line CONFIGS needs one. +ifeq ($(origin CONFIGS),command line) +override CONFIGS := $(PROFILE_CONFIGS) +else +CONFIGS := $(PROFILE_CONFIGS) +endif endif From 7620644adf935611aa7729db3e89181eb33a3b94 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 30 Oct 2026 21:01:34 -0700 Subject: [PATCH 300/336] rtu: register the collector insertion compares a cycle early The deepest RTU path (22 levels) ran from the box PE result through the one-hot row mux, the insertion t-compares and an indexed write chain over the whole collector array into coll_ordt. It is split three ways, with cycles unchanged: - Each collector row decodes its own write enables and insertion slot, so every coll_ordt/coll_ordoff entry is one mux. coll_ctx/coll_last keep their indexed write so they still map to LUTRAM. - The box PE exports t_near_pre, the next result's t_near one cycle ahead (its enable is tied high in the scheduler, so it never stalls). Each entry of the next result's row, and the result landing now, are compared against it and registered with a per-slot select; when the result landing now inserts into the same row, the select picks the compare of the entry each slot shifts to. The count mask stays in the write cycle, so a row allocation needs no forwarding. - yld_q/cbtype_q/objv_q get per-context next-state logic in the same last-write-wins order, so the barrier walker's per-lane writes no longer chain behind the EXEC write. Beacon (rtu / gfx DUT): coll_ordt 22->5 / 22->5, coll_ordoff 20->5, yld_q 20->14 / 21->14, objv_q 17->14 / 18->14; deepest rtu 22->18, gfx 22->20; no register got deeper. Verified: circt-lec proves the collector update equal under the registered-compare invariant at 4 rows x widths 2/4/6/8, 8x4 and 16x2 (16 rows at widths 4-8 timed out undecided) and rejects two known-bad forwarding variants; the yield-flag next state is proven equal at five slot x lane shapes. SimX rt_suite (36) + multicore smokes pass; rtlsim rt_suite (35), the rt_raycast parity case and the three rt_raycast perf_gate cases pass with identical cycles on all 39 runs (rt_raycast 159694), so SimX needs no change; model_parity rt_raycast gap 2.2%. Verilator -Wall clean at BVH widths 0/2/4/8. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/rtu/VX_rtu_box_pe.sv | 9 +- hw/rtl/rtu/VX_rtu_scheduler.sv | 258 +++++++++++++-------- hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv | 3 +- 3 files changed, 175 insertions(+), 95 deletions(-) diff --git a/hw/rtl/rtu/VX_rtu_box_pe.sv b/hw/rtl/rtu/VX_rtu_box_pe.sv index 446e42a1e6..b8638e656b 100644 --- a/hw/rtl/rtu/VX_rtu_box_pe.sv +++ b/hw/rtl/rtu/VX_rtu_box_pe.sv @@ -62,7 +62,10 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // it instead of doing so on the cycle the result lands output wire [TAG_WIDTH-1:0] tag_out_pre, output wire hit, - output wire [31:0] t_near + output wire [31:0] t_near, + // the next result's t_near, one cycle ahead (meaningful when that result + // is valid) + output wire [31:0] t_near_pre ); localparam F = LATENCY_FMA; localparam LAT_FMA = 3 * F; // origin - ro, + corner, * inv_d @@ -318,12 +321,13 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // ── stage 7: hit = lo <= hi; t_near = lo ───────────────────────── // lo and hi are never NaN; t_near is non-negative for t_min >= 0 and +0 for // a zero, so the consumer may order it as an unsigned integer. + wire [31:0] t_near_w = (lo_all_r[30:0] == 31'd0) ? 32'd0 : lo_all_r; reg hit_r; reg [31:0] t_near_r; always_ff @(posedge clk) begin if (enable) begin hit_r <= f32_le(lo_all_r, hi_all_r); - t_near_r <= (lo_all_r[30:0] == 31'd0) ? 32'd0 : lo_all_r; + t_near_r <= t_near_w; end end @@ -368,5 +372,6 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign tag_out_pre = tag_out_pre_w; assign hit = hit_r; assign t_near = t_near_r; + assign t_near_pre = t_near_w; endmodule diff --git a/hw/rtl/rtu/VX_rtu_scheduler.sv b/hw/rtl/rtu/VX_rtu_scheduler.sv index dab898a532..2cc93d2066 100644 --- a/hw/rtl/rtu/VX_rtu_scheduler.sv +++ b/hw/rtl/rtu/VX_rtu_scheduler.sv @@ -702,6 +702,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wire box_feed, box_feed_raw; wire box_valid_out, box_hit; wire [31:0] box_t_near; + wire [31:0] box_t_near_pre; wire [COLL_IDW+32-1:0] box_tag_out; wire [COLL_IDW+32-1:0] box_tag_pre; wire [COLL_IDW-1:0] box_coll = box_tag_out[COLL_IDW+32-1 : 32]; @@ -733,12 +734,14 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( .tag_out (box_tag_out), .tag_out_pre (box_tag_pre), .hit (box_hit), - .t_near (box_t_near) + .t_near (box_t_near), + .t_near_pre (box_t_near_pre) ); end else begin : g_no_box_pe assign box_valid_out = 1'b0; assign box_hit = 1'b0; assign box_t_near = '0; + assign box_t_near_pre = '0; assign box_tag_out = '0; assign box_tag_pre = '0; `UNUSED_VAR ({box_feed, box_feed_raw, feed_ci, walk_inv_d}) @@ -746,10 +749,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( `UNUSED_VAR ({node, node_kind, node_lines, leaf_lines, stacktop_q, pathv_q}) end - // Row select for the insertion read. It depends only on the tag, which the - // box PE delivers a cycle ahead of its result, so the decode is registered - // here rather than repeated on the result cycle -- where it fans out across - // every row and lands on the path that ends at the ordering registers. + // Row decode for the collector write, registered off the early tag. wire [COLL_IDW-1:0] box_coll_pre = box_tag_pre[COLL_IDW+32-1 : 32]; `UNUSED_VAR (box_tag_pre[31:0]) reg [COLL_SIZE-1:0] box_coll_hot; @@ -761,43 +761,59 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end - wire [COLL_SIZE-1:0][NODE_W*32+RTU_CHILD_BITS-1:0] coll_row_in; - for (genvar c = 0; c < COLL_SIZE; ++c) begin : g_coll_row_in - assign coll_row_in[c] = {coll_ordt[c], coll_ordcnt[c]}; - end + wire coll_pushable = box_hit && (box_childoff != 32'd0); - wire [NODE_W-1:0][31:0] coll_ordt_sel; - wire [RTU_CHILD_BITS-1:0] coll_ordcnt_sel; - VX_onehot_mux #( - .DATAW (NODE_W * 32 + RTU_CHILD_BITS), - .N (COLL_SIZE) - ) coll_row_mux ( - .data_in (coll_row_in), - .sel_in (box_coll_hot), - .data_out ({coll_ordt_sel, coll_ordcnt_sel}) - ); + // Insertion t-compares, taken a cycle ahead off the PE's early row and + // t_near: each entry of that row, and the result landing now, against the + // next result's t. If the result landing now inserts into the same row, the + // registered select picks the compare of the entry each slot shifts to. + wire [NODE_W-1:0] ins_t_le; + if (RTU_BVH_WIDTH != 0) begin : g_ins_pre + reg [NODE_W-1:0] pre_le_q; + reg new_le_q; + reg [NODE_W-1:0] take_new_q; + reg [NODE_W-1:0] take_prev_q; + + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_le_sel + localparam PREV = (oc != 0) ? (oc - 1) : 0; + assign ins_t_le[oc] = take_new_q[oc] ? new_le_q + : take_prev_q[oc] ? pre_le_q[PREV] + : pre_le_q[oc]; + end - // Insertion decisions off the admit thermometer: the collected list is - // sorted ascending and the count mask is a prefix, so the admit bits are a - // prefix of ones and the insertion slot is the first zero. Neighbouring-bit - // decode -- no popcount and no index compare on the ordering-write path. - wire [NODE_W-1:0] ins_le, ins_here, ins_shift; - if (RTU_BVH_WIDTH != 0) begin : g_ins_slots - for (genvar oc = 0; oc < RTU_BVH_WIDTH; ++oc) begin : g_le - assign ins_le[oc] = (RTU_CHILD_BITS'(oc) < coll_ordcnt_sel) - && (coll_ordt_sel[oc] <= box_t_near); + wire [NODE_W-1:0][31:0] pre_ordt = coll_ordt[box_coll_pre]; + wire [RTU_CHILD_BITS-1:0] pre_ordcnt = coll_ordcnt[box_coll_pre]; + wire pre_fwd = box_valid_out && (box_coll == box_coll_pre); + wire pre_proc = coll_proc[box_coll_pre]; + wire fwd_proc = pre_fwd && pre_proc; + wire fwd_ins = pre_fwd && ~pre_proc && coll_pushable; + + // this cycle's insertion into the forwarded row + wire [NODE_W-1:0] cur_le; + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_cur_le + assign cur_le[oc] = (RTU_CHILD_BITS'(oc) < pre_ordcnt) && ins_t_le[oc]; end - assign ins_here[0] = ~ins_le[0]; - assign ins_shift[0] = 1'b0; - for (genvar oc = 1; oc < RTU_BVH_WIDTH; ++oc) begin : g_slot - assign ins_here[oc] = ~ins_le[oc] && ins_le[oc-1]; - assign ins_shift[oc] = ~ins_le[oc-1]; + + wire [NODE_W-1:0] pre_le, take_new, take_prev; + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_pre_le + localparam PREV = (oc != 0) ? (oc - 1) : 0; + wire cur_here = ~cur_le[oc] && ((oc == 0) || cur_le[PREV]); + wire cur_shift = (oc != 0) && ~cur_le[PREV]; + assign take_new[oc] = ((oc == 0) && fwd_proc) || (fwd_ins && cur_here); + assign take_prev[oc] = fwd_ins && cur_shift; + assign pre_le[oc] = (pre_ordt[oc] <= box_t_near_pre); end - end else begin : g_no_ins_slots - `UNUSED_VAR ({coll_ordt_sel, coll_ordcnt_sel}) - assign {ins_le, ins_here, ins_shift} = '0; + + always @(posedge clk) begin + pre_le_q <= pre_le; + new_le_q <= (box_t_near <= box_t_near_pre); + take_new_q <= take_new; + take_prev_q <= take_prev; + end + end else begin : g_no_ins_pre + assign ins_t_le = '0; + `UNUSED_VAR ({box_t_near_pre, box_coll_pre}) end - wire coll_pushable = box_hit && (box_childoff != 32'd0); // ── tri PE ──────────────────────────────────────────────────────── wire tri_feed; @@ -1970,8 +1986,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( done_q <= '0; mask_q <= '0; hit_q <= '0; - yld_q <= '0; - objv_q <= '0; attr_q <= '0; running <= '0; finalised <= '0; @@ -1994,7 +2008,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( mask_q[32'(slot_start_slot)*NUM_LANES + k] <= slot_start_mask[k]; done_q[32'(slot_start_slot)*NUM_LANES + k] <= ~slot_start_mask[k]; hit_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; - yld_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; attr_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; live_q[32'(slot_start_slot)*NUM_LANES + k] <= slot_start_mask[k]; seed_v_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; @@ -2019,14 +2032,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( if (exec_hit_set) begin hit_q[sel_q] <= 1'b1; end - if (exec_yld_set) begin - yld_q[sel_q] <= 1'b1; - cbtype_q[sel_q] <= exec_cbtype; - objv_q[sel_q] <= exec_objv; - end - if (exec_yld_clr) begin - yld_q[sel_q] <= 1'b0; - end lvl_q_arr[sel_q] <= word_n.lvl; if (fresh_q || rewalk_q || sp_clr) begin sp_q_arr[sel_q] <= '0; @@ -2064,7 +2069,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( attr_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; seed_v_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; end - yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; live_q[32'(bw_slot)*NUM_LANES + k] <= res_rewalk[k]; if (res_rewalk[k]) begin done_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; @@ -2076,18 +2080,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( finalised[bw_slot] <= 1'b0; end end else begin - for (k = 0; k < NUM_LANES; k = k + 1) begin - if (bw_copy_mask[k]) begin - yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; - cbtype_q[32'(bw_slot)*NUM_LANES + k] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_CHS); - objv_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; - end - if (bw_miss_mask[k]) begin - yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; - cbtype_q[32'(bw_slot)*NUM_LANES + k] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_MISS); - objv_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; - end - end finalised[bw_slot] <= 1'b1; end end @@ -2103,6 +2095,49 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end + // Yield flags, per context: the barrier walker's per-lane writes decode to + // each context instead of chaining behind the EXEC write. + for (genvar i = 0; i < NUM_CTX; ++i) begin : g_yld_upd + localparam SLOT = i / NUM_LANES; + localparam LANE = i % NUM_LANES; + wire start_i = slot_start_valid && (slot_start_slot == SLOT_W'(SLOT)); + wire exec_i = x_valid && (sel_q == CTX_TAG_W'(i)); + wire bw_i = bw_job_done && (bw_slot == SLOT_W'(SLOT)); + wire res_i = bw_i && bw_is_res; + wire chs_i = bw_i && ~bw_is_res && bw_copy_mask[LANE]; + wire miss_i = bw_i && ~bw_is_res && bw_miss_mask[LANE]; + always_ff @(posedge clk) begin + if (reset) begin + yld_q[i] <= 1'b0; + objv_q[i] <= 1'b0; + end else begin + if (res_i) begin + yld_q[i] <= 1'b0; + end else if (chs_i || miss_i) begin + yld_q[i] <= 1'b1; + end else if (exec_i && exec_yld_clr) begin + yld_q[i] <= 1'b0; + end else if (exec_i && exec_yld_set) begin + yld_q[i] <= 1'b1; + end else if (start_i) begin + yld_q[i] <= 1'b0; + end + if (miss_i) begin + cbtype_q[i] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_MISS); + end else if (chs_i) begin + cbtype_q[i] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_CHS); + end else if (exec_i && exec_yld_set) begin + cbtype_q[i] <= exec_cbtype; + end + if (chs_i || miss_i) begin + objv_q[i] <= 1'b0; + end else if (exec_i && exec_yld_set) begin + objv_q[i] <= exec_objv; + end + end + end + end + // a resume's committed t seeds the lane's re-walk always_ff @(posedge clk) begin if (bw_wr_gnt && (bw_state == BW_WR) && bw_is_res && (bw_field == 3'd0)) begin @@ -2127,37 +2162,76 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // ── collector update ────────────────────────────────────────────── always_ff @(posedge clk) begin - if (reset) begin - coll_busy <= '0; - end else begin - if (x_valid && coll_alloc_r) begin - coll_busy[coll_alloc_idx] <= 1'b1; - coll_proc[coll_alloc_idx] <= box_raw_r; - coll_ctx[coll_alloc_idx] <= sel_q; - coll_cnt[coll_alloc_idx] <= '0; - coll_last[coll_alloc_idx] <= box_raw_r ? RTU_CHILD_BITS'(0) : last_child; - coll_ordcnt[coll_alloc_idx] <= '0; - end - if (x_valid && coll_free_r) begin - coll_busy[word_x.coll_id] <= 1'b0; + if (~reset && x_valid && coll_alloc_r) begin + coll_ctx[coll_alloc_idx] <= sel_q; + coll_last[coll_alloc_idx] <= box_raw_r ? RTU_CHILD_BITS'(0) : last_child; + end + end + + // Each row decodes its own write enables and insertion slot, so the + // ordering write is one mux per entry. The admit bits are a prefix of ones + // (the list is sorted ascending and the count mask is a prefix), so the + // insertion slot is the first zero: a neighbouring-bit decode. + for (genvar c = 0; c < COLL_SIZE; ++c) begin : g_coll_upd + wire alloc_c = x_valid && coll_alloc_r && (coll_alloc_idx == COLL_IDW'(c)); + wire free_c = x_valid && coll_free_r && (word_x.coll_id == COLL_IDW'(c)); + wire box_c = box_valid_out && box_coll_hot[c]; + wire proc_c = box_c && coll_proc[c]; + wire tree_c = box_c && ~coll_proc[c]; + wire ins_c = tree_c && coll_pushable; + + always_ff @(posedge clk) begin + if (reset) begin + coll_busy[c] <= 1'b0; + end else begin + if (free_c) begin + coll_busy[c] <= 1'b0; + end else if (alloc_c) begin + coll_busy[c] <= 1'b1; + end + if (alloc_c) begin + coll_proc[c] <= box_raw_r; + end + if (tree_c) begin + coll_cnt[c] <= coll_cnt[c] + RTU_CHILD_BITS'(1); + end else if (alloc_c) begin + coll_cnt[c] <= '0; + end + if (ins_c) begin + coll_ordcnt[c] <= coll_ordcnt[c] + RTU_CHILD_BITS'(1); + end else if (alloc_c) begin + coll_ordcnt[c] <= '0; + end + if (proc_c) begin + coll_prochit[c] <= box_hit; + end end - if (box_valid_out) begin - if (coll_proc[box_coll]) begin - coll_prochit[box_coll] <= box_hit; - coll_ordt[box_coll][0] <= box_t_near; - end else begin - coll_cnt[box_coll] <= coll_cnt[box_coll] + RTU_CHILD_BITS'(1); - if (coll_pushable) begin - for (integer oc = 0; oc < RTU_BVH_WIDTH; oc = oc + 1) begin - if (ins_here[oc]) begin - coll_ordoff[box_coll][oc[IDXW-1:0]] <= box_childoff; - coll_ordt[box_coll][oc[IDXW-1:0]] <= box_t_near; - end else if (ins_shift[oc]) begin - coll_ordoff[box_coll][oc[IDXW-1:0]] <= coll_ordoff[box_coll][oc[IDXW-1:0]-1]; - coll_ordt[box_coll][oc[IDXW-1:0]] <= coll_ordt[box_coll][oc[IDXW-1:0]-1]; - end - end - coll_ordcnt[box_coll] <= coll_ordcnt[box_coll] + RTU_CHILD_BITS'(1); + end + + wire [NODE_W-1:0] ins_le; + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_le + assign ins_le[oc] = (RTU_CHILD_BITS'(oc) < coll_ordcnt[c]) && ins_t_le[oc]; + end + + // a procedural box stores its t in slot 0 + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_slot + localparam PREV = (oc != 0) ? (oc - 1) : 0; + wire ins_here = ~ins_le[oc] && ((oc == 0) || ins_le[PREV]); + wire ins_shift = (oc != 0) && ~ins_le[PREV]; + wire set_t = ((oc == 0) && proc_c) || (ins_c && ins_here); + wire set_off = ins_c && ins_here; + wire shift_c = ins_c && ins_shift; + always_ff @(posedge clk) begin + if (~reset) begin + if (set_t) begin + coll_ordt[c][oc] <= box_t_near; + end else if (shift_c) begin + coll_ordt[c][oc] <= coll_ordt[c][PREV]; + end + if (set_off) begin + coll_ordoff[c][oc] <= box_childoff; + end else if (shift_c) begin + coll_ordoff[c][oc] <= coll_ordoff[c][PREV]; end end end @@ -2290,7 +2364,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign obj_vld = objv_q; assign attr_vld = attr_q; - `UNUSED_VAR ({f_aligned, s1_fresh, ray_q, ins_le, ins_here, ins_shift}) + `UNUSED_VAR ({f_aligned, s1_fresh, ray_q}) `ifdef DBG_TRACE_RTU always_ff @(posedge clk) begin diff --git a/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv b/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv index f89563d573..5ff50ede0a 100644 --- a/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv +++ b/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv @@ -105,7 +105,8 @@ module VX_rtu_box_pe_tb import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( .tag_out (tag_out), `UNUSED_PIN (tag_out_pre), .hit (hit), - .t_near (t_near) + .t_near (t_near), + `UNUSED_PIN (t_near_pre) ); endmodule From a5161bbf7fc7e89e7439f565f554e6e039791a73 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 30 Oct 2026 22:46:30 -0700 Subject: [PATCH 301/336] config: let a test's own flags replace a profile's setting 926658724 kept a test Makefile's appended -D flags under a hardware profile, but a flag for a knob the profile also sets still landed next to the profile's: both -DVX_CFG_RTU_TLAS_ENABLE and _DISABLE reached the compiler, and the #ifdef saw the enable. So the 16 flat-scene RT smoke tests, which build a plain triangle list, still ran the flat TLAS walker on rtx5080 (a profile with TLAS on) and missed. - merge_configs: a later flag for a knob replaces an earlier one, as on a compiler command line; with nothing replaced the caller's flags still pass through verbatim, and merging a merged list is a no-op. - tests/regression/common.mk folds the profile again after the app's own flags, so the app's override wins in its compile and in the runtime it builds. - The flat-scene tests now pin -DVX_CFG_RTU_TLAS_DISABLE beside BVH_WIDTH=0: a TLAS-enabled flat walker expects a TLAS scene. Verified on SimX XLEN 64: on rtx5080 without VM, rt_smoke, _chs, _multitri (previously failing), _miss, _shadow, _tlas, _cull_mask and _deep_tlas pass; rt_smoke and rt_smoke_tlas pass without a profile; sgemm and sgemm_tcu_wg pass on a100. The merge keeps quoting when nothing is replaced and returns a merged list unchanged. Co-Authored-By: Claude Opus 5.5 --- ci/gen_config.py | 15 ++++++++++++--- tests/raytracing/rt_smoke/Makefile | 5 +++-- tests/raytracing/rt_smoke_ahs/Makefile | 5 +++-- tests/raytracing/rt_smoke_ahs_mixed/Makefile | 5 +++-- tests/raytracing/rt_smoke_async_batch/Makefile | 5 +++-- tests/raytracing/rt_smoke_chs/Makefile | 5 +++-- tests/raytracing/rt_smoke_cull_back/Makefile | 5 +++-- tests/raytracing/rt_smoke_is/Makefile | 5 +++-- tests/raytracing/rt_smoke_miss/Makefile | 5 +++-- tests/raytracing/rt_smoke_multitri/Makefile | 5 +++-- tests/raytracing/rt_smoke_numctx/Makefile | 5 +++-- tests/raytracing/rt_smoke_recursive/Makefile | 5 +++-- tests/raytracing/rt_smoke_reform/Makefile | 5 +++-- tests/raytracing/rt_smoke_reform_mw/Makefile | 5 +++-- tests/raytracing/rt_smoke_reform_sbt/Makefile | 5 +++-- tests/raytracing/rt_smoke_sbt/Makefile | 5 +++-- tests/raytracing/rt_smoke_shadow/Makefile | 5 +++-- tests/regression/common.mk | 6 ++++++ 18 files changed, 66 insertions(+), 35 deletions(-) diff --git a/ci/gen_config.py b/ci/gen_config.py index 58aa770a45..0d5f58ed99 100755 --- a/ci/gen_config.py +++ b/ci/gen_config.py @@ -594,11 +594,20 @@ def _canonical_define(tok: str, enums: Dict[str, EnumSpec]) -> Optional[str]: def merge_configs(prof: Profile, cflags: str, enums: Dict[str, EnumSpec]) -> str: explicit = shlex.split(cflags) if cflags else [] - given = {_canonical_define(t, enums) for t in explicit} + knobs = [_canonical_define(t, enums) for t in explicit] + given = set(knobs) given.discard(None) merged = [t for t in profile_defines(prof) if _canonical_define(t, enums) not in given] - # The caller's flags pass through verbatim, quoting included. - return " ".join(merged + ([cflags.strip()] if cflags.strip() else [])) + # A later flag for a knob replaces an earlier one, as on a compiler command + # line: a Makefile appends its own flags after an already merged profile. + last = {k: i for i, k in enumerate(knobs) if k is not None} + kept = [t for i, (t, k) in enumerate(zip(explicit, knobs)) if k is None or last[k] == i] + if len(kept) == len(explicit): + # Nothing replaced: the caller's flags pass through verbatim, quoting included. + caller = [cflags.strip()] if cflags.strip() else [] + else: + caller = [shlex.join(kept)] + return " ".join(merged + caller) # ----------------------------- diff --git a/tests/raytracing/rt_smoke/Makefile b/tests/raytracing/rt_smoke/Makefile index 0874691901..b72f9c9f11 100644 --- a/tests/raytracing/rt_smoke/Makefile +++ b/tests/raytracing/rt_smoke/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke diff --git a/tests/raytracing/rt_smoke_ahs/Makefile b/tests/raytracing/rt_smoke_ahs/Makefile index f6052b4538..7991aa54f7 100644 --- a/tests/raytracing/rt_smoke_ahs/Makefile +++ b/tests/raytracing/rt_smoke_ahs/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_ahs diff --git a/tests/raytracing/rt_smoke_ahs_mixed/Makefile b/tests/raytracing/rt_smoke_ahs_mixed/Makefile index 400ebe2a57..0cbb329586 100644 --- a/tests/raytracing/rt_smoke_ahs_mixed/Makefile +++ b/tests/raytracing/rt_smoke_ahs_mixed/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_ahs_mixed diff --git a/tests/raytracing/rt_smoke_async_batch/Makefile b/tests/raytracing/rt_smoke_async_batch/Makefile index 18c4957554..65cac992c1 100644 --- a/tests/raytracing/rt_smoke_async_batch/Makefile +++ b/tests/raytracing/rt_smoke_async_batch/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE # The whole point of the test: four traces issued back to back before any wait. # Four traces in flight is four slots, and the default pool holds one per core. CONFIGS += -DVX_CFG_RTU_NUM_SLOTS=4 diff --git a/tests/raytracing/rt_smoke_chs/Makefile b/tests/raytracing/rt_smoke_chs/Makefile index e12fc4293a..51336c5dd8 100644 --- a/tests/raytracing/rt_smoke_chs/Makefile +++ b/tests/raytracing/rt_smoke_chs/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_chs diff --git a/tests/raytracing/rt_smoke_cull_back/Makefile b/tests/raytracing/rt_smoke_cull_back/Makefile index dbd4074892..1f65b72d2a 100644 --- a/tests/raytracing/rt_smoke_cull_back/Makefile +++ b/tests/raytracing/rt_smoke_cull_back/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_cull_back diff --git a/tests/raytracing/rt_smoke_is/Makefile b/tests/raytracing/rt_smoke_is/Makefile index 46b5149f15..065923ac22 100644 --- a/tests/raytracing/rt_smoke_is/Makefile +++ b/tests/raytracing/rt_smoke_is/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_is diff --git a/tests/raytracing/rt_smoke_miss/Makefile b/tests/raytracing/rt_smoke_miss/Makefile index 6b9b4c4bcb..3c96b4ada8 100644 --- a/tests/raytracing/rt_smoke_miss/Makefile +++ b/tests/raytracing/rt_smoke_miss/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_miss diff --git a/tests/raytracing/rt_smoke_multitri/Makefile b/tests/raytracing/rt_smoke_multitri/Makefile index e8b594dc18..209b953dc5 100644 --- a/tests/raytracing/rt_smoke_multitri/Makefile +++ b/tests/raytracing/rt_smoke_multitri/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_multitri diff --git a/tests/raytracing/rt_smoke_numctx/Makefile b/tests/raytracing/rt_smoke_numctx/Makefile index 4c240bd8c4..680cfe6b74 100644 --- a/tests/raytracing/rt_smoke_numctx/Makefile +++ b/tests/raytracing/rt_smoke_numctx/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE # Two resident traces: RTU_NUM_SLOTS = 2 DERIVES RTU_NUM_CTX = 8 (a slot owns one # context per thread of the warp it traces). Exercises the multi-slot RTU — two warps # traversing concurrently, per-slot completion barriers, and the window-write arbiter diff --git a/tests/raytracing/rt_smoke_recursive/Makefile b/tests/raytracing/rt_smoke_recursive/Makefile index 7dc5dc0eb2..f80ce73dd8 100644 --- a/tests/raytracing/rt_smoke_recursive/Makefile +++ b/tests/raytracing/rt_smoke_recursive/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE # The sub-ray is traced from inside the closest-hit callback, so the warp holds # its parent's slot while it asks for a second one. Two traces in flight per warp # is two slots; the default pool (one per core) cannot serve it and the warp diff --git a/tests/raytracing/rt_smoke_reform/Makefile b/tests/raytracing/rt_smoke_reform/Makefile index ea6ab993c8..635b932a4e 100644 --- a/tests/raytracing/rt_smoke_reform/Makefile +++ b/tests/raytracing/rt_smoke_reform/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_reform diff --git a/tests/raytracing/rt_smoke_reform_mw/Makefile b/tests/raytracing/rt_smoke_reform_mw/Makefile index c3fea042d2..5b7d917f99 100644 --- a/tests/raytracing/rt_smoke_reform_mw/Makefile +++ b/tests/raytracing/rt_smoke_reform_mw/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE # Reformation batches candidates from several warps into one callback dispatch, # so several warps have to be tracing AT THE SAME TIME. One slot per core would # serialise them and the test would pass without ever exercising what it is for. diff --git a/tests/raytracing/rt_smoke_reform_sbt/Makefile b/tests/raytracing/rt_smoke_reform_sbt/Makefile index e0e479021a..bb04bb17ac 100644 --- a/tests/raytracing/rt_smoke_reform_sbt/Makefile +++ b/tests/raytracing/rt_smoke_reform_sbt/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_reform_sbt diff --git a/tests/raytracing/rt_smoke_sbt/Makefile b/tests/raytracing/rt_smoke_sbt/Makefile index 6f8f29081a..bfca028c5e 100644 --- a/tests/raytracing/rt_smoke_sbt/Makefile +++ b/tests/raytracing/rt_smoke_sbt/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_sbt diff --git a/tests/raytracing/rt_smoke_shadow/Makefile b/tests/raytracing/rt_smoke_shadow/Makefile index 6662e36517..86d5090d45 100644 --- a/tests/raytracing/rt_smoke_shadow/Makefile +++ b/tests/raytracing/rt_smoke_shadow/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_shadow diff --git a/tests/regression/common.mk b/tests/regression/common.mk index dcb15840dc..ac503bee3b 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -21,6 +21,12 @@ ifneq ($(findstring -DVX_CFG_TCU_WGMMA_ENABLE,$(CONFIGS)),) CONFIGS := $(if $(findstring -DWGMMA_RS,$(CONFIGS))$(findstring -DWGMMA_SS,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DWGMMA_RS) endif +# config.mk folded the profile in before this app appended its own flags; fold +# again so an app flag replaces the profile's setting of the same knob. +ifneq ($(strip $(PROFILE)),) +CONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --xlen=$(XLEN) --merge-configs --cflags='$(CONFIGS)') +endif + XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') ifneq (,$(filter -DVX_CFG_EXT_C_ENABLE, $(XCONFIGS))) From c240fcb8e687fed1414ec481bdd7edcd7cd47077 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 31 Oct 2026 00:15:56 -0700 Subject: [PATCH 302/336] pending_size: select the next count from precomputed steps The cache bank's MSHR pending counter (INCRW 1, DECRW 2) gets its decrement from the tag-compare release, so it arrives late; the wide-step path then ran it through a subtract, an add and four compares before the size and flag registers (cache DUT deepest path, 17 levels). For small widths (at most 8 possible steps) every next size and its four flags are now computed from size_r alone, and each (incr, decr) pair is decoded straight into a one-hot select of the precomputed step, so no adder sits on the late inputs. Wider counters keep the adder path. Beacon (cache DUT): full_r 17 -> 16, nothing deeper (lmem, raster, core unchanged). A Verilator testbench running the previous and new module side by side over 8 shapes (SIZE 3..64, INCRW/DECRW up to 3, both branches) for 200k cycles each, with fill/drain phases reaching full and empty, matches on every output every cycle; a variant with an off-by-one almost-full compare is caught on all six select-step shapes. rtlsim cycles are identical to the parent on sgemm -n32 and dxa_copy -d1/-d2/-d3 (DXA + L2). Verilator -Wall lint clean at XLEN 32 and 64. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/libs/VX_pending_size.sv | 67 +++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/hw/rtl/libs/VX_pending_size.sv b/hw/rtl/libs/VX_pending_size.sv index b94889e6e1..a9a1958544 100644 --- a/hw/rtl/libs/VX_pending_size.sv +++ b/hw/rtl/libs/VX_pending_size.sv @@ -64,7 +64,72 @@ module VX_pending_size #( reg empty_r, alm_empty_r; reg full_r, alm_full_r; - if (INCRW != 1 || DECRW != 1) begin : g_wide_step + localparam MAX_INCR = (1 << INCRW) - 1; + localparam MAX_DECR = (1 << DECRW) - 1; + localparam NUM_STEPS = MAX_INCR + MAX_DECR + 1; + + if ((INCRW != 1 || DECRW != 1) && (NUM_STEPS <= 8)) begin : g_select_step + + // incr/decr typically arrive late, so every next size and its flags + // are computed from size_r alone and they only select one. + localparam DELTAW = `MIN(SIZEW, `MAX(INCRW, DECRW)+1); + + logic [SIZEW-1:0] size_r; + wire [NUM_STEPS-1:0][SIZEW+3:0] step_data; + + for (genvar s = 0; s < NUM_STEPS; ++s) begin : g_steps + wire [SIZEW-1:0] step_size = size_r + SIZEW'(s - MAX_DECR); + assign step_data[s] = {step_size, + (step_size == SIZEW'(0)), + (step_size == SIZEW'(SIZE)), + (step_size <= SIZEW'(ALM_EMPTY)), + (step_size >= SIZEW'(ALM_FULL))}; + end + + // each (incr, decr) pair is decoded directly, so no adder sits on them + wire [NUM_STEPS-1:0] step_hot; + for (genvar s = 0; s < NUM_STEPS; ++s) begin : g_hot + wire [MAX_INCR:0][MAX_DECR:0] pair_hit; + for (genvar i = 0; i <= MAX_INCR; ++i) begin : g_i + for (genvar j = 0; j <= MAX_DECR; ++j) begin : g_j + assign pair_hit[i][j] = (i - j + MAX_DECR == s) && (incr == INCRW'(i)) && (decr == DECRW'(j)); + end + end + assign step_hot[s] = (| pair_hit); + end + + wire [SIZEW-1:0] size_n; + wire empty_n, full_n, alm_empty_n, alm_full_n; + VX_onehot_mux #( + .DATAW (SIZEW + 4), + .N (NUM_STEPS) + ) step_mux ( + .data_in (step_data), + .sel_in (step_hot), + .data_out ({size_n, empty_n, full_n, alm_empty_n, alm_full_n}) + ); + + always @(posedge clk) begin + if (reset) begin + empty_r <= 1; + full_r <= 0; + alm_empty_r <= 1; + alm_full_r <= 0; + size_r <= '0; + end else begin + `ASSERT((DELTAW'(incr) <= DELTAW'(decr)) || (size_n >= size_r), ("runtime error: counter overflow")); + `ASSERT((DELTAW'(incr) >= DELTAW'(decr)) || (size_n <= size_r), ("runtime error: counter underflow")); + empty_r <= empty_n; + full_r <= full_n; + alm_empty_r <= alm_empty_n; + alm_full_r <= alm_full_n; + size_r <= size_n; + end + end + + assign size = size_r; + + end else if (INCRW != 1 || DECRW != 1) begin : g_wide_step localparam DELTAW = `MIN(SIZEW, `MAX(INCRW, DECRW)+1); From 55b77804f7437c504c828ec07c31bc5a3b49ac9f Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 31 Oct 2026 03:48:55 -0700 Subject: [PATCH 303/336] dxa: shorten the row-wrap and tiled-destination address paths Both DXA units sat at 16 levels (Beacon); the work is cycle-identical. - addr_gen: the row-wrap path ran from a 32-bit dim_count == tile-1 compare through the step_delta mux, the cursor add, a 32-bit span add and calc_num_lines' +LINE-1 add into lines_left_r. The per-dim "at last index" flags are now registered with the odometer (count == tile-2 when it steps, tile == 1 when it wraps to 0), and row_len + LINE - 1 is latched per transfer, so the wrap starts at a flop and loses an adder. Only the low bits of row_len are kept for the last-CL end offset. - smem_wr: the per-CL tiled destination chained four variable shifts and two adds. The added fields are disjoint by their masks (n_in < 2^lg_tcN, the k remainders below their shifts), so the term is one OR of three masked shifts whose masks and shift amounts are latched per transfer, like calc_sh2_q. Beacon (dxa DUT): calc_intra_r 16 -> 4, lines_left_r 16 -> 11, last_vend_r 13 -> 10, gmem_cursor_r 13 -> 11, cl_addr_r 12 -> 9; both units 16 -> 15; nothing deeper (dim_at_last_r new at 9). Verified: a Verilator testbench running the previous and new addr_gen side by side over 20000 random 1-4D transfers (~1.04M beats, random back-pressure) matches every output every beat, and catches a variant with a wrong wrap flag on the first transfer; the smem_wr term matches the previous formula bit-exact on 1.5M random and edge-biased inputs in both modes and rejects a known-bad variant. rtlsim cycles identical to the parent: dxa_copy -d1/-d2/-d3 (6846/57193/34780), sgemm -n32, sgemm_tcu_wg_dxa (57895); sgemm_tcu_wg_sp_dxa passes. Verilator -Wall clean at XLEN 32 and 64. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/dxa/VX_dxa_addr_gen.sv | 62 +++++++++++++++++++++-------------- hw/rtl/dxa/VX_dxa_smem_wr.sv | 26 ++++++++++----- 2 files changed, 55 insertions(+), 33 deletions(-) diff --git a/hw/rtl/dxa/VX_dxa_addr_gen.sv b/hw/rtl/dxa/VX_dxa_addr_gen.sv index df1d34bbc0..f7067aee95 100644 --- a/hw/rtl/dxa/VX_dxa_addr_gen.sv +++ b/hw/rtl/dxa/VX_dxa_addr_gen.sv @@ -74,7 +74,8 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // ---- Registered state ---- reg active_r; reg [`VX_CFG_MEM_ADDR_WIDTH-1:0] gmem_cursor_r; // Current row's GMEM byte base - reg [31:0] row_len_r; // row_len_bytes (constant) + reg [CL_OFF_BITS-1:0] row_len_off_r; // row_len_bytes within a CL (constant) + reg [31:0] row_len_ceil_r; // row_len_bytes + LINE - 1 // Inner-loop counters/flags (advance every beat). reg [GMEM_ADDR_WIDTH-1:0] cl_addr_r; // Current CL address (GMEM) @@ -84,7 +85,9 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Outer-dim odometer. reg [DXA_MAX_OUTER_DIMS-1:0][31:0] dim_count_r; // Current index per dim - reg [DXA_MAX_OUTER_DIMS-1:0][31:0] dim_last_r; // Last index per dim (tile-1) + reg [DXA_MAX_OUTER_DIMS-1:0][31:0] dim_penult_r; // Index before the last (tile-2) + reg [DXA_MAX_OUTER_DIMS-1:0] dim_single_r; // tile == 1: index 0 is the last + reg [DXA_MAX_OUTER_DIMS-1:0] dim_at_last_r; // dim_count_r == tile-1 reg [DXA_MAX_OUTER_DIMS-1:0][31:0] delta_r; // Wrap delta per dim reg [DXA_MAX_OUTER_DIMS-1:0][31:0] oob_limit_r; // OOB limit per dim @@ -150,14 +153,12 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( || (dim_count_r[3] >= oob_limit_r[3]); // ---- Last-row detection: all outer dims at their last index ---- - // An odometer index reaches its bound by equality, not magnitude: with - // dim_count in [0, tile-1], (dim_count+1 >= tile) ⟺ (dim_count == tile-1). - // Equality is ~2 LUT levels vs a CARRY8 chain, which keeps the once-per-row - // step_delta / num_lines wrap path short. - wire dim0_last = (dim_count_r[0] == dim_last_r[0]); - wire dim1_last = (dim_count_r[1] == dim_last_r[1]); - wire dim2_last = (dim_count_r[2] == dim_last_r[2]); - wire dim3_last = (dim_count_r[3] == dim_last_r[3]); + // Registered alongside the odometer, so the once-per-row step_delta / + // num_lines wrap path starts at a flop instead of a 32-bit compare. + wire dim0_last = dim_at_last_r[0]; + wire dim1_last = dim_at_last_r[1]; + wire dim2_last = dim_at_last_r[2]; + wire dim3_last = dim_at_last_r[3]; wire is_last_outer = dim0_last && dim1_last && dim2_last && dim3_last; // ---- Output ---- @@ -204,7 +205,8 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] next_cursor = gmem_cursor_r + step_delta; wire [31:0] start_span = 32'(setup_params.initial_gmem_base[CL_OFF_BITS-1:0]) + setup_params.row_len_bytes; - wire [31:0] wrap_span = 32'(next_cursor[CL_OFF_BITS-1:0]) + row_len_r; + wire [CL_OFF_BITS-1:0] wrap_end_off = next_cursor[CL_OFF_BITS-1:0] + row_len_off_r; + wire [31:0] wrap_lines = (32'(next_cursor[CL_OFF_BITS-1:0]) + row_len_ceil_r) >> CL_OFF_BITS; function automatic logic [31:0] calc_num_lines(input logic [31:0] span); calc_num_lines = (span + GMEM_LINE_SIZE - 1) >> CL_OFF_BITS; @@ -245,8 +247,9 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( end else if (start) begin active_r <= 1'b1; gmem_cursor_r <= setup_params.initial_gmem_base; - row_len_r <= setup_params.row_len_bytes; - cl_addr_r <= setup_params.initial_gmem_base[`VX_CFG_MEM_ADDR_WIDTH-1:CL_OFF_BITS]; + row_len_off_r <= setup_params.row_len_bytes[CL_OFF_BITS-1:0]; + row_len_ceil_r <= setup_params.row_len_bytes + GMEM_LINE_SIZE - 1; + cl_addr_r <= setup_params.initial_gmem_base[`VX_CFG_MEM_ADDR_WIDTH-1:CL_OFF_BITS]; lines_left_r <= calc_num_lines(start_span); is_first_r <= 1'b1; last_vend_r <= calc_last_vend(start_span[CL_OFF_BITS-1:0]); @@ -267,8 +270,10 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( n_base_r <= '0; tiled_base_r <= setup_params.initial_smem_base; for (int d = 0; d < DXA_MAX_OUTER_DIMS; d++) begin - dim_count_r[d] <= '0; - dim_last_r[d] <= setup_params.dim_tiles[d] - 32'd1; + dim_count_r[d] <= '0; + dim_penult_r[d] <= setup_params.dim_tiles[d] - 32'd2; + dim_single_r[d] <= (setup_params.dim_tiles[d] == 32'd1); + dim_at_last_r[d] <= (setup_params.dim_tiles[d] == 32'd1); delta_r[d] <= setup_params.delta[d]; oob_limit_r[d] <= setup_params.oob_limit[d]; end @@ -293,9 +298,9 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // next beat is the new row's CL0. gmem_cursor_r <= next_cursor; cl_addr_r <= next_cursor[`VX_CFG_MEM_ADDR_WIDTH-1:CL_OFF_BITS]; - lines_left_r <= calc_num_lines(wrap_span); + lines_left_r <= wrap_lines; is_first_r <= 1'b1; - last_vend_r <= calc_last_vend(wrap_span[CL_OFF_BITS-1:0]); + last_vend_r <= calc_last_vend(wrap_end_off); // K-major: SMEM jumps back to the next row base (initial + // (i1+1) * esize). Maintained via km_row_base_r which we bump // by esize at each row wrap. Row-major path keeps its linear @@ -308,18 +313,25 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Tiled scatter: each new row restarts at N=0 (tile-relative). n_base_r <= '0; if (dim0_steps) begin - dim_count_r[0] <= dim_count_r[0] + 1; + dim_count_r[0] <= dim_count_r[0] + 1; + dim_at_last_r[0] <= (dim_count_r[0] == dim_penult_r[0]); end else begin - dim_count_r[0] <= '0; + dim_count_r[0] <= '0; + dim_at_last_r[0] <= dim_single_r[0]; if (dim1_steps) begin - dim_count_r[1] <= dim_count_r[1] + 1; + dim_count_r[1] <= dim_count_r[1] + 1; + dim_at_last_r[1] <= (dim_count_r[1] == dim_penult_r[1]); end else begin - dim_count_r[1] <= '0; + dim_count_r[1] <= '0; + dim_at_last_r[1] <= dim_single_r[1]; if (dim2_steps) begin - dim_count_r[2] <= dim_count_r[2] + 1; + dim_count_r[2] <= dim_count_r[2] + 1; + dim_at_last_r[2] <= (dim_count_r[2] == dim_penult_r[2]); end else begin - dim_count_r[2] <= '0; - dim_count_r[3] <= dim_count_r[3] + 1; + dim_count_r[2] <= '0; + dim_at_last_r[2] <= dim_single_r[2]; + dim_count_r[3] <= dim_count_r[3] + 1; + dim_at_last_r[3] <= (dim_count_r[3] == dim_penult_r[3]); end end end @@ -336,6 +348,8 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // km_step is LMEM-bounded; only the low DXA_SMEM_ADDR_W bits feed smem_byte_addr_r. `UNUSED_VAR (km_step[31:DXA_SMEM_ADDR_W]) + // the outermost dim only steps (all-last ends the transfer), never wraps to 0 + `UNUSED_VAR (dim_single_r[DXA_MAX_OUTER_DIMS-1]) `ifdef DBG_TRACE_DXA always @(posedge clk) begin diff --git a/hw/rtl/dxa/VX_dxa_smem_wr.sv b/hw/rtl/dxa/VX_dxa_smem_wr.sv index 05dff2b23a..d535ecfe7c 100644 --- a/hw/rtl/dxa/VX_dxa_smem_wr.sv +++ b/hw/rtl/dxa/VX_dxa_smem_wr.sv @@ -167,6 +167,9 @@ module VX_dxa_smem_wr import VX_gpu_pkg::*, VX_dxa_pkg::*; #( reg [DXA_SMEM_ADDR_W-1:0] tiled_step_q; reg [DXA_SMEM_ADDR_W-1:0] tiled_wrap_q; reg [4:0] calc_sh2_q; + reg [15:0] intra_khi_mask_q, intra_klo_mask_q; + reg [4:0] intra_khi_sh_q, intra_n_sh_q; + reg [3:0] esize_q; wire is_flat_w = (dest_mode_q == DXA_DEST_FLAT); wire [15:0] tcn_mask_w = (16'd1 << lg_tcN_q) - 16'd1; wire [4:0] step_sh_w = is_flat_w ? (5'(lg_ratio_q) + 5'(esize)) @@ -183,6 +186,16 @@ module VX_dxa_smem_wr import VX_gpu_pkg::*, VX_dxa_pkg::*; #( calc_sh2_q <= is_flat_w ? (5'(lg_tcN_q) + 5'(lg_tcN_q) + 5'd1 + 5'(lg_ratio_q) + 5'(esize)) : (5'(lg_tcN_q) + 5'(lg_bkK_q) + 5'(esize)); + // intra-block term of the per-CL dest calc (stage 1 below), as disjoint + // k/n bit fields shifted into place: + // FLAT: k[hi] << (lg_tcN+esize) | n_in << (lg_ratio+esize) | k[lo] << esize + // hi = ((2 << lg_tcN) - 1) << lg_ratio, lo = (1 << lg_ratio) - 1 + // BM: n_in << (lg_bkK+esize) | k[lo] << esize, lo = (1 << lg_bkK) - 1 + intra_khi_mask_q <= is_flat_w ? (((16'd2 << lg_tcN_q) - 16'd1) << lg_ratio_q) : 16'd0; + intra_khi_sh_q <= 5'(lg_tcN_q) + 5'(esize); + intra_n_sh_q <= step_sh_w; + intra_klo_mask_q <= (16'd1 << (is_flat_w ? lg_ratio_q : lg_bkK_q)) - 16'd1; + esize_q <= esize; end // ════════════════════════════════════════════════════════════════════ @@ -426,23 +439,18 @@ module VX_dxa_smem_wr import VX_gpu_pkg::*, VX_dxa_pkg::*; #( wire [15:0] calc_k = calc_pend ? pend_k_r : defer_k_r; wire [15:0] calc_n = calc_pend ? pend_n_r : defer_n_r; - wire [15:0] ratio_mask_w = (16'd1 << lg_ratio_q) - 16'd1; - wire [15:0] ktck_mask_w = (16'd2 << lg_tcN_q) - 16'd1; - wire [15:0] kw_mask_w = (16'd1 << lg_bkK_q) - 16'd1; wire [15:0] calc_n_blk = calc_n >> lg_tcN_q; wire [15:0] calc_n_in = calc_n & tcn_mask_q; wire [15:0] f_k_word = calc_k >> lg_ratio_q; - wire [15:0] f_elem = calc_k & ratio_mask_w; wire [15:0] f_k_blk = f_k_word >> (lg_tcN_q + 4'd1); - wire [15:0] f_kw_in = f_k_word & ktck_mask_w; wire [15:0] b_k_blk = calc_k >> lg_bkK_q; - wire [15:0] b_r_in = calc_k & kw_mask_w; wire [15:0] calc_k_blk = is_flat_w ? f_k_blk : b_k_blk; wire [31:0] calc_blk_w = (32'(calc_k_blk) << lg_nsteps_q) + 32'(calc_n_blk); - wire [31:0] calc_intra_w = is_flat_w - ? (((((32'(f_kw_in) << lg_tcN_q) + 32'(calc_n_in)) << lg_ratio_q) + 32'(f_elem)) << esize) - : (((32'(calc_n_in) << lg_bkK_q) + 32'(b_r_in)) << esize); + // the masked fields are disjoint, so the nested shift-adds are one OR + wire [31:0] calc_intra_w = (32'(calc_k & intra_khi_mask_q) << intra_khi_sh_q) + | (32'(calc_n_in) << intra_n_sh_q) + | (32'(calc_k & intra_klo_mask_q) << esize_q); reg [31:0] calc_blk_r, calc_intra_r; reg calc2_valid_r, calc2_pend_r; From 361c402d266841a711af7fdf7850ee5d2017f9c9 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 31 Oct 2026 03:49:37 -0700 Subject: [PATCH 304/336] rtu: byte-align the fetched lines and take EXEC's compares at ALIGN Two cycle-neutral cuts on the RTU scheduler's EXEC paths: - ctx_store: EXEC shifted the fetched lines to the structure's byte offset before decoding them. The shift now happens at ALIGN and is registered (f_aligned_q replaces fbuf_q); its amount is the low bits of structaddr_q, which only the low bits of the scene base and cur_off reach, so ALIGN forms it with a narrow add. - EXEC's 32-bit compares (coll_t0 < best_t, above-floor and before-yield for coll_t0 and the triangle t, yld_t >= trit, tri_i + 1 < tri_n, inst_idx + 1 == inst_cnt) and the next instance's record offset are registered at ALIGN from the sources EXEC's operands are captured from, replacing the key-only precompute. Only PROC_WAIT, TRI_WAIT and INST_NEXT read them, which a fresh or re-walk pass never enters, so word_x equals word_q there; yld_q is still read live at EXEC. Beacon (rtu / gfx DUT): deepest 18 -> 15 / 20 -> 20, ctx_store 18 -> 13 / 17 -> 14, commit_fifo 15 -> 7 / 15 -> 8, box PE sr_corner 15 -> 12; no register deeper. A registered wake pre-decode for the context arbiter was tried and dropped (no gain in rtu, gfx 20 -> 23). Verified: circt-lec proves the aligned buffer equal at 32/1024 and 48/1536 address/buffer widths and the compares and offset equal at FLAT 0 and 1 (with the fresh/re-walk override in the harness), and rejects four known-bad variants. SimX rt_suite + multicore smokes 40/40; rtlsim rt_suite, the rt_raycast parity case and the three rt_raycast perf_gate cases 39/39 with identical cycles on all runs (rt_raycast 159694); model_parity gap 2.2%. Verilator -Wall clean at BVH widths 0/2/4/8. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/rtu/VX_rtu_scheduler.sv | 90 +++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/hw/rtl/rtu/VX_rtu_scheduler.sv b/hw/rtl/rtu/VX_rtu_scheduler.sv index 2cc93d2066..62f0af6453 100644 --- a/hw/rtl/rtu/VX_rtu_scheduler.sv +++ b/hw/rtl/rtu/VX_rtu_scheduler.sv @@ -322,14 +322,22 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [CTX_TAG_W-1:0] sel_q; reg fresh_q; reg rewalk_q; - // candidate key order, precomputed at ALIGN off the store row: the key of - // the record under test ({instance, record offset}) against the floor and - // the staged candidate, so EXEC only adds the t compares - reg key_gt_floor_q; - reg key_lt_yld_q; + // EXEC's compares, precomputed at ALIGN off the sources its operands are + // captured from. Only states a fresh or re-walk pass never enters read + // them, so word_x is word_q there. *_before_q leaves out yld_q, which + // EXEC reads live. + reg t0_lt_best_q; // procedural t below the committed hit + reg t0_above_q; // ... above the resume floor + reg t0_before_q; // ... ahead of the staged candidate + reg tri_above_q; + reg tri_before_q; + reg tri_occl_q; // a hit at trit occludes the staged candidate + reg tri_more_q; // a further triangle in the loop + reg inst_last_q; // the last instance of the leaf + reg [31:0] inst_next_off_q; // the next instance's record offset ctx_state_t word_q; lane_ray_t ray_q; - reg [BUF_BITS-1:0] fbuf_q; + reg [BUF_BITS-1:0] f_aligned_q; // fetched lines, byte-aligned to the structure reg [STK_ENTW-1:0] stacktop_q; reg [PATHW-1:0] pathv_q; // rank path entry below the current node reg [ADDRW-1:0] structaddr_q; @@ -526,6 +534,24 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [COLL_SIZE-1:0] coll_prochit; wire [31:0] cand_ki_al = cs_word.in_blas ? cs_word.inst_id : 32'd0; + wire key_gt_floor_al = {cand_ki_al, cs_word.cur_off} > {cs_word.floor_ki, cs_word.floor_ko}; + wire key_lt_yld_al = {cand_ki_al, cs_word.cur_off} < {cs_word.yld_ki, cs_word.yld_ko}; + wire [31:0] coll_t0_al = coll_ordt[cs_word.coll_id][0]; + wire [31:0] trit_al = trires_rdata[95:64]; + wire [31:0] inst_next_al = cs_word.inst_idx + 32'd1; + function automatic logic above_floor_al(input logic [31:0] t); + above_floor_al = !cs_word.has_floor + || (t > cs_word.floor_t) + || ((t == cs_word.floor_t) && key_gt_floor_al); + endfunction + function automatic logic before_yld_al(input logic [31:0] t); + before_yld_al = (t < cs_word.yld_t) + || ((t == cs_word.yld_t) && key_lt_yld_al); + endfunction + // the structure's byte offset in its line: the low bits of structaddr_q, + // which only the low bits of its two addends reach + wire [RTU_LINE_SEL_BITS-1:0] f_off_al = slot_scene[s1_slot][RTU_LINE_SEL_BITS-1:0] + + ((s1_fresh || s1_rewalk) ? RTU_LINE_SEL_BITS'(0) : cs_word.cur_off[RTU_LINE_SEL_BITS-1:0]); // ═══════════════════════ stage advance ════════════════════════════ always_ff @(posedge clk) begin @@ -547,11 +573,18 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( sel_q <= s1_sel; fresh_q <= s1_fresh; rewalk_q <= s1_rewalk; - key_gt_floor_q <= {cand_ki_al, cs_word.cur_off} > {cs_word.floor_ki, cs_word.floor_ko}; - key_lt_yld_q <= {cand_ki_al, cs_word.cur_off} < {cs_word.yld_ki, cs_word.yld_ko}; + t0_lt_best_q <= (coll_t0_al < cs_word.best_t); + t0_above_q <= above_floor_al(coll_t0_al); + t0_before_q <= before_yld_al(coll_t0_al); + tri_above_q <= above_floor_al(trit_al); + tri_before_q <= before_yld_al(trit_al); + tri_occl_q <= (cs_word.yld_t >= trit_al); + tri_more_q <= ((cs_word.tri_i + 32'd1) < cs_word.tri_n); + inst_last_q <= (inst_next_al == cs_word.inst_cnt); + inst_next_off_q <= cs_word.inst_base + (inst_next_al * 32'(RTU_INST_STRIDE)); word_q <= cs_word; ray_q <= lane_ray_t'(ray_rdata); - fbuf_q <= fbuf; + f_aligned_q <= fbuf >> {f_off_al, 3'b000}; stacktop_q <= stk_rdata; pathv_q <= path_rdata; // a fresh context's store row is stale and a re-walk restarts: @@ -572,9 +605,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end // ═══════════════════════ EXEC decode ══════════════════════════════ - wire [RTU_LINE_SEL_BITS-1:0] f_off = structaddr_q[RTU_LINE_SEL_BITS-1:0]; - wire [RTU_LINE_SEL_BITS+2:0] f_shift = {f_off, 3'b000}; - wire [BUF_BITS-1:0] f_aligned = fbuf_q >> f_shift; + wire [RTU_LINE_SEL_BITS-1:0] f_off = structaddr_q[RTU_LINE_SEL_BITS-1:0]; + wire [BUF_BITS-1:0] f_aligned = f_aligned_q; wire [31:0] f_off32 = 32'(f_off); wire [LB-1:0] tri_rec_lines = @@ -1275,16 +1307,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // Candidate order: ascending (t, key), key = {instance id, record offset}. // A candidate is staged only above the floor and ahead of the staged one. wire [31:0] cand_ki = word_q.in_blas ? word_q.inst_id : 32'd0; - function automatic logic above_floor(input logic [31:0] t); - above_floor = !word_q.has_floor - || (t > word_q.floor_t) - || ((t == word_q.floor_t) && key_gt_floor_q); - endfunction - function automatic logic before_yld(input logic [31:0] t); - before_yld = !yld_q[sel_q] - || (t < word_q.yld_t) - || ((t == word_q.yld_t) && key_lt_yld_q); - endfunction + wire yld_sel = yld_q[sel_q]; // EXEC outcome (combinational) ctx_state_t word_n; @@ -1589,9 +1612,9 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_PROC_WAIT: begin // woken by the collector: the raw AABB result landed if (coll_hit_q - && (coll_t0_q < word_x.best_t) - && above_floor(coll_t0_q) - && before_yld(coll_t0_q)) begin + && t0_lt_best_q + && t0_above_q + && (!yld_sel || t0_before_q)) begin cf_din_r.kind = CK_YLDP; cf_din_r.t = coll_t0_q; cf_din_r.prim = word_x.prim_base; @@ -1678,13 +1701,13 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( exec_hit_set = 1'b1; word_n.best_t = trit_q; // a closer opaque hit occludes a farther candidate - if (yld_q[sel_q] && (word_x.yld_t >= trit_q)) begin + if (yld_sel && tri_occl_q) begin exec_yld_clr = 1'b1; end if (term_first) begin word_n.cstate = CS_DONE; exec_done = 1'b1; - end else if ((word_x.tri_i + 32'd1) < word_x.tri_n) begin + end else if (tri_more_q) begin word_n.tri_i = word_x.tri_i + 32'd1; word_n.cur_off = word_x.cur_off + 32'(RTU_TRI_STRIDE); word_n.cstate = CS_LTRI_REQ0; @@ -1695,8 +1718,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end end else if (tri_pass - && above_floor(trit_q) - && before_yld(trit_q)) begin + && tri_above_q + && (!yld_sel || tri_before_q)) begin cf_din_r.kind = CK_YLDA; cf_din_r.t = trit_q; cf_din_r.u = triu_q; @@ -1718,7 +1741,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( word_n.yld_t = trit_q; word_n.yld_ki = cand_ki; word_n.yld_ko = word_x.cur_off; - if ((word_x.tri_i + 32'd1) < word_x.tri_n) begin + if (tri_more_q) begin word_n.tri_i = word_x.tri_i + 32'd1; word_n.cur_off = word_x.cur_off + 32'(RTU_TRI_STRIDE); word_n.cstate = CS_LTRI_REQ0; @@ -1728,7 +1751,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wake_self = 1'b1; end end else begin - if ((word_x.tri_i + 32'd1) < word_x.tri_n) begin + if (tri_more_q) begin word_n.tri_i = word_x.tri_i + 32'd1; word_n.cur_off = word_x.cur_off + 32'(RTU_TRI_STRIDE); word_n.cstate = CS_LTRI_REQ0; @@ -1867,7 +1890,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // path ends at the instance it resumed. word_n.in_blas = 1'b0; word_n.follow = 1'b0; - if ((word_x.inst_idx + 32'd1) == word_x.inst_cnt) begin + if (inst_last_q) begin if (FLAT) begin word_n.cstate = CS_DONE; exec_done = 1'b1; @@ -1877,8 +1900,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end else begin word_n.inst_idx = word_x.inst_idx + 32'd1; - word_n.cur_off = word_x.inst_base - + ((word_x.inst_idx + 32'd1) * 32'(RTU_INST_STRIDE)); + word_n.cur_off = inst_next_off_q; word_n.cstate = CS_INST_REQ; wake_self = 1'b1; end From 15de4b470b64f9178ce88b5c43588422987f21f8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 31 Oct 2026 09:50:33 -0700 Subject: [PATCH 305/336] cta_dispatch: elaborate the small and single-lane shapes 6f08ef66e broke two configurations: - Verilator -Wall (CMPCONST) rejected the remaining-warps thermometer in shapes where 2*NUM_LANES thresholds exceed what full_warps_r can hold (the sgemm_tcu_wg* defaults, NUM_WARPS=4 with ISSUE_WIDTH=4), so those five tests no longer built on rtlsim. A threshold at or above 2^FULL_W-1 is now the constant 0 the compare already was. - CIRCT could not legalize the init-window write thread_idx_init_r[init_cnt_r] with one lane: the array index is zero bits wide (comb.icmp on i0), so the fpga_gate core DUT failed before synthesis. The write now decodes per lane, like the thread_idx_r write above it; inside the branch init_cnt_r < NUM_LANES, so it is the same element. Both are exact rewrites. Verified: the fpga_gate core DUT CIRCT- elaborates again (it failed at 6f08ef66e and passed at its parent); Verilator -Wall clean at XLEN 32 and 64 on the five tests' resolved CONFIGS, the core DUT config, default, 1x1 and 64x32; rtlsim passes sgemm_tcu_wg (16338 cycles), _mx, _sp, _dxa, _sp_dxa and sgemm. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_cta_dispatch.sv | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index 4d0196faff..7c1aa9d71d 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -269,7 +269,12 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( wire [FULL_W-1:0] full_warps_r = block_size_r[CTA_TID_WIDTH:NT_BITS]; wire [2*NUM_LANES-1:0] full_r_gt, rem_r_gt; for (genvar j = 0; j < 2*NUM_LANES; ++j) begin : g_rem_therm - assign full_r_gt[j] = (full_warps_r > FULL_W'(j)); + // a block holds at most 2^FULL_W-1 full warps + if (j < (1 << FULL_W) - 1) begin : g_full_cmp + assign full_r_gt[j] = (full_warps_r > FULL_W'(j)); + end else begin : g_full_never + assign full_r_gt[j] = 1'b0; + end if (j == 0) begin : g_first assign rem_r_gt[j] = full_r_gt[j] | part_warp_r; end else begin : g_rest @@ -711,7 +716,11 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // window constants: step warp_step_r once per cycle if (init_busy) begin if (int'(init_cnt_r) < NUM_LANES) begin - thread_idx_init_r[init_cnt_r] <= init_next; + for (integer k = 0; k < NUM_LANES; ++k) begin + if (int'(init_cnt_r) == k) begin + thread_idx_init_r[k] <= init_next; + end + end init_cnt_r <= init_cnt_r + LCNT_WIDTH'(1); end else begin group_step_r <= init_next; From f3b0aab16d6e0f3deea18d828d3174d6524731c9 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 31 Oct 2026 14:03:24 -0700 Subject: [PATCH 306/336] raster: the early-Z DCR gate resets off A draw that never programs the OM depth state must not cull, or read a depth buffer at a power-on address. Under VX_CFG_RASTER_EARLYZ_ENABLE the early-Z safe flag and the depth function now reset to off and ALWAYS. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/raster/VX_raster_dcr.sv | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hw/rtl/raster/VX_raster_dcr.sv b/hw/rtl/raster/VX_raster_dcr.sv index 6297c29a87..f535112ae6 100644 --- a/hw/rtl/raster/VX_raster_dcr.sv +++ b/hw/rtl/raster/VX_raster_dcr.sv @@ -32,7 +32,9 @@ module VX_raster_dcr import VX_gpu_pkg::*, VX_raster_pkg::*; output raster_dcrs_t raster_dcrs ); `UNUSED_SPARAM (INSTANCE_ID) +`ifndef VX_CFG_RASTER_EARLYZ_ENABLE `UNUSED_VAR (reset) +`endif // Decode write strobe from the req-style DCR bus wire write_valid = dcr_bus_if.req_valid && dcr_bus_if.req_data.rw; @@ -47,8 +49,16 @@ module VX_raster_dcr import VX_gpu_pkg::*, VX_raster_pkg::*; // DCR registers raster_dcrs_t dcrs; - // DCRs write + // DCRs write. The early-Z gate resets off: a draw that never programs the + // OM depth state must not cull, or read a depth buffer at a power-on + // address. always @(posedge clk) begin + `ifdef VX_CFG_RASTER_EARLYZ_ENABLE + if (reset) begin + dcrs.earlyz_safe <= 1'b0; + dcrs.depth_func <= OM_DEPTH_FUNC_BITS'(`VX_OM_DEPTH_FUNC_ALWAYS); + end else + `endif if (write_valid) begin case (write_addr) `VX_DCR_RASTER_TBUF_ADDR: begin From bff0b624665959529eda6952d4487f9efe39376f Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 31 Oct 2026 14:03:39 -0700 Subject: [PATCH 307/336] vm: translate the socket units' memory traffic through a unit MMU Under VM the RTU's rtcache misses and the DXA's global reads now go through a socket unit MMU, a VX_mmu instance with its own TLB (VX_CFG_UTLB_SIZE), so a BVH root, an acceleration-structure pointer or a tensor descriptor is a virtual address like any kernel pointer. - The clients own the request register in front of the MMU: the rtcache registers its memory requests and the DXA registers its gmem output under VM, in RTL and SimX; the MMU has no input stage (REQ_IN_BUF removed). - A flush token holds on a lane while that lane has parked requests, on data-side MMUs with writing clients, so it reaches the cache behind the lane's earlier requests. Under VM the core's dcache flush also waits on its own dMMU lanes (lane_idle) and is latched until the request drops. - A data-side permission fault caught on a hit is answered with zeroed data from a one-entry per-lane slot. - New gfxvm logic_gate DUT, and vm.yaml cases running rt_smoke and sgemm2_dxa at a minted VA on simx and rtlsim, plus rt_raycast parity. Beacon: the deepest path is unchanged (22 in the DUTs, 29 in the full GPU). Moving the request register into the clients moves the TLB lookup's ready path onto their output buffers: rtcache 2-4 -> 13 levels, DXA ready output 1 -> 13 in the gfxvm top. This depth was the MMU input buffer's before, and is intended; the logic_gate baseline needs a re-record for it. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 5 + ci/testcases/logic_gate.yaml | 8 + ci/testcases/vm.yaml | 33 ++++ docs/designs/virtual_memory_subsystem.md | 118 ++++++++++++-- hw/rtl/VX_define.vh | 10 ++ hw/rtl/VX_gpu_pkg.sv | 16 +- hw/rtl/VX_socket.sv | 135 ++++++++++++++-- hw/rtl/core/VX_core.sv | 20 +++ hw/rtl/dxa/VX_dxa_core.sv | 9 +- hw/rtl/vm/VX_mmu.sv | 193 +++++++++++++++++++---- hw/rtl/vm/VX_mmu_snoop.sv | 37 ++++- hw/syn/xilinx/dut/catalog.mk | 10 +- hw/syn/yosys/dut/catalog.mk | 7 +- hw/unittest/gfx/VX_gfx_top.sv | 131 +++++++++++++++ hw/unittest/vm/VX_vm_top.sv | 23 ++- sim/simx/cluster.cpp | 26 ++- sim/simx/constants.h | 9 ++ sim/simx/dxa/dxa_core.cpp | 9 ++ sim/simx/dxa/dxa_core.h | 1 + sim/simx/mem/cache.cpp | 21 +++ sim/simx/mem/cache.h | 4 + sim/simx/mem/mmu.cpp | 70 ++++++-- sim/simx/mem/mmu.h | 14 +- sim/simx/socket.cpp | 76 ++++++++- sim/simx/socket.h | 17 +- sw/runtime/include/dxa.h | 6 +- 26 files changed, 917 insertions(+), 91 deletions(-) diff --git a/VX_config.toml b/VX_config.toml index ccfa148a0e..b00033eb8e 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -179,6 +179,11 @@ VX_CFG_DTLB_SIZE = 16 VX_CFG_ITLB_SIZE = 8 VX_CFG_L1_TLB_MSHR_SIZE = 4 +# Socket unit MMU: one TLB per socket for the socket-resident units' memory +# ports (RTU rtcache misses, DXA gmem reads), kept apart from the dTLB so +# BVH traffic cannot evict the cores' translations. +VX_CFG_UTLB_SIZE = 8 + # Shared cluster TLB and page-table walker complex. VX_CFG_L2_TLB_SIZE = 512 VX_CFG_L2_TLB_NUM_WAYS = 4 diff --git a/ci/testcases/logic_gate.yaml b/ci/testcases/logic_gate.yaml index 011c1806f6..05909f5588 100644 --- a/ci/testcases/logic_gate.yaml +++ b/ci/testcases/logic_gate.yaml @@ -188,6 +188,14 @@ builds: clock_mhz: 250 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 +# The same slice under VM: the unit MMUs in front of the graphics caches and the +# DXA, which neither gfx (VM off) nor vm (no units) builds. +- id: gfxvm + group: graphics + dut: gfxvm + clock_mhz: 250 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + # --- tensor (combined subsystem) --------------------------------------------- # Full tensor subsystem in one wrapper: TCU (WGMMA + MX) + dxa behind a shared L2, diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index 3540d0e161..8dd1b6fb44 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -244,6 +244,28 @@ tests: app: sgemm configs: -DVX_CFG_VM_ENABLE -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 +# Socket unit MMU: the RTU's BVH fetches and the DXA's global reads translate, +# so a scene or a tile source at a minted VA (!= PA) is read correctly (both +# apps check their results; dxa_copy does not). +- id: vm-rtu + via: blackbox + drivers: + - simx + - rtlsim + app: rt_smoke + configs+: -DVX_CFG_EXT_RTU_ENABLE + env: + VORTEX_RANDOMIZE_VA: 1 +- id: vm-dxa + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_dxa + configs+: -DVX_CFG_EXT_DXA_ENABLE + env: + VORTEX_RANDOMIZE_VA: 1 + # --------------------------------------------------------------------------- # check: model_parity — same VM-enabled app on both drivers (xlen 32); exact # instret + cycles within tolerance. SimX is the timing model of the RTL. @@ -258,6 +280,17 @@ tests: via: blackbox app: vecadd args: -n16384 +- id: parity-vm-rt_raycast + check: model_parity + via: blackbox + app: rt_raycast + configs+: -DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_EXT_DXA_ENABLE +- id: parity-vm-dxa_copy + check: model_parity + via: blackbox + app: dxa_copy + configs+: -DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_EXT_DXA_ENABLE + args: -d3 # The RTL command processor's DMA walks the page table for every upload and # download; only xrt runs it (simx and rtlsim use the C++ command processor). - id: cp-dma-walk diff --git a/docs/designs/virtual_memory_subsystem.md b/docs/designs/virtual_memory_subsystem.md index 99701ab776..7efb3e8ae7 100644 --- a/docs/designs/virtual_memory_subsystem.md +++ b/docs/designs/virtual_memory_subsystem.md @@ -160,8 +160,11 @@ kernel image once the loader knows its extents. Buffers allocated with (`VX_CFG_VM_PINNED_REGION_SIZE`), so a kernel can hold PA-stable pointers while paged and identity translations contend in the same TLBs. -The graphics fixed-function masters (RASTER, TEX, OM) and the RTU issue -physical addresses and never pass through an MMU; see +The socket-resident units translate too: the RTU's rtcache misses and the +DXA's global reads go through the socket **unit MMU** (§3.8), so a BVH root, +an acceleration-structure pointer or a tensor descriptor is a virtual address +like any kernel pointer. The graphics fixed-function masters (RASTER, TEX, OM) +still issue physical addresses and never pass through an MMU; see [`graphics_hardware_stack.md`](graphics_hardware_stack.md). --- @@ -212,7 +215,7 @@ mutually exclusive by priority: | `cat_bypass` | `valid ∧ req_bypass` | fields forwarded unchanged | | `cat_hit` | `valid ∧ ¬bypass ∧ ¬mshr_match ∧ cam_hit ∧ perm_ok` | `PA = {ppn_spliced, addr[11:0]}` forwarded; MRU bump | | `cat_park` | `valid ∧ ¬bypass ∧ (mshr_match ∨ ¬cam_hit)` | lowest parking lane parks; others hold and retry | -| `cat_pfault` | `valid ∧ ¬bypass ∧ ¬mshr_match ∧ cam_hit ∧ ¬perm_ok` | accepted and dropped; fault sideband pulses | +| `cat_pfault` | `valid ∧ ¬bypass ∧ ¬mshr_match ∧ cam_hit ∧ ¬perm_ok` | accepted, never sent to memory; a data-side load or atomic is answered with zeroed data (§3.5); fault sideband pulses | `mshr_match` outranks `cam_hit` on purpose: a request to a page whose walk is still landing must queue behind that page's parked requests, or a same-lane @@ -267,7 +270,10 @@ or atomic still owes the pipeline a response, so `VX_mmu` injects a zeroed response carrying the parked tag ahead of the cache reply for that lane (`kill_needs_rsp = (EXEC_SIDE == 0) ∧ (¬rw ∨ amo)`). A plain store and every instruction fetch retire silently — a fabricated instruction word would be -decoded as real. +decoded as real. A permission fault caught on +a hit is answered the same way: the load or atomic waits in a one-entry +per-lane slot (the lane holds while it is full) and its zeroed response goes +out behind any kill on that lane. Ordering follows from the structure. The stage is hit-under-miss, so only same-page order is guaranteed: accesses to one VPN share an entry and drain in @@ -278,6 +284,20 @@ joiners: the next same-VPN request re-walks rather than inheriting the kill. Allocation is refused during a flush, because every entry clears at that edge and an accepted request would vanish with neither replay nor kill. +A cache flush token bypasses translation, so on its own it would overtake its +lane's parked requests. Each lane counts the requests it has in the miss +station (up on a park, down on the replay or kill that drains one), and a +flush token on a lane holds while that count is non-zero: it reaches the cache +behind every earlier request of its lane, which is the order a +`FLUSH_FORWARD` cache relies on to forward the token behind that port's +writes. Only a data-side MMU with writing clients counts (the dMMU and the +cluster unit MMU): a read or fetch overtaken by a token is still answered +correctly. The token of one lane does not wait on another lane's parks; the +core covers that case for the dcache: under VM its flush request also waits +until its own dMMU lanes are idle (`lane_idle`, which counts the parks), as it +waits for its stores, and is then latched until the flush request drops, since +the token's own trip through the MMU makes a lane busy again. + ### 3.6 Flush and drain `flush_clear = flush_if.req ∧ tlb_empty`: the CAM and the miss station clear @@ -295,6 +315,52 @@ request, one lane per core, `ITLB_SIZE` (8) entries. Fetch reaches it through icache stream; that request carries the FLUSH attribute and bypasses translation. +### 3.8 The socket unit MMU + +The socket-resident units reach memory through the socket memory arbiter, not +through the dcache, so the dMMU never sees them. Under VM a third `VX_mmu` +instance per socket, `ummu` +([`VX_socket.sv`](../../hw/rtl/VX_socket.sv)), sits **on their memory side**: +between the rtcache's miss port and the DXA's gmem port on one side and the +socket arbiter's `RTCACHE_MEM_ARB_IDX` / `DXA_MEM_ARB_IDX` inputs on the other, +one lane per port (`UMMU_RTCACHE_IDX`, `UMMU_DXA_IDX` in +[`VX_gpu_pkg.sv`](../../hw/rtl/VX_gpu_pkg.sv)), each lane an L1 line wide with +the socket arbiter's tag. It is the same module as the dMMU, with four +differences: + +- **Its own TLB**, `VX_CFG_UTLB_SIZE` (8) entries, so BVH and tensor streams + cannot evict the cores' translations, and its misses ride a third socket + `tlb_bus` into `tlb_socket_arb` (3 inputs), on to the cluster TLB and the + walker like any L1 miss. +- **A registered reply** (`RSP_OUT_BUF = 2`). Each reply leaves through an + MMU-owned stage, so the select that injects kill and permission-fault + answers stays off the units' response logic. The requests arrive from + flops their clients own: the unit caches register their memory requests + (`MEM_OUT_BUF`), and the DXA registers its gmem output under VM. The TLB + lookup therefore starts at a client flop and its accept ends at the + client's registered ready, never deeper inside the unit. A unit access pays + the reply stage plus the MMU's own output stage, under VM only. +- **`WRITE_ENABLE = 0`.** Its clients only read, so a parked request keeps + `{rw, addr, attr, tag}` and the replay restores zero data and all byte + enables; the miss-station payload is a line's address, not a line. +- **The unit caches are virtually indexed and tagged**, since translation + happens after them. They are invalidated whenever translation changes: the + socket snoop's `SATP_HI` flush (§8.2) also flushes the unit MMU's TLB and, + from a latch of its own, injects a flush token into the rtcache through its + `VX_dcr_flush` leg (OR-ed with the core-driven graphics flush), held until + the rtcache reports done. The two legs are independent: a cache flush waits + for its outstanding misses, which may need a unit-TLB park, and a TLB under + flush refuses parks, so holding the TLB flush for the cache's walk could + deadlock. The rtcache is read-only, so the flush only invalidates. + A single address space has no synonyms, so a VIVT read-only cache flushed + on every `satp` change is exact. + +Without VM, or with neither RTU nor DXA (`SOCKET_UNIT_MMU_ENABLE` undefined), +none of this exists: the unit ports wire straight into the arbiter as before. +The unit MMU's `empty` holds the socket `busy`, but not the per-core +`mmu_drained` barrier gate: that gate orders a core's own loads and stores, +none of which pass through the unit MMU. + --- ## 4. The miss/fill fabric @@ -317,11 +383,11 @@ elastic slice (`OUT_BUF = 3`). The id therefore grows deterministically ``` L1_TLB_ID_WIDTH = clog2(L1_TLB_MSHR_SIZE) the L1 miss-station slot -TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + sel(2) + {dMMU, iMMU} +TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + sel(SOCKET_NUM_MMUS) + {dMMU, iMMU[, unit MMU]} TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + sel(NUM_SOCKETS) + socket ``` -The socket arbiter merges the dMMU and iMMU ports onto `cluster_tlb_bus_if`, +The socket arbiter merges the dMMU, iMMU and (when present) unit MMU ports onto `cluster_tlb_bus_if`, the cluster arbiter merges every socket onto the cluster TLB's client port. A fill routes back to exactly one L1 slot with no table lookups anywhere. @@ -516,7 +582,8 @@ held until the next DCR access retires it, muxed ahead of the cluster reply. socket's DCR chain, forwarding it to the cores unchanged while snooping the satp writes. It derives `vm_active` from the mode field (`SATP_LO[31]` on Sv32, `SATP_HI[31:28] ≠ 0` on Sv39) and, on the `SATP_HI` -write, requests a flush of both L1 MMUs, held until both report done. The +write, requests a flush of every L1 MMU, held until all report done; with a +unit MMU the rtcache invalidation of §3.8 runs as a separate leg. The flush is **self-timed**: satp only changes between kernels, once the TLBs are already drained, so no completion handshake crosses the socket boundary. This is what keeps `satp`, flush, and fault ports off the socket interface — only @@ -584,10 +651,16 @@ SimObjects bound through channels: `ReqIn/ReqOut` per port, a `Tlb` CAM (`mmu_tlb.cpp`, MRU replacement, superpage splice), an `L1_TLB_MSHR_SIZE`-entry miss station with two parked requests per entry, per-port replay queues that drain ahead of new input, - per-port kill-response queues, and the `TlbMissOut/TlbFillIn` link. Fills - install, then re-check each parked request's own permission and either - replay it translated or kill it. Responses pass through with no added - latency: the stage's cost is charged on the request path. + per-port kill-response queues, a one-entry permission-fault slot per port + (a port stalls its next fault while it is full), and the + `TlbMissOut/TlbFillIn` link. Fills install, then re-check each parked + request's own permission and either replay it translated or kill it. With + writing data-side clients a flush token holds while its port has requests + in the miss station. With a registered reply (the unit MMU) each response + leaves a cycle late; otherwise responses pass through. The DXA's gmem + output carries its own request register under VM, as in the RTL. + The dcache flush needs no MMU wait: SimX flushes the caches from the host + after the run has drained, when every MMU is already empty. - **`L2Tlb`** ([`tlb_l2.cpp`](../../sim/simx/mem/tlb_l2.cpp)) — one request per tick accepted round-robin over clients into a pipe that retires after `VX_CFG_L2_TLB_LATENCY` ticks; megapage `Tlb` probed first, then the @@ -600,7 +673,10 @@ SimObjects bound through channels: walker index in the tag, and the first-fault latch. Wiring ([`cluster.cpp`](../../sim/simx/cluster.cpp)): every core's two `Mmu` -instances are clients of the cluster `L2Tlb`; the `Ptw` shares L2-cache port 0 +instances are clients of the cluster `L2Tlb`, followed by each socket's unit +`Mmu` ([`socket.cpp`](../../sim/simx/socket.cpp)), placed as in the RTL +between the rtcache / DXA gmem ports and the socket arbiter; `set_satp` flushes +its TLB and invalidates the unit caches (`Cache::invalidate`); the `Ptw` shares L2-cache port 0 with socket 0 through a **priority** arbiter that favours PTE fetches — every translated access behind a walk is blocked until it completes, and the walkers issue far too little traffic to starve the demand stream (the reverse @@ -670,6 +746,7 @@ untranslated. | `VX_CFG_VM_ENABLE` | false | build the MMU, TLB fabric, cluster TLB and walker at all | | `VX_CFG_DTLB_SIZE` | 16 | socket dMMU CAM entries | | `VX_CFG_ITLB_SIZE` | 8 | socket iMMU CAM entries | +| `VX_CFG_UTLB_SIZE` | 8 | socket unit MMU CAM entries (RTU, DXA) | | `VX_CFG_L1_TLB_MSHR_SIZE` | 4 | L1 miss-station entries (distinct outstanding VPNs); 2 tokens each | | `VX_CFG_L2_TLB_SIZE` / `NUM_WAYS` | 512 / 4 | cluster TLB set array | | `VX_CFG_L2_TLB_MEGA_SIZE` | 8 | cluster TLB superpage CAM | @@ -705,6 +782,9 @@ sizes must be powers of two ≥ 2 (static asserts). | `tlb-concurrent`, `mc-l2-amo`, `mc-l3` | 2–4 cores, L2 / L3 | shared cluster TLB and walker across cores and cache levels | | `vm-fault` | `vm_fault -t0` | unmapped page → device error and first-fault report | | `parity-vm-sgemm/vecadd` | `check: model_parity` | exact instret + cycles within tolerance, simx vs rtlsim | +| `vm-rtu` | `rt_smoke`, `EXT_RTU`, randomized VA | rtcache misses through the socket unit MMU | +| `vm-dxa` | `sgemm2_dxa`, `EXT_DXA`, randomized VA | DXA gmem reads through the socket unit MMU | +| `parity-vm-rt_raycast`, `parity-vm-dxa_copy` | `check: model_parity`, RTU + DXA | unit-MMU timing, simx vs rtlsim | `vm_fault -t1` (a store to a read-only page) is not in CI: it needs the L1 permission-fault *report*, which the RTL does not surface (§9, §13). @@ -726,6 +806,8 @@ Grades: ✅ done · ⚠️ partial · ❌ pending. | area | state | note | |---|:--:|---| | Socket L1 translation stage, lane-parallel, hit-under-miss | ✅ | RTL + SimX | +| Socket unit MMU for RTU and DXA, VIVT rtcache invalidated on `satp` | ✅ | RTL + SimX | +| TEX, RASTER, OM translation | ❌ | still physical (`VX_MEM_PHYS` buffers) | | VPN-dedup miss stations with per-entry requester queues, ordered drain | ✅ | L1 and cluster | | Narrow miss/fill fabric with id-prepending arbiters | ✅ | socket + cluster levels | | Cluster TLB: pipelined set-associative array + megapage CAM | ✅ | RTL + SimX | @@ -742,6 +824,18 @@ Grades: ✅ done · ⚠️ partial · ❌ pending. | Chip-wide L3 TLB, TLB inclusion policy, a `VX_dma` block with its own TLB | ❌ | not started | | ASIDs, A/D write-back, PMP, Sv48/Sv57 | ❌ | deliberate: single global address space; runtime pre-sets A/D | +**Known limitation.** The `satp` commit's invalidation of the rtcache shares +that cache's flush leg with the core-driven graphics flush: the two requests +are OR-ed onto one `VX_dcr_flush`, so whichever is raised second completes on +the walk the first one started — a core flush raised while the `satp` request +is high completes on the `satp` flush's done, and the reverse. Both directions +are benign for these read-only caches: the walk blocks the cache's inputs and +invalidates every line, so a walk that completes after a request was raised +leaves no line the request could still see. The runtime also commits `satp` +once, before the first launch, with no core flush in progress. A writing cache, +or a commit that must complete strictly after a given core flush, would need +its own requester. + **Deliberate simplifications** (recorded so they are not mistaken for bugs): there is one global address space and no ASID — `satp` is programmed once per address space, never per dispatch, and the ASID field is ignored diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index 7607b2bdf6..b2416abaa9 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -83,6 +83,16 @@ `define EXT_GFX_CLUSTER_ENABLE `endif +// Under VM, the socket unit MMU translates the memory ports of the +// socket-resident units: the rtcache misses and the DXA gmem reads. +`ifdef VX_CFG_VM_ENABLE +`ifdef VX_CFG_EXT_RTU_ENABLE + `define SOCKET_UNIT_MMU_ENABLE +`elsif VX_CFG_EXT_DXA_ENABLE + `define SOCKET_UNIT_MMU_ENABLE +`endif +`endif + // Early-Z occlusion cull requires BOTH the rasterizer (produces the covered-quad // waves + depth plane) and the OM (owns the ocache the depth read is coherent // with). It is illegal without them — reading committed depth needs the ocache. diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 192d1aa688..fa83dc9d9b 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1974,6 +1974,11 @@ package VX_gpu_pkg; localparam DXA_MEM_ARB_IDX = RTCACHE_MEM_ARB_IDX + `VX_CFG_EXT_RTU_ENABLED; localparam SOCKET_MEM_ARB_REQS = 2 + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_DXA_ENABLED; + // Socket unit MMU ports (SOCKET_UNIT_MMU_ENABLE), in socket arb order. + localparam UMMU_RTCACHE_IDX = 0; + localparam UMMU_DXA_IDX = UMMU_RTCACHE_IDX + `VX_CFG_EXT_RTU_ENABLED; + localparam UMMU_NUM_REQS = `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_DXA_ENABLED; + // Widest tag among the socket arb inputs (DXA adapts to the given budget) `ifdef VX_CFG_EXT_TEX_ENABLE localparam SOCKET_MEM_TAG_W1 = `MAX(L1_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH); @@ -2037,9 +2042,14 @@ package VX_gpu_pkg; // TLB miss/fill fabric id widths. The id starts as the L1 miss-station slot // and each arb level prepends its grant index so fills route back exactly. localparam L1_TLB_ID_WIDTH = `CLOG2(`VX_CFG_L1_TLB_MSHR_SIZE); - // A socket exposes two L1 MMUs (dtlb + itlb), regardless of SOCKET_SIZE, each - // driving one bus into the socket TLB arbiter. - localparam TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + `ARB_SEL_BITS(2, 1); + // A socket exposes two L1 MMUs (dtlb + itlb), regardless of SOCKET_SIZE, plus + // the unit MMU when present, each driving one bus into the socket TLB arbiter. +`ifdef SOCKET_UNIT_MMU_ENABLE + localparam SOCKET_NUM_MMUS = 3; +`else + localparam SOCKET_NUM_MMUS = 2; +`endif + localparam TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + `ARB_SEL_BITS(SOCKET_NUM_MMUS, 1); localparam TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + `ARB_SEL_BITS(NUM_SOCKETS, 1); localparam L2_TLB_SLOT_WIDTH = `CLOG2(`VX_CFG_L2_TLB_MSHR_SIZE); // Device-level walker: cluster L2-TLB miss buses arb into one walker. diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index b5bd01799a..b8acb3ca50 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -276,20 +276,29 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_tlb_flush_if immu_flush_if(); VX_mmu_fault_if dmmu_fault_if(); VX_mmu_fault_if immu_fault_if(); +`ifdef SOCKET_UNIT_MMU_ENABLE + VX_tlb_flush_if ummu_flush_if(); + VX_dcr_flush_if ucache_flush_if(); +`endif VX_mmu_snoop mmu_snoop ( - .clk (clk), - .reset (reset), - .dcr_bus_in_if (dcr_bus_if), - .dcr_bus_out_if (socket_dcr_bus_if), - .vm_active (vm_active), - .dmmu_flush_if (dmmu_flush_if), - .immu_flush_if (immu_flush_if) + .clk (clk), + .reset (reset), + .dcr_bus_in_if (dcr_bus_if), + .dcr_bus_out_if (socket_dcr_bus_if), + .vm_active (vm_active), + .dmmu_flush_if (dmmu_flush_if), + `ifdef SOCKET_UNIT_MMU_ENABLE + .ummu_flush_if (ummu_flush_if), + .ucache_flush_if (ucache_flush_if), + `endif + .immu_flush_if (immu_flush_if) ); - // dtlb + itlb each drive one tlb_bus into the socket arbiter below. + // dtlb + itlb (+ the unit MMU) each drive one tlb_bus into the socket + // arbiter below. VX_tlb_bus_if #(.ID_WIDTH (L1_TLB_ID_WIDTH)) - socket_tlb_bus_if [2] (); + socket_tlb_bus_if [SOCKET_NUM_MMUS] (); wire dmmu_empty, immu_empty; @@ -316,6 +325,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .tlb_bus_if (socket_tlb_bus_if[0]), .flush_if (dmmu_flush_if), .fault_if (dmmu_fault_if), + .lane_idle (dmmu_lane_idle), .empty (dmmu_empty) ); @@ -338,6 +348,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .tlb_bus_if (socket_tlb_bus_if[1]), .flush_if (immu_flush_if), .fault_if (immu_fault_if), + `UNUSED_PIN (lane_idle), .empty (immu_empty) ); @@ -354,8 +365,18 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; // Socket MMU drain status, broadcast to every core so its busy/barrier logic // waits for in-flight translations. Shared MMU: a core waits for the whole - // socket to drain (conservative, always correct). + // socket to drain (conservative, always correct). The unit MMU carries no + // core load or store, so it holds only the socket busy, not the barriers. wire socket_mmu_drained = dmmu_empty && immu_empty; + // Per dMMU lane: none of that lane's requests is in the MMU; each core's + // dcache flush waits only on its own lanes. + wire [`VX_CFG_SOCKET_SIZE * DCACHE_NUM_REQS-1:0] dmmu_lane_idle; +`ifdef SOCKET_UNIT_MMU_ENABLE + wire ummu_empty; + wire unit_mmu_drained = ummu_empty; +`else + wire unit_mmu_drained = 1'b1; +`endif `ifdef PERF_ENABLE // Surfacing these through sysmem_perf needs a dedicated mmu slot; until that @@ -364,9 +385,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `UNUSED_VAR (immu_perf) `endif - // Merge the two MMU tlb buses onto the one cluster-facing bus. + // Merge the MMU tlb buses onto the one cluster-facing bus. VX_tlb_bus_arb #( - .NUM_INPUTS (2), + .NUM_INPUTS (SOCKET_NUM_MMUS), .ID_WIDTH_IN (L1_TLB_ID_WIDTH), .OUT_BUF (3) ) tlb_socket_arb ( @@ -797,6 +818,78 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; ); `endif // VX_CFG_EXT_DXA_ENABLE +`ifdef SOCKET_UNIT_MMU_ENABLE + /////////////////////////////////////////////////////////////////////////// + // Socket unit MMU — translates the socket-resident units' memory ports + // (rtcache misses, DXA gmem reads) between them and the socket memory + // arbiter, so the unit caches are virtually indexed and tagged. Its own + // TLB keeps BVH and tensor streams from evicting the dTLB's entries, and + // its misses ride the third socket tlb_bus. + /////////////////////////////////////////////////////////////////////////// + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) unit_mem_bus_if[UMMU_NUM_REQS](); + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) unit_phys_bus_if[UMMU_NUM_REQS](); + + +`ifdef VX_CFG_EXT_RTU_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_RTCACHE_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); +`endif +`ifdef VX_CFG_EXT_DXA_ENABLE + `ASSIGN_VX_MEM_BUS_IF (unit_mem_bus_if[UMMU_DXA_IDX], dxa_gmem_bus_if[0]); +`endif + + VX_mmu_fault_if ummu_fault_if(); + +`ifdef PERF_ENABLE + mmu_perf_t ummu_perf; + `UNUSED_VAR (ummu_perf) +`endif + + VX_mmu #( + .INSTANCE_ID ("ummu"), + .NUM_REQS (UMMU_NUM_REQS), + .TLB_SIZE (`VX_CFG_UTLB_SIZE), + .EXEC_SIDE (0), + .WRITE_ENABLE (0), + .RSP_OUT_BUF (2), + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) ummu ( + .clk (clk), + .reset (reset), + .vm_active (vm_active), + `ifdef PERF_ENABLE + .mmu_perf (ummu_perf), + `endif + .core_bus_if (unit_mem_bus_if), + .mem_bus_if (unit_phys_bus_if), + .tlb_bus_if (socket_tlb_bus_if[2]), + .flush_if (ummu_flush_if), + .fault_if (ummu_fault_if), + `UNUSED_PIN (lane_idle), + .empty (ummu_empty) + ); + + // Faults are reported by the cluster walker, as for the dMMU/iMMU. + `UNUSED_VAR (ummu_fault_if.valid) + `UNUSED_VAR (ummu_fault_if.va) + `UNUSED_VAR (ummu_fault_if.access) + `UNUSED_VAR (ummu_fault_if.amo) + +`ifndef VX_CFG_EXT_RTU_ENABLE + // no unit cache to invalidate + assign ucache_flush_if.done = 1'b1; + `UNUSED_VAR (ucache_flush_if.req) +`endif +`endif // SOCKET_UNIT_MMU_ENABLE + /////////////////////////////////////////////////////////////////////////// wire mem_arb_empty; @@ -823,11 +916,20 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_EXT_TEX_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[TCACHE_MEM_ARB_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif + `ifdef SOCKET_UNIT_MMU_ENABLE + `ifdef VX_CFG_EXT_RTU_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_RTCACHE_IDX]); + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[DXA_MEM_ARB_IDX], unit_phys_bus_if[UMMU_DXA_IDX]); + `endif + `else `ifdef VX_CFG_EXT_RTU_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif `ifdef VX_CFG_EXT_DXA_ENABLE `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[DXA_MEM_ARB_IDX], dxa_gmem_bus_if[0]); + `endif `endif VX_mem_bus_arb #( @@ -895,7 +997,13 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; assign tcache_flush_done_w = 1'b1; `endif `ifdef VX_CFG_EXT_RTU_ENABLE +`ifdef SOCKET_UNIT_MMU_ENABLE + // The rtcache is virtually tagged: the SATP commit invalidates it too. + assign rtcache_flush_if.req = socket_flush_req || ucache_flush_if.req; + assign ucache_flush_if.done = rtcache_flush_if.done; +`else assign rtcache_flush_if.req = socket_flush_req; +`endif assign rtcache_flush_done_w = rtcache_flush_if.done; `else assign rtcache_flush_done_w = 1'b1; @@ -961,6 +1069,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_VM_ENABLE .mmu_drained (socket_mmu_drained), + .dcache_mmu_drained (& dmmu_lane_idle[core_id * DCACHE_NUM_REQS +: DCACHE_NUM_REQS]), `endif `ifdef EXT_GFX_ANY_ENABLE @@ -989,7 +1098,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; assign per_core_kmu_fire[i] = per_core_kmu_bus_if[i].valid && per_core_kmu_bus_if[i].ready; end `ifdef VX_CFG_VM_ENABLE - wire mmu_busy = ~socket_mmu_drained; + wire mmu_busy = ~(socket_mmu_drained && unit_mmu_drained); `else wire mmu_busy = 1'b0; `endif diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index c68f1eda23..26891662aa 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -46,6 +46,8 @@ import VX_tcu_pkg::*; // Translation happens at the socket (one MMU per L1 cache): the core emits // virtual addresses, and busy/barrier logic waits on the socket MMU draining. input wire mmu_drained, + // This core's own dMMU lanes hold none of its requests. + input wire dcache_mmu_drained, `endif `ifdef VX_CFG_EXT_DXA_ENABLE @@ -163,7 +165,25 @@ import VX_tcu_pkg::*; // Stores are ack-less and lag warp exit, so the dcache flush waits until every // store of this core has reached L1. wire store_drained = (&lsu_sched_empty) && mem_unit_empty; +`ifdef VX_CFG_VM_ENABLE + // Under VM a store may still be parked on a TLB miss in this core's dMMU + // lanes, which a flush token on another lane would overtake: the flush also + // waits for those lanes. The request is level-held and the token's own trip + // through the MMU clears the lanes' idle, so the start is latched until the + // flush request drops. + reg dcache_flush_go_r; + always @(posedge clk) begin + if (reset || ~dcr_flush_if.req) begin + dcache_flush_go_r <= 1'b0; + end else if (store_drained && dcache_mmu_drained) begin + dcache_flush_go_r <= 1'b1; + end + end + assign dcr_flush_dcache_if.req = dcr_flush_if.req + && ((store_drained && dcache_mmu_drained) || dcache_flush_go_r); +`else assign dcr_flush_dcache_if.req = dcr_flush_if.req && store_drained; +`endif // Only the dcache flush walks the shared levels (the icache does not // forward its flush), so the icache flushes alongside it. assign dcr_flush_icache_if.req = dcr_flush_if.req; diff --git a/hw/rtl/dxa/VX_dxa_core.sv b/hw/rtl/dxa/VX_dxa_core.sv index af7d5c3021..e0a1330af7 100644 --- a/hw/rtl/dxa/VX_dxa_core.sv +++ b/hw/rtl/dxa/VX_dxa_core.sv @@ -180,13 +180,20 @@ module VX_dxa_core import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // ================================================================ // Output arbitration // ================================================================ +`ifdef SOCKET_UNIT_MMU_ENABLE + localparam GMEM_OUT_REG = 1; +`else + localparam GMEM_OUT_REG = 0; +`endif VX_mem_bus_arb #( .NUM_INPUTS (`VX_CFG_NUM_DXA_CORES), .NUM_OUTPUTS (GMEM_OUT_PORTS), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (WORKER_GMEM_TAG_WIDTH), .ARBITER ("R"), - .REQ_OUT_BUF ((`VX_CFG_NUM_DXA_CORES > 1) ? 3 : 0) + // Registered whenever its consumer needs a flop to start from: the + // arbiter's merge, or the unit MMU's TLB lookup under VM. + .REQ_OUT_BUF (((`VX_CFG_NUM_DXA_CORES > 1) || GMEM_OUT_REG) ? 3 : 0) ) gmem_arb ( .clk (clk), .reset (reset), diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index 9166421400..9e67270382 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -28,6 +28,12 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, parameter REPLAY_DEPTH = 2, parameter EXEC_SIDE = 0, + // 0: the clients only read, so a parked request keeps no write data or + // byte enables (don't-care on a read). + parameter WRITE_ENABLE = 1, + // Response stage per lane toward the client (TO_OUT_BUF encoding). Keeps + // the injected-response select off the client's response cone. + parameter RSP_OUT_BUF = 0, parameter DATA_SIZE = DCACHE_WORD_SIZE, parameter TAG_WIDTH = DCACHE_TAG_WIDTH_BASE, parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE), @@ -49,10 +55,39 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_tlb_flush_if.slave flush_if, VX_mmu_fault_if.master fault_if, + // Per lane: nothing of the lane's is in the MMU (front stages, fault slot, + // and, where flushes are ordered, the miss station). + output wire [NUM_REQS-1:0] lane_idle, output wire empty ); `UNUSED_PARAM (INSTANCE_ID) + // The client's bus as the MMU drives it: requests pass straight through to + // the lookup (the client registers its own request output), responses + // through an optional MMU-owned stage. + VX_mem_bus_if #( + .DATA_SIZE (DATA_SIZE), + .TAG_WIDTH (TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH), + .ATTR_WIDTH (ATTR_WIDTH) + ) req_if [NUM_REQS] (); + + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_rsp_buf + VX_mem_bus_slice #( + .DATA_SIZE (DATA_SIZE), + .TAG_WIDTH (TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH), + .ATTR_WIDTH (ATTR_WIDTH), + .REQ_OUT_BUF (0), + .RSP_OUT_BUF (RSP_OUT_BUF) + ) rsp_buf ( + .clk (clk), + .reset (reset), + .bus_in_if (core_bus_if[l]), + .bus_out_if (req_if[l]) + ); + end + localparam DATA_WIDTH = DATA_SIZE * 8; localparam LANE_W = `UP(`CLOG2(NUM_REQS)); localparam PAGE_OFFSET_W = `VX_VM_PAGE_LOG2_SIZE - `CLOG2(DATA_SIZE); @@ -65,7 +100,9 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( localparam F_ADDR_LO = F_DATA_LO + DATA_WIDTH; localparam F_RW_LO = F_ADDR_LO + ADDR_WIDTH; localparam FIELDS_W = F_RW_LO + 1; - localparam PAYLOAD_W = LANE_W + FIELDS_W; + // Parked fields: a read-only client's drop {data, byteen}. + localparam PFIELDS_W = (WRITE_ENABLE != 0) ? FIELDS_W : (FIELDS_W - DATA_WIDTH - DATA_SIZE); + localparam PAYLOAD_W = LANE_W + PFIELDS_W; function automatic tlb_access_e access_of (input logic rw); @@ -94,17 +131,17 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [NUM_REQS-1:0] mshr_match; for (genvar l = 0; l < NUM_REQS; ++l) begin : g_decode - assign req_valid[l] = core_bus_if[l].req_valid; - assign req_rw[l] = core_bus_if[l].req_data.rw; - assign req_addr[l] = core_bus_if[l].req_data.addr; - assign req_attr[l] = core_bus_if[l].req_data.attr[ATTR_WIDTH-1:0]; + assign req_valid[l] = req_if[l].req_valid; + assign req_rw[l] = req_if[l].req_data.rw; + assign req_addr[l] = req_if[l].req_data.addr; + assign req_attr[l] = req_if[l].req_data.attr[ATTR_WIDTH-1:0]; assign req_fields[l] = { - core_bus_if[l].req_data.rw, - core_bus_if[l].req_data.addr, - core_bus_if[l].req_data.data, - core_bus_if[l].req_data.byteen, - core_bus_if[l].req_data.attr[ATTR_WIDTH-1:0], - core_bus_if[l].req_data.tag[TAG_WIDTH-1:0] + req_if[l].req_data.rw, + req_if[l].req_data.addr, + req_if[l].req_data.data, + req_if[l].req_data.byteen, + req_if[l].req_data.attr[ATTR_WIDTH-1:0], + req_if[l].req_data.tag[TAG_WIDTH-1:0] }; assign req_bypass[l] = ~vm_active || req_attr[l][MEM_ATTR_FLUSH_OFFS] @@ -195,10 +232,15 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [NUM_REQS-1:0] cat_park; wire [NUM_REQS-1:0] cat_hit; wire [NUM_REQS-1:0] cat_pfault; + wire [NUM_REQS-1:0] pf_accept; // permission fault taken this cycle + wire [NUM_REQS-1:0] pf_rsp_busy; // a fault response is owed on the lane + wire [NUM_REQS-1:0] lane_parked; // the lane has requests in the miss station for (genvar l = 0; l < NUM_REQS; ++l) begin : g_cat + // A flush token holds behind its lane's parked requests (see g_parked). + wire flush_hold = req_attr[l][MEM_ATTR_FLUSH_OFFS] && lane_parked[l]; assign cam_acc[l] = req_acc[l]; - assign cat_bypass[l] = req_valid[l] && req_bypass[l]; + assign cat_bypass[l] = req_valid[l] && req_bypass[l] && !flush_hold; assign cat_park[l] = req_valid[l] && !req_bypass[l] && (mshr_match[l] || !cam_hit[l]); assign cat_hit[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && cam_perm[l]; assign cat_pfault[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && !cam_perm[l]; @@ -235,13 +277,30 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( assign park_vpn = park_out[PARK_W-1 -: TLB_VPN_WIDTH]; assign park_access = tlb_access_e'(park_out[FIELDS_W+1 +: $bits(tlb_access_e)]); assign park_amo = park_out[FIELDS_W]; - assign park_payload = {park_lane, park_out[FIELDS_W-1:0]}; // --------------------------------------------------------------------- // Replay decode + translation splice // --------------------------------------------------------------------- - wire [LANE_W-1:0] replay_lane = replay_payload[FIELDS_W +: LANE_W]; - wire [FIELDS_W-1:0] replay_fields = replay_payload[FIELDS_W-1:0]; + wire [LANE_W-1:0] replay_lane = replay_payload[PFIELDS_W +: LANE_W]; + wire [PFIELDS_W-1:0] replay_pfields = replay_payload[PFIELDS_W-1:0]; + wire [PFIELDS_W-1:0] park_pfields; + wire [FIELDS_W-1:0] replay_fields; + + if (WRITE_ENABLE != 0) begin : g_park_wr + assign park_pfields = park_out[FIELDS_W-1:0]; + assign replay_fields = replay_pfields; + end else begin : g_park_ro + // A read's data and byte enables are don't-care below the L1; the + // replay presents it as a full-line read. + assign park_pfields = {park_out[FIELDS_W-1:F_ADDR_LO], park_out[F_BYTEEN_LO-1:0]}; + assign replay_fields = {replay_pfields[PFIELDS_W-1:F_BYTEEN_LO], DATA_WIDTH'(0), {DATA_SIZE{1'b1}}, + replay_pfields[F_BYTEEN_LO-1:0]}; + `UNUSED_VAR (park_out[F_ADDR_LO-1:F_BYTEEN_LO]) + `RUNTIME_ASSERT (~(park_valid && park_ready) || ~park_out[F_RW_LO], + ("%t: *** %s read-only MMU parked a write, addr=0x%0h", $time, INSTANCE_ID, + park_out[F_ADDR_LO +: ADDR_WIDTH])) + end + assign park_payload = {park_lane, park_pfields}; wire replay_rw = replay_fields[F_RW_LO]; wire [ADDR_WIDTH-1:0] replay_raddr = replay_fields[F_ADDR_LO +: ADDR_WIDTH]; wire [ATTR_WIDTH-1:0] replay_rattr = replay_fields[F_ATTR_LO +: ATTR_WIDTH]; @@ -334,38 +393,116 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( assign cam_access_hit[l] = cat_hit[l] && !replay_to_lane[l] && pipe_ready[l]; + // A permission fault on a hit never reaches memory, but a data-side + // load or atomic still owes its requester a response, as a kill does: + // it waits in a per-lane slot and is answered with zeroed data. + wire pf_rsp_valid; + wire [TAG_WIDTH-1:0] pf_rsp_tag; + if (EXEC_SIDE == 0) begin : g_pf_rsp + wire pf_needs_rsp = !req_rw[l] || req_amo[l]; + reg pf_rsp_valid_r; + reg [TAG_WIDTH-1:0] pf_rsp_tag_r; + wire pf_rsp_fire = pf_rsp_valid_r && !kill_to_lane[l] && req_if[l].rsp_ready; + // An empty slot loads unconditionally so the tag's enable stays off + // the CAM lookup cone; only the valid bit depends on the fault. + always @(posedge clk) begin + if (reset) begin + pf_rsp_valid_r <= 1'b0; + end else if (pf_rsp_valid_r) begin + pf_rsp_valid_r <= ~pf_rsp_fire; + end else begin + pf_rsp_valid_r <= cat_pfault[l] && pf_needs_rsp; + end + end + always @(posedge clk) begin + if (~pf_rsp_valid_r) begin + pf_rsp_tag_r <= req_fields[l][F_TAG_LO +: TAG_WIDTH]; + end + end + assign pf_accept[l] = cat_pfault[l] && !(pf_needs_rsp && pf_rsp_valid_r); + assign pf_rsp_valid = pf_rsp_valid_r; + assign pf_rsp_tag = pf_rsp_tag_r; + end else begin : g_pf_no_rsp + // A fetch is answered by the fault report alone. + assign pf_accept[l] = cat_pfault[l]; + assign pf_rsp_valid = 1'b0; + assign pf_rsp_tag = '0; + end + assign pf_rsp_busy[l] = pf_rsp_valid; + // Request accept back to the client. - assign core_bus_if[l].req_ready = + assign req_if[l].req_ready = cat_bypass[l] ? (!replay_to_lane[l] && pipe_ready[l]) : cat_hit[l] ? (!replay_to_lane[l] && pipe_ready[l]) : cat_park[l] ? (park_sel[l] && park_ready) - : cat_pfault[l] ? 1'b1 + : cat_pfault[l] ? pf_accept[l] : 1'b0; - // Response path wires straight back, except a kill teardown injects a - // zeroed response ahead of the cache reply for its lane. - assign core_rsp_ready[l] = core_bus_if[l].rsp_ready; - assign core_bus_if[l].rsp_valid = kill_to_lane[l] ? 1'b1 : mem_bus_if[l].rsp_valid; - assign core_bus_if[l].rsp_data.data = kill_to_lane[l] ? '0 : mem_bus_if[l].rsp_data.data; - assign core_bus_if[l].rsp_data.tag = kill_to_lane[l] ? kill_tag : mem_bus_if[l].rsp_data.tag; - assign mem_bus_if[l].rsp_ready = core_bus_if[l].rsp_ready && ~kill_to_lane[l]; + // Response path wires straight back, except a kill teardown or a + // permission fault injects a zeroed response ahead of the cache reply. + // The injected tag is chosen ahead of the one select on the reply. + wire inj_rsp = kill_to_lane[l] || pf_rsp_valid; + wire [TAG_WIDTH-1:0] inj_tag = kill_to_lane[l] ? kill_tag : pf_rsp_tag; + assign core_rsp_ready[l] = req_if[l].rsp_ready; + assign req_if[l].rsp_valid = inj_rsp ? 1'b1 : mem_bus_if[l].rsp_valid; + assign req_if[l].rsp_data.data = inj_rsp ? '0 : mem_bus_if[l].rsp_data.data; + assign req_if[l].rsp_data.tag = inj_rsp ? inj_tag : mem_bus_if[l].rsp_data.tag; + assign mem_bus_if[l].rsp_ready = req_if[l].rsp_ready && ~inj_rsp; end assign replay_ready = replay_valid && (replay_perm ? pipe_ready[replay_lane] : kill_ready); + // --------------------------------------------------------------------- + // Per-lane parked count. A flush token bypasses translation, so it would + // overtake its lane's parked requests; it holds until they have left the + // miss station instead, reaching the cache behind them as a lane's + // requests always do (a FLUSH_FORWARD cache forwards it behind that + // port's writes). Only a data-side MMU with writing clients needs it: a + // read overtaken by a flush token is still answered correctly. + // --------------------------------------------------------------------- + localparam PCNT_W = `CLOG2(MSHR_SIZE * REPLAY_DEPTH + 2); + wire park_take = park_valid && park_ready; + wire [LANE_W-1:0] park_take_lane = park_lane; + wire drain_take = replay_ready || (kill_valid && kill_ready); + + if ((WRITE_ENABLE != 0) && (EXEC_SIDE == 0)) begin : g_flush_order + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_parked + reg [PCNT_W-1:0] parked_cnt_r; + wire inc = park_take && (park_take_lane == LANE_W'(l)); + wire dec = drain_take && (replay_lane == LANE_W'(l)); + always @(posedge clk) begin + if (reset) begin + parked_cnt_r <= '0; + end else if (inc && !dec) begin + parked_cnt_r <= parked_cnt_r + PCNT_W'(1); + end else if (dec && !inc) begin + parked_cnt_r <= parked_cnt_r - PCNT_W'(1); + end + end + `RUNTIME_ASSERT (reset || ~(dec && !inc) || (parked_cnt_r != '0), + ("%t: *** %s lane %0d drained more than it parked", $time, INSTANCE_ID, l)) + assign lane_parked[l] = (parked_cnt_r != '0); + end + end else begin : g_no_flush_order + assign lane_parked = '0; + `UNUSED_VAR (park_take) + `UNUSED_VAR (park_take_lane) + `UNUSED_VAR (drain_take) + end + // --------------------------------------------------------------------- // Fault sideband to the fault-latch surface. Structural faults come from // the walker via the MSHR; permission faults are detected here on hits // and replays. // --------------------------------------------------------------------- - wire perm_fault_any = (| cat_pfault) || replay_drop; + wire perm_fault_any = (| pf_accept) || replay_drop; reg [TLB_VPN_WIDTH-1:0] perm_fault_vpn; reg perm_fault_amo; always @(*) begin perm_fault_vpn = replay_vpn; perm_fault_amo = replay_amo; for (int l = NUM_REQS-1; l >= 0; --l) begin - if (cat_pfault[l]) begin + if (pf_accept[l]) begin perm_fault_vpn = req_vpn[l]; perm_fault_amo = req_amo[l]; end @@ -398,7 +535,9 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [NUM_REQS-1:0] pipe_busy; for (genvar l = 0; l < NUM_REQS; ++l) begin : g_busy assign pipe_busy[l] = core_bus_if[l].req_valid // incoming (client held) - || mem_bus_if[l].req_valid; // output stage + || mem_bus_if[l].req_valid // output stage + || pf_rsp_busy[l]; // owed fault response + assign lane_idle[l] = ~pipe_busy[l] && ~lane_parked[l]; end assign empty = ~(| pipe_busy) && tlb_empty; diff --git a/hw/rtl/vm/VX_mmu_snoop.sv b/hw/rtl/vm/VX_mmu_snoop.sv index 0d442636dc..fb9179a40a 100644 --- a/hw/rtl/vm/VX_mmu_snoop.sv +++ b/hw/rtl/vm/VX_mmu_snoop.sv @@ -17,7 +17,9 @@ // device VX_mmu_dcr one level up) — forwarding DCR downstream to the cores — // while snooping the SATP writes to derive the translation-enable bit and to // self-time the L1 TLB flush. This keeps satp/flush off the socket boundary: -// only the TLB miss fabric crosses it. +// only the TLB miss fabric crosses it. With a unit MMU, the same write also +// flushes its TLB and invalidates the unit caches, which are virtually indexed +// and tagged (they sit in front of the unit MMU). module VX_mmu_snoop import VX_gpu_pkg::*; ( input wire clk, input wire reset, @@ -30,6 +32,11 @@ module VX_mmu_snoop import VX_gpu_pkg::*; ( // L1 dTLB / iTLB flush. VX_tlb_flush_if.master dmmu_flush_if, +`ifdef SOCKET_UNIT_MMU_ENABLE + // Unit MMU TLB flush and unit cache invalidation. + VX_tlb_flush_if.master ummu_flush_if, + VX_dcr_flush_if.master ucache_flush_if, +`endif VX_tlb_flush_if.master immu_flush_if ); wire dcr_wr = dcr_bus_in_if.req_valid && dcr_bus_in_if.req_data.rw; @@ -55,22 +62,44 @@ module VX_mmu_snoop import VX_gpu_pkg::*; ( assign vm_active = vm_active_r; // Self-timed flush: the SATP_HI write (the address-space commit) requests a - // flush of both L1 TLBs, held until they drain and flash-clear. satp changes + // flush of every L1 TLB, held until they drain and flash-clear. satp changes // only on an address-space change, issued between kernels once the TLBs are // already drained, so completion needs no handshake back to the device. - wire flush_done = dmmu_flush_if.done && immu_flush_if.done; +`ifdef SOCKET_UNIT_MMU_ENABLE + wire tlb_flush_done = dmmu_flush_if.done && immu_flush_if.done && ummu_flush_if.done; +`else + wire tlb_flush_done = dmmu_flush_if.done && immu_flush_if.done; +`endif reg tlb_flush_r; always @(posedge clk) begin if (reset) begin tlb_flush_r <= 1'b0; end else if (is_satp_hi) begin tlb_flush_r <= 1'b1; - end else if (flush_done) begin + end else if (tlb_flush_done) begin tlb_flush_r <= 1'b0; end end assign dmmu_flush_if.req = tlb_flush_r; assign immu_flush_if.req = tlb_flush_r; +`ifdef SOCKET_UNIT_MMU_ENABLE + assign ummu_flush_if.req = tlb_flush_r; + + // The unit-cache invalidation is its own leg: a cache flush waits for its + // outstanding misses, which may need a unit-TLB park, so it must not hold + // the TLB legs (whose flush refuses parks) for its walk. + reg ucache_flush_r; + always @(posedge clk) begin + if (reset) begin + ucache_flush_r <= 1'b0; + end else if (is_satp_hi) begin + ucache_flush_r <= 1'b1; + end else if (ucache_flush_if.done) begin + ucache_flush_r <= 1'b0; + end + end + assign ucache_flush_if.req = ucache_flush_r; +`endif // Forward the DCR downstream verbatim: the socket has no MMU readback, so // requests pass through and responses return unchanged. diff --git a/hw/syn/xilinx/dut/catalog.mk b/hw/syn/xilinx/dut/catalog.mk index 12167b355b..172de1c912 100644 --- a/hw/syn/xilinx/dut/catalog.mk +++ b/hw/syn/xilinx/dut/catalog.mk @@ -10,7 +10,7 @@ # Add a DUT: append its name to DUTS and define the five variables. No new # directory, no dispatcher edit. -DUTS := cache core cp dxa fpu gfx issue lmem mem_unit mufu om raster rtu \ +DUTS := cache core cp dxa fpu gfx gfxvm issue lmem mem_unit mufu om raster rtu \ scope tcu tensor tex top vm vortex # --------------------------------------------------------------------------- @@ -82,6 +82,14 @@ gfx_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/cache -I$ gfx_PKG = $(RTL_DIR)/fpu/VX_fpu_pkg.sv $(RTL_DIR)/dxa/VX_dxa_pkg.sv $(RTL_DIR)/tex/VX_tex_pkg.sv \ $(RTL_DIR)/rtu/VX_rtu_pkg.sv $(RTL_DIR)/raster/VX_raster_pkg.sv $(RTL_DIR)/om/VX_om_pkg.sv +# The graphics slice under VM: the unit MMUs between the graphics caches / DXA +# and memory, which the gfx DUT (VM off) and the vm DUT (no units) never build. +gfxvm_PRJ := VX_gfx_top +gfxvm_IP := 1 +gfxvm_CFG := $(gfx_CFG) -DVX_CFG_VM_ENABLE +gfxvm_INC = $(gfx_INC) +gfxvm_PKG = $(gfx_PKG) $(RTL_DIR)/vm/VX_tlb_pkg.sv + # NOTE: the original issue/Makefile listed $(FPU_INCLUDE) twice; a duplicate -I # is a no-op, so it is listed once here. issue_PRJ := VX_issue_top diff --git a/hw/syn/yosys/dut/catalog.mk b/hw/syn/yosys/dut/catalog.mk index bef32bddfa..2392cbf40d 100644 --- a/hw/syn/yosys/dut/catalog.mk +++ b/hw/syn/yosys/dut/catalog.mk @@ -9,7 +9,7 @@ # synthesizes, so both gates measure the same modules and a divergence between # them is meaningful rather than an artefact of two DUT lists. -DUTS := cache core lmem tcu gfx tex raster om rtu dxa mufu vm tensor top vortex +DUTS := cache core lmem tcu gfx gfxvm tex raster om rtu dxa mufu vm tensor top vortex UNITTEST_DIR ?= $(VORTEX_HOME)/hw/unittest AFU_RTL_DIR ?= $(VORTEX_HOME)/hw/rtl/afu @@ -46,6 +46,11 @@ gfx_CFG := -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_EXT_RASTER_E -DVX_CFG_SOCKET_SIZE=2 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 \ -DVX_CFG_L2_ENABLE -DVX_CFG_L2_SIZE=262144 +# The graphics slice under VM, as the Xilinx catalog's gfxvm. +gfxvm_TOP := VX_gfx_top +gfxvm_INC := $(gfx_INC) +gfxvm_CFG := $(gfx_CFG) -DVX_CFG_VM_ENABLE + tex_TOP := VX_tex_core_top tex_INC := -I$(UNITTEST_DIR)/tex_core tex_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TEX_ENABLE diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index 09159d41da..86890d34e2 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -21,6 +21,11 @@ // socket slice: // VX_tex_bus_arb -> VX_tex_core xN -> tcache (VX_cache_cluster) // VX_rtu_bus_arb -> VX_rtu_core xN -> rtcache (VX_cache_cluster) +// VX_mmu : under VM, the socket unit MMU between the rtcache / +// DXA gmem ports and the arbiter; its tlb_bus, satp +// mode and TLB flush are top ports. The 3-input socket +// TLB arbiter and the snoop's unit-MMU and unit-cache +// flush legs are elaborated only by the full-GPU build // VX_mem_bus_arb : the socket memory arbiter with its full production // input set {icache, dcache, tcache, rtcache, DXA gmem}; // the icache/dcache/DXA slots are driven from top ports @@ -47,6 +52,9 @@ // top-level port and every DUT output is observed at a top-level port. module VX_gfx_top import VX_gpu_pkg::* +`ifdef SOCKET_UNIT_MMU_ENABLE + , VX_tlb_pkg::* +`endif `ifdef VX_CFG_EXT_TEX_ENABLE , VX_tex_pkg::* `endif @@ -60,6 +68,11 @@ module VX_gfx_top import VX_gpu_pkg::* parameter SFU_LANES = `VX_CFG_NUM_SFU_LANES, parameter L2_MEM_ADDR_W = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(L2_SECTOR_SIZE), parameter EXT_ADDR_W = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(`VX_CFG_L1_LINE_SIZE) +`ifdef SOCKET_UNIT_MMU_ENABLE + , parameter UTLB_ID_W = `UP(L1_TLB_ID_WIDTH), + parameter UTLB_REQ_W = UTLB_ID_W + $bits(tlb_access_e) + 1 + TLB_VPN_WIDTH, + parameter UTLB_RSP_W = UTLB_ID_W + 1 + TLB_LEVEL_WIDTH + TLB_PPN_WIDTH + TLB_FLAGS_WIDTH +`endif ) ( input wire clk, input wire reset, @@ -177,6 +190,22 @@ module VX_gfx_top import VX_gpu_pkg::* input wire flush_req, output wire flush_done, +`ifdef SOCKET_UNIT_MMU_ENABLE + // ----------------------------------------------------------------------- + // Socket unit MMU: satp mode, TLB flush, drain, and its tlb_bus + // ----------------------------------------------------------------------- + input wire vm_active, + input wire ummu_flush_req, + output wire ummu_flush_done, + output wire ummu_empty, + output wire utlb_req_valid, + output wire [UTLB_REQ_W-1:0] utlb_req_data, + input wire utlb_req_ready, + input wire utlb_rsp_valid, + input wire [UTLB_RSP_W-1:0] utlb_rsp_data, + output wire utlb_rsp_ready, +`endif + // ----------------------------------------------------------------------- // L2 memory side (master) // ----------------------------------------------------------------------- @@ -491,6 +520,91 @@ module VX_gfx_top import VX_gpu_pkg::* assign flush_done_w[1] = 1'b1; `endif // VX_CFG_EXT_RTU_ENABLE +`ifdef SOCKET_UNIT_MMU_ENABLE + /////////////////////////////////////////////////////////////////////////// + // Socket unit MMU, wired as VX_socket wires it + /////////////////////////////////////////////////////////////////////////// + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) unit_mem_bus_if[UMMU_NUM_REQS](); + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) unit_phys_bus_if[UMMU_NUM_REQS](); + +`ifdef VX_CFG_EXT_RTU_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_RTCACHE_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); +`endif +`ifdef VX_CFG_EXT_DXA_ENABLE + // DXA gmem slot (read-only tile fetches from the ports) + assign unit_mem_bus_if[UMMU_DXA_IDX].req_valid = dxa_mem_req_valid; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.rw = 1'b0; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.byteen = '1; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.addr = dxa_mem_req_addr; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.attr = '0; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.data = '0; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.tag = dxa_mem_req_tag; + assign dxa_mem_req_ready = unit_mem_bus_if[UMMU_DXA_IDX].req_ready; + assign dxa_mem_rsp_valid = unit_mem_bus_if[UMMU_DXA_IDX].rsp_valid; + assign dxa_mem_rsp_data = unit_mem_bus_if[UMMU_DXA_IDX].rsp_data.data; + assign dxa_mem_rsp_tag = unit_mem_bus_if[UMMU_DXA_IDX].rsp_data.tag; + assign unit_mem_bus_if[UMMU_DXA_IDX].rsp_ready = dxa_mem_rsp_ready; +`endif + + VX_tlb_bus_if #( + .ID_WIDTH (L1_TLB_ID_WIDTH) + ) utlb_bus_if(); + assign utlb_req_valid = utlb_bus_if.req_valid; + assign utlb_req_data = utlb_bus_if.req_data; + assign utlb_bus_if.req_ready = utlb_req_ready; + assign utlb_bus_if.rsp_valid = utlb_rsp_valid; + assign utlb_bus_if.rsp_data = utlb_rsp_data; + assign utlb_rsp_ready = utlb_bus_if.rsp_ready; + + VX_tlb_flush_if ummu_flush_if(); + assign ummu_flush_if.req = ummu_flush_req; + assign ummu_flush_done = ummu_flush_if.done; + +`ifdef PERF_ENABLE + mmu_perf_t ummu_perf; + `UNUSED_VAR (ummu_perf) +`endif + + VX_mmu_fault_if ummu_fault_if(); + `UNUSED_VAR (ummu_fault_if.valid) + `UNUSED_VAR (ummu_fault_if.va) + `UNUSED_VAR (ummu_fault_if.access) + `UNUSED_VAR (ummu_fault_if.amo) + + VX_mmu #( + .INSTANCE_ID ("ummu"), + .NUM_REQS (UMMU_NUM_REQS), + .TLB_SIZE (`VX_CFG_UTLB_SIZE), + .EXEC_SIDE (0), + .WRITE_ENABLE (0), + .RSP_OUT_BUF (2), + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) ummu ( + .clk (clk), + .reset (reset), + .vm_active (vm_active), + `ifdef PERF_ENABLE + .mmu_perf (ummu_perf), + `endif + .core_bus_if (unit_mem_bus_if), + .mem_bus_if (unit_phys_bus_if), + .tlb_bus_if (utlb_bus_if), + .flush_if (ummu_flush_if), + .fault_if (ummu_fault_if), + `UNUSED_PIN (lane_idle), + .empty (ummu_empty) + ); +`endif + /////////////////////////////////////////////////////////////////////////// // Socket memory arbiter: {icache, dcache, tcache, rtcache, DXA} -> trunk /////////////////////////////////////////////////////////////////////////// @@ -543,10 +657,20 @@ module VX_gfx_top import VX_gpu_pkg::* `ifdef VX_CFG_EXT_TEX_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[TCACHE_MEM_ARB_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif + `ifdef SOCKET_UNIT_MMU_ENABLE + `ifdef VX_CFG_EXT_RTU_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_RTCACHE_IDX]); + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[DXA_MEM_ARB_IDX], unit_phys_bus_if[UMMU_DXA_IDX]); + `endif + `else `ifdef VX_CFG_EXT_RTU_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif + `endif `ifdef VX_CFG_EXT_DXA_ENABLE + `ifndef SOCKET_UNIT_MMU_ENABLE // DXA gmem slot (read-only tile fetches from the ports) assign socket_arb_in_if[DXA_MEM_ARB_IDX].req_valid = dxa_mem_req_valid; assign socket_arb_in_if[DXA_MEM_ARB_IDX].req_data.rw = 1'b0; @@ -560,6 +684,7 @@ module VX_gfx_top import VX_gpu_pkg::* assign dxa_mem_rsp_data = socket_arb_in_if[DXA_MEM_ARB_IDX].rsp_data.data; assign dxa_mem_rsp_tag = socket_arb_in_if[DXA_MEM_ARB_IDX].rsp_data.tag; assign socket_arb_in_if[DXA_MEM_ARB_IDX].rsp_ready = dxa_mem_rsp_ready; + `endif `else `UNUSED_VAR (dxa_mem_req_valid) `UNUSED_VAR (dxa_mem_req_addr) @@ -714,6 +839,12 @@ module VX_gfx_top import VX_gpu_pkg::* end `endif // EXT_GFX_CLUSTER_ENABLE + // Under VM the walker's PTE fetches take an L2 client slot; the walker is + // covered by the vm DUT, so the slot idles here. + if (PTW_ON_L2 != 0) begin : g_ptw_l2_idle + `INIT_VX_MEM_BUS_IF (l2_core_if[L2_PTW_IDX]) + end + assign flush_done = & flush_done_w; /////////////////////////////////////////////////////////////////////////// diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 056711d448..c578de445c 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -255,7 +255,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // --- socket miss ports (dTLB, iTLB) into the socket arbiter --- VX_tlb_bus_if #(.ID_WIDTH (L1_TLB_ID_WIDTH)) - socket_tlb_bus_if [2] (); + socket_tlb_bus_if [SOCKET_NUM_MMUS] (); VX_mmu #( .INSTANCE_ID ("dmmu"), @@ -273,6 +273,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .tlb_bus_if (socket_tlb_bus_if[0]), .flush_if (dmmu_flush_if), .fault_if (dmmu_fault_if), + `UNUSED_PIN (lane_idle), .empty (dmmu_empty[s]) ); @@ -383,6 +384,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .tlb_bus_if (socket_tlb_bus_if[1]), .flush_if (immu_flush_if), .fault_if (immu_fault_if), + `UNUSED_PIN (lane_idle), .empty (immu_empty[s]) ); @@ -423,8 +425,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `UNUSED_PIN (req_empty) ); end else begin : g_no_immu - // Keep the socket arbiter 2-wide (so TLB_SOCKET_ID_WIDTH matches the - // package) with the iTLB port idle. + // Keep the socket arbiter at the package's width (so + // TLB_SOCKET_ID_WIDTH matches) with the iTLB port idle. assign socket_tlb_bus_if[1].req_valid = 1'b0; assign socket_tlb_bus_if[1].req_data = '0; assign socket_tlb_bus_if[1].rsp_ready = 1'b0; @@ -453,9 +455,20 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `UNUSED_VAR (ic_mem_rsp_tag[s]) end - // --- socket TLB arbiter: {dTLB, iTLB} -> cluster fabric --- + // A build with socket units adds the unit MMU's port (SOCKET_NUM_MMUS); + // this sandbox has no units, so that port idles. + for (genvar t = 2; t < SOCKET_NUM_MMUS; ++t) begin : g_unit_tlb_idle + assign socket_tlb_bus_if[t].req_valid = 1'b0; + assign socket_tlb_bus_if[t].req_data = '0; + assign socket_tlb_bus_if[t].rsp_ready = 1'b0; + `UNUSED_VAR (socket_tlb_bus_if[t].req_ready) + `UNUSED_VAR (socket_tlb_bus_if[t].rsp_valid) + `UNUSED_VAR (socket_tlb_bus_if[t].rsp_data) + end + + // --- socket TLB arbiter: {dTLB, iTLB[, unit]} -> cluster fabric --- VX_tlb_bus_arb #( - .NUM_INPUTS (2), + .NUM_INPUTS (SOCKET_NUM_MMUS), .ID_WIDTH_IN (L1_TLB_ID_WIDTH), .OUT_BUF (3) ) tlb_socket_arb ( diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 4919c6b6aa..f08072a6b1 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -56,7 +56,9 @@ class Cluster::Impl { for (uint32_t i = 0; i < sockets_per_cluster; ++i) { uint32_t socket_id = cluster_id * sockets_per_cluster + i; snprintf(sname, 100, "%s-socket%d", name.c_str(), i); - sockets_.at(i) = Socket::Create(sname, socket_id, simobject_); + // A socket's unit MMU is a cluster-TLB client after every core's two. + uint32_t unit_tlb_client = sockets_per_cluster * VX_CFG_SOCKET_SIZE * 2 + i; + sockets_.at(i) = Socket::Create(sname, socket_id, simobject_, unit_tlb_client); } // Global-barrier event links: each core's arrive end fans into the @@ -149,9 +151,15 @@ class Cluster::Impl { #endif // cluster-resident gfx caches #ifdef VX_CFG_VM_ENABLE - // Shared cluster L2 TLB. Every core's two L1 TLBs are clients; its - // misses export on the walker link the processor binds at the device. - uint32_t num_tlb_clients = sockets_per_cluster * VX_CFG_SOCKET_SIZE * 2; + // Shared cluster L2 TLB. Every core's two L1 TLBs are clients, then each + // socket's unit MMU; its misses export on the walker link the processor + // binds at the device. + uint32_t num_core_tlb_clients = sockets_per_cluster * VX_CFG_SOCKET_SIZE * 2; + #ifdef SIMX_SOCKET_UNIT_MMU + uint32_t num_tlb_clients = num_core_tlb_clients + sockets_per_cluster; + #else + uint32_t num_tlb_clients = num_core_tlb_clients; + #endif snprintf(sname, 100, "%s-l2tlb", name.c_str()); l2tlb_ = L2Tlb::Create(sname, num_tlb_clients); @@ -165,6 +173,13 @@ class Cluster::Impl { } } } + #ifdef SIMX_SOCKET_UNIT_MMU + for (uint32_t s = 0; s < sockets_per_cluster; ++s) { + auto& socket = sockets_.at(s); + socket->tlb_miss_out().bind(&l2tlb_->ReqIn.at(socket->unit_tlb_client())); + l2tlb_->RspOut.at(socket->unit_tlb_client()).bind(&socket->tlb_fill_in()); + } + #endif #endif @@ -442,6 +457,9 @@ class Cluster::Impl { for (uint32_t c = 0; c < cores_per_socket_; ++c) { socket->core(c)->set_satp(value); } + #ifdef SIMX_SOCKET_UNIT_MMU + socket->set_satp(value); + #endif } } diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 941eeddea2..c7a7c32810 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -89,6 +89,15 @@ inline constexpr uint32_t NUM_SOCKETS = __UP(VX_CFG_NUM_CORES / VX_CFG_SOCKE static_assert((VX_CFG_NUM_CORES % VX_CFG_SOCKET_SIZE) == 0, "NUM_CORES must be a multiple of SOCKET_SIZE"); +// Under VM, a socket's unit MMU translates the memory ports of its +// socket-resident units (rtcache misses, DXA gmem reads), in socket arb order. +#if defined(VX_CFG_VM_ENABLE) && (defined(VX_CFG_EXT_RTU_ENABLE) || defined(VX_CFG_EXT_DXA_ENABLE)) +#define SIMX_SOCKET_UNIT_MMU +#endif +inline constexpr uint32_t UMMU_RTCACHE_IDX = 0; +inline constexpr uint32_t UMMU_DXA_IDX = UMMU_RTCACHE_IDX + VX_CFG_EXT_RTU_ENABLED; +inline constexpr uint32_t UMMU_NUM_REQS = VX_CFG_EXT_RTU_ENABLED + VX_CFG_EXT_DXA_ENABLED; + // Demand clients of the cluster L2 (sockets and gfx caches), excluding the // walker's PTE-fetch slot. inline constexpr uint32_t VX_CFG_L2_DEMAND_REQS = NUM_SOCKETS * VX_CFG_L1_MEM_PORTS; diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 61a697eb0a..bc908e448e 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -935,7 +935,16 @@ DxaCore::DxaCore(const SimContext& ctx, const char* name, Socket* socket) snprintf(sname, 100, "%s-gmem-arb", name); gmem_arb_ = MemArbiter::Create(sname, ArbiterType::RoundRobin, VX_CFG_NUM_DXA_CORES, kDxaMemPorts); for (uint32_t i = 0; i < kDxaMemPorts; ++i) { + #ifdef SIMX_SOCKET_UNIT_MMU + // The unit MMU's lookup starts from a registered request: the DXA owns it. + snprintf(sname, 100, "%s-gmem-out%d", name, i); + auto out_reg = MemReqSlice::Create(sname, 1); + gmem_arb_->ReqOut.at(i).bind(&out_reg->In); + out_reg->Out.bind(&gmem_req_out.at(i)); + gmem_out_regs_.push_back(out_reg); + #else gmem_arb_->ReqOut.at(i).bind(&gmem_req_out.at(i)); + #endif gmem_rsp_in.at(i).bind(&gmem_arb_->RspIn.at(i)); } diff --git a/sim/simx/dxa/dxa_core.h b/sim/simx/dxa/dxa_core.h index 7f5c1219ea..cb49c85f29 100644 --- a/sim/simx/dxa/dxa_core.h +++ b/sim/simx/dxa/dxa_core.h @@ -58,6 +58,7 @@ class DxaCore : public SimObject { std::vector> gmem_req_out; std::vector> gmem_rsp_in; MemArbiter::Ptr gmem_arb_; + std::vector gmem_out_regs_; // LMEM row width the RTL drains per cycle: one DXA write beat covers the // full banked row (DXA_LMEM_WORD_SIZE = LMEM_NUM_BANKS * XLEN/8). The diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 7e09ea14ed..91b8d0b0ee 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -794,6 +794,14 @@ class CacheBank : public SimObject { // a writeback request for every dirty line via mem_req_out (write-back only; // write-through caches have nothing to evict). flush_done() reports when the // walk finishes AND the cache is otherwise idle. + // Drop every line, as an address-space change does to a read-only + // virtually tagged cache. Called between launches, with no line in flight. + void invalidate() { + for (auto &set : sets_) { + set.reset(); + } + } + void flush_begin(bool keep_lines) { keep_lines_ = keep_lines; if (!config_.write_back || keep_lines) { @@ -3283,6 +3291,15 @@ class Cache::Impl { return perf_stats; } + void invalidate() { + if (config_.bypass) { + return; + } + for (auto &bank : banks_) { + bank->invalidate(); + } + } + void flush_begin(bool l1_only) { if (config_.bypass) return; // The shared last-level cache stays the point of coherence across an @@ -3401,6 +3418,10 @@ bool Cache::flush_done() const { return impl_->flush_done(); } +void Cache::invalidate() { + impl_->invalidate(); +} + void Cache::keep_lines() { impl_->keep_lines(); } diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index fd8fafae0c..9f5de60bac 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -100,6 +100,10 @@ class Cache : public SimObject { // A shared last-level cache keeps its lines into the next launch. void keep_lines(); + // Drop every line (a read-only, virtually tagged cache on an + // address-space change). Called between launches. + void invalidate(); + protected: void on_reset(); void on_tick(); diff --git a/sim/simx/mem/mmu.cpp b/sim/simx/mem/mmu.cpp index e2d578ba9e..c254b68580 100644 --- a/sim/simx/mem/mmu.cpp +++ b/sim/simx/mem/mmu.cpp @@ -19,7 +19,9 @@ Mmu::Mmu(const SimContext& ctx, uint32_t num_ports, uint32_t tlb_size, uint32_t client_id, - bool exec_side) + bool exec_side, + bool rsp_reg, + bool write_enable) : SimObject(ctx, name) , ReqIn(num_ports, this) , RspOut(num_ports, this) @@ -30,10 +32,13 @@ Mmu::Mmu(const SimContext& ctx, , num_ports_(num_ports) , client_id_(client_id) , exec_side_(exec_side) + , rsp_reg_(rsp_reg) + , write_enable_(write_enable) , tlb_(tlb_size) , mshr_(VX_CFG_L1_TLB_MSHR_SIZE) , replay_(num_ports) , fault_rsp_(num_ports) + , pf_slot_(num_ports) {} Mmu::~Mmu() {} @@ -48,6 +53,9 @@ void Mmu::on_reset() { for (auto& q : fault_rsp_) { q = {}; } + for (auto& s : pf_slot_) { + s.reset(); + } reports_ = {}; // Physical pages move under a virtual address between launches, so // cached translations cannot outlive one — the shared levels clear here @@ -81,6 +89,11 @@ bool Mmu::empty() const { return false; } } + for (auto& s : pf_slot_) { + if (s) { + return false; + } + } return reports_.empty(); } @@ -139,14 +152,32 @@ void Mmu::kill_request(uint32_t port, const MemReq& req) { if (exec_side_) { return; } - // Same rule the caches apply: only a plain store retires without one. - if (req.op != MemOp::ST || req.flags.strsp) { + if (needs_kill_rsp(req)) { auto data = std::make_shared(); data->fill(0); fault_rsp_.at(port).push(MemRsp(req.tag, req.hart_id, req.uuid, data)); } } +bool Mmu::needs_kill_rsp(const MemReq& req) const { + // Same rule the caches apply: only a plain store retires without one. + return !exec_side_ && (req.op != MemOp::ST || req.flags.strsp); +} + +bool Mmu::port_parked(uint32_t port) const { + for (auto& e : mshr_) { + if (!e.valid) { + continue; + } + for (auto& pr : e.parked) { + if (pr.first == port) { + return true; + } + } + } + return false; +} + void Mmu::report_fault(const MemReq& req) { TlbReq report; report.vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; @@ -161,21 +192,27 @@ void Mmu::on_tick() { // 1) Forward downstream responses upstream unchanged. Kill responses go // first: they belong to accesses that will never reach the cache, and the // warps waiting on them cannot drain until they land. + uint64_t rsp_delay = rsp_reg_ ? 1 : 0; for (uint32_t p = 0; p < num_ports_; ++p) { if (RspOut.at(p).full()) { continue; } if (!fault_rsp_.at(p).empty()) { - RspOut.at(p).send(fault_rsp_.at(p).front(), 0); + RspOut.at(p).send(fault_rsp_.at(p).front(), rsp_delay); fault_rsp_.at(p).pop(); continue; } + if (pf_slot_.at(p)) { + RspOut.at(p).send(*pf_slot_.at(p), rsp_delay); + pf_slot_.at(p).reset(); + continue; + } if (RspIn.at(p).empty()) { continue; } - // Responses pass straight through with no added latency: the translation - // stage's cost is charged on the request path, not the reply path. - RspOut.at(p).send(RspIn.at(p).peek(), 0); + // Responses pass straight through, through the port's response stage + // when registered: the translation is charged on the request path. + RspOut.at(p).send(RspIn.at(p).peek(), rsp_delay); RspIn.at(p).pop(); } @@ -264,9 +301,14 @@ void Mmu::on_tick() { if (ReqIn.at(p).empty()) { continue; } - const MemReq& req = ReqIn.at(p).peek(); + MemReq req = ReqIn.at(p).peek(); if (!needs_translation(req)) { + // A flush token reaches the cache behind its port's parked requests + // (writing data-side clients only: an overtaken read is still correct). + if (req.flags.flush && write_enable_ && !exec_side_ && port_parked(p)) { + continue; + } if (ReqOut.at(p).try_send(req)) { ReqIn.at(p).pop(); } @@ -279,8 +321,16 @@ void Mmu::on_tick() { // A cached translation still has to satisfy the access: the entry // was installed for whichever intent first missed on this page. if (!tlb_perm_ok(res.flags, access_of(req), memop_is_amo(req.op))) { + bool needs_rsp = needs_kill_rsp(req); + if (needs_rsp && pf_slot_.at(p)) { + continue; // the port's fault slot is still owed out + } DT(3, this->name() << " perm-fault: vpn=0x" << std::hex << vpn << std::dec); - kill_request(p, req); + if (needs_rsp) { + auto data = std::make_shared(); + data->fill(0); + pf_slot_.at(p) = MemRsp(req.tag, req.hart_id, req.uuid, data); + } report_fault(req); ReqIn.at(p).pop(); continue; @@ -299,7 +349,7 @@ void Mmu::on_tick() { if (id < 0) { id = mshr_alloc(vpn); } - if (id >= 0 && mshr_[id].parked.size() < 2) { + if (id >= 0 && mshr_[id].parked.size() < MAX_PARKED_PER_ENTRY) { mshr_[id].parked.emplace_back(p, req); ReqIn.at(p).pop(); } diff --git a/sim/simx/mem/mmu.h b/sim/simx/mem/mmu.h index e71edd5bc6..846e1bdfbf 100644 --- a/sim/simx/mem/mmu.h +++ b/sim/simx/mem/mmu.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -54,7 +55,9 @@ class Mmu : public SimObject { uint32_t num_ports, uint32_t tlb_size, uint32_t client_id, - bool exec_side); + bool exec_side, + bool rsp_reg = false, + bool write_enable = true); ~Mmu(); @@ -86,6 +89,8 @@ class Mmu : public SimObject { }; bool needs_translation(const MemReq& req) const; + bool port_parked(uint32_t port) const; + bool needs_kill_rsp(const MemReq& req) const; int mshr_find(uint64_t vpn) const; int mshr_alloc(uint64_t vpn); TlbAccess access_of(const MemReq& req) const; @@ -95,12 +100,19 @@ class Mmu : public SimObject { uint32_t num_ports_; uint32_t client_id_; bool exec_side_; + // Each response leaves a cycle late through its own stage. + bool rsp_reg_; + // Clients may write: a flush token is ordered behind the port's parks. + bool write_enable_; std::unique_ptr satp_; Tlb tlb_; std::vector mshr_; std::vector> replay_; // per-port, drains before new input std::vector> fault_rsp_;// per-port, kills faulted accesses + // Per-port single slot answering a permission fault on a hit; a lane whose + // slot is still full stalls its next such fault. + std::vector> pf_slot_; std::queue reports_; // permission faults awaiting the walker // Requests of one VPN that a single miss-station entry can absorb. diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 9c631bea4c..4b54013a36 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -17,6 +17,9 @@ #include "cluster.h" #include "constants.h" #include "types.h" +#ifdef SIMX_SOCKET_UNIT_MMU +#include "mem/mmu.h" +#endif #ifdef VX_CFG_EXT_DXA_ENABLE #include "dxa_core.h" #include "local_mem.h" @@ -204,6 +207,16 @@ class Socket::Impl { dxa_core_ = DxaCore::Create(sname, simobject_); #endif +#ifdef SIMX_SOCKET_UNIT_MMU + // ── Socket unit MMU ───────────────────────────────────────────────── + // Translates the unit caches' and DXA's memory ports between them and the + // socket memory arbiter, so those caches are virtually tagged. Each + // response leaves through a registered stage; its clients only read. + snprintf(sname, 100, "%s-unit_mmu", name.c_str()); + unit_mmu_ = Mmu::Create(sname, UMMU_NUM_REQS, VX_CFG_UTLB_SIZE, + simobject_->unit_tlb_client(), false, true, false); +#endif + // find overlap uint32_t overlap = __MIN(VX_CFG_ICACHE_MEM_PORTS, VX_CFG_L1_MEM_PORTS); @@ -244,6 +257,23 @@ class Socket::Impl { sock_arb->RspOut.at(0).bind(&icaches_->mem_rsp_in.at(0)); dcaches_->mem_req_out.at(0).bind(&sock_arb->ReqIn.at(1)); sock_arb->RspOut.at(1).bind(&dcaches_->mem_rsp_in.at(0)); + #if defined(VX_CFG_EXT_RTU_ENABLE) || defined(VX_CFG_EXT_DXA_ENABLE) + // A unit port reaches the arbiter through the unit MMU under VM. + auto bind_unit_port = [&](uint32_t arb_idx, uint32_t ummu_idx, + SimChannel& req_src, + SimChannel& rsp_dst) { + #ifdef SIMX_SOCKET_UNIT_MMU + req_src.bind(&unit_mmu_->ReqIn.at(ummu_idx)); + unit_mmu_->RspOut.at(ummu_idx).bind(&rsp_dst); + unit_mmu_->ReqOut.at(ummu_idx).bind(&sock_arb->ReqIn.at(arb_idx)); + sock_arb->RspOut.at(arb_idx).bind(&unit_mmu_->RspIn.at(ummu_idx)); + #else + __unused(ummu_idx); + req_src.bind(&sock_arb->ReqIn.at(arb_idx)); + sock_arb->RspOut.at(arb_idx).bind(&rsp_dst); + #endif + }; + #endif #ifdef VX_CFG_EXT_TEX_ENABLE constexpr uint32_t kTexArbIdx = 2; tcache_->mem_req_out.at(0).bind(&sock_arb->ReqIn.at(kTexArbIdx)); @@ -251,13 +281,11 @@ class Socket::Impl { #endif #ifdef VX_CFG_EXT_RTU_ENABLE constexpr uint32_t kRtuArbIdx = 2 + VX_CFG_EXT_TEX_ENABLED; - rtcache_->mem_req_out.at(0).bind(&sock_arb->ReqIn.at(kRtuArbIdx)); - sock_arb->RspOut.at(kRtuArbIdx).bind(&rtcache_->mem_rsp_in.at(0)); + bind_unit_port(kRtuArbIdx, UMMU_RTCACHE_IDX, rtcache_->mem_req_out.at(0), rtcache_->mem_rsp_in.at(0)); #endif #ifdef VX_CFG_EXT_DXA_ENABLE constexpr uint32_t kDxaArbIdx = 2 + VX_CFG_EXT_TEX_ENABLED + VX_CFG_EXT_RTU_ENABLED; - dxa_core_->gmem_req_out.at(0).bind(&sock_arb->ReqIn.at(kDxaArbIdx)); - sock_arb->RspOut.at(kDxaArbIdx).bind(&dxa_core_->gmem_rsp_in.at(0)); + bind_unit_port(kDxaArbIdx, UMMU_DXA_IDX, dxa_core_->gmem_req_out.at(0), dxa_core_->gmem_rsp_in.at(0)); #endif bind_mem_port(0, sock_arb->ReqOut.at(0), sock_arb->RspIn.at(0)); @@ -381,6 +409,12 @@ class Socket::Impl { if (core->running()) return true; } + #ifdef SIMX_SOCKET_UNIT_MMU + // Parked translations hold no channel packet. + if (!unit_mmu_->empty()) { + return true; + } + #endif return false; } @@ -467,6 +501,19 @@ class Socket::Impl { RtuCore::Ptr& rtu_core() { return rtu_core_; } #endif +#ifdef SIMX_SOCKET_UNIT_MMU + SimChannel& tlb_miss_out() { return unit_mmu_->TlbMissOut; } + SimChannel& tlb_fill_in() { return unit_mmu_->TlbFillIn; } + // The unit caches are virtually tagged: a satp commit flushes the unit + // TLB and invalidates them. + void set_satp(uint64_t satp) { + unit_mmu_->set_satp(satp); + #ifdef VX_CFG_EXT_RTU_ENABLE + rtcache_->invalidate(); + #endif + } +#endif + private: Socket* simobject_; std::vector cores_; @@ -486,6 +533,9 @@ class Socket::Impl { Cache::Ptr rtcache_; RtuBusArbiter::Ptr rtu_bus_arb_; #endif +#ifdef SIMX_SOCKET_UNIT_MMU + Mmu::Ptr unit_mmu_; +#endif }; /////////////////////////////////////////////////////////////////////////////// @@ -493,12 +543,14 @@ class Socket::Impl { Socket::Socket(const SimContext& ctx, const char* name, uint32_t socket_id, - Cluster* cluster) + Cluster* cluster, + uint32_t unit_tlb_client) : SimObject(ctx, name) , mem_req_out(VX_CFG_L1_MEM_PORTS, this) , mem_rsp_in(VX_CFG_L1_MEM_PORTS, this) , socket_id_(socket_id) , cluster_(cluster) + , unit_tlb_client_(unit_tlb_client) , impl_(new Impl(this)) {} @@ -571,3 +623,17 @@ RtuCore::Ptr& Socket::rtu_core() { return impl_->rtu_core(); } #endif + +#ifdef SIMX_SOCKET_UNIT_MMU +SimChannel& Socket::tlb_miss_out() { + return impl_->tlb_miss_out(); +} + +SimChannel& Socket::tlb_fill_in() { + return impl_->tlb_fill_in(); +} + +void Socket::set_satp(uint64_t satp) { + impl_->set_satp(satp); +} +#endif diff --git a/sim/simx/socket.h b/sim/simx/socket.h index 15127a24e9..70c8878ea5 100644 --- a/sim/simx/socket.h +++ b/sim/simx/socket.h @@ -14,7 +14,9 @@ #pragma once #include "types.h" +#include "constants.h" #include "cache.h" +#include "mem/tlb_types.h" #ifdef VX_CFG_EXT_DXA_ENABLE #include "dxa_core.h" #endif @@ -54,12 +56,16 @@ class Socket : public SimObject { Socket(const SimContext& ctx, const char* name, uint32_t socket_id, - Cluster* cluster); + Cluster* cluster, + uint32_t unit_tlb_client = 0); ~Socket(); uint32_t id() const { return socket_id_; } + // The unit MMU's client index at the cluster TLB. + uint32_t unit_tlb_client() const { return unit_tlb_client_; } + Cluster* cluster() const { return cluster_; } bool running() const; @@ -98,12 +104,21 @@ class Socket : public SimObject { RtuCore::Ptr& rtu_core(); #endif +#ifdef SIMX_SOCKET_UNIT_MMU + // Unit MMU: its miss/fill link to the cluster TLB, and the device satp + // (Cluster::set_mmu_satp), which flushes its TLB on an actual change. + SimChannel& tlb_miss_out(); + SimChannel& tlb_fill_in(); + void set_satp(uint64_t satp); +#endif + protected: void on_reset(); private: uint32_t socket_id_; Cluster* cluster_; + uint32_t unit_tlb_client_; class Impl; Impl* impl_; diff --git a/sw/runtime/include/dxa.h b/sw/runtime/include/dxa.h index 6f0a3f4262..ee77ba8a0d 100644 --- a/sw/runtime/include/dxa.h +++ b/sw/runtime/include/dxa.h @@ -93,9 +93,9 @@ enum class Layout : uint32_t { // 5D: 21 DCR writes // // `base_addr` is written to VX_DCR_DXA_DESC_BASE_{LO,HI} and is -// consumed by the DXA worker's AXI master, which bypasses the -// per-core MMU. The backing buffer must be allocated with -// VX_MEM_PHYS (see vortex2.h). +// consumed by the DXA worker's memory port, which the socket unit +// MMU translates under VM: it is a device virtual address like any +// buffer address (vx_buffer_address). // Program a 1D DXA descriptor (7 DCR writes). // size0: dimension size (elements) From 0d854e1851137ad013411ad86a5c31b9011e695c Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 31 Oct 2026 14:03:53 -0700 Subject: [PATCH 308/336] vm: translate the TEX unit's tcache misses through the socket unit MMU Under VM the tcache's misses join the RTU's and the DXA's on the socket unit MMU (UMMU_TCACHE_IDX), so a texture base is a virtual address like any kernel pointer; the address-space flush also reaches the tcache's VX_dcr_flush leg. - The unit MMU registers its park (PARK_REG): the winning lane is staged one cycle and allocates the next, deduplicating against the staged VPN (ALLOC_PROBE). An entry that filled and freed meanwhile is re-armed as filled and replays from its kept translation, so the page is neither walked nor installed twice. SimX models the re-arm: the entry holds its miss-station slot and drains two cycles later, and a hit on a page with a live entry queues behind it, as in the RTL. - The TLB CAM carries each entry's superpage splice mask, decoded ahead of the mux, so a hit only ANDs it in. Beacon on the vm DUT: deepest path 25 -> 23; in the full GPU the dMMU and iMMU output stages go 24 -> 17. - SimX holds a flush only on MMUs with writing clients. - vm.yaml runs gfx_tex at a minted VA on simx and rtlsim, plus its parity. gfx_pipeline_tex and draw3d with only TEX enabled hit the known pinned-buffer collision at this commit; the cluster-unit change that follows fixes them. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 4 +- ci/testcases/vm.yaml | 16 +++ docs/designs/graphics_hardware_stack.md | 10 +- docs/designs/virtual_memory_subsystem.md | 88 ++++++++----- hw/rtl/VX_define.vh | 6 +- hw/rtl/VX_gpu_pkg.sv | 5 +- hw/rtl/VX_socket.sv | 43 ++++--- hw/rtl/tex/VX_tex_define.vh | 4 + hw/rtl/vm/VX_mmu.sv | 85 ++++++++++--- hw/rtl/vm/VX_tlb_cam.sv | 22 +++- hw/rtl/vm/VX_tlb_l1.sv | 2 + hw/rtl/vm/VX_tlb_mshr.sv | 62 ++++++++- hw/unittest/gfx/VX_gfx_top.sv | 15 ++- sim/simx/constants.h | 10 +- sim/simx/mem/mmu.cpp | 153 ++++++++++++++++++++--- sim/simx/mem/mmu.h | 29 ++++- sim/simx/socket.cpp | 13 +- sw/runtime/common/device.cpp | 10 +- tests/graphics/gfx_pipeline_tex/main.cpp | 2 +- tests/graphics/gfx_tex/main.cpp | 17 ++- tests/graphics/gfx_tex4/main.cpp | 24 ++-- 21 files changed, 475 insertions(+), 145 deletions(-) diff --git a/VX_config.toml b/VX_config.toml index b00033eb8e..485dbb0af7 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -180,8 +180,8 @@ VX_CFG_ITLB_SIZE = 8 VX_CFG_L1_TLB_MSHR_SIZE = 4 # Socket unit MMU: one TLB per socket for the socket-resident units' memory -# ports (RTU rtcache misses, DXA gmem reads), kept apart from the dTLB so -# BVH traffic cannot evict the cores' translations. +# ports (TEX tcache misses, RTU rtcache misses, DXA gmem reads), kept apart +# from the dTLB so texture and BVH traffic cannot evict the cores' translations. VX_CFG_UTLB_SIZE = 8 # Shared cluster TLB and page-table walker complex. diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index 8dd1b6fb44..debba0e557 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -265,6 +265,16 @@ tests: configs+: -DVX_CFG_EXT_DXA_ENABLE env: VORTEX_RANDOMIZE_VA: 1 +- id: vm-tex + via: blackbox + drivers: + - simx + - rtlsim + app: gfx_tex + configs+: -DVX_CFG_EXT_TEX_ENABLE + args: -i toad.png -r toad_ref_f0.png -f 0 -g 0 + env: + VORTEX_RANDOMIZE_VA: 1 # --------------------------------------------------------------------------- # check: model_parity — same VM-enabled app on both drivers (xlen 32); exact @@ -291,6 +301,12 @@ tests: app: dxa_copy configs+: -DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_EXT_DXA_ENABLE args: -d3 +- id: parity-vm-gfx_tex + check: model_parity + via: blackbox + app: gfx_tex + configs+: -DVX_CFG_EXT_TEX_ENABLE + args: -i toad.png -r toad_ref_f0.png -f 0 -g 0 # The RTL command processor's DMA walks the page table for every upload and # download; only xrt runs it (simx and rtlsim use the C++ command processor). - id: cp-dma-walk diff --git a/docs/designs/graphics_hardware_stack.md b/docs/designs/graphics_hardware_stack.md index 43132045a6..2946fa43a4 100644 --- a/docs/designs/graphics_hardware_stack.md +++ b/docs/designs/graphics_hardware_stack.md @@ -302,11 +302,13 @@ kernel), never by the host. ## 8. VM / pinned-buffer tie-in -Under `VX_CFG_VM_ENABLE` the per-core MMU translates VA→PA for kernel LSU -traffic, but the RASTER/TEX/OM AXI masters **bypass** the MMU and use the -physical addresses written into their DCRs. `VX_MEM_PHYS` buffers are +Under `VX_CFG_VM_ENABLE` the socket MMUs translate VA→PA for kernel LSU +traffic and, through the socket unit MMU, for the TEX unit's tcache misses +(see [`virtual_memory_subsystem.md`](virtual_memory_subsystem.md) §3.8), so a +texture is an ordinary buffer. The RASTER/OM masters still **bypass** the MMU +and use the physical addresses written into their DCRs. `VX_MEM_PHYS` buffers are identity-mapped and carved from a dedicated pinned slab so VA == PA. DCR -writes targeting graphics buffer-address registers are validated against +writes targeting raster/OM buffer-address registers are validated against the pinned slab on the CP submit path (returning `VX_ERR_INVALID_VALUE` for a PA outside the slab). The slab size is `VX_CFG_VM_PINNED_REGION_SIZE` (overridable via `VORTEX_VM_PINNED_SIZE`). Tests allocate every HW-bound diff --git a/docs/designs/virtual_memory_subsystem.md b/docs/designs/virtual_memory_subsystem.md index 7efb3e8ae7..32e3f7b5c6 100644 --- a/docs/designs/virtual_memory_subsystem.md +++ b/docs/designs/virtual_memory_subsystem.md @@ -160,11 +160,12 @@ kernel image once the loader knows its extents. Buffers allocated with (`VX_CFG_VM_PINNED_REGION_SIZE`), so a kernel can hold PA-stable pointers while paged and identity translations contend in the same TLBs. -The socket-resident units translate too: the RTU's rtcache misses and the -DXA's global reads go through the socket **unit MMU** (§3.8), so a BVH root, -an acceleration-structure pointer or a tensor descriptor is a virtual address -like any kernel pointer. The graphics fixed-function masters (RASTER, TEX, OM) -still issue physical addresses and never pass through an MMU; see +The socket-resident units translate too: the TEX unit's tcache misses, the +RTU's rtcache misses and the DXA's global reads go through the socket **unit +MMU** (§3.8), so a texture base, a BVH root, an acceleration-structure pointer +or a tensor descriptor is a virtual address like any kernel pointer. The +cluster-resident fixed-function masters (RASTER, OM) still issue physical +addresses and never pass through an MMU; see [`graphics_hardware_stack.md`](graphics_hardware_stack.md). --- @@ -321,14 +322,15 @@ The socket-resident units reach memory through the socket memory arbiter, not through the dcache, so the dMMU never sees them. Under VM a third `VX_mmu` instance per socket, `ummu` ([`VX_socket.sv`](../../hw/rtl/VX_socket.sv)), sits **on their memory side**: -between the rtcache's miss port and the DXA's gmem port on one side and the -socket arbiter's `RTCACHE_MEM_ARB_IDX` / `DXA_MEM_ARB_IDX` inputs on the other, -one lane per port (`UMMU_RTCACHE_IDX`, `UMMU_DXA_IDX` in +between the tcache's and rtcache's miss ports and the DXA's gmem port on one +side and the socket arbiter's `TCACHE_MEM_ARB_IDX` / `RTCACHE_MEM_ARB_IDX` / +`DXA_MEM_ARB_IDX` inputs on the other, one lane per port (`UMMU_TCACHE_IDX`, +`UMMU_RTCACHE_IDX`, `UMMU_DXA_IDX` in [`VX_gpu_pkg.sv`](../../hw/rtl/VX_gpu_pkg.sv)), each lane an L1 line wide with -the socket arbiter's tag. It is the same module as the dMMU, with four +the socket arbiter's tag. It is the same module as the dMMU, with five differences: -- **Its own TLB**, `VX_CFG_UTLB_SIZE` (8) entries, so BVH and tensor streams +- **Its own TLB**, `VX_CFG_UTLB_SIZE` (8) entries, so texture, BVH and tensor streams cannot evict the cores' translations, and its misses ride a third socket `tlb_bus` into `tlb_socket_arb` (3 inputs), on to the cluster TLB and the walker like any L1 miss. @@ -340,23 +342,39 @@ differences: lookup therefore starts at a client flop and its accept ends at the client's registered ready, never deeper inside the unit. A unit access pays the reply stage plus the MMU's own output stage, under VM only. +- **A registered park** (`PARK_REG = 1`). The lane that wins the park + arbitration is first registered in a one-entry stage and allocates in the + miss station the next cycle, deduplicating against the staged VPN itself + (`ALLOC_PROBE`) since the lane's probe is a cycle old by then. This keeps + the lane arbitration off the miss station's update cone. A later request + on the staged page holds until the stage drains, so it cannot overtake the + staged one. The stage takes a park only while empty, so the lanes park at + most every other cycle, and a park (new or joining) pays one more cycle. An + entry that filled and freed while the park was staged still matches: it is + re-armed as filled and replays from its kept translation (`retired_r`), + so the staged park neither walks the page again nor installs it twice. - **`WRITE_ENABLE = 0`.** Its clients only read, so a parked request keeps `{rw, addr, attr, tag}` and the replay restores zero data and all byte enables; the miss-station payload is a line's address, not a line. - **The unit caches are virtually indexed and tagged**, since translation happens after them. They are invalidated whenever translation changes: the socket snoop's `SATP_HI` flush (§8.2) also flushes the unit MMU's TLB and, - from a latch of its own, injects a flush token into the rtcache through its - `VX_dcr_flush` leg (OR-ed with the core-driven graphics flush), held until - the rtcache reports done. The two legs are independent: a cache flush waits - for its outstanding misses, which may need a unit-TLB park, and a TLB under - flush refuses parks, so holding the TLB flush for the cache's walk could - deadlock. The rtcache is read-only, so the flush only invalidates. + from a latch of its own, injects a flush token into the tcache and the + rtcache through their `VX_dcr_flush` legs (OR-ed with the core-driven + graphics flush), held until both caches report done. The two legs are + independent: a cache flush waits for its outstanding misses, which may need + a unit-TLB park, and a TLB under flush refuses parks, so holding the TLB + flush for a cache's walk could deadlock. Both caches are read-only, so the + flush only invalidates. A single address space has no synonyms, so a VIVT read-only cache flushed on every `satp` change is exact. -Without VM, or with neither RTU nor DXA (`SOCKET_UNIT_MMU_ENABLE` undefined), +Without VM, or with none of TEX, RTU and DXA (`SOCKET_UNIT_MMU_ENABLE` undefined), none of this exists: the unit ports wire straight into the arbiter as before. +A unit's DCR base address is a virtual address, which on Sv32 spans the +whole 32-bit space, so under VM the TEX base field (`TEX_ADDR_BITS`, a +cache-block index) is 26 bits; without VM, 25 bits cover the physical range. + The unit MMU's `empty` holds the socket `busy`, but not the per-core `mmu_drained` barrier gate: that gate orders a core's own loads and stores, none of which pass through the unit MMU. @@ -583,7 +601,7 @@ socket's DCR chain, forwarding it to the cores unchanged while snooping the satp writes. It derives `vm_active` from the mode field (`SATP_LO[31]` on Sv32, `SATP_HI[31:28] ≠ 0` on Sv39) and, on the `SATP_HI` write, requests a flush of every L1 MMU, held until all report done; with a -unit MMU the rtcache invalidation of §3.8 runs as a separate leg. The +unit MMU the tcache and rtcache invalidation of §3.8 runs as a separate leg. The flush is **self-timed**: satp only changes between kernels, once the TLBs are already drained, so no completion handshake crosses the socket boundary. This is what keeps `satp`, flush, and fault ports off the socket interface — only @@ -675,8 +693,8 @@ SimObjects bound through channels: Wiring ([`cluster.cpp`](../../sim/simx/cluster.cpp)): every core's two `Mmu` instances are clients of the cluster `L2Tlb`, followed by each socket's unit `Mmu` ([`socket.cpp`](../../sim/simx/socket.cpp)), placed as in the RTL -between the rtcache / DXA gmem ports and the socket arbiter; `set_satp` flushes -its TLB and invalidates the unit caches (`Cache::invalidate`); the `Ptw` shares L2-cache port 0 +between the tcache / rtcache / DXA gmem ports and the socket arbiter; `set_satp` +flushes its TLB and invalidates the unit caches (`Cache::invalidate`); the `Ptw` shares L2-cache port 0 with socket 0 through a **priority** arbiter that favours PTE fetches — every translated access behind a walk is blocked until it completes, and the walkers issue far too little traffic to starve the demand stream (the reverse @@ -746,7 +764,7 @@ untranslated. | `VX_CFG_VM_ENABLE` | false | build the MMU, TLB fabric, cluster TLB and walker at all | | `VX_CFG_DTLB_SIZE` | 16 | socket dMMU CAM entries | | `VX_CFG_ITLB_SIZE` | 8 | socket iMMU CAM entries | -| `VX_CFG_UTLB_SIZE` | 8 | socket unit MMU CAM entries (RTU, DXA) | +| `VX_CFG_UTLB_SIZE` | 8 | socket unit MMU CAM entries (TEX, RTU, DXA) | | `VX_CFG_L1_TLB_MSHR_SIZE` | 4 | L1 miss-station entries (distinct outstanding VPNs); 2 tokens each | | `VX_CFG_L2_TLB_SIZE` / `NUM_WAYS` | 512 / 4 | cluster TLB set array | | `VX_CFG_L2_TLB_MEGA_SIZE` | 8 | cluster TLB superpage CAM | @@ -785,6 +803,8 @@ sizes must be powers of two ≥ 2 (static asserts). | `vm-rtu` | `rt_smoke`, `EXT_RTU`, randomized VA | rtcache misses through the socket unit MMU | | `vm-dxa` | `sgemm2_dxa`, `EXT_DXA`, randomized VA | DXA gmem reads through the socket unit MMU | | `parity-vm-rt_raycast`, `parity-vm-dxa_copy` | `check: model_parity`, RTU + DXA | unit-MMU timing, simx vs rtlsim | +| `vm-tex` | `gfx_tex`, `EXT_TEX`, randomized VA | tcache misses through the socket unit MMU | +| `parity-vm-gfx_tex` | `check: model_parity`, TEX | unit-MMU texture timing, simx vs rtlsim | `vm_fault -t1` (a store to a read-only page) is not in CI: it needs the L1 permission-fault *report*, which the RTL does not surface (§9, §13). @@ -806,8 +826,8 @@ Grades: ✅ done · ⚠️ partial · ❌ pending. | area | state | note | |---|:--:|---| | Socket L1 translation stage, lane-parallel, hit-under-miss | ✅ | RTL + SimX | -| Socket unit MMU for RTU and DXA, VIVT rtcache invalidated on `satp` | ✅ | RTL + SimX | -| TEX, RASTER, OM translation | ❌ | still physical (`VX_MEM_PHYS` buffers) | +| Socket unit MMU for TEX, RTU and DXA, VIVT tcache/rtcache invalidated on `satp` | ✅ | RTL + SimX | +| RASTER, OM translation | ❌ | still physical (`VX_MEM_PHYS` buffers) | | VPN-dedup miss stations with per-entry requester queues, ordered drain | ✅ | L1 and cluster | | Narrow miss/fill fabric with id-prepending arbiters | ✅ | socket + cluster levels | | Cluster TLB: pipelined set-associative array + megapage CAM | ✅ | RTL + SimX | @@ -824,17 +844,17 @@ Grades: ✅ done · ⚠️ partial · ❌ pending. | Chip-wide L3 TLB, TLB inclusion policy, a `VX_dma` block with its own TLB | ❌ | not started | | ASIDs, A/D write-back, PMP, Sv48/Sv57 | ❌ | deliberate: single global address space; runtime pre-sets A/D | -**Known limitation.** The `satp` commit's invalidation of the rtcache shares -that cache's flush leg with the core-driven graphics flush: the two requests -are OR-ed onto one `VX_dcr_flush`, so whichever is raised second completes on -the walk the first one started — a core flush raised while the `satp` request -is high completes on the `satp` flush's done, and the reverse. Both directions -are benign for these read-only caches: the walk blocks the cache's inputs and -invalidates every line, so a walk that completes after a request was raised -leaves no line the request could still see. The runtime also commits `satp` -once, before the first launch, with no core flush in progress. A writing cache, -or a commit that must complete strictly after a given core flush, would need -its own requester. +**Known limitation.** The `satp` commit's invalidation of the tcache and the +rtcache shares each cache's flush leg with the core-driven graphics flush: the +two requests are OR-ed onto one `VX_dcr_flush`, so whichever is raised second +completes on the walk the first one started — a core flush raised while the +`satp` request is high completes on the `satp` flush's done, and the reverse. +Both directions are benign for these read-only caches: the walk blocks the +cache's inputs and invalidates every line, so a walk that completes after a +request was raised leaves no line the request could still see. The runtime +also commits `satp` once, before the first launch, with no core flush in +progress. A writing cache, or a commit that must complete strictly after a +given core flush, would need its own requester. **Deliberate simplifications** (recorded so they are not mistaken for bugs): there is one global address space and no ASID — `satp` is programmed diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index b2416abaa9..e6cabb35a2 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -84,9 +84,11 @@ `endif // Under VM, the socket unit MMU translates the memory ports of the -// socket-resident units: the rtcache misses and the DXA gmem reads. +// socket-resident units: the tcache and rtcache misses and the DXA gmem reads. `ifdef VX_CFG_VM_ENABLE -`ifdef VX_CFG_EXT_RTU_ENABLE +`ifdef VX_CFG_EXT_TEX_ENABLE + `define SOCKET_UNIT_MMU_ENABLE +`elsif VX_CFG_EXT_RTU_ENABLE `define SOCKET_UNIT_MMU_ENABLE `elsif VX_CFG_EXT_DXA_ENABLE `define SOCKET_UNIT_MMU_ENABLE diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index fa83dc9d9b..4bf3565d3f 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1975,9 +1975,10 @@ package VX_gpu_pkg; localparam SOCKET_MEM_ARB_REQS = 2 + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_DXA_ENABLED; // Socket unit MMU ports (SOCKET_UNIT_MMU_ENABLE), in socket arb order. - localparam UMMU_RTCACHE_IDX = 0; + localparam UMMU_TCACHE_IDX = 0; + localparam UMMU_RTCACHE_IDX = UMMU_TCACHE_IDX + `VX_CFG_EXT_TEX_ENABLED; localparam UMMU_DXA_IDX = UMMU_RTCACHE_IDX + `VX_CFG_EXT_RTU_ENABLED; - localparam UMMU_NUM_REQS = `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_DXA_ENABLED; + localparam UMMU_NUM_REQS = `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_DXA_ENABLED; // Widest tag among the socket arb inputs (DXA adapts to the given budget) `ifdef VX_CFG_EXT_TEX_ENABLE diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index b8acb3ca50..b5bea180cc 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -821,10 +821,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef SOCKET_UNIT_MMU_ENABLE /////////////////////////////////////////////////////////////////////////// // Socket unit MMU — translates the socket-resident units' memory ports - // (rtcache misses, DXA gmem reads) between them and the socket memory - // arbiter, so the unit caches are virtually indexed and tagged. Its own - // TLB keeps BVH and tensor streams from evicting the dTLB's entries, and - // its misses ride the third socket tlb_bus. + // (tcache and rtcache misses, DXA gmem reads) between them and the socket + // memory arbiter, so the unit caches are virtually indexed and tagged. Its + // own TLB keeps texture, BVH and tensor streams from evicting the dTLB's + // entries, and its misses ride the third socket tlb_bus. /////////////////////////////////////////////////////////////////////////// VX_mem_bus_if #( @@ -838,6 +838,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; ) unit_phys_bus_if[UMMU_NUM_REQS](); +`ifdef VX_CFG_EXT_TEX_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_TCACHE_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); +`endif `ifdef VX_CFG_EXT_RTU_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_RTCACHE_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif @@ -859,6 +862,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .EXEC_SIDE (0), .WRITE_ENABLE (0), .RSP_OUT_BUF (2), + .PARK_REG (1), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) ) ummu ( @@ -883,7 +887,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `UNUSED_VAR (ummu_fault_if.access) `UNUSED_VAR (ummu_fault_if.amo) -`ifndef VX_CFG_EXT_RTU_ENABLE +`ifndef EXT_GFX_ANY_ENABLE // no unit cache to invalidate assign ucache_flush_if.done = 1'b1; `UNUSED_VAR (ucache_flush_if.req) @@ -913,10 +917,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[ICACHE_MEM_ARB_IDX], icache_mem_bus_if[0], SOCKET_MEM_TAG_WIDTH, ICACHE_MEM_TAG_WIDTH, UUID_WIDTH); `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[DCACHE_MEM_ARB_IDX], dcache_mem_bus_if[0], SOCKET_MEM_TAG_WIDTH, DCACHE_MEM_TAG_WIDTH, UUID_WIDTH); + `ifdef SOCKET_UNIT_MMU_ENABLE `ifdef VX_CFG_EXT_TEX_ENABLE - `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[TCACHE_MEM_ARB_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[TCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_TCACHE_IDX]); `endif - `ifdef SOCKET_UNIT_MMU_ENABLE `ifdef VX_CFG_EXT_RTU_ENABLE `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_RTCACHE_IDX]); `endif @@ -924,6 +928,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[DXA_MEM_ARB_IDX], unit_phys_bus_if[UMMU_DXA_IDX]); `endif `else + `ifdef VX_CFG_EXT_TEX_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[TCACHE_MEM_ARB_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); + `endif `ifdef VX_CFG_EXT_RTU_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif @@ -990,20 +997,26 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; wire tcache_flush_done_w; wire rtcache_flush_done_w; +`ifdef SOCKET_UNIT_MMU_ENABLE + // The unit caches are virtually tagged: the SATP commit invalidates them too. + wire unit_cache_flush_req = ucache_flush_if.req; + assign ucache_flush_if.done = tcache_flush_done_w && rtcache_flush_done_w; +`else + wire unit_cache_flush_req = 1'b0; +`endif +`ifndef VX_CFG_EXT_TEX_ENABLE +`ifndef VX_CFG_EXT_RTU_ENABLE + `UNUSED_VAR (unit_cache_flush_req) // no socket-resident unit cache +`endif +`endif `ifdef VX_CFG_EXT_TEX_ENABLE - assign tcache_flush_if.req = socket_flush_req; + assign tcache_flush_if.req = socket_flush_req || unit_cache_flush_req; assign tcache_flush_done_w = tcache_flush_if.done; `else assign tcache_flush_done_w = 1'b1; `endif `ifdef VX_CFG_EXT_RTU_ENABLE -`ifdef SOCKET_UNIT_MMU_ENABLE - // The rtcache is virtually tagged: the SATP commit invalidates it too. - assign rtcache_flush_if.req = socket_flush_req || ucache_flush_if.req; - assign ucache_flush_if.done = rtcache_flush_if.done; -`else - assign rtcache_flush_if.req = socket_flush_req; -`endif + assign rtcache_flush_if.req = socket_flush_req || unit_cache_flush_req; assign rtcache_flush_done_w = rtcache_flush_if.done; `else assign rtcache_flush_done_w = 1'b1; diff --git a/hw/rtl/tex/VX_tex_define.vh b/hw/rtl/tex/VX_tex_define.vh index 7748778be7..d8e95e985a 100644 --- a/hw/rtl/tex/VX_tex_define.vh +++ b/hw/rtl/tex/VX_tex_define.vh @@ -25,8 +25,12 @@ `define TEX_FXD_HALF (`TEX_FXD_ONE >> 1) `define TEX_FXD_MASK (`TEX_FXD_ONE - 1) +// Texture base: a cache-block index. Under VM it is a virtual address, which +// spans the whole 32-bit space on Sv32. `ifdef VX_CFG_XLEN_64 `define TEX_ADDR_BITS 32 +`elsif VX_CFG_VM_ENABLE +`define TEX_ADDR_BITS 26 `else `define TEX_ADDR_BITS 25 `endif diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index 9e67270382..5b4979afc3 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -34,6 +34,10 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // Response stage per lane toward the client (TO_OUT_BUF encoding). Keeps // the injected-response select off the client's response cone. parameter RSP_OUT_BUF = 0, + // 1: a parking lane is first registered, and allocates in the miss + // station the next cycle. Takes the lane arbitration off the miss + // station's update cone; a lane on the staged page holds meanwhile. + parameter PARK_REG = 0, parameter DATA_SIZE = DCACHE_WORD_SIZE, parameter TAG_WIDTH = DCACHE_TAG_WIDTH_BASE, parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE), @@ -187,6 +191,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .MSHR_SIZE (MSHR_SIZE), .REPLAY_DEPTH (REPLAY_DEPTH), .PAYLOAD_W (PAYLOAD_W), + .ALLOC_PROBE (PARK_REG), .ID_WIDTH (ID_WIDTH) ) tlb ( .clk (clk), @@ -232,6 +237,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [NUM_REQS-1:0] cat_park; wire [NUM_REQS-1:0] cat_hit; wire [NUM_REQS-1:0] cat_pfault; + wire [NUM_REQS-1:0] stage_hold; // the lane's page is in the park stage wire [NUM_REQS-1:0] pf_accept; // permission fault taken this cycle wire [NUM_REQS-1:0] pf_rsp_busy; // a fault response is owed on the lane wire [NUM_REQS-1:0] lane_parked; // the lane has requests in the miss station @@ -239,11 +245,12 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( for (genvar l = 0; l < NUM_REQS; ++l) begin : g_cat // A flush token holds behind its lane's parked requests (see g_parked). wire flush_hold = req_attr[l][MEM_ATTR_FLUSH_OFFS] && lane_parked[l]; + wire xlat = req_valid[l] && !req_bypass[l] && !stage_hold[l]; assign cam_acc[l] = req_acc[l]; assign cat_bypass[l] = req_valid[l] && req_bypass[l] && !flush_hold; - assign cat_park[l] = req_valid[l] && !req_bypass[l] && (mshr_match[l] || !cam_hit[l]); - assign cat_hit[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && cam_perm[l]; - assign cat_pfault[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && !cam_perm[l]; + assign cat_park[l] = xlat && (mshr_match[l] || !cam_hit[l]); + assign cat_hit[l] = xlat && !mshr_match[l] && cam_hit[l] && cam_perm[l]; + assign cat_pfault[l] = xlat && !mshr_match[l] && cam_hit[l] && !cam_perm[l]; end // Park arbitration: at most one lane parks a miss per cycle (lowest lane). @@ -252,13 +259,16 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( localparam PARK_W = TLB_VPN_WIDTH + $bits(tlb_access_e) + 1 + FIELDS_W; wire [NUM_REQS-1:0] park_sel; + wire sel_valid; + wire sel_ready; + wire [LANE_W-1:0] sel_lane; VX_priority_encoder #( .N (NUM_REQS) ) park_enc ( .data_in (cat_park), .onehot_out (park_sel), - .index_out (park_lane), - .valid_out (park_valid) + .index_out (sel_lane), + .valid_out (sel_valid) ); wire [NUM_REQS-1:0][PARK_W-1:0] park_src; @@ -274,9 +284,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .sel_in (park_sel), .data_out (park_out) ); - assign park_vpn = park_out[PARK_W-1 -: TLB_VPN_WIDTH]; - assign park_access = tlb_access_e'(park_out[FIELDS_W+1 +: $bits(tlb_access_e)]); - assign park_amo = park_out[FIELDS_W]; + wire [TLB_VPN_WIDTH-1:0] sel_vpn = park_out[PARK_W-1 -: TLB_VPN_WIDTH]; // --------------------------------------------------------------------- // Replay decode + translation splice @@ -296,11 +304,59 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( assign replay_fields = {replay_pfields[PFIELDS_W-1:F_BYTEEN_LO], DATA_WIDTH'(0), {DATA_SIZE{1'b1}}, replay_pfields[F_BYTEEN_LO-1:0]}; `UNUSED_VAR (park_out[F_ADDR_LO-1:F_BYTEEN_LO]) - `RUNTIME_ASSERT (~(park_valid && park_ready) || ~park_out[F_RW_LO], + `RUNTIME_ASSERT (~(sel_valid && sel_ready) || ~park_out[F_RW_LO], ("%t: *** %s read-only MMU parked a write, addr=0x%0h", $time, INSTANCE_ID, park_out[F_ADDR_LO +: ADDR_WIDTH])) end - assign park_payload = {park_lane, park_pfields}; + + // The selected lane's park, straight into the miss station or through a + // one-entry stage. The stage accepts only while empty, so its ready is a + // flop; it allocates the next cycle against its own VPN (ALLOC_PROBE). + localparam STAGE_W = TLB_VPN_WIDTH + $bits(tlb_access_e) + 1 + LANE_W + PFIELDS_W; + wire [STAGE_W-1:0] sel_park = {sel_vpn, park_out[FIELDS_W+1 +: $bits(tlb_access_e)], + park_out[FIELDS_W], sel_lane, park_pfields}; + wire [STAGE_W-1:0] stage_park; + wire stage_valid; + + if (PARK_REG != 0) begin : g_park_reg + reg stage_valid_r; + reg [STAGE_W-1:0] stage_r; + always @(posedge clk) begin + if (reset) begin + stage_valid_r <= 1'b0; + end else if (sel_valid && sel_ready) begin + stage_valid_r <= 1'b1; + end else if (park_ready) begin + stage_valid_r <= 1'b0; + end + end + always @(posedge clk) begin + if (sel_ready) begin + stage_r <= sel_park; + end + end + assign sel_ready = ~stage_valid_r; + assign stage_valid = stage_valid_r; + assign stage_park = stage_r; + // Until it allocates, a later request on the staged page holds: it would + // miss the miss-station probe and could overtake the staged request. + wire [TLB_VPN_WIDTH-1:0] stage_vpn = stage_r[STAGE_W-1 -: TLB_VPN_WIDTH]; + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_hold + assign stage_hold[l] = stage_valid_r && (req_vpn[l] == stage_vpn); + end + end else begin : g_park_direct + assign sel_ready = park_ready; + assign stage_valid = sel_valid; + assign stage_park = sel_park; + assign stage_hold = '0; + end + + assign park_valid = stage_valid; + assign park_vpn = stage_park[STAGE_W-1 -: TLB_VPN_WIDTH]; + assign park_access = tlb_access_e'(stage_park[LANE_W + PFIELDS_W + 1 +: $bits(tlb_access_e)]); + assign park_amo = stage_park[LANE_W + PFIELDS_W]; + assign park_lane = stage_park[PFIELDS_W +: LANE_W]; + assign park_payload = stage_park[LANE_W + PFIELDS_W - 1 : 0]; wire replay_rw = replay_fields[F_RW_LO]; wire [ADDR_WIDTH-1:0] replay_raddr = replay_fields[F_ADDR_LO +: ADDR_WIDTH]; wire [ATTR_WIDTH-1:0] replay_rattr = replay_fields[F_ATTR_LO +: ATTR_WIDTH]; @@ -434,7 +490,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( assign req_if[l].req_ready = cat_bypass[l] ? (!replay_to_lane[l] && pipe_ready[l]) : cat_hit[l] ? (!replay_to_lane[l] && pipe_ready[l]) - : cat_park[l] ? (park_sel[l] && park_ready) + : cat_park[l] ? (park_sel[l] && sel_ready) : cat_pfault[l] ? pf_accept[l] : 1'b0; @@ -461,8 +517,9 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // read overtaken by a flush token is still answered correctly. // --------------------------------------------------------------------- localparam PCNT_W = `CLOG2(MSHR_SIZE * REPLAY_DEPTH + 2); - wire park_take = park_valid && park_ready; - wire [LANE_W-1:0] park_take_lane = park_lane; + // A park leaves its lane when the stage (or the miss station) takes it. + wire park_take = sel_valid && sel_ready; + wire [LANE_W-1:0] park_take_lane = sel_lane; wire drain_take = replay_ready || (kill_valid && kill_ready); if ((WRITE_ENABLE != 0) && (EXEC_SIDE == 0)) begin : g_flush_order @@ -539,6 +596,6 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( || pf_rsp_busy[l]; // owed fault response assign lane_idle[l] = ~pipe_busy[l] && ~lane_parked[l]; end - assign empty = ~(| pipe_busy) && tlb_empty; + assign empty = ~(| pipe_busy) && tlb_empty && ~(PARK_REG != 0 && stage_valid); endmodule diff --git a/hw/rtl/vm/VX_tlb_cam.sv b/hw/rtl/vm/VX_tlb_cam.sv index 950d974626..18c15614c5 100644 --- a/hw/rtl/vm/VX_tlb_cam.sv +++ b/hw/rtl/vm/VX_tlb_cam.sv @@ -63,10 +63,19 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( // Selectable fields packed once so each lane picks the winning entry with a // one-hot mux rather than encoding a hit index and re-indexing entries_r. - localparam SEL_W = TLB_PPN_WIDTH + TLB_FLAGS_WIDTH + TLB_LEVEL_WIDTH; + // Each entry also carries its superpage splice mask, decoded from its level + // ahead of the mux, so a hit only ANDs it in rather than decoding the + // selected level after the select. + localparam LOW_W = `UP((TLB_PT_LEVELS - 1) * TLB_LEVEL_BITS); + localparam SEL_W = TLB_PPN_WIDTH + TLB_FLAGS_WIDTH + TLB_LEVEL_WIDTH + LOW_W; wire [TLB_SIZE-1:0][SEL_W-1:0] entry_sel; for (genvar i = 0; i < TLB_SIZE; ++i) begin : g_entry_sel - assign entry_sel[i] = {entries_r[i].ppn, entries_r[i].flags, entries_r[i].level}; + wire [TLB_VPN_WIDTH-1:0] low_mask = ~tlb_vpn_mask(entries_r[i].level); + assign entry_sel[i] = {entries_r[i].ppn, entries_r[i].flags, entries_r[i].level, + low_mask[LOW_W-1:0]}; + if (TLB_VPN_WIDTH > LOW_W) begin : g_hi + `UNUSED_VAR (low_mask[TLB_VPN_WIDTH-1:LOW_W]) // a leaf is below the root level + end end for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lookup @@ -106,11 +115,12 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( end assign lookup_perm[l] = (| (hit_onehot & perm_vec)); - wire [TLB_PPN_WIDTH-1:0] sel_ppn = sel[TLB_LEVEL_WIDTH+TLB_FLAGS_WIDTH +: TLB_PPN_WIDTH]; - wire [TLB_FLAGS_WIDTH-1:0] sel_flags = sel[TLB_LEVEL_WIDTH +: TLB_FLAGS_WIDTH]; - wire [TLB_LEVEL_WIDTH-1:0] sel_level = sel[0 +: TLB_LEVEL_WIDTH]; + wire [TLB_PPN_WIDTH-1:0] sel_ppn = sel[LOW_W+TLB_LEVEL_WIDTH+TLB_FLAGS_WIDTH +: TLB_PPN_WIDTH]; + wire [TLB_FLAGS_WIDTH-1:0] sel_flags = sel[LOW_W+TLB_LEVEL_WIDTH +: TLB_FLAGS_WIDTH]; + wire [TLB_LEVEL_WIDTH-1:0] sel_level = sel[LOW_W +: TLB_LEVEL_WIDTH]; + wire [LOW_W-1:0] sel_low = sel[0 +: LOW_W]; - wire [TLB_VPN_WIDTH-1:0] low = lookup_vpn[l] & ~tlb_vpn_mask(sel_level); + wire [TLB_VPN_WIDTH-1:0] low = TLB_VPN_WIDTH'(lookup_vpn[l][LOW_W-1:0] & sel_low); // Splice the superpage's intra-page index from the VPN into the PPN. assign lookup_ppn[l] = sel_ppn | TLB_PPN_WIDTH'(low); assign lookup_flags[l] = sel_flags; diff --git a/hw/rtl/vm/VX_tlb_l1.sv b/hw/rtl/vm/VX_tlb_l1.sv index b9416d2964..32ff7092bb 100644 --- a/hw/rtl/vm/VX_tlb_l1.sv +++ b/hw/rtl/vm/VX_tlb_l1.sv @@ -27,6 +27,7 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, parameter REPLAY_DEPTH = 2, parameter PAYLOAD_W = 1, + parameter ALLOC_PROBE = 0, parameter ID_WIDTH = `CLOG2(MSHR_SIZE) ) ( input wire clk, @@ -140,6 +141,7 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .QDATA_W (PAYLOAD_W), .QDEPTH (REPLAY_DEPTH), .DEDUP_LIVE_EXCLUDES_FAULT (1), + .ALLOC_PROBE (ALLOC_PROBE), .ID_WIDTH (ID_WIDTH) ) mshr ( .clk (clk), diff --git a/hw/rtl/vm/VX_tlb_mshr.sv b/hw/rtl/vm/VX_tlb_mshr.sv index 32c8cb47da..0dd99d2ba0 100644 --- a/hw/rtl/vm/VX_tlb_mshr.sv +++ b/hw/rtl/vm/VX_tlb_mshr.sv @@ -24,6 +24,11 @@ // per-lane program order for same-page accesses, which the requester relies on. // DEDUP_LIVE_EXCLUDES_FAULT drops faulted entries from dedup (a new same-VPN // request re-walks); with it clear the request joins and shares the fault. +// ALLOC_PROBE dedups the allocate against its own VPN instead: for a requester +// that registers its park, whose lane probe is a cycle stale by then. An entry +// that filled and freed while the park was registered still matches (retired_r): +// the allocate re-arms it as filled, replaying from its kept translation rather +// than walking the page again. // // The associative state (valid/vpn/control) is held in flip-flops — an all-entry // parallel compare cannot come from an addressable RAM. The wide single-read @@ -39,6 +44,7 @@ module VX_tlb_mshr import VX_tlb_pkg::*; #( parameter QDATA_W = 1, parameter QDEPTH = 2, parameter DEDUP_LIVE_EXCLUDES_FAULT = 0, + parameter ALLOC_PROBE = 0, parameter ID_WIDTH = `CLOG2(MSHR_SIZE) ) ( input wire clk, @@ -137,9 +143,32 @@ module VX_tlb_mshr import VX_tlb_pkg::*; #( ); end - // Allocate reuses the probing lane's dedup result. - wire has_match = probe_match[alloc_lane]; - wire [IDX_W-1:0] match_slot = probe_slot[alloc_lane]; + wire has_match; + wire [IDX_W-1:0] match_slot; + wire rearm; // the allocate re-arms a retired entry + wire [MSHR_SIZE-1:0] retired; // freed after a good fill, translation kept + if (ALLOC_PROBE != 0) begin : g_alloc_probe + wire [MSHR_SIZE-1:0] m; + for (genvar e = 0; e < MSHR_SIZE; ++e) begin : g_m + assign m[e] = (live[e] || retired[e]) && (vpn_r[e] == alloc_vpn); + end + VX_priority_encoder #( + .N (MSHR_SIZE) + ) alloc_enc ( + .data_in (m), + `UNUSED_PIN (onehot_out), + .index_out (match_slot), + .valid_out (has_match) + ); + assign rearm = has_match && retired[match_slot]; + `UNUSED_VAR (alloc_lane) + `UNUSED_VAR (probe_slot) + end else begin : g_lane_probe + // Allocate reuses the probing lane's dedup result. + assign has_match = probe_match[alloc_lane]; + assign match_slot = probe_slot[alloc_lane]; + assign rearm = 1'b0; + end wire has_free; wire [IDX_W-1:0] free_slot; @@ -401,6 +430,10 @@ module VX_tlb_mshr import VX_tlb_pkg::*; #( fault_r <= '0; amo_r <= '0; end else begin + if (alloc_fire && rearm) begin + valid_r[match_slot] <= 1'b1; + filled_r[match_slot] <= 1'b1; + end if (alloc_fire && !has_match) begin valid_r[free_slot] <= 1'b1; issued_r[free_slot] <= 1'b0; @@ -450,6 +483,29 @@ module VX_tlb_mshr import VX_tlb_pkg::*; #( end end + // Retired entries: only a registered park can find its entry freed between + // its lane's probe and its allocate. The entry keeps issued_r set when it + // frees, so a re-arm restores valid and filled only. + if (ALLOC_PROBE != 0) begin : g_retired + reg [MSHR_SIZE-1:0] retired_r; + always @(posedge clk) begin + if (reset || flush) begin + retired_r <= '0; + end else begin + if (drain_free && !fault_r[drain_sel_r]) begin + retired_r[drain_sel_r] <= 1'b1; + end + if (alloc_fire) begin + retired_r[alloc_target] <= 1'b0; + end + end + end + assign retired = retired_r; + end else begin : g_no_retired + assign retired = '0; + `UNUSED_VAR (retired) + end + assign empty = ~(| valid_r); endmodule diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index 86890d34e2..48ac6a995d 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -21,8 +21,8 @@ // socket slice: // VX_tex_bus_arb -> VX_tex_core xN -> tcache (VX_cache_cluster) // VX_rtu_bus_arb -> VX_rtu_core xN -> rtcache (VX_cache_cluster) -// VX_mmu : under VM, the socket unit MMU between the rtcache / -// DXA gmem ports and the arbiter; its tlb_bus, satp +// VX_mmu : under VM, the socket unit MMU between the tcache / +// rtcache / DXA gmem ports and the arbiter; its tlb_bus, satp // mode and TLB flush are top ports. The 3-input socket // TLB arbiter and the snoop's unit-MMU and unit-cache // flush legs are elaborated only by the full-GPU build @@ -535,6 +535,9 @@ module VX_gfx_top import VX_gpu_pkg::* .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) ) unit_phys_bus_if[UMMU_NUM_REQS](); +`ifdef VX_CFG_EXT_TEX_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_TCACHE_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); +`endif `ifdef VX_CFG_EXT_RTU_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_RTCACHE_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif @@ -586,6 +589,7 @@ module VX_gfx_top import VX_gpu_pkg::* .EXEC_SIDE (0), .WRITE_ENABLE (0), .RSP_OUT_BUF (2), + .PARK_REG (1), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) ) ummu ( @@ -654,10 +658,10 @@ module VX_gfx_top import VX_gpu_pkg::* assign dc_mem_rsp_tag[0] = socket_arb_in_if[DCACHE_MEM_ARB_IDX].rsp_data.tag; assign socket_arb_in_if[DCACHE_MEM_ARB_IDX].rsp_ready = dc_mem_rsp_ready[0]; + `ifdef SOCKET_UNIT_MMU_ENABLE `ifdef VX_CFG_EXT_TEX_ENABLE - `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[TCACHE_MEM_ARB_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[TCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_TCACHE_IDX]); `endif - `ifdef SOCKET_UNIT_MMU_ENABLE `ifdef VX_CFG_EXT_RTU_ENABLE `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_RTCACHE_IDX]); `endif @@ -665,6 +669,9 @@ module VX_gfx_top import VX_gpu_pkg::* `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[DXA_MEM_ARB_IDX], unit_phys_bus_if[UMMU_DXA_IDX]); `endif `else + `ifdef VX_CFG_EXT_TEX_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[TCACHE_MEM_ARB_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); + `endif `ifdef VX_CFG_EXT_RTU_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif diff --git a/sim/simx/constants.h b/sim/simx/constants.h index c7a7c32810..0b96539cbd 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -90,13 +90,15 @@ static_assert((VX_CFG_NUM_CORES % VX_CFG_SOCKET_SIZE) == 0, "NUM_CORES must be a multiple of SOCKET_SIZE"); // Under VM, a socket's unit MMU translates the memory ports of its -// socket-resident units (rtcache misses, DXA gmem reads), in socket arb order. -#if defined(VX_CFG_VM_ENABLE) && (defined(VX_CFG_EXT_RTU_ENABLE) || defined(VX_CFG_EXT_DXA_ENABLE)) +// socket-resident units (tcache and rtcache misses, DXA gmem reads), in socket +// arb order. +#if defined(VX_CFG_VM_ENABLE) && (defined(VX_CFG_EXT_TEX_ENABLE) || defined(VX_CFG_EXT_RTU_ENABLE) || defined(VX_CFG_EXT_DXA_ENABLE)) #define SIMX_SOCKET_UNIT_MMU #endif -inline constexpr uint32_t UMMU_RTCACHE_IDX = 0; +inline constexpr uint32_t UMMU_TCACHE_IDX = 0; +inline constexpr uint32_t UMMU_RTCACHE_IDX = UMMU_TCACHE_IDX + VX_CFG_EXT_TEX_ENABLED; inline constexpr uint32_t UMMU_DXA_IDX = UMMU_RTCACHE_IDX + VX_CFG_EXT_RTU_ENABLED; -inline constexpr uint32_t UMMU_NUM_REQS = VX_CFG_EXT_RTU_ENABLED + VX_CFG_EXT_DXA_ENABLED; +inline constexpr uint32_t UMMU_NUM_REQS = VX_CFG_EXT_TEX_ENABLED + VX_CFG_EXT_RTU_ENABLED + VX_CFG_EXT_DXA_ENABLED; // Demand clients of the cluster L2 (sockets and gfx caches), excluding the // walker's PTE-fetch slot. diff --git a/sim/simx/mem/mmu.cpp b/sim/simx/mem/mmu.cpp index c254b68580..020d552641 100644 --- a/sim/simx/mem/mmu.cpp +++ b/sim/simx/mem/mmu.cpp @@ -21,7 +21,8 @@ Mmu::Mmu(const SimContext& ctx, uint32_t client_id, bool exec_side, bool rsp_reg, - bool write_enable) + bool write_enable, + bool park_reg) : SimObject(ctx, name) , ReqIn(num_ports, this) , RspOut(num_ports, this) @@ -34,6 +35,7 @@ Mmu::Mmu(const SimContext& ctx, , exec_side_(exec_side) , rsp_reg_(rsp_reg) , write_enable_(write_enable) + , park_reg_(park_reg) , tlb_(tlb_size) , mshr_(VX_CFG_L1_TLB_MSHR_SIZE) , replay_(num_ports) @@ -56,6 +58,7 @@ void Mmu::on_reset() { for (auto& s : pf_slot_) { s.reset(); } + stage_.reset(); reports_ = {}; // Physical pages move under a virtual address between launches, so // cached translations cannot outlive one — the shared levels clear here @@ -71,6 +74,9 @@ void Mmu::set_satp(uint64_t satp) { } satp_ = std::make_unique(satp); tlb_.flush(); // sfence.vma + for (auto& e : mshr_) { + e.retired = false; + } } bool Mmu::empty() const { @@ -94,6 +100,9 @@ bool Mmu::empty() const { return false; } } + if (stage_) { + return false; + } return reports_.empty(); } @@ -129,6 +138,8 @@ int Mmu::mshr_alloc(uint64_t vpn) { if (!mshr_[i].valid) { mshr_[i].valid = true; mshr_[i].issued = false; + mshr_[i].retired = false; + mshr_[i].rearmed = false; mshr_[i].vpn = vpn; mshr_[i].parked.clear(); return (int)i; @@ -165,6 +176,9 @@ bool Mmu::needs_kill_rsp(const MemReq& req) const { } bool Mmu::port_parked(uint32_t port) const { + if (stage_ && stage_->port == port) { + return true; + } for (auto& e : mshr_) { if (!e.valid) { continue; @@ -178,6 +192,82 @@ bool Mmu::port_parked(uint32_t port) const { return false; } +void Mmu::retire(int id, uint64_t ppn, uint32_t flags, uint32_t level) { + // A freed entry keeps its translation only behind a registered park, the + // one requester that can find its entry freed after probing it. + uint64_t vpn = mshr_[id].vpn; + mshr_[id] = MshrEntry(); + if (park_reg_) { + mshr_[id].vpn = vpn; + mshr_[id].retired = true; + mshr_[id].r_ppn = ppn; + mshr_[id].r_flags = flags; + mshr_[id].r_level = level; + } +} + +int Mmu::retired_find(uint64_t vpn) const { + for (size_t i = 0; i < mshr_.size(); ++i) { + if (!mshr_[i].valid && mshr_[i].retired && mshr_[i].vpn == vpn) { + return (int)i; + } + } + return -1; +} + +void Mmu::replay_or_kill(uint32_t port, const MemReq& req, uint64_t ppn, + uint32_t flags, uint32_t level) { + // Requests of differing intent chain onto one entry, so the walk's own + // check covers only the request that allocated it. Each parked request is + // re-checked against the flags the walk brought back. + if (!tlb_perm_ok(flags, access_of(req), memop_is_amo(req.op))) { + kill_request(port, req); + report_fault(req); + DT(3, this->name() << " perm-fault: vpn=0x" << std::hex + << (req.addr >> VX_VM_PAGE_LOG2_SIZE) << std::dec); + return; + } + uint64_t page_mask = (uint64_t(1) << VX_VM_PAGE_LOG2_SIZE) - 1; + uint64_t low_mask = (uint64_t(1) << (level * TLB_VPN_LEVEL_BITS)) - 1; + uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; + MemReq translated = req; + translated.addr = ((((ppn & ~low_mask) | (vpn & low_mask))) << VX_VM_PAGE_LOG2_SIZE) + | (req.addr & page_mask); + replay_.at(port).push(translated); +} + +void Mmu::stage_drain() { + // The staged park allocates (or joins) a cycle after it was taken, against + // its own VPN; an entry that filled and freed meanwhile re-arms from its + // kept translation. + if (!stage_ || SimPlatform::instance().cycles() < stage_->cycle) { + return; + } + auto& st = *stage_; + int id = mshr_find(st.vpn); + if (id < 0) { + int r = retired_find(st.vpn); + if (r >= 0) { + // Re-armed: the entry is held again and drains as a filled one does, + // two cycles on, absorbing same-page joins meanwhile. + auto& e = mshr_[r]; + e.valid = true; + e.issued = true; + e.rearmed = true; + e.rearm_cycle = SimPlatform::instance().cycles() + 2; + e.parked.clear(); + e.parked.emplace_back(st.port, st.req); + stage_.reset(); + return; + } + id = mshr_alloc(st.vpn); + } + if (id >= 0 && mshr_[id].parked.size() < MAX_PARKED_PER_ENTRY) { + mshr_[id].parked.emplace_back(st.port, st.req); + stage_.reset(); + } +} + void Mmu::report_fault(const MemReq& req) { TlbReq report; report.vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; @@ -228,35 +318,36 @@ void Mmu::on_tick() { // Dropping a fill would leave its parked requests waiting forever. __assert(id >= 0 && id < (int)mshr_.size() && mshr_[id].valid, "TLB fill does not match an outstanding miss"); - uint64_t page_mask = (uint64_t(1) << VX_VM_PAGE_LOG2_SIZE) - 1; - uint32_t shift = rsp.level * TLB_VPN_LEVEL_BITS; - uint64_t low_mask = (uint64_t(1) << shift) - 1; if (!rsp.fault) { tlb_.fill(mshr_[id].vpn, rsp.ppn, rsp.flags, rsp.level); } for (auto& [port, req] : mshr_[id].parked) { - // Requests of differing intent chain onto one entry, so the walk's - // own check covers only the request that allocated it. Each parked - // request is re-checked against the flags the walk brought back. if (rsp.fault) { kill_request(port, req); - } else if (!tlb_perm_ok(rsp.flags, access_of(req), memop_is_amo(req.op))) { - kill_request(port, req); - report_fault(req); - DT(3, this->name() << " perm-fault: vpn=0x" << std::hex - << (req.addr >> VX_VM_PAGE_LOG2_SIZE) << std::dec); } else { - MemReq translated = req; - uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; - uint64_t ppn = (rsp.ppn & ~low_mask) | (vpn & low_mask); - translated.addr = (ppn << VX_VM_PAGE_LOG2_SIZE) | (req.addr & page_mask); - replay_.at(port).push(translated); + replay_or_kill(port, req, rsp.ppn, rsp.flags, rsp.level); } } - mshr_[id] = MshrEntry(); + if (rsp.fault) { + mshr_[id] = MshrEntry(); + } else { + retire(id, rsp.ppn, rsp.flags, rsp.level); + } TlbFillIn.pop(); } + // 2b) Drain re-armed entries once their fill-equivalent cycle comes. + for (size_t i = 0; i < mshr_.size(); ++i) { + auto& e = mshr_[i]; + if (!e.valid || !e.rearmed || SimPlatform::instance().cycles() < e.rearm_cycle) { + continue; + } + for (auto& [port, req] : e.parked) { + replay_or_kill(port, req, e.r_ppn, e.r_flags, e.r_level); + } + retire((int)i, e.r_ppn, e.r_flags, e.r_level); + } + // 3) Issue pending miss requests (one per tick over the shared link). for (size_t i = 0; i < mshr_.size(); ++i) { auto& e = mshr_[i]; @@ -290,6 +381,16 @@ void Mmu::on_tick() { // while older requests are still parked, so only same-address order is // guaranteed — those share a VPN, hence one entry and its arrival-order // parked list. Replays drain ahead of new input on the same port. + // With a registered park, the stage's state at the start of the cycle + // decides both whether it takes a park and which page holds. + bool stage_free = !stage_.has_value(); + std::optional hold_vpn; + if (stage_) { + hold_vpn = stage_->vpn; + } + if (park_reg_) { + stage_drain(); + } for (uint32_t p = 0; p < num_ports_; ++p) { if (!replay_.at(p).empty()) { if (ReqOut.at(p).try_send(replay_.at(p).front())) { @@ -316,8 +417,14 @@ void Mmu::on_tick() { } uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; + if (hold_vpn && *hold_vpn == vpn) { + continue; // its page is in the park stage + } auto res = tlb_.lookup(vpn); - if (res.hit) { + // A page with a live miss-station entry (a re-armed one still holds its + // requests) parks behind it, as the RTL's station probe does, so a hit + // cannot overtake an older same-page request. + if (res.hit && !(park_reg_ && mshr_find(vpn) >= 0)) { // A cached translation still has to satisfy the access: the entry // was installed for whichever intent first missed on this page. if (!tlb_perm_ok(res.flags, access_of(req), memop_is_amo(req.op))) { @@ -345,6 +452,14 @@ void Mmu::on_tick() { // Park the miss and keep the port flowing (hit-under-miss). // Same-VPN misses chain on one entry, preserving arrival order; // a full entry or a full station stalls this port only. + if (park_reg_) { + if (stage_free) { + stage_ = Staged{p, req, vpn, SimPlatform::instance().cycles() + 1}; + stage_free = false; + ReqIn.at(p).pop(); + } + continue; + } int id = mshr_find(vpn); if (id < 0) { id = mshr_alloc(vpn); diff --git a/sim/simx/mem/mmu.h b/sim/simx/mem/mmu.h index 846e1bdfbf..90ff0822b5 100644 --- a/sim/simx/mem/mmu.h +++ b/sim/simx/mem/mmu.h @@ -57,7 +57,8 @@ class Mmu : public SimObject { uint32_t client_id, bool exec_side, bool rsp_reg = false, - bool write_enable = true); + bool write_enable = true, + bool park_reg = false); ~Mmu(); @@ -85,6 +86,15 @@ class Mmu : public SimObject { bool valid = false; bool issued = false; uint64_t vpn = 0; + // Freed after a good fill: a registered park whose lane probe saw this + // entry re-arms it from the kept translation instead of walking again. + bool retired = false; + // Re-armed from its kept translation: drains, as if filled, from this cycle. + bool rearmed = false; + uint64_t rearm_cycle = 0; + uint64_t r_ppn = 0; + uint32_t r_flags = 0; + uint32_t r_level = 0; std::vector> parked; // (port, request) }; @@ -93,6 +103,11 @@ class Mmu : public SimObject { bool needs_kill_rsp(const MemReq& req) const; int mshr_find(uint64_t vpn) const; int mshr_alloc(uint64_t vpn); + int retired_find(uint64_t vpn) const; + void replay_or_kill(uint32_t port, const MemReq& req, uint64_t ppn, + uint32_t flags, uint32_t level); + void stage_drain(); + void retire(int id, uint64_t ppn, uint32_t flags, uint32_t level); TlbAccess access_of(const MemReq& req) const; void kill_request(uint32_t port, const MemReq& req); void report_fault(const MemReq& req); @@ -104,6 +119,18 @@ class Mmu : public SimObject { bool rsp_reg_; // Clients may write: a flush token is ordered behind the port's parks. bool write_enable_; + // A registered park: a lane's miss (new or joining) is first held in a + // one-entry stage and reaches the miss station the next cycle at the + // earliest; the stage takes a request only while empty, and a lane whose + // page is staged holds meanwhile. + bool park_reg_; + struct Staged { + uint32_t port; + MemReq req; + uint64_t vpn; + uint64_t cycle; + }; + std::optional stage_; std::unique_ptr satp_; Tlb tlb_; diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 4b54013a36..cb16ceaa86 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -211,10 +211,11 @@ class Socket::Impl { // ── Socket unit MMU ───────────────────────────────────────────────── // Translates the unit caches' and DXA's memory ports between them and the // socket memory arbiter, so those caches are virtually tagged. Each - // response leaves through a registered stage; its clients only read. + // response leaves through a registered stage, and a new miss is held a + // cycle in a park stage before it allocates; its clients only read. snprintf(sname, 100, "%s-unit_mmu", name.c_str()); unit_mmu_ = Mmu::Create(sname, UMMU_NUM_REQS, VX_CFG_UTLB_SIZE, - simobject_->unit_tlb_client(), false, true, false); + simobject_->unit_tlb_client(), false, true, false, true); #endif // find overlap @@ -257,7 +258,6 @@ class Socket::Impl { sock_arb->RspOut.at(0).bind(&icaches_->mem_rsp_in.at(0)); dcaches_->mem_req_out.at(0).bind(&sock_arb->ReqIn.at(1)); sock_arb->RspOut.at(1).bind(&dcaches_->mem_rsp_in.at(0)); - #if defined(VX_CFG_EXT_RTU_ENABLE) || defined(VX_CFG_EXT_DXA_ENABLE) // A unit port reaches the arbiter through the unit MMU under VM. auto bind_unit_port = [&](uint32_t arb_idx, uint32_t ummu_idx, SimChannel& req_src, @@ -273,11 +273,9 @@ class Socket::Impl { sock_arb->RspOut.at(arb_idx).bind(&rsp_dst); #endif }; - #endif #ifdef VX_CFG_EXT_TEX_ENABLE constexpr uint32_t kTexArbIdx = 2; - tcache_->mem_req_out.at(0).bind(&sock_arb->ReqIn.at(kTexArbIdx)); - sock_arb->RspOut.at(kTexArbIdx).bind(&tcache_->mem_rsp_in.at(0)); + bind_unit_port(kTexArbIdx, UMMU_TCACHE_IDX, tcache_->mem_req_out.at(0), tcache_->mem_rsp_in.at(0)); #endif #ifdef VX_CFG_EXT_RTU_ENABLE constexpr uint32_t kRtuArbIdx = 2 + VX_CFG_EXT_TEX_ENABLED; @@ -508,6 +506,9 @@ class Socket::Impl { // TLB and invalidates them. void set_satp(uint64_t satp) { unit_mmu_->set_satp(satp); + #ifdef VX_CFG_EXT_TEX_ENABLE + tcache_->invalidate(); + #endif #ifdef VX_CFG_EXT_RTU_ENABLE rtcache_->invalidate(); #endif diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index 81e34a506d..a6f255e713 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -832,14 +832,14 @@ vx_result_t Device::cp_poll_seqnum_(uint64_t target) { vx_result_t Device::cp_submit_dcr_write(uint32_t addr, uint32_t value) { // VM safety check: when VM is active and the pinned slab is configured, // every HW-addr DCR must reference a buffer in the pinned slab — the HW - // master bypasses the per-core MMU and would otherwise dereference a stale VA. + // master bypasses the MMU and would otherwise dereference a stale VA. + // TEX (and DXA, RTU) translate through the socket unit MMU instead, so their + // addresses are VAs like any buffer address. // - // TEX / RASTER / OM address DCRs encode value as a cache-block index - // (pa = value << 6). Split BASE_LO/HI DCR pairs (e.g. DXA) are validated - // by the caller, not here. + // RASTER / OM address DCRs encode value as a cache-block index + // (pa = value << 6). if (vm_enabled_ && pinned_mem_) { switch (addr) { - case VX_DCR_TEX_ADDR: case VX_DCR_RASTER_TBUF_ADDR: case VX_DCR_RASTER_PBUF_ADDR: case VX_DCR_OM_CBUF_ADDR: diff --git a/tests/graphics/gfx_pipeline_tex/main.cpp b/tests/graphics/gfx_pipeline_tex/main.cpp index 2c3df6eaac..e5ccf748fc 100644 --- a/tests/graphics/gfx_pipeline_tex/main.cpp +++ b/tests/graphics/gfx_pipeline_tex/main.cpp @@ -176,7 +176,7 @@ int main(int argc, char** argv) { std::vector texbuf; std::vector mip_offsets; RT_CHECK(GenerateMipmaps(texbuf, mip_offsets, tex_pixels.data(), FORMAT_A8R8G8B8, tw, th, tw * 4)); vx_buffer_h tex_b = nullptr; uint64_t tex_addr = 0; - RT_CHECK(vx_buffer_create(dev, texbuf.size(), VX_MEM_READ | VX_MEM_PHYS, &tex_b)); + RT_CHECK(vx_buffer_create(dev, texbuf.size(), VX_MEM_READ, &tex_b)); RT_CHECK(vx_buffer_address(tex_b, &tex_addr)); RT_CHECK(vx_enqueue_write(q, tex_b, 0, texbuf.data(), texbuf.size(), 0, nullptr, nullptr)); diff --git a/tests/graphics/gfx_tex/main.cpp b/tests/graphics/gfx_tex/main.cpp index 4fe29506c7..5dc512ee95 100644 --- a/tests/graphics/gfx_tex/main.cpp +++ b/tests/graphics/gfx_tex/main.cpp @@ -1,11 +1,10 @@ // Texture regression test using the KMU launch API. // -// Pinned-buffer contract: -// src_buffer → TEX HW (VX_DCR_TEX_ADDR) → VX_MEM_PHYS (identity-mapped) -// dst_buffer → kernel-only (LSU stores) → not pinned -// Only src_buffer is read by the TEX fixed-function block. dst_buffer is -// written by kernel code through the per-core MMU, so it gets a regular -// minted VA under VM. +// Buffers: +// src_buffer → TEX HW (VX_DCR_TEX_ADDR) +// dst_buffer → kernel-only (LSU stores) +// Both get a regular minted VA under VM: the TEX unit's tcache misses +// translate through the socket unit MMU like the kernel's loads and stores. // // Host loads a PNG, generates a mipmap chain, configures TEX stage 0 // DCRs, then launches a 2D grid where each thread samples one output @@ -209,9 +208,9 @@ int main(int argc, char *argv[]) { RT_CHECK(vx_module_get_kernel(module_, border_mode ? "kernel_main_border" : "main", &kernel)); // ---- allocate device buffers ----------------------------------------- - // src_buffer is bound to the TEX unit (VX_DCR_TEX_ADDR) which - // bypasses the per-core MMU — needs a physical address. - RT_CHECK(vx_buffer_create(device, src_bufsize, VX_MEM_READ | VX_MEM_PHYS, &src_buffer)); + // src_buffer is bound to the TEX unit (VX_DCR_TEX_ADDR); under VM its + // reads translate through the socket unit MMU like any kernel access. + RT_CHECK(vx_buffer_create(device, src_bufsize, VX_MEM_READ, &src_buffer)); RT_CHECK(vx_buffer_address(src_buffer, &src_addr)); // dst_buffer is kernel-written via the LSU only — no HW-block reader, // no need to pin (the per-core MMU translates the VA for the kernel). diff --git a/tests/graphics/gfx_tex4/main.cpp b/tests/graphics/gfx_tex4/main.cpp index 5f4702572e..a2c171e600 100644 --- a/tests/graphics/gfx_tex4/main.cpp +++ b/tests/graphics/gfx_tex4/main.cpp @@ -1,11 +1,10 @@ // Texture regression test using the KMU launch API. // -// Pinned-buffer contract: -// src_buffer → TEX HW (VX_DCR_TEX_ADDR) → VX_MEM_PHYS (identity-mapped) -// dst_buffer → kernel-only (LSU stores) → not pinned -// Only src_buffer is read by the TEX fixed-function block. dst_buffer is -// written by kernel code through the per-core MMU, so it gets a regular -// minted VA under VM. +// Buffers: +// src_buffer → TEX HW (VX_DCR_TEX_ADDR) +// dst_buffer → kernel-only (LSU stores) +// Both get a regular minted VA under VM: the TEX unit's tcache misses +// translate through the socket unit MMU like the kernel's loads and stores. // // Host loads a PNG, generates a mipmap chain, configures TEX stage 0 // DCRs, then launches a 2D grid where each thread samples one output @@ -197,14 +196,11 @@ int main(int argc, char *argv[]) { RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); // ---- allocate device buffers ----------------------------------------- - // HW path: src_buffer is bound to the TEX unit (VX_DCR_TEX_ADDR), which reads - // it through the TEX/tcache path bypassing the per-core MMU — needs a physical - // (pinned) address. SW path (-S): the kernel reads the texture through the - // LSU/dcache like any other resident buffer, so allocate it as a normal - // kernel-read buffer (a PHYS buffer read via the LSU is not dcache-coherent - // with the host upload). - uint32_t src_flags = VX_MEM_READ | (sw_path ? 0u : (uint32_t)VX_MEM_PHYS); - RT_CHECK(vx_buffer_create(device, src_bufsize, src_flags, &src_buffer)); + // HW path: src_buffer is bound to the TEX unit (VX_DCR_TEX_ADDR), whose + // tcache translates through the socket unit MMU. SW path (-S): the kernel + // reads the texture through the LSU/dcache. Either way it is a normal + // kernel-read buffer. + RT_CHECK(vx_buffer_create(device, src_bufsize, VX_MEM_READ, &src_buffer)); RT_CHECK(vx_buffer_address(src_buffer, &src_addr)); // dst_buffer is kernel-written via the LSU only — no HW-block reader, // no need to pin (the per-core MMU translates the VA for the kernel). From ec353adac76674149dede3d3450fea5838e78433 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 31 Oct 2026 14:04:01 -0700 Subject: [PATCH 309/336] vm: translate raster and OM through a cluster unit MMU Under VM the cluster-resident raster and OM units reach the L2 through a fourth VX_mmu, cmmu, between the rcache's and ocache's memory ports and their L2 client slots, so tile, primitive, color and depth buffers are ordinary virtual buffers. It has its own TLB (VX_CFG_CTLB_SIZE), the registered reply and park of the socket unit MMU, and joins the cluster TLB arbiter after the sockets. WRITE_ENABLE follows OM, since the ocache writes through it. The rcache and ocache are virtually tagged, so the device's SATP_HI flush also reaches VX_graphics: the rcache invalidates, the OM flush sequencer drains its fragments and the ocache's writes into the L2, and only then is the cmmu TLB flushed; mmu_flush_done waits on the L2 TLB and on cmmu. The rcache and ocache register their memory requests (MEM_OUT_BUF). Beacon: the rcache and ocache output buffers go 5-8 -> 12 levels, the TLB lookup's ready path now ending on the client-owned register (as for the socket units); the deepest path is unchanged. The logic_gate baseline needs a re-record for it. vm.yaml runs gfx_raster and gfx_om at a minted VA on simx and rtlsim, plus their parity. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 4 + ci/testcases/vm.yaml | 32 ++++++ docs/designs/graphics_hardware_stack.md | 21 ++-- docs/designs/virtual_memory_subsystem.md | 75 +++++++++---- hw/rtl/VX_cluster.sv | 110 +++++++++++++++++++- hw/rtl/VX_define.vh | 7 ++ hw/rtl/VX_gpu_pkg.sv | 13 ++- hw/rtl/VX_graphics.sv | 49 ++++++++- hw/rtl/Vortex.sv | 11 ++ hw/rtl/om/VX_om_define.vh | 4 + hw/rtl/raster/VX_raster_define.vh | 4 + hw/unittest/gfx/VX_gfx_top.sv | 106 ++++++++++++++++++- sim/simx/cluster.cpp | 67 ++++++++++-- sim/simx/constants.h | 8 ++ sw/runtime/common/device.cpp | 31 ------ sw/runtime/common/graphics.cpp | 12 +-- sw/runtime/common/vm.cpp | 11 +- tests/graphics/gfx_draw3d/main.cpp | 37 +++---- tests/graphics/gfx_om/main.cpp | 19 ++-- tests/graphics/gfx_pipeline_om/main.cpp | 12 +-- tests/graphics/gfx_pipeline_raster/main.cpp | 10 +- tests/graphics/gfx_pipeline_tex/main.cpp | 12 +-- tests/graphics/gfx_raster/main.cpp | 26 ++--- 23 files changed, 534 insertions(+), 147 deletions(-) diff --git a/VX_config.toml b/VX_config.toml index 485dbb0af7..82f527aac1 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -184,6 +184,10 @@ VX_CFG_L1_TLB_MSHR_SIZE = 4 # from the dTLB so texture and BVH traffic cannot evict the cores' translations. VX_CFG_UTLB_SIZE = 8 +# Cluster unit MMU: one TLB per cluster for the rcache and ocache memory ports +# (raster and OM), a client of the cluster TLB like a socket. +VX_CFG_CTLB_SIZE = 8 + # Shared cluster TLB and page-table walker complex. VX_CFG_L2_TLB_SIZE = 512 VX_CFG_L2_TLB_NUM_WAYS = 4 diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index debba0e557..89ff412191 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -275,6 +275,26 @@ tests: args: -i toad.png -r toad_ref_f0.png -f 0 -g 0 env: VORTEX_RANDOMIZE_VA: 1 +- id: vm-raster + via: blackbox + drivers: + - simx + - rtlsim + app: gfx_raster + configs+: -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + args: -w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png + env: + VORTEX_RANDOMIZE_VA: 1 +- id: vm-om + via: blackbox + drivers: + - simx + - rtlsim + app: gfx_om + configs+: -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + args: -w 128 -h 128 -d -b -r whitebox_db_128.png + env: + VORTEX_RANDOMIZE_VA: 1 # --------------------------------------------------------------------------- # check: model_parity — same VM-enabled app on both drivers (xlen 32); exact @@ -307,6 +327,18 @@ tests: app: gfx_tex configs+: -DVX_CFG_EXT_TEX_ENABLE args: -i toad.png -r toad_ref_f0.png -f 0 -g 0 +- id: parity-vm-gfx_raster + check: model_parity + via: blackbox + app: gfx_raster + configs+: -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + args: -w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png +- id: parity-vm-gfx_om + check: model_parity + via: blackbox + app: gfx_om + configs+: -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + args: -w 128 -h 128 -d -b -r whitebox_db_128.png # The RTL command processor's DMA walks the page table for every upload and # download; only xrt runs it (simx and rtlsim use the C++ command processor). - id: cp-dma-walk diff --git a/docs/designs/graphics_hardware_stack.md b/docs/designs/graphics_hardware_stack.md index 2946fa43a4..669ff6568a 100644 --- a/docs/designs/graphics_hardware_stack.md +++ b/docs/designs/graphics_hardware_stack.md @@ -300,19 +300,14 @@ kernel), never by the host. --- -## 8. VM / pinned-buffer tie-in - -Under `VX_CFG_VM_ENABLE` the socket MMUs translate VA→PA for kernel LSU -traffic and, through the socket unit MMU, for the TEX unit's tcache misses -(see [`virtual_memory_subsystem.md`](virtual_memory_subsystem.md) §3.8), so a -texture is an ordinary buffer. The RASTER/OM masters still **bypass** the MMU -and use the physical addresses written into their DCRs. `VX_MEM_PHYS` buffers are -identity-mapped and carved from a dedicated pinned slab so VA == PA. DCR -writes targeting raster/OM buffer-address registers are validated against -the pinned slab on the CP submit path (returning `VX_ERR_INVALID_VALUE` -for a PA outside the slab). The slab size is `VX_CFG_VM_PINNED_REGION_SIZE` -(overridable via `VORTEX_VM_PINNED_SIZE`). Tests allocate every HW-bound -buffer with `VX_MEM_PHYS` and omit it for write-only LSU buffers. The +## 8. Virtual memory + +Under `VX_CFG_VM_ENABLE` every graphics master translates: TEX through the +socket unit MMU, RASTER and OM through the cluster unit MMU in front of the +rcache and ocache (see +[`virtual_memory_subsystem.md`](virtual_memory_subsystem.md) §3.8-3.9), so the +addresses in the TEX/RASTER/OM DCRs are ordinary device virtual addresses and +a texture, tile, primitive, color or depth buffer needs no `VX_MEM_PHYS`. The VM/MMU subsystem is documented in [`virtual_memory_subsystem.md`](virtual_memory_subsystem.md). diff --git a/docs/designs/virtual_memory_subsystem.md b/docs/designs/virtual_memory_subsystem.md index 32e3f7b5c6..2b2cc7abef 100644 --- a/docs/designs/virtual_memory_subsystem.md +++ b/docs/designs/virtual_memory_subsystem.md @@ -164,9 +164,9 @@ The socket-resident units translate too: the TEX unit's tcache misses, the RTU's rtcache misses and the DXA's global reads go through the socket **unit MMU** (§3.8), so a texture base, a BVH root, an acceleration-structure pointer or a tensor descriptor is a virtual address like any kernel pointer. The -cluster-resident fixed-function masters (RASTER, OM) still issue physical -addresses and never pass through an MMU; see -[`graphics_hardware_stack.md`](graphics_hardware_stack.md). +cluster-resident raster and OM units translate through the cluster **unit +MMU** (§3.9), so their tile, primitive, color and depth buffers are ordinary +buffers too; see [`graphics_hardware_stack.md`](graphics_hardware_stack.md). --- @@ -379,6 +379,36 @@ The unit MMU's `empty` holds the socket `busy`, but not the per-core `mmu_drained` barrier gate: that gate orders a core's own loads and stores, none of which pass through the unit MMU. +### 3.9 The cluster unit MMU + +Raster and OM live in the cluster (`VX_graphics`) and reach the L2 directly +through the rcache and the ocache, past every socket. Under VM a fourth +`VX_mmu`, `cmmu` ([`VX_cluster.sv`](../../hw/rtl/VX_cluster.sv)), sits between +those caches' memory ports and their L2 client slots (`L2_GFX_RASTER_IDX`, +`L2_GFX_OM_IDX`), one lane each (`CMMU_RCACHE_IDX`, `CMMU_OCACHE_IDX`), with +the same registered reply (`RSP_OUT_BUF = 2`) and registered park +(`PARK_REG = 1`) as the socket unit MMU; the rcache and ocache register their +memory requests (`MEM_OUT_BUF`). Its TLB is +`VX_CFG_CTLB_SIZE` (8) entries, and its misses enter the cluster TLB arbiter +as one more client after the sockets (`CLUSTER_NUM_TLB_PORTS`), its slot id +zero-extended to the socket buses' width. `WRITE_ENABLE` follows OM, since the +ocache writes (write-through) through it, so its park stage holds a full +line. `vm_active` comes from the device's +`satp` mode, fanned to the clusters from [`Vortex.sv`](../../hw/rtl/Vortex.sv). + +The rcache and ocache are virtually tagged, so the device's `SATP_HI` flush +(`mmu_flush_req`, §8.1) also reaches `VX_graphics` as an address-space-change +requester: the rcache invalidates through its `VX_dcr_flush` leg, and the OM +flush sequencer takes it as an L1-only round, draining the OM's fragments and +the ocache's writes into the L2 before invalidating. Only then does the +cluster flush the `cmmu` TLB those writes went through; the cluster's +`mmu_flush_done` waits on the L2 TLB and on `cmmu`. `cmmu.empty` joins the +cluster `busy`. With both units, the raster's early-Z depth reads go through +the ocache and so through `cmmu` too; its enable (`OM_EARLYZ_SAFE`) and depth +function reset off, so a draw that never programs the OM depth state reads no +depth buffer. As for TEX, the raster and OM buffer-base fields +(`RASTER_ADDR_BITS`, `OM_ADDR_BITS`) widen to 26 bits on Sv32 under VM. + --- ## 4. The miss/fill fabric @@ -402,7 +432,7 @@ elastic slice (`OUT_BUF = 3`). The id therefore grows deterministically ``` L1_TLB_ID_WIDTH = clog2(L1_TLB_MSHR_SIZE) the L1 miss-station slot TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + sel(SOCKET_NUM_MMUS) + {dMMU, iMMU[, unit MMU]} -TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + sel(NUM_SOCKETS) + socket +TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + sel(CLUSTER_NUM_TLB_PORTS) + socket [or cluster unit MMU] ``` The socket arbiter merges the dMMU, iMMU and (when present) unit MMU ports onto `cluster_tlb_bus_if`, @@ -633,7 +663,8 @@ DCRs. Then: maps of §2.4. - **`vx_mem_alloc`** takes a PA from the global pool, mints a VA (identity by default; `VORTEX_RANDOMIZE_VA=1` picks a random page-aligned - base seeded by `VORTEX_VA_SEED`, keeping multi-page buffers VA-contiguous) + base seeded by `VORTEX_VA_SEED`, keeping multi-page buffers VA-contiguous + and outside the OM aperture, which the LSU decodes by address) and installs one leaf PTE per page. Re-mapping a VA already covered by a leaf is idempotent when the translation matches (identity superpages installed at init are legitimately re-covered) and a conflict otherwise. @@ -692,9 +723,11 @@ SimObjects bound through channels: Wiring ([`cluster.cpp`](../../sim/simx/cluster.cpp)): every core's two `Mmu` instances are clients of the cluster `L2Tlb`, followed by each socket's unit -`Mmu` ([`socket.cpp`](../../sim/simx/socket.cpp)), placed as in the RTL +`Mmu` ([`socket.cpp`](../../sim/simx/socket.cpp)) and the cluster's unit `Mmu` +in front of the rcache / ocache, placed as in the RTL between the tcache / rtcache / DXA gmem ports and the socket arbiter; `set_satp` -flushes its TLB and invalidates the unit caches (`Cache::invalidate`); the `Ptw` shares L2-cache port 0 +flushes its TLB and invalidates the unit caches (`Cache::invalidate`; the +ocache writes through and holds no write between launches); the `Ptw` shares L2-cache port 0 with socket 0 through a **priority** arbiter that favours PTE fetches — every translated access behind a walk is blocked until it completes, and the walkers issue far too little traffic to starve the demand stream (the reverse @@ -765,6 +798,7 @@ untranslated. | `VX_CFG_DTLB_SIZE` | 16 | socket dMMU CAM entries | | `VX_CFG_ITLB_SIZE` | 8 | socket iMMU CAM entries | | `VX_CFG_UTLB_SIZE` | 8 | socket unit MMU CAM entries (TEX, RTU, DXA) | +| `VX_CFG_CTLB_SIZE` | 8 | cluster unit MMU CAM entries (RASTER, OM) | | `VX_CFG_L1_TLB_MSHR_SIZE` | 4 | L1 miss-station entries (distinct outstanding VPNs); 2 tokens each | | `VX_CFG_L2_TLB_SIZE` / `NUM_WAYS` | 512 / 4 | cluster TLB set array | | `VX_CFG_L2_TLB_MEGA_SIZE` | 8 | cluster TLB superpage CAM | @@ -805,6 +839,9 @@ sizes must be powers of two ≥ 2 (static asserts). | `parity-vm-rt_raycast`, `parity-vm-dxa_copy` | `check: model_parity`, RTU + DXA | unit-MMU timing, simx vs rtlsim | | `vm-tex` | `gfx_tex`, `EXT_TEX`, randomized VA | tcache misses through the socket unit MMU | | `parity-vm-gfx_tex` | `check: model_parity`, TEX | unit-MMU texture timing, simx vs rtlsim | +| `vm-raster` | `gfx_raster`, `EXT_RASTER` + `EXT_OM`, randomized VA | rcache misses through the cluster unit MMU | +| `vm-om` | `gfx_om -d -b`, `EXT_RASTER` + `EXT_OM`, randomized VA | ocache reads and write-throughs through the cluster unit MMU | +| `parity-vm-gfx_raster`, `parity-vm-gfx_om` | `check: model_parity`, RASTER + OM | cluster-unit-MMU timing, simx vs rtlsim | `vm_fault -t1` (a store to a read-only page) is not in CI: it needs the L1 permission-fault *report*, which the RTL does not surface (§9, §13). @@ -827,7 +864,7 @@ Grades: ✅ done · ⚠️ partial · ❌ pending. |---|:--:|---| | Socket L1 translation stage, lane-parallel, hit-under-miss | ✅ | RTL + SimX | | Socket unit MMU for TEX, RTU and DXA, VIVT tcache/rtcache invalidated on `satp` | ✅ | RTL + SimX | -| RASTER, OM translation | ❌ | still physical (`VX_MEM_PHYS` buffers) | +| Cluster unit MMU for RASTER and OM, rcache invalidated / ocache drained and invalidated on `satp` | ✅ | RTL + SimX | | VPN-dedup miss stations with per-entry requester queues, ordered drain | ✅ | L1 and cluster | | Narrow miss/fill fabric with id-prepending arbiters | ✅ | socket + cluster levels | | Cluster TLB: pipelined set-associative array + megapage CAM | ✅ | RTL + SimX | @@ -844,17 +881,17 @@ Grades: ✅ done · ⚠️ partial · ❌ pending. | Chip-wide L3 TLB, TLB inclusion policy, a `VX_dma` block with its own TLB | ❌ | not started | | ASIDs, A/D write-back, PMP, Sv48/Sv57 | ❌ | deliberate: single global address space; runtime pre-sets A/D | -**Known limitation.** The `satp` commit's invalidation of the tcache and the -rtcache shares each cache's flush leg with the core-driven graphics flush: the -two requests are OR-ed onto one `VX_dcr_flush`, so whichever is raised second -completes on the walk the first one started — a core flush raised while the -`satp` request is high completes on the `satp` flush's done, and the reverse. -Both directions are benign for these read-only caches: the walk blocks the -cache's inputs and invalidates every line, so a walk that completes after a -request was raised leaves no line the request could still see. The runtime -also commits `satp` once, before the first launch, with no core flush in -progress. A writing cache, or a commit that must complete strictly after a -given core flush, would need its own requester. +**Known limitation.** The `satp` commit's invalidation of the tcache, the +rtcache and the rcache shares each cache's flush leg with the core-driven +graphics flush: the two requests are OR-ed onto one `VX_dcr_flush`, so +whichever is raised second completes on the walk the first one started — a +core flush raised while the `satp` request is high completes on the `satp` +flush's done, and the reverse. Both directions are benign for these read-only +caches: the walk blocks the cache's inputs and invalidates every line, so a +walk that completes after a request was raised leaves no line the request +could still see. The runtime also commits `satp` once, before the first +launch, with no core flush in progress. A writing cache, or a commit that must +complete strictly after a given core flush, would need its own requester. **Deliberate simplifications** (recorded so they are not mistaken for bugs): there is one global address space and no ASID — `satp` is programmed diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index fb2fe0adce..caf860a54c 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -54,6 +54,10 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_mem_bus_if.slave dev_ptw_mem_if, input wire mmu_flush_req, output wire mmu_flush_done, +`ifdef CLUSTER_UNIT_MMU_ENABLE + // Translation active vs BARE (satp.MODE), for the cluster unit MMU. + input wire mmu_vm_active, +`endif `endif // Command-processor DMA, entering as an L2 client when this cluster's L2 @@ -324,14 +328,15 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; // --------------------------------------------------------------------- // Shared cluster TLB + page-table walker // --------------------------------------------------------------------- + // Every socket's tlb_bus, then the cluster unit MMU's (when present). VX_tlb_bus_if #(.ID_WIDTH (TLB_SOCKET_ID_WIDTH)) - per_socket_tlb_bus_if [NUM_SOCKETS] (); + per_socket_tlb_bus_if [CLUSTER_NUM_TLB_PORTS] (); VX_tlb_bus_if #(.ID_WIDTH (TLB_CLUSTER_ID_WIDTH)) l2_client_if (); VX_tlb_bus_if #(.ID_WIDTH (L2_TLB_SLOT_WIDTH)) l2_ptw_if (); VX_tlb_bus_arb #( - .NUM_INPUTS (NUM_SOCKETS), + .NUM_INPUTS (CLUSTER_NUM_TLB_PORTS), .ID_WIDTH_IN (TLB_SOCKET_ID_WIDTH), .OUT_BUF (3) ) tlb_cluster_arb ( @@ -374,9 +379,21 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; // TLB flush off the SATP DCR write, and the device walker reports its // own done leg at the top. assign l2_flush_if.req = mmu_flush_req; +`ifdef CLUSTER_UNIT_MMU_ENABLE + // The rcache and ocache are virtually tagged: the address-space change + // drains the ocache's writes and invalidates both, then flushes the + // cluster unit MMU's TLB that those writes went through. + VX_dcr_flush_if gfx_mmu_flush_if(); + VX_tlb_flush_if cmmu_flush_if(); + wire cmmu_empty; + assign gfx_mmu_flush_if.req = mmu_flush_req; + assign cmmu_flush_if.req = mmu_flush_req && gfx_mmu_flush_if.done; + assign mmu_flush_done = l2_flush_if.done && cmmu_flush_if.done; + wire mmu_busy = ~(l2_empty && cmmu_empty); +`else assign mmu_flush_done = l2_flush_if.done; - wire mmu_busy = ~l2_empty; +`endif `else wire mmu_busy = 1'b0; `endif @@ -465,9 +482,95 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif .dcr_bus_if (gfx_dcr_bus_if), .cluster_flush_if (cluster_flush_if), + `ifdef CLUSTER_UNIT_MMU_ENABLE + .mmu_flush_if (gfx_mmu_flush_if), + `endif .busy (gfx_busy) ); +`ifdef CLUSTER_UNIT_MMU_ENABLE + /////////////////////////////////////////////////////////////////////////// + // Cluster unit MMU — translates the rcache and ocache memory ports on + // their way to the L2, so those caches are virtually indexed and tagged. + // Its misses go straight to the cluster TLB as one more client. + /////////////////////////////////////////////////////////////////////////// + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) gfx_mem_bus_if[CMMU_NUM_REQS](); + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) gfx_phys_bus_if[CMMU_NUM_REQS](); + +`ifdef VX_CFG_EXT_RASTER_ENABLE + `ASSIGN_VX_MEM_BUS_IF (gfx_mem_bus_if[CMMU_RCACHE_IDX], rcache_l2_bus_if); + `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_GFX_RASTER_IDX], gfx_phys_bus_if[CMMU_RCACHE_IDX]); +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + `ASSIGN_VX_MEM_BUS_IF (gfx_mem_bus_if[CMMU_OCACHE_IDX], ocache_l2_bus_if); + `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_GFX_OM_IDX], gfx_phys_bus_if[CMMU_OCACHE_IDX]); +`endif + + // Its miss bus enters the cluster TLB arbiter after the sockets' buses, + // its slot id zero-extended to their width. + VX_tlb_bus_if #( + .ID_WIDTH (L1_TLB_ID_WIDTH) + ) cmmu_tlb_bus_if(); + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_valid = cmmu_tlb_bus_if.req_valid; + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_data.id = `UP(TLB_SOCKET_ID_WIDTH)'(cmmu_tlb_bus_if.req_data.id); + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_data.access = cmmu_tlb_bus_if.req_data.access; + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_data.amo = cmmu_tlb_bus_if.req_data.amo; + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_data.vpn = cmmu_tlb_bus_if.req_data.vpn; + assign cmmu_tlb_bus_if.req_ready = per_socket_tlb_bus_if[NUM_SOCKETS].req_ready; + assign cmmu_tlb_bus_if.rsp_valid = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_valid; + assign cmmu_tlb_bus_if.rsp_data.id = `UP(L1_TLB_ID_WIDTH)'(per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.id); + assign cmmu_tlb_bus_if.rsp_data.fault = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.fault; + assign cmmu_tlb_bus_if.rsp_data.level = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.level; + assign cmmu_tlb_bus_if.rsp_data.ppn = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.ppn; + assign cmmu_tlb_bus_if.rsp_data.flags = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.flags; + assign per_socket_tlb_bus_if[NUM_SOCKETS].rsp_ready = cmmu_tlb_bus_if.rsp_ready; + + VX_mmu_fault_if cmmu_fault_if(); + // Faults are reported by the walker, as for the socket MMUs. + `UNUSED_VAR (cmmu_fault_if.valid) + `UNUSED_VAR (cmmu_fault_if.va) + `UNUSED_VAR (cmmu_fault_if.access) + `UNUSED_VAR (cmmu_fault_if.amo) + +`ifdef PERF_ENABLE + mmu_perf_t cmmu_perf; + `UNUSED_VAR (cmmu_perf) +`endif + + VX_mmu #( + .INSTANCE_ID ("cmmu"), + .NUM_REQS (CMMU_NUM_REQS), + .TLB_SIZE (`VX_CFG_CTLB_SIZE), + .EXEC_SIDE (0), + .WRITE_ENABLE (`VX_CFG_EXT_OM_ENABLED), + .RSP_OUT_BUF (2), + .PARK_REG (1), + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) cmmu ( + .clk (clk), + .reset (reset), + .vm_active (mmu_vm_active), + `ifdef PERF_ENABLE + .mmu_perf (cmmu_perf), + `endif + .core_bus_if (gfx_mem_bus_if), + .mem_bus_if (gfx_phys_bus_if), + .tlb_bus_if (cmmu_tlb_bus_if), + .flush_if (cmmu_flush_if), + .fault_if (cmmu_fault_if), + `UNUSED_PIN (lane_idle), + .empty (cmmu_empty) + ); +`else `ifdef VX_CFG_EXT_RASTER_ENABLE `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_GFX_RASTER_IDX], rcache_l2_bus_if); `endif @@ -475,6 +578,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_EXT_OM_ENABLE `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_GFX_OM_IDX], ocache_l2_bus_if); `endif +`endif // CLUSTER_UNIT_MMU_ENABLE `elsif EXT_GFX_ANY_ENABLE // No cluster-resident graphics caches: the socket-local flush legs diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index e6cabb35a2..d3ff4a3849 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -83,6 +83,13 @@ `define EXT_GFX_CLUSTER_ENABLE `endif +// Under VM, the cluster unit MMU translates the rcache and ocache memory ports. +`ifdef VX_CFG_VM_ENABLE +`ifdef EXT_GFX_CLUSTER_ENABLE + `define CLUSTER_UNIT_MMU_ENABLE +`endif +`endif + // Under VM, the socket unit MMU translates the memory ports of the // socket-resident units: the tcache and rtcache misses and the DXA gmem reads. `ifdef VX_CFG_VM_ENABLE diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 4bf3565d3f..a9f7ab8c1e 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -2009,6 +2009,11 @@ package VX_gpu_pkg; localparam L2_GFX_RASTER_IDX = L2_SOCKET_REQS; localparam L2_GFX_OM_IDX = L2_GFX_RASTER_IDX + `VX_CFG_EXT_RASTER_ENABLED; + // Cluster unit MMU ports (CLUSTER_UNIT_MMU_ENABLE), in L2 client order. + localparam CMMU_RCACHE_IDX = 0; + localparam CMMU_OCACHE_IDX = CMMU_RCACHE_IDX + `VX_CFG_EXT_RASTER_ENABLED; + localparam CMMU_NUM_REQS = `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_OM_ENABLED; + // With one cluster, an L2 and no L3, that L2 is the LLC: the device // walker's PTE fetches ride it through a dedicated client slot, so a // leaf-PTE miss is a cache lookup rather than a DRAM round trip. In @@ -2051,7 +2056,13 @@ package VX_gpu_pkg; localparam SOCKET_NUM_MMUS = 2; `endif localparam TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + `ARB_SEL_BITS(SOCKET_NUM_MMUS, 1); - localparam TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + `ARB_SEL_BITS(NUM_SOCKETS, 1); + // The cluster TLB's clients: every socket, plus the cluster unit MMU. +`ifdef CLUSTER_UNIT_MMU_ENABLE + localparam CLUSTER_NUM_TLB_PORTS = NUM_SOCKETS + 1; +`else + localparam CLUSTER_NUM_TLB_PORTS = NUM_SOCKETS; +`endif + localparam TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + `ARB_SEL_BITS(CLUSTER_NUM_TLB_PORTS, 1); localparam L2_TLB_SLOT_WIDTH = `CLOG2(`VX_CFG_L2_TLB_MSHR_SIZE); // Device-level walker: cluster L2-TLB miss buses arb into one walker. localparam TLB_DEV_ID_WIDTH = L2_TLB_SLOT_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_CLUSTERS, 1); diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index 2e0fa10725..257db1f23e 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -65,6 +65,12 @@ module VX_graphics import VX_gpu_pkg::* // Cluster-level flush trigger. VX_dcr_flush_if.slave cluster_flush_if, +`ifdef CLUSTER_UNIT_MMU_ENABLE + // Address-space change (SATP commit): the rcache and ocache are virtually + // tagged, so the ocache drains its writes and both invalidate. + VX_dcr_flush_if.slave mmu_flush_if, +`endif + // Producer busy — the raster engine signals frame drain out-of-band (it has // no in-band `done`); the cluster ORs this into the device busy aggregation. output wire busy @@ -249,7 +255,11 @@ module VX_graphics import VX_gpu_pkg::* ) rcache_flushable_bus_if [`VX_CFG_NUM_RASTER_CORES * RCACHE_NUM_REQS] (); VX_dcr_flush_if rcache_flush_if(); +`ifdef CLUSTER_UNIT_MMU_ENABLE + assign rcache_flush_if.req = cluster_flush_if.req || mmu_flush_if.req; +`else assign rcache_flush_if.req = cluster_flush_if.req; +`endif VX_dcr_flush #( .WORD_SIZE (RCACHE_WORD_SIZE), @@ -502,6 +512,15 @@ module VX_graphics import VX_gpu_pkg::* reg omf_cluster_done_r; reg omf_l1_only_r; // every requester of this round is L1-only wire omf_cluster_req = cluster_flush_if.req && ~omf_cluster_done_r; + // The address-space change is an L1-only requester: the ocache drains its + // writes and invalidates, and the L2 (physically tagged) is left alone. +`ifdef CLUSTER_UNIT_MMU_ENABLE + reg omf_mmu_r; + reg omf_mmu_done_r; + wire omf_mmu_req = mmu_flush_if.req && ~omf_mmu_done_r; +`else + wire omf_mmu_req = 1'b0; +`endif // The cluster flush takes its kind from the broadcast write that starts it, // which every core receives too. The cores raise the cluster flush request @@ -552,7 +571,7 @@ module VX_graphics import VX_gpu_pkg::* end case (omf_state_r) OMF_IDLE: begin - if ((| steer_flush_req) || omf_cluster_req) begin + if ((| steer_flush_req) || omf_cluster_req || omf_mmu_req) begin omf_steers_r <= steer_flush_req; omf_cluster_r <= omf_cluster_req; omf_l1_only_r <= (& (~steer_flush_req | steer_flush_l1_only)) @@ -584,6 +603,26 @@ module VX_graphics import VX_gpu_pkg::* end assign ocache_flush_if.req = (omf_state_r == OMF_FLUSH); +`ifdef CLUSTER_UNIT_MMU_ENABLE + // The address-space change joins a round like the cluster flush does. + always @(posedge clk) begin + if (reset) begin + omf_mmu_r <= 1'b0; + omf_mmu_done_r <= 1'b0; + end else begin + if (~mmu_flush_if.req) begin + omf_mmu_done_r <= 1'b0; + end + if ((omf_state_r == OMF_IDLE) && ((| steer_flush_req) || omf_cluster_req || omf_mmu_req)) begin + omf_mmu_r <= omf_mmu_req; + end + if ((omf_state_r == OMF_FLUSH) && ocache_flush_if.done && omf_mmu_r) begin + omf_mmu_done_r <= 1'b1; + end + end + end +`endif + // OM-core flush chain carries the OM tag + 1 flush bit. When early-Z widens // the shared bus tag (a wider early-Z requester), the flushed OM bus is // zero-extended up to OCACHE_BUS_TAG_WIDTH below. Each OM input is a @@ -688,6 +727,14 @@ module VX_graphics import VX_gpu_pkg::* `endif assign cluster_flush_if.done = rcache_flush_done & ocache_flush_done; +`ifdef CLUSTER_UNIT_MMU_ENABLE +`ifdef VX_CFG_EXT_OM_ENABLE + assign mmu_flush_if.done = rcache_flush_done & omf_mmu_done_r; +`else + assign mmu_flush_if.done = rcache_flush_done; +`endif +`endif + // Producer busy = a raster engine still draining a frame (out-of-band // drain) or an OM core with fragments in flight (vx_om4 is fire-and-forget, // so nothing else holds the device busy until the ROP commits). diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 0c9ba0c713..8dd18a2b12 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -303,6 +303,14 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `ifdef VX_CFG_VM_ENABLE wire [`VX_CFG_XLEN-1:0] mmu_satp; wire mmu_flush_req; +`ifdef CLUSTER_UNIT_MMU_ENABLE + // Translation active vs BARE (satp.MODE), for the cluster unit MMUs. +`ifdef VX_VM_ADDR_MODE_SV32 + wire mmu_vm_active = mmu_satp[31]; +`else + wire mmu_vm_active = (| mmu_satp[63:60]); +`endif +`endif wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_flush_done; // Per-cluster L2-TLB miss export buses (used by the device-level walker). @@ -466,6 +474,9 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .dev_ptw_mem_if (per_cluster_ptw_mem_if[cluster_id]), .mmu_flush_req (mmu_flush_req), .mmu_flush_done (cl_mmu_flush_done[cluster_id]), + `ifdef CLUSTER_UNIT_MMU_ENABLE + .mmu_vm_active (mmu_vm_active), + `endif `endif .cp_mem_if (per_cluster_cp_mem_if[cluster_id]), diff --git a/hw/rtl/om/VX_om_define.vh b/hw/rtl/om/VX_om_define.vh index d6f9d12a8d..045fab1dc5 100644 --- a/hw/rtl/om/VX_om_define.vh +++ b/hw/rtl/om/VX_om_define.vh @@ -18,8 +18,12 @@ `include "VX_define.vh" +// Buffer bases: cache-block indices. Under VM they are virtual addresses, +// which span the whole 32-bit space on Sv32. `ifdef VX_CFG_XLEN_64 `define OM_ADDR_BITS 32 +`elsif VX_CFG_VM_ENABLE +`define OM_ADDR_BITS 26 `else `define OM_ADDR_BITS 25 `endif diff --git a/hw/rtl/raster/VX_raster_define.vh b/hw/rtl/raster/VX_raster_define.vh index b8920cba7f..b4c95d2ba2 100644 --- a/hw/rtl/raster/VX_raster_define.vh +++ b/hw/rtl/raster/VX_raster_define.vh @@ -18,8 +18,12 @@ `include "VX_define.vh" +// Buffer bases: cache-block indices. Under VM they are virtual addresses, +// which span the whole 32-bit space on Sv32. `ifdef VX_CFG_XLEN_64 `define RASTER_ADDR_BITS 32 +`elsif VX_CFG_VM_ENABLE +`define RASTER_ADDR_BITS 26 `else `define RASTER_ADDR_BITS 25 `endif diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index 48ac6a995d..a48dcfbe40 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -30,6 +30,8 @@ // input set {icache, dcache, tcache, rtcache, DXA gmem}; // the icache/dcache/DXA slots are driven from top ports // cluster tail: +// VX_mmu : under VM, the cluster unit MMU between the rcache / +// ocache ports and the L2 (tlb_bus and flush as ports) // VX_graphics : the production container — raster launch fork, // VX_raster_core xN + rcache, the OM trunk stage // (VX_om_steer/VX_om_ingress per trunk port), @@ -52,7 +54,7 @@ // top-level port and every DUT output is observed at a top-level port. module VX_gfx_top import VX_gpu_pkg::* -`ifdef SOCKET_UNIT_MMU_ENABLE +`ifdef VX_CFG_VM_ENABLE , VX_tlb_pkg::* `endif `ifdef VX_CFG_EXT_TEX_ENABLE @@ -68,7 +70,7 @@ module VX_gfx_top import VX_gpu_pkg::* parameter SFU_LANES = `VX_CFG_NUM_SFU_LANES, parameter L2_MEM_ADDR_W = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(L2_SECTOR_SIZE), parameter EXT_ADDR_W = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(`VX_CFG_L1_LINE_SIZE) -`ifdef SOCKET_UNIT_MMU_ENABLE +`ifdef VX_CFG_VM_ENABLE , parameter UTLB_ID_W = `UP(L1_TLB_ID_WIDTH), parameter UTLB_REQ_W = UTLB_ID_W + $bits(tlb_access_e) + 1 + TLB_VPN_WIDTH, parameter UTLB_RSP_W = UTLB_ID_W + 1 + TLB_LEVEL_WIDTH + TLB_PPN_WIDTH + TLB_FLAGS_WIDTH @@ -190,11 +192,27 @@ module VX_gfx_top import VX_gpu_pkg::* input wire flush_req, output wire flush_done, +`ifdef VX_CFG_VM_ENABLE + input wire vm_active, +`endif +`ifdef CLUSTER_UNIT_MMU_ENABLE + // ----------------------------------------------------------------------- + // Cluster unit MMU: the address-space change, drain, and its tlb_bus + // ----------------------------------------------------------------------- + input wire cmmu_flush_req, + output wire cmmu_flush_done, + output wire cmmu_empty, + output wire ctlb_req_valid, + output wire [UTLB_REQ_W-1:0] ctlb_req_data, + input wire ctlb_req_ready, + input wire ctlb_rsp_valid, + input wire [UTLB_RSP_W-1:0] ctlb_rsp_data, + output wire ctlb_rsp_ready, +`endif `ifdef SOCKET_UNIT_MMU_ENABLE // ----------------------------------------------------------------------- - // Socket unit MMU: satp mode, TLB flush, drain, and its tlb_bus + // Socket unit MMU: TLB flush, drain, and its tlb_bus // ----------------------------------------------------------------------- - input wire vm_active, input wire ummu_flush_req, output wire ummu_flush_done, output wire ummu_empty, @@ -795,6 +813,14 @@ module VX_gfx_top import VX_gpu_pkg::* ) ocache_l2_bus_if(); `endif +`ifdef CLUSTER_UNIT_MMU_ENABLE + VX_dcr_flush_if gfx_mmu_flush_if(); + VX_tlb_flush_if cmmu_flush_if(); + assign gfx_mmu_flush_if.req = cmmu_flush_req; + assign cmmu_flush_if.req = cmmu_flush_req && gfx_mmu_flush_if.done; + assign cmmu_flush_done = cmmu_flush_if.done; +`endif + VX_graphics #( .INSTANCE_ID ("cluster0") ) graphics ( @@ -813,6 +839,9 @@ module VX_gfx_top import VX_gpu_pkg::* `endif .dcr_bus_if (gfx_dcr_bus_if), .cluster_flush_if (cluster_flush_if), + `ifdef CLUSTER_UNIT_MMU_ENABLE + .mmu_flush_if (gfx_mmu_flush_if), + `endif .busy (busy) ); @@ -826,12 +855,81 @@ module VX_gfx_top import VX_gpu_pkg::* end `endif +`ifdef CLUSTER_UNIT_MMU_ENABLE + // Cluster unit MMU, wired as VX_cluster wires it. + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) gfx_mem_bus_if[CMMU_NUM_REQS](); + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) gfx_phys_bus_if[CMMU_NUM_REQS](); + +`ifdef VX_CFG_EXT_RASTER_ENABLE + `ASSIGN_VX_MEM_BUS_IF (gfx_mem_bus_if[CMMU_RCACHE_IDX], rcache_l2_bus_if); + `ASSIGN_VX_MEM_BUS_IF (l2_core_if[L2_GFX_RASTER_IDX], gfx_phys_bus_if[CMMU_RCACHE_IDX]); +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + `ASSIGN_VX_MEM_BUS_IF (gfx_mem_bus_if[CMMU_OCACHE_IDX], ocache_l2_bus_if); + `ASSIGN_VX_MEM_BUS_IF (l2_core_if[L2_GFX_OM_IDX], gfx_phys_bus_if[CMMU_OCACHE_IDX]); +`endif + + VX_tlb_bus_if #( + .ID_WIDTH (L1_TLB_ID_WIDTH) + ) ctlb_bus_if(); + assign ctlb_req_valid = ctlb_bus_if.req_valid; + assign ctlb_req_data = ctlb_bus_if.req_data; + assign ctlb_bus_if.req_ready = ctlb_req_ready; + assign ctlb_bus_if.rsp_valid = ctlb_rsp_valid; + assign ctlb_bus_if.rsp_data = ctlb_rsp_data; + assign ctlb_rsp_ready = ctlb_bus_if.rsp_ready; + + VX_mmu_fault_if cmmu_fault_if(); + `UNUSED_VAR (cmmu_fault_if.valid) + `UNUSED_VAR (cmmu_fault_if.va) + `UNUSED_VAR (cmmu_fault_if.access) + `UNUSED_VAR (cmmu_fault_if.amo) + +`ifdef PERF_ENABLE + mmu_perf_t cmmu_perf; + `UNUSED_VAR (cmmu_perf) +`endif + + VX_mmu #( + .INSTANCE_ID ("cmmu"), + .NUM_REQS (CMMU_NUM_REQS), + .TLB_SIZE (`VX_CFG_CTLB_SIZE), + .EXEC_SIDE (0), + .WRITE_ENABLE (`VX_CFG_EXT_OM_ENABLED), + .RSP_OUT_BUF (2), + .PARK_REG (1), + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) cmmu ( + .clk (clk), + .reset (reset), + .vm_active (vm_active), + `ifdef PERF_ENABLE + .mmu_perf (cmmu_perf), + `endif + .core_bus_if (gfx_mem_bus_if), + .mem_bus_if (gfx_phys_bus_if), + .tlb_bus_if (ctlb_bus_if), + .flush_if (cmmu_flush_if), + .fault_if (cmmu_fault_if), + `UNUSED_PIN (lane_idle), + .empty (cmmu_empty) + ); +`else `ifdef VX_CFG_EXT_RASTER_ENABLE `ASSIGN_VX_MEM_BUS_IF (l2_core_if[L2_GFX_RASTER_IDX], rcache_l2_bus_if); `endif `ifdef VX_CFG_EXT_OM_ENABLE `ASSIGN_VX_MEM_BUS_IF (l2_core_if[L2_GFX_OM_IDX], ocache_l2_bus_if); `endif +`endif // CLUSTER_UNIT_MMU_ENABLE `else // !EXT_GFX_CLUSTER_ENABLE // No cluster-resident graphics: the trunk connects straight to the L2. diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index f08072a6b1..283324fc77 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -19,6 +19,9 @@ #include "constants.h" #include "types.h" #include "debug.h" +#ifdef SIMX_CLUSTER_UNIT_MMU +#include "mem/mmu.h" +#endif #ifdef VX_CFG_EXT_OM_ENABLE #include "om_core.h" #include "om_unit.h" @@ -135,6 +138,22 @@ class Cluster::Impl { l2arb->RspOut.at(kL2Rows * port + 0).bind(&sockets_.at(i)->mem_rsp_in.at(j)); } } + // A gfx cache's memory port reaches the L2 arbiter through the cluster + // unit MMU under VM. + auto bind_gfx_port = [&](uint32_t arb_idx, uint32_t cmmu_idx, + SimChannel& req_src, + SimChannel& rsp_dst) { + #ifdef SIMX_CLUSTER_UNIT_MMU + req_src.bind(&gfx_mmu_->ReqIn.at(cmmu_idx)); + gfx_mmu_->RspOut.at(cmmu_idx).bind(&rsp_dst); + gfx_mmu_->ReqOut.at(cmmu_idx).bind(&l2arb->ReqIn.at(arb_idx)); + l2arb->RspOut.at(arb_idx).bind(&gfx_mmu_->RspIn.at(cmmu_idx)); + #else + __unused(cmmu_idx); + req_src.bind(&l2arb->ReqIn.at(arb_idx)); + l2arb->RspOut.at(arb_idx).bind(&rsp_dst); + #endif + }; // L2 arb outputs → l2cache (after all rows are bound). for (uint32_t i = 0; i < VX_CFG_L2_DEMAND_REQS; ++i) { l2arb->ReqOut.at(i).bind(&l2cache_->core_req_in.at(i)); @@ -156,9 +175,15 @@ class Cluster::Impl { // binds at the device. uint32_t num_core_tlb_clients = sockets_per_cluster * VX_CFG_SOCKET_SIZE * 2; #ifdef SIMX_SOCKET_UNIT_MMU - uint32_t num_tlb_clients = num_core_tlb_clients + sockets_per_cluster; + uint32_t num_unit_tlb_clients = sockets_per_cluster; + #else + uint32_t num_unit_tlb_clients = 0; + #endif + #ifdef SIMX_CLUSTER_UNIT_MMU + uint32_t gfx_tlb_client = num_core_tlb_clients + num_unit_tlb_clients; + uint32_t num_tlb_clients = gfx_tlb_client + 1; #else - uint32_t num_tlb_clients = num_core_tlb_clients; + uint32_t num_tlb_clients = num_core_tlb_clients + num_unit_tlb_clients; #endif snprintf(sname, 100, "%s-l2tlb", name.c_str()); l2tlb_ = L2Tlb::Create(sname, num_tlb_clients); @@ -180,6 +205,18 @@ class Cluster::Impl { l2tlb_->RspOut.at(socket->unit_tlb_client()).bind(&socket->tlb_fill_in()); } #endif + #ifdef SIMX_CLUSTER_UNIT_MMU + // Cluster unit MMU: between the rcache / ocache memory ports and the L2, + // so those caches are virtually tagged. Its TLB is the cluster TLB's last + // client. Like the socket unit MMU, each response leaves through a + // registered stage and a new miss is held a cycle before it allocates; + // the ocache writes, so a flush token is ordered behind its parks. + snprintf(sname, 100, "%s-gfx_mmu", name.c_str()); + gfx_mmu_ = Mmu::Create(sname, CMMU_NUM_REQS, VX_CFG_CTLB_SIZE, gfx_tlb_client, + false, true, VX_CFG_EXT_OM_ENABLED != 0, true); + gfx_mmu_->TlbMissOut.bind(&l2tlb_->ReqIn.at(gfx_tlb_client)); + l2tlb_->RspOut.at(gfx_tlb_client).bind(&gfx_mmu_->TlbFillIn); + #endif #endif @@ -224,8 +261,7 @@ class Cluster::Impl { // ocache memory side → l2arb. constexpr uint32_t kOmRow = 1; for (uint32_t i = 0; i < kOcacheMemPorts; ++i) { - ocache->mem_req_out.at(i).bind(&l2arb->ReqIn.at(kL2Rows * i + kOmRow)); - l2arb->RspOut.at(kL2Rows * i + kOmRow).bind(&ocache->mem_rsp_in.at(i)); + bind_gfx_port(kL2Rows * i + kOmRow, CMMU_OCACHE_IDX, ocache->mem_req_out.at(i), ocache->mem_rsp_in.at(i)); } // Per-core SFU.om_req_out (OmUnit decodes onto it) → OmCore::om_req_in[cid], @@ -288,8 +324,7 @@ class Cluster::Impl { // rcache memory side → l2arb. constexpr uint32_t kRasterRow = 1 + VX_CFG_EXT_OM_ENABLED; for (uint32_t i = 0; i < kRcacheMemPorts; ++i) { - rcache->mem_req_out.at(i).bind(&l2arb->ReqIn.at(kL2Rows * i + kRasterRow)); - l2arb->RspOut.at(kL2Rows * i + kRasterRow).bind(&rcache->mem_rsp_in.at(i)); + bind_gfx_port(kL2Rows * i + kRasterRow, CMMU_RCACHE_IDX, rcache->mem_req_out.at(i), rcache->mem_rsp_in.at(i)); } // Cluster-level RasterBus arbiter: NUM_CORES_PER_CLUSTER inputs (one per @@ -348,6 +383,11 @@ class Cluster::Impl { if (l2tlb_->busy()) { return true; } + #endif + #ifdef SIMX_CLUSTER_UNIT_MMU + if (!gfx_mmu_->empty()) { + return true; + } #endif return false; } @@ -461,6 +501,18 @@ class Cluster::Impl { socket->set_satp(value); #endif } + #ifdef SIMX_CLUSTER_UNIT_MMU + // The rcache and ocache are virtually tagged: a satp commit flushes the + // cluster unit TLB and invalidates them. The ocache writes through, and a + // commit falls between launches, so it holds no write to drain first. + gfx_mmu_->set_satp(value); + #ifdef VX_CFG_EXT_RASTER_ENABLE + rcache_->invalidate(); + #endif + #ifdef VX_CFG_EXT_OM_ENABLE + ocache_->invalidate(); + #endif + #endif } SimChannel& ptw_req_out() { @@ -641,6 +693,9 @@ class Cluster::Impl { uint32_t cores_per_socket_; #ifdef VX_CFG_VM_ENABLE L2Tlb::Ptr l2tlb_; +#ifdef SIMX_CLUSTER_UNIT_MMU + Mmu::Ptr gfx_mmu_; +#endif #endif #ifdef VX_CFG_EXT_OM_ENABLE OmCore::Ptr om_core_; diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 0b96539cbd..ea6b196cfe 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -100,6 +100,14 @@ inline constexpr uint32_t UMMU_RTCACHE_IDX = UMMU_TCACHE_IDX + VX_CFG_EXT_TEX_EN inline constexpr uint32_t UMMU_DXA_IDX = UMMU_RTCACHE_IDX + VX_CFG_EXT_RTU_ENABLED; inline constexpr uint32_t UMMU_NUM_REQS = VX_CFG_EXT_TEX_ENABLED + VX_CFG_EXT_RTU_ENABLED + VX_CFG_EXT_DXA_ENABLED; +// Under VM, the cluster unit MMU translates the rcache and ocache memory ports. +#if defined(VX_CFG_VM_ENABLE) && (defined(VX_CFG_EXT_RASTER_ENABLE) || defined(VX_CFG_EXT_OM_ENABLE)) +#define SIMX_CLUSTER_UNIT_MMU +#endif +inline constexpr uint32_t CMMU_RCACHE_IDX = 0; +inline constexpr uint32_t CMMU_OCACHE_IDX = CMMU_RCACHE_IDX + VX_CFG_EXT_RASTER_ENABLED; +inline constexpr uint32_t CMMU_NUM_REQS = VX_CFG_EXT_RASTER_ENABLED + VX_CFG_EXT_OM_ENABLED; + // Demand clients of the cluster L2 (sockets and gfx caches), excluding the // walker's PTE-fetch slot. inline constexpr uint32_t VX_CFG_L2_DEMAND_REQS = NUM_SOCKETS * VX_CFG_L1_MEM_PORTS; diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index a6f255e713..1ba76afe9e 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -830,37 +830,6 @@ vx_result_t Device::cp_poll_seqnum_(uint64_t target) { } vx_result_t Device::cp_submit_dcr_write(uint32_t addr, uint32_t value) { - // VM safety check: when VM is active and the pinned slab is configured, - // every HW-addr DCR must reference a buffer in the pinned slab — the HW - // master bypasses the MMU and would otherwise dereference a stale VA. - // TEX (and DXA, RTU) translate through the socket unit MMU instead, so their - // addresses are VAs like any buffer address. - // - // RASTER / OM address DCRs encode value as a cache-block index - // (pa = value << 6). - if (vm_enabled_ && pinned_mem_) { - switch (addr) { - case VX_DCR_RASTER_TBUF_ADDR: - case VX_DCR_RASTER_PBUF_ADDR: - case VX_DCR_OM_CBUF_ADDR: - case VX_DCR_OM_ZBUF_ADDR: { - const uint64_t pa = uint64_t(value) << 6; - if (pa < pinned_base_ || - pa >= pinned_base_ + pinned_size_) { - std::cerr << "[VXDRV] dcr 0x" << std::hex << addr - << " value 0x" << value << " (pa 0x" << pa - << ") not in pinned slab [0x" << pinned_base_ - << ", 0x" << (pinned_base_ + pinned_size_) - << ") — buffer needs VX_MEM_PHYS" - << std::dec << std::endl; - return VX_ERR_INVALID_VALUE; - } - break; - } - default: - break; - } - } // CMD_DCR_WRITE on-wire layout (cmd_size=20): // bytes 0..3 header { opcode=0x04, flags=0, reserved=0 } // bytes 4..11 arg0 DCR addr diff --git a/sw/runtime/common/graphics.cpp b/sw/runtime/common/graphics.cpp index 8b3d055dc1..7b44ead983 100644 --- a/sw/runtime/common/graphics.cpp +++ b/sw/runtime/common/graphics.cpp @@ -310,7 +310,7 @@ vx_result_t FrontEndPool::init(vx_device_h dev, vx_kernel_h setup_k, return vx_buffer_address(b, out_addr); }; const uint32_t W = VX_MEM_WRITE; // host-write-intent scratch - const uint32_t RWP = VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS; // pinned FF-read outputs + const uint32_t RW = VX_MEM_READ | VX_MEM_WRITE; // FF-read outputs vx_result_t r; // No slot_prim/slot_bbox scratch: setup_k counts survivors and EMIT recomputes @@ -319,9 +319,9 @@ vx_result_t FrontEndPool::init(vx_device_h dev, vx_kernel_h setup_k, // Two-heap residency split: the device-only shader scratch regions live // in ONE pooled slab (collapsing 12 separate allocations into a single resident - // buffer), while the FF-pinned-PA outputs the raster unit reads (prim, tilebuf) - // keep their own VX_MEM_PHYS buffers — both because they need the pinned heap and - // because callers read them back at offset 0 (prim_buffer()/tile_buffer()). + // buffer), while the outputs the raster unit reads (prim, tilebuf) keep their + // own buffers, because callers read them back at offset 0 + // (prim_buffer()/tile_buffer()). constexpr uint64_t SLAB_ALIGN = 64; // sub-region alignment (cache line) auto align_up = [](uint64_t v, uint64_t a) { return (v + a - 1) / a * a; }; @@ -353,9 +353,9 @@ vx_result_t FrontEndPool::init(vx_device_h dev, vx_kernel_h setup_k, } // FF-pinned-PA outputs (read by the raster unit; read back by the host at off 0). - if ((r = mk(NT * MS * PRIM_SZ, RWP, &impl_->prim)) != VX_SUCCESS) return r; + if ((r = mk(NT * MS * PRIM_SZ, RW, &impl_->prim)) != VX_SUCCESS) return r; impl_->prim_buf = impl_->bufs.back(); - if ((r = mk(B * HDR_SZ + (uint64_t)keys_cap * 4, RWP, &impl_->tilebuf)) != VX_SUCCESS) return r; + if ((r = mk(B * HDR_SZ + (uint64_t)keys_cap * 4, RW, &impl_->tilebuf)) != VX_SUCCESS) return r; impl_->tile_buf = impl_->bufs.back(); return VX_SUCCESS; } diff --git a/sw/runtime/common/vm.cpp b/sw/runtime/common/vm.cpp index 2286093e9d..051fed853f 100644 --- a/sw/runtime/common/vm.cpp +++ b/sw/runtime/common/vm.cpp @@ -34,6 +34,12 @@ uint32_t pte_flags_from_access(uint32_t access_flags) { return pte; } +// The OM aperture is decoded by address ahead of translation (the LSU steers +// it to the OM unit), so a randomized VA must never land in it. +bool overlaps_om_aperture(uint64_t va, uint64_t size) { + return va < uint64_t(VX_MEM_OM_END_ADDR) && (va + size) > uint64_t(VX_MEM_OM_BASE_ADDR); +} + // Ceil-log2 — used for the per-level VPN field width. constexpr unsigned vm_clog2(uint64_t n) { unsigned r = 0; @@ -175,7 +181,7 @@ uint64_t VMManager::map_p2v(uint64_t ppn, uint32_t flags) { uint64_t random_page_offset = dist(rng_); uint64_t candidate_va = va_range_start + (random_page_offset << VX_VM_PAGE_LOG2_SIZE); - bool in_use = false; + bool in_use = overlaps_om_aperture(candidate_va, VX_VM_PAGE_SIZE); for (const auto& mapping : addr_mapping) { if (mapping.second == (candidate_va >> VX_VM_PAGE_LOG2_SIZE)) { in_use = true; @@ -239,7 +245,8 @@ int VMManager::phy_to_virt_map(uint64_t size, uint64_t* dev_pAddr, uint32_t flag uint64_t random_page_offset = dist(rng_); uint64_t candidate_va = va_range_start + (random_page_offset << VX_VM_PAGE_LOG2_SIZE); - bool range_available = true; + bool range_available = + !overlaps_om_aperture(candidate_va, num_pages << VX_VM_PAGE_LOG2_SIZE); for (uint64_t i = 0; i < num_pages && range_available; ++i) { uint64_t test_vpn = (candidate_va >> VX_VM_PAGE_LOG2_SIZE) + i; for (const auto& mapping : addr_mapping) { diff --git a/tests/graphics/gfx_draw3d/main.cpp b/tests/graphics/gfx_draw3d/main.cpp index 3d0f2841ea..fd61f639c8 100644 --- a/tests/graphics/gfx_draw3d/main.cpp +++ b/tests/graphics/gfx_draw3d/main.cpp @@ -1,15 +1,14 @@ // 3D rendering regression test: full TEX + RASTER + OM pipeline // against a CGLTrace replay. // -// Pinned-buffer contract: -// tile_buffer → RASTER HW (VX_DCR_RASTER_TBUF_ADDR) → pinned -// prim_buffer → RASTER HW (VX_DCR_RASTER_PBUF_ADDR) → pinned -// tex_buffer → TEX HW (VX_DCR_TEX_ADDR) → pinned -// depth_buffer → OM HW (VX_DCR_OM_ZBUF_ADDR) → pinned -// color_buffer → OM HW (VX_DCR_OM_CBUF_ADDR) → pinned -// All five buffers are accessed by fixed-function HW whose AXI master -// bypasses the per-core MMU, so under VM they must be -// identity-mapped (allocated with VX_MEM_PHYS). +// Buffers read or written by fixed-function HW: +// tile_buffer → RASTER HW (VX_DCR_RASTER_TBUF_ADDR) +// prim_buffer → RASTER HW (VX_DCR_RASTER_PBUF_ADDR) +// tex_buffer → TEX HW (VX_DCR_TEX_ADDR) +// depth_buffer → OM HW (VX_DCR_OM_ZBUF_ADDR) +// color_buffer → OM HW (VX_DCR_OM_CBUF_ADDR) +// Under VM their caches translate through the socket / cluster unit MMUs, +// so all five are ordinary buffers at minted virtual addresses. #include #include @@ -326,10 +325,10 @@ int render(const CGLTrace& trace) { if (tile_buffer != nullptr) { vx_buffer_release(tile_buffer); tile_buffer = nullptr; } if (prim_buffer != nullptr) { vx_buffer_release(prim_buffer); prim_buffer = nullptr; } // With the FF unit present, tile_buffer / prim_buffer are bound to the raster - // unit (via VX_DCR_RASTER_T/PBUF_ADDR) which bypasses the per-core MMU. - // Software-routed, the shader walks the primitives through the LSU instead + // unit (via VX_DCR_RASTER_T/PBUF_ADDR), whose rcache translates through the + // cluster unit MMU under VM. Software-routed, the shader walks the primitives through the LSU instead // and the tile buffer goes unused (the SW walk is binning-independent). - uint32_t rast_flags = VX_MEM_READ | (ff_raster ? (uint32_t)VX_MEM_PHYS : 0u); + uint32_t rast_flags = VX_MEM_READ; RT_CHECK(vx_buffer_create(device, tilebuf.size(), rast_flags, &tile_buffer)); RT_CHECK(vx_buffer_address(tile_buffer, &tilebuf_addr)); RT_CHECK(vx_buffer_create(device, primbuf.size(), rast_flags, &prim_buffer)); @@ -474,9 +473,10 @@ int render(const CGLTrace& trace) { // allocate texture memory if (tex_buffer != nullptr) { vx_buffer_release(tex_buffer); tex_buffer = nullptr; } - // With the FF unit present, tex_buffer is bound to it (VX_DCR_TEX_ADDR) and - // bypasses the MMU; software-routed, the shader loads it through the LSU. - uint32_t tex_flags = VX_MEM_READ | (ff_tex ? (uint32_t)VX_MEM_PHYS : 0u); + // With the FF unit present, tex_buffer is bound to it (VX_DCR_TEX_ADDR), + // whose tcache translates through the socket unit MMU under VM; + // software-routed, the shader loads it through the LSU. + uint32_t tex_flags = VX_MEM_READ; RT_CHECK(vx_buffer_create(device, texbuf.size(), tex_flags, &tex_buffer)); RT_CHECK(vx_buffer_address(tex_buffer, &texbuf_addr)); std::cout << "tex_buffer=0x" << std::hex << texbuf_addr << std::dec << std::endl; @@ -753,9 +753,10 @@ int main(int argc, char *argv[]) { cbuf_size = dst_width * cbuf_pitch; // With the FF unit present, depth_buffer / color_buffer are bound to the OM - // unit (via VX_DCR_OM_Z/CBUF_ADDR) and bypass the MMU; software-routed, the - // shader read-modify-writes them through the LSU. - uint32_t om_flags = VX_MEM_READ_WRITE | (ff_om ? (uint32_t)VX_MEM_PHYS : 0u); + // unit (via VX_DCR_OM_Z/CBUF_ADDR), whose ocache translates through the + // cluster unit MMU under VM; software-routed, the shader read-modify-writes + // them through the LSU. + uint32_t om_flags = VX_MEM_READ_WRITE; RT_CHECK(vx_buffer_create(device, zbuf_size, om_flags, &depth_buffer)); RT_CHECK(vx_buffer_address(depth_buffer, &zbuf_addr)); RT_CHECK(vx_buffer_create(device, cbuf_size, om_flags, &color_buffer)); diff --git a/tests/graphics/gfx_om/main.cpp b/tests/graphics/gfx_om/main.cpp index 0d5bbae377..11cc1a2ffe 100644 --- a/tests/graphics/gfx_om/main.cpp +++ b/tests/graphics/gfx_om/main.cpp @@ -1,10 +1,10 @@ // Output-merger regression test. // -// Pinned-buffer contract: -// depth_buffer → OM HW (VX_DCR_OM_ZBUF_ADDR) → VX_MEM_PHYS (identity-mapped) -// color_buffer → OM HW (VX_DCR_OM_CBUF_ADDR) → VX_MEM_PHYS (identity-mapped) -// Both are read+written by the OM fixed-function block, which bypasses -// the per-core MMU; under VM the buffers must be identity-mapped. +// Buffers: +// depth_buffer → OM HW (VX_DCR_OM_ZBUF_ADDR) +// color_buffer → OM HW (VX_DCR_OM_CBUF_ADDR) +// Both are read+written by the OM fixed-function block through the ocache, +// whose misses translate through the cluster unit MMU under VM. #include #include @@ -173,12 +173,9 @@ int main(int argc, char *argv[]) { cbuf_pitch = dst_width * 4; cbuf_size = dst_height * cbuf_pitch; - // HW path: depth_buffer / color_buffer are bound to the OM unit (via - // VX_DCR_OM_ZBUF_ADDR / VX_DCR_OM_CBUF_ADDR), which reads/writes them through - // the OM/ocache path bypassing the per-core MMU — both need physical (pinned) - // addresses. SW path (-S): the kernel does the merge through the LSU/dcache, - // so allocate them as normal kernel-accessed buffers. - uint32_t om_flags = VX_MEM_READ_WRITE | (sw_path ? 0u : (uint32_t)VX_MEM_PHYS); + // HW path: the OM unit reads/writes them through the ocache (translated by + // the cluster unit MMU); SW path (-S): the kernel merges through the LSU. + uint32_t om_flags = VX_MEM_READ_WRITE; RT_CHECK(vx_buffer_create(device, zbuf_size, om_flags, &depth_buffer)); RT_CHECK(vx_buffer_address(depth_buffer, &zbuf_addr)); RT_CHECK(vx_buffer_create(device, cbuf_size, om_flags, &color_buffer)); diff --git a/tests/graphics/gfx_pipeline_om/main.cpp b/tests/graphics/gfx_pipeline_om/main.cpp index 6aa21eab7c..0300fe12e8 100644 --- a/tests/graphics/gfx_pipeline_om/main.cpp +++ b/tests/graphics/gfx_pipeline_om/main.cpp @@ -1,7 +1,7 @@ // gfx_v2 device front end -> RASTER, end to end. // // The fused setup+binning pipeline runs on the SIMT cores to produce RASTER's -// tilebuf + primbuf (pinned), which are bound to the RASTER unit and rasterised +// tilebuf + primbuf, which are bound to the RASTER unit and rasterised // by a trivial fragment kernel. Two validations: (1) the device buffers match // host Binning() (localises any front-end bug), and (2) the rendered image // matches the gfx-v1 reference PNG (proves the device front end drives the FF @@ -156,11 +156,11 @@ int main(int argc, char** argv) { RT_CHECK(vx_module_get_kernel(mod, "binning_k", &k_binning)); RT_CHECK(vx_module_get_kernel(mod, "main", &k_frag)); - // OM writes the color buffer via its AXI master (bypasses the MMU) -> pinned. + // OM writes the color buffer through the ocache. uint32_t cbuf_stride = 4, cbuf_pitch = dst_width * cbuf_stride, cbuf_size = dst_height * cbuf_pitch; vx_buffer_h color_buffer = nullptr; uint64_t cbuf_addr = 0; - RT_CHECK(vx_buffer_create(dev, cbuf_size, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &color_buffer)); + RT_CHECK(vx_buffer_create(dev, cbuf_size, VX_MEM_READ | VX_MEM_WRITE, &color_buffer)); RT_CHECK(vx_buffer_address(color_buffer, &cbuf_addr)); { std::vector clr(cbuf_size / 4, 0xff000000); vx_event_h e = nullptr; RT_CHECK(vx_enqueue_write(q, color_buffer, 0, clr.data(), cbuf_size, 0, nullptr, &e)); @@ -195,7 +195,7 @@ int main(int argc, char** argv) { const size_t PRIM_SZ = sizeof(rast_prim_t), HDR_SZ = sizeof(rast_bin_header_t); const size_t TILEBUF_SZ = (size_t)B * HDR_SZ + (size_t)Kcap * sizeof(uint32_t); - // Buffers. prim/tilebuf are pinned (RASTER's AXI master reads them). + // Buffers. The raster unit reads prim/tilebuf through the rcache. vx_buffer_h verts_b, slot_prim_b, slot_bbox_b, keep_b, offset_b, tsum_b, prim_b, bbox_b, bcount_b, boffset_b, keys_b, btsum_b, thist_b, bincount_b, binbase_b, tilebuf_b, meta_b; RT_CHECK(vx_buffer_create(dev, 3 * ntri * sizeof(setup_vertex_t), VX_MEM_READ, &verts_b)); @@ -204,7 +204,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_buffer_create(dev, ntri * sizeof(uint32_t), VX_MEM_WRITE, &keep_b)); RT_CHECK(vx_buffer_create(dev, (ntri + 1) * sizeof(uint32_t), VX_MEM_WRITE, &offset_b)); RT_CHECK(vx_buffer_create(dev, T * sizeof(uint32_t), VX_MEM_WRITE, &tsum_b)); - RT_CHECK(vx_buffer_create(dev, ntri * MS * PRIM_SZ, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &prim_b)); + RT_CHECK(vx_buffer_create(dev, ntri * MS * PRIM_SZ, VX_MEM_READ | VX_MEM_WRITE, &prim_b)); RT_CHECK(vx_buffer_create(dev, ntri * MS * sizeof(setup_bbox_t), VX_MEM_WRITE, &bbox_b)); RT_CHECK(vx_buffer_create(dev, ntri * MS * sizeof(uint32_t), VX_MEM_WRITE, &bcount_b)); RT_CHECK(vx_buffer_create(dev, (ntri * MS + 1) * sizeof(uint32_t), VX_MEM_WRITE, &boffset_b)); @@ -213,7 +213,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_buffer_create(dev, T * B * sizeof(uint32_t), VX_MEM_WRITE, &thist_b)); RT_CHECK(vx_buffer_create(dev, B * sizeof(uint32_t), VX_MEM_WRITE, &bincount_b)); RT_CHECK(vx_buffer_create(dev, B * sizeof(uint32_t), VX_MEM_WRITE, &binbase_b)); - RT_CHECK(vx_buffer_create(dev, TILEBUF_SZ, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &tilebuf_b)); + RT_CHECK(vx_buffer_create(dev, TILEBUF_SZ, VX_MEM_READ | VX_MEM_WRITE, &tilebuf_b)); RT_CHECK(vx_buffer_create(dev, 3 * sizeof(uint32_t), VX_MEM_WRITE, &meta_b)); pipe_arg_t pa = {}; diff --git a/tests/graphics/gfx_pipeline_raster/main.cpp b/tests/graphics/gfx_pipeline_raster/main.cpp index 3138be19bb..f314dce328 100644 --- a/tests/graphics/gfx_pipeline_raster/main.cpp +++ b/tests/graphics/gfx_pipeline_raster/main.cpp @@ -1,7 +1,7 @@ // gfx_v2 device front end -> RASTER, end to end. // // The fused setup+binning pipeline runs on the SIMT cores to produce RASTER's -// tilebuf + primbuf (pinned), which are bound to the RASTER unit and rasterised +// tilebuf + primbuf, which are bound to the RASTER unit and rasterised // by a trivial fragment kernel. Two validations: (1) the device buffers match // host Binning() (localises any front-end bug), and (2) the rendered image // matches the gfx-v1 reference PNG (proves the device front end drives the FF @@ -157,7 +157,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_module_get_kernel(mod, "binning_k", &k_binning)); RT_CHECK(vx_module_get_kernel(mod, "main", &k_frag)); - // color buffer (kernel-written via LSU; not pinned) + // color buffer (kernel-written via LSU) uint32_t cbuf_stride = 4, cbuf_pitch = dst_width * cbuf_stride, cbuf_size = dst_height * cbuf_pitch; vx_buffer_h color_buffer = nullptr; uint64_t cbuf_addr = 0; @@ -196,7 +196,7 @@ int main(int argc, char** argv) { const size_t PRIM_SZ = sizeof(rast_prim_t), HDR_SZ = sizeof(rast_bin_header_t); const size_t TILEBUF_SZ = (size_t)B * HDR_SZ + (size_t)Kcap * sizeof(uint32_t); - // Buffers. prim/tilebuf are pinned (RASTER's AXI master reads them). + // Buffers. The raster unit reads prim/tilebuf through the rcache. vx_buffer_h verts_b, slot_prim_b, slot_bbox_b, keep_b, offset_b, tsum_b, prim_b, bbox_b, bcount_b, boffset_b, keys_b, btsum_b, thist_b, bincount_b, binbase_b, tilebuf_b, meta_b; RT_CHECK(vx_buffer_create(dev, 3 * ntri * sizeof(setup_vertex_t), VX_MEM_READ, &verts_b)); @@ -205,7 +205,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_buffer_create(dev, ntri * sizeof(uint32_t), VX_MEM_WRITE, &keep_b)); RT_CHECK(vx_buffer_create(dev, (ntri + 1) * sizeof(uint32_t), VX_MEM_WRITE, &offset_b)); RT_CHECK(vx_buffer_create(dev, T * sizeof(uint32_t), VX_MEM_WRITE, &tsum_b)); - RT_CHECK(vx_buffer_create(dev, ntri * MS * PRIM_SZ, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &prim_b)); + RT_CHECK(vx_buffer_create(dev, ntri * MS * PRIM_SZ, VX_MEM_READ | VX_MEM_WRITE, &prim_b)); RT_CHECK(vx_buffer_create(dev, ntri * MS * sizeof(setup_bbox_t), VX_MEM_WRITE, &bbox_b)); RT_CHECK(vx_buffer_create(dev, ntri * MS * sizeof(uint32_t), VX_MEM_WRITE, &bcount_b)); RT_CHECK(vx_buffer_create(dev, (ntri * MS + 1) * sizeof(uint32_t), VX_MEM_WRITE, &boffset_b)); @@ -214,7 +214,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_buffer_create(dev, T * B * sizeof(uint32_t), VX_MEM_WRITE, &thist_b)); RT_CHECK(vx_buffer_create(dev, B * sizeof(uint32_t), VX_MEM_WRITE, &bincount_b)); RT_CHECK(vx_buffer_create(dev, B * sizeof(uint32_t), VX_MEM_WRITE, &binbase_b)); - RT_CHECK(vx_buffer_create(dev, TILEBUF_SZ, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &tilebuf_b)); + RT_CHECK(vx_buffer_create(dev, TILEBUF_SZ, VX_MEM_READ | VX_MEM_WRITE, &tilebuf_b)); RT_CHECK(vx_buffer_create(dev, 3 * sizeof(uint32_t), VX_MEM_WRITE, &meta_b)); pipe_arg_t pa = {}; diff --git a/tests/graphics/gfx_pipeline_tex/main.cpp b/tests/graphics/gfx_pipeline_tex/main.cpp index e5ccf748fc..1aa25eeb61 100644 --- a/tests/graphics/gfx_pipeline_tex/main.cpp +++ b/tests/graphics/gfx_pipeline_tex/main.cpp @@ -164,13 +164,13 @@ int main(int argc, char** argv) { RT_CHECK(vx_module_get_kernel(mod, "binning_k", &k_binning)); RT_CHECK(vx_module_get_kernel(mod, "main", &k_frag)); - // OM color buffer (pinned: OM's AXI master writes it). + // OM color buffer (the OM writes it through the ocache). const uint32_t cbuf_stride = 4, cbuf_pitch = dst_width * cbuf_stride, cbuf_size = dst_height * cbuf_pitch; vx_buffer_h cbuf_b = nullptr; uint64_t cbuf_addr = 0; - RT_CHECK(vx_buffer_create(dev, cbuf_size, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &cbuf_b)); + RT_CHECK(vx_buffer_create(dev, cbuf_size, VX_MEM_READ | VX_MEM_WRITE, &cbuf_b)); RT_CHECK(vx_buffer_address(cbuf_b, &cbuf_addr)); - // Texture: checkerboard -> mipmaps -> pinned tex buffer (TEX's AXI master). + // Texture: checkerboard -> mipmaps -> tex buffer (read through the tcache). const uint32_t tw = 64, th = 64; std::vector tex_pixels = gen_texture(tw, th); std::vector texbuf; std::vector mip_offsets; @@ -272,8 +272,8 @@ int main(int argc, char** argv) { // reads. std::vector a_binbuf = repack_bins(gold); vx_buffer_h a_tb = nullptr, a_pb = nullptr; uint64_t a_tb_addr = 0, a_pb_addr = 0; - RT_CHECK(vx_buffer_create(dev, a_binbuf.size(), VX_MEM_READ | VX_MEM_PHYS, &a_tb)); - RT_CHECK(vx_buffer_create(dev, a_primbuf.size(), VX_MEM_READ | VX_MEM_PHYS, &a_pb)); + RT_CHECK(vx_buffer_create(dev, a_binbuf.size(), VX_MEM_READ, &a_tb)); + RT_CHECK(vx_buffer_create(dev, a_primbuf.size(), VX_MEM_READ, &a_pb)); RT_CHECK(vx_buffer_address(a_tb, &a_tb_addr)); RT_CHECK(vx_buffer_address(a_pb, &a_pb_addr)); RT_CHECK(vx_enqueue_write(q, a_tb, 0, a_binbuf.data(), a_binbuf.size(), 0, nullptr, nullptr)); @@ -408,7 +408,7 @@ int main(int argc, char** argv) { const uint32_t zbuf_pitch = dst_width * 4, zbuf_size = dst_height * zbuf_pitch; vx_buffer_h zbuf_b = nullptr; uint64_t zbuf_addr = 0; - RT_CHECK(vx_buffer_create(dev, zbuf_size, VX_MEM_READ_WRITE | VX_MEM_PHYS, &zbuf_b)); + RT_CHECK(vx_buffer_create(dev, zbuf_size, VX_MEM_READ_WRITE, &zbuf_b)); RT_CHECK(vx_buffer_address(zbuf_b, &zbuf_addr)); frag_arg_t fa_d = fa; fa_d.prim_addr = prim_addr; fa_d.depth_enabled = 1; diff --git a/tests/graphics/gfx_raster/main.cpp b/tests/graphics/gfx_raster/main.cpp index 5ca49d0a68..eaaa27e6c1 100644 --- a/tests/graphics/gfx_raster/main.cpp +++ b/tests/graphics/gfx_raster/main.cpp @@ -1,12 +1,11 @@ // Vortex rasterizer regression test. // -// Pinned-buffer layout: -// tile_buffer → RASTER HW (VX_DCR_RASTER_TBUF_ADDR) → pinned -// prim_buffer → RASTER HW (VX_DCR_RASTER_PBUF_ADDR) → pinned -// color_buffer → kernel-only (LSU stores from shaded quads) → not pinned -// The raster fixed-function block reads tile_buffer + prim_buffer via its -// own AXI master (no per-core MMU). The kernel writes color_buffer via -// the LSU, which goes through the MMU — no pinning needed. +// Buffers: +// tile_buffer → RASTER HW (VX_DCR_RASTER_TBUF_ADDR) +// prim_buffer → RASTER HW (VX_DCR_RASTER_PBUF_ADDR) +// color_buffer → kernel-only (LSU stores from shaded quads) +// The raster fixed-function block reads tile_buffer + prim_buffer through the +// rcache, whose misses translate through the cluster unit MMU under VM. #include #include @@ -212,15 +211,12 @@ int render(const CGLTrace& trace) { // allocate tile memory if (tile_buffer != nullptr) { vx_buffer_release(tile_buffer); tile_buffer = nullptr; } if (prim_buffer != nullptr) { vx_buffer_release(prim_buffer); prim_buffer = nullptr; } - // HW path: tile_buffer / prim_buffer are bound to the raster unit (via - // VX_DCR_RASTER_T/PBUF_ADDR), which reads them through its own AXI master - // bypassing the per-core MMU — both need physical (pinned) addresses. SW path - // (-z): the kernel reads prim_buffer through the LSU, so allocate it as a - // normal kernel-read buffer (tile_buffer is unused by the SW walk). - uint32_t prim_flags = VX_MEM_READ | (use_sw ? 0u : (uint32_t)VX_MEM_PHYS); - RT_CHECK(vx_buffer_create(device, tilebuf.size(), VX_MEM_READ | VX_MEM_PHYS, &tile_buffer)); + // HW path: the raster unit reads them through the rcache (translated by + // the cluster unit MMU); SW path (-z): the kernel reads prim_buffer through + // the LSU (tile_buffer is unused by the SW walk). + RT_CHECK(vx_buffer_create(device, tilebuf.size(), VX_MEM_READ, &tile_buffer)); RT_CHECK(vx_buffer_address(tile_buffer, &tilebuf_addr)); - RT_CHECK(vx_buffer_create(device, primbuf.size(), prim_flags, &prim_buffer)); + RT_CHECK(vx_buffer_create(device, primbuf.size(), VX_MEM_READ, &prim_buffer)); RT_CHECK(vx_buffer_address(prim_buffer, &primbuf_addr)); std::cout << "tile_buffer=0x" << std::hex << tilebuf_addr << std::dec << std::endl; std::cout << "prim_buffer=0x" << std::hex << primbuf_addr << std::dec << std::endl; From 74b523a296de50e90fcc22659a87cde74bd32585 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 06:39:27 -0800 Subject: [PATCH 310/336] rams: fold asynchronous reads into block RAM in the synthesis sources Since b6263ad26 Vivado synthesized CIRCT's whole-design netlist, so that seq-retime-firmem-reads could fold a register from another module into an asynchronous block RAM. On that netlist Vivado lost Fmax (core 295 -> 286 MHz, cache 300 -> 276, TCU 291.5 -> 276) and its FSM extraction, and top did not build. Deciding a fold needs the whole design; applying it does not need a new netlist. hw/scripts/ram_fold.py elaborates the preprocessed sources with slang (CIRCT's front end), follows each block-RAM-sized combinational VX_dp_ram read back to the registers that feed it, in any module, and patches the copies: every signal on the path gets a twin carrying its next value (a register's twin copies its clocked block, keeping only its own assignments), ports and interface signals get twins, and the RAM takes the registered-address form, which reads exactly what the combinational read did. Edits only add code, a patch that adds an elaboration error is dropped, and a RAM the pass cannot fold soundly stays as written, with the reason in ram_fold.rpt. Every synthesis flow but Altera OPAE reads the folded preprocessed sources (gen_sources.sh -P -F; Foldsim and FireSim fold after swapping in their shell modules); Yosys, Beacon and Foldsim elaborate them with CIRCT. The flows need pyslang 11 or 12 (RAM_FOLD_PYTHON); CI installs it. The RAM library gives each read form one behaviour in simulation and synthesis: the simulation-only read-first model of combinational reads, RADDR_REG, RDW_ASSERT and the asynchronous block-RAM branch are gone; VX_sp_ram's single-lane write-first synthesis registered the data while its simulation model registered the address, and now registers the address; on ASIC the SRAM macro serves registered reads only, with a forward of the lanes written at the captured address for the registered-address form. Synthesis results move with this change (Vivado reads RTL again; Yosys and Beacon see the folded RAM): the FPGA, ASIC and logic gates are re-run and their baselines re-recorded after review. Verified: read forms against a behavioural reference, simulation, synthesis and ASIC models of VX_dp_ram and VX_sp_ram (15/15); ci/test_ram_fold.py (4 RAMs folded across an interface array, pipe register, function, concatenations and shared instance bodies, 20k-cycle lockstep, refusals, a known-bad patch caught); rtlsim built from folded preprocessed sources with every combinational read folded where sound and a next-address check, against unfolded: 10 regression apps with identical output and cycle counts, check never fired; the core gate configuration folds the IPDOM store (240x48) with no new slang or Verilator errors. Not run yet: the gates, the Foldsim/FireSim and Altera flows. Co-Authored-By: Claude Opus 5.5 --- .github/actions/setup-vortex/action.yml | 2 +- .github/workflows/fpga_gate.yml | 5 + ci/test_ram_fold.py | 487 ++++++++++ docs/building_toolchain.md | 9 +- docs/coding_guidelines_verilog.md | 3 + docs/designs/foldsim.md | 8 +- docs/hardware_library.md | 100 ++- docs/synthesis_analysis.md | 17 + hw/rtl/core/VX_bar_unit.sv | 3 +- hw/rtl/core/VX_ipdom_stack.sv | 3 +- hw/rtl/libs/VX_dp_ram.sv | 171 ++-- hw/rtl/libs/VX_fifo_queue.sv | 5 +- hw/rtl/libs/VX_sp_ram.sv | 203 ++--- hw/rtl/tcu/VX_tcu_meta.sv | 5 +- hw/scripts/circt_elaborate.sh | 4 +- hw/scripts/gen_sources.sh | 29 +- hw/scripts/ram_fold.py | 1078 +++++++++++++++++++++++ hw/syn/altera/dut/common.mk | 2 +- hw/syn/common.mk | 10 +- hw/syn/firesim/Makefile | 3 + hw/syn/foldsim/Makefile | 3 + hw/syn/synopsys/Makefile | 2 +- hw/syn/xilinx/common.mk | 7 +- hw/syn/xilinx/dut/common.mk | 5 +- hw/syn/xilinx/sandbox/Makefile | 3 +- hw/syn/yosys/Makefile | 2 +- 26 files changed, 1867 insertions(+), 302 deletions(-) create mode 100644 ci/test_ram_fold.py create mode 100644 hw/scripts/ram_fold.py diff --git a/.github/actions/setup-vortex/action.yml b/.github/actions/setup-vortex/action.yml index b27d3c1305..bd5d05da13 100644 --- a/.github/actions/setup-vortex/action.yml +++ b/.github/actions/setup-vortex/action.yml @@ -75,7 +75,7 @@ runs: - name: Install test-harness Python deps shell: bash - run: pip install --quiet pytest pyyaml + run: pip install --quiet pytest pyyaml "pyslang>=11,<13" # Re-export the SST / gem5 / openmpi tool env. ci/{sst,toolchain}_install.sh # emit these to $GITHUB_ENV, but that is per-job — the setup job that built diff --git a/.github/workflows/fpga_gate.yml b/.github/workflows/fpga_gate.yml index ab154187a9..232b517d14 100644 --- a/.github/workflows/fpga_gate.yml +++ b/.github/workflows/fpga_gate.yml @@ -53,6 +53,11 @@ jobs: echo "state=$HOME/.cache/vortex/fpga_gate.$(echo '${{ github.repository }}' | tr / _).sha" >> "$GITHUB_OUTPUT" echo "gating $SHA" + # The synthesis flows fold asynchronous RAM reads into block RAM with + # pyslang (hw/scripts/ram_fold.py); a no-op once installed. + - name: Install pyslang + run: python3 -m pip install --quiet --user "pyslang>=11,<13" + # The command is the one the catalog declares (ci/testcases/fpga_gate.yaml's # single `run:` case), so what CI runs and what the catalog says cannot # drift apart. Dispatch inputs only narrow it, and reach the shell through diff --git a/ci/test_ram_fold.py b/ci/test_ram_fold.py new file mode 100644 index 0000000000..2d2799d48d --- /dev/null +++ b/ci/test_ram_fold.py @@ -0,0 +1,487 @@ +#!/usr/bin/env python3 +"""hw/scripts/ram_fold.py on a small design that crosses every construct the +pass handles, simulated in lockstep against the unfolded design. + +The design reads four block-RAM-sized RAMs, one per generate iteration, at an +address that comes from registers in other modules: a library pipe register +with a partial reset, reached through an interface array, a function, a +concatenated assign target and a declaration initializer, plus a hand-written +always_ff register. The four RAMs live in instances of one module, so slang +analyzes one body and the pass maps the other three onto it. + +Needs Verilator (VERILATOR or PATH), circt-verilog (CIRCT_VERILOG or PATH) and +pyslang >= 11 in RAM_FOLD_PYTHON (default: this interpreter). +""" + +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[1] +FOLD = ROOT / "hw/scripts/ram_fold.py" +GEN = ROOT / "hw/scripts/gen_sources.sh" +LIBS = [ROOT / "hw/rtl/VX_platform.vh", ROOT / "hw/rtl/VX_scope.vh", + ROOT / "hw/rtl/libs/VX_dp_ram.sv", ROOT / "hw/rtl/libs/VX_pipe_register.sv"] + +DESIGN = r''' +`include "VX_platform.vh" + +typedef struct packed { + logic [3:0] hi; + logic [3:0] lo; +} rf_pair_t; + +interface rf_if; + logic [7:0] addr; + rf_pair_t pair; + modport master (output addr, pair); + modport slave (input addr, pair); +endinterface + +function automatic logic [7:0] rf_swz(input logic [7:0] x); + return {x[3:0], x[7:4]}; +endfunction + +module rf_src ( + input wire clk, + input wire reset, + input wire en, + input wire [7:0] din, + input wire [7:0] din2, + rf_if o +); + VX_pipe_register #( + .DATAW (16), + .RESETW (1) + ) pr ( + .clk (clk), + .reset (reset), + .enable (en), + .data_in ({din, din2}), + .data_out ({o.addr, o.pair}) + ); +endmodule + +module rf_leaf #( + parameter SIZE = 256 +) ( + input wire clk, + input wire reset, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + input wire [7:0] raddr, + output wire [15:0] rdata +); + VX_dp_ram #( + .DATAW (16), + .SIZE (SIZE), + .RDW_MODE ("R") + ) ram ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (raddr), + .rdata (rdata) + ); +endmodule + +module rf_leaf_in ( + input wire clk, + input wire reset, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + input wire [7:0] raddr, + output wire [15:0] rdata +); + VX_dp_ram #( + .DATAW (16), + .SIZE (256), + .RDW_MODE ("R") + ) ram ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (raddr), + .rdata (rdata) + ); +endmodule + +// instantiated once foldable and once not: neither instance may fold +module rf_mixed ( + input wire clk, + input wire reset, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + input wire [7:0] raddr, + output wire [15:0] rdata +); + VX_dp_ram #( + .DATAW (16), + .SIZE (256), + .RDW_MODE ("R") + ) ram ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (raddr), + .rdata (rdata) + ); +endmodule + +module rf_mid ( + input wire clk, + input wire reset, + rf_if i, + input wire [7:0] bias, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + output wire [15:0] rdata +); + reg [7:0] cnt_r; + always_ff @(posedge clk) begin + if (reset) begin + cnt_r <= '0; + end else if (write) begin + cnt_r <= cnt_r + 8'd1; + end + end + wire [3:0] lo_part, hi_part; + assign {hi_part, lo_part} = i.pair; + wire [7:0] mixed; + assign mixed = rf_swz(i.addr) ^ {hi_part, lo_part} ^ cnt_r; + wire [7:0] addr = mixed + bias; + rf_leaf leaf ( + .clk (clk), + .reset (reset), + .write (write), + .waddr (waddr), + .wdata (wdata), + .raddr (addr), + .rdata (rdata) + ); +endmodule + +// three addresses the pass must refuse: another clock, a register assigned +// inside a concatenation, a register written by two blocks +module rf_odd ( + input wire clk, + input wire clk2, + input wire reset, + input wire [7:0] din, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + output wire [2:0][15:0] rdata +); + reg [7:0] slow_r, cat_r, tag_r, two_r; + always_ff @(posedge clk2) begin + slow_r <= din; + end + always_ff @(posedge clk) begin + {cat_r, tag_r} <= {din, ~din}; + end + always_ff @(posedge clk) begin + two_r[3:0] <= din[3:0]; + end + always_ff @(posedge clk) begin + two_r[7:4] <= din[7:4]; + end + VX_dp_ram #( + .DATAW (16), + .SIZE (256) + ) u_slow ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (slow_r), + .rdata (rdata[0]) + ); + VX_dp_ram #( + .DATAW (16), + .SIZE (256) + ) u_cat ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (cat_r), + .rdata (rdata[1]) + ); + VX_dp_ram #( + .DATAW (16), + .SIZE (256) + ) u_two ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (two_r), + .rdata (rdata[2]) + ); +endmodule + +module rf_top ( + input wire clk, + input wire clk2, + input wire reset, + input wire en, + input wire [7:0] din0, + input wire [7:0] din1, + input wire [7:0] din2, + input wire [7:0] bias_in, + input wire [7:0] top_addr, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + output wire [9:0][15:0] rdata +); + reg [7:0] bias_r; + always_ff @(posedge clk) begin + bias_r <= bias_in; + end + rf_if ifs [4] (); + for (genvar g = 0; g < 4; ++g) begin : g_lane + rf_src src ( + .clk (clk), + .reset (reset), + .en (en), + .din (din0 + 8'(g)), + .din2 (g[0] ? din1 : din2), + .o (ifs[g]) + ); + rf_mid mid ( + .clk (clk), + .reset (reset), + .i (ifs[g]), + .bias (bias_r), + .write (write), + .waddr (waddr), + .wdata (wdata), + .rdata (rdata[g]) + ); + end + rf_mixed mixed_reg ( + .clk (clk), + .reset (reset), + .write (write), + .waddr (waddr), + .wdata (wdata), + .raddr (bias_r), + .rdata (rdata[5]) + ); + rf_mixed mixed_in ( + .clk (clk), + .reset (reset), + .write (write), + .waddr (waddr), + .wdata (wdata), + .raddr (top_addr), + .rdata (rdata[6]) + ); + rf_odd odd ( + .clk (clk), + .clk2 (clk2), + .reset (reset), + .din (din0), + .write (write), + .waddr (waddr), + .wdata (wdata), + .rdata (rdata[9:7]) + ); + // read straight from a top-level input: nothing to fold + rf_leaf_in direct ( + .clk (clk), + .reset (reset), + .write (write), + .waddr (waddr), + .wdata (wdata), + .raddr (top_addr), + .rdata (rdata[4]) + ); +endmodule +''' + +TB = r''' +module tb; + reg clk = 0, clk2 = 0; + reg reset = 1, en = 1, write = 0; + reg [7:0] din0 = 0, din1 = 0, din2 = 0, bias_in = 0, top_addr = 0, waddr = 0; + reg [15:0] wdata = 0; + wire [9:0][15:0] rdata; + rf_top dut (.*); + always #5 clk = ~clk; + always #7 clk2 = ~clk2; + integer f, c; + initial begin + f = $fopen("trace.txt", "w"); + // every RAM word written before anything is traced + for (c = 0; c < 256; c = c + 1) begin + @(negedge clk); + if (c > 3) reset = 0; + write = 1; waddr = c[7:0]; wdata = 16'(c * 7919); + end + for (c = 0; c < 20000; c = c + 1) begin + @(negedge clk); + $fwrite(f, "%0d %h\n", c, rdata); + en = $urandom_range(3) != 0; + write = $urandom_range(1); + din0 = 8'($urandom); din1 = 8'($urandom); din2 = 8'($urandom); + bias_in = 8'($urandom); top_addr = 8'($urandom); + waddr = 8'($urandom); wdata = 16'($urandom); + end + $fclose(f); + $finish; + end +endmodule +''' + + +def tool(env, name): + path = os.environ.get(env) or shutil.which(name) + return path if path and os.path.exists(path) else None + + +VERILATOR = tool("VERILATOR", "verilator") +CIRCT = tool("CIRCT_VERILOG", "circt-verilog") +PYTHON = os.environ.get("RAM_FOLD_PYTHON", sys.executable) + + +def have_pyslang(): + return subprocess.run([PYTHON, "-c", "import pyslang.analysis"], capture_output=True).returncode == 0 + + +@unittest.skipUnless(VERILATOR and CIRCT and have_pyslang(), + "needs verilator, circt-verilog and pyslang >= 11") +class RamFoldTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.tmp = pathlib.Path(tempfile.mkdtemp(prefix="ram_fold_test_")) + rtl = cls.tmp / "rtl" + rtl.mkdir() + for f in LIBS: + shutil.copy(f, rtl) + (rtl / "rf_design.sv").write_text(DESIGN) + (cls.tmp / "tb.sv").write_text(TB) + env = dict(os.environ, CIRCT_VERILOG=CIRCT, RAM_FOLD_PYTHON=PYTHON) + # the flow's own path: gen_sources.sh preprocesses, then folds (-F) + for name, fold in (("orig", []), ("flow", ["-F"])): + r = subprocess.run(["bash", str(GEN), "-DSIMULATION", "-I" + str(rtl), "-P", *fold, + "-Trf_top", "-C" + str(cls.tmp / name)], + cwd=cls.tmp, env=env, capture_output=True, text=True) + assert r.returncode == 0, r.stdout + r.stderr + if fold: + cls.flow_out = r.stdout + # the same fold with the next-address check, for simulation + shutil.copytree(cls.tmp / "orig", cls.tmp / "chk") + r = subprocess.run([PYTHON, str(FOLD), "--top", "rf_top", "--check", "--diff", "chk.diff", + *sorted(str(p) for p in (cls.tmp / "chk").glob("*.sv"))], + cwd=cls.tmp, capture_output=True, text=True) + assert r.returncode == 0, r.stdout + r.stderr + cls.report = r.stdout + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp, ignore_errors=True) + + def simulate(self, name): + src = self.tmp / name + obj = self.tmp / ("obj_" + name) + files = [str(p) for p in sorted(src.glob("*.sv"))] + r = subprocess.run([VERILATOR, "--binary", "--timing", "-Wno-fatal", "-Wno-lint", "-Wno-style", + "--x-initial", "0", "--top-module", "tb", "-I" + str(src), "-Mdir", str(obj), + *files, str(self.tmp / "tb.sv")], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout[-3000:] + r.stderr[-3000:]) + run = self.tmp / ("run_" + name) + run.mkdir(exist_ok=True) + r = subprocess.run([str(obj / "Vtb"), "+verilator+seed+7"], cwd=run, capture_output=True, text=True) + return r.stdout + r.stderr, (run / "trace.txt").read_text() + + def test_report(self): + folded = [l for l in self.report.splitlines() if l.startswith("fold ")] + kept = [l for l in self.report.splitlines() if l.startswith("keep ")] + self.assertEqual(len(folded), 4, self.report) + self.assertEqual(len(kept), 6, self.report) + self.assertTrue(all(".mid.leaf.ram" in l for l in folded), self.report) + why = {l.split()[1]: l for l in kept} + self.assertIn("top-level input", why["rf_top.direct.ram"]) + self.assertIn("top-level input", why["rf_top.mixed_in.ram"]) + # the foldable instance shares its instantiation with one that is not + self.assertIn("rf_top.mixed_in.ram cannot fold", why["rf_top.mixed_reg.ram"]) + self.assertIn("not the RAM's clock", why["rf_top.odd.u_slow"]) + self.assertIn("assigned as part of", why["rf_top.odd.u_cat"]) + self.assertIn("procedural blocks", why["rf_top.odd.u_two"]) + + def test_flow_patch_is_the_pass_patch(self): + # gen_sources.sh -F folds exactly as the pass does on its own + self.assertEqual(self.flow_out.count("fold "), 4, self.flow_out) + plain = self.tmp / "plain" + shutil.copytree(self.tmp / "orig", plain) + r = subprocess.run([PYTHON, str(FOLD), "--top", "rf_top", + *sorted(str(p) for p in plain.glob("*.sv"))], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + for f in sorted((self.tmp / "flow").glob("*.sv")): + self.assertEqual(f.read_text(), (plain / f.name).read_text(), f.name) + + def test_rams_take_the_registered_address_form(self): + leaf = (self.tmp / "chk" / "rf_design.sv").read_text() + self.assertIn(".OUT_REG (1)", leaf) + self.assertIn('.RDW_MODE ("W")', leaf) + self.assertIn(".raddr (raddr__nx)", leaf) + + def test_lockstep(self): + out_o, trace_o = self.simulate("orig") + out_c, trace_c = self.simulate("chk") + self.assertNotIn("ram_fold:", out_c, out_c[-2000:]) + self.assertEqual(trace_o.count("\n"), 20000) + self.assertTrue(trace_o == trace_c, "folded design diverges from the original") + + def test_known_bad_patch_is_caught(self): + # read the counter's current value instead of its next one: a fold off by a cycle + bad = self.tmp / "bad" + if not bad.exists(): + shutil.copytree(self.tmp / "chk", bad) + d = bad / "rf_design.sv" + text, n = re.subn(r"(assign mixed__nx = .*?)cnt_r__nx", r"\1cnt_r", d.read_text()) + self.assertEqual(n, 1) + d.write_text(text) + out_b, trace_b = self.simulate("bad") + _, trace_o = self.simulate("orig") + self.assertIn("ram_fold:", out_b) + self.assertNotEqual(trace_b, trace_o) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index 2302218fa6..7f38db077a 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -1130,10 +1130,11 @@ for the backend's architecture and ## 14. CIRCT -**Purpose**: the SystemVerilog frontend of every synthesis flow. Each flow -reads the netlist CIRCT elaborates (`hw/scripts/circt_elaborate.sh`, -`gen_sources.sh -E`; see the hardware library's "Block RAM and asynchronous -reads"), and Foldsim builds on it. +**Purpose**: the SystemVerilog frontend of every synthesis flow. Its +preprocessor writes the sources each flow reads (`gen_sources.sh -P`, then the +RAM fold, `-F`; see the hardware library's "Block RAM and asynchronous reads"); +Vivado reads them as they are, and Yosys, Beacon and Foldsim read the netlist +CIRCT elaborates from them (`hw/scripts/circt_elaborate.sh`). > **Baseline:** Vortex's CIRCT fork, branch `vortex_3.x` of > `vortexgpgpu/circt`: upstream CIRCT with the import, lowering and diff --git a/docs/coding_guidelines_verilog.md b/docs/coding_guidelines_verilog.md index 3222a6d340..790601eb11 100644 --- a/docs/coding_guidelines_verilog.md +++ b/docs/coding_guidelines_verilog.md @@ -248,6 +248,9 @@ Strive for moderate combinatorial logic depths that balance latency with synthes ## 11. Reuse the Hardware IP Library Before writing new RTL, consult the hardware IP library in [hw/rtl/libs/](../hw/rtl/libs/) — the [hardware_library.md](hardware_library.md) reference catalogs the reusable, parameterized modules it provides: elastic buffers and flow control, arbiters, mux/demux, stream fork/join/pack/dispatch, crossbars and interconnect, encoders/decoders, arithmetic (multipliers, dividers, adders, CSA trees), RAM/FIFO primitives, memory adapters, and bit-manipulation utilities. Prefer instantiating an existing library module over hand-rolling equivalent logic: the library modules carry consistent valid/ready handshake semantics, inherit the FPGA/ASIC synthesis support, and are already verified, so reuse avoids duplicating tested logic and the subtle handshake/timing bugs that re-implementation invites. If a needed primitive is genuinely missing, add it to the library rather than embedding a one-off in a block. + +**Memories go through `VX_dp_ram` / `VX_sp_ram`.** Never hand-write a RAM array. Choose the read form the design needs; a combinational read (`OUT_REG=0`) of a block-RAM-sized `VX_dp_ram` becomes block RAM through the synthesis flows' RAM fold, which computes its address one cycle early from the registers that feed it, wherever they are. Keep that address computable: drive it from registers on the RAM's clock through assigns, ports, interfaces and functions, not from a top-level input or an `always_comb` block; connect the modules on its path by name; and give a RAM whose read form is a parameter one instantiation per form, since the fold edits the instantiation every instance shares. `/ram_fold.rpt` names every RAM the fold kept and why. See [Block RAM and asynchronous reads](hardware_library.md#block-ram-and-asynchronous-reads). + ## 12. Module & Interface Declarations & Instantiations Declare and instantiate modules **and parameterized interfaces** with one diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md index 39a4805d9c..9f8b46a509 100644 --- a/docs/designs/foldsim.md +++ b/docs/designs/foldsim.md @@ -90,10 +90,10 @@ compiler's `foldsim-find-pipelines`). `hw/scripts/check_ip_contracts.py` checks each contract's latency against the core's generation, the RTL and the configuration; the in-process simulator runs the cores' models (`sim/foldsim/xilinx_fp_models.sv`). `-DVX_CFG_FPU_TYPE_STD` in `CONFIGS` -selects the soft FPU. A RAM that asks for block RAM but reads asynchronously reads -synchronously once the target is flattened (CIRCT `seq-retime-firmem-reads`, -see the hardware library's "Block RAM and asynchronous reads"); -`gen/memories.rpt` lists each such RAM and the decision. +selects the soft FPU. A block-RAM-sized RAM that reads asynchronously gets a registered read +from the RAM fold before the target is lifted (`/ram_fold.rpt`, see the +hardware library's "Block RAM and asynchronous reads"); `gen/memories.rpt` lists +the RAMs that still read asynchronously. --- diff --git a/docs/hardware_library.md b/docs/hardware_library.md index 094d08460a..1b81ed2428 100644 --- a/docs/hardware_library.md +++ b/docs/hardware_library.md @@ -1220,16 +1220,15 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn | `OUT_REG` | 0 | Output register enable | | `LUTRAM` | 0 | Force LUTRAM implementation | | `RDW_MODE` | "W" | Read-during-write mode: "W" (write-first), "R" (read-first) | -| `RADDR_REG` | 0 | Read address registered hint | | `INIT_ENABLE` | 0 | Enable initialization | | `INIT_FILE` | "" | Initialization memory file | **Features:** - Separate read and write ports with independent addresses - Bit-level write enable (`WRENW` granularity) -- Write-first or read-first semantics for simultaneous read/write to same address +- Three read forms: registered address, registered data, combinational (see [Block RAM and asynchronous reads](#block-ram-and-asynchronous-reads)) - FPGA: infers LUTRAM or BRAM based on `LUTRAM` parameter -- ASIC: uses register-based implementation +- ASIC: a registered read of block-RAM size maps to the SRAM macro (`VX_dp_ram_asic`); the registered-address form forwards the lanes written at the captured address around it. Other RAMs are flops - Optional output register for timing improvement - File-based initialization for preloaded content @@ -1254,7 +1253,7 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn **Features:** - Single address shared between read and write - Three read-during-write modes including "N" (no-change) which holds previous read data -- A forced-BRAM, read-first RAM read asynchronously is tagged block RAM (`ram_style = "block"`); the synthesis elaboration makes its read synchronous (see [Block RAM and asynchronous reads](#block-ram-and-asynchronous-reads)) +- Read forms and block-RAM rules as in VX_dp_ram (see [Block RAM and asynchronous reads](#block-ram-and-asynchronous-reads)) - Same FPGA/ASIC implementation paths as VX_dp_ram --- @@ -1305,40 +1304,65 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn ### Block RAM and asynchronous reads -Block RAMs and SRAM macros only read synchronously. A `VX_dp_ram` or -`VX_sp_ram` that must be block RAM (`FORCE_BRAM`) but reads asynchronously -(`OUT_REG=0`, read-first) is written as a plain array with the standard -synthesis attribute `(* ram_style = "block" *)` (`USE_BLOCK_BRAM`) and an -asynchronous read. Every synthesis flow elaborates the design with CIRCT -(`hw/scripts/circt_elaborate.sh`, `gen_sources.sh -E`), whose -`seq-retime-firmem-reads` pass then reads such a RAM one edge earlier, at the -address its address registers are about to take, so that it maps to a block -RAM. The modules between the RAM and its registers are inlined for it, and -only those. - -The attribute is a contract: - -- `ram_style = "block"`: the RAM is converted. If it cannot be (its address - depends on anything but registers and logic, or a register has an - asynchronous reset), the elaboration stops and names the reason; -- `ram_style = "distributed"`: the RAM stays as it is; -- no attribute: the RAM stays as it is (`circt_elaborate.sh -m auto` converts - those it can). - -The converted read is exact on every cycle after the first clock edge. Each -elaboration writes `.memories.rpt`, one line per asynchronous-read -RAM with its size, hint and decision. - -A converted RAM returns the data written in the same cycle, as the -asynchronous read did (write-first). The elaborated netlist states this to -Vivado with `rw_addr_collision = "yes"`, and the ports of every RAM share one -clock net, so Vivado keeps each RAM's read-under-write behavior. A block RAM -whose ports Vivado takes as independently clocked returns undefined bits when -a read meets a write to the same address; simulation never shows it. To -check a netlist, read the block RAM primitives' `WRITE_MODE_A/B` and -`CLOCK_DOMAINS`, not the synthesis report's mapping table. - ---- +Block RAMs and SRAM macros read only synchronously. `VX_dp_ram` and +`VX_sp_ram` have three read forms: + +| Form | Parameters | `rdata` | Implementation | +|------|------------|---------|----------------| +| Registered address | `OUT_REG=1`, `RDW_MODE="W"` | `ram[raddr captured on read]`; sees a write on the same edge | block RAM (with a collision bypass) or distributed RAM | +| Registered data | `OUT_REG=1`, `RDW_MODE="R"` (`"N"` on `VX_sp_ram` also holds it across a write) | `ram[raddr]` captured on read; does not see a write on the same edge | block RAM or distributed RAM | +| Combinational | `OUT_REG=0` | `ram[raddr]` | distributed RAM or flops, unless the RAM fold turns it into the registered-address form | + +The form fixes the behaviour, identically in simulation and synthesis; size +and `LUTRAM` only pick the implementation (`FORCE_BRAM`), locally, inside the +RAM. + +**The RAM fold.** Write a read in whatever form the design needs; a +combinational read of a `VX_dp_ram` is fine at any size. Every synthesis flow +but Altera OPAE (which does not preprocess) runs `hw/scripts/ram_fold.py` on its preprocessed sources (`gen_sources.sh -P +-F`) before any tool reads them. For each block-RAM-sized `VX_dp_ram` that reads +combinationally and does not ask for `LUTRAM`, it computes the read address one +cycle early and gives the RAM the registered-address form, reading on every +cycle (`read` tied high), which reads exactly what the combinational read did: + +- The pass elaborates the design with slang (CIRCT's front end) and follows the + read address back to the registers that feed it, through assigns, ports, + interfaces, struct fields, functions and generate scopes, in any module. +- Each signal on that path gets a twin, `__nx`, carrying its next value: a + copy of the signal's own driver with its operands renamed. A register's twin + is a combinational copy of its clocked block that keeps only the assignments + to that register; their right-hand sides still read current values. Ports + and interface signals on the path get twin ports and members. +- The edits only add code, except the RAM's parameters and address + connection. The patched sources are elaborated again, and the patch is + dropped if it adds an error. + +`/ram_fold.rpt` lists every candidate RAM, folded or kept with the reason; +`/ram_fold.diff` is the patch. A RAM is kept, and stays distributed RAM, when: + +- its address depends on a top-level input, or on a signal driven by an + `always_comb` block or by a clocked block with blocking assignments; +- an instance on the path is an element of an instance array, or connects its + ports by position; +- another RAM made by the same instantiation statement cannot fold, or + registers its read (`OUT_REG` from a parameter, as in `VX_fifo_queue`): the patch + edits the statement, which all its instances share; +- a register on the path is written by more than one block, inside a + concatenation, or on a clock other than the RAM's; +- the address reaches a signal through a module body slang analyzed only as + another instance and the signal lies outside it, or a signal on the path has + no driver. + +`ram_fold.py --check` adds a simulation check that each folded RAM's next +address is its address a cycle later; `ci/test_ram_fold.py` runs a folded design +in lockstep against the original, with a known-bad patch the check must catch. + +The fold needs a python3 with pyslang 11 or 12 (`pip install "pyslang>=11,<13"`), named by +`RAM_FOLD_PYTHON` (default `python3`, `hw/syn/common.mk`). Vivado reads the +folded preprocessed sources, so its own RAM, FSM and shift-register inference +sees the RTL as written. The CIRCT elaboration of the same sources feeds Yosys, +Beacon and Foldsim; its `.memories.rpt` lists the RAMs that still read +combinationally. ### VX_fifo_queue diff --git a/docs/synthesis_analysis.md b/docs/synthesis_analysis.md index af1349ccda..1cbd13a011 100644 --- a/docs/synthesis_analysis.md +++ b/docs/synthesis_analysis.md @@ -64,6 +64,23 @@ NUM_CORES=16 # equivalent to -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=16 -DVX NUM_CORES=32 # equivalent to -DVX_CFG_NUM_CLUSTERS=2 -DVX_CFG_NUM_CORES=16 -DVX_CFG_L2_ENABLE ``` +### Block RAM for asynchronous reads (the RAM fold) + +Every flow reads preprocessed copies of the sources (`gen_sources.sh -P`) that +the RAM fold has patched first (`-F`, [hw/scripts/ram_fold.py](../hw/scripts/ram_fold.py)): +a block-RAM-sized `VX_dp_ram` that reads combinationally gets its read address one +cycle early and a registered read, so it maps to block RAM or an SRAM macro. The +build's source folder holds `ram_fold.rpt` (each candidate RAM, folded or kept +with the reason) and `ram_fold.diff` (the patch). The fold needs a python3 with +pyslang 11 or 12, named by `RAM_FOLD_PYTHON` (default `python3`): + +```bash +pip install "pyslang>=11,<13" +RAM_FOLD_PYTHON=/path/to/python3 make core # when python3 has no pyslang +``` + +See [Block RAM and asynchronous reads](hardware_library.md#block-ram-and-asynchronous-reads). + ### Using PREFIX for Isolated Builds Use `PREFIX=` to keep builds separate. Each flow creates a build directory derived from `PREFIX` so that multiple configurations can coexist without overwriting each other: diff --git a/hw/rtl/core/VX_bar_unit.sv b/hw/rtl/core/VX_bar_unit.sv index 95f0e80c98..f6239012e3 100644 --- a/hw/rtl/core/VX_bar_unit.sv +++ b/hw/rtl/core/VX_bar_unit.sv @@ -229,8 +229,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( VX_dp_ram #( .DATAW (1), .SIZE (1 << BAR_ADDR_BITS), - .RDW_MODE ("W"), - .RADDR_REG(1) + .RDW_MODE ("W") ) barrier_phase_store ( .clk (clk), .reset (reset), diff --git a/hw/rtl/core/VX_ipdom_stack.sv b/hw/rtl/core/VX_ipdom_stack.sv index 270bfe0f25..813af395bd 100644 --- a/hw/rtl/core/VX_ipdom_stack.sv +++ b/hw/rtl/core/VX_ipdom_stack.sv @@ -102,8 +102,7 @@ module VX_ipdom_stack import VX_gpu_pkg::*; #( VX_dp_ram #( .DATAW (BRAM_DATAW), .SIZE (BRAM_SIZE), - .RDW_MODE ("R"), - .RADDR_REG(1) + .RDW_MODE ("R") ) ipdom_store ( .clk (clk), .reset (reset), diff --git a/hw/rtl/libs/VX_dp_ram.sv b/hw/rtl/libs/VX_dp_ram.sv index 8cd451e282..93893811da 100644 --- a/hw/rtl/libs/VX_dp_ram.sv +++ b/hw/rtl/libs/VX_dp_ram.sv @@ -67,6 +67,15 @@ end `endif +// Read forms. The form fixes the behaviour; size and LUTRAM alone pick block or +// distributed RAM. +// OUT_REG=1, RDW_MODE "W": registered address. raddr is the address's next +// value; rdata = ram[raddr captured on read], seeing a write on the same edge. +// OUT_REG=1, RDW_MODE "R": registered data. rdata = ram[raddr] captured on +// read, not seeing a write on the same edge. +// OUT_REG=0: combinational, rdata = ram[raddr]. Never block RAM by itself: the +// synthesis flows' RAM fold (hw/scripts/ram_fold.py) turns a block-RAM-sized +// one into the registered-address form when its address comes from registers. `TRACING_OFF module VX_dp_ram #( parameter DATAW = 1, @@ -75,8 +84,6 @@ module VX_dp_ram #( parameter OUT_REG = 0, parameter LUTRAM = 0, parameter `STRING RDW_MODE = "W", // W: write-first, R: read-first - parameter RADDR_REG = 0, // read address registered hint - parameter RDW_ASSERT = 0, parameter RESET_RAM = 0, parameter INIT_ENABLE = 0, parameter INIT_FILE = "", @@ -95,38 +102,59 @@ module VX_dp_ram #( ); localparam WSELW = DATAW / WRENW; `UNUSED_PARAM (LUTRAM) - `UNUSED_PARAM (RADDR_REG) `STATIC_ASSERT(!(WRENW * WSELW != DATAW), ("invalid parameter")) `STATIC_ASSERT((RDW_MODE == "R" || RDW_MODE == "W"), ("invalid parameter")) - `UNUSED_PARAM (RDW_ASSERT) `ifdef SYNTHESIS localparam FORCE_BRAM = !LUTRAM && `FORCE_BRAM(SIZE, DATAW); - // Async reads have no native block-RAM form: a forced-BRAM RAM read - // asynchronously is tagged block RAM, and the synthesis elaboration makes its - // read synchronous, reading at the address its registers are about to take - // (CIRCT seq-retime-firmem-reads). That only pays off for a read-first RAM; a - // write-first one adds a read/write collision bypass that is larger than - // plain distributed RAM. - localparam FORCE_BRAM_ASYNC = FORCE_BRAM && (OUT_REG == 0) && (RDW_MODE == "R"); `ifdef ASIC - if (FORCE_BRAM && (OUT_REG != 0 || RADDR_REG != 0)) begin : g_asic + // The macro reads synchronously, so a combinational read stays in logic. + if (FORCE_BRAM && OUT_REG != 0) begin : g_asic + `UNUSED_VAR (reset) + wire [WRENW-1:0] wr_lanes = (WRENW != 1) ? wren : {WRENW{1'b1}}; + wire [DATAW-1:0] macro_rdata; VX_dp_ram_asic #( .DATAW (DATAW), .SIZE (SIZE), .WRENW (WRENW) ) dp_ram_asic ( .clk (clk), - .reset (reset), + .reset (1'b0), .read (read), .write (write), - .wren (wren), + .wren (wr_lanes), .waddr (waddr), .wdata (wdata), .raddr (raddr), - .rdata (rdata) + .rdata (macro_rdata) ); + if (RDW_MODE == "W") begin : g_write_first + // The macro reads before it writes and then holds its output, while + // this form reads the captured address continuously: the lanes written + // at that address since the read are forwarded around the macro. + reg [ADDRW-1:0] raddr_r; + reg [WRENW-1:0] fwd_en_r; + reg [DATAW-1:0] fwd_data_r; + wire [ADDRW-1:0] fwd_addr = read ? raddr : raddr_r; + wire [WRENW-1:0] fwd_lanes = {WRENW{write && waddr == fwd_addr}} & wr_lanes; + always @(posedge clk) begin + if (read) begin + raddr_r <= raddr; + end + fwd_en_r <= (read ? '0 : fwd_en_r) | fwd_lanes; + for (integer i = 0; i < WRENW; ++i) begin + if (fwd_lanes[i]) begin + fwd_data_r[i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; + end + end + end + for (genvar i = 0; i < WRENW; ++i) begin : g_lanes + assign rdata[i * WSELW +: WSELW] = fwd_en_r[i] ? fwd_data_r[i * WSELW +: WSELW] : macro_rdata[i * WSELW +: WSELW]; + end + end else begin : g_read_first + assign rdata = macro_rdata; + end end else begin : g_no_asic `else if (1) begin : g_no_asic @@ -288,73 +316,37 @@ module VX_dp_ram #( end end else begin : g_async `UNUSED_VAR (read) - if (FORCE_BRAM_ASYNC) begin : g_bram - if (RDW_MODE == "W") begin : g_write_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[raddr]; - end else begin : g_no_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[raddr]; + if (RDW_MODE == "W") begin : g_write_first + if (WRENW != 1) begin : g_wren + `RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_WREN end - end else begin : g_read_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[raddr]; - end else begin : g_no_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[raddr]; + assign rdata = ram[raddr]; + end else begin : g_no_wren + `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_ALL end + assign rdata = ram[raddr]; end - end else begin : g_auto - if (RDW_MODE == "W") begin : g_write_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[raddr]; - end else begin : g_no_wren - `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[raddr]; + end else begin : g_read_first + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_WREN end - end else begin : g_read_first - if (WRENW != 1) begin : g_wren - `NO_RW_RAM_CHECK `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[raddr]; - end else begin : g_no_wren - `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[raddr]; + assign rdata = ram[raddr]; + end else begin : g_no_wren + `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_ALL end + assign rdata = ram[raddr]; end end end @@ -407,30 +399,7 @@ module VX_dp_ram #( end end else begin : g_async `UNUSED_VAR (read) - if (RDW_MODE == "W") begin : g_write_first - assign rdata = ram[raddr]; - end else begin : g_read_first - reg [DATAW-1:0] prev_data; - reg [ADDRW-1:0] prev_waddr; - reg prev_write; - - always @(posedge clk) begin - if (reset) begin - prev_write <= 0; - prev_data <= '0; - prev_waddr <= '0; - end else begin - prev_write <= write; - prev_data <= ram[waddr]; - prev_waddr <= waddr; - end - end - - assign rdata = (prev_write && (prev_waddr == raddr)) ? prev_data : ram[raddr]; - if (RDW_ASSERT) begin : g_rw_asert - `RUNTIME_ASSERT(~read || (rdata == ram[raddr]), ("read after write hazard")) - end - end + assign rdata = ram[raddr]; end `endif diff --git a/hw/rtl/libs/VX_fifo_queue.sv b/hw/rtl/libs/VX_fifo_queue.sv index 22adee2f56..48dc0846eb 100644 --- a/hw/rtl/libs/VX_fifo_queue.sv +++ b/hw/rtl/libs/VX_fifo_queue.sv @@ -97,7 +97,7 @@ module VX_fifo_queue #( // present the look-ahead address (rd_ptr_n, the next read pointer): the // registered read then returns ram[rd_ptr_r] on the same cycle the async // read would have, making the output bit-identical while letting Vivado - // infer native BRAM (no async patch). + // infer native BRAM. wire [ADDRW-1:0] rd_addr = (OUT_REG != 0) ? rd_ptr_n : rd_ptr_r; VX_dp_ram #( @@ -105,8 +105,7 @@ module VX_fifo_queue #( .SIZE (DEPTH), .OUT_REG (OUT_REG), .LUTRAM (LUTRAM), - .RDW_MODE ("W"), - .RADDR_REG (OUT_REG == 0) + .RDW_MODE ("W") ) dp_ram ( .clk (clk), .reset (reset), diff --git a/hw/rtl/libs/VX_sp_ram.sv b/hw/rtl/libs/VX_sp_ram.sv index 319bcffb87..13e00641fc 100644 --- a/hw/rtl/libs/VX_sp_ram.sv +++ b/hw/rtl/libs/VX_sp_ram.sv @@ -67,6 +67,9 @@ end `endif +// Read forms, as in VX_dp_ram. OUT_REG=1 with RDW_MODE "W" registers the +// address, with "R" registers the data, with "N" registers the data and holds it +// across a write. OUT_REG=0 is combinational and never block RAM. `TRACING_OFF module VX_sp_ram #( parameter DATAW = 1, @@ -75,8 +78,6 @@ module VX_sp_ram #( parameter OUT_REG = 0, parameter LUTRAM = 0, parameter `STRING RDW_MODE = "W", // W: write-first, R: read-first, N: no-change - parameter RADDR_REG = 0, // read address registered hint - parameter RDW_ASSERT = 0, parameter RESET_RAM = 0, parameter INIT_ENABLE = 0, parameter INIT_FILE = "", @@ -94,37 +95,56 @@ module VX_sp_ram #( ); localparam WSELW = DATAW / WRENW; `UNUSED_PARAM (LUTRAM) - `UNUSED_PARAM (RADDR_REG) `STATIC_ASSERT(!(WRENW * WSELW != DATAW), ("invalid parameter")) `STATIC_ASSERT((RDW_MODE == "R" || RDW_MODE == "W" || RDW_MODE == "N"), ("invalid parameter")) - `UNUSED_PARAM (RDW_ASSERT) `ifdef SYNTHESIS localparam FORCE_BRAM = !LUTRAM && `FORCE_BRAM(SIZE, DATAW); - // Async reads have no native block-RAM form: a forced-BRAM RAM read - // asynchronously is tagged block RAM, and the synthesis elaboration makes its - // read synchronous, reading at the address its registers are about to take - // (CIRCT seq-retime-firmem-reads). That only pays off for a read-first RAM; a - // write-first one adds a read/write collision bypass that is larger than - // plain distributed RAM. - localparam FORCE_BRAM_ASYNC = FORCE_BRAM && (OUT_REG == 0) && (RDW_MODE == "R"); `ifdef ASIC - if (FORCE_BRAM && (OUT_REG != 0 || RADDR_REG != 0)) begin : g_asic + // The macro reads synchronously, so a combinational read stays in logic. + if (FORCE_BRAM && OUT_REG != 0) begin : g_asic + `UNUSED_VAR (reset) + wire [WRENW-1:0] wr_lanes = (WRENW != 1) ? wren : {WRENW{1'b1}}; + wire [DATAW-1:0] macro_rdata; VX_sp_ram_asic #( .DATAW (DATAW), .SIZE (SIZE), .WRENW (WRENW) ) sp_ram_asic ( .clk (clk), - .reset (reset), - .read (read), + .reset (1'b0), + .read ((RDW_MODE == "N") ? (read && ~write) : read), .write (write), - .wren (wren), + .wren (wr_lanes), .addr (addr), .wdata (wdata), - .rdata (rdata) + .rdata (macro_rdata) ); + if (RDW_MODE == "W") begin : g_write_first + // As in VX_dp_ram: the lanes written at the captured address since + // the read are forwarded around the read-first macro. + reg [ADDRW-1:0] addr_r; + reg [WRENW-1:0] fwd_en_r; + reg [DATAW-1:0] fwd_data_r; + wire [WRENW-1:0] fwd_lanes = {WRENW{write && (read || addr == addr_r)}} & wr_lanes; + always @(posedge clk) begin + if (read) begin + addr_r <= addr; + end + fwd_en_r <= (read ? '0 : fwd_en_r) | fwd_lanes; + for (integer i = 0; i < WRENW; ++i) begin + if (fwd_lanes[i]) begin + fwd_data_r[i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; + end + end + end + for (genvar i = 0; i < WRENW; ++i) begin : g_lanes + assign rdata[i * WSELW +: WSELW] = fwd_en_r[i] ? fwd_data_r[i * WSELW +: WSELW] : macro_rdata[i * WSELW +: WSELW]; + end + end else begin : g_read_first + assign rdata = macro_rdata; + end end else begin : g_no_asic `else if (1) begin : g_no_asic @@ -144,20 +164,16 @@ module VX_sp_ram #( end assign rdata = ram[addr_r]; end else begin : g_no_wren - `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; `RAM_INITIALIZATION - reg [DATAW-1:0] rdata_r; + reg [ADDRW-1:0] addr_r; always @(posedge clk) begin `RAM_WRITE_ALL if (read) begin - if (write) begin - rdata_r <= wdata; - end else begin - rdata_r <= ram[addr]; - end + addr_r <= addr; end end - assign rdata = rdata_r; + assign rdata = ram[addr_r]; end end else if (RDW_MODE == "R") begin : g_read_first if (WRENW != 1) begin : g_wren @@ -214,7 +230,7 @@ module VX_sp_ram #( // columns whose long routes dominate the path delay. if (RDW_MODE == "W") begin : g_write_first if (WRENW != 1) begin : g_wren - `USE_FAST_BRAM `RAM_ARRAY_WREN + `RW_RAM_CHECK `USE_FAST_BRAM `RAM_ARRAY_WREN `RAM_INITIALIZATION reg [ADDRW-1:0] addr_r; always @(posedge clk) begin @@ -225,20 +241,16 @@ module VX_sp_ram #( end assign rdata = ram[addr_r]; end else begin : g_no_wren - `USE_FAST_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RW_RAM_CHECK `USE_FAST_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; `RAM_INITIALIZATION - reg [DATAW-1:0] rdata_r; + reg [ADDRW-1:0] addr_r; always @(posedge clk) begin `RAM_WRITE_ALL if (read) begin - if (write) begin - rdata_r <= wdata; - end else begin - rdata_r <= ram[addr]; - end + addr_r <= addr; end end - assign rdata = rdata_r; + assign rdata = ram[addr_r]; end end else if (RDW_MODE == "R") begin : g_read_first if (WRENW != 1) begin : g_wren @@ -292,7 +304,7 @@ module VX_sp_ram #( end else begin : g_auto if (RDW_MODE == "W") begin : g_write_first if (WRENW != 1) begin : g_wren - `RAM_ARRAY_WREN + `RW_RAM_CHECK `RAM_ARRAY_WREN `RAM_INITIALIZATION reg [ADDRW-1:0] addr_r; always @(posedge clk) begin @@ -303,20 +315,16 @@ module VX_sp_ram #( end assign rdata = ram[addr_r]; end else begin : g_no_wren - reg [DATAW-1:0] ram [0:SIZE-1]; + `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; `RAM_INITIALIZATION - reg [DATAW-1:0] rdata_r; + reg [ADDRW-1:0] addr_r; always @(posedge clk) begin `RAM_WRITE_ALL if (read) begin - if (write) begin - rdata_r <= wdata; - end else begin - rdata_r <= ram[addr]; - end + addr_r <= addr; end end - assign rdata = rdata_r; + assign rdata = ram[addr_r]; end end else if (RDW_MODE == "R") begin : g_read_first if (WRENW != 1) begin : g_wren @@ -370,73 +378,37 @@ module VX_sp_ram #( end end else begin : g_async `UNUSED_VAR (read) - if (FORCE_BRAM_ASYNC) begin : g_bram - if (RDW_MODE == "W") begin : g_write_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[addr]; - end else begin : g_no_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[addr]; + if (RDW_MODE == "W") begin : g_write_first + if (WRENW != 1) begin : g_wren + `RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_WREN end - end else begin : g_read_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[addr]; - end else begin : g_no_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[addr]; + assign rdata = ram[addr]; + end else begin : g_no_wren + `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_ALL end + assign rdata = ram[addr]; end - end else begin : g_auto - if (RDW_MODE == "W") begin : g_write_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[addr]; - end else begin : g_no_wren - `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[addr]; + end else begin : g_read_first + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_WREN end - end else begin : g_read_first - if (WRENW != 1) begin : g_wren - `NO_RW_RAM_CHECK `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[addr]; - end else begin : g_no_wren - `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[addr]; + assign rdata = ram[addr]; + end else begin : g_no_wren + `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_ALL end + assign rdata = ram[addr]; end end end @@ -497,28 +469,7 @@ module VX_sp_ram #( end end else begin : g_async `UNUSED_VAR (read) - if (RDW_MODE == "W") begin : g_write_first - assign rdata = ram[addr]; - end else begin : g_read_first - reg [DATAW-1:0] prev_data; - reg [ADDRW-1:0] prev_waddr; - reg prev_write; - always @(posedge clk) begin - if (reset) begin - prev_write <= 0; - prev_data <= '0; - prev_waddr <= '0; - end else begin - prev_write <= write; - prev_data <= ram[addr]; - prev_waddr <= addr; - end - end - assign rdata = (prev_write && (prev_waddr == addr)) ? prev_data : ram[addr]; - if (RDW_ASSERT) begin : g_rw_asert - `RUNTIME_ASSERT(~read || (rdata == ram[addr]), ("read after write hazard")) - end - end + assign rdata = ram[addr]; end `endif diff --git a/hw/rtl/tcu/VX_tcu_meta.sv b/hw/rtl/tcu/VX_tcu_meta.sv index 68bf326390..46a00f459b 100644 --- a/hw/rtl/tcu/VX_tcu_meta.sv +++ b/hw/rtl/tcu/VX_tcu_meta.sv @@ -27,7 +27,7 @@ // // Read latency contract: the read address (rd_wid) is the already-registered // execute_if header wid, so the RAM is read combinationally in the issue -// cycle (OUT_REG=0, RADDR_REG=1). The sparse mux / MX scale extraction in +// cycle (OUT_REG=0). The sparse mux / MX scale extraction in // VX_tcu_core consume the result in that same cycle. module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( @@ -211,8 +211,7 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .WRENW (MERGED_COLS), .LUTRAM (0), .OUT_REG (0), - .RDW_MODE ("W"), - .RADDR_REG(1) // rd_wid is registered! + .RDW_MODE ("W") ) meta_ram ( .clk (clk), .reset (reset), diff --git a/hw/scripts/circt_elaborate.sh b/hw/scripts/circt_elaborate.sh index 387cf9af86..d6f86f95ec 100755 --- a/hw/scripts/circt_elaborate.sh +++ b/hw/scripts/circt_elaborate.sh @@ -14,7 +14,9 @@ # limitations under the License. # Elaborates SystemVerilog sources with CIRCT and writes the design as one -# file, the netlist every synthesis flow reads: +# file, the netlist Yosys, Beacon and Foldsim read (Vivado reads the sources +# themselves). Asynchronous RAM reads are made synchronous before this, in the +# sources (gen_sources.sh -F); here: # # - memories are detected, and a RAM that asks for block RAM with its # synthesis attribute (ram_style = "block", ...) reads synchronously, so diff --git a/hw/scripts/gen_sources.sh b/hw/scripts/gen_sources.sh index 71d68a8e6d..06b23d0c35 100755 --- a/hw/scripts/gen_sources.sh +++ b/hw/scripts/gen_sources.sh @@ -34,6 +34,7 @@ top_module="" copy_folder="" preprocessor=0 elaborate=0 +fold=0 stubs=() defines_str="" @@ -50,7 +51,7 @@ add_option() { } # Parse command arguments -while getopts D:G:T:I:J:O:H:C:X:PEh flag +while getopts D:G:T:I:J:O:H:C:X:PEFh flag do case "${flag}" in D) defines+=( ${OPTARG} ) @@ -76,9 +77,11 @@ do ;; E) elaborate=1 ;; + F) fold=1 + ;; X) stubs+=( "${OPTARG}" ) ;; - h) echo "Usage: [-D] [-G=] [-T] [-I] [-J] [-O] [-C: copy to] [-H] [-P: macro preprocessing] [-E: elaborate into one netlist] [-X] [-h help] [files...]" + h) echo "Usage: [-D] [-G=] [-T] [-I] [-J] [-O] [-C: copy to] [-H] [-P: macro preprocessing] [-F: fold asynchronous RAM reads into block RAM] [-E: elaborate into one netlist] [-X] [-h help] [files...]" exit 0 ;; \?) echo "Invalid option: -$OPTARG" 1>&2 @@ -192,6 +195,28 @@ if [ -n "$copy_folder" ]; then fi fi +# Optional RAM fold (-F): patch the preprocessed copies so that every +# block-RAM-sized VX_dp_ram that reads combinationally reads a registered next +# address instead, which every synthesis tool maps to block RAM. The report +# and the patch land in the copy folder. RAM_FOLD_PYTHON is a python3 with +# pyslang 11 or 12. +if [ "$fold" != 0 ]; then + if [ -z "$copy_folder" ] || [ "$preprocessor" = 0 ] || [ -z "$top_module" ]; then + echo "gen_sources.sh: error: -F needs preprocessed copies (-P -C) and a top module (-T)" >&2 + exit 1 + fi + fold_files=() + while IFS= read -r f; do + fold_files+=( "$f" ) + done < <(find "$(realpath "$copy_folder")" -maxdepth 1 -type f \( -name "*.v" -o -name "*.sv" \) | sort) + if ! "${RAM_FOLD_PYTHON:-python3}" "$SCRIPT_DIR"/ram_fold.py --top "$top_module" $defines_str \ + --report "$copy_folder/ram_fold.rpt" --diff "$copy_folder/ram_fold.diff" "${fold_files[@]}"; then + echo "gen_sources.sh: error: ram_fold.py failed (RAM_FOLD_PYTHON='${RAM_FOLD_PYTHON:-python3}'," \ + "which needs pyslang 11 or 12)" >&2 + exit 1 + fi +fi + # Optional filelist generation # Order-sensitive consumers (Questa, VCS) need every package compiled before # its importers. Detect packages by content, not filename: vendored IP ships diff --git a/hw/scripts/ram_fold.py b/hw/scripts/ram_fold.py new file mode 100644 index 0000000000..ecedc8e46d --- /dev/null +++ b/hw/scripts/ram_fold.py @@ -0,0 +1,1078 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""RAM fold: block RAM for asynchronous reads, as a patch of the sources. + +A block RAM reads synchronously. A VX_dp_ram of block-RAM size that reads +combinationally (OUT_REG=0) becomes one when its read address is computed one +cycle early: the RAM then registers that next address itself (OUT_REG=1, +RDW_MODE "W"), which reads exactly what the combinational read did on every +cycle (docs/hardware_library.md, read forms). + +The next address is the address logic evaluated on the next values of the +registers that feed it, wherever in the hierarchy they are. The pass elaborates +the design with slang, the front end CIRCT uses, follows the address back to +those registers, and patches the sources it was given: + + - every signal on that cone gets a twin __nx carrying its next value, + driven by a copy of the signal's own driver with its operands renamed; + - a register's twin is a combinational copy of its always_ff block that keeps + only the assignments to that register (their right-hand sides still read + current values, as non-blocking assignments do); + - a port on the cone gets a twin port, connected at every instantiation; + - an interface signal gets a twin member; + - the RAM takes the twin address and the registered-address form, reading + every cycle. + +Edits only add code, except the RAM's own parameters and address connection. +The patched sources are elaborated again; a patch that adds an error is dropped +whole and every RAM keeps its original, combinational read. Nothing is folded across a +construct the pass does not handle: it reports the RAM and leaves it alone. +""" + +import argparse +import difflib +import os +import re +import shlex +import sys + +import pyslang +from pyslang import analysis, ast, driver + +SFX = "__nx" +SK = ast.SymbolKind +EK = ast.ExpressionKind + +RAM_MODULES = ("VX_dp_ram",) + +CONST_KINDS = {SK.Parameter, SK.TypeParameter, SK.EnumValue, SK.Genvar, SK.Specparam} + +PURE_SYSTEM_CALLS = {"$clog2", "$bits", "$signed", "$unsigned", "$countones", "$size", + "$high", "$low", "$left", "$right", "$onehot", "$onehot0", "$isunknown"} + + +class Unsupported(Exception): + pass + + +def force_bram(depth, width): + # `FORCE_BRAM in VX_platform.vh + return depth >= 64 and depth * width >= 2048 + + +def const_int(v): + s = str(v) + m = re.match(r"^(?:\d+)?'s?[dhbo]?([0-9a-fA-F_xXzZ]+)$", s) + if s.isdigit() or (s.startswith("-") and s[1:].isdigit()): + return int(s) + if m: + base = {"d": 10, "h": 16, "b": 2, "o": 8} + b = re.search(r"'s?([dhbo])", s) + return int(m.group(1).replace("_", ""), base[b.group(1)] if b else 10) + if s.startswith('"'): + return s + raise ValueError(s) + + +class Sources: + """File contents and text edits, by byte offset.""" + + def __init__(self, sm): + self.sm = sm + self.data = {} + + def file(self, loc): + return os.path.realpath(self.sm.getFullPath(loc.buffer)) + + def bytes(self, path): + if path not in self.data: + with open(path, "rb") as f: + self.data[path] = f.read() + return self.data[path] + + def span(self, rng): + path = self.file(rng.start) + return path, rng.start.offset, rng.end.offset + + def text(self, rng): + path, s, e = self.span(rng) + return self.bytes(path)[s:e].decode() + + def indent(self, path, offset): + b = self.bytes(path) + line = b.rfind(b"\n", 0, offset) + 1 + m = re.match(rb"[ \t]*", b[line:]) + return m.group(0).decode() + + +class Edits: + """Keyed insertions and replacements; the same key must carry the same edit.""" + + def __init__(self): + self.items = {} + self.order = [] + + def add(self, key, path, start, end, text): + e = (path, start, end, text) + if key in self.items: + if self.items[key] != e: + raise Unsupported("conflicting edits at %s:%d" % (os.path.basename(path), start)) + return + self.items[key] = e + self.order.append(key) + + def merge(self, other): + for k in other.order: + self.add(k, *other.items[k]) + + def apply(self, srcs): + out = {} + by_file = {} + for n, k in enumerate(self.order): + path, s, e, t = self.items[k] + by_file.setdefault(path, []).append((s, e, n, t)) + for path, lst in by_file.items(): + data = srcs.bytes(path) + # replacements must not overlap each other or contain insertions + reps = [(s, e) for s, e, _, _ in lst if e > s] + for s, e in reps: + for s2, e2, _, _ in lst: + if (s2, e2) != (s, e) and s < s2 < e or (e2 > s2 and s2 < e and s < e2 and (s2, e2) != (s, e)): + raise Unsupported("overlapping edits in %s at %d" % (os.path.basename(path), s)) + # apply back to front; at one offset, insertions keep their order + lst.sort(key=lambda x: (x[0], x[2])) + res = bytearray() + pos = 0 + for s, e, _, t in lst: + res += data[pos:s] + res += t.encode() + pos = max(pos, e) if e > s else max(pos, s) + res += data[pos:] + out[path] = bytes(res) + return out + + +class Design: + """An elaborated design and its indices.""" + + def __init__(self, args, files): + self.dr = driver.Driver() + self.dr.addStandardArgs() + # no error limit: slang stops its driver analysis once the limit is reached + cmd = ["slang", "--allow-use-before-declare", "--allow-hierarchical-const", + "--ignore-unknown-modules", "--disable-instance-caching", "--error-limit=0"] + args + files + if not self.dr.parseCommandLine(shlex.join(cmd), driver.CommandLineOptions()): + raise SystemExit("ram_fold: bad slang arguments") + if not (self.dr.processOptions() and self.dr.parseAllSources()): + raise SystemExit("ram_fold: cannot parse the sources") + self.comp = self.dr.createCompilation() + self.root = self.comp.getRoot() + self.errors = self._errors() + self.am = analysis.AnalysisManager() + self.am.analyze(self.comp) + self.srcs = Sources(self.dr.sourceManager) + self.parent = {} # instance path -> parent instance + self.owner = {} # value symbol path -> instance + self.by_path = {} # value symbol path -> symbol + self.insts = {} # instance path -> instance + self.iface_maps = {} + self.iface_users = {} # interface instance path -> [(instance, port)] + self.rams = [] + self.arrayed = set() # instances that are elements of an instance array + for top in self.root.topInstances: + self.parent[top.hierarchicalPath] = None + self.insts[top.hierarchicalPath] = top + self._index(top.body, top) + + + def _errors(self): + de = pyslang.DiagnosticEngine(self.dr.sourceManager) + cl = pyslang.TextDiagnosticClient() + de.addClient(cl) + errs = set() + for d in self.comp.getAllDiagnostics(): + if d.isError(): + cl.clear() + de.issue(d) + errs.add(re.sub(r":\d+:\d+:", ":", cl.getString().split("\n")[0])) + return errs + + def _index(self, scope, inst): + for m in scope: + k = m.kind + if k == SK.Instance: + self.parent[m.hierarchicalPath] = inst + self.insts[m.hierarchicalPath] = m + self._ifaces(m) + if m.definition.name in RAM_MODULES: + self.rams.append(m) + self._index(m.body, m) + elif k in (SK.GenerateBlock, SK.GenerateBlockArray): + if k == SK.GenerateBlock and m.isUninstantiated: + continue + self._index(m, inst) + elif k in (SK.Net, SK.Variable): + self.owner[m.hierarchicalPath] = inst + self.by_path[m.hierarchicalPath] = m + elif k == SK.InstanceArray: + self._index_array(m, inst) + + def _ifaces(self, inst): + for p in inst.body.portList: + if p.kind == SK.InterfacePort: + c = inst.getPortConnection(p) + if c is not None and c.ifaceConn is not None and c.ifaceConn[0] is not None: + self.iface_users.setdefault(c.ifaceConn[0].hierarchicalPath, []).append((inst, p)) + + def _index_array(self, arr, inst): + for e in arr: + if e.kind == SK.InstanceArray: + self._index_array(e, inst) + elif e.kind == SK.Instance: + self.parent[e.hierarchicalPath] = inst + self.insts[e.hierarchicalPath] = e + self.arrayed.add(e.hierarchicalPath) + self._ifaces(e) + self._index(e.body, e) + + def param(self, inst, name): + for p in inst.body.parameters: + if p.name == name: + return p.value + raise KeyError(name) + + def port(self, inst, name): + for p in inst.body.portList: + if p.name == name: + return p + raise KeyError(name) + + def port_of(self, inst, sym): + for p in inst.body.portList: + if p.kind == SK.Port and p.internalSymbol is not None and \ + p.internalSymbol.hierarchicalPath == sym.hierarchicalPath: + return p + return None + + +def leaf_name_span(srcs, node, name): + """The byte span of the last identifier of a value reference.""" + path, s, e = srcs.span(node.sourceRange) + txt = srcs.bytes(path)[s:e].decode() + m = None + for m in re.finditer(r"\b%s\b" % re.escape(name), txt): + pass + if m is None: + raise Unsupported("cannot locate '%s' in '%s'" % (name, txt)) + return path, s + m.start(), s + m.end() + + +def chain_of(node): + """(root value expression, selector path) of a select/member chain.""" + path = [] + while True: + k = node.kind + if k == EK.ElementSelect: + c = node.selector.constant + path.append(("idx", str(c) if c is not None else None)) + node = node.value + elif k == EK.RangeSelect: + path.append(("rng", None)) + node = node.value + elif k == EK.MemberAccess: + path.append(("mem", node.member.name)) + node = node.value + elif k == EK.Conversion and node.isImplicit if hasattr(node, "isImplicit") else False: + node = node.operand + else: + break + if node.kind in (EK.NamedValue, EK.HierarchicalValue): + return node, tuple(reversed(path)) + return None, () + + +def reroot(p, a, b): + """p re-rooted from the hierarchy under a to the one under b, or None.""" + if p == a or p.startswith(a + ".") or p.startswith(a + "["): + return b + p[len(a):] + return None + + +def path_text(path): + return "".join("[%s]" % (v if v is not None else "*") if k != "mem" else "." + v for k, v in path) + + +def compatible(a, b): + for x, y in zip(a, b): + if x[0] != y[0]: + return True + if x[0] == "mem" and x[1] != y[1]: + return False + if x[0] == "idx" and x[1] is not None and y[1] is not None and x[1] != y[1]: + return False + return True + + +class Fold: + """Folds one RAM; its edits are merged only if it succeeds.""" + + def __init__(self, design): + self.d = design + self.srcs = design.srcs + self.edits = Edits() + self.declared = set() + self.done = set() + + # ---- canonical bodies ------------------------------------------------- + # + # slang analyzes one body per set of identically parameterized instances, so + # the drivers of a signal in any other instance are found on its counterpart + # in that canonical body, and whatever those drivers reference is mapped back. + + def canon_path(self, path): + """(path of the analyzed counterpart of path, mapping back).""" + tr = [] + for _ in range(64): + x = self.d.insts.get(path) or self.d.owner.get(path) + moved = False + while x is not None: + cb = x.canonicalBody if not x.isInterface else None + ci = cb.parentInstance if cb is not None else None + if ci is not None and ci.hierarchicalPath != x.hierarchicalPath: + tr.append((ci.hierarchicalPath, x.hierarchicalPath)) + path = reroot(path, x.hierarchicalPath, ci.hierarchicalPath) + moved = True + break + x = self.d.parent.get(x.hierarchicalPath) + if not moved: + break + return path, tuple(tr) + + def canon(self, sym): + """(counterpart of sym that carries drivers, mapping back).""" + path, tr = self.canon_path(sym.hierarchicalPath) + csym = self.d.by_path.get(path) + if csym is None: + raise Unsupported("no canonical counterpart of '%s'" % sym.hierarchicalPath) + return csym, tr + + def iface_drivers(self, sym, inst): + """Drivers of an interface signal in the modules connected to it whose + bodies slang analyzed only as another instance: [(driver, csym, tr)].""" + rel = sym.hierarchicalPath[len(inst.hierarchicalPath):] + found = [] + for user, port in self.d.iface_users.get(inst.hierarchicalPath, []): + upath, tr = self.canon_path(user.hierarchicalPath) + if not tr: + continue # analyzed as itself: its drivers are on sym already + cuser = self.d.insts.get(upath) + if cuser is None: + continue + cport = next((p for p in cuser.body.portList if p.name == port.name), None) + c = cuser.getPortConnection(cport) if cport is not None else None + if c is None or c.ifaceConn is None or c.ifaceConn[0] is None: + continue + csym = self.d.by_path.get(c.ifaceConn[0].hierarchicalPath + rel) + if csym is None: + continue + for drv in self.d.am.getDrivers(csym): + if reroot(drv.containingSymbol.hierarchicalPath, upath, upath) is not None: + found.append((drv, csym, tr)) + return found + + def back(self, path, tr): + for cp, ap in reversed(tr): + q = reroot(path, cp, ap) + if q is None: + # a signal of an interface the shared body reaches through a port + for ic, ia in self.iface_map(cp, ap): + q = reroot(path, ic, ia) + if q is not None: + break + if q is None: + raise Unsupported("a shared instance body reads '%s' outside itself" % path) + path = q + return path + + def iface_map(self, cp, ap): + """(interface instance path, its counterpart) for the interface ports of + instance cp and its copy ap.""" + if (cp, ap) in self.d.iface_maps: + return self.d.iface_maps[(cp, ap)] + ci, ai = self.d.insts.get(cp), self.d.insts.get(ap) + res = [] + if ci is not None and ai is not None: + for cport, aport in zip(ci.body.portList, ai.body.portList): + if cport.kind != SK.InterfacePort: + continue + cc, ac = ci.getPortConnection(cport), ai.getPortConnection(aport) + if cc is None or ac is None or cc.ifaceConn is None or ac.ifaceConn is None: + continue + c0, a0 = cc.ifaceConn[0], ac.ifaceConn[0] + if c0 is not None and a0 is not None: + res.append((c0.hierarchicalPath, a0.hierarchicalPath)) + self.d.iface_maps[(cp, ap)] = res + return res + + def act(self, sym, tr): + if not tr: + return sym + s = self.d.by_path.get(self.back(sym.hierarchicalPath, tr)) + if s is None: + raise Unsupported("no instance counterpart of '%s'" % sym.hierarchicalPath) + return s + + def act_inst(self, inst, tr): + if not tr: + return inst + i = self.d.insts.get(self.back(inst.hierarchicalPath, tr)) + if i is None: + raise Unsupported("no instance counterpart of '%s'" % inst.hierarchicalPath) + return i + + def scope_key(self, sym, tr): + p = sym.hierarchicalPath + for cp, ap in reversed(tr): + p = reroot(p, cp, ap) or p + return p + + def clock_root(self, expr, tr=()): + """Where a clock comes from: its net, followed up through input ports.""" + while True: + while expr.kind == EK.Conversion: + expr = expr.operand + if expr.kind != EK.NamedValue: + return None + sym = self.act(self.value_symbol(expr), tr) + tr = () + inst = self.d.owner.get(sym.hierarchicalPath) + port = self.d.port_of(inst, sym) if inst is not None else None + if port is None or port.direction != ast.ArgumentDirection.In or \ + self.d.parent.get(inst.hierarchicalPath) is None: + return sym.hierarchicalPath + conn = inst.getPortConnection(port) + if conn is None or conn.expression is None: + return None + expr = conn.expression + + # ---- references ------------------------------------------------------- + + def value_symbol(self, node): + sym = node.symbol + if sym.kind == SK.ModportPort: + sym = sym.internalSymbol + return sym + + def leaves(self, expr, tr=()): + """Non-constant value references in expr: (node, symbol, path).""" + paths = {} + found = [] + + def visit(n): + if not isinstance(n, ast.Expression): + return ast.VisitAction.Advance + k = n.kind + if k in (EK.ElementSelect, EK.RangeSelect, EK.MemberAccess): + root, p = chain_of(n) + if root is not None: + key = root.sourceRange.start.offset + if key not in paths or len(p) > len(paths[key]): + paths[key] = p + elif k in (EK.NamedValue, EK.HierarchicalValue): + sym = self.value_symbol(n) + if sym.kind in CONST_KINDS: + return ast.VisitAction.Advance + if sym.kind not in (SK.Net, SK.Variable): + raise Unsupported("reference to %s '%s'" % (sym.kind.name, sym.name)) + found.append((n, self.act(sym, tr))) + elif k == EK.Call: + self.check_pure(n) + elif k == EK.Assignment: + raise Unsupported("assignment inside an expression") + return ast.VisitAction.Advance + + expr.visit(visit) + return [(n, s, paths.get(n.sourceRange.start.offset, ())) for n, s in found] + + def check_pure(self, call): + sub = call.subroutine + name = call.subroutineName + if call.isSystemCall: + if name not in PURE_SYSTEM_CALLS: + raise Unsupported("system call %s" % name) + return + if sub.kind != SK.Subroutine: + raise Unsupported("call to %s" % name) + + def visit(n): + if isinstance(n, ast.Expression) and n.kind in (EK.NamedValue, EK.HierarchicalValue): + s = n.symbol + if s.kind in (SK.Net, SK.Variable) and not self.inside(s, sub): + raise Unsupported("function %s reads '%s'" % (name, s.name)) + return ast.VisitAction.Advance + + sub.visit(visit) + + @staticmethod + def inside(sym, scope_sym): + p = sym.lexicalPath if hasattr(sym, "lexicalPath") else "" + return p.startswith(scope_sym.lexicalPath) if hasattr(scope_sym, "lexicalPath") else False + + def nx_expr(self, expr, extra=(), tr=()): + """The source text of expr evaluated on next values.""" + path, s, e = self.srcs.span(expr.sourceRange) + reps = [] + for node, sym, p in self.leaves(expr, tr): + if extra and node.sourceRange.start.offset == s: + p = p + tuple(extra) + name = self.twin(sym, p) + fp, ls, le = leaf_name_span(self.srcs, node, sym.name) + if fp != path: + raise Unsupported("expression spans files") + reps.append((ls, le, name)) + return self.substitute(path, s, e, reps) + + def substitute(self, path, s, e, reps): + data = self.srcs.bytes(path) + out, pos = [], s + for ls, le, t in sorted(set(reps)): + out.append(data[pos:ls].decode()) + out.append(t) + pos = le + out.append(data[pos:e].decode()) + return "".join(out) + + def rename_root(self, lhs, tr=()): + """The text of an lvalue with its root signal renamed to its twin.""" + root, _ = chain_of(lhs) + if root is None: + raise Unsupported("lvalue '%s'" % self.srcs.text(lhs.sourceRange)) + for n, _, _ in self.leaves(lhs, tr): + if n.sourceRange.start.offset != root.sourceRange.start.offset: + raise Unsupported("lvalue with a variable index '%s'" % self.srcs.text(lhs.sourceRange)) + sym = self.value_symbol(root) + path, s, e = self.srcs.span(lhs.sourceRange) + _, ls, le = leaf_name_span(self.srcs, root, sym.name) + return self.substitute(path, s, e, [(ls, le, sym.name + SFX)]), sym + + # ---- twins ------------------------------------------------------------ + + def twin(self, sym, path=()): + key = sym.hierarchicalPath + name = sym.name + SFX + inst = self.d.owner.get(key) + if inst is None: + raise Unsupported("no owner for '%s'" % key) + port = self.d.port_of(inst, sym) if inst.isModule or inst.isInterface else None + if key not in self.declared: + self.declared.add(key) + self.declare(sym, inst, port) + if port is not None and port.direction == ast.ArgumentDirection.In: + if ("in", key, path) not in self.done: + self.done.add(("in", key, path)) + self.input_port(sym, inst, port, path) + return name + if sym.kind == SK.Net and sym.initializer is not None: + return name # driven by its declaration, which declare() copied + # every part read must be driven: a twin read without a driver would be + # undriven logic that still elaborates + csym, tr = self.canon(sym) + drivers = [(drv, csym, tr) for drv in self.d.am.getDrivers(csym)] + if inst.isInterface: + drivers += self.iface_drivers(sym, inst) + if not drivers: + raise Unsupported("'%s' is undriven" % sym.hierarchicalPath) + blocks = {(self.scope_key(drv.containingSymbol, t), drv.containingSymbol.syntax.sourceRange.start.offset) + for drv, c, t in drivers if drv.containingSymbol.kind == SK.ProceduralBlock} + if len(blocks) > 1: + raise Unsupported("'%s' is driven by %d procedural blocks" % (sym.hierarchicalPath, len(blocks))) + if not any([self.driver(sym, inst, c, drv, path, t) for drv, c, t in drivers]): + raise Unsupported("no driver of '%s'%s" % (sym.hierarchicalPath, path_text(path))) + return name + + def declare(self, sym, inst, port): + name = sym.name + SFX + if port is not None: + self.declare_port(port, name) + return + decl = sym.syntax + while decl is not None and decl.kind not in (pyslang.syntax.SyntaxKind.DataDeclaration, + pyslang.syntax.SyntaxKind.NetDeclaration): + decl = decl.parent + if decl is None: + raise Unsupported("no declaration for '%s'" % sym.name) + path, ds, de = self.srcs.span(decl.sourceRange) + _, vs, ve = self.srcs.span(sym.syntax.sourceRange) + data = self.srcs.bytes(path) + prefix = data[ds:self.srcs.span(decl.declarators[0].sourceRange)[1]].decode() + dims = "".join(str(x) for x in sym.syntax.dimensions).strip() + init = "" + if sym.kind == SK.Net and sym.initializer is not None: + init = " = " + self.nx_expr(sym.initializer) + ind = self.srcs.indent(path, ds) + text = "\n%s%s%s%s%s;" % (ind, prefix.strip() + " ", name, (" " + dims) if dims else "", init) + self.edits.add(("decl", path, de, name), path, de, de, text) + if inst.isInterface: + self.modports(sym, inst, name) + + def declare_port(self, port, name): + syn = port.syntax + if syn is not None and syn.kind == pyslang.syntax.SyntaxKind.Declarator: + syn = syn.parent + if syn is None or syn.kind != pyslang.syntax.SyntaxKind.ImplicitAnsiPort: + raise Unsupported("port '%s' is not an ANSI port" % port.name) + header = syn + while header is not None and not str(header.header).strip(): + header = self.prev_port(header) + if header is None: + raise Unsupported("port '%s' has no header" % port.name) + dims = "".join(str(x) for x in syn.declarator.dimensions).strip() + # appended after the last port, so an instantiation that connects by + # position elsewhere keeps its ports + ports = [p for p in syn.parent.ports if isinstance(p, pyslang.syntax.SyntaxNode)] + path, ps, _ = self.srcs.span(syn.sourceRange) + _, _, pe = self.srcs.span(ports[-1].sourceRange) + ind = self.srcs.indent(path, ps) + text = ",\n%s%s %s%s" % (ind, str(header.header).strip(), name, (" " + dims) if dims else "") + self.edits.add(("port", path, pe, name), path, pe, pe, text) + + @staticmethod + def prev_port(syn): + lst = syn.parent.ports if hasattr(syn.parent, "ports") else syn.parent + prev = None + for item in lst: + if item is syn or (hasattr(item, "sourceRange") and item.sourceRange.start.offset == syn.sourceRange.start.offset): + return prev + if isinstance(item, pyslang.syntax.SyntaxNode) and item.kind == pyslang.syntax.SyntaxKind.ImplicitAnsiPort: + prev = item + return None + + def modports(self, sym, inst, name): + for m in inst.body: + if m.kind != SK.Modport: + continue + for p in m: + if p.name == sym.name: + path, s, e = self.srcs.span(p.syntax.sourceRange) + self.edits.add(("modport", path, e, name), path, e, e, ", " + name) + + def input_port(self, sym, inst, port, path): + parent = self.d.parent.get(inst.hierarchicalPath) + if parent is None: + raise Unsupported("'%s' is a top-level input" % sym.hierarchicalPath) + conn = inst.getPortConnection(port) + if conn is None or conn.expression is None: + raise Unsupported("input '%s' of %s is unconnected" % (port.name, inst.hierarchicalPath)) + text = self.nx_expr(conn.expression, path) + self.connect(inst, port.name + SFX, text) + + def connect(self, inst, port_name, text): + if inst.hierarchicalPath in self.d.arrayed: + raise Unsupported("instance array %s on the address cone" % inst.hierarchicalPath) + syn = inst.syntax + conns = [c for c in syn.connections if isinstance(c, pyslang.syntax.SyntaxNode)] + if not conns or any(c.kind != pyslang.syntax.SyntaxKind.NamedPortConnection for c in conns): + raise Unsupported("instance %s does not connect its ports by name" % inst.name) + path, s, e = self.srcs.span(conns[-1].sourceRange) + ind = self.srcs.indent(path, s) + self.edits.add(("conn", path, e, port_name), path, e, e, + ",\n%s.%s (%s)" % (ind, port_name, text)) + + def driver(self, sym, inst, csym, drv, path, tr): + cs = drv.containingSymbol + k = cs.kind + if k == SK.ContinuousAssign: + lhs, rhs = cs.assignment.left, cs.assignment.right + if lhs.kind == EK.Concatenation: + return self.concat_assign(sym, csym, cs, lhs, rhs, path, tr) + root, lpath = chain_of(lhs) + if root is None: + raise Unsupported("assign to '%s'" % self.srcs.text(lhs.sourceRange)) + if not compatible(lpath, path): + return False + key = ("assign", self.scope_key(cs, tr), cs.syntax.sourceRange.start.offset, + self.srcs.file(cs.syntax.sourceRange.start)) + if key in self.done: + return True + self.done.add(key) + ltxt, _ = self.rename_root(lhs, tr) + rtxt = self.nx_expr(rhs, (), tr) + stmt = cs.syntax.parent + while stmt is not None and stmt.kind != pyslang.syntax.SyntaxKind.ContinuousAssign: + stmt = stmt.parent + fpath, ss, se = self.srcs.span(stmt.sourceRange) + ind = self.srcs.indent(fpath, ss) + self.edits.add(("assign", fpath, se, ltxt), fpath, se, se, + "\n%sassign %s = %s;" % (ind, ltxt, rtxt)) + return True + elif k == SK.ProceduralBlock: + self.register(sym, csym, cs, tr) + return True + elif k == SK.Instance: + return self.child_output(sym, inst, self.act_inst(cs, tr), path) + else: + raise Unsupported("'%s' is driven by a %s" % (sym.name, k.name)) + + def concat_assign(self, sym, csym, cs, lhs, rhs, path, tr): + """assign {.., sym, ..} = e: the twin of e, sliced.""" + ops = list(lhs.operands) + mine = [] + for i, op in enumerate(ops): + root, p = chain_of(op) + if root is None: + raise Unsupported("assign to '%s'" % self.srcs.text(lhs.sourceRange)) + if self.value_symbol(root).hierarchicalPath == csym.hierarchicalPath and compatible(p, path): + mine.append(i) + if not mine: + return False + key = ("cassign", self.scope_key(cs, tr), cs.syntax.sourceRange.start.offset, + self.srcs.file(cs.syntax.sourceRange.start), sym.hierarchicalPath) + if key in self.done: + return True + self.done.add(key) + stmt = cs.syntax.parent + while stmt is not None and stmt.kind != pyslang.syntax.SyntaxKind.ContinuousAssign: + stmt = stmt.parent + fpath, ss, se = self.srcs.span(stmt.sourceRange) + ind = self.srcs.indent(fpath, ss) + whole = self.srcs.text(lhs.sourceRange) + tname = "%s_cat%d%s" % (sym.name, cs.syntax.sourceRange.start.offset, SFX) + rtxt = self.nx_expr(rhs, (), tr) + self.edits.add(("catw", fpath, se, tname), fpath, se, se, + "\n%swire [$bits(%s)-1:0] %s = %s;" % (ind, whole, tname, rtxt)) + for i in mine: + later = [self.srcs.text(o.sourceRange) for o in ops[i + 1:]] + off = "$bits({%s})" % ", ".join(later) if later else "0" + ltxt, _ = self.rename_root(ops[i], tr) + self.edits.add(("cats", fpath, se, ltxt), fpath, se, se, + "\n%sassign %s = %s[%s +: $bits(%s)];" % (ind, ltxt, tname, off, self.srcs.text(ops[i].sourceRange))) + return True + + # ---- registers -------------------------------------------------------- + + def register(self, sym, csym, block, tr): + key = ("reg", self.scope_key(block, tr), block.syntax.sourceRange.start.offset, + self.srcs.file(block.syntax.sourceRange.start), sym.name) + if key in self.done: + return + self.done.add(key) + pk = block.procedureKind + if pk not in (ast.ProceduralBlockKind.AlwaysFF, ast.ProceduralBlockKind.Always): + raise Unsupported("'%s' is driven by an %s block" % (sym.name, pk.name)) + timed = block.body + if timed.kind != ast.StatementKind.Timed: + raise Unsupported("'%s' is driven by an untimed block" % sym.name) + tc = self.srcs.text(timed.timing.sourceRange) + if not re.fullmatch(r"(@\s*\(\s*)?posedge\s+\w+(\s*\))?", tc.strip()): + raise Unsupported("'%s' is clocked by '%s'" % (sym.name, tc)) + ev = getattr(timed.timing, "expr", None) + clk = self.clock_root(ev, tr) if ev is not None else None + if clk is None or clk != self.clock: + raise Unsupported("'%s' is clocked by '%s', not the RAM's clock" % (sym.hierarchicalPath, clk or tc)) + body = timed.stmt + path, bs, be = self.srcs.span(body.sourceRange) + reps = [] + self.keep_only(csym, body, reps) + btxt = self.substitute(path, bs, be, reps) + name = sym.name + SFX + bpath, s, e = self.srcs.span(block.syntax.sourceRange) + ind = self.srcs.indent(bpath, s) + body_ind = "\n".join((" " + l) if l.strip() else l for l in btxt.split("\n")) + text = "\n%salways @(*) begin\n%s %s = %s;\n%s %s\n%send" % (ind, ind, name, sym.name, ind, body_ind.lstrip(), ind) + self.edits.add(key[2:], bpath, e, e, text) + + def keep_only(self, sym, st, reps): + """Rewrite a clocked statement to compute sym's twin and nothing else.""" + k = st.kind + SKd = ast.StatementKind + if k == SKd.Block: + self.keep_only(sym, st.body, reps) + elif k == SKd.List: + for s in st.list: + self.keep_only(sym, s, reps) + elif k == SKd.Conditional: + self.keep_only(sym, st.ifTrue, reps) + if st.ifFalse is not None: + self.keep_only(sym, st.ifFalse, reps) + elif k == SKd.Case: + for it in st.items: + self.keep_only(sym, it.stmt, reps) + if st.defaultCase is not None: + self.keep_only(sym, st.defaultCase, reps) + elif k == SKd.ForLoop: + self.keep_only(sym, st.body, reps) + elif k in (SKd.Empty, SKd.VariableDeclaration): + pass + elif k == SKd.ExpressionStatement: + e = st.expr + path, s, en = self.srcs.span(st.sourceRange) + if e.kind == EK.Assignment: + root, _ = chain_of(e.left) + rsym = self.value_symbol(root) if root is not None else None + if e.isNonBlocking: + if rsym is None and any(x.hierarchicalPath == sym.hierarchicalPath + for _, x, _ in self.leaves(e.left)): + raise Unsupported("'%s' is assigned as part of '%s'" + % (sym.name, self.srcs.text(e.left.sourceRange))) + if rsym is not None and rsym.hierarchicalPath == sym.hierarchicalPath: + _, ls, le = leaf_name_span(self.srcs, root, sym.name) + reps.append((ls, le, sym.name + SFX)) + op = e.syntax.operatorToken.range + reps.append((op.start.offset, op.end.offset, "=")) + else: + reps.append((s, en, ";")) + return + raise Unsupported("blocking assignment in the block driving '%s'" % sym.name) + if e.kind == EK.Call and e.isSystemCall: + reps.append((s, en, ";")) + return + raise Unsupported("statement '%s' in the block driving '%s'" % (self.srcs.text(st.sourceRange)[:40], sym.name)) + else: + raise Unsupported("%s statement in the block driving '%s'" % (k.name, sym.name)) + + # ---- child outputs ---------------------------------------------------- + + def child_output(self, sym, inst, child, path): + covered = False + for conn in child.portConnections: + port = conn.port + if port.kind != SK.Port or port.direction != ast.ArgumentDirection.Out or conn.expression is None: + continue + lv = conn.expression + if lv.kind == EK.Assignment: + lv = lv.left + refs = [(n, s, p) for n, s, p in self.leaves(lv) if s.hierarchicalPath == sym.hierarchicalPath] + if not refs: + continue + key = ("out", child.hierarchicalPath, port.name, sym.hierarchicalPath, path) + if key in self.done: + covered = True + continue + self.done.add(key) + self.twin(port.internalSymbol) + pname = port.name + SFX + if lv.kind == EK.Concatenation: + covered = self.concat_output(sym, child, pname, lv, path) or covered + else: + root, p = chain_of(lv) + if root is None or not compatible(p, path): + continue + ltxt, _ = self.rename_root(lv) + self.connect(child, pname, ltxt) + covered = True + return covered + + def concat_output(self, sym, child, pname, lv, path): + ops = list(lv.operands) + fname = "%s_%s" % (child.name, pname) + stmt = child.syntax.parent + fpath, ss, se = self.srcs.span(stmt.sourceRange) + ind = self.srcs.indent(fpath, ss) + whole = self.srcs.text(lv.sourceRange) + self.edits.add(("fwire", fpath, ss, fname), fpath, ss, ss, + "wire [$bits(%s)-1:0] %s;\n%s" % (whole, fname, ind)) + self.connect(child, pname, fname) + covered = False + for i, op in enumerate(ops): + root, p = chain_of(op) + if root is None or self.value_symbol(root).hierarchicalPath != sym.hierarchicalPath: + continue + if not compatible(p, path): + continue + later = [self.srcs.text(o.sourceRange) for o in ops[i + 1:]] + off = "$bits({%s})" % ", ".join(later) if later else "0" + ltxt, _ = self.rename_root(op) + self.edits.add(("fslice", fpath, se, ltxt), fpath, se, se, + "\n%sassign %s = %s[%s +: $bits(%s)];" % (ind, ltxt, fname, off, self.srcs.text(op.sourceRange))) + covered = True + return covered + + # ---- the RAM ---------------------------------------------------------- + + def ram(self, ram, check): + parent = self.d.parent[ram.hierarchicalPath] + syn = ram.syntax + stmt = syn.parent + raddr = ram.getPortConnection(self.d.port(ram, "raddr")) + if raddr is None or raddr.expression is None: + raise Unsupported("raddr unconnected") + clk = ram.getPortConnection(self.d.port(ram, "clk")) + self.clock = self.clock_root(clk.expression) if clk is not None and clk.expression is not None else None + if self.clock is None: + raise Unsupported("cannot follow the RAM's clock") + nx = self.nx_expr(raddr.expression) + # parameters + params = stmt.parameters + if params is None: + raise Unsupported("RAM without a parameter list") + named = {} + for p in params.parameters: + if not isinstance(p, pyslang.syntax.SyntaxNode): + continue + if p.kind != pyslang.syntax.SyntaxKind.NamedParamAssignment: + raise Unsupported("RAM parameters not assigned by name") + named[p.name.valueText] = p + for pname, val in (("OUT_REG", "1"), ("RDW_MODE", '"W"')): + if pname in named: + rng = named[pname].expr.sourceRange + path, s, e = self.srcs.span(rng) + self.edits.add(("rparam", path, s, pname), path, s, e, val) + else: + path, s, e = self.srcs.span(params.openParen.range) + self.edits.add(("rparam+", path, e, pname), path, e, e, "\n .%s (%s)," % (pname, val)) + path, s, e = self.srcs.span(raddr.expression.sourceRange) + self.edits.add(("raddr", path, s), path, s, e, nx) + read = ram.getPortConnection(self.d.port(ram, "read")) + if read is not None and read.expression is not None: + path, s, e = self.srcs.span(read.expression.sourceRange) + self.edits.add(("read", path, s), path, s, e, "1'b1") + else: + self.connect(ram, "read", "1'b1") + if check: + self.check_assert(ram, raddr.expression, nx) + return nx + + def check_assert(self, ram, orig, nx): + clk = ram.getPortConnection(self.d.port(ram, "clk")).expression + rst = ram.getPortConnection(self.d.port(ram, "reset")).expression + stmt = ram.syntax.parent + path, s, e = self.srcs.span(stmt.sourceRange) + ind = self.srcs.indent(path, s) + n = ram.name + o = self.srcs.text(orig.sourceRange) + c, r = self.srcs.text(clk.sourceRange), self.srcs.text(rst.sourceRange) + text = ("\n%sreg [$bits(%s)-1:0] %s_fold_raddr_r;\n%sreg %s_fold_ok_r = 0;\n" + "%salways @(posedge %s) begin\n%s %s_fold_raddr_r <= %s;\n%s %s_fold_ok_r <= !%s;\n" + "%s if (%s_fold_ok_r && !%s && %s_fold_raddr_r !== %s)\n" + "%s $error(\"ram_fold: %%m.%s next address %%h, address %%h\", %s_fold_raddr_r, %s);\n%send" + % (ind, o, n, ind, n, ind, c, ind, n, nx, ind, n, r, ind, n, r, n, o, ind, n, n, o, ind)) + self.edits.add(("check", path, e, n), path, e, e, text) + + +def run(args): + files = list(args.files) + design = Design(args.slang, files) + report = [] + edits = Edits() + targets = {} + for ram in design.rams: + try: + d = const_int(design.param(ram, "SIZE")) + w = const_int(design.param(ram, "DATAW")) + oreg = const_int(design.param(ram, "OUT_REG")) + lut = const_int(design.param(ram, "LUTRAM")) + except (KeyError, ValueError) as ex: + report.append("skip %s: parameters (%s)" % (ram.hierarchicalPath, ex)) + continue + if oreg != 0 or not args.all and (lut != 0 or not force_bram(d, w)): + continue + targets[ram.hierarchicalPath] = (ram, d, w) + # A fold edits the RAM's instantiation statement, which every instance of its + # module shares: all RAMs it makes fold together or none does. + groups = {} + for p, (ram, d, w) in sorted(targets.items()): + syn = ram.syntax.parent.sourceRange + groups.setdefault((design.srcs.file(syn.start), syn.start.offset), []) + for ram in design.rams: + syn = ram.syntax.parent.sourceRange + key = (design.srcs.file(syn.start), syn.start.offset) + if key in groups: + groups[key].append(ram) + folded = [] + for key in sorted(groups): + rams = groups[key] + f = Fold(design) + cur = rams[0] + try: + for cur in rams: + if const_int(design.param(cur, "OUT_REG")) != 0: + raise Unsupported("its instantiation also makes a RAM that registers its read") + nxs = [] + for cur in rams: + nxs.append(f.ram(cur, args.check)) + trial = Edits() + trial.merge(edits) + trial.merge(f.edits) + trial.apply(design.srcs) + edits = trial + for ram, nx in zip(rams, nxs): + folded.append(ram.hierarchicalPath) + report.append("fold %s (%sx%s): raddr <- %s" % ( + ram.hierarchicalPath, design.param(ram, "SIZE"), design.param(ram, "DATAW"), nx)) + except Exception as ex: + if not isinstance(ex, Unsupported): + ex = "internal error: %s: %s" % (type(ex).__name__, ex) + for ram in rams: + why = str(ex) if ram is cur else "%s cannot fold: %s" % (cur.hierarchicalPath, ex) + report.append("keep %s (%sx%s): %s" % ( + ram.hierarchicalPath, design.param(ram, "SIZE"), design.param(ram, "DATAW"), why)) + patched = edits.apply(design.srcs) if folded else {} + return design, report, patched, folded + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--top", action="append", required=True, help="top module (slang --top)") + ap.add_argument("--report", help="write the fold report here") + ap.add_argument("--diff", help="write the patch as a unified diff here") + ap.add_argument("--check", action="store_true", + help="add a simulation check that each folded RAM's next address is its address a cycle later") + ap.add_argument("--all", action="store_true", + help="fold every combinational VX_dp_ram read, whatever its size (testing the pass)") + ap.add_argument("--dry-run", action="store_true", help="report only, do not patch") + ap.add_argument("files", nargs="*") + args, slang = ap.parse_known_args() + args.slang = [x for t in args.top for x in ("--top", t)] + [a for a in slang if not a.endswith((".sv", ".v"))] + args.files = [a for a in args.files] + [a for a in slang if a.endswith((".sv", ".v"))] + + design, report, patched, folded = run(args) + + diff = [] + if patched and not args.dry_run: + originals = {p: design.srcs.bytes(p) for p in patched} + for p, b in patched.items(): + with open(p, "wb") as f: + f.write(b) + new = Design(args.slang, args.files) + added = new.errors - design.errors + if added: + for p, b in originals.items(): + with open(p, "wb") as f: + f.write(b) + report.append("REVERTED: the patched sources do not elaborate:") + report += [" " + e for e in sorted(added)] + folded = [] + else: + still = [r.hierarchicalPath for r in new.rams + if r.hierarchicalPath in folded and const_int(new.param(r, "OUT_REG")) != 1] + if still: + raise SystemExit("ram_fold: internal error, not folded after patching: %s" % still) + for p, b in patched.items(): + diff += difflib.unified_diff(originals[p].decode().splitlines(True), + b.decode().splitlines(True), fromfile=p, tofile=p) + if args.diff: + with open(args.diff, "w") as f: + f.writelines(diff) + + text = "\n".join(report) + ("\n" if report else "") + if args.report: + with open(args.report, "w") as f: + f.write(text) + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hw/syn/altera/dut/common.mk b/hw/syn/altera/dut/common.mk index 962d005e46..d0d786d1df 100644 --- a/hw/syn/altera/dut/common.mk +++ b/hw/syn/altera/dut/common.mk @@ -54,7 +54,7 @@ all: gen-sources $(PROJECT).sta.rpt $(PROJECT).pow.rpt gen-sources: src src: mkdir -p src - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -Csrc $(RTL_PKGS) + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -F -Csrc $(RTL_PKGS) syn: $(PROJECT).syn.rpt diff --git a/hw/syn/common.mk b/hw/syn/common.mk index f254765268..333f6a10d1 100644 --- a/hw/syn/common.mk +++ b/hw/syn/common.mk @@ -23,8 +23,10 @@ endif # Tool install prefixes (overridable; default under $(TOOLDIR)). CIRCT is the # SystemVerilog frontend of every flow: its preprocessor (gen_sources.sh -P) -# and its elaboration into the one netlist a synthesis tool reads -# (circt_elaborate.sh; gen_sources.sh -E). +# feeds Vivado, and its elaboration into one netlist (circt_elaborate.sh; +# gen_sources.sh -E) feeds Yosys, Beacon and Foldsim. Before either, the RAM +# fold (gen_sources.sh -F, hw/scripts/ram_fold.py) patches the preprocessed +# sources so that asynchronous RAM reads map to block RAM. CIRCT_PATH ?= $(TOOLDIR)/circt YOSYS_PATH ?= $(TOOLDIR)/yosys STA_PATH ?= $(TOOLDIR)/sta @@ -41,5 +43,7 @@ FOLDSIM_LIFT_TOOL ?= $(FOLDSIM_PATH)/bin/foldsim-lift YOSYS ?= $(YOSYS_PATH)/bin/yosys STA ?= $(STA_PATH)/bin/sta VERILATOR ?= $(VERILATOR_PATH)/bin/verilator +# a python3 with pyslang 11 or 12 (pip install "pyslang>=11,<13") +RAM_FOLD_PYTHON ?= python3 -export CIRCT_PATH FOLDSIM_PATH FOLDSIM_SHELL FOLDSIM_LIFT_TOOL YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL FOLDSIM FOLDSIM_OPT YOSYS STA VERILATOR +export CIRCT_PATH FOLDSIM_PATH FOLDSIM_SHELL FOLDSIM_LIFT_TOOL YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL FOLDSIM FOLDSIM_OPT YOSYS STA VERILATOR RAM_FOLD_PYTHON diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index c5f02886e0..e94cb94ccd 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -368,6 +368,9 @@ $(VIVADO_SRCS_LIST): $(RTL_SRCS) { echo "ERROR: $$f no longer selects its vendor core by VENDOR_OK"; exit 1; }; \ sed -i 's/localparam VENDOR_OK = 1;/localparam VENDOR_OK = 0;/' $(VIVADO_SRCS_DIR)/$$f; \ done + @# the RAM fold runs here, not in gen_sources.sh (-F), so it patches the sources as read + $(RAM_FOLD_PYTHON) $(SCRIPT_DIR)/ram_fold.py --top VX_firesim_wrap --report $(VIVADO_SRCS_DIR)/ram_fold.rpt \ + --diff $(VIVADO_SRCS_DIR)/ram_fold.diff $$(grep -E '\.s?v$$' $(VIVADO_SRCS_LIST)) @# gen_sources.sh lists packages in find(1) order, which is directory order, @# not dependency order. Synthesis reorders on its own; a simulator compiles @# the list as given. Under -DSIMULATION VX_trace_pkg imports VX_gpu_pkg, and diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile index ae4feee805..62df5ff58a 100644 --- a/hw/syn/foldsim/Makefile +++ b/hw/syn/foldsim/Makefile @@ -158,6 +158,9 @@ $(SRCS_LIST): $(RTL_SRCS) $(IP_STUBS) $(SCRIPT_DIR)/xilinx_ip_gen.tcl rm -f $(SRCS_DIR)/$$f; \ grep -v "/$$f$$" $@.tmp > $@.tmp2 && mv $@.tmp2 $@.tmp; \ done + @# the RAM fold runs here, not in gen_sources.sh (-F), so it patches the sources as read + $(RAM_FOLD_PYTHON) $(SCRIPT_DIR)/ram_fold.py --top $(TARGET_TOP) --report $(SRCS_DIR)/ram_fold.rpt \ + --diff $(SRCS_DIR)/ram_fold.diff $$(grep -E '\.s?v$$' $@.tmp) @if grep -qE 'localparam +VX_MEM_PORTS *= *0 *;' $(SRCS_DIR)/VX_gpu_pkg.sv; then \ echo "ERROR: generated sources resolve VX_MEM_PORTS to 0 (no memory interface)"; exit 1; \ fi diff --git a/hw/syn/synopsys/Makefile b/hw/syn/synopsys/Makefile index 61bdbd8341..f457f6f24c 100644 --- a/hw/syn/synopsys/Makefile +++ b/hw/syn/synopsys/Makefile @@ -154,7 +154,7 @@ all: synthesis gen-sources: $(BUILD_DIR)/sources.txt $(BUILD_DIR)/sources.txt: - $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -C$(BUILD_DIR)/src -O$(BUILD_DIR)/sources.txt + $(SCRIPT_DIR)/gen_sources.sh -P -F -T$(TOP_LEVEL_ENTITY) $(CFLAGS) -C$(BUILD_DIR)/src -O$(BUILD_DIR)/sources.txt synthesis: $(BUILD_DIR)/sources.txt cd $(BUILD_DIR); \ diff --git a/hw/syn/xilinx/common.mk b/hw/syn/xilinx/common.mk index 7591821ef4..0a10d046ae 100644 --- a/hw/syn/xilinx/common.mk +++ b/hw/syn/xilinx/common.mk @@ -104,13 +104,12 @@ XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_D check-axil-blocks: $(if $(filter-out 0,$(PLATFORM_AXIL_BLOCK_BYTES)),@python3 $(SCRIPT_DIR)/check_axil_blocks.py $(RTL_DIR)/cp/VX_cp_axil_regfile.sv) -# The sources are elaborated by CIRCT into the one netlist Vivado reads -# (gen_sources.sh -E); the Xilinx floating-point cores stay instances, bound to +# Vivado reads the preprocessed sources (gen_sources.sh -P), so its own +# inference sees the RTL as written; the Xilinx floating-point cores bind to # the cores xilinx_ip_gen.tcl generates. gen-sources: $(BUILD_DIR)/sources.txt $(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt \ - -Tvortex_afu -E -X$(SCRIPT_DIR)/xilinx_ip_stubs.sv + mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P -F -Tvortex_afu $(CFLAGS) -Csrc -Osources.txt # The preprocessed sources bake in the kernel's memory port count (the flow's # AFU_MEM_PORTS): regenerate them whenever the configuration changes it. diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index a36655468e..fbc0b57493 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -60,8 +60,7 @@ all: $(PROJECT).xpr gen-sources: project_1/sources.txt project_1/sources.txt: mkdir -p project_1 - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -Cproject_1/src -Oproject_1/sources.txt $(RTL_PKGS) \ - -E -X$(SCRIPT_DIR)/xilinx_ip_stubs.sv + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -F -Cproject_1/src -Oproject_1/sources.txt $(RTL_PKGS) build: $(PROJECT).xpr $(PROJECT).xpr: project_1/sources.txt @@ -109,7 +108,7 @@ BEACON_MAP = $(if $(beacon_keep),foldsim-keep-modules{modules=$(beacon_keep)} BEACON_PORTS = $(if $(beacon_keep),ports=beacon/ports.csv port-modules=$(beacon_keep)) beacon: mkdir -p beacon - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -Obeacon/sources.f $(RTL_PKGS) + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -F -Cbeacon/src -Obeacon/sources.f $(RTL_PKGS) $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t $(TOP_LEVEL_ENTITY) -l beacon/sources.f \ -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r beacon/memories.rpt -o beacon/netlist.mlir $(FOLDSIM_OPT) beacon/netlist.mlir --pass-pipeline='builtin.module($(BEACON_MAP))' \ diff --git a/hw/syn/xilinx/sandbox/Makefile b/hw/syn/xilinx/sandbox/Makefile index d2eda595eb..b82cee5f6c 100644 --- a/hw/syn/xilinx/sandbox/Makefile +++ b/hw/syn/xilinx/sandbox/Makefile @@ -74,8 +74,7 @@ simulate.tcl: $(SRC_DIR)/simulate.tcl gen-sources: project_1/sources.txt project_1/sources.txt: mkdir -p project_1 - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -P -Cproject_1/src -Oproject_1/sources.txt \ - -TVortex_top -E -X$(SCRIPT_DIR)/xilinx_ip_stubs.sv + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -P -F -TVortex_top -Cproject_1/src -Oproject_1/sources.txt build: done.dcp done.dcp: project_1/sources.txt kernel.bin.coe project.tcl diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index b2c3351ac8..2613bc6044 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -202,7 +202,7 @@ $(BUILD_DIR): mkdir -p $@ $(BUILD_DIR)/src: | $(BUILD_DIR) - $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -C$(BUILD_DIR)/src + $(SCRIPT_DIR)/gen_sources.sh -P -F -T$(TOP_LEVEL_ENTITY) $(CFLAGS) -C$(BUILD_DIR)/src # CIRCT elaborates the design into one Verilog-2005 file; the black-box # modules (SRAM macros) stay instances with their parameter values, bound to From fc795c16e3bb9bc4d9be81d73bb83ad240944913 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 06:54:33 -0800 Subject: [PATCH 311/336] fifo_queue: a block-RAM-sized queue reads its head through a registered address A queue without an output register reads its head, ram[rd_ptr_r], combinationally, so at block-RAM size it became distributed RAM: the graphics DUTs hold four 1024x132 and four 64x130 of them. The RAM fold cannot convert them, because the queue's RAM instantiation also makes the registered-read RAM of OUT_REG=1 queues. At block-RAM size the RAM now registers rd_ptr_r's next value itself (0 on reset, as rd_ptr_r), which reads the same word on every cycle. Smaller queues keep the combinational read. Verified: the old and new queue in lockstep for 200k cycles of fill/drain traffic at 1024x132, 64x130 and 8x16, simulation and Vivado synthesis models, data_out and empty identical on every cycle. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/libs/VX_fifo_queue.sv | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/hw/rtl/libs/VX_fifo_queue.sv b/hw/rtl/libs/VX_fifo_queue.sv index 48dc0846eb..145989ee3c 100644 --- a/hw/rtl/libs/VX_fifo_queue.sv +++ b/hw/rtl/libs/VX_fifo_queue.sv @@ -92,18 +92,28 @@ module VX_fifo_queue #( end end - // OUT_REG=0: asynchronous read of rd_ptr_r (combinational head). + // OUT_REG=0: the head is ram[rd_ptr_r]. Read combinationally, except at + // block-RAM size, where the RAM registers rd_ptr_r's next value itself + // (registered-address form) and so reads the same word on every cycle. // OUT_REG=1: synchronous BRAM. rd_ptr_r already leads the head by one, so // present the look-ahead address (rd_ptr_n, the next read pointer): the // registered read then returns ram[rd_ptr_r] on the same cycle the async // read would have, making the output bit-identical while letting Vivado // infer native BRAM. - wire [ADDRW-1:0] rd_addr = (OUT_REG != 0) ? rd_ptr_n : rd_ptr_r; + localparam RAM_OUT_REG = (OUT_REG != 0) || (LUTRAM == 0 && `FORCE_BRAM(DEPTH, DATAW)); + wire [ADDRW-1:0] rd_addr; + if (OUT_REG != 0) begin : g_rd_lookahead + assign rd_addr = rd_ptr_n; + end else if (RAM_OUT_REG) begin : g_rd_next + assign rd_addr = reset ? '0 : rd_ptr_n; + end else begin : g_rd_head + assign rd_addr = rd_ptr_r; + end VX_dp_ram #( .DATAW (DATAW), .SIZE (DEPTH), - .OUT_REG (OUT_REG), + .OUT_REG (RAM_OUT_REG), .LUTRAM (LUTRAM), .RDW_MODE ("W") ) dp_ram ( From 777748f7eb66083e313fc6e2b2b26d68a45618d4 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 09:24:51 -0800 Subject: [PATCH 312/336] lsu: explicit local (vx_lds/vx_sts) and global (vx_ldg/vx_stg) loads and stores Mainstream GPU ISAs give shared-memory and global accesses their own instructions (NVIDIA LDS/STS and LDG/STG, AMD ds_* and global_*) on top of generic loads that test the shared window in the LSU. This adds the same pair to Vortex, as the base of wide local loads: a SimX measurement of sgemm2 with one vx_flds.128 per four A-tile reads cuts cycles by 9% to 20% where the kernel is issue-bound. vx_lds/vx_sts (custom-3) address local memory by the low LMEM_LOG_SIZE bits of the effective address, so a local pointer may hold the window address or a bare offset, and casts stay no-ops. imm[11] selects the FP register file; offsets are unscaled bytes (loads simm11, stores simm9 with the size in imm[10:9]). The request is marked local by one op_args bit, the struct's last spare bit, so INST_ARGS_BITS stays 27. SimX moves a local address into the window so every unit that classifies by address sees it as local. vx_ldg/vx_stg (custom-0 funct7 0x08-0x17) take the generic path: the local window is reserved, so no global pointer lies in it. The 5-bit offset is unscaled bytes; scaling it by the access size put a funct3 shift ahead of the decode mux and cost two Beacon levels on decode -> ibuffer. The cache hint is decoded and ignored until a hint is measured to help. Standard loads and stores stay generic, so libraries, inline assembly and pointers loaded from memory keep working on local addresses. Verified: tests/regression/lds_ldg (every form, the offset limits, window and bare-offset local addresses, generic store then LDS) passes on simx, rtlsim and a debug-trace rtlsim; lds-ldg-1/-2 (simx, rtlsim) and the parity-lds-ldg model-parity case pass; with SimX local routing disabled the test fails with 288 errors; lmem-access* and parity-sgemm2-lmem* (13) and the full simx regression (96) pass. Beacon core against the parent: no regression; the instruction buffer's worst end point stays at 10 levels, with more bits at 9-10 from the wider decode mux (watch; core's deepest path is 15). Not run: FPGA/ASIC synthesis of this change, XLEN 64, xrt. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 6 + ci/testcases/regression.yaml | 21 ++ .../custom_accelerator_isa_extensions.md | 14 +- hw/rtl/VX_gpu_pkg.sv | 6 +- hw/rtl/VX_trace_pkg.sv | 21 ++ hw/rtl/core/VX_decode.sv | 62 +++++ hw/rtl/core/VX_lsu_block.sv | 17 +- sim/simx/decode.cpp | 62 +++++ sim/simx/lsu_unit.cpp | 10 + sim/simx/types.h | 1 + tests/regression/Makefile | 2 +- tests/regression/lds_ldg/Makefile | 16 ++ tests/regression/lds_ldg/common.h | 40 +++ tests/regression/lds_ldg/kernel.cpp | 229 ++++++++++++++++++ tests/regression/lds_ldg/main.cpp | 170 +++++++++++++ 15 files changed, 668 insertions(+), 9 deletions(-) create mode 100644 tests/regression/lds_ldg/Makefile create mode 100644 tests/regression/lds_ldg/common.h create mode 100644 tests/regression/lds_ldg/kernel.cpp create mode 100644 tests/regression/lds_ldg/main.cpp diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index e38cc8f049..1792f1603d 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -265,6 +265,12 @@ tests: app: lmem_access args: -n16 -s configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 +# Explicit local and global loads and stores: SimX and RTL route them alike. +- id: parity-lds-ldg + check: model_parity + via: blackbox + app: lds_ldg + configs: -DVX_CFG_NUM_THREADS=16 - id: parity-sgemm2-lmem check: model_parity via: blackbox diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 099df6752b..aeddaab0a2 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -62,6 +62,27 @@ tests: configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 shape: threads: 16 +# Explicit local (vx_lds/vx_sts) and global (vx_ldg/vx_stg) loads and stores: +# every width, the offset limits, and local addresses given as a bare offset. +- id: lds-ldg-1 + via: blackbox + drivers: + - simx + - rtlsim + app: lds_ldg + shape: + threads: 16 +# Two LSU blocks: local requests on the second engine. +- id: lds-ldg-2 + via: blackbox + drivers: + - simx + - rtlsim + app: lds_ldg + configs: -DVX_CFG_ISSUE_WIDTH=2 + shape: + threads: 32 + warps: 8 - id: occupancy-1 via: blackbox drivers: diff --git a/docs/designs/custom_accelerator_isa_extensions.md b/docs/designs/custom_accelerator_isa_extensions.md index a4593709a3..30eadb3624 100644 --- a/docs/designs/custom_accelerator_isa_extensions.md +++ b/docs/designs/custom_accelerator_isa_extensions.md @@ -226,6 +226,8 @@ the table in the same change. | | | `0x02` | tensor core (WMMA/WGMMA) | R | | | | `0x03` | DXA async copy | R | | | | `0x04` | packed loads (`vx_packl*`) | R | +| | | `0x08`–`0x0F` | `vx_ldg`: `funct7[2]` FP, `[1:0]` cache hint; `rs2` = uimm5 byte offset | R | +| | | `0x10`–`0x17` | `vx_stg`: `funct7[2]` FP, `[1:0]` cache hint; `rd` = uimm5 byte offset | R | | custom-1 `0x2B` | `funct3` | `0` | `vx_wgather` | R4 | | | | `3` | `vx_om_export` | R4 | | | | `5` | `vx_tex` | R4 | @@ -233,7 +235,17 @@ the table in the same change. | | | `7` | `vx_rt_trace`, `vx_rt_wait` | R4 | | | | `1`, `2`, `4` | free | | | custom-2 `0x5B` | `funct7` | `0x00` | MUFU special functions; `fn` in `rs2`, `fmt` in `funct3` | R | -| custom-3 `0x7B` | | | free | | +| custom-3 `0x7B` | `funct3` | `0`–`6` | `vx_lds` (LOAD widths); `imm[11]` FP, `imm[10:0]` byte offset | I | +| | | `7` | `vx_sts`; `imm[11]` FP, `imm[10:9]` size, `imm[8:0]` byte offset | S | + +`vx_lds`/`vx_sts` address local memory by the low `LMEM_LOG_SIZE` bits of the +effective address, so a local pointer may hold the window address or a bare +offset. `vx_ldg`/`vx_stg` take the generic path: the local-memory window is +reserved, so no global pointer falls in it. Their hint is 0 (default), 1 +(`.cg`) or 2 (`.cs`); 3 is reserved. The FP forms with `funct3`/size 0 and the +integer-only codes with `imm[11]` set are reserved for wide local accesses, as +are `vx_ldg` with `funct3` 7 and `vx_stg` with `funct3` 4–7. Standard loads and +stores stay generic. --- diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index a9f7ab8c1e..605ce2a150 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1159,10 +1159,10 @@ package VX_gpu_pkg; // words, and VX_gfx_uops emits one uop per set bit. // Zero = not an export. Read ONLY by VX_gfx_uops; the LSU never sees it set // (the expander clears it), so the LSU stays a general-purpose unit. - // Spends two of the struct's spare padding bits -- INST_ARGS_BITS is - // unchanged (the PACKAGE_ASSERT below proves it). + // is_local marks vx_lds/vx_sts: the access goes to local memory at the + // low bits of its address, whatever the address window says. typedef struct packed { - logic [(INST_ARGS_BITS-1-1-12-2-1-1-$bits(amo_op_e)-2-2)-1:0] __padding; + logic is_local; logic [1:0] export_mask; logic amo_valid; logic amo_unsigned; diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index 27a24f4a41..3349343a85 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -180,6 +180,27 @@ package VX_trace_pkg; `TRACE(level, ("PACKLH.F")) end endcase + end else if (op_args.lsu.is_local) begin + if (op_args.lsu.is_float) begin + `TRACE(level, ("VX_F")) + end else begin + `TRACE(level, ("VX_")) + end + if (op_args.lsu.is_store) begin + `TRACE(level, ("STS.")) + end else begin + `TRACE(level, ("LDS.")) + end + case (INST_LSU_BITS'(op_type)) + INST_LSU_LB, INST_LSU_SB: `TRACE(level, ("B")) + INST_LSU_LH, INST_LSU_SH: `TRACE(level, ("H")) + INST_LSU_LW, INST_LSU_SW: `TRACE(level, ("W")) + INST_LSU_LD, INST_LSU_SD: `TRACE(level, ("D")) + INST_LSU_LBU: `TRACE(level, ("BU")) + INST_LSU_LHU: `TRACE(level, ("HU")) + INST_LSU_LWU: `TRACE(level, ("WU")) + default: `TRACE(level, ("?")) + endcase end else if (op_args.lsu.is_float) begin case (INST_LSU_BITS'(op_type)) INST_LSU_LW: `TRACE(level, ("FLW")) diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 636e6f1075..a21bfe6984 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -222,6 +222,7 @@ module VX_decode import // address of every store in the program. op_args.lsu.export_mask = 2'b00; `endif + op_args.lsu.is_local = 1'b0; case (opcode) INST_I: begin @@ -488,6 +489,36 @@ module VX_decode import `USED_IREG (rs2); `endif end + `ifdef VX_CFG_LMEM_ENABLE + // vx_lds/vx_sts: imm[11] selects the FP register file; a store + // carries its size in imm[10:9]. + INST_EXT4: begin + ex_type = EX_LSU; + op_args.lsu.is_float = instr[31]; + op_args.lsu.pack = 0; + op_args.lsu.is_local = 1; + `USED_IREG (rs1); + if (funct3 == 3'h7) begin + op_type = INST_OP_BITS'({2'b10, instr[30:29]}); + op_args.lsu.is_store = 1; + op_args.lsu.offset = {{3{s_imm[8]}}, s_imm[8:0]}; + `ifdef VX_CFG_EXT_F_ENABLE + `USED_REG (instr[31], rs2, 1'b1); + `else + `USED_IREG (rs2); + `endif + end else begin + op_type = INST_OP_BITS'({1'b0, funct3}); + op_args.lsu.is_store = 0; + op_args.lsu.offset = {u_12[10], u_12[10:0]}; + `ifdef VX_CFG_EXT_F_ENABLE + `USED_REG (instr[31], rd, 1'b1); + `else + `USED_IREG (rd); + `endif + end + end + `endif `ifdef VX_CFG_EXT_F_ENABLE INST_FMADD, // 7'b1000011 INST_FMSUB, // 7'b1000111 @@ -815,6 +846,37 @@ module VX_decode import default:; endcase end + // vx_ldg/vx_stg: global accesses through the generic path, the + // window being reserved for local memory. funct7[2] selects the + // FP register file; the cache hint in funct7[1:0] is advisory. + 7'h08, 7'h09, 7'h0A, 7'h0B, 7'h0C, 7'h0D, 7'h0E, 7'h0F: begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'({1'b0, funct3}); + op_args.lsu.is_store = 0; + op_args.lsu.is_float = funct7[2]; + op_args.lsu.pack = 0; + op_args.lsu.offset = 12'(rs2); + `USED_IREG (rs1); + `ifdef VX_CFG_EXT_F_ENABLE + `USED_REG (funct7[2], rd, 1'b1); + `else + `USED_IREG (rd); + `endif + end + 7'h10, 7'h11, 7'h12, 7'h13, 7'h14, 7'h15, 7'h16, 7'h17: begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'({2'b10, funct3[1:0]}); + op_args.lsu.is_store = 1; + op_args.lsu.is_float = funct7[2]; + op_args.lsu.pack = 0; + op_args.lsu.offset = 12'(rd); + `USED_IREG (rs1); + `ifdef VX_CFG_EXT_F_ENABLE + `USED_REG (funct7[2], rs2, 1'b1); + `else + `USED_IREG (rs2); + `endif + end default:; endcase end diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 43f764107e..eee402e9d5 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -137,6 +137,15 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // carried through the scheduler's USER channel and out via the // lsu_mem_if's per-lane req_data.user field. + // vx_lds/vx_sts address local memory by the low bits alone, so a bare + // offset that falls in another window is still local. +`ifdef VX_CFG_LMEM_ENABLE + wire req_local = execute_if.data.op_args.lsu.is_local; +`else + wire req_local = 1'b0; + `UNUSED_VAR (execute_if.data.op_args.lsu.is_local) +`endif + mem_bus_attr_t [NUM_LANES-1:0] mem_req_attr_struct; wire [NUM_LANES-1:0][MEM_ATTR_WIDTH-1:0] mem_req_attr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_mem_req_attr @@ -159,7 +168,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // the store off the L1->L2 trunk on that bit alone. wire [MEM_ADDRW-1:0] om_addr_start = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_OM_BASE_ADDR) >> MEM_ASHIFT); wire [MEM_ADDRW-1:0] om_addr_end = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_OM_END_ADDR) >> MEM_ASHIFT); - wire lane_is_om = (block_addr >= om_addr_start) && (block_addr < om_addr_end); + wire lane_is_om = (block_addr >= om_addr_start) && (block_addr < om_addr_end) && ~req_local; assign mem_req_attr_struct[i].is_addr_om = lane_is_om; `else wire lane_is_om = 1'b0; @@ -168,12 +177,12 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // a fence's token carries its kind: io for a fence ordering only device // accesses, and for an acquire fence's (byte enables set: no drain owed) assign mem_req_attr_struct[i].is_addr_io = req_is_fence ? (req_fence_io || req_fence_acq) - : (((block_addr >= io_addr_start) && (block_addr < io_addr_end)) + : (((block_addr >= io_addr_start) && (block_addr < io_addr_end) && ~req_local) || lane_is_om); `ifdef VX_CFG_LMEM_ENABLE // is local memory address - assign mem_req_attr_struct[i].is_addr_local = - (block_addr[MEM_ADDRW-1:LMEM_BLK_LSB] == LMEM_BLK_BASE[MEM_ADDRW-1:LMEM_BLK_LSB]); + assign mem_req_attr_struct[i].is_addr_local = req_local + || (block_addr[MEM_ADDRW-1:LMEM_BLK_LSB] == LMEM_BLK_BASE[MEM_ADDRW-1:LMEM_BLK_LSB]); `else assign mem_req_attr_struct[i].is_addr_local = 1'b0; `endif diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index bdc564d16e..c9805efef2 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -267,6 +267,10 @@ static op_string_t op_string(const Instr &instr) { case LsuType::LOAD: { auto lsuArgs = std::get(instrArgs); bool dst_float = (instr.get_dest_reg().type == RegType::Float); + if (lsuArgs.is_local) { + static const char* const widths[] = {"B", "H", "W", "D", "BU", "HU", "WU"}; + return {std::string(dst_float ? "VX_FLDS." : "VX_LDS.") + widths[lsuArgs.width], to_hex_str(lsuArgs.offset)}; + } if (instr.is_macro_op() && dst_float && lsuArgs.width == 0) { return {"PACKLB.F", ""}; } @@ -299,6 +303,10 @@ static op_string_t op_string(const Instr &instr) { auto lsuArgs = std::get(instrArgs); // For stores, the data source (rs2) carries the type info. bool src_float = (instr.get_src_reg(1).type == RegType::Float); + if (lsuArgs.is_local) { + static const char* const widths[] = {"B", "H", "W", "D"}; + return {std::string(src_float ? "VX_FSTS." : "VX_STS.") + widths[lsuArgs.width], to_hex_str(lsuArgs.offset)}; + } if (src_float) { switch (lsuArgs.width) { case 1: return {"FSH", to_hex_str(lsuArgs.offset)}; @@ -1047,6 +1055,33 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { } instr->set_macro_op(); } break; + // vx_ldg/vx_stg: global accesses through the generic path, the window + // being reserved for local memory. funct7[2] selects the FP register + // file; the cache hint in funct7[1:0] is advisory. + case 0x08: case 0x09: case 0x0a: case 0x0b: + case 0x0c: case 0x0d: case 0x0e: case 0x0f: { + bool is_float = (funct7 >> 2) & 0x1; + if ((funct7 & 0x3) == 3 || funct3 == 7 || (is_float && (funct3 == 0 || funct3 > 3))) { + std::abort(); + } + instr->set_fu_type(FUType::LSU); + instr->set_op_type(LsuType::LOAD); + instr->set_dest_reg(rd, is_float ? RegType::Float : RegType::Integer); + instr->set_src_reg(0, rs1, RegType::Integer); + instr->set_args(IntrLsuArgs{funct3, /*stride*/ 0, int32_t(rs2)}); + } break; + case 0x10: case 0x11: case 0x12: case 0x13: + case 0x14: case 0x15: case 0x16: case 0x17: { + bool is_float = (funct7 >> 2) & 0x1; + if ((funct7 & 0x3) == 3 || funct3 > 3 || (is_float && funct3 == 0)) { + std::abort(); + } + instr->set_fu_type(FUType::LSU); + instr->set_op_type(LsuType::STORE); + instr->set_src_reg(0, rs1, RegType::Integer); + instr->set_src_reg(1, rs2, is_float ? RegType::Float : RegType::Integer); + instr->set_args(IntrLsuArgs{funct3, /*stride*/ 0, int32_t(rd)}); + } break; default: std::abort(); } @@ -1223,6 +1258,33 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_dest_reg(rd, RegType::Float); instr->set_src_reg(0, rs1, RegType::Float); } break; +#endif +#ifdef VX_CFG_LMEM_ENABLE + case Opcode::EXT4: { + // vx_lds/vx_sts: imm[11] selects the FP register file; a store carries + // its size in imm[10:9]. The remaining codes are reserved. + instr->set_fu_type(FUType::LSU); + instr->set_src_reg(0, rs1, RegType::Integer); + bool is_float = (funct7 >> 6) & 0x1; + if (funct3 == 7) { + uint32_t size = (funct7 >> 4) & 0x3; + if (is_float && size == 0) { + std::abort(); + } + uint32_t imm9 = ((funct7 & 0xf) << width_reg) | rd; + instr->set_src_reg(1, rs2, is_float ? RegType::Float : RegType::Integer); + instr->set_op_type(LsuType::STORE); + instr->set_args(IntrLsuArgs{size, /*stride*/ 0, (int32_t)sext(imm9, 9), /*is_local*/ true}); + } else { + if (is_float && (funct3 == 0 || funct3 > 3)) { + std::abort(); + } + uint32_t imm11 = (code >> shift_rs2) & 0x7ff; + instr->set_dest_reg(rd, is_float ? RegType::Float : RegType::Integer); + instr->set_op_type(LsuType::LOAD); + instr->set_args(IntrLsuArgs{funct3, /*stride*/ 0, (int32_t)sext(imm11, 11), /*is_local*/ true}); + } + } break; #endif default: std::abort(); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 038be7bbfc..e1db778b3a 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -164,6 +164,7 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { bool is_write; int64_t offset; uint32_t stride; + [[maybe_unused]] bool is_local = false; if (is_amo) { auto amo_args = std::get(trace->instr_ptr->get_args()); data_bytes = 1u << (amo_args.width & 0x3); @@ -176,6 +177,7 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { is_write = (lsu_tag && *lsu_tag == LsuType::STORE); offset = lsu_args.offset; stride = lsu_args.stride; + is_local = lsu_args.is_local; if (is_write && lsu_args.width > 3) std::abort(); } @@ -199,6 +201,14 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { // upper bits into the 64-bit address field. e.addr = Word(rs1_data[t].i + (uint64_t)stride * rs2_data[t].u + offset); e.size = data_bytes; + #ifdef VX_CFG_LMEM_ENABLE + if (is_local) { + // Local memory decodes only the low bits; the window address + // carries the request to it. + constexpr uint64_t lmem_mask = (uint64_t(1) << VX_CFG_LMEM_LOG_SIZE) - 1; + e.addr = uint64_t(VX_MEM_LMEM_BASE_ADDR) | (e.addr & lmem_mask); + } else + #endif if (StackInterleave::contains(e.addr)) { // Interleaving is word-granular; a wider access would straddle two threads. if (e.size > StackInterleave::WORD_SIZE) { diff --git a/sim/simx/types.h b/sim/simx/types.h index 60fd7b531a..9beeb26df5 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -384,6 +384,7 @@ struct IntrLsuArgs { uint32_t width : 3; uint32_t stride : 4; // per-uop multiplier: 0 = regular LD/ST; 0..N-1 = packLD uop_idx int32_t offset; // immediate addr offset (signed); 0 for packLD uops + bool is_local = false; // vx_lds/vx_sts: local memory at the low address bits }; // A fence carries no access width; its width field says what it orders. diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 025260ccdf..e9255f220c 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax uniform_ld lmem_access mufu zfhmin \ + softmax uniform_ld lmem_access lds_ldg mufu zfhmin \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/lds_ldg/Makefile b/tests/regression/lds_ldg/Makefile new file mode 100644 index 0000000000..d5465bb33c --- /dev/null +++ b/tests/regression/lds_ldg/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := lds_ldg + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/lds_ldg/common.h b/tests/regression/lds_ldg/common.h new file mode 100644 index 0000000000..9d089a0a70 --- /dev/null +++ b/tests/regression/lds_ldg/common.h @@ -0,0 +1,40 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Local memory per block: RO_WORDS read-only words, then SLOT_WORDS words per +// thread that it stores to. +#define RO_WORDS 1024 +#define SLOT_WORDS 5 + +// Global input words read past each thread's index (the largest LDG offset). +#define IN_PAD 8 + +// Initial value of local word i and global input word i: bytes differ from +// word to word and about half of them have the sign bit set, so sub-word loads +// see both sign- and zero-extended values. +static inline uint32_t ro_word(uint32_t i) { + return (i * 0x9E3779B1u) ^ 0xA5C30000u; +} +static inline uint32_t in_word(uint32_t i) { + return (i * 0x85EBCA77u) ^ 0x5AC30000u; +} + +// The words a thread stores: local slot word k, and global output word k. +static inline uint32_t slot_word(uint32_t tid, uint32_t k) { + return 0xC0000000u | (tid << 4) | k; +} +static inline uint32_t out_word(uint32_t tid, uint32_t k) { + return 0x30000000u | (tid << 4) | k; +} + +typedef struct { + uint32_t lmem_words; // local-memory words per block + uint64_t in_addr; // global input words, one per thread + IN_PAD + uint64_t out_addr; // global output, SLOT_WORDS words per thread + uint64_t err_addr; // out: per-thread error count + uint64_t dump_addr; // out: each block's final local memory +} kernel_arg_t; + +#endif diff --git a/tests/regression/lds_ldg/kernel.cpp b/tests/regression/lds_ldg/kernel.cpp new file mode 100644 index 0000000000..87dac5c237 --- /dev/null +++ b/tests/regression/lds_ldg/kernel.cpp @@ -0,0 +1,229 @@ +#include +#include "common.h" + +// vx_lds/vx_sts (custom-3) and vx_ldg/vx_stg (custom-0), written as .insn so +// the encodings are tested independently of the compiler. +// vx_lds: I-type, funct3 = LOAD width, imm[11] = FP, imm[10:0] = offset +// vx_sts: S-type, funct3 = 7, imm[11] = FP, imm[10:9] = size, imm[8:0] = offset +// vx_ldg: R-type, funct7 = 0b0001_F_HH, funct3 = LOAD width, rs2 = uimm5 offset +// vx_stg: R-type, funct7 = 0b0010_F_HH, funct3 = size, rd = uimm5 offset + +namespace { + +template +inline uint32_t lds(uintptr_t p) { + uint32_t v; + asm volatile (".insn i 0x7b, %2, %0, %1, %3" : "=r"(v) : "r"(p), "i"(F3), "i"(OFF & 0x7ff) : "memory"); + return v; +} + +template +inline uint32_t flds_w(uintptr_t p) { + float v; + asm volatile (".insn i 0x7b, 2, %0, %1, %2" : "=f"(v) : "r"(p), "i"((OFF & 0x7ff) - 2048) : "memory"); + uint32_t bits; + __builtin_memcpy(&bits, &v, sizeof(bits)); + return bits; +} + +constexpr int sts_imm(int fp, int size, int off) { + return ((fp << 11) | (size << 9) | (off & 0x1ff)) - (fp ? 4096 : 0); +} + +template +inline void sts(uintptr_t p, uint32_t v) { + asm volatile (".insn s 0x7b, 7, %0, %2(%1)" :: "r"(v), "r"(p), "i"(sts_imm(0, SIZE, OFF)) : "memory"); +} + +template +inline void fsts_w(uintptr_t p, uint32_t bits) { + float v; + __builtin_memcpy(&v, &bits, sizeof(v)); + asm volatile (".insn s 0x7b, 7, %0, %2(%1)" :: "f"(v), "r"(p), "i"(sts_imm(1, 2, OFF)) : "memory"); +} + +} // namespace + +#define VX_LDG(f3, f7, imm, p) ({ \ + uint32_t _v; \ + asm volatile (".insn r 0x0b, " #f3 ", " #f7 ", %0, %1, x" #imm : "=r"(_v) : "r"(p) : "memory"); \ + _v; }) + +#define VX_FLDG_W(f7, imm, p) ({ \ + float _f; \ + asm volatile (".insn r 0x0b, 2, " #f7 ", %0, %1, x" #imm : "=f"(_f) : "r"(p) : "memory"); \ + uint32_t _v; \ + __builtin_memcpy(&_v, &_f, sizeof(_v)); \ + _v; }) + +#define VX_STG(f3, f7, imm, p, v) \ + asm volatile (".insn r 0x0b, " #f3 ", " #f7 ", x" #imm ", %0, %1" :: "r"(p), "r"(v) : "memory") + +#define VX_FSTG_W(f7, imm, p, bits) do { \ + float _f; \ + uint32_t _b = (bits); \ + __builtin_memcpy(&_f, &_b, sizeof(_f)); \ + asm volatile (".insn r 0x0b, 2, " #f7 ", x" #imm ", %0, %1" :: "r"(p), "f"(_f) : "memory"); \ +} while (0) + +static inline uint32_t sext8(uint32_t v) { return (uint32_t)(int32_t)(int8_t)v; } +static inline uint32_t sext16(uint32_t v) { return (uint32_t)(int32_t)(int16_t)v; } + +// Each thread checks, against the values the host and the generic path +// define: +// vx_lds - words at the offset limits, through the window address and +// through the bare offset (local memory decodes only the low +// bits); bytes and half-words, signed and unsigned; an FP word +// vx_sts - words at the offset limits, half-words through the bare +// offset, bytes, an FP word; each read back by a generic load +// ordering - a generic store, then a vx_lds of the same word +// vx_ldg - every width from offset 0 to the 31-byte limit, the FP form, +// every hint +// vx_stg - words, half-words, bytes up to the 31-byte limit, the FP form +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = reinterpret_cast(__local_mem()); + auto in = reinterpret_cast(arg->in_addr); + auto out = reinterpret_cast(arg->out_addr); + auto err = reinterpret_cast(arg->err_addr); + auto dump = reinterpret_cast(arg->dump_addr); + const uint32_t lid = threadIdx.x; + const uint32_t bsize = blockDim.x; + const uint32_t tid = blockIdx.x * bsize + lid; + const uint32_t words = arg->lmem_words; + + for (uint32_t i = lid; i < RO_WORDS; i += bsize) { + lmem[i] = ro_word(i); + } + __syncthreads(); + + uint32_t errors = 0; + const uintptr_t window = reinterpret_cast(lmem); + const uintptr_t bare = window - VX_MEM_LMEM_BASE_ADDR; + + // reads reach qi - 256 .. qi + 255 words, inside the read-only words + const uint32_t qi = 384 + (lid * 7) % 256; + const uintptr_t q = window + qi * 4; + const uintptr_t qb = bare + qi * 4; + if (lds<2, 0>(q) != ro_word(qi)) { + ++errors; + } + if (lds<2, 1020>(q) != ro_word(qi + 255)) { + ++errors; + } + if (lds<2, -1024>(q) != ro_word(qi - 256)) { + ++errors; + } + if (lds<2, 0>(qb) != ro_word(qi)) { + ++errors; + } + if (lds<2, 1020>(qb) != ro_word(qi + 255)) { + ++errors; + } + if (lds<2, -1024>(qb) != ro_word(qi - 256)) { + ++errors; + } + + { + const uint32_t v = ro_word(qi); + const uint32_t bo = lid & 3; + const uint32_t ho = lid & 1; + if (lds<0, 0>(q + bo) != sext8(v >> (8 * bo))) { + ++errors; + } + if (lds<4, 0>(q + bo) != ((v >> (8 * bo)) & 0xff)) { + ++errors; + } + if (lds<1, 0>(q + 2 * ho) != sext16(v >> (16 * ho))) { + ++errors; + } + if (lds<5, 0>(q + 2 * ho) != ((v >> (16 * ho)) & 0xffff)) { + ++errors; + } + if (lds<4, 3>(q) != (v >> 24)) { + ++errors; + } + if (lds<0, -1>(q) != sext8(ro_word(qi - 1) >> 24)) { + ++errors; + } + if (lds<1, -2>(qb) != sext16(ro_word(qi - 1) >> 16)) { + ++errors; + } + if (flds_w<4>(q) != ro_word(qi + 1)) { + ++errors; + } + if (flds_w<-1024>(qb) != ro_word(qi - 256)) { + ++errors; + } + } + + { + const uint32_t si = RO_WORDS + lid * SLOT_WORDS; + const uintptr_t s = window + si * 4; + const uintptr_t sb = bare + si * 4; + sts<2, -256>(s + 256, slot_word(tid, 0)); + sts<2, 252>(s + 4 - 252, slot_word(tid, 1)); + sts<1, 0>(sb + 8, slot_word(tid, 2)); + sts<1, 2>(sb + 8, slot_word(tid, 2) >> 16); + fsts_w<12>(s, slot_word(tid, 3)); + sts<0, 16>(s, slot_word(tid, 4)); + sts<0, 17>(s, slot_word(tid, 4) >> 8); + sts<0, -238>(s + 256, slot_word(tid, 4) >> 16); + sts<0, 19>(sb, slot_word(tid, 4) >> 24); + for (uint32_t k = 0; k < SLOT_WORDS; ++k) { + if (lmem[si + k] != slot_word(tid, k)) { + ++errors; + } + } + lmem[si] = ~slot_word(tid, 0); + if (lds<2, 0>(s) != ~slot_word(tid, 0)) { + ++errors; + } + lmem[si] = slot_word(tid, 0); + } + + { + const uintptr_t g = reinterpret_cast(in + tid); + if (VX_LDG(2, 0x08, 0, g) != in_word(tid)) { + ++errors; + } + if (VX_LDG(2, 0x09, 28, g) != in_word(tid + 7)) { + ++errors; + } + if (VX_LDG(0, 0x0a, 1, g) != sext8(in_word(tid) >> 8)) { + ++errors; + } + if (VX_LDG(4, 0x09, 31, g) != (in_word(tid + 7) >> 24)) { + ++errors; + } + if (VX_LDG(1, 0x08, 30, g) != sext16(in_word(tid + 7) >> 16)) { + ++errors; + } + if (VX_LDG(5, 0x08, 0, g) != (in_word(tid) & 0xffff)) { + ++errors; + } + if (VX_FLDG_W(0x0c, 12, g) != in_word(tid + 3)) { + ++errors; + } + if (VX_FLDG_W(0x0e, 28, g) != in_word(tid + 7)) { + ++errors; + } + + const uintptr_t o = reinterpret_cast(out + tid * SLOT_WORDS); + VX_STG(2, 0x10, 0, o, out_word(tid, 0)); + VX_STG(2, 0x11, 28, o + 4 - 28, out_word(tid, 1)); + VX_STG(1, 0x12, 8, o, out_word(tid, 2)); + VX_STG(1, 0x10, 10, o, out_word(tid, 2) >> 16); + VX_FSTG_W(0x14, 12, o, out_word(tid, 3)); + VX_STG(0, 0x10, 16, o, out_word(tid, 4)); + VX_STG(0, 0x12, 31, o + 17 - 31, out_word(tid, 4) >> 8); + VX_STG(0, 0x10, 18, o, out_word(tid, 4) >> 16); + VX_STG(0, 0x10, 19, o, out_word(tid, 4) >> 24); + } + + err[tid] = errors; + + __syncthreads(); + for (uint32_t i = lid; i < words; i += bsize) { + dump[blockIdx.x * words + i] = lmem[i]; + } +} diff --git a/tests/regression/lds_ldg/main.cpp b/tests/regression/lds_ldg/main.cpp new file mode 100644 index 0000000000..7735266ed4 --- /dev/null +++ b/tests/regression/lds_ldg/main.cpp @@ -0,0 +1,170 @@ +// lds_ldg — the explicit local (vx_lds/vx_sts) and global (vx_ldg/vx_stg) +// loads and stores return and store what the generic path does, at every +// width and at the limits of their offsets. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "k:h")) != -1) { + switch (c) { + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // One block per core holding all its warps, two waves of blocks. + uint64_t nt = 0; + uint64_t nw = 0; + uint64_t nc = 0; + uint64_t lmem_size = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + CHECK(vx_device_query(dev, VX_CAPS_LOCAL_MEM_SIZE, &lmem_size)); + const uint32_t block = static_cast(nt * nw); + const uint32_t grid = static_cast(nc * 2); + const uint32_t num_threads = block * grid; + const uint32_t words = RO_WORDS + block * SLOT_WORDS; + std::cout << "lds_ldg: block=" << block << " grid=" << grid << " words=" << words << std::endl; + if (words * sizeof(uint32_t) > lmem_size) { + std::cout << "local memory too small\nFAILED!" << std::endl; + return 1; + } + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + const uint32_t in_words = num_threads + IN_PAD; + std::vector h_in(in_words); + for (uint32_t i = 0; i < in_words; ++i) { + h_in[i] = in_word(i); + } + + vx_buffer_h in_buf = nullptr; + vx_buffer_h out_buf = nullptr; + vx_buffer_h err_buf = nullptr; + vx_buffer_h dump_buf = nullptr; + CHECK(vx_buffer_create(dev, in_words * sizeof(uint32_t), VX_MEM_READ, &in_buf)); + CHECK(vx_buffer_create(dev, num_threads * SLOT_WORDS * sizeof(uint32_t), VX_MEM_WRITE, &out_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); + CHECK(vx_buffer_create(dev, grid * words * sizeof(uint32_t), VX_MEM_WRITE, &dump_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.lmem_words = words; + CHECK(vx_buffer_address(in_buf, &kernel_arg.in_addr)); + CHECK(vx_buffer_address(out_buf, &kernel_arg.out_addr)); + CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); + CHECK(vx_buffer_address(dump_buf, &kernel_arg.dump_addr)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid; + li.block_dim[0]= block; + li.lmem_size = words * sizeof(uint32_t); + + std::vector h_out(num_threads * SLOT_WORDS); + std::vector h_err(num_threads); + std::vector h_dump(grid * words); + vx_event_h write_ev = nullptr; + vx_event_h launch_ev = nullptr; + vx_event_h read_ev = nullptr; + CHECK(vx_enqueue_write(q, in_buf, 0, h_in.data(), in_words * sizeof(uint32_t), 0, nullptr, &write_ev)); + CHECK(vx_enqueue_launch(q, &li, 1, &write_ev, &launch_ev)); + CHECK(vx_enqueue_read(q, h_out.data(), out_buf, 0, h_out.size() * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_dump.data(), dump_buf, 0, h_dump.size() * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::printf("*** thread %u: %u wrong values\n", t, h_err[t]); + } + errors += h_err[t]; + } + for (uint32_t k = 0; k < SLOT_WORDS; ++k) { + uint32_t actual = h_out[t * SLOT_WORDS + k]; + if (actual != out_word(t, k)) { + if (errors < 16) { + std::printf("*** out[%u][%u] expected=0x%x actual=0x%x\n", t, k, out_word(t, k), actual); + } + ++errors; + } + } + } + for (uint32_t b = 0; b < grid; ++b) { + for (uint32_t i = 0; i < words; ++i) { + uint32_t expected = (i < RO_WORDS) ? ro_word(i) + : slot_word(b * block + (i - RO_WORDS) / SLOT_WORDS, (i - RO_WORDS) % SLOT_WORDS); + uint32_t actual = h_dump[b * words + i]; + if (actual != expected) { + if (errors < 16) { + std::printf("*** block %u lmem[%u] expected=0x%x actual=0x%x\n", b, i, expected, actual); + } + ++errors; + } + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_event_release(write_ev); + vx_buffer_release(dump_buf); + vx_buffer_release(err_buf); + vx_buffer_release(out_buf); + vx_buffer_release(in_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} From 8abe25dc44395b6b2c6b1f3b1dfa0b43dd783673 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 13:04:39 -0800 Subject: [PATCH 313/336] beacon: count area in logic_gate (LUT, FF, LUTRAM, BRAM, URAM, DSP) Beacon gated depth only, so a resource jump -- a memory leaving block RAM, DSPs multiplying -- surfaced only in a nightly Vivado run, hours later. This phase's core build is the example: LUTRAM +154% and block RAM -45% took a 79-minute fpga_gate build to see. foldsim's Beacon now counts area too (foldsim-estimate-area, foldsim d8350bc), and Vortex gates it beside depth: - The DUT and Yosys Beacon targets map the netlist once more for area and run foldsim-estimate-area with the same target. - beacon_summary.py records the design's lut, ff, lutram, bram, uram and dsp in beacon_summary.csv, each unit's area in beacon_units.json, and writes beacon_area.txt: the units by LUTs and every memory with its placement, generate copies folded. - logic_gate gates each resource against the baseline at 5%, fpga_gate's LUT tolerance, through the usual threshold path (catalog thresholds and --metric-threshold override it), and shows the columns. - ci/beacon.py run counts area; diff adds the design's resources and the units whose own area moved, and exits 3 on an area-only regression, which --against reports as AREA. A crash exits 4, so it reads as a failed run, not as DEEPER. - docs/designs/beacon.md (the area mapping, the model and its LUT scale, outputs, the gate rule, validation against fresh fpga_gate builds), continuous_integration.md and AGENTS.md. Against the fresh U55C fpga_gate builds of core, cache, tcu and tensor, Beacon's DSPs are within two, block RAM within 14%, LUTs within -14% to +21% and flip-flops 10-32% high; logic_gate compares each DUT with its own baseline, so the per-family LUT scale cancels there. Verified: logic_gate -b om -b tcu end to end with the new foldsim (columns, beacon_area.txt; NO-BASELINE as expected), beacon.py diff on a synthetic area change (exit 3, attributed to the one unit) and on identical runs (exit 0), the area pass on all 13 logic_gate netlists. Not run: the full catalog, the gpu group, the Yosys beacon target end to end. Landing order: refresh the foldsim prebuilt with d8350bc before pushing this, or CI's foldsim-opt lacks foldsim-estimate-area and every logic_gate build fails. The Beacon baselines need a human re-record (the new foldsim makes them STALE in any case). Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- ci/beacon.py | 120 ++++++++++++++-- ci/logic_gate.py | 19 +-- ci/synth_gate.py | 79 ++++++---- docs/designs/beacon.md | 190 +++++++++++++++++++++---- docs/designs/continuous_integration.md | 15 +- hw/scripts/beacon_summary.py | 109 +++++++++++++- hw/syn/xilinx/dut/common.mk | 19 +-- hw/syn/yosys/Makefile | 7 +- 9 files changed, 469 insertions(+), 93 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7a6753dde9..995d39cf47 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,8 +86,8 @@ See [docs/testing.md](docs/testing.md) and [docs/debugging.md](docs/debugging.md - **RTL coverage path is `xrt`, not `rtlsim`.** When discussing or planning RTL verification, `xrt` is the canonical path — `rtlsim` bypasses the AFU surface. `rtlsim` remains useful for fast iteration on processor RTL; `xrt` is what proves the full integration. - **`ci/regression.sh` is the canonical source of tested configurations.** Use it to discover supported parameter combinations before inventing ad hoc ones. - **Perf-regression baselines (`ci/baselines/perf/*.json`) are golden data — never hand-edit them, and never "fix" a red perf gate by bumping the number.** They are regenerated only by `pytest ci -m perf_gate --update-baselines` (a human-run, reviewed step), and CI must never pass that flag. A `perf_gate` failure means real cycles moved: root-cause it, or — if the change is intended — regenerate the baseline so the diff shows the perf delta for review. Same rule as image goldens and `known_issue:`. -- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,beacon}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/logic_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `logic_gate` means a unit got deeper, and its reasons name the unit. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). -- **Beacon first for any RTL change that can move timing; synthesis only to confirm.** A change to a pipeline, arbiter, queue, interface or datapath can add logic levels that only the nightly `fpga_gate` would find, hours later. Beacon finds them in minutes, so iterate on it, not on Vivado or Yosys: run `ci/logic_gate.py -b ` on the parent revision once (its own build tree, e.g. `git worktree add ../ref HEAD` and `--build-dir build32_beacon_ref`), then `ci/logic_gate.py -b --against build32_beacon_ref` after each edit, and fix what `/beacon_diff.txt` reports until nothing fails (`ci/beacon.py paths` shows a deep unit's paths op by op). Run synthesis once per change, on the DUTs whose depth moved, and feed the result back with `ci/beacon.py validate`. A change to the cluster, socket, L2 or anything only the whole GPU assembles also runs the `gpu` group (`ci/logic_gate.py -b gpu -j 1`, ~30 min and ~95 GB per build). A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. Beacon does not see wires, placement or congestion; a change that moves those still needs synthesis. See [docs/designs/beacon.md §6.3](docs/designs/beacon.md). +- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,beacon}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/logic_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `logic_gate` means a unit got deeper or the design's area grew past 5 %, and its reasons name the unit or the resource (`beacon_area.txt` has the area by unit and each memory's placement). Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). +- **Beacon first for any RTL change that can move timing; synthesis only to confirm.** A change to a pipeline, arbiter, queue, interface or datapath can add logic levels that only the nightly `fpga_gate` would find, hours later. Beacon finds them in minutes, so iterate on it, not on Vivado or Yosys: run `ci/logic_gate.py -b ` on the parent revision once (its own build tree, e.g. `git worktree add ../ref HEAD` and `--build-dir build32_beacon_ref`), then `ci/logic_gate.py -b --against build32_beacon_ref` after each edit, and fix what `/beacon_diff.txt` reports until nothing fails (`ci/beacon.py paths` shows a deep unit's paths op by op). Run synthesis once per change, on the DUTs whose depth moved, and feed the result back with `ci/beacon.py validate`. A change to the cluster, socket, L2 or anything only the whole GPU assembles also runs the `gpu` group (`ci/logic_gate.py -b gpu -j 1`, ~30 min and ~95 GB per build). A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. `beacon_diff.txt` also lists each resource of the design (LUT, FF, distributed and block RAM, DSP) and the units whose area moved, so a memory that leaves block RAM shows in minutes. Beacon does not see wires, placement or congestion; a change that moves those still needs synthesis. See [docs/designs/beacon.md §6.3](docs/designs/beacon.md). - **SimX is the RTL's timing model — keep them in lockstep.** Any change that moves RTL cycles (pipeline structure, arbitration, queue depths, cache/memory behavior) must land together with the matching SimX timing-model update, and vice-versa. The `model_parity` CI gate enforces this: a `check: model_parity` case (`ci/testcases/core.yaml` + per-extension parity categories) runs the same app/args/configs on simx and rtlsim and asserts exact retired-instruction match plus cycle agreement within the case tolerance (default 5%). Never widen a tolerance to absorb a divergence — model the behavior. When adding a hardware feature, add or extend a parity case that exercises it. - **When RTL debugging stalls, switch to the SimX-as-oracle pattern.** For numerical bugs, deep pipeline races, or any failure mode where rtlsim is "close but wrong": (1) build/extend the SimX C++ model so it mirrors the *new* RTL architecture and gets to PASS; (2) add matching trace dumps to both SimX and RTL (cycle, FU events, SRAM addresses+data, hazards) — same CSV format on both sides; (3) diff trace files — the first divergence is the bug. Don't keep guessing from output values; localize via trace diff. See [docs/debugging.md](docs/debugging.md#simx-as-oracle-for-rtl-debug). diff --git a/ci/beacon.py b/ci/beacon.py index 652834e4bd..a2af644667 100755 --- a/ci/beacon.py +++ b/ci/beacon.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""beacon -- run Beacon, foldsim's logic-level count, on a Vortex netlist, and -read its results by RTL unit or against Vivado. +"""beacon -- run Beacon, foldsim's logic-level and area count, on a Vortex +netlist, and read its results by RTL unit or against Vivado. run map an elaborated CIRCT netlist and count the logic levels of - every end point (foldsim-estimate-timing) + every end point (foldsim-estimate-timing) and its LUTs, + flip-flops, memories and DSPs (foldsim-estimate-area) summarize the deep end points grouped by RTL unit, optionally beside Vivado's failing ones paths every deep path op by op, by RTL unit (one per pair of @@ -191,7 +192,8 @@ def unit(key): def cmd_run(args): - """Map (unless the input is already mapped bytecode), then count levels.""" + """Map (unless the input is already mapped bytecode), then count levels; + from a netlist, also map for area and count it.""" opt = os.environ.get("FOLDSIM_OPT") or "foldsim-opt" os.makedirs(args.out, exist_ok=True) # Map to the cells of the target the levels are counted on. @@ -223,6 +225,17 @@ def cmd_run(args): steps.append(("levels", [opt, mapped, "-o", os.devnull, "--pass-pipeline=builtin.module(" "foldsim-estimate-timing{%s})" % opts])) + if not args.design.endswith(".mlirbc"): + # Area reads the netlist mapped again, for area. + top = [o for o in args.options if o.startswith(("top=", "black-boxes="))] + area = " ".join([target, "output=%s/area.csv" % args.out, + "memories=%s/area_memories.csv" % args.out, + "report=%s/area.txt" % args.out] + top) + steps.append(("area", [opt, args.design, "-o", os.devnull, + "--pass-pipeline=builtin.module(" + "foldsim-beacon-map{%s area}," + "foldsim-estimate-area{%s})" % (target, area)] + + threads)) for name, cmd in steps: start = time.time() rc = subprocess.call(cmd) @@ -546,7 +559,10 @@ def cmd_diff(args): """A change's effect on depth: every register at or above --min-levels in either run whose depth moved, worst first. Exits 1 on a regression: a register --fail-delta or more levels deeper, one that reaches --fix-levels - from below, or one only the change has at --fix-levels. + from below, or one only the change has at --fix-levels. If both runs + counted area, also the design's resources and the units whose area moved; + exits 3 if only area regressed: a resource of the design more than + --area-tolerance larger. Against routed Vivado runs of the same DUT's revisions, a register that got four or more levels deeper lost slack every time, while a rise of one @@ -566,10 +582,19 @@ def top_mean(regs, n=50): depths = sorted((v[0] for v in regs.values()), reverse=True)[:n] return sum(depths) / len(depths) if depths else 0.0 + area_before, area_after = load_area(args.before), load_area(args.after) + area_head = '' + if bool(area_before) != bool(area_after): + print('area: only the %s run counted it; not compared' + % ('reference' if area_before else 'changed')) + if area_before and area_after: + tb, ta = area_before[''], area_after[''] + area_head = '; ' + ', '.join( + f'{k} {tb[k]:g} -> {ta[k]:g}' for k in ('lut', 'bram', 'dsp')) print(f'deepest path {deepest(before)} -> {deepest(after)} levels; ' f'mean of the 50 deepest {top_mean(before):.1f} -> {top_mean(after):.1f}; ' f'registers at or above {args.min_levels} levels ' - f'{watched(before)} -> {watched(after)}') + f'{watched(before)} -> {watched(after)}{area_head}') deeper, shallower, new, gone = [], [], [], [] for key, (lv, start) in after.items(): @@ -630,9 +655,75 @@ def show(title, rows, moved): for b, a, (cls, u) in moved[:args.top]: print(f'{b:4d} -> {a:<4d} {cls + ": " if cls else ""}{u}') + area_regress = False + if area_before and area_after: + area_regress = show_area(area_before, area_after, args) + # A register the reference lacks may be a renamed one, so it fails only at # the fix threshold, as a new unit does in logic_gate. - return 1 if regress or any(r[0] >= args.fix_levels for r in new) else 0 + if regress or any(r[0] >= args.fix_levels for r in new): + return 1 + return 3 if area_regress else 0 + + +AREA_KEYS = ('lut', 'ff', 'lutram', 'bram', 'uram', 'dsp') + + +def load_area(run_dir): + """The design's resources ('') and each folded unit's own, summed over its + copies, from a run's area.csv, or None. A unit's own resources leave out + its children's, so a change shows at the unit it is in. `lut` counts + distributed RAM too, as Vivado's utilization does.""" + path = f'{run_dir}/area.csv' + if not os.path.exists(path): + return None + units = {} + with open(path) as fh: + for row in csv.DictReader(fh): + prefix = 'total_' if not row['path'] else '' + res = {k: float(row[prefix + k]) for k in AREA_KEYS} + res['lut'] += res['lutram'] + key = unit(norm(row['path']) + '/x') if row['path'] else '' + if key in units: + for k in AREA_KEYS: + units[key][k] += res[k] + else: + units[key] = res + return units if '' in units else None + + +def show_area(before, after, args): + """The design's resources before and after, and the units whose own LUTs, + memories or DSPs moved most; True if a resource of the design grew more + than --area-tolerance.""" + tb, ta = before[''], after[''] + print('\narea (design)') + regress = False + for k in AREA_KEYS: + b, a = tb[k], ta[k] + pct = (a - b) / b * 100 if b else (100.0 if a else 0.0) + flag = '' + if pct > 100 * args.area_tolerance: + flag, regress = ' REGRESSION', True + print(f' {k:7s} {b:10g} -> {a:<10g} {pct:+6.1f}%{flag}') + moved = [] + for key in set(before) | set(after): + if not key: + continue + b = before.get(key, dict.fromkeys(AREA_KEYS, 0.0)) + a = after.get(key, dict.fromkeys(AREA_KEYS, 0.0)) + if any(abs(a[k] - b[k]) >= 0.5 for k in AREA_KEYS): + moved.append((abs(a['lut'] - b['lut']) + 100 * ( + abs(a['bram'] - b['bram']) + abs(a['uram'] - b['uram']) + + abs(a['dsp'] - b['dsp'])), key, b, a)) + if moved: + moved.sort(key=lambda r: -r[0]) + print(f'\nunits whose own area moved ({len(moved)}), all copies') + for _, key, b, a in moved[:args.top]: + parts = [f'{k} {b[k]:g} -> {a[k]:g}' for k in AREA_KEYS + if abs(a[k] - b[k]) >= 0.5] + print(f' {key}: ' + ', '.join(parts)) + return regress def cmd_buffers(args): @@ -1088,7 +1179,8 @@ def main(): p.add_argument('--top', type=int, default=40) p.set_defaults(fn=cmd_ports) - p = sub.add_parser("diff", help="a change's effect on depth, register by register") + p = sub.add_parser("diff", help="a change's effect on depth, register by " + "register, and on area, by unit") p.add_argument('before', help="run directory of the reference (the parent revision)") p.add_argument('after', help="run directory of the change") p.add_argument('--min-levels', type=int, default=8, @@ -1102,6 +1194,9 @@ def main(): p.add_argument('--top', type=int, default=40) p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], help="the runs' column to compare") + p.add_argument('--area-tolerance', type=float, default=0.05, + help="a resource of the design that grows by more than this " + "fraction fails (exit 3)") p.set_defaults(fn=cmd_diff) p = sub.add_parser("buffers", help="elastic buffers that pass deep paths") @@ -1142,4 +1237,11 @@ def main(): if __name__ == '__main__': - sys.exit(main()) + # Exit codes 1 and 3 are regressions to logic_gate (DEEPER, AREA); a crash + # must read as a failed run, not as one. + try: + sys.exit(main()) + except Exception: + import traceback + traceback.print_exc() + sys.exit(4) diff --git a/ci/logic_gate.py b/ci/logic_gate.py index 61a2df84e8..f6f1773657 100755 --- a/ci/logic_gate.py +++ b/ci/logic_gate.py @@ -1,18 +1,21 @@ #!/usr/bin/env python3 -"""logic_gate — logic-level regression gate (Beacon). - -Gates the depth of every RTL unit of the DUT catalog: foldsim-estimate-timing -counts the logic levels of every register-to-register path of the CIRCT netlist -the Xilinx flow elaborates, as Vivado counts them on DEVICE's family, without -running Vivado. A unit that gets one level deeper than its baseline fails, by name, in -seconds -- where ci/fpga_gate.py finds the Fmax it costs the next night. The +"""logic_gate — logic-level and area regression gate (Beacon). + +Gates the depth of every RTL unit of the DUT catalog, and each DUT's area: +foldsim-estimate-timing counts the logic levels of every register-to-register +path of the CIRCT netlist the Xilinx flow elaborates, as Vivado counts them on +DEVICE's family, and foldsim-estimate-area its LUTs, flip-flops, distributed +and block RAM and DSPs, without running Vivado. A unit that gets one level +deeper than its baseline fails, by name, and a resource more than 5% larger, +in seconds -- where ci/fpga_gate.py finds the Fmax or area it costs the next +night. The gating machinery is shared with the synthesis gates and lives in ci/synth_gate.py; this entry point only pins the tool. Spec : ci/testcases/logic_gate.yaml (what to measure) Baselines : ci/baselines/synthesis/beacon/.json (last accepted levels) Builds : hw/syn/xilinx/dut/logic_gate_ (no Vivado) - Metrics : /beacon_summary.csv, beacon_units.json + Metrics : /beacon_summary.csv, beacon_units.json, beacon_area.txt (written by hw/syn/xilinx/dut's `beacon` target) It needs CIRCT and foldsim (ci/toolchain_install.sh --circt --foldsim); the diff --git a/ci/synth_gate.py b/ci/synth_gate.py index 90e92b982c..a4237c0756 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -102,6 +102,9 @@ class Tool: summary = "synth_summary.csv" # Default gate tolerance, overridden by --threshold and the catalog. tolerance = TOLERANCE + # Per-metric defaults over `tolerance`, overridden by --metric-threshold + # and the catalog. + metric_tolerance = {} integral = () # recorded as ints; every other metric is a rounded float columns = () # (header, metric, format, width) for the report table @@ -458,36 +461,49 @@ def describe_baseline(self, base): class Beacon(Tool): - """Beacon: logic levels per RTL unit (hw/syn/xilinx/dut, target - `beacon`). + """Beacon: logic levels per RTL unit and the design's area + (hw/syn/xilinx/dut, target `beacon`). foldsim-estimate-timing ranks every register-to-register path of the CIRCT netlist the Xilinx flow elaborates by its logic levels, counted as Vivado - counts them on DEVICE's family. It needs no Vivado and no licence and - takes seconds per DUT, so it can run on every change where the Fmax gate - runs nightly. + counts them on DEVICE's family; foldsim-estimate-area counts its LUTs, + flip-flops, distributed and block RAM, UltraRAM and DSPs, placing each + memory as Vivado infers it. It needs no Vivado and no licence and takes + seconds per DUT, so it can run on every change where the Fmax gate runs + nightly. Depth is deterministic -- the same RTL and tools give the same levels -- so the gate has no tolerance: a unit that gets deeper fails, by name, until it - is fixed or a human re-records it. What it gates is depth, not Fmax: - Beacon does not see wires (docs/designs/continuous_integration.md 3.5). + is fixed or a human re-records it. Area is as deterministic, but a feature + adds some by design, so a resource of the design fails past its baseline + by more than fpga_gate's LUT tolerance (metric_tolerance), and a memory + that moves between distributed and block RAM shows as both columns + moving. What it gates is depth and cells, not Fmax or placement: Beacon + does not see wires (docs/designs/continuous_integration.md 3.5). """ name = "beacon" gate = "logic_gate" flow = "hw/syn/xilinx/dut" - metrics = ("max_levels", "deep_endpoints", "watch_endpoints", "endpoints", - "build_time_s") - gated = ("max_levels",) - integral = metrics + area_metrics = ("lut", "ff", "lutram", "bram", "uram", "dsp") + metrics = ("max_levels", "deep_endpoints", "watch_endpoints", + "endpoints") + area_metrics + ("build_time_s",) + gated = ("max_levels",) + area_metrics + integral = ("max_levels", "deep_endpoints", "watch_endpoints", + "endpoints", "lut", "ff", "lutram", "uram", "dsp", + "build_time_s") key_metric = "max_levels" summary = "beacon_summary.csv" tolerance = 0.0 + metric_tolerance = dict.fromkeys(area_metrics, 0.05) columns = (("max_lv", "max_levels", "%d", 7), ("deep", "deep_endpoints", "%d", 9), ("watch", "watch_endpoints", "%d", 9), - ("ends", "endpoints", "%d", 9)) + ("ends", "endpoints", "%d", 9), + ("lut", "lut", "%d", 9), ("ff", "ff", "%d", 9), + ("lutram", "lutram", "%d", 9), ("bram", "bram", "%g", 8), + ("uram", "uram", "%d", 6), ("dsp", "dsp", "%d", 6)) # Elaboration and mapping of the widest DUTs; the level count itself is # seconds. @@ -505,12 +521,14 @@ class Beacon(Tool): ("circt_elaborate.sh", "elaboration"), (elab_done, "mapping"), ("foldsim-estimate-timing", "levels"), + ("foldsim-estimate-area", "area"), ("beacon_summary.py", "reporting"), ) error_re = re.compile(r"^(?:ERROR|FATAL|Error)\b|: error:|^\S+: \*\*\* ") UNITS = "beacon_units.json" PATHS = "beacon_paths.txt" + AREA = "beacon_area.txt" def env(self, args): return {"circt": tool_version("circt-verilog", "--version", @@ -607,13 +625,17 @@ def compare(self, metrics, base): def evidence(self, work): # Every unit at or above the watch threshold, its paths op by op: a - # failing unit is a heading there. - path = os.path.join(work, self.PATHS) - return [path] if os.path.exists(path) else [] + # failing unit is a heading there; the area by unit and each memory's + # placement. + return [p for p in (os.path.join(work, self.PATHS), + os.path.join(work, self.AREA)) + if os.path.exists(p)] def describe(self, m): - return "max %d levels, %d end points >= %s" % ( - m["max_levels"], m["deep_endpoints"], m.get("fix_levels")) + area = (", %d LUTs, %g BRAM, %d DSP" % (m["lut"], m["bram"], m["dsp"]) + if m.get("lut") is not None else "") + return "max %d levels, %d end points >= %s%s" % ( + m["max_levels"], m["deep_endpoints"], m.get("fix_levels"), area) def describe_baseline(self, base): return "max %d levels" % base["max_levels"] @@ -1364,9 +1386,10 @@ def cell(key, fmt): def compare_against(results, ref_dir, tool): - """Each finished build's Beacon end points against the same build in - ref_dir, register by register; return the builds a register got deeper - in, as failing verdicts.""" + """Each finished build's Beacon run against the same build in ref_dir: + end points register by register, and area by unit; return the builds a + register got deeper in, or whose area grew past the tolerance, as failing + verdicts.""" script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "beacon.py") failures = [] for r in results: @@ -1388,10 +1411,15 @@ def compare_against(results, ref_dir, tool): with open(out) as fh: head = fh.readline().strip() print("\n%s against %s: %s%s\n see %s" - % (b["id"], ref_dir, head, " -- REGRESSION" if rc == 1 else "", out)) + % (b["id"], ref_dir, head, " -- REGRESSION" if rc in (1, 3) else "", + out)) if rc == 1: failures.append((b["id"], "DEEPER", ["a register got deeper than in " "the reference: %s" % out])) + elif rc == 3: + failures.append((b["id"], "AREA", ["the design's area grew past the " + "tolerance against the " + "reference: %s" % out])) elif rc: failures.append((b["id"], "BUILD-FAIL", ["beacon.py diff failed: %s" % out])) return failures @@ -1441,9 +1469,10 @@ def main(argv=None, default_tool="xilinx"): ap.add_argument("--against", metavar="DIR", help="Beacon only: the build tree of a reference run, " "typically the parent revision's; each build's end " - "points are compared register by register with the " - "same build there (ci/beacon.py diff), and a register " - "that got deeper fails the run") + "points are compared register by register, and its " + "area by unit, with the same build there " + "(ci/beacon.py diff); a register that got deeper, or " + "area grown past the tolerance, fails the run") ap.add_argument("--xlen", type=int, default=32, choices=(32, 64), help="ISA width of the builds that do not pin their own " "(default: 32)") @@ -1477,7 +1506,7 @@ def main(argv=None, default_tool="xilinx"): # happens whenever this is piped -- CI, nohup, a log file). sys.stdout.reconfigure(line_buffering=True) - overrides = dict(METRIC_TOLERANCE) + overrides = dict(METRIC_TOLERANCE, **tool.metric_tolerance) for spec in args.metric_threshold: metric, _, value = spec.partition("=") if metric not in tool.metrics or not value: diff --git a/docs/designs/beacon.md b/docs/designs/beacon.md index f29a70e846..90e51bf777 100644 --- a/docs/designs/beacon.md +++ b/docs/designs/beacon.md @@ -1,21 +1,22 @@ -# Beacon — logic depth of every path, before Vivado +# Beacon — logic depth and area, before Vivado Beacon counts the logic levels of every register-to-register path -of a Vortex design the way Vivado counts them, in seconds and without +of a Vortex design the way Vivado counts them, and the LUTs, flip-flops, +distributed and block RAM and DSPs it maps to, in seconds and without Vivado. It names the RTL units whose paths are too deep for the target clock, -gates their depth on every RTL change (`logic_gate`), and is the first step -of RTL timing work: run Beacon, fix the deep units, then confirm with one -Vivado run. +gates their depth and the design's area on every RTL change (`logic_gate`), +and is the first step of RTL timing work: run Beacon, fix the deep units, +then confirm with one Vivado run. | § | contents | |---|---| | [1](#1-overview) | Overview and how to run it | | [2](#2-system-architecture) | System architecture: where each piece lives | | [3](#3-compiler-architecture) | Compiler architecture: the two passes | -| [4](#4-the-model) | The model: levels per element, devices | +| [4](#4-the-model) | The model: levels per element, devices, area (§4.5) | | [5](#5-outputs) | Outputs | | [6](#6-integration) | Integration: the DUT flow, `logic_gate`, timing work | -| [7](#7-validation) | Validation against Vivado; the learned calibration (§7.1) | +| [7](#7-validation) | Validation against Vivado; the learned calibration (§7.1); area (§7.2) | | [8](#8-diagnostics) | Diagnostics beyond levels | | [9](#9-invariants-and-limitations) | Invariants and limitations | @@ -38,7 +39,8 @@ One DUT of the synthesis catalog, from a configured build tree: ```bash make -C hw/syn/xilinx/dut tcu-beacon # build_tcu/beacon_paths.txt every path at >= 8 levels, op by op, by unit -# build_tcu/beacon_units.json deepest end point per RTL unit +# build_tcu/beacon_units.json deepest end point per RTL unit, area per unit +# build_tcu/beacon_area.txt LUTs, FFs, memories and DSPs, by unit; each memory's placement ``` The whole catalog as a gate, and a baseline update (a human step): @@ -54,7 +56,7 @@ Any CIRCT netlist, with `ci/beacon.py` (subcommands `run`, `foldsim-opt` from the `PATH` or `FOLDSIM_OPT`): ```bash -ci/beacon.py run design.mlir runs/full # map, then count levels +ci/beacon.py run design.mlir runs/full # map, then count levels and area ci/beacon.py run runs/full/mapped.mlirbc runs/socket \ top=VX_socket__lift black-boxes=VX_core__lift # one unit of a mapped design ci/beacon.py summarize runs/full --min-levels 12 # deep end points by RTL unit @@ -62,7 +64,7 @@ ci/beacon.py paths runs/full --unit lsu_unit # its paths, op by op ci/beacon.py validate runs/socket ep.csv --native VX_socket__lift=dut ``` -Or the two passes directly: +Or the passes directly: ```bash foldsim-opt design.mlir --emit-bytecode -o mapped.mlirbc \ @@ -71,6 +73,10 @@ foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( foldsim-estimate-timing{top=VX_core_top target=virtexuplusHBM output=endpoints.csv instances=instances.csv report=report.txt top-paths=50 trace=lsu_unit})' +foldsim-opt design.mlir -o /dev/null --pass-pipeline='builtin.module( + foldsim-beacon-map{target=virtexuplusHBM area}, + foldsim-estimate-area{top=VX_core_top target=virtexuplusHBM + output=area.csv memories=area_memories.csv report=area.txt})' ``` | `foldsim-estimate-timing` option | meaning | @@ -86,6 +92,15 @@ foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( | `retime-levels=` | also estimate the depth after Vivado's retiming to this period (§4.4) | | `ports=`, `port-modules=` | the depth at each port of the named modules' instances (§8) | +| `foldsim-estimate-area` option | meaning | +|---|---| +| `top=`, `target=`, `black-boxes=` | as for `foldsim-estimate-timing`; map with `foldsim-beacon-map{area}` for the same target | +| `output=` | the instance table of §5 (`area.csv`) | +| `memories=` | every memory, its shape and where it goes (`area_memories.csv`) | +| `report=` | the design's totals, the mapping's LUTs by op, the vendor IP | +| `lut-scale=` | device LUTs per mapped LUT (default: the target's, §4.5) | +| `modules=`, `state-threads=` | per module, its totals and its state's cost folded over that many threads (JSON; foldsim's fold-degree search prices copies with it) | + ## 2. System architecture ![Beacon system](../assets/img/beacon_system.svg) @@ -96,13 +111,13 @@ never builds it. The integration is in Vortex: | Piece | Where | Role | |---|---|---| -| `foldsim-beacon-map`, `foldsim-estimate-timing` | foldsim `lib/Transforms/Beacon.cpp`, `EstimateTiming.cpp` | map to the target's cells; count levels | +| `foldsim-beacon-map`, `foldsim-estimate-timing`, `foldsim-estimate-area` | foldsim `lib/Transforms/Beacon.cpp`, `EstimateTiming.cpp`, `EstimateArea.cpp` | map to the target's cells; count levels; count area | | `ci/beacon.py` | `ci/` | run Beacon on any netlist; read it by unit; compare with Vivado | | `xilinx_cone_dump.tcl`, `xilinx_cone_probe.tcl`, `xilinx_wire_probe.tcl` | `hw/scripts` | Vivado probes for what levels cannot show (§8) | -| target `beacon` | `hw/syn/xilinx/dut/common.mk` | one DUT: sources → netlist → levels → summary | +| target `beacon` | `hw/syn/xilinx/dut/common.mk` | one DUT: sources → netlist → levels and area → summary | | `circt_elaborate.sh -f mlir` | `hw/scripts` | the lowered HW/Comb/Seq netlist, no `firtool` | | `gen_platform.py --emit=family` | `hw/scripts` | the family of `DEVICE`, from `hw/syn/xilinx/platforms` | -| `beacon_summary.py` | `hw/scripts` | `beacon_summary.csv` and per-unit `beacon_units.json` | +| `beacon_summary.py` | `hw/scripts` | `beacon_summary.csv`, per-unit `beacon_units.json`, `beacon_area.txt` | | `Beacon` backend, `logic_gate.py` | `ci/synth_gate.py`, `ci/` | the gate, beside `fpga_gate` and `asic_gate` | | `logic_gate.yml` | `.github/workflows` | runs it on pull requests that touch RTL | @@ -117,9 +132,9 @@ black boxes (`xilinx_ip_stubs.sv`). ### 3.1 `foldsim-beacon-map` -Both passes load the same **target** from foldsim's target table +Every pass loads the same **target** from foldsim's target table (`foldsim/Beacon.h`): its cells for the mapping, its DSPs and delays for the -level count. A target is named after its FPGA family or its PDK, so every +level count, its LUT scale for the area (§4.5). A target is named after its FPGA family or its PDK, so every flow passes the name it already has: the DUT flow the family `gen_platform.py --emit=family` gives for `DEVICE`, the ASIC flow `PDK`, `ci/beacon.py run` its `target=`. The mapping is a pipeline of CIRCT's @@ -149,6 +164,20 @@ synthesis passes plus three of foldsim's, per module: The full GPU maps in about a minute and 18–20 GB. +With `area`, the same lowering maps the design for `foldsim-estimate-area` +instead. A depth-oriented cover duplicates logic to save levels, so its +cells are no measure of area; the area mapping is what Vivado does off the +critical paths: + +- every module stays an instance, so each one's resources stay its own + (inlining moves the count by 1–4 %), and each module maps once however + many instances it has; +- no mux balancing, no maximum AND cover and no SOP balancing; +- variadic logic lowered sharing common subsets (`reuse-subsets`), and the + cut mapper's area strategy. + +A DUT of the catalog maps for area in 1–16 s. + ### 3.2 `foldsim-estimate-timing` The estimator works on the elaborated design: one **context** per instance, @@ -168,6 +197,14 @@ and one **node** per (context, value, bit). It never enumerates paths: Time and memory are linear in the design's bits: the full GPU runs in 16–21 s and 2.7 GB, a DUT of the catalog in seconds. +### 3.3 `foldsim-estimate-area` + +The area estimator counts each module's own ops once (§4.5), then sums the +instance tree from `top`: a module's total is its own resources and its +instances' totals. It writes one row per instance, depth first, with the +instance's own resources and its subtree's, and one row per memory with the +placement chosen for it. It takes under a second on a DUT. + ## 4. The model ### 4.1 Levels per element @@ -255,6 +292,47 @@ harness makes them. It ranks no better than plain levels, so `levels` stays the ranking; `retimed ≥ 12` at `retime-levels=10` is a short list of fix candidates retiming is unlikely to absorb. +### 4.5 Area + +Per op of the area mapping (§3.1): + +| Element | FPGA | ASAP7 | +|---|---|---| +| LUT (`comb.truth_table`) | 1 LUT | 1 cell | +| Adder or subtracter | a LUT per bit into the carry chain | a prefix adder: w · (2 + ⌈log₂ w⌉) cells | +| Order compare | a LUT per two bits | 2 cells per bit | +| Multiply by a constant | a LUT per bit (shifts and adders) | 3 cells per bit | +| Multiply, ≤ 10 variable operand bits | a table: a LUT per output bit (at most as many as the operands' variable bits), per 64 entries past six bits | the same in cells | +| Multiply, wider | DSPs: the fewer of ⌈a/A⌉·⌈b/B⌉ in either orientation over the significant bits (§4.2) | partial products and a Dadda tree: 3·a·b cells | +| Divide or remainder (output bits: at most the dividend's) | a table over ≤ 10 variable bits; a constant divisor, 2 LUTs per bit; else a subtracter per quotient bit, a LUT per result bit | the same, but 3 cells per subtracter bit | +| Register | a flip-flop per bit | a flip-flop per bit | +| Vendor IP (`xil_*`) | its measured LUTs, flip-flops and DSPs | — | +| SRAM macro (black box) | — | a macro | + +**Memories** go where Vivado infers them. A `ram_style` the RTL asks for wins +(`VX_dp_ram` writes `block` for `FORCE_BRAM` sizes and `distributed` for +`LUTRAM`; the RAM fold, `ram_fold.py`, registers the read of the RAMs it +folds, so `VX_dp_ram` writes `block` for them too). Otherwise a +read of latency 0 is distributed RAM, and a registered read of at least 32 +words and 2 Kb is block RAM. The shapes are the same on every FPGA family: + +| Placement | Cost | +|---|---| +| distributed RAM | 0.57 LUTs per bit up to 32 words (RAM32M16), 1.14 per 64 words beyond (RAM64M8), per read port, plus a multiplexer tree over the 64-word LUTs | +| block RAM | the fewest 36 Kb blocks of any shape (512 × 72 … 32K × 1) or 18 Kb halves; a copy per two ports, a read-write port one of them | +| UltraRAM (`ultra`) | 4096 × 72 each | +| `registers` | flip-flops and a read multiplexer | + +On ASAP7, a memory without a macro is flip-flops, a write decoder and a +read multiplexer tree. + +**The LUT scale.** The cut mapper has none of Vivado's LUT combining or +clock-enable and reset pins, and maps module by module, so it covers a +design with 2.1–3.5× the logic LUTs Vivado leaves. The target's `lutScale` converts the mapping's LUTs to +Vivado's: 0.337 on the LUT6 families, the geometric mean of Vivado's logic +LUTs over Beacon's on routed U55C builds (§7.2). Vendor IP and distributed +RAM, and the memories' read multiplexers, are not scaled; `lut-scale=1` gives the raw cover. + ## 5. Outputs | File | Contents | @@ -262,10 +340,19 @@ candidates retiming is unlikely to absorb. | `endpoints.csv` | one row per end point bit: `levels,retimed,kind,start_kind,fanout,crossings,start_inst,start,end_inst,end` | | `instances.csv` | the instance tree: `id,parent,instance,module` (the `*_inst` columns' ids) | | `report.txt` | counts, combinational loops, ops without a model, the histogram of end points by levels, the `top-paths` deepest paths op by op, and the deepest path of each `trace=` match | -| `beacon_summary.csv` | the DUT flow's one row: max levels, end points ≥ 12, ≥ 8, all | +| `beacon_summary.csv` | the DUT flow's one row: max levels, end points ≥ 12, ≥ 8, all, and the area | | `paths.jsonl` | `paths=`: one JSON line per pair of start and end registers whose deepest bit has at least `path-levels` levels: that bit's levels, the pair's bit count, start, end, and `ops`, the path from the end back to the start as `[levels, kind, fanout, instance id, name, source]` | | `beacon_units.json` | the deepest end point of every unit at ≥ 8 levels, and the 10 deepest paths | | `beacon_paths.txt` | the DUT flow's `paths.jsonl` at ≥ 8 levels, by unit (`ci/beacon.py paths` for any run) | +| `area.csv` | one row per instance, depth first: `depth,path,module`, its own `lut,lutram,ff,bram,uram,dsp,macros`, then its subtree's (`total_*`); `lut` is logic LUTs, scaled (§4.5) | +| `area_memories.csv` | one row per memory: its instance, name, depth, width, read, write and read-write ports, read latency, `ram_style`, placement and cost | +| `area.txt` | the design's totals, the mapping's LUTs by op before scaling, and the vendor IP | +| `beacon_area.txt` | `area.txt`, the units with ≥ 1 % of the LUTs (one copy's subtree, × copies), and every memory with its placement | + +`beacon_summary.csv` also holds the design's `lut` (distributed RAM +included, as Vivado's utilization counts it), `ff`, `lutram`, `bram`, +`uram` and `dsp`, and `beacon_units.json` an `area` map of every unit's +resources (one copy's subtree) and copy count. A path in `report.txt`, deepest first, one op per line: levels at the op, its kind, the fanout of its bit, and its instance path and name: @@ -322,11 +409,12 @@ On the full GPU, the 3,283 pairs at ≥ 12 levels fold into 169 paths in repository's); 2. `circt_elaborate.sh -f mlir` into `beacon/netlist.mlir`, the FP IPs as black boxes, with every op's source location; -3. `foldsim-beacon-map`, then `foldsim-estimate-timing`, both with +3. `foldsim-beacon-map`, then `foldsim-estimate-timing`; and + `foldsim-beacon-map{area}`, then `foldsim-estimate-area`; all with `target=` the family `gen_platform.py --device=$(DEVICE) --emit=family` names; -4. `beacon_summary.py`: `beacon_summary.csv`, `beacon_units.json` and - `beacon_paths.txt` (§5). +4. `beacon_summary.py`: `beacon_summary.csv`, `beacon_units.json`, + `beacon_paths.txt` and `beacon_area.txt` (§5). The ASIC flow has the same target, with the same sources, defines and SRAM black boxes as its synthesis, and `target=$(PDK)`: @@ -349,12 +437,20 @@ each against `ci/baselines/synthesis/beacon/`: - a **new unit** at or above the fix threshold (12) fails; - a unit that gets shallower, or leaves the watch list, is an improvement to lock in by re-recording; -- end point counts are recorded, not gated (they grow with lanes and warps). +- end point counts are recorded, not gated (they grow with lanes and warps); +- a **resource** of the design (`lut`, `ff`, `lutram`, `bram`, `uram`, + `dsp`) more than 5 % above its baseline fails, `fpga_gate`'s LUT + tolerance (catalog `thresholds` and `--metric-threshold` override it); + 5 % below is an improvement to lock in. A memory that moves between + distributed and block RAM shows as both columns moving. A failing build names its `beacon_paths.txt`, where the failing unit is a -heading with its paths op by op; CI uploads it with the build's logs. +heading with its paths op by op, and its `beacon_area.txt`; CI uploads them +with the build's logs. -Depth is deterministic, so the tolerance is zero. The baseline fingerprint +Depth is deterministic, so the tolerance is zero. Area is as +deterministic, but a feature adds some by design, so it has the Fmax gate's +tolerance. The baseline fingerprint is the CIRCT and foldsim versions, `DEVICE` and `XLEN`: a new foldsim reads as STALE, not as a regression. Baselines are golden data, re-recorded only by a human (`--update-baseline`), as for the other gates. @@ -385,7 +481,10 @@ fixed while the change is still open, and synthesis runs once, to confirm. deepest register, the mean of the 50 deepest and the count at the watch threshold, before and after; then it lists the registers that got deeper, new deep registers, and the ones that got shallower, then the units whose - deepest register moved. The run fails (`DEEPER`) on a register 4 or more + deepest register moved. With area, the headline adds the LUTs, block RAMs + and DSPs before and after, and a section lists each resource of the + design and the units whose own area moved, all copies; a resource more + than 5 % larger fails the run (`AREA`). The run fails (`DEEPER`) on a register 4 or more levels deeper, one that reaches the fix threshold (12) from below, or a new one at it. These limits come from 13 routed U55C builds: a register 4 or more levels deeper lost slack in Vivado in every case (170 of 170), while @@ -405,9 +504,9 @@ fixed while the change is still open, and synthesis runs once, to confirm. run with Vivado's end point table, and a disagreement there is a Beacon bug to fix in foldsim. -Beacon counts logic levels; it does not see wire delay, placement, SLR -crossings, congestion or area. A change that moves those (a wide fan-out, a -floorplan, a larger RAM) still needs synthesis before it lands. Compare a fix +Beacon counts logic levels and cells; it does not see wire delay, +placement, SLR crossings or congestion. A change that moves those (a wide +fan-out, a floorplan) still needs synthesis before it lands. Compare a fix against a native gate (`hw/syn/xilinx/dut` or the OOC gates), not the folded design (§9). @@ -540,6 +639,37 @@ the same RTL (it1, it2) reaches 0.80. AP for failing registers goes from So the model is a ranking for where to look first, not a gate. `diff` and `logic_gate` keep gating on levels. +### 7.2 Area + +Against the routed `fpga_gate` builds of the same sources (U55C, Vivado +2024.1, 1 November 2026): + +| DUT | LUT (Beacon / Vivado) | FF | distributed RAM LUTs | block RAM | DSP | +|---|---|---|---|---|---| +| core | 148,956 / 174,006 (0.86) | 166,850 / 146,008 (1.14) | 6,838 / 8,528 | 129 / 137 | 97 / 96 | +| cache | 37,466 / 31,039 (1.21) | 33,270 / 25,280 (1.32) | 3,784 / 2,722 | 32 / 32 | 0 / 0 | +| tcu | 122,262 / 117,087 (1.04) | 47,095 / 42,675 (1.10) | 2,083 / 1,360 | 0 / 0 | 769 / 768 | +| tensor | 264,677 / 233,153 (1.14) | 160,881 / 138,986 (1.16) | 12,806 / 7,844 | 114.5 / 133 | 267 / 265 | + +- **DSPs** are within two per DUT. Vivado decides some narrow products + (8 × 9, say) by a heuristic of its own, between DSPs and LUTs; Beacon puts + every product of more than ten variable bits in DSPs. +- **Block RAM** is exact on cache, 6 % short on core and 14 % short on + tensor; which of tensor's memories Vivado puts in block RAM that Beacon + does not is not yet traced. +- **Flip-flops** run 10–32 % high. Vivado can move pipeline registers into + DSPs (`AREG`, `MREG`, `PREG`) and block RAM output registers, and shift + registers into SRLs, none of which Beacon models; how much each accounts + for is not yet measured. +- **LUTs** are a calibrated estimate, within −14 % to +21 % on these DUTs; + the scale is one number per family (§4.5), so a DUT whose logic packs + better or worse than the average moves by that much. `logic_gate` + compares a DUT with its own baseline, so the scale cancels there. + +The repository's Vivado baselines (`ci/baselines/synthesis/xilinx/`) date +from 5 October 2026 and predate later RTL changes, so they are not a +reference for this table. + ## 8. Diagnostics beyond levels **Port depths.** `port-modules=` names modules whose instances' ports @@ -600,7 +730,13 @@ Beacon cannot answer, run on a checkpoint into LUTRAM and cuts some chains, so its paths differ from the RTL's. The native gates are the reference for native RTL. - **Black boxes.** Vendor IPs and `black-boxes=` modules end and start - paths at their pins: their insides are not timed. + paths at their pins: their insides are not timed. Vendor IP's area is + its measured cost per instance; a `black-boxes=` module's is left out. +- **Area is a count, not a utilization report.** The mapping is a cut + mapper's cover scaled to Vivado's average (§4.5, §7.2), memories go by + Vivado's inference rules, and what Vivado packs into DSP, block RAM and + SRL cells Beacon counts as fabric. Logic that nothing observable reads is + counted; Vivado removes it. - **Devices.** The V80 and the U55C (Vivado), and ASAP7 RVT TT (the Yosys flow). The ASAP7 numbers are Beacon's period estimate, a sizing guide, not a sign-off: they come within about 10–30 % of OpenSTA. diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index 9caf98dc5c..90facb0f39 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -278,17 +278,18 @@ so the parity and perf views of one workload are two cases. that costs **timing closure or area**: they synthesize a catalog of designs and assert the results against the target clock and area budget each build declares, recording every run's metrics in golden baselines. `logic_gate` -catches one that makes a unit **deeper**, in seconds and without synthesis. +catches one that makes a unit **deeper** or the design **larger**, in seconds +and without synthesis. | | `fpga_gate` | `asic_gate` | `logic_gate` | |---|---|---|---| -| flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | CIRCT → `foldsim-estimate-timing` (logic levels as Vivado counts them on `DEVICE`'s family) | +| flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | CIRCT → `foldsim-estimate-timing` (logic levels as Vivado counts them on `DEVICE`'s family) and `foldsim-estimate-area` (its cells, memories and DSPs) | | design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | `hw/syn/xilinx/dut`, target `beacon` | -| gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | `max_levels` against the baseline, and the deepest end point of every RTL unit | +| gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | `max_levels` against the baseline, the deepest end point of every RTL unit, and `lut`, `ff`, `lutram`, `bram`, `uram`, `dsp` against the baseline | | also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | end point bits at ≥ 12 and ≥ 8 levels, the deepest paths | | comparable when | same device, optimization level, xlen | same PDK, VT, corner, xlen | same CIRCT, foldsim, xlen | | runner | self-hosted, licensed Vivado | hosted, one job per design | hosted, one job for the catalog; the opt-in `gpu` group self-hosted | -| default tolerance | Fmax 2 %, LUT 5 % | Fmax 2 %, cell area 5 % | none: Beacon is deterministic | +| default tolerance | Fmax 2 %, LUT 5 % | Fmax 2 %, cell area 5 % | depth none: Beacon is deterministic; area 5 % | None is a pytest cell. A cell is a build tree plus a driver; a gate build is neither, and the synthesis gates are hours long. They are standalone scripts @@ -392,13 +393,17 @@ Beacon has profiles for the V80 and the U55C only, and fails on any other | no new unit at the fix threshold (12 levels at 300 MHz) | a new deep cone is a regression even with no baseline to compare | | a shallower unit, or one that drops below the watch threshold (8), reads `IMPROVED` | the gain must be recorded | | no output of a build's `interfaces` modules reached combinationally from outside the instance in more bits than its baseline | modules register their outgoing interfaces: a path that runs through a core, socket or cluster boundary is what fails routing across SLRs, and Beacon names it by port | +| no resource of the design (`lut`, `ff`, `lutram`, `bram`, `uram`, `dsp`) more than 5 % above its baseline | area found in minutes, not in a nightly Vivado run; a memory that moves between distributed and block RAM moves two columns, and `beacon_area.txt` names each memory's placement | `interfaces` lists modules by name; their instances keep their ports through Beacon's mapping (`BEACON_INTERFACES` in the DUT flow), and `ports.csv` counts, per output port, the bits a combinational path from outside the instance reaches. -There is no tolerance: the same RTL and tools give the same levels. A new CIRCT +Depth has no tolerance: the same RTL and tools give the same levels. Area is +as deterministic, but a feature adds some by design, so it has `fpga_gate`'s +LUT tolerance; its LUTs are the area mapping's scaled to Vivado's average +([beacon.md §4.5](beacon.md)), so it gates their change, not a budget. A new CIRCT or foldsim can map the same RTL differently, so both versions are in the config hash and a tool change reads `STALE`. `make -C hw/syn/xilinx/dut -beacon` runs Beacon on one DUT by hand; its `beacon_paths.txt` lists every deep path op by op, diff --git a/hw/scripts/beacon_summary.py b/hw/scripts/beacon_summary.py index 31cc0e1d25..60c117e3cd 100644 --- a/hw/scripts/beacon_summary.py +++ b/hw/scripts/beacon_summary.py @@ -1,16 +1,23 @@ #!/usr/bin/env python3 -"""Summarize a Beacon run (foldsim-estimate-timing) for logic_gate. +"""Summarize a Beacon run (foldsim-estimate-timing, foldsim-estimate-area) +for logic_gate. Beacon writes one row per end point bit: the logic levels of its longest -path, its start and its end. ci/logic_gate.py gates depth per RTL unit, so -this collapses the table into what it compares: +path, its start and its end; and one row per instance: its resources and its +subtree's. ci/logic_gate.py gates depth per RTL unit and the design's area, +so this collapses the tables into what it compares: beacon_summary.csv one row: the deepest end point, the end point bits at or - above the fix and watch thresholds, and all end points + above the fix and watch thresholds, all end points, and + the design's LUTs, flip-flops, distributed RAM LUTs, + block RAMs, UltraRAMs and DSPs (area.csv) beacon_units.json the deepest end point of every unit at or above the - watch threshold, and the deepest paths (diagnostics) + watch threshold, the deepest paths (diagnostics), and + `area`: each unit's resources, one copy's subtree beacon_paths.txt if Beacon wrote paths.jsonl: every path at or above the watch threshold, op by op, by unit (write_paths) + beacon_area.txt if Beacon wrote area.csv: the area summary, the units + by LUTs, and where each memory goes (write_area) If Beacon wrote ports.csv (BEACON_INTERFACES), beacon_units.json also holds `interfaces`: per output port of those modules' instances, folded like @@ -38,6 +45,10 @@ COLUMNS = ("top", "max_levels", "deep_endpoints", "watch_endpoints", "endpoints") +AREA = ("lut", "ff", "lutram", "bram", "uram", "dsp") +# Units of at least this share of the design's LUTs are listed in +# beacon_area.txt. +AREA_UNIT_SHARE = 0.01 INDEX = re.compile(r"_\d+(?=_|/|$)") @@ -113,14 +124,21 @@ def main(): summary["max_levels"] = max( (int(r["levels"]) for r in csv.DictReader(fh)), default=0) + area = read_area(args.run_dir) + columns = COLUMNS + if area: + summary.update(area["total"]) + columns = COLUMNS + AREA with open(os.path.join(args.output_dir, "beacon_summary.csv"), "w", newline="") as fh: - w = csv.DictWriter(fh, fieldnames=COLUMNS) + w = csv.DictWriter(fh, fieldnames=columns) w.writeheader() w.writerow(summary) doc = {"fix_levels": FIX_LEVELS, "watch_levels": WATCH_LEVELS, "units": dict(sorted(units.items())), "deepest_paths": ["%d %s -> %s" % d for d in deepest]} + if area: + doc["area"] = area["units"] ports = os.path.join(args.run_dir, "ports.csv") if os.path.exists(ports): doc["interfaces"] = interfaces(ports) @@ -137,7 +155,84 @@ def main(): n, u = write_paths(fh, args.run_dir, WATCH_LEVELS, paths=paths) print("beacon: %d paths >= %d levels in %d units -> beacon_paths.txt" % (n, WATCH_LEVELS, u)) - + if area: + with open(os.path.join(args.output_dir, "beacon_area.txt"), "w") as fh: + write_area(fh, args.run_dir, area) + t = area["total"] + print("beacon: %s %d LUTs (%d distributed RAM), %d FFs, %g block " + "RAMs, %d UltraRAMs, %d DSPs -> beacon_area.txt" + % (args.top, t["lut"], t["lutram"], t["ff"], t["bram"], + t["uram"], t["dsp"])) + + +def read_area(run_dir): + """The design's resources and each unit's from a run's area.csv, or None. + `lut` counts every LUT, distributed RAM included, as Vivado's utilization + does. A unit's resources are one copy's subtree; `copies` counts its + instances.""" + path = os.path.join(run_dir, "area.csv") + if not os.path.exists(path): + return None + total, units = None, {} + with open(path) as fh: + for row in csv.DictReader(fh): + res = {k: float(row["total_" + k]) for k in AREA} + res["lut"] += res["lutram"] + res = {k: (round(v, 1) if k == "bram" else int(round(v))) + for k, v in res.items()} + if total is None: + total = res + continue + u = fold(row["path"]) + if u in units: + units[u]["copies"] += 1 + else: + units[u] = dict(res, module=row["module"], copies=1) + return {"total": total, "units": dict(sorted(units.items()))} + + +def write_area(out, run_dir, area): + """The area report, the units holding at least AREA_UNIT_SHARE of the + LUTs, and every memory with its placement.""" + with open(os.path.join(run_dir, "area.txt")) as fh: + out.write(fh.read()) + t = area["total"] + floor = AREA_UNIT_SHARE * t["lut"] + out.write("\nUnits with at least %g%% of the LUTs, one copy's subtree " + "(x copies):\n" % (100 * AREA_UNIT_SHARE)) + out.write(" %9s %8s %7s %7s %5s %5s %s\n" + % ("lut", "ff", "lutram", "bram", "uram", "dsp", "unit")) + rows = sorted(area["units"].items(), key=lambda kv: -kv[1]["lut"]) + for u, r in rows: + if r["lut"] * r["copies"] < floor: + continue + out.write(" %9d %8d %7d %7g %5d %5d %s (%s) x%d\n" + % (r["lut"], r["ff"], r["lutram"], r["bram"], r["uram"], + r["dsp"], u, r["module"], r["copies"])) + mems = os.path.join(run_dir, "area_memories.csv") + if not os.path.exists(mems): + return + out.write("\nMemories (depth x width, read latency, ram_style -> " + "placement and cost, x copies):\n") + groups = collections.OrderedDict() + with open(mems) as fh: + for r in csv.DictReader(fh): + name = fold("%s/%s" % (r["path"], r["name"]) if r["path"] + else r["name"]) + key = (name, r["depth"], r["width"], r["latency"], r["style"], + r["placement"]) + if key in groups: + groups[key][1] += 1 + else: + groups[key] = [r, 1] + for (name, depth, width, latency, style, placement), (r, n) in \ + groups.items(): + cost = ("%s LUTs" % r["lutram"] if placement == "lutram" else + "%s blocks" % r["bram"] if placement == "bram" else + "%s UltraRAMs" % r["uram"] if placement == "uram" else + "%s FFs" % r["ff"]) + out.write(" %6sx%-5s lat %s %-11s -> %-6s %-12s %s x%d\n" + % (depth, width, latency, style, placement, cost, name, n)) def fold(name): return INDEX.sub("_*", norm(name)) diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index fbc0b57493..fe8a851365 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -83,15 +83,17 @@ power: TOOL_DIR=$(SCRIPT_DIR) SAIF_FILE=$(SAIF_FILE) SAIF_INST=$(SAIF_INST) \ $(VIVADO) -mode batch -source $(SCRIPT_DIR)/xilinx_power_analysis.tcl -# Beacon (foldsim-estimate-timing): the logic levels of every -# register-to-register path of the design, counted as Vivado counts them on -# DEVICE's family (hw/syn/xilinx/platforms), by RTL unit, in seconds and -# without Vivado. It elaborates the same sources and -# defines as gen-sources into a CIRCT netlist of its own, with the Xilinx -# floating-point cores as black boxes. ci/logic_gate.py gates +# Beacon (foldsim-estimate-timing, foldsim-estimate-area): the logic levels of +# every register-to-register path of the design, counted as Vivado counts them +# on DEVICE's family (hw/syn/xilinx/platforms), by RTL unit, and its LUTs, +# flip-flops, distributed and block RAM and DSPs, in seconds and without +# Vivado. It elaborates the same sources and defines as gen-sources into a +# CIRCT netlist of its own, with the Xilinx floating-point cores as black +# boxes, mapped once for depth and once for area. ci/logic_gate.py gates # beacon_summary.csv and beacon_units.json; beacon_paths.txt has every path # of at least BEACON_PATH_LEVELS (beacon_summary.py's watch threshold), op by -# op, by unit. +# op, by unit, and beacon_area.txt the area by unit and where each memory +# goes. BEACON_PATH_LEVELS ?= 8 # The Beacon target: the family of the device, as hw/syn/xilinx/platforms names it. BEACON_TARGET = $(shell python3 $(SCRIPT_DIR)/gen_platform.py --device=$(DEVICE) --emit=family) @@ -114,11 +116,12 @@ beacon: $(FOLDSIM_OPT) beacon/netlist.mlir --pass-pipeline='builtin.module($(BEACON_MAP))' \ --emit-bytecode -o beacon/mapped.mlirbc $(FOLDSIM_OPT) beacon/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) target=$(BEACON_TARGET) output=beacon/endpoints.csv instances=beacon/instances.csv report=beacon/report.txt paths=beacon/paths.jsonl path-levels=$(BEACON_PATH_LEVELS) $(BEACON_PORTS)})" + $(FOLDSIM_OPT) beacon/netlist.mlir -o /dev/null --pass-pipeline="builtin.module(foldsim-beacon-map{target=$(BEACON_TARGET) area},foldsim-estimate-area{top=$(TOP_LEVEL_ENTITY) target=$(BEACON_TARGET) output=beacon/area.csv memories=beacon/area_memories.csv report=beacon/area.txt})" python3 $(SCRIPT_DIR)/beacon_summary.py beacon --top $(TOP_LEVEL_ENTITY) clean: ifndef RESUME - rm -rf project_1 beacon beacon_summary.csv beacon_units.json beacon_paths.txt + rm -rf project_1 beacon beacon_summary.csv beacon_units.json beacon_paths.txt beacon_area.txt rm -rf .Xil rm -f *.rpt rm -f *.log diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index 2613bc6044..11cf88bd11 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -285,8 +285,9 @@ timing: $(SYN_SRC_DEPS) $(SYN_LIB_DEPS) | $(BUILD_DIR) @$(SRC_DIR)/synth_summary.py --reports $(BUILD_DIR)/reports \ --top $(TOP_LEVEL_ENTITY) --clock-mhz $(CLOCK_FREQ) -# Beacon (foldsim-estimate-timing, target=$(PDK)): the gate stages of every -# register-to-register path, with the same sources and SRAM black boxes as +# Beacon (foldsim-estimate-timing, foldsim-estimate-area, target=$(PDK)): the +# gate stages of every register-to-register path and the cells, flip-flops and +# SRAM macros of the design, with the same sources and SRAM black boxes as # synthesis, in seconds and without Yosys; report.txt converts the deepest # path to a period with the library's delays. docs/designs/beacon.md. beacon: $(BUILD_DIR)/src @@ -296,7 +297,9 @@ beacon: $(BUILD_DIR)/src $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/netlist.mlir --pass-pipeline='builtin.module(foldsim-beacon-map{target=$(PDK)})' \ --emit-bytecode -o $(BUILD_DIR)/beacon/mapped.mlirbc $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) target=$(PDK) output=$(BUILD_DIR)/beacon/endpoints.csv instances=$(BUILD_DIR)/beacon/instances.csv report=$(BUILD_DIR)/beacon/report.txt})' + $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/netlist.mlir -o /dev/null --pass-pipeline='builtin.module(foldsim-beacon-map{target=$(PDK) area},foldsim-estimate-area{top=$(TOP_LEVEL_ENTITY) target=$(PDK) output=$(BUILD_DIR)/beacon/area.csv memories=$(BUILD_DIR)/beacon/area_memories.csv report=$(BUILD_DIR)/beacon/area.txt})' @grep "^deepest path" $(BUILD_DIR)/beacon/report.txt + @head -1 $(BUILD_DIR)/beacon/area.txt gate-sim: techmap asap7-models @if [ "$(PDK)" != "asap7" ]; then echo "ERROR: gate-sim requires PDK=asap7 functional models" >&2; exit 1; fi From 71cb2934bcb6739f2a00c62ff5c2da10f977552e Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 13:43:35 -0800 Subject: [PATCH 314/336] lsu: wide local loads (vx_flds.128, vx_flds.64) sgemm2's inner loop spends two local-memory loads per FMA; with a 16-byte load for the A row it spends 1.25, as NVIDIA's LDS.128 loop does, and issues 13.6-22.6% fewer instructions per tile. vx_flds.128/.64 (custom-3, imm[11]=1, funct3 0/4; XLEN 32 with F and local memory) load 4 or 2 consecutive words into an aligned group of FP registers as one issued instruction and one LSU/local-memory request. The local-memory engine serves the request in word phases, reusing the request's conflict matrices, and each phase returns one word per lane at today's width; each response is one commit beat writing fd + phase. Each phase releases its register on its last packet (per-phase release), so a dependent instruction on fd can start before fd + 3 arrives. - decode/SimX decode: the two forms, gated on LDS_WIDE_ENABLE. - scoreboard: a wide load reserves and tests its whole group; the group's other registers are tested against the registered in-use set, so SimX frees a group one cycle after a single register, and a pending group clear skips a register reserved again in between. - commit: the early release travels one-hot per warp (rel_wis), decoded ahead of the commit arbiter's output register like eop_wis. - VX_mem_scheduler (shared lib): an RSP_SIDE_WIDTH sideband carried with each request to its response; at the default 0 the logic is the same as before (all five other instances unchanged in behavior). - LSU, lmem switch/unit/engine, lsu_mem_if: the phase count and the sideband from LSU to local memory and back. - tests/regression/lds_ldg: wide loads at broadcast, bank-conflict, permuted and divergent lanes, offset limits, back-to-back and with scalar writes to the group in between (including an FP op on fd + 2 between two wide loads of the group); the wide part runs where the ISA has it (XLEN 32 + F), the scalar part everywhere. - CI: lds-ldg-3 (NUM_LSU_LANES=4), lds-ldg-4 (SIMD_WIDTH=1), parity-lds-ldg-lanes. Performance (SimX, sgemm2 with the local accesses hand-written as .insn, since LLVM does not emit wide loads yet; 16 shapes): geomean -5.4% cycles; -16.6%/-18.9% on RTX 5080-like 4-core BS 16, about -10% on the default config, 0 to -9% at 16x16, +1% to +2% at BS 32 (latency-bound). No compiled kernel changes until LLVM vectorizes addrspace(3) loads. Verified: lds_ldg on rtlsim (default, LSU_LANES=4, SIMD_WIDTH=1, SIMD_WIDTH=4 x 16 threads) and SimX (same, plus XLEN 64 with the wide part skipped); rtlsim and SimX agree within 0.2% on lds_ldg cycles; the new FP-op case fails on the SimX without the group-clear fix (32 errors) and passes with it; CI cases lds-ldg/lmem-access/parity-sgemm2-lmem; Beacon core: deepest path 15 -> 15, mean of the 50 deepest 11.8 -> 11.8, +11 LUTs for rel_wis. Before the rel_wis and group-clear changes: make -C tests/regression run-simx 75/75, the XLEN 64 build (simx, rtlsim, opaesim), Beacon lmem 9 -> 9. Not run: a SIMD_WIDTH=1 parity case (SimX and rtlsim already differ by ~20% there without wide loads, e.g. sgemm2 -n32 -t4), perf_gate (default sgemm2 does not use the instructions), FPGA/ASIC synthesis. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 6 + ci/testcases/regression.yaml | 20 +++ .../custom_accelerator_isa_extensions.md | 10 +- hw/rtl/VX_gpu_pkg.sv | 15 ++ hw/rtl/VX_trace_pkg.sv | 33 ++-- hw/rtl/core/VX_commit_slice.sv | 41 ++++- hw/rtl/core/VX_core.sv | 7 +- hw/rtl/core/VX_decode.sv | 15 +- hw/rtl/core/VX_dispatcher.sv | 2 +- hw/rtl/core/VX_lsu_block.sv | 110 +++++++++++- hw/rtl/core/VX_lsu_mem_path.sv | 9 +- hw/rtl/core/VX_lsu_scheduler.sv | 16 ++ hw/rtl/core/VX_mem_unit.sv | 3 +- hw/rtl/core/VX_scoreboard.sv | 79 ++++++++- hw/rtl/core/VX_uop_sequencer.sv | 2 +- hw/rtl/libs/VX_mem_scheduler.sv | 43 ++++- hw/rtl/mem/VX_lmem_engine.sv | 113 ++++++++++-- hw/rtl/mem/VX_lmem_switch.sv | 29 +++- hw/rtl/mem/VX_lmem_unit.sv | 48 ++++- hw/rtl/mem/VX_lsu_adapter.sv | 1 + hw/rtl/mem/VX_lsu_mem_arb.sv | 11 +- hw/rtl/mem/VX_lsu_mem_if.sv | 5 +- hw/rtl/om/VX_om_mem.sv | 3 + hw/rtl/raster/VX_raster_earlyz.sv | 3 + hw/rtl/raster/VX_raster_mem.sv | 3 + hw/rtl/tcu/VX_tcu_agu.sv | 1 + hw/rtl/tex/VX_tex_mem.sv | 3 + hw/unittest/issue/VX_issue_top.sv | 1 + hw/unittest/lmem_unit/VX_lmem_unit_top.sv | 12 +- hw/unittest/mem_unit/VX_mem_unit_top.sv | 7 +- hw/unittest/tensor/VX_tensor_top.sv | 7 +- sim/simx/core.cpp | 3 + sim/simx/decode.cpp | 21 ++- sim/simx/instr_trace.h | 11 ++ sim/simx/lsu_unit.cpp | 62 ++++++- sim/simx/mem/local_mem.cpp | 30 +++- sim/simx/scoreboard.cpp | 65 +++++-- sim/simx/scoreboard.h | 6 + sim/simx/types.h | 8 + tests/regression/lds_ldg/common.h | 11 +- tests/regression/lds_ldg/kernel.cpp | 164 ++++++++++++++++++ tests/regression/lds_ldg/main.cpp | 20 ++- 42 files changed, 935 insertions(+), 124 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 1792f1603d..cd41c45087 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -271,6 +271,12 @@ tests: via: blackbox app: lds_ldg configs: -DVX_CFG_NUM_THREADS=16 +# Fewer LSU lanes than SIMD lanes: a wide load's phases complete across packets. +- id: parity-lds-ldg-lanes + check: model_parity + via: blackbox + app: lds_ldg + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_LSU_LANES=4 - id: parity-sgemm2-lmem check: model_parity via: blackbox diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index aeddaab0a2..ff04a92e95 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -83,6 +83,26 @@ tests: shape: threads: 32 warps: 8 +# Fewer LSU lanes than SIMD lanes: a warp's request goes out in four packets. +- id: lds-ldg-3 + via: blackbox + drivers: + - simx + - rtlsim + app: lds_ldg + configs: -DVX_CFG_NUM_LSU_LANES=4 + shape: + threads: 16 +# One SIMD lane: the dispatcher splits every warp into single-lane packets. +- id: lds-ldg-4 + via: blackbox + drivers: + - simx + - rtlsim + app: lds_ldg + configs: -DVX_CFG_SIMD_WIDTH=1 + shape: + threads: 4 - id: occupancy-1 via: blackbox drivers: diff --git a/docs/designs/custom_accelerator_isa_extensions.md b/docs/designs/custom_accelerator_isa_extensions.md index 30eadb3624..efc0a428b6 100644 --- a/docs/designs/custom_accelerator_isa_extensions.md +++ b/docs/designs/custom_accelerator_isa_extensions.md @@ -242,10 +242,12 @@ the table in the same change. effective address, so a local pointer may hold the window address or a bare offset. `vx_ldg`/`vx_stg` take the generic path: the local-memory window is reserved, so no global pointer falls in it. Their hint is 0 (default), 1 -(`.cg`) or 2 (`.cs`); 3 is reserved. The FP forms with `funct3`/size 0 and the -integer-only codes with `imm[11]` set are reserved for wide local accesses, as -are `vx_ldg` with `funct3` 7 and `vx_stg` with `funct3` 4–7. Standard loads and -stores stay generic. +(`.cg`) or 2 (`.cs`); 3 is reserved. With `imm[11]` set, `funct3` 0 is +`vx_flds.128` and `funct3` 4 is `vx_flds.64` (XLEN 32): one request loads four +or two consecutive words into the quad- or pair-aligned FP registers starting at +`fd`. The other integer-only codes with `imm[11]` set and the FP store with +size 0 are reserved for wide local accesses, as are `vx_ldg` with `funct3` 7 +and `vx_stg` with `funct3` 4–7. Standard loads and stores stay generic. --- diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 605ce2a150..d05d41b24f 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1436,6 +1436,7 @@ package VX_gpu_pkg; logic [UUID_WIDTH-1:0] uuid; logic [ISSUE_WIS_W-1:0] wis; logic [PER_ISSUE_WARPS-1:0] eop_wis; // one-hot wis, set on eop + logic [PER_ISSUE_WARPS-1:0] rel_wis; // one-hot wis, releases rd ahead of eop logic [NCTA_WIDTH-1:0] cta_id; logic [SIMD_IDX_W-1:0] sid; logic [`VX_CFG_SIMD_WIDTH-1:0] tmask; @@ -1672,7 +1673,21 @@ package VX_gpu_pkg; logic [LSU_CLIENT_TAG_WIDTH-1:0] tag; } lsu_req_data_t; + // Wide local loads (vx_flds.64/.128) fill an aligned FP register group, + // one word-phase at a time. + localparam LDS_WIDE_ENABLE = `VX_CFG_LMEM_ENABLED && `VX_CFG_EXT_F_ENABLED && (`VX_CFG_XLEN == 32); + + // A local response's phase of a wide load. hold marks a phase before the + // last, whose lanes stay outstanding; phase_end marks its last pass. + typedef struct packed { + logic hold; + logic phase_end; + logic [1:0] phase; + } lsu_rsp_side_t; + localparam LSU_RSP_SIDE_W = LDS_WIDE_ENABLE ? $bits(lsu_rsp_side_t) : 0; + typedef struct packed { + logic [`UP(LSU_RSP_SIDE_W)-1:0] side; logic [`VX_CFG_NUM_LSU_LANES-1:0] mask; logic [`VX_CFG_NUM_LSU_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] data; logic [LSU_CLIENT_TAG_WIDTH-1:0] tag; diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index 3349343a85..dcfd34d2fd 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -44,7 +44,8 @@ package VX_trace_pkg; task trace_ex_op(input int level, input [EX_BITS-1:0] ex_type, input [INST_OP_BITS-1:0] op_type, - input op_args_t op_args + input op_args_t op_args, + input [BYTESEL_BITS-1:0] bytesel = BYTESEL_DEFAULT ); case (ex_type) EX_ALU: begin @@ -191,16 +192,26 @@ package VX_trace_pkg; end else begin `TRACE(level, ("LDS.")) end - case (INST_LSU_BITS'(op_type)) - INST_LSU_LB, INST_LSU_SB: `TRACE(level, ("B")) - INST_LSU_LH, INST_LSU_SH: `TRACE(level, ("H")) - INST_LSU_LW, INST_LSU_SW: `TRACE(level, ("W")) - INST_LSU_LD, INST_LSU_SD: `TRACE(level, ("D")) - INST_LSU_LBU: `TRACE(level, ("BU")) - INST_LSU_LHU: `TRACE(level, ("HU")) - INST_LSU_LWU: `TRACE(level, ("WU")) - default: `TRACE(level, ("?")) - endcase + // a full-size bytesel with offset 1 or 3 is a wide load + if ((bytesel[BYTESEL_BITS-1 -: XLENB_W] == XLENB_W'(XLENB-1)) + && ((bytesel[XLENB_W-1:0] | XLENB_W'(2)) == XLENB_W'(3))) begin + if (bytesel[1]) begin + `TRACE(level, ("128")) + end else begin + `TRACE(level, ("64")) + end + end else begin + case (INST_LSU_BITS'(op_type)) + INST_LSU_LB, INST_LSU_SB: `TRACE(level, ("B")) + INST_LSU_LH, INST_LSU_SH: `TRACE(level, ("H")) + INST_LSU_LW, INST_LSU_SW: `TRACE(level, ("W")) + INST_LSU_LD, INST_LSU_SD: `TRACE(level, ("D")) + INST_LSU_LBU: `TRACE(level, ("BU")) + INST_LSU_LHU: `TRACE(level, ("HU")) + INST_LSU_LWU: `TRACE(level, ("WU")) + default: `TRACE(level, ("?")) + endcase + end end else if (op_args.lsu.is_float) begin case (INST_LSU_BITS'(op_type)) INST_LSU_LW: `TRACE(level, ("FLW")) diff --git a/hw/rtl/core/VX_commit_slice.sv b/hw/rtl/core/VX_commit_slice.sv index b2a1133d24..aa0fb9c200 100644 --- a/hw/rtl/core/VX_commit_slice.sv +++ b/hw/rtl/core/VX_commit_slice.sv @@ -30,27 +30,50 @@ module VX_commit_slice import VX_gpu_pkg::*; #( ); `UNUSED_SPARAM (INSTANCE_ID) localparam OUT_DATAW = $bits(commit_t); + localparam REL_W = LDS_WIDE_ENABLE ? PER_ISSUE_WARPS : 0; + localparam ARB_DATAW = PER_ISSUE_WARPS + REL_W + OUT_DATAW; VX_commit_if commit_arb_if(); wire [PER_ISSUE_WARPS-1:0] commit_eop_wis; + wire [PER_ISSUE_WARPS-1:0] commit_rel_wis; wire [NUM_EX_UNITS-1:0] valid_in; - wire [NUM_EX_UNITS-1:0][PER_ISSUE_WARPS+OUT_DATAW-1:0] data_in; + wire [NUM_EX_UNITS-1:0][ARB_DATAW-1:0] data_in; + wire [ARB_DATAW-1:0] data_out; wire [NUM_EX_UNITS-1:0] ready_in; // Decoded ahead of the output register so each warp's release reads - // its own flop. + // its own flop. A wide load's beat (full-size bytesel with an odd offset) + // writes a whole register, and its offset bit 1 releases it. for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_data_in wire [PER_ISSUE_WARPS-1:0] eop_wis = PER_ISSUE_WARPS'(commit_if[j].data.eop) << wid_to_wis(commit_if[j].data.wid); + if (LDS_WIDE_ENABLE && (j == EX_LSU)) begin : g_wide + wire [XLENB_W-1:0] size = commit_if[j].data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; + wire [XLENB_W-1:0] off = commit_if[j].data.bytesel[0 +: XLENB_W]; + wire wide = (size == XLENB_W'(XLENB-1)) && off[0]; + commit_t data; + always @(*) begin + data = commit_if[j].data; + if (wide) begin + data.bytesel = BYTESEL_DEFAULT; + end + end + wire [PER_ISSUE_WARPS-1:0] rel_wis = PER_ISSUE_WARPS'(wide && off[1]) + << wid_to_wis(commit_if[j].data.wid); + assign data_in[j] = {eop_wis, rel_wis, data}; + end else if (LDS_WIDE_ENABLE) begin : g_narrow + assign data_in[j] = {eop_wis, PER_ISSUE_WARPS'(0), commit_if[j].data}; + end else begin : g_no_wide + assign data_in[j] = {eop_wis, commit_if[j].data}; + end assign valid_in[j] = commit_if[j].valid; - assign data_in[j] = {eop_wis, commit_if[j].data}; assign commit_if[j].ready = ready_in[j]; end VX_stream_arb #( .NUM_INPUTS (NUM_EX_UNITS), - .DATAW (PER_ISSUE_WARPS + OUT_DATAW), + .DATAW (ARB_DATAW), .ARBITER ("P"), .OUT_BUF (1) ) commit_arb ( @@ -59,12 +82,19 @@ module VX_commit_slice import VX_gpu_pkg::*; #( .valid_in (valid_in), .ready_in (ready_in), .data_in (data_in), - .data_out ({commit_eop_wis, commit_arb_if.data}), + .data_out (data_out), .valid_out (commit_arb_if.valid), .ready_out (commit_arb_if.ready), `UNUSED_PIN (sel_out) ); + if (LDS_WIDE_ENABLE) begin : g_rel + assign {commit_eop_wis, commit_rel_wis, commit_arb_if.data} = data_out; + end else begin : g_no_rel + assign {commit_eop_wis, commit_arb_if.data} = data_out; + assign commit_rel_wis = '0; + end + // notify scheduler wire [PER_ISSUE_WARPS-1:0] committed_wis_n = commit_arb_if.valid ? commit_eop_wis : '0; `BUFFER(committed_wis, committed_wis_n); @@ -94,6 +124,7 @@ module VX_commit_slice import VX_gpu_pkg::*; #( assign writeback_if.data.uuid = commit_arb_if.data.uuid; assign writeback_if.data.wis = wid_to_wis(commit_arb_if.data.wid); assign writeback_if.data.eop_wis = commit_eop_wis; + assign writeback_if.data.rel_wis = commit_rel_wis; assign writeback_if.data.cta_id = commit_arb_if.data.cta_id; assign writeback_if.data.sid = commit_arb_if.data.sid; assign writeback_if.data.PC = commit_arb_if.data.PC; diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 26891662aa..78d400aca0 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -88,9 +88,10 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty; `ifdef VX_CFG_LMEM_ENABLE VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH), + .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); `endif VX_mem_bus_if #( diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index a21bfe6984..f2bfba8a3b 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -107,6 +107,11 @@ module VX_decode import wire [11:0] i_imm = is_itype_sh ? {7'b0, instr[24:20]} : u_12; `endif wire [11:0] s_imm = {funct7, rd}; +`ifdef VX_CFG_LMEM_ENABLE +`ifdef VX_CFG_EXT_F_ENABLE + wire lds_wide = LDS_WIDE_ENABLE && instr[31] && (funct3[1:0] == 2'b00); +`endif +`endif wire [11:0] b_imm = {instr[31], instr[7], instr[30:25], instr[11:8]}; wire [19:0] jal_imm = {instr[31], instr[19:12], instr[20], instr[30:21]}; @@ -512,6 +517,14 @@ module VX_decode import op_args.lsu.is_store = 0; op_args.lsu.offset = {u_12[10], u_12[10:0]}; `ifdef VX_CFG_EXT_F_ENABLE + // vx_flds.128 (funct3 0) and vx_flds.64 (funct3 4) load 4 or 2 + // words into an aligned FP register group, one word per beat. + // A full-size bytesel with an odd offset, which no other + // instruction produces, carries the group size. + if (lds_wide) begin + op_type = INST_OP_BITS'(INST_LSU_LW); + bytesel = {XLENB_W'(XLENB-1), XLENB_W'({~funct3[2], 1'b1})}; + end `USED_REG (instr[31], rd, 1'b1); `else `USED_IREG (rd); @@ -1074,7 +1087,7 @@ module VX_decode import `TRACE(1, ("%t: %s decode: wid=%0d, cta_id=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, decode_if.data.wid, decode_if.data.cta_id, to_fullPC(decode_if.data.PC))) VX_trace_pkg::trace_ex_type(1, decode_if.data.ex_type); `TRACE(1, (", op=")) - VX_trace_pkg::trace_ex_op(1, decode_if.data.ex_type, decode_if.data.op_type, decode_if.data.op_args); + VX_trace_pkg::trace_ex_op(1, decode_if.data.ex_type, decode_if.data.op_type, decode_if.data.op_args, decode_if.data.bytesel); `TRACE(1, (", tmask=%b, wb=%b, rd_xregs=%b, wr_xregs=%b, used_rs=%b, rd=", decode_if.data.tmask, decode_if.data.wb, decode_if.data.rd_xregs, decode_if.data.wr_xregs, decode_if.data.used_rs)) VX_trace_pkg::trace_reg_idx(1, decode_if.data.rd); `TRACE(1, (", rs1=")) diff --git a/hw/rtl/core/VX_dispatcher.sv b/hw/rtl/core/VX_dispatcher.sv index 9c17114d77..148e11aeec 100644 --- a/hw/rtl/core/VX_dispatcher.sv +++ b/hw/rtl/core/VX_dispatcher.sv @@ -113,7 +113,7 @@ module VX_dispatcher import VX_gpu_pkg::*; #( `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, wis_to_wid(dispatch_if[ex].data.wis, slice_id), dispatch_if[ex].data.cta_id, dispatch_if[ex].data.sid, to_fullPC(dispatch_if[ex].data.PC))) VX_trace_pkg::trace_ex_type(1, ex); `TRACE(1, (", op=")) - VX_trace_pkg::trace_ex_op(1, ex, dispatch_if[ex].data.op_type, dispatch_if[ex].data.op_args); + VX_trace_pkg::trace_ex_op(1, ex, dispatch_if[ex].data.op_type, dispatch_if[ex].data.op_args, dispatch_if[ex].data.bytesel); `TRACE(1, (", tmask=%b, wb=%b, wr_xregs=%b, rd=%0d, rs1_data=", dispatch_if[ex].data.tmask, dispatch_if[ex].data.wb, dispatch_if[ex].data.wr_xregs, dispatch_if[ex].data.rd)) `TRACE_ARRAY1D(1, "0x%0h", dispatch_if[ex].data.rs1_data, `VX_CFG_SIMD_WIDTH) `TRACE(1, (", rs2_data=")) diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index eee402e9d5..671d82608c 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -146,6 +146,18 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `UNUSED_VAR (execute_if.data.op_args.lsu.is_local) `endif + // A wide local load names its group size, as log2 words, in the atomic op + // of a request that is not an atomic. + wire [1:0] req_wide; + if (LDS_WIDE_ENABLE) begin : g_req_wide + wire [XLENB_W-1:0] bytesel_size = execute_if.data.header.bytesel[BYTESEL_BITS-1 -: XLENB_W]; + wire [XLENB_W-1:0] bytesel_off = execute_if.data.header.bytesel[0 +: XLENB_W]; + wire is_wide = (bytesel_size == XLENB_W'(XLENB-1)) && bytesel_off[0]; + assign req_wide = is_wide ? (bytesel_off[1] ? 2'd2 : 2'd1) : 2'd0; + end else begin : g_no_req_wide + assign req_wide = 2'd0; + end + mem_bus_attr_t [NUM_LANES-1:0] mem_req_attr_struct; wire [NUM_LANES-1:0][MEM_ATTR_WIDTH-1:0] mem_req_attr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_mem_req_attr @@ -189,7 +201,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `ifdef VX_CFG_EXT_A_ENABLE amo_req_t lane_amo; assign lane_amo.amo_valid = execute_if.data.op_args.lsu.amo_valid; - assign lane_amo.amo_op = execute_if.data.op_args.lsu.amo_op; + assign lane_amo.amo_op = (LDS_WIDE_ENABLE && ~execute_if.data.op_args.lsu.amo_valid) + ? amo_op_e'($bits(amo_op_e)'(req_wide)) : execute_if.data.op_args.lsu.amo_op; // amo_unsigned distinguishes signed/unsigned AMOMIN/MAX variants // (decoder collapses MINU/MAXU into MIN/MAX + this bit). assign lane_amo.amo_unsigned = execute_if.data.op_args.lsu.amo_unsigned; @@ -208,7 +221,12 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `else // EXT_A disabled: tie the AMO sideband to zero so the bits don't // propagate as 'x under --x-assign unique. - assign mem_req_attr_struct[i].amo = '0; + amo_req_t lane_amo; + always @(*) begin + lane_amo = '0; + lane_amo.amo_op = amo_op_e'($bits(amo_op_e)'(req_wide)); + end + assign mem_req_attr_struct[i].amo = lane_amo; `endif assign mem_req_attr[i] = mem_req_attr_struct[i]; end @@ -232,6 +250,13 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire mem_rsp_eop; wire mem_rsp_ready; + // a wide load response's phase, and whether it completes that phase for + // the whole instruction, which releases the phase's register + wire mem_rsp_hold; + wire mem_rsp_phase_end; + wire [1:0] mem_rsp_phase; + wire mem_rsp_phase_rel; + wire mem_req_fire = mem_req_valid && mem_req_ready; wire mem_rsp_fire = mem_rsp_valid && mem_rsp_ready; @@ -333,6 +358,20 @@ module VX_lsu_block import VX_gpu_pkg::*; #( $time, execute_if.data.header.wid, to_fullPC(execute_if.data.header.PC), full_addr[i], inst_lsu_wsize(execute_if.data.op_type), execute_if.data.header.uuid)) end + // A wide load's lanes address whole groups and fill an aligned register group. + if (LDS_WIDE_ENABLE) begin : g_wide_align + wire lsu_req_fire = execute_if.valid && execute_if.ready; + wire [3:0] group_amask = req_wide[1] ? 4'hf : 4'h7; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lanes + `RUNTIME_ASSERT((~lsu_req_fire || (req_wide == 2'd0) || ~execute_if.data.header.tmask[i] || ((full_addr[i][3:0] & group_amask) == 4'h0)), + ("%t: misaligned wide local load, wid=%0d, PC=0x%0h, addr=0x%0h! (#%0d)", + $time, execute_if.data.header.wid, to_fullPC(execute_if.data.header.PC), full_addr[i], execute_if.data.header.uuid)) + end + `RUNTIME_ASSERT((~lsu_req_fire || (req_wide == 2'd0) || ((execute_if.data.header.rd[1:0] & {req_wide[1], 1'b1}) == 2'b00)), + ("%t: misaligned wide local load register group, wid=%0d, PC=0x%0h, rd=%0d! (#%0d)", + $time, execute_if.data.header.wid, to_fullPC(execute_if.data.header.PC), execute_if.data.header.rd, execute_if.data.header.uuid)) + end + // store data formatting for (genvar i = 0; i < NUM_LANES; ++i) begin : g_mem_req_data always @(*) begin @@ -412,6 +451,35 @@ module VX_lsu_block import VX_gpu_pkg::*; #( assign mem_rsp_sop_pkt = pkt_sop[pkt_raddr]; assign mem_rsp_eop_pkt = mem_rsp_eop && pkt_eop[pkt_raddr] && (pkt_ctr[pkt_raddr] == 1); + + // A wide load's packets serve their phases independently, so a phase + // before the last is complete once every packet has served it. + if (LDS_WIDE_ENABLE) begin : g_phase_ctr + reg [`VX_CFG_LSU_PENDING_SIZE-1:0][2:0][PID_BITS:0] phase_ctr; + wire phase_dec = mem_rsp_fire && mem_rsp_hold && mem_rsp_phase_end; + + always @(posedge clk) begin + for (integer s = 0; s < `VX_CFG_LSU_PENDING_SIZE; ++s) begin + for (integer k = 0; k < 3; ++k) begin + if (reset) begin + phase_ctr[s][k] <= '0; + end else if (mem_req_rd_sop_fire && (pkt_waddr == LSUQ_SIZEW'(s))) begin + phase_ctr[s][k] <= (PID_BITS+1)'(1); + end else begin + phase_ctr[s][k] <= phase_ctr[s][k] + + (PID_BITS+1)'(mem_req_rd_fire && (pkt_waddr == LSUQ_SIZEW'(s))) + - (PID_BITS+1)'(phase_dec && (pkt_raddr == LSUQ_SIZEW'(s)) && (mem_rsp_phase == 2'(k))); + end + end + end + end + + wire [PID_BITS:0] rsp_phase_ctr = phase_ctr[pkt_raddr][(mem_rsp_phase == 2'd3) ? 2'd0 : mem_rsp_phase]; + assign mem_rsp_phase_rel = mem_rsp_phase_end + && (mem_rsp_hold ? (pkt_eop[pkt_raddr] && (rsp_phase_ctr == 1)) : mem_rsp_eop_pkt); + end else begin : g_no_phase_ctr + assign mem_rsp_phase_rel = 1'b0; + end `RUNTIME_ASSERT(~(mem_req_rd_fire && full), ("allocator full!")) `RUNTIME_ASSERT(~(mem_req_rd_sop_fire && pkt_ctr[pkt_waddr] != 0), ("oops! broken sop request!")) // pkt_ctr is guarded against a same-cycle same-slot request/response by @@ -424,7 +492,9 @@ module VX_lsu_block import VX_gpu_pkg::*; #( assign pkt_waddr = 0; assign mem_rsp_sop_pkt = mem_rsp_sop; assign mem_rsp_eop_pkt = mem_rsp_eop; + assign mem_rsp_phase_rel = mem_rsp_phase_end; `UNUSED_VAR (pkt_raddr) + `UNUSED_VAR (mem_rsp_hold) end // pack memory request tag @@ -482,6 +552,19 @@ module VX_lsu_block import VX_gpu_pkg::*; #( assign mem_rsp_eop = client_if.rsp_data.eop; assign client_if.rsp_ready = mem_rsp_ready; + if (LDS_WIDE_ENABLE) begin : g_rsp_side + lsu_rsp_side_t rsp_side; + assign rsp_side = lsu_rsp_side_t'(client_if.rsp_data.side); + assign mem_rsp_hold = rsp_side.hold; + assign mem_rsp_phase_end = rsp_side.phase_end; + assign mem_rsp_phase = rsp_side.phase; + end else begin : g_no_rsp_side + assign mem_rsp_hold = 1'b0; + assign mem_rsp_phase_end = 1'b0; + assign mem_rsp_phase = 2'd0; + `UNUSED_VAR (client_if.rsp_data.side) + end + lsu_header_t rsp_hdr; wire [INST_LSU_BITS-1:0] rsp_op_type; wire [NUM_LANES-1:0][REQ_ASHIFT-1:0] rsp_align; @@ -546,12 +629,35 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // result + // A wide load's beat writes the register of its phase; its bytesel + // offset bit 1 says whether the beat releases that register. + wire rsp_wide; + wire rsp_quad; + if (LDS_WIDE_ENABLE) begin : g_rsp_wide + wire [XLENB_W-1:0] bytesel_size = rsp_hdr.bytesel[BYTESEL_BITS-1 -: XLENB_W]; + wire [XLENB_W-1:0] bytesel_off = rsp_hdr.bytesel[0 +: XLENB_W]; + assign rsp_wide = (bytesel_size == XLENB_W'(XLENB-1)) && bytesel_off[0]; + assign rsp_quad = bytesel_off[1]; + end else begin : g_no_rsp_wide + assign rsp_wide = 1'b0; + assign rsp_quad = 1'b0; + `UNUSED_VAR ({mem_rsp_hold, mem_rsp_phase_end, mem_rsp_phase_rel}) + end + lsu_header_t rsp_hdr2; always @(*) begin rsp_hdr2 = rsp_hdr; rsp_hdr2.tmask = mem_rsp_mask; rsp_hdr2.sop = mem_rsp_sop_pkt; rsp_hdr2.eop = mem_rsp_eop_pkt; + if (rsp_wide) begin + if (rsp_quad) begin + rsp_hdr2.rd[1:0] = mem_rsp_phase; + end else begin + rsp_hdr2.rd[0] = mem_rsp_phase[0]; + end + rsp_hdr2.bytesel = {XLENB_W'(XLENB-1), XLENB_W'({mem_rsp_phase_rel, 1'b1})}; + end end VX_elastic_buffer #( diff --git a/hw/rtl/core/VX_lsu_mem_path.sv b/hw/rtl/core/VX_lsu_mem_path.sv index 6cef8767b3..c04a458dc8 100644 --- a/hw/rtl/core/VX_lsu_mem_path.sv +++ b/hw/rtl/core/VX_lsu_mem_path.sv @@ -49,9 +49,10 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH), + .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) ) lsu_mem_if(); // scheduler: client 0 is the block's lanes, client 1 the TCU metadata AGU @@ -206,6 +207,8 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( .out_rsp_tag (dcache_coalesced_if.rsp_data.tag), .out_rsp_ready (dcache_coalesced_if.rsp_ready) ); + assign lsu_dcache_if.rsp_data.side = '0; + `UNUSED_VAR (dcache_coalesced_if.rsp_data.side) end else begin : g_no_coalescing diff --git a/hw/rtl/core/VX_lsu_scheduler.sv b/hw/rtl/core/VX_lsu_scheduler.sv index f093e8cd7e..bba1047cbe 100644 --- a/hw/rtl/core/VX_lsu_scheduler.sv +++ b/hw/rtl/core/VX_lsu_scheduler.sv @@ -68,6 +68,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( wire [SCHED_TAG_WIDTH-1:0] sched_rsp_tag; wire sched_rsp_sop; wire sched_rsp_eop; + wire [`UP(LSU_RSP_SIDE_W)-1:0] sched_rsp_side; wire sched_rsp_ready; // Flatten interface array into per-signal vectors so the arbiter @@ -109,6 +110,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign cli_rsp_data[0].tag = sched_rsp_tag; assign cli_rsp_data[0].sop = sched_rsp_sop; assign cli_rsp_data[0].eop = sched_rsp_eop; + assign cli_rsp_data[0].side = sched_rsp_side; assign sched_rsp_ready = cli_rsp_ready[0]; end else begin : g_arb // Round-robin priority. rr_ptr rotates after every accepted request. @@ -186,6 +188,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign cli_rsp_data[i].tag = sched_rsp_tag[LSU_CLIENT_TAG_WIDTH-1:0]; assign cli_rsp_data[i].sop = sched_rsp_sop; assign cli_rsp_data[i].eop = sched_rsp_eop; + assign cli_rsp_data[i].side = sched_rsp_side; end // sched_rsp_ready is driven by the granted client's ready. logic sched_rsp_ready_l; @@ -215,6 +218,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0] lsu_mem_rsp_mask; wire [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] lsu_mem_rsp_data; wire [LSU_TAG_WIDTH-1:0] lsu_mem_rsp_tag; + wire lsu_mem_rsp_hold; wire lsu_mem_rsp_ready; wire sched_req_queue_empty; @@ -233,6 +237,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( .PENDING_SIZE (PENDING_SIZE), .UUID_WIDTH (UUID_WIDTH), .RSP_PARTIAL (1), + .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W), .MEM_OUT_BUF (0), .CORE_OUT_BUF(0), .LUTRAM (1) @@ -260,6 +265,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( .core_rsp_tag (sched_rsp_tag), .core_rsp_sop (sched_rsp_sop), .core_rsp_eop (sched_rsp_eop), + .core_rsp_side (sched_rsp_side), .core_rsp_ready (sched_rsp_ready), .mem_req_valid (lsu_mem_req_valid), @@ -276,6 +282,8 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( .mem_rsp_mask (lsu_mem_rsp_mask), .mem_rsp_data (lsu_mem_rsp_data), .mem_rsp_tag (lsu_mem_rsp_tag), + .mem_rsp_hold (lsu_mem_rsp_hold), + .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_rsp_ready) ); @@ -296,6 +304,14 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign lsu_mem_rsp_mask = lsu_mem_if.rsp_data.mask; assign lsu_mem_rsp_data = lsu_mem_if.rsp_data.data; assign lsu_mem_rsp_tag = lsu_mem_if.rsp_data.tag; + if (LSU_RSP_SIDE_W != 0) begin : g_rsp_hold + lsu_rsp_side_t rsp_side; + assign rsp_side = lsu_rsp_side_t'(lsu_mem_if.rsp_data.side); + assign lsu_mem_rsp_hold = rsp_side.hold; + `UNUSED_VAR (rsp_side) + end else begin : g_no_rsp_hold + assign lsu_mem_rsp_hold = 1'b0; + end assign lsu_mem_if.rsp_ready = lsu_mem_rsp_ready; assign empty = sched_req_queue_empty; diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 6c4f5d09b2..029ac0f27f 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -175,7 +175,8 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .TAG_WIDTH (LSU_TAG_WIDTH), .DMA_ENABLE (LMEM_DMA_EN), .DMA_TAG_WIDTH (LMEM_DMA_TAG_WIDTH), - .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) + .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), + .WIDE_ENABLE (LDS_WIDE_ENABLE) ) lmem_unit ( .clk (clk), .reset (reset), diff --git a/hw/rtl/core/VX_scoreboard.sv b/hw/rtl/core/VX_scoreboard.sv index 922ef621d9..5ccbfee990 100644 --- a/hw/rtl/core/VX_scoreboard.sv +++ b/hw/rtl/core/VX_scoreboard.sv @@ -103,6 +103,7 @@ module VX_scoreboard import VX_gpu_pkg::*; #( wire staging_fire = staging_if[w].valid && staging_if[w].ready; wire writeback_fire = writeback_if.valid && writeback_if.data.eop_wis[w]; + wire writeback_rel = writeback_if.valid && writeback_if.data.rel_wis[w]; wire [NUM_OPDS-1:0] [NUM_REGS_BITS-1:0] ibf_opds, stg_opds; assign ibf_opds = {ibuffer_if[w].data.rs3, ibuffer_if[w].data.rs2, ibuffer_if[w].data.rs1, ibuffer_if[w].data.rd}; @@ -115,6 +116,24 @@ module VX_scoreboard import VX_gpu_pkg::*; #( wire [NUM_XREGS-1:0] ibf_xregs_mask = ibuffer_if[w].data.rd_xregs | ibuffer_if[w].data.wr_xregs; wire [NUM_XREGS-1:0] stg_xregs_mask = staging_if[w].data.rd_xregs | staging_if[w].data.wr_xregs; + // A wide load's rd names an aligned group of two or four registers. + wire ibf_wide, ibf_quad, stg_wide, stg_quad; + if (LDS_WIDE_ENABLE) begin : g_wide + wire [XLENB_W-1:0] ibf_size = ibuffer_if[w].data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; + wire [XLENB_W-1:0] ibf_off = ibuffer_if[w].data.bytesel[0 +: XLENB_W]; + wire [XLENB_W-1:0] stg_size = staging_if[w].data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; + wire [XLENB_W-1:0] stg_off = staging_if[w].data.bytesel[0 +: XLENB_W]; + assign ibf_wide = (ibf_size == XLENB_W'(XLENB-1)) && ibf_off[0]; + assign ibf_quad = ibf_wide && ibf_off[1]; + assign stg_wide = (stg_size == XLENB_W'(XLENB-1)) && stg_off[0]; + assign stg_quad = stg_wide && stg_off[1]; + end else begin : g_no_wide + assign ibf_wide = 1'b0; + assign ibf_quad = 1'b0; + assign stg_wide = 1'b0; + assign stg_quad = 1'b0; + end + wire [NUM_OPDS-1:0][REG_TYPES-1:0][RV_REGS-1:0] ibf_opd_mask, stg_opd_mask; for (genvar i = 0; i < NUM_OPDS; ++i) begin : g_opd_masks @@ -124,6 +143,20 @@ module VX_scoreboard import VX_gpu_pkg::*; #( end end + // the registers the staging instr reserves: its rd, or rd's group + wire [NUM_REGS-1:0] stg_rsv_mask; + if (LDS_WIDE_ENABLE) begin : g_rsv_group + for (genvar r = 0; r < NUM_REGS; ++r) begin : g_r + localparam [NUM_REGS_BITS-1:0] R = NUM_REGS_BITS'(r); + assign stg_rsv_mask[r] = stg_used_rs[0] + && (stg_opds[0][NUM_REGS_BITS-1:2] == R[NUM_REGS_BITS-1:2]) + && (stg_quad || (stg_opds[0][1] == R[1])) + && (stg_wide || (stg_opds[0][0] == R[0])); + end + end else begin : g_rsv_single + assign stg_rsv_mask = stg_opd_mask[0]; + end + // Writeback release feeds wb_inuse_regs, which the busy check reads; the // staging reserve is added on top to form the next state, inuse_regs_n. reg [NUM_REGS-1:0] wb_inuse_regs; @@ -131,10 +164,10 @@ module VX_scoreboard import VX_gpu_pkg::*; #( always @(*) begin wb_inuse_regs = inuse_regs; wb_inuse_xregs = inuse_xregs; + if ((writeback_fire || writeback_rel) && writeback_if.data.wb) begin + wb_inuse_regs[writeback_if.data.rd] = 0; // release rd + end if (writeback_fire) begin - if (writeback_if.data.wb) begin - wb_inuse_regs[writeback_if.data.rd] = 0; // release rd - end wb_inuse_xregs &= ~writeback_if.data.wr_xregs; // release special regs end end @@ -144,7 +177,7 @@ module VX_scoreboard import VX_gpu_pkg::*; #( inuse_xregs_n = wb_inuse_xregs; if (staging_fire) begin if (staging_if[w].data.wb) begin - inuse_regs_n |= stg_opd_mask[0]; // reserve rd + inuse_regs_n |= stg_rsv_mask; // reserve rd or its group end inuse_xregs_n |= staging_if[w].data.wr_xregs; // reserve special regs end @@ -166,17 +199,45 @@ module VX_scoreboard import VX_gpu_pkg::*; #( // against the released in-use set once and the outcomes select bits. wire [NUM_REGS-1:0] ibf_regs_mask = ibf_opd_mask[0] | ibf_opd_mask[1] | ibf_opd_mask[2] | ibf_opd_mask[3]; wire [NUM_REGS-1:0] stg_regs_mask = stg_opd_mask[0] | stg_opd_mask[1] | stg_opd_mask[2] | stg_opd_mask[3]; - wire ibf_busy = | (wb_inuse_regs & ibf_regs_mask); - wire stg_busy = | (wb_inuse_regs & stg_regs_mask); + wire ibf_group_busy, stg_group_busy; + if (LDS_WIDE_ENABLE) begin : g_group_busy + // A destination group's other registers are tested against the + // registered in-use set, reduced per pair and quad ahead, off the + // operand masks' reduction. Missing a release in this cycle only + // delays the group by a cycle. + wire [NUM_REGS/2-1:0] inuse_pairs; + wire [NUM_REGS/4-1:0] inuse_quads; + for (genvar g = 0; g < NUM_REGS/2; ++g) begin : g_pairs + assign inuse_pairs[g] = | inuse_regs[2*g +: 2]; + end + for (genvar g = 0; g < NUM_REGS/4; ++g) begin : g_quads + assign inuse_quads[g] = | inuse_regs[4*g +: 4]; + end + assign ibf_group_busy = ibf_quad ? inuse_quads[ibf_opds[0][NUM_REGS_BITS-1:2]] + : (ibf_wide && inuse_pairs[ibf_opds[0][NUM_REGS_BITS-1:1]]); + assign stg_group_busy = stg_quad ? inuse_quads[stg_opds[0][NUM_REGS_BITS-1:2]] + : (stg_wide && inuse_pairs[stg_opds[0][NUM_REGS_BITS-1:1]]); + end else begin : g_no_group_busy + assign ibf_group_busy = 1'b0; + assign stg_group_busy = 1'b0; + end + wire ibf_busy = (| (wb_inuse_regs & ibf_regs_mask)) || ibf_group_busy; + wire stg_busy = (| (wb_inuse_regs & stg_regs_mask)) || stg_group_busy; wire ibf_xbusy = | (wb_inuse_xregs & ibf_xregs_mask); wire stg_xbusy = | (wb_inuse_xregs & stg_xregs_mask); // On fire the staging instr also reserves its rd and wr_xregs. Its // successor sees that reservation only through a register it names, - // and a lone staging instr always names its own rd. + // and a lone staging instr always names its own rd. A register group + // matches any register in it. wire [NUM_OPDS-1:0] ibf_reads_rd; for (genvar i = 0; i < NUM_OPDS; ++i) begin : g_ibf_reads_rd - assign ibf_reads_rd[i] = ibf_used_rs[i] && (ibf_opds[i] == stg_opds[0]); + wire any_bit1 = stg_quad || ((i == 0) && ibf_quad); + wire any_bit0 = stg_wide || ((i == 0) && ibf_wide); + assign ibf_reads_rd[i] = ibf_used_rs[i] + && (ibf_opds[i][NUM_REGS_BITS-1:2] == stg_opds[0][NUM_REGS_BITS-1:2]) + && (any_bit1 || (ibf_opds[i][1] == stg_opds[0][1])) + && (any_bit0 || (ibf_opds[i][0] == stg_opds[0][0])); end wire stg_rsv_rd = staging_if[w].data.wb; wire ibf_rsv = (stg_rsv_rd && (| ibf_reads_rd)) @@ -254,7 +315,7 @@ module VX_scoreboard import VX_gpu_pkg::*; #( w, to_fullPC(staging_if[w].data.PC), staging_if[w].data.tmask, timeout_ctr, operands_busy, staging_if[w].data.uuid)) - `RUNTIME_ASSERT(~(writeback_fire && writeback_if.data.wb) || inuse_regs[writeback_if.data.rd] != 0, + `RUNTIME_ASSERT(~((writeback_fire || writeback_rel) && writeback_if.data.wb) || inuse_regs[writeback_if.data.rd] != 0, ("invalid writeback register: wid=%0d, PC=0x%0h, tmask=%b, rd=%0d (#%0d)", w, to_fullPC(writeback_if.data.PC), writeback_if.data.tmask, writeback_if.data.rd, writeback_if.data.uuid)) diff --git a/hw/rtl/core/VX_uop_sequencer.sv b/hw/rtl/core/VX_uop_sequencer.sv index 2b888a8cce..f91d9ec6f0 100644 --- a/hw/rtl/core/VX_uop_sequencer.sv +++ b/hw/rtl/core/VX_uop_sequencer.sv @@ -234,7 +234,7 @@ module VX_uop_sequencer import `TRACE(1, ("%t: %s decode: wid=%0d, cta_id=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, WARP_ID, output_if.data.cta_id, to_fullPC(output_if.data.PC))) VX_trace_pkg::trace_ex_type(1, output_if.data.ex_type); `TRACE(1, (", op=")) - VX_trace_pkg::trace_ex_op(1, output_if.data.ex_type, output_if.data.op_type, output_if.data.op_args); + VX_trace_pkg::trace_ex_op(1, output_if.data.ex_type, output_if.data.op_type, output_if.data.op_args, output_if.data.bytesel); `TRACE(1, (", tmask=%b, wb=%b, rd_xregs=%b, wr_xregs=%b, used_rs=%b, rd=", output_if.data.tmask, output_if.data.wb, output_if.data.rd_xregs, output_if.data.wr_xregs, output_if.data.used_rs)) VX_trace_pkg::trace_reg_idx(1, output_if.data.rd); `TRACE(1, (", rs1=")) diff --git a/hw/rtl/libs/VX_mem_scheduler.sv b/hw/rtl/libs/VX_mem_scheduler.sv index 5ac1bca106..11b3d65b10 100644 --- a/hw/rtl/libs/VX_mem_scheduler.sv +++ b/hw/rtl/libs/VX_mem_scheduler.sv @@ -43,6 +43,10 @@ module VX_mem_scheduler #( // all feeding the one output register -- which then cannot be placed near // its own data. Distributed RAM lands in the slices that hold that register. parameter REQQ_LUTRAM = 0, + // Nonzero: a memory response carries a side word to its core response, and + // a hold bit that keeps the lanes it serves outstanding. Needs RSP_PARTIAL + // and no coalescing. + parameter RSP_SIDE_WIDTH = 0, parameter WORD_WIDTH = WORD_SIZE * 8, parameter LINE_WIDTH = LINE_SIZE * 8, @@ -85,6 +89,7 @@ module VX_mem_scheduler #( output wire [TAG_WIDTH-1:0] core_rsp_tag, output wire core_rsp_sop, output wire core_rsp_eop, + output wire [`UP(RSP_SIDE_WIDTH)-1:0] core_rsp_side, input wire core_rsp_ready, // Memory request @@ -103,6 +108,8 @@ module VX_mem_scheduler #( input wire [MEM_CHANNELS-1:0] mem_rsp_mask, input wire [MEM_CHANNELS-1:0][LINE_WIDTH-1:0] mem_rsp_data, input wire [MEM_TAG_WIDTH-1:0] mem_rsp_tag, + input wire mem_rsp_hold, + input wire [`UP(RSP_SIDE_WIDTH)-1:0] mem_rsp_side, output wire mem_rsp_ready ); localparam BATCH_SEL_WIDTH = `UP(MEM_BATCH_BITS); @@ -117,6 +124,7 @@ module VX_mem_scheduler #( `STATIC_ASSERT ((MEM_CHANNELS <= CORE_REQS), ("invalid parameter")) `STATIC_ASSERT (`IS_DIVISBLE(CORE_REQS * WORD_SIZE, LINE_SIZE), ("invalid parameter")) `STATIC_ASSERT ((TAG_WIDTH >= UUID_WIDTH), ("invalid parameter")) + `STATIC_ASSERT ((RSP_SIDE_WIDTH == 0) || ((RSP_PARTIAL != 0) && !COALESCE_ENABLE), ("invalid parameter")) `RUNTIME_ASSERT((~core_req_valid || core_req_mask != 0), ("invalid request mask")) wire ibuf_push; @@ -477,7 +485,11 @@ module VX_mem_scheduler #( assign crsp_valid = mem_rsp_valid_s; assign crsp_mask = mem_rsp_mask_s; assign crsp_sop = 1'b1; - assign crsp_eop = 1'b1; + if (RSP_SIDE_WIDTH != 0) begin : g_hold + assign crsp_eop = ~mem_rsp_hold; + end else begin : g_no_hold + assign crsp_eop = 1'b1; + end assign crsp_data = mem_rsp_data_s; assign mem_rsp_ready_s = crsp_ready; @@ -493,7 +505,14 @@ module VX_mem_scheduler #( assign curr_mask[r] = (BATCH_SEL_WIDTH'(i) == rsp_batch_idx) && mem_rsp_mask_s[j]; end - assign rsp_rem_mask_n = rsp_rem_mask[ibuf_raddr] & ~curr_mask; + wire [CORE_REQS-1:0] rsp_retire_mask; + if (RSP_SIDE_WIDTH != 0) begin : g_hold + assign rsp_retire_mask = mem_rsp_hold ? '0 : curr_mask; + end else begin : g_no_hold + assign rsp_retire_mask = curr_mask; + end + + assign rsp_rem_mask_n = rsp_rem_mask[ibuf_raddr] & ~rsp_retire_mask; wire mem_rsp_fire_s = mem_rsp_valid_s && mem_rsp_ready_s; @@ -597,8 +616,22 @@ module VX_mem_scheduler #( // Send response to caller + localparam CRSP_DATAW = CORE_REQS + 1 + 1 + (CORE_REQS * WORD_WIDTH) + TAG_WIDTH; + + wire [CRSP_DATAW+RSP_SIDE_WIDTH-1:0] crsp_buf_in, crsp_buf_out; + + if (RSP_SIDE_WIDTH != 0) begin : g_crsp_side + assign crsp_buf_in = {mem_rsp_side, crsp_mask, crsp_sop, crsp_eop, crsp_data, crsp_tag}; + assign {core_rsp_side, core_rsp_mask, core_rsp_sop, core_rsp_eop, core_rsp_data, core_rsp_tag} = crsp_buf_out; + end else begin : g_crsp_no_side + assign crsp_buf_in = {crsp_mask, crsp_sop, crsp_eop, crsp_data, crsp_tag}; + assign {core_rsp_mask, core_rsp_sop, core_rsp_eop, core_rsp_data, core_rsp_tag} = crsp_buf_out; + assign core_rsp_side = '0; + `UNUSED_VAR ({mem_rsp_hold, mem_rsp_side}) + end + VX_elastic_buffer #( - .DATAW (CORE_REQS + 1 + 1 + (CORE_REQS * WORD_WIDTH) + TAG_WIDTH), + .DATAW (CRSP_DATAW + RSP_SIDE_WIDTH), .SIZE (`TO_OUT_BUF_SIZE(CORE_OUT_BUF)), .OUT_REG (`TO_OUT_BUF_REG(CORE_OUT_BUF)) ) rsp_buf ( @@ -606,8 +639,8 @@ module VX_mem_scheduler #( .reset (reset), .valid_in (crsp_valid), .ready_in (crsp_ready), - .data_in ({crsp_mask, crsp_sop, crsp_eop, crsp_data, crsp_tag}), - .data_out ({core_rsp_mask, core_rsp_sop, core_rsp_eop, core_rsp_data, core_rsp_tag}), + .data_in (crsp_buf_in), + .data_out (crsp_buf_out), .valid_out (core_rsp_valid), .ready_out (core_rsp_ready) ); diff --git a/hw/rtl/mem/VX_lmem_engine.sv b/hw/rtl/mem/VX_lmem_engine.sv index ef083fc8e0..3b79fe9728 100644 --- a/hw/rtl/mem/VX_lmem_engine.sv +++ b/hw/rtl/mem/VX_lmem_engine.sv @@ -35,6 +35,13 @@ // payload mux. The first pass's winners come from S0; a replay finds the next // winners in the cycle after a pass, so a conflicting request replays at one // pass per two cycles. The last pass frees S1 in its own cycle. +// +// A wide read takes two or four words per lane, its lanes' base words aligned +// to the group. It is served one word-phase at a time: a phase sets the low +// word-address bits of every lane to the phase number, which leaves each pair +// of lanes' bank and word equality unchanged, so every phase reuses the +// request's matrices and first winners, and starts in the cycle after the +// previous phase's last pass. module VX_lmem_engine import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter NUM_LANES = 4, @@ -49,11 +56,14 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( parameter ATTR_WIDTH = 1, parameter NUM_BLOCKS = 1, parameter RSP_QUEUE_SIZE = 8, + // serve wide reads + parameter WIDE_ENABLE = 0, parameter WORD_WIDTH = WORD_SIZE * 8, parameter BLOCK_WIDTH = `UP(`CLOG2(NUM_BLOCKS)), parameter DONE_WIDTH = `CLOG2(NUM_BLOCKS + 1), - parameter BANK_ADDR_WIDTH = ADDR_WIDTH - `CLOG2(NUM_BANKS) + parameter BANK_ADDR_WIDTH = ADDR_WIDTH - `CLOG2(NUM_BANKS), + localparam RSP_SIDE_WIDTH = WIDE_ENABLE ? $bits(lsu_rsp_side_t) : 0 ) ( input wire clk, input wire reset, @@ -71,6 +81,8 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( input wire [NUM_LANES-1:0][ADDR_WIDTH-1:0] req_addr, input wire [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] req_payload, input wire [TAG_WIDTH-1:0] req_tag, + // a read's words per lane, as log2 (up to 4 words) + input wire [1:0] req_wide, output wire req_ready, // the engine holds a request; the one it holds is a store or an atomic @@ -100,6 +112,8 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( output wire [NUM_LANES-1:0] rsp_mask, output wire [NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data, output wire [TAG_WIDTH-1:0] rsp_tag, + // the response's phase of a wide read (lsu_rsp_side_t) + output wire [`UP(RSP_SIDE_WIDTH)-1:0] rsp_side, input wire rsp_take, input wire [DONE_WIDTH-1:0] rsp_done, @@ -114,9 +128,13 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS); localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); localparam CREDIT_WIDTH = `CLOG2(RSP_QUEUE_SIZE + 1); - localparam RSP_DATAW = BLOCK_WIDTH + NUM_LANES + NUM_LANES * WORD_WIDTH + TAG_WIDTH; + localparam WIDE_WIDTH = WIDE_ENABLE ? 2 : 0; + // the response's side word travels with its tag + localparam TAGX_WIDTH = TAG_WIDTH + RSP_SIDE_WIDTH; + localparam RSP_DATAW = BLOCK_WIDTH + NUM_LANES + NUM_LANES * WORD_WIDTH + TAGX_WIDTH; `STATIC_ASSERT(ADDR_WIDTH > BANK_SEL_BITS, ("invalid parameter")) + `STATIC_ASSERT((WIDE_ENABLE == 0) || (ADDR_WIDTH >= 2), ("invalid parameter")) // S0: the engine's input queue. Holding two requests, it takes one in any // cycle that starts with at most one, so its ready is a register and the @@ -135,10 +153,27 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0][ADDR_WIDTH-1:0] s0_addr; wire [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] s0_payload; wire [TAG_WIDTH-1:0] s0_tag; + wire [1:0] s0_wide; wire s1_free; + wire [S0_DATAW-1:0] s0_base_in, s0_base_out; + wire [S0_DATAW+WIDE_WIDTH-1:0] s0_buf_in, s0_buf_out; + + assign s0_base_in = {req_block, req_rw, req_read, req_amo, req_attr, req_mask, req_addr, req_payload, req_tag}; + assign {s0_block, s0_rw, s0_read, s0_amo, s0_attr, s0_mask, s0_addr, s0_payload, s0_tag} = s0_base_out; + + if (WIDE_ENABLE != 0) begin : g_s0_wide + assign s0_buf_in = {req_wide, s0_base_in}; + assign {s0_wide, s0_base_out} = s0_buf_out; + end else begin : g_s0_no_wide + assign s0_buf_in = s0_base_in; + assign s0_base_out = s0_buf_out; + assign s0_wide = 2'b00; + `UNUSED_VAR (req_wide) + end + VX_elastic_buffer #( - .DATAW (S0_DATAW), + .DATAW (S0_DATAW + WIDE_WIDTH), .SIZE (2), .OUT_REG (1) ) s0_buf ( @@ -146,8 +181,8 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( .reset (reset), .valid_in (req_valid), .ready_in (req_ready), - .data_in ({req_block, req_rw, req_read, req_amo, req_attr, req_mask, req_addr, req_payload, req_tag}), - .data_out ({s0_block, s0_rw, s0_read, s0_amo, s0_attr, s0_mask, s0_addr, s0_payload, s0_tag}), + .data_in (s0_buf_in), + .data_out (s0_buf_out), .valid_out (s0_valid), .ready_out (s1_free) ); @@ -203,6 +238,14 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( reg [TAG_WIDTH-1:0] s1_tag; reg [NUM_LANES-1:0][NUM_LANES-1:0] s1_same_bank; reg [NUM_LANES-1:0][NUM_LANES-1:0] s1_same_word; + // a wide read's group size, its phase, whether the phase is its last, and + // the request's first pass, which every phase starts from + reg s1_quad; + reg [1:0] s1_phase; + reg s1_final; + reg [NUM_LANES-1:0] s1_mask0; + reg [NUM_LANES-1:0] s1_first0; + reg s1_one_pass0; // A pass serves the registered bank winners and the pending read lanes of // their words. After a pass that leaves lanes pending, the next cycle finds @@ -228,10 +271,15 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( reg [CREDIT_WIDTH-1:0] credits; wire has_rsp = ~s1_rw; wire pass_fire = s1_valid && s1_ready && pass_en && (~has_rsp || (credits != '0)); - wire s1_release = pass_fire && s1_last; + wire last_phase = (WIDE_ENABLE == 0) || s1_final; + wire phase_end = pass_fire && s1_last; + wire s1_release = phase_end && last_phase; + wire s1_advance = phase_end && ~last_phase; assign s1_free = ~s1_valid || s1_release; wire s1_load = s0_valid && s1_free; + wire [1:0] phase_n = s1_phase + 2'd1; + always @(posedge clk) begin if (reset) begin s1_valid <= 1'b0; @@ -255,6 +303,25 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( s1_tag <= s0_tag; s1_same_bank <= s0_same_bank; s1_same_word <= s0_same_word; + s1_quad <= s0_wide[1]; + s1_phase <= 2'd0; + s1_final <= (s0_wide == 2'd0); + s1_mask0 <= s0_mask; + s1_first0 <= s0_first; + s1_one_pass0 <= s0_one_pass; + end else if (s1_advance) begin + s1_ready <= 1'b1; + s1_last <= s1_one_pass0; + s1_winners <= s1_first0; + s1_pending <= s1_mask0; + s1_phase <= phase_n; + s1_final <= (phase_n == {s1_quad, 1'b1}); + for (integer i = 0; i < NUM_LANES; ++i) begin + s1_addr[i][0] <= phase_n[0]; + if (s1_quad) begin + s1_addr[i][1] <= phase_n[1]; + end + end end else if (pass_fire) begin s1_ready <= 1'b0; s1_pending <= s1_pending & ~served; @@ -265,6 +332,18 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( end end + wire [TAGX_WIDTH-1:0] s1_tagx; + if (WIDE_ENABLE != 0) begin : g_s1_side + lsu_rsp_side_t s1_side; + assign s1_side.hold = ~s1_final; + assign s1_side.phase_end = s1_last; + assign s1_side.phase = s1_phase; + assign s1_tagx = {s1_side, s1_tag}; + end else begin : g_s1_no_side + assign s1_tagx = s1_tag; + `UNUSED_VAR ({s1_quad, s1_phase, s1_final, s1_mask0, s1_first0, s1_one_pass0, phase_n}) + end + // stores and atomics held between S0's two entries and S1 reg [1:0] num_writes; wire write_in = req_valid && req_ready && ~req_read; @@ -349,17 +428,17 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( wire [BLOCK_WIDTH-1:0] m2_block; wire [NUM_LANES-1:0] m2_mask; wire [NUM_LANES-1:0][BANK_SEL_WIDTH-1:0] m2_bank; - wire [TAG_WIDTH-1:0] m2_tag; + wire [TAGX_WIDTH-1:0] m2_tag; VX_pipe_register #( - .DATAW (1 + BLOCK_WIDTH + NUM_LANES + NUM_LANES * BANK_SEL_WIDTH + TAG_WIDTH), + .DATAW (1 + BLOCK_WIDTH + NUM_LANES + NUM_LANES * BANK_SEL_WIDTH + TAGX_WIDTH), .RESETW (1), .DEPTH (2) ) rsp_meta_pipe ( .clk (clk), .reset (reset), .enable (1'b1), - .data_in ({pass_fire && has_rsp, s1_block, served, s1_bank, s1_tag}), + .data_in ({pass_fire && has_rsp, s1_block, served, s1_bank, s1_tagx}), .data_out ({m2_valid, m2_block, m2_mask, m2_bank, m2_tag}) ); @@ -374,7 +453,7 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( wire [BLOCK_WIDTH-1:0] m3_block; wire [NUM_LANES-1:0] m3_mask; wire [NUM_LANES-1:0][WORD_WIDTH-1:0] m3_data; - wire [TAG_WIDTH-1:0] m3_tag; + wire [TAGX_WIDTH-1:0] m3_tag; VX_pipe_register #( .DATAW (RSP_DATAW + 1), @@ -413,8 +492,16 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( `UNUSED_PIN (alm_full), `UNUSED_PIN (size) ); + wire [TAGX_WIDTH-1:0] rsp_tagx; assign rsp_valid = ~rsp_empty || m3_valid; - assign {rsp_block, rsp_mask, rsp_data, rsp_tag} = rsp_empty ? {m3_block, m3_mask, m3_data, m3_tag} : rsp_head; + assign {rsp_block, rsp_mask, rsp_data, rsp_tagx} = rsp_empty ? {m3_block, m3_mask, m3_data, m3_tag} : rsp_head; + + if (WIDE_ENABLE != 0) begin : g_rsp_side + assign {rsp_side, rsp_tag} = rsp_tagx; + end else begin : g_rsp_no_side + assign rsp_tag = rsp_tagx; + assign rsp_side = '0; + end // a response's slot is free again once its block has taken it on wire credit_take = pass_fire && has_rsp; @@ -433,8 +520,8 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( `ifdef DBG_TRACE_MEM always @(posedge clk) begin if (pass_fire) begin - `TRACE(2, ("%t: %s pass: rw=%b, amo=%b, pending=%b, served=%b, tag=0x%0h\n", - $time, INSTANCE_ID, s1_rw, s1_amo, s1_pending, served, s1_tag)) + `TRACE(2, ("%t: %s pass: rw=%b, amo=%b, pending=%b, served=%b, phase=%0d, tag=0x%0h\n", + $time, INSTANCE_ID, s1_rw, s1_amo, s1_pending, served, s1_phase, s1_tag)) end end `endif diff --git a/hw/rtl/mem/VX_lmem_switch.sv b/hw/rtl/mem/VX_lmem_switch.sv index c155fe903f..ae8aa381ec 100644 --- a/hw/rtl/mem/VX_lmem_switch.sv +++ b/hw/rtl/mem/VX_lmem_switch.sv @@ -106,9 +106,29 @@ module VX_lmem_switch import VX_gpu_pkg::*; #( .ready_out (local_out_if.req_ready) ); + // The local responses carry a sideband; the global ones leave it clear. + wire [1:0][RSP_DATAW+LSU_RSP_SIDE_W-1:0] rsp_arb_in; + wire [RSP_DATAW+LSU_RSP_SIDE_W-1:0] rsp_arb_out; + + wire [RSP_DATAW-1:0] local_rsp = {local_out_if.rsp_data.mask, local_out_if.rsp_data.data, local_out_if.rsp_data.tag}; + wire [RSP_DATAW-1:0] global_rsp = {global_out_if.rsp_data.mask, global_out_if.rsp_data.data, global_out_if.rsp_data.tag}; + `UNUSED_VAR (global_out_if.rsp_data.side) + + if (LSU_RSP_SIDE_W != 0) begin : g_rsp_side + assign rsp_arb_in[1] = {local_out_if.rsp_data.side, local_rsp}; + assign rsp_arb_in[0] = {LSU_RSP_SIDE_W'(0), global_rsp}; + assign {lsu_in_if.rsp_data.side, lsu_in_if.rsp_data.mask, lsu_in_if.rsp_data.data, lsu_in_if.rsp_data.tag} = rsp_arb_out; + end else begin : g_no_rsp_side + assign rsp_arb_in[1] = local_rsp; + assign rsp_arb_in[0] = global_rsp; + assign {lsu_in_if.rsp_data.mask, lsu_in_if.rsp_data.data, lsu_in_if.rsp_data.tag} = rsp_arb_out; + assign lsu_in_if.rsp_data.side = '0; + `UNUSED_VAR (local_out_if.rsp_data.side) + end + VX_stream_arb #( .NUM_INPUTS (2), - .DATAW (RSP_DATAW), + .DATAW (RSP_DATAW + LSU_RSP_SIDE_W), .ARBITER (ARBITER), .OUT_BUF (RSP_OUT_BUF) ) rsp_arb ( @@ -122,11 +142,8 @@ module VX_lmem_switch import VX_gpu_pkg::*; #( local_out_if.rsp_ready, global_out_if.rsp_ready }), - .data_in ({ - local_out_if.rsp_data, - global_out_if.rsp_data - }), - .data_out (lsu_in_if.rsp_data), + .data_in (rsp_arb_in), + .data_out (rsp_arb_out), .valid_out (lsu_in_if.rsp_valid), .ready_out (lsu_in_if.rsp_ready), `UNUSED_PIN (sel_out) diff --git a/hw/rtl/mem/VX_lmem_unit.sv b/hw/rtl/mem/VX_lmem_unit.sv index a48400fb16..f7cf4f4a80 100644 --- a/hw/rtl/mem/VX_lmem_unit.sv +++ b/hw/rtl/mem/VX_lmem_unit.sv @@ -32,6 +32,10 @@ // request is seen, and it is accepted in the cycle that follows. An atomic owns // its bank for three cycles (read, capture, write back), so no pass or DMA access // is issued in the two cycles after an atomic pass. +// +// A wide read names its group size in the atomic op of a request that is not +// an atomic. Its lanes' addresses are aligned down to the group here, and the +// engine serves the group one word-phase at a time. module VX_lmem_unit import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", @@ -55,7 +59,10 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( parameter DMA_TAG_WIDTH = 1, // Synthesize atomic-op + LR/SC logic in the banks - parameter AMO_ENABLE = 0 + parameter AMO_ENABLE = 0, + + // Serve wide reads + parameter WIDE_ENABLE = 0 ) ( input wire clk, input wire reset, @@ -97,6 +104,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( localparam PAYLOAD_WIDTH = WORD_WIDTH + WORD_SIZE + LANE_SB_WIDTH; // common to all lanes: the atomic op and its signedness localparam ATTR_WIDTH = AMO_OP_WIDTH + 1; + localparam RSP_SIDE_WIDTH = WIDE_ENABLE ? $bits(lsu_rsp_side_t) : 0; `STATIC_ASSERT(ADDR_WIDTH == (BANK_ADDR_WIDTH + `CLOG2(NUM_BANKS)), ("invalid parameter")) `STATIC_ASSERT(`IS_POW2(NUM_BANKS), ("invalid parameter")) @@ -112,6 +120,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][ADDR_WIDTH-1:0] in_addr; wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] in_payload; wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] in_tag; + wire [NUM_BLOCKS-1:0][1:0] in_wide; wire [NUM_BLOCKS-1:0] in_ready; for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_in @@ -125,9 +134,23 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( assign in_attr[k] = AMO_ENABLE ? {attr0.amo.amo_op, attr0.amo.amo_unsigned} : '0; assign in_mask[k] = lsu_mem_if[k].req_data.mask; assign in_tag[k] = lsu_mem_if[k].req_data.tag; + if (WIDE_ENABLE != 0) begin : g_wide + wire [AMO_OP_WIDTH-1:0] attr0_op = attr0.amo.amo_op; + assign in_wide[k] = attr0.amo.amo_valid ? 2'b00 : attr0_op[1:0]; + `UNUSED_VAR (attr0_op) + end else begin : g_no_wide + assign in_wide[k] = 2'b00; + end `UNUSED_VAR (attr0) for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lanes - assign in_addr[k][i] = lsu_mem_if[k].req_data.addr[i][ADDR_WIDTH-1:0]; + wire [ADDR_WIDTH-1:0] lane_addr = lsu_mem_if[k].req_data.addr[i][ADDR_WIDTH-1:0]; + if (WIDE_ENABLE != 0) begin : g_wide_addr + assign in_addr[k][i] = {lane_addr[ADDR_WIDTH-1:2], + lane_addr[1] && ~in_wide[k][1], + lane_addr[0] && (in_wide[k] == 2'b00)}; + end else begin : g_addr + assign in_addr[k][i] = lane_addr; + end if (AMO_ENABLE != 0) begin : g_amo mem_bus_attr_t lane_attr; assign lane_attr = mem_bus_attr_t'(lsu_mem_if[k].req_data.user[i]); @@ -177,6 +200,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0] a_rsp_mask; wire [NUM_LANES-1:0][WORD_WIDTH-1:0] a_rsp_data; wire [TAG_WIDTH-1:0] a_rsp_tag; + wire [`UP(RSP_SIDE_WIDTH)-1:0] a_rsp_side; wire [NUM_LANES-1:0] a_perf_served; wire a_perf_store; @@ -194,6 +218,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0] b_rsp_mask; wire [NUM_LANES-1:0][WORD_WIDTH-1:0] b_rsp_data; wire [TAG_WIDTH-1:0] b_rsp_tag; + wire [`UP(RSP_SIDE_WIDTH)-1:0] b_rsp_side; wire [NUM_LANES-1:0] b_perf_served; wire b_perf_replay; @@ -311,7 +336,8 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .PAYLOAD_WIDTH (PAYLOAD_WIDTH), .ATTR_WIDTH (ATTR_WIDTH), .NUM_BLOCKS (NUM_BLOCKS), - .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE) + .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE), + .WIDE_ENABLE (WIDE_ENABLE) ) engine_a ( .clk (clk), .reset (reset), @@ -325,6 +351,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .req_addr (in_addr[a_req_block]), .req_payload (in_payload[a_req_block]), .req_tag (in_tag[a_req_block]), + .req_wide (in_wide[a_req_block]), .req_ready (a_req_ready), .busy (a_busy), .holds_write (a_holds_write), @@ -342,6 +369,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .rsp_mask (a_rsp_mask), .rsp_data (a_rsp_data), .rsp_tag (a_rsp_tag), + .rsp_side (a_rsp_side), .rsp_take (a_rsp_take), .rsp_done (a_rsp_done), .perf_served (a_perf_served), @@ -371,7 +399,8 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .PAYLOAD_WIDTH (1), .ATTR_WIDTH (ATTR_WIDTH), .NUM_BLOCKS (NUM_BLOCKS), - .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE) + .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE), + .WIDE_ENABLE (WIDE_ENABLE) ) engine_b ( .clk (clk), .reset (reset), @@ -385,6 +414,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .req_addr (in_addr[b_req_block]), .req_payload (b_req_payload), .req_tag (in_tag[b_req_block]), + .req_wide (in_wide[b_req_block]), .req_ready (b_req_ready), .busy (b_busy), .holds_write (b_holds_write), @@ -402,6 +432,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .rsp_mask (b_rsp_mask), .rsp_data (b_rsp_data), .rsp_tag (b_rsp_tag), + .rsp_side (b_rsp_side), .rsp_take (b_rsp_take), .rsp_done (b_rsp_done), .perf_served (b_perf_served), @@ -420,6 +451,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( assign b_rsp_mask = '0; assign b_rsp_data = '0; assign b_rsp_tag = '0; + assign b_rsp_side = '0; assign b_perf_served = '0; assign b_perf_replay = 1'b0; `UNUSED_VAR (b_bank_rdata) @@ -448,6 +480,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( reg [NUM_LANES-1:0] rsp_mask_r; reg [NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data_r; reg [TAG_WIDTH-1:0] rsp_tag_r; + reg [`UP(RSP_SIDE_WIDTH)-1:0] rsp_side_r; wire rsp_fire = rsp_valid_r && lsu_mem_if[k].rsp_ready; wire rsp_free = ~rsp_valid_r || lsu_mem_if[k].rsp_ready; @@ -470,6 +503,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( rsp_mask_r <= a_hit ? a_rsp_mask : b_rsp_mask; rsp_data_r <= a_hit ? a_rsp_data : b_rsp_data; rsp_tag_r <= a_hit ? a_rsp_tag : b_rsp_tag; + rsp_side_r <= a_hit ? a_rsp_side : b_rsp_side; end end @@ -477,6 +511,12 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( assign lsu_mem_if[k].rsp_data.mask = rsp_mask_r; assign lsu_mem_if[k].rsp_data.data = rsp_data_r; assign lsu_mem_if[k].rsp_data.tag = rsp_tag_r; + if (WIDE_ENABLE != 0) begin : g_rsp_side + assign lsu_mem_if[k].rsp_data.side = rsp_side_r; + end else begin : g_no_rsp_side + assign lsu_mem_if[k].rsp_data.side = '0; + `UNUSED_VAR (rsp_side_r) + end end `POP_COUNT(a_rsp_done, a_done); diff --git a/hw/rtl/mem/VX_lsu_adapter.sv b/hw/rtl/mem/VX_lsu_adapter.sv index 4b64b82b00..5504d31131 100644 --- a/hw/rtl/mem/VX_lsu_adapter.sv +++ b/hw/rtl/mem/VX_lsu_adapter.sv @@ -117,5 +117,6 @@ module VX_lsu_adapter import VX_gpu_pkg::*; #( .tag_out (lsu_mem_if.rsp_data.tag), .ready_out (lsu_mem_if.rsp_ready) ); + assign lsu_mem_if.rsp_data.side = '0; endmodule diff --git a/hw/rtl/mem/VX_lsu_mem_arb.sv b/hw/rtl/mem/VX_lsu_mem_arb.sv index 90df8a233a..3b0817a7c3 100644 --- a/hw/rtl/mem/VX_lsu_mem_arb.sv +++ b/hw/rtl/mem/VX_lsu_mem_arb.sv @@ -164,6 +164,7 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( bus_out_if[i].rsp_data.data, rsp_tag_out }; + `UNUSED_VAR (bus_out_if[i].rsp_data.side) assign bus_out_if[i].rsp_ready = rsp_ready_in[i]; end @@ -195,8 +196,13 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( TAG_WIDTH'(bus_out_if[i].rsp_data.tag) }; `else - assign rsp_data_in[i] = bus_out_if[i].rsp_data; + assign rsp_data_in[i] = { + bus_out_if[i].rsp_data.mask, + bus_out_if[i].rsp_data.data, + bus_out_if[i].rsp_data.tag + }; `endif + `UNUSED_VAR (bus_out_if[i].rsp_data.side) assign bus_out_if[i].rsp_ready = rsp_ready_in[i]; end @@ -222,7 +228,8 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_output assign bus_in_if[i].rsp_valid = rsp_valid_out[i]; - assign bus_in_if[i].rsp_data = rsp_data_out[i]; + assign {bus_in_if[i].rsp_data.mask, bus_in_if[i].rsp_data.data, bus_in_if[i].rsp_data.tag} = rsp_data_out[i]; + assign bus_in_if[i].rsp_data.side = '0; assign rsp_ready_out[i] = bus_in_if[i].rsp_ready; end diff --git a/hw/rtl/mem/VX_lsu_mem_if.sv b/hw/rtl/mem/VX_lsu_mem_if.sv index 31af0acb68..f335b4b8a6 100644 --- a/hw/rtl/mem/VX_lsu_mem_if.sv +++ b/hw/rtl/mem/VX_lsu_mem_if.sv @@ -18,7 +18,9 @@ interface VX_lsu_mem_if import VX_gpu_pkg::*; #( parameter DATA_SIZE = 1, parameter TAG_WIDTH = 1, parameter USER_WIDTH = MEM_ATTR_WIDTH, - parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE) + parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE), + // response sideband width; zero keeps a single bit that endpoints tie off + parameter RSP_SIDE_WIDTH = 0 ) (); typedef struct packed { @@ -37,6 +39,7 @@ interface VX_lsu_mem_if import VX_gpu_pkg::*; #( } req_data_t; typedef struct packed { + logic [`UP(RSP_SIDE_WIDTH)-1:0] side; logic [NUM_LANES-1:0] mask; logic [NUM_LANES-1:0][DATA_SIZE*8-1:0] data; tag_t tag; diff --git a/hw/rtl/om/VX_om_mem.sv b/hw/rtl/om/VX_om_mem.sv index f2c5e90097..a3f56c89cd 100644 --- a/hw/rtl/om/VX_om_mem.sv +++ b/hw/rtl/om/VX_om_mem.sv @@ -283,6 +283,7 @@ module VX_om_mem import VX_gpu_pkg::*; import VX_om_pkg::*; #( .core_rsp_tag (mrsp_tag), `UNUSED_PIN (core_rsp_sop), `UNUSED_PIN (core_rsp_eop), + `UNUSED_PIN (core_rsp_side), .core_rsp_ready (mrsp_ready), // Memory request @@ -301,6 +302,8 @@ module VX_om_mem import VX_gpu_pkg::*; import VX_om_pkg::*; #( .mem_rsp_mask (lsu_mem_if.rsp_data.mask), .mem_rsp_data (lsu_mem_if.rsp_data.data), .mem_rsp_tag (lsu_mem_if.rsp_data.tag), + .mem_rsp_hold (1'b0), + .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_if.rsp_ready) ); diff --git a/hw/rtl/raster/VX_raster_earlyz.sv b/hw/rtl/raster/VX_raster_earlyz.sv index c95061aa05..fe472bab08 100644 --- a/hw/rtl/raster/VX_raster_earlyz.sv +++ b/hw/rtl/raster/VX_raster_earlyz.sv @@ -336,6 +336,7 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX `UNUSED_PIN (core_rsp_tag), `UNUSED_PIN (core_rsp_sop), `UNUSED_PIN (core_rsp_eop), + `UNUSED_PIN (core_rsp_side), .core_rsp_ready (mrsp_ready), // Memory request @@ -354,6 +355,8 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX .mem_rsp_mask (lsu_mem_if.rsp_data.mask), .mem_rsp_data (lsu_mem_if.rsp_data.data), .mem_rsp_tag (lsu_mem_if.rsp_data.tag), + .mem_rsp_hold (1'b0), + .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_if.rsp_ready) ); diff --git a/hw/rtl/raster/VX_raster_mem.sv b/hw/rtl/raster/VX_raster_mem.sv index b677ce96d5..df67986e36 100644 --- a/hw/rtl/raster/VX_raster_mem.sv +++ b/hw/rtl/raster/VX_raster_mem.sv @@ -385,6 +385,7 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( .core_rsp_tag (sched_rsp_tag), `UNUSED_PIN (core_rsp_sop), `UNUSED_PIN (core_rsp_eop), + `UNUSED_PIN (core_rsp_side), .core_rsp_ready (mem_rsp_ready), // Memory request @@ -403,6 +404,8 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( .mem_rsp_mask (lsu_mem_if.rsp_data.mask), .mem_rsp_data (lsu_mem_if.rsp_data.data), .mem_rsp_tag (lsu_mem_if.rsp_data.tag), + .mem_rsp_hold (1'b0), + .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_if.rsp_ready) ); diff --git a/hw/rtl/tcu/VX_tcu_agu.sv b/hw/rtl/tcu/VX_tcu_agu.sv index 315917365f..c5fb6dfe9f 100644 --- a/hw/rtl/tcu/VX_tcu_agu.sv +++ b/hw/rtl/tcu/VX_tcu_agu.sv @@ -136,6 +136,7 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( assign client_if.req_data = req_data; assign client_if.rsp_ready = 1'b1; `UNUSED_VAR (client_if.rsp_data.sop) + `UNUSED_VAR (client_if.rsp_data.side) `UNUSED_VAR (client_if.rsp_data.tag) `RUNTIME_ASSERT(~client_if.rsp_valid || (issued_r && ~done_r), diff --git a/hw/rtl/tex/VX_tex_mem.sv b/hw/rtl/tex/VX_tex_mem.sv index 597c45cd95..26f1464b73 100644 --- a/hw/rtl/tex/VX_tex_mem.sv +++ b/hw/rtl/tex/VX_tex_mem.sv @@ -171,6 +171,7 @@ module VX_tex_mem import VX_gpu_pkg::*; import VX_tex_pkg::*; #( .core_rsp_tag (mem_rsp_tag), `UNUSED_PIN (core_rsp_sop), `UNUSED_PIN (core_rsp_eop), + `UNUSED_PIN (core_rsp_side), .core_rsp_ready (mem_rsp_ready), // Memory request @@ -189,6 +190,8 @@ module VX_tex_mem import VX_gpu_pkg::*; import VX_tex_pkg::*; #( .mem_rsp_mask (lsu_mem_if.rsp_data.mask), .mem_rsp_data (lsu_mem_if.rsp_data.data), .mem_rsp_tag (lsu_mem_if.rsp_data.tag), + .mem_rsp_hold (1'b0), + .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_if.rsp_ready) ); diff --git a/hw/unittest/issue/VX_issue_top.sv b/hw/unittest/issue/VX_issue_top.sv index 01de117a5e..eb78dccd84 100644 --- a/hw/unittest/issue/VX_issue_top.sv +++ b/hw/unittest/issue/VX_issue_top.sv @@ -111,6 +111,7 @@ module VX_issue_top import VX_gpu_pkg::*; #( assign writeback_if[i].data.uuid = writeback_uuid[i]; assign writeback_if[i].data.wis = writeback_wis[i]; assign writeback_if[i].data.eop_wis = PER_ISSUE_WARPS'(writeback_eop[i]) << writeback_wis[i]; + assign writeback_if[i].data.rel_wis = '0; assign writeback_if[i].data.sid = writeback_sid[i]; assign writeback_if[i].data.tmask = writeback_tmask[i]; assign writeback_if[i].data.PC = writeback_PC[i]; diff --git a/hw/unittest/lmem_unit/VX_lmem_unit_top.sv b/hw/unittest/lmem_unit/VX_lmem_unit_top.sv index 0e6dabe6f2..e75c0c0a49 100644 --- a/hw/unittest/lmem_unit/VX_lmem_unit_top.sv +++ b/hw/unittest/lmem_unit/VX_lmem_unit_top.sv @@ -52,6 +52,7 @@ module VX_lmem_unit_top import VX_gpu_pkg::*; #( output wire [NUM_BLOCKS-1:0][NUM_LANES-1:0] rsp_mask, output wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data, output wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] rsp_tag, + output wire [NUM_BLOCKS-1:0][`UP(LSU_RSP_SIDE_W)-1:0] rsp_side, input wire [NUM_BLOCKS-1:0] rsp_ready, // DMA requests @@ -73,9 +74,10 @@ module VX_lmem_unit_top import VX_gpu_pkg::*; #( localparam DMA_REQ_DATAW = 1 + DMA_ADDR_WIDTH + DMA_DATA_SIZE * 8 + DMA_DATA_SIZE + MEM_ATTR_WIDTH + DMA_TAG_WIDTH; VX_lsu_mem_if #( - .NUM_LANES (NUM_LANES), - .DATA_SIZE (WORD_SIZE), - .TAG_WIDTH (TAG_WIDTH) + .NUM_LANES (NUM_LANES), + .DATA_SIZE (WORD_SIZE), + .TAG_WIDTH (TAG_WIDTH), + .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) ) lsu_mem_if[NUM_BLOCKS](); VX_mem_bus_if #( @@ -108,6 +110,7 @@ module VX_lmem_unit_top import VX_gpu_pkg::*; #( assign rsp_mask[k] = lsu_mem_if[k].rsp_data.mask; assign rsp_data[k] = lsu_mem_if[k].rsp_data.data; assign rsp_tag[k] = lsu_mem_if[k].rsp_data.tag; + assign rsp_side[k] = lsu_mem_if[k].rsp_data.side; assign lsu_mem_if[k].rsp_ready = rsp_ready[k]; end @@ -141,7 +144,8 @@ module VX_lmem_unit_top import VX_gpu_pkg::*; #( .TAG_WIDTH (TAG_WIDTH), .DMA_ENABLE (1), .DMA_TAG_WIDTH (DMA_TAG_WIDTH), - .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) + .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), + .WIDE_ENABLE (LDS_WIDE_ENABLE) ) lmem_unit ( .clk (clk), .reset (reset), diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index 12d186a47d..c5a435cbaa 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -47,9 +47,10 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( ); `ifdef VX_CFG_LMEM_ENABLE VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH), + .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); `endif diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 741521a4dc..62fdb24f14 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -223,9 +223,10 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( wire [NUM_LSU_BLKS-1:0] block_store_empty; VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH), + .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); VX_mem_bus_if #( diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 7479a64a02..9cd1d0c422 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -632,6 +632,9 @@ class Core::Impl { trace->op_type = instr->get_op_type(); trace->dst_reg = instr->get_dest_reg(); trace->dst_bytesel = instr->get_dst_bytesel(); + if (auto lsu_args = std::get_if(&instr->get_args())) { + trace->dst_span = lsu_args->nregs; + } for (uint32_t i = 0; i < NUM_SRC_REGS; ++i) { trace->src_regs[i] = instr->get_src_reg(i); } diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index c9805efef2..94adeaa7b3 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -267,6 +267,9 @@ static op_string_t op_string(const Instr &instr) { case LsuType::LOAD: { auto lsuArgs = std::get(instrArgs); bool dst_float = (instr.get_dest_reg().type == RegType::Float); + if (lsuArgs.nregs > 1) { + return {(lsuArgs.nregs == 4) ? "VX_FLDS.128" : "VX_FLDS.64", to_hex_str(lsuArgs.offset)}; + } if (lsuArgs.is_local) { static const char* const widths[] = {"B", "H", "W", "D", "BU", "HU", "WU"}; return {std::string(dst_float ? "VX_FLDS." : "VX_LDS.") + widths[lsuArgs.width], to_hex_str(lsuArgs.offset)}; @@ -1276,13 +1279,27 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_op_type(LsuType::STORE); instr->set_args(IntrLsuArgs{size, /*stride*/ 0, (int32_t)sext(imm9, 9), /*is_local*/ true}); } else { - if (is_float && (funct3 == 0 || funct3 > 3)) { + // vx_flds.128 (funct3 0) and vx_flds.64 (funct3 4) load words into an + // aligned quad or pair of FP registers; defined on XLEN 32 only + uint32_t nregs = 1; + if (is_float && (funct3 == 0 || funct3 == 4)) { + #if defined(VX_CFG_XLEN_32) && defined(VX_CFG_EXT_F_ENABLE) + nregs = (funct3 == 0) ? 4 : 2; + #else + std::abort(); + #endif + } else if (is_float && funct3 > 3) { + std::abort(); + } + if (rd % nregs != 0) { + std::cout << "Error: vx_flds." << (32 * nregs) << " destination f" << rd + << " is not aligned to " << nregs << " registers" << std::endl; std::abort(); } uint32_t imm11 = (code >> shift_rs2) & 0x7ff; instr->set_dest_reg(rd, is_float ? RegType::Float : RegType::Integer); instr->set_op_type(LsuType::LOAD); - instr->set_args(IntrLsuArgs{funct3, /*stride*/ 0, (int32_t)sext(imm11, 11), /*is_local*/ true}); + instr->set_args(IntrLsuArgs{(nregs > 1) ? 2u : funct3, /*stride*/ 0, (int32_t)sext(imm11, 11), /*is_local*/ true, nregs}); } } break; #endif diff --git a/sim/simx/instr_trace.h b/sim/simx/instr_trace.h index 63a275bba8..8db44adbe2 100644 --- a/sim/simx/instr_trace.h +++ b/sim/simx/instr_trace.h @@ -52,6 +52,14 @@ struct instr_trace_t { // Byte write enable uint8_t dst_bytesel; + // Registers from dst_reg up that the scoreboard reserves and releases: a + // wide local load reserves its quad or pair at issue, and each word-phase + // releases its own register. + uint32_t dst_span; + + // A wide local load's words 1..3, per thread, until their phase retires. + std::vector wide_data; + std::shared_ptr instr_ptr; int pid; @@ -90,6 +98,7 @@ struct instr_trace_t { , src_data(NUM_SRC_REGS, std::vector(VX_CFG_NUM_THREADS)) , dst_data(VX_CFG_NUM_THREADS) , dst_bytesel(0xFF) + , dst_span(1) , pid(-1) , sop(true) , eop(true) @@ -116,6 +125,8 @@ struct instr_trace_t { , src_data(rhs.src_data) , dst_data(rhs.dst_data) , dst_bytesel(rhs.dst_bytesel) + , dst_span(rhs.dst_span) + , wide_data(rhs.wide_data) , instr_ptr(rhs.instr_ptr) , pid(rhs.pid) , sop(rhs.sop) diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index e1db778b3a..8538c79669 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -161,6 +161,7 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { const bool is_cas = is_amo && (*amo_tag == AmoType::AMOCAS); uint32_t num_threads = VX_CFG_NUM_THREADS; uint32_t data_bytes; + uint32_t align_bytes; bool is_write; int64_t offset; uint32_t stride; @@ -168,12 +169,15 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { if (is_amo) { auto amo_args = std::get(trace->instr_ptr->get_args()); data_bytes = 1u << (amo_args.width & 0x3); + align_bytes = data_bytes; is_write = false; // AMO never writes via the regular store path offset = 0; // AMO has no immediate offset stride = 0; // single-word access — addr = rs1[t] } else { auto lsu_args = std::get(trace->instr_ptr->get_args()); data_bytes = 1u << (lsu_args.width & 0x3); + // a wide local load reads its words from one aligned group per lane + align_bytes = data_bytes * lsu_args.nregs; is_write = (lsu_tag && *lsu_tag == LsuType::STORE); offset = lsu_args.offset; stride = lsu_args.stride; @@ -224,9 +228,9 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { // The hardware LSU enforces the same contract, so modelling // anything looser here would let a shader pass in simulation and fail // on the device. - if ((e.addr % e.size) != 0) { + if ((e.addr % align_bytes) != 0) { std::cout << "Error: misaligned memory access: addr=0x" << std::hex - << e.addr << std::dec << ", size=" << e.size << std::endl; + << e.addr << std::dec << ", size=" << align_bytes << std::endl; std::abort(); } if (is_write || is_amo) { @@ -309,13 +313,18 @@ void LsuUnit::process_response_step(uint32_t b) { : (uint32_t)__builtin_ctz(data_bs); bool nan_box = dst_float && (data_bytes < 8); bool half_float = VX_CFG_EXT_ZFHMIN_ENABLED && dst_float && (entry.lsu_args.width == 1); + // a wide local load's phase k returns word k of each lane's group + uint32_t phase = lsu_rsp.phase; + if (phase != 0 && trace->wide_data.empty()) { + trace->wide_data.resize(3 * VX_CFG_NUM_THREADS, 0); + } for (uint32_t lane = 0; lane < lsu_rsp.mask.size(); ++lane) { if (!lsu_rsp.mask.test(lane)) continue; const auto& lane_info = entry.lanes.at(lane); assert(lsu_rsp.data.at(lane) && "LOAD response must carry line payload"); - uint32_t off = lane_info.addr & (VX_CFG_MEM_BLOCK_SIZE - 1); + uint32_t off = (lane_info.addr & (VX_CFG_MEM_BLOCK_SIZE - 1)) + phase * data_bytes; uint64_t read_data = 0; std::memcpy(&read_data, lsu_rsp.data.at(lane)->data() + off, data_bytes); // Format the loaded value at low bits per RISC-V load semantics. @@ -342,10 +351,14 @@ void LsuUnit::process_response_step(uint32_t b) { std::abort(); } // Place at bytesel-specified position; OR-in NaN-box for Float dest. - auto& dst = entry.trace->dst_data.at(lane_info.tid); uint32_t box_bits = half_float ? 16 : 32; - dst.u64 = (formatted << (8 * byte_off)) - | (nan_box ? nanbox::box(0, box_bits) : 0); + uint64_t value = (formatted << (8 * byte_off)) + | (nan_box ? nanbox::box(0, box_bits) : 0); + if (phase != 0) { + trace->wide_data.at(lane_info.tid * 3 + phase - 1) = value; + } else { + trace->dst_data.at(lane_info.tid).u64 = value; + } } } entry.count -= lsu_rsp.mask.count(); // track remaining @@ -353,7 +366,27 @@ void LsuUnit::process_response_step(uint32_t b) { if (is_final) { state.pending_reqs.release(lsu_rsp.tag); } - if (is_final && entry.eop) { + if (entry.is_load && entry.lsu_args.nregs > 1 && lsu_rsp.phase_last) { + // A wide load's word-phase is complete: its beat writes register + // fd + phase and releases it, so a consumer of fd need not wait for + // fd + 3. The last phase's beat retires the instruction. + instr_trace_t* beat = trace; + if (!is_final) { + auto beat_alloc = core_->trace_pool().allocate(1); + beat = new (beat_alloc) instr_trace_t(*trace); + beat->eop = false; + } + uint32_t phase = lsu_rsp.phase; + if (phase != 0) { + for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { + beat->dst_data.at(t).u64 = trace->wide_data.at(t * 3 + phase - 1); + } + } + beat->dst_reg.idx += phase; + beat->dst_span = 1; + beat->wide_data.clear(); + output.send(beat, 2); + } else if (is_final && entry.eop) { // the response buffer, then the result arbiter's register output.send(trace, 2); } else { @@ -634,8 +667,12 @@ void LsuUnit::process_request_step(uint32_t b) { // signed/unsigned into one op + flag. lsu_req.op = is_amo ? amo_to_memop(*amo_tag) : (is_write ? MemOp::ST : MemOp::LD); + uint32_t nregs = 1; if (is_amo) { lsu_req.flags.amo_unsigned = amo_is_unsigned(*amo_tag); + } else { + nregs = std::get(trace->instr_ptr->get_args()).nregs; + lsu_req.flags.nwords_m1 = nregs - 1; } uint32_t t0 = state.addr_list.size() - state.remain_addrs; uint32_t beat_n = std::min(VX_CFG_NUM_LSU_LANES, state.remain_addrs); @@ -672,6 +709,13 @@ void LsuUnit::process_request_step(uint32_t b) { uint32_t count = lsu_req.mask.count(); bool is_eop = (state.remain_addrs == 0); + // A wide load's phases release its registers, so all its lanes must + // ride one request; the dispatcher's lane split guarantees it. + if (nregs > 1 && !is_eop) { + std::abort(); + } + // every word-phase answers each lane once + uint32_t rsp_count = count * nregs; uint32_t tag = 0; if (!is_write || is_amo) { @@ -692,7 +736,7 @@ void LsuUnit::process_request_step(uint32_t b) { --state.staged_reads; state.head_staged = false; } - tag = state.pending_reqs.allocate({trace, count, is_eop, std::move(lane_entries), entry_args, true}); + tag = state.pending_reqs.allocate({trace, rsp_count, is_eop, std::move(lane_entries), entry_args, true}); } lsu_req.tag = tag; lsu_req.cid = trace->cid; @@ -707,7 +751,7 @@ void LsuUnit::process_request_step(uint32_t b) { core_->perf_stats().stores += count; } else { core_->perf_stats().loads += count; - pending_loads_ += count; + pending_loads_ += rsp_count; } } diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 337b574605..e568ab4a06 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -80,6 +80,7 @@ class LocalMem::Impl { // the next pass's bank winners are known: a pass that leaves lanes // pending is followed by a cycle that finds the next ones bool ready = false; + uint32_t phase = 0; // a wide load's word-phase in service std::deque rsp_queue; }; @@ -144,7 +145,9 @@ class LocalMem::Impl { // a plain read also serves every pending lane reading the same word. A // store or an atomic takes its bank alone, so lanes writing one word land // in lane order and the highest lane's bytes are the ones left. - BitVector<> select_pass(const LsuReq& req, const BitVector<>& pending) const { + // A wide load's word-phase reads at a fixed byte offset from each lane's + // aligned group, so its conflicts are those of the group addresses. + BitVector<> select_pass(const LsuReq& req, const BitVector<>& pending, uint32_t offset) const { uint32_t num_banks = 1u << config_.B; std::vector winner(num_banks, -1); BitVector<> served(pending.size()); @@ -153,11 +156,12 @@ class LocalMem::Impl { if (!pending.test(i)) { continue; } - uint32_t b = this->bank_of(req.addrs.at(i)); + uint64_t addr = req.addrs.at(i) + offset; + uint32_t b = this->bank_of(addr); if (winner.at(b) == -1) { winner.at(b) = i; served.set(i); - } else if (join && this->bank_word(req.addrs.at(i)) == this->bank_word(req.addrs.at(winner.at(b)))) { + } else if (join && this->bank_word(addr) == this->bank_word(req.addrs.at(winner.at(b)) + offset)) { served.set(i); } } @@ -313,11 +317,12 @@ class LocalMem::Impl { if (has_rsp && rsp_count >= RSP_QUEUE_SIZE) { return false; } - auto served = this->select_pass(req, e.pending); + auto served = this->select_pass(req, e.pending, e.phase * config_.word_size); LsuRsp rsp(config_.num_lanes); - rsp.tag = req.tag; - rsp.cid = req.cid; - rsp.uuid = req.uuid; + rsp.tag = req.tag; + rsp.cid = req.cid; + rsp.uuid = req.uuid; + rsp.phase = e.phase; for (uint32_t i = 0; i < config_.num_lanes; ++i) { if (!served.test(i)) { continue; @@ -329,7 +334,8 @@ class LocalMem::Impl { } e.pending.reset(i); } - DT(4, simobject_->name() << " pass: served=" << served << ", " << req); + rsp.phase_last = e.pending.none(); + DT(4, simobject_->name() << " pass: phase=" << e.phase << ", served=" << served << ", " << req); if (has_rsp) { e.rsp_queue.push_back({SimPlatform::instance().cycles() + RSP_LATENCY, e.s1.block, std::move(rsp)}); } @@ -339,6 +345,13 @@ class LocalMem::Impl { } #endif if (e.pending.none()) { + if (e.phase < req.flags.nwords_m1) { + // the next word-phase serves every lane again, from the next cycle + ++e.phase; + e.pending = req.mask; + return false; + } + e.phase = 0; e.s1_valid = false; return true; } @@ -459,6 +472,7 @@ class LocalMem::Impl { e.s0.clear(); e.s1_valid = false; e.pending.reset(); + e.phase = 0; e.rsp_queue.clear(); } for (auto& r : rsp_regs_) { diff --git a/sim/simx/scoreboard.cpp b/sim/simx/scoreboard.cpp index 7701aa15cb..6c76af3e2b 100644 --- a/sim/simx/scoreboard.cpp +++ b/sim/simx/scoreboard.cpp @@ -19,10 +19,14 @@ using namespace vortex; Scoreboard::Scoreboard(const SimContext& ctx, const char* name) : SimObject(ctx, name) , in_use_regs_(VX_CFG_NUM_WARPS) + , in_use_groups_(VX_CFG_NUM_WARPS) , in_use_fcsr_(VX_CFG_NUM_WARPS, 0) { for (auto& in_use_reg : in_use_regs_) { in_use_reg.resize((int)RegType::Count); } + for (auto& in_use_group : in_use_groups_) { + in_use_group.resize((int)RegType::Count); + } this->on_reset(); } @@ -35,10 +39,16 @@ void Scoreboard::on_reset() { mask.reset(); } } + for (auto& in_use_group : in_use_groups_) { + for (auto& mask : in_use_group) { + mask.reset(); + } + } std::fill(in_use_fcsr_.begin(), in_use_fcsr_.end(), 0); owners_.clear(); commit_counts_.clear(); pending_releases_.clear(); + pending_group_releases_.clear(); } void Scoreboard::on_tick() { @@ -53,7 +63,15 @@ void Scoreboard::on_tick() { } pending_releases_.pop_front(); } - if (pending_releases_.empty()) { + while (!pending_group_releases_.empty() && pending_group_releases_.front().due <= now) { + auto& r = pending_group_releases_.front(); + // a register reserved again since its release stays in use + if (!in_use_regs_.at(r.wid).at((int)r.reg.type).test(r.reg.idx)) { + in_use_groups_.at(r.wid).at((int)r.reg.type).reset(r.reg.idx); + } + pending_group_releases_.pop_front(); + } + if (pending_releases_.empty() && pending_group_releases_.empty()) { this->tick_sleep(); } } @@ -65,8 +83,11 @@ bool Scoreboard::in_use(instr_trace_t* trace) const { } if (trace->wb) { assert(trace->dst_reg.type != RegType::None); - if (in_use_regs_.at(trace->wid).at((int)trace->dst_reg.type).test(trace->dst_reg.idx)) { - return true; + auto& dst_in_use = (trace->dst_span > 1) ? in_use_groups_ : in_use_regs_; + for (uint32_t i = 0; i < trace->dst_span; ++i) { + if (dst_in_use.at(trace->wid).at((int)trace->dst_reg.type).test(trace->dst_reg.idx + i)) { + return true; + } } } for (uint32_t i = 0; i < trace->src_regs.size(); ++i) { @@ -83,10 +104,12 @@ std::vector Scoreboard::get_uses(instr_trace_t* trace) co std::vector out; if (trace->wb) { assert(trace->dst_reg.type != RegType::None); - if (in_use_regs_.at(trace->wid).at((int)trace->dst_reg.type).test(trace->dst_reg.idx)) { - uint32_t reg_id = get_reg_id(trace->dst_reg, trace->wid); - auto owner = owners_.at(reg_id); - out.push_back({trace->dst_reg.type, trace->dst_reg.idx, owner->fu_type, owner->op_type, owner->uuid}); + for (uint32_t i = 0; i < trace->dst_span; ++i) { + RegOpd reg{trace->dst_reg.type, trace->dst_reg.idx + i}; + if (in_use_regs_.at(trace->wid).at((int)reg.type).test(reg.idx)) { + auto owner = owners_.at(get_reg_id(reg, trace->wid)); + out.push_back({reg.type, reg.idx, owner->fu_type, owner->op_type, owner->uuid}); + } } } for (uint32_t i = 0; i < trace->src_regs.size(); ++i) { @@ -102,21 +125,29 @@ std::vector Scoreboard::get_uses(instr_trace_t* trace) co } void Scoreboard::reserve(instr_trace_t* trace) { - uint32_t reg_id = get_reg_id(trace->dst_reg, trace->wid); assert(trace->wb); - in_use_regs_.at(trace->wid).at((int)trace->dst_reg.type).set(trace->dst_reg.idx); - assert(owners_.count(reg_id) == 0); - owners_[reg_id] = trace; + for (uint32_t i = 0; i < trace->dst_span; ++i) { + RegOpd reg{trace->dst_reg.type, trace->dst_reg.idx + i}; + uint32_t reg_id = get_reg_id(reg, trace->wid); + in_use_regs_.at(trace->wid).at((int)reg.type).set(reg.idx); + in_use_groups_.at(trace->wid).at((int)reg.type).set(reg.idx); + assert(owners_.count(reg_id) == 0); + owners_[reg_id] = trace; + } } void Scoreboard::release(instr_trace_t* trace) { - uint32_t reg_id = get_reg_id(trace->dst_reg, trace->wid); assert(trace->wb); - assert(in_use_regs_.at(trace->wid).at((int)trace->dst_reg.type).test(trace->dst_reg.idx)); - assert(owners_.count(reg_id) != 0); - commit_counts_.erase(reg_id); - pending_releases_.push_back({trace->wid, trace->dst_reg, 0, - SimPlatform::instance().cycles() + kReleaseDelay}); + commit_counts_.erase(get_reg_id(trace->dst_reg, trace->wid)); + for (uint32_t i = 0; i < trace->dst_span; ++i) { + RegOpd reg{trace->dst_reg.type, trace->dst_reg.idx + i}; + assert(in_use_regs_.at(trace->wid).at((int)reg.type).test(reg.idx)); + assert(owners_.count(get_reg_id(reg, trace->wid)) != 0); + pending_releases_.push_back({trace->wid, reg, 0, + SimPlatform::instance().cycles() + kReleaseDelay}); + pending_group_releases_.push_back({trace->wid, reg, 0, + SimPlatform::instance().cycles() + kGroupReleaseDelay}); + } this->tick_wake(); } diff --git a/sim/simx/scoreboard.h b/sim/simx/scoreboard.h index d5514bc654..377a019e32 100644 --- a/sim/simx/scoreboard.h +++ b/sim/simx/scoreboard.h @@ -72,6 +72,10 @@ class Scoreboard : public SimObject { // A committed result is written back from the commit arbiter's output // register, and the issue stage reads a registered readiness. static constexpr uint32_t kReleaseDelay = 2; + // A wide load tests its destination group against the in-use state as + // registered at the start of the cycle, so it sees a release one cycle + // after other instructions do. + static constexpr uint32_t kGroupReleaseDelay = kReleaseDelay + 1; struct pending_release_t { uint32_t wid; @@ -81,8 +85,10 @@ class Scoreboard : public SimObject { }; std::vector> in_use_regs_; + std::vector> in_use_groups_; std::vector in_use_fcsr_; std::deque pending_releases_; + std::deque pending_group_releases_; std::unordered_map owners_; std::unordered_map commit_counts_; diff --git a/sim/simx/types.h b/sim/simx/types.h index 9beeb26df5..24d0b3689b 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -385,6 +385,7 @@ struct IntrLsuArgs { uint32_t stride : 4; // per-uop multiplier: 0 = regular LD/ST; 0..N-1 = packLD uop_idx int32_t offset; // immediate addr offset (signed); 0 for packLD uops bool is_local = false; // vx_lds/vx_sts: local memory at the low address bits + uint32_t nregs = 1; // vx_flds.64/.128: registers fd.. loaded one word-phase each }; // A fence carries no access width; its width field says what it orders. @@ -469,6 +470,7 @@ struct MemFlags { uint32_t dxa_notify_done : 1; // bit 7 uint32_t dxa_notify_bar_id : 16; // bits 8..23 #endif + uint32_t nwords_m1 : 2; // wide local load: words per lane - 1, served in word-phases }; }; @@ -1210,6 +1212,10 @@ struct LsuRsp { uint64_t tag; uint32_t cid; uint64_t uuid; + // Wide local load: the word-phase this pass served, and whether it was the + // phase's last pass. + uint32_t phase; + bool phase_last; // TLM data: per-lane block populated for loads/AMO returns. std::vector> data; @@ -1219,6 +1225,8 @@ struct LsuRsp { , tag (0) , cid(0) , uuid(0) + , phase(0) + , phase_last(true) , data(size) {} diff --git a/tests/regression/lds_ldg/common.h b/tests/regression/lds_ldg/common.h index 9d089a0a70..0933e9f46b 100644 --- a/tests/regression/lds_ldg/common.h +++ b/tests/regression/lds_ldg/common.h @@ -4,9 +4,15 @@ #include // Local memory per block: RO_WORDS read-only words, then SLOT_WORDS words per -// thread that it stores to. +// thread that it stores to, then, from a 16-byte boundary, WIDE_WORDS words per +// thread that it stores to and reads back with wide loads. #define RO_WORDS 1024 #define SLOT_WORDS 5 +#define WIDE_WORDS 4 + +static inline uint32_t wide_base(uint32_t block) { + return (RO_WORDS + block * SLOT_WORDS + 3) & ~3u; +} // Global input words read past each thread's index (the largest LDG offset). #define IN_PAD 8 @@ -28,6 +34,9 @@ static inline uint32_t slot_word(uint32_t tid, uint32_t k) { static inline uint32_t out_word(uint32_t tid, uint32_t k) { return 0x30000000u | (tid << 4) | k; } +static inline uint32_t wide_word(uint32_t tid, uint32_t k) { + return 0x6C000000u | (tid << 4) | k; +} typedef struct { uint32_t lmem_words; // local-memory words per block diff --git a/tests/regression/lds_ldg/kernel.cpp b/tests/regression/lds_ldg/kernel.cpp index 87dac5c237..600c40f445 100644 --- a/tests/regression/lds_ldg/kernel.cpp +++ b/tests/regression/lds_ldg/kernel.cpp @@ -4,6 +4,8 @@ // vx_lds/vx_sts (custom-3) and vx_ldg/vx_stg (custom-0), written as .insn so // the encodings are tested independently of the compiler. // vx_lds: I-type, funct3 = LOAD width, imm[11] = FP, imm[10:0] = offset +// vx_flds.128 / .64: vx_lds with imm[11] = 1 and funct3 = 0 / 4; fd is an +// aligned quad / pair, the base 16 / 8-byte aligned // vx_sts: S-type, funct3 = 7, imm[11] = FP, imm[10:9] = size, imm[8:0] = offset // vx_ldg: R-type, funct7 = 0b0001_F_HH, funct3 = LOAD width, rs2 = uimm5 offset // vx_stg: R-type, funct7 = 0b0010_F_HH, funct3 = size, rd = uimm5 offset @@ -66,6 +68,44 @@ inline void fsts_w(uintptr_t p, uint32_t bits) { asm volatile (".insn r 0x0b, 2, " #f7 ", x" #imm ", %0, %1" :: "r"(p), "f"(_f) : "memory"); \ } while (0) +// vx_flds is defined on XLEN 32 with the F extension. +#if (__riscv_xlen == 32) && defined(__riscv_f) +#define HAS_FLDS 1 +#endif + +// A wide load and the reads of its registers sit in one asm block, so the +// compiler cannot reuse them in between. fd + 3 (fd + 1) is read first: it is +// the last phase's register. +#define VX_FLDS128(r0, r1, r2, r3, off, p, w) \ + asm volatile (".insn i 0x7b, 0, " r0 ", %4, %5\n\t" \ + "fmv.x.w %3, " r3 "\n\t" \ + "fmv.x.w %0, " r0 "\n\t" \ + "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %2, " r2 \ + : "=&r"((w)[0]), "=&r"((w)[1]), "=&r"((w)[2]), "=&r"((w)[3]) \ + : "r"(p), "i"(((off) & 0x7ff) - 2048) \ + : r0, r1, r2, r3, "memory") + +#define VX_FLDS64(r0, r1, off, p, w) \ + asm volatile (".insn i 0x7b, 4, " r0 ", %2, %3\n\t" \ + "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %0, " r0 \ + : "=&r"((w)[0]), "=&r"((w)[1]) \ + : "r"(p), "i"(((off) & 0x7ff) - 2048) \ + : r0, r1, "memory") + +#ifdef HAS_FLDS +static inline uint32_t ro_misses(const uint32_t* w, uint32_t n, uint32_t first) { + uint32_t misses = 0; + for (uint32_t k = 0; k < n; ++k) { + if (w[k] != ro_word(first + k)) { + ++misses; + } + } + return misses; +} +#endif + static inline uint32_t sext8(uint32_t v) { return (uint32_t)(int32_t)(int8_t)v; } static inline uint32_t sext16(uint32_t v) { return (uint32_t)(int32_t)(int16_t)v; } @@ -77,6 +117,13 @@ static inline uint32_t sext16(uint32_t v) { return (uint32_t)(int32_t)(int16_t)v // vx_sts - words at the offset limits, half-words through the bare // offset, bytes, an FP word; each read back by a generic load // ordering - a generic store, then a vx_lds of the same word +// vx_flds.128/.64 - a warp-uniform group (broadcast), a 16-byte lane stride +// (every lane on the same banks), a lane permutation, half the +// lanes under a divergent branch; fd = f0 and f28; two wide loads +// back to back; scalar writes to fd + 2 just before and just +// after; a wide load, an FP op on fd + 2 that waits for it, +// and a wide load of the same group again; offsets -1024 and 1008 through the window address and +// the bare offset; a generic store, then a wide load of the words // vx_ldg - every width from offset 0 to the 31-byte limit, the FP form, // every hint // vx_stg - words, half-words, bytes up to the 31-byte limit, the FP form @@ -220,6 +267,123 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { VX_STG(0, 0x10, 19, o, out_word(tid, 4) >> 24); } +#ifdef HAS_FLDS + { + uint32_t w[8]; + const uint32_t gu = 384; + VX_FLDS128("f8", "f9", "f10", "f11", 0, window + gu * 4, w); + errors += ro_misses(w, 4, gu); + VX_FLDS64("f8", "f9", 8, bare + gu * 4, w); + errors += ro_misses(w, 2, gu + 2); + + // a warp's lanes stay 16 bytes apart, and gs + 7 stays in the read-only + // words for any block size + const uint32_t gs = 256 + (lid % 128) * 4; + VX_FLDS128("f12", "f13", "f14", "f15", 0, window + gs * 4, w); + errors += ro_misses(w, 4, gs); + VX_FLDS64("f14", "f15", 0, window + gs * 4, w); + errors += ro_misses(w, 2, gs); + + const uint32_t gp = 512 + ((lid * 7 + 3) % 64) * 4; + VX_FLDS128("f16", "f17", "f18", "f19", 0, bare + gp * 4, w); + errors += ro_misses(w, 4, gp); + VX_FLDS64("f16", "f17", 8, window + gp * 4, w); + errors += ro_misses(w, 2, gp + 2); + + if (lid & 1) { + VX_FLDS128("f20", "f21", "f22", "f23", 0, window + gs * 4, w); + errors += ro_misses(w, 4, gs); + VX_FLDS64("f22", "f23", 8, window + gp * 4, w); + errors += ro_misses(w, 2, gp + 2); + } + + VX_FLDS128("f0", "f1", "f2", "f3", 0, window + gp * 4, w); + errors += ro_misses(w, 4, gp); + VX_FLDS128("f28", "f29", "f30", "f31", 0, bare + gs * 4, w); + errors += ro_misses(w, 4, gs); + VX_FLDS64("f30", "f31", 0, window + gu * 4, w); + errors += ro_misses(w, 2, gu); + + asm volatile (".insn i 0x7b, 0, f8, %8, %9\n\t" + ".insn i 0x7b, 0, f12, %8, %10\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11\n\t" + "fmv.x.w %4, f12\n\t" + "fmv.x.w %5, f13\n\t" + "fmv.x.w %6, f14\n\t" + "fmv.x.w %7, f15" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]), + "=&r"(w[4]), "=&r"(w[5]), "=&r"(w[6]), "=&r"(w[7]) + : "r"(window + gs * 4), "i"(-2048), "i"((16 & 0x7ff) - 2048) + : "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "memory"); + errors += ro_misses(w, 4, gs); + errors += ro_misses(w + 4, 4, gs + 4); + + const uint32_t marker = 0x7F00D000u | lid; + asm volatile ("fmv.w.x f10, %1\n\t" + ".insn i 0x7b, 0, f8, %2, %3\n\t" + "fmv.x.w %0, f10" + : "=&r"(w[2]) + : "r"(marker), "r"(window + gp * 4), "i"(-2048) + : "f8", "f9", "f10", "f11", "memory"); + if (w[2] != ro_word(gp + 2)) { + ++errors; + } + asm volatile (".insn i 0x7b, 0, f8, %4, %5\n\t" + "fmv.w.x f10, %6\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) + : "r"(window + gp * 4), "i"(-2048), "r"(marker) + : "f8", "f9", "f10", "f11", "memory"); + if (w[0] != ro_word(gp) || w[1] != ro_word(gp + 1) || w[2] != marker || w[3] != ro_word(gp + 3)) { + ++errors; + } + asm volatile (".insn i 0x7b, 0, f8, %4, %5\n\t" + "fadd.s f10, f10, f10\n\t" + ".insn i 0x7b, 0, f8, %4, %6\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) + : "r"(window + gs * 4), "i"(-2048), "i"((16 & 0x7ff) - 2048) + : "f8", "f9", "f10", "f11", "memory"); + errors += ro_misses(w, 4, gs + 4); + + const uint32_t go = 512; + VX_FLDS128("f4", "f5", "f6", "f7", -1024, window + go * 4 + 1024, w); + errors += ro_misses(w, 4, go); + VX_FLDS128("f4", "f5", "f6", "f7", 1008, window + go * 4 - 1008, w); + errors += ro_misses(w, 4, go); + VX_FLDS128("f24", "f25", "f26", "f27", -1024, bare + gs * 4 + 1024, w); + errors += ro_misses(w, 4, gs); + VX_FLDS128("f24", "f25", "f26", "f27", 1008, bare + gs * 4 - 1008, w); + errors += ro_misses(w, 4, gs); + VX_FLDS64("f26", "f27", -1024, window + gp * 4 + 1024, w); + errors += ro_misses(w, 2, gp); + + const uint32_t wi = wide_base(bsize) + lid * WIDE_WORDS; + for (uint32_t k = 0; k < WIDE_WORDS; ++k) { + lmem[wi + k] = wide_word(tid, k); + } + VX_FLDS128("f16", "f17", "f18", "f19", 0, window + wi * 4, w); + VX_FLDS64("f20", "f21", 8, bare + wi * 4, w + 4); + for (uint32_t k = 0; k < WIDE_WORDS; ++k) { + if (w[k] != wide_word(tid, k)) { + ++errors; + } + } + if (w[4] != wide_word(tid, 2) || w[5] != wide_word(tid, 3)) { + ++errors; + } + } +#endif + err[tid] = errors; __syncthreads(); diff --git a/tests/regression/lds_ldg/main.cpp b/tests/regression/lds_ldg/main.cpp index 7735266ed4..597bade802 100644 --- a/tests/regression/lds_ldg/main.cpp +++ b/tests/regression/lds_ldg/main.cpp @@ -1,6 +1,6 @@ -// lds_ldg — the explicit local (vx_lds/vx_sts) and global (vx_ldg/vx_stg) -// loads and stores return and store what the generic path does, at every -// width and at the limits of their offsets. +// lds_ldg — the explicit local (vx_lds/vx_sts, vx_flds.64/.128) and global +// (vx_ldg/vx_stg) loads and stores return and store what the generic path +// does, at every width and at the limits of their offsets. #include #include "common.h" @@ -48,14 +48,20 @@ int main(int argc, char** argv) { uint64_t nw = 0; uint64_t nc = 0; uint64_t lmem_size = 0; + uint64_t isa = 0; CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); CHECK(vx_device_query(dev, VX_CAPS_LOCAL_MEM_SIZE, &lmem_size)); + CHECK(vx_device_query(dev, VX_CAPS_ISA_FLAGS, &isa)); + // the kernel's wide loads (vx_flds) exist on XLEN 32 with F + const bool wide = (VX_ISA_ARCH(isa) == 32) && (isa & VX_ISA_STD_F); const uint32_t block = static_cast(nt * nw); const uint32_t grid = static_cast(nc * 2); const uint32_t num_threads = block * grid; - const uint32_t words = RO_WORDS + block * SLOT_WORDS; + const uint32_t slot_end = RO_WORDS + block * SLOT_WORDS; + const uint32_t wide_start = wide_base(block); + const uint32_t words = wide ? (wide_start + block * WIDE_WORDS) : slot_end; std::cout << "lds_ldg: block=" << block << " grid=" << grid << " words=" << words << std::endl; if (words * sizeof(uint32_t) > lmem_size) { std::cout << "local memory too small\nFAILED!" << std::endl; @@ -136,8 +142,12 @@ int main(int argc, char** argv) { } for (uint32_t b = 0; b < grid; ++b) { for (uint32_t i = 0; i < words; ++i) { + if (i >= slot_end && i < wide_start) { + continue; // alignment padding, never written + } uint32_t expected = (i < RO_WORDS) ? ro_word(i) - : slot_word(b * block + (i - RO_WORDS) / SLOT_WORDS, (i - RO_WORDS) % SLOT_WORDS); + : (i < slot_end) ? slot_word(b * block + (i - RO_WORDS) / SLOT_WORDS, (i - RO_WORDS) % SLOT_WORDS) + : wide_word(b * block + (i - wide_start) / WIDE_WORDS, (i - wide_start) % WIDE_WORDS); uint32_t actual = h_dump[b * words + i]; if (actual != expected) { if (errors < 16) { From 112a91d9928d8c7776b3274ffb389874c96f5501 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 17:21:54 -0800 Subject: [PATCH 315/336] beacon: document the area mapping's rewrites and the 14-DUT accuracy Beacon's area estimate now prices what Vivado builds: clock enables and synchronous resets on the flip-flop pins, bits a select passes through, register bits that never leave 0, rom_style tables as memories, associative tables as flip-flops, byte-masked block RAM per lane, and the vendor FP IP at its routed medians (foldsim ec18c67). Section 4.5 describes the rules; 7.2 replaces the four-DUT table with all 14 fpga_gate DUTs (mean LUT error 41 % -> 31 %, lutScale 0.337 -> 0.391) and names what is still off: logic Vivado trims across module boundaries, MUXF7/F8 packing, DSP inference on tex/gfx/om. Also: the gate ran Vivado 2024.2.2, not 2024.1, and lutScale is fitted on the U55C only (the V80 shares it; asap7 is unfitted). Landing order is unchanged: the foldsim prebuilt must carry ec18c67 before this branch is pushed. Co-Authored-By: Claude Opus 5.5 --- docs/designs/beacon.md | 144 ++++++++++++++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 29 deletions(-) diff --git a/docs/designs/beacon.md b/docs/designs/beacon.md index 90e51bf777..29d1448b04 100644 --- a/docs/designs/beacon.md +++ b/docs/designs/beacon.md @@ -294,7 +294,34 @@ candidates retiming is unlikely to absorb. ### 4.5 Area -Per op of the area mapping (§3.1): +The area mapping (§3.1) first takes out of the netlist what Vivado keeps +out of LUTs: + +- **Clock enables and synchronous resets** (`foldsim-register-enables`). A + select between a register's next value and the register itself is the + flip-flop's CE pin, and a select of a constant under it the R or S pin; + neither takes a LUT (Vivado's `VX_pipe_register`s with reset and enable + have none). The register becomes a `seq.compreg.ce`; its enable and reset + are one gate per register, not per bit. +- **Register bits that never leave 0** (`foldsim-constant-registers`). A + flip-flop powers up at 0, so a bit that only loads 0 or itself, and resets + to 0 if at all, is the constant 0, and the register narrows to its other + bits (`VX_cache_amo`'s write-back offset, of which only bit 5 varies, makes + its shift a select of two amounts). +- **Bits a select passes through** (`foldsim-narrow-muxes`). A procedural + write of part of a vector in a loop elaborates to a select per iteration + of the whole vector, whose arms agree on every bit outside that part; + lowered to and-inverter gates, each such bit stays a pair of gates and + every iteration's condition reaches it. `VX_cache_amo`'s read-forward + loop over the L2's 512-bit word (64 bytes of 4 write-back slots and the + settling entry) is a chain of 320 whole-word selects, every one reaching + every bit. Each select narrows to the runs where its arms differ. +- **Styled ROMs** (`foldsim-styled-roms`). A constant table on a wire with + `rom_style` `block` or `distributed` becomes a memory of its shape, placed + as below (`VX_mufu`'s table: one block RAM, as Vivado places it, instead + of 12,321 mapped LUTs of logic). + +Then, per op: | Element | FPGA | ASAP7 | |---|---|---| @@ -306,7 +333,7 @@ Per op of the area mapping (§3.1): | Multiply, wider | DSPs: the fewer of ⌈a/A⌉·⌈b/B⌉ in either orientation over the significant bits (§4.2) | partial products and a Dadda tree: 3·a·b cells | | Divide or remainder (output bits: at most the dividend's) | a table over ≤ 10 variable bits; a constant divisor, 2 LUTs per bit; else a subtracter per quotient bit, a LUT per result bit | the same, but 3 cells per subtracter bit | | Register | a flip-flop per bit | a flip-flop per bit | -| Vendor IP (`xil_*`) | its measured LUTs, flip-flops and DSPs | — | +| Vendor IP (`xil_*`) | its logic LUTs, flip-flops and DSPs: for `xil_fma`, `xil_fdiv` and `xil_fsqrt` the medians of their routed instances in the U55C builds (616, 743, 412 LUTs; 1,064, 1,360, 786 flip-flops) | — | | SRAM macro (black box) | — | a macro | **Memories** go where Vivado infers them. A `ram_style` the RTL asks for wins @@ -323,15 +350,41 @@ words and 2 Kb is block RAM. The shapes are the same on every FPGA family: | UltraRAM (`ultra`) | 4096 × 72 each | | `registers` | flip-flops and a read multiplexer | +Two shapes override the inference: + +- A memory distributed RAM cannot hold is **flip-flops**: one read of + latency 0 at each of its words, at constant addresses (an associative + lookup, as the MSHR's `addr_table`, `word_table` and `sector_table` are), + or more than seven read ports besides a read-write port. Its + constant-address reads are wires (in a copy folded over threads, a + multiplexer over the threads), every other read a multiplexer per bit, + and each word a write enable on its flip-flops' CE pins. The L2 MSHR goes from 629 to 348 distributed RAM LUTs + and from 167 to 567 flip-flops per bank, against Vivado's 352 and 579. +- A block RAM writes 8- or 9-bit bytes under a mask, so a memory masked in + **lanes that are not whole bytes** (neither a multiple of 8 nor of 9 bits) + is a block RAM per lane: the L2 tag store (64 × 136 in eight 17-bit + lanes) is eight 18 Kb halves, the I-cache's (64 × 80 in four 20-bit lanes) + four, as Vivado builds them. +- A memory without a write port is a **ROM**: in distributed form, logic + (a table over the address per bit and read, nothing for one word), not + distributed RAM, which is how Vivado reports it. A memory nothing reads + costs nothing. + On ASAP7, a memory without a macro is flip-flops, a write decoder and a read multiplexer tree. **The LUT scale.** The cut mapper has none of Vivado's LUT combining or -clock-enable and reset pins, and maps module by module, so it covers a -design with 2.1–3.5× the logic LUTs Vivado leaves. The target's `lutScale` converts the mapping's LUTs to -Vivado's: 0.337 on the LUT6 families, the geometric mean of Vivado's logic -LUTs over Beacon's on routed U55C builds (§7.2). Vendor IP and distributed -RAM, and the memories' read multiplexers, are not scaled; `lut-scale=1` gives the raw cover. +wide multiplexers (MUXF7/F8), and maps module by module, without the logic +Vivado trims across module boundaries, so it covers a design with 2.6× the +logic LUTs Vivado leaves at the geometric mean, from 1.4× (`lmem`) to 7.5× +(`rtu`) (§7.2). The target's `lutScale` +converts the mapping's LUTs to Vivado's: 0.391 on the LUT6 families, the +geometric mean over the 14 routed U55C `fpga_gate` DUTs of Vivado's logic +LUTs, less what Beacon prices outside the mapping (vendor IP, memory +multiplexers), over Beacon's mapped LUTs (§7.2). The V80 shares that value +without a fit of its own, and `asap7` has none (1.0): its cell counts are +not yet compared with Yosys. Vendor IP and distributed RAM, and the +memories' read multiplexers, are not scaled; `lut-scale=1` gives the raw cover. ## 5. Outputs @@ -642,29 +695,62 @@ So the model is a ranking for where to look first, not a gate. `diff` and ### 7.2 Area Against the routed `fpga_gate` builds of the same sources (U55C, Vivado -2024.1, 1 November 2026): +2024.2.2, 1 November 2026): -| DUT | LUT (Beacon / Vivado) | FF | distributed RAM LUTs | block RAM | DSP | -|---|---|---|---|---|---| -| core | 148,956 / 174,006 (0.86) | 166,850 / 146,008 (1.14) | 6,838 / 8,528 | 129 / 137 | 97 / 96 | -| cache | 37,466 / 31,039 (1.21) | 33,270 / 25,280 (1.32) | 3,784 / 2,722 | 32 / 32 | 0 / 0 | -| tcu | 122,262 / 117,087 (1.04) | 47,095 / 42,675 (1.10) | 2,083 / 1,360 | 0 / 0 | 769 / 768 | -| tensor | 264,677 / 233,153 (1.14) | 160,881 / 138,986 (1.16) | 12,806 / 7,844 | 114.5 / 133 | 267 / 265 | - -- **DSPs** are within two per DUT. Vivado decides some narrow products - (8 × 9, say) by a heuristic of its own, between DSPs and LUTs; Beacon puts - every product of more than ten variable bits in DSPs. -- **Block RAM** is exact on cache, 6 % short on core and 14 % short on - tensor; which of tensor's memories Vivado puts in block RAM that Beacon - does not is not yet traced. -- **Flip-flops** run 10–32 % high. Vivado can move pipeline registers into - DSPs (`AREG`, `MREG`, `PREG`) and block RAM output registers, and shift - registers into SRLs, none of which Beacon models; how much each accounts - for is not yet measured. -- **LUTs** are a calibrated estimate, within −14 % to +21 % on these DUTs; - the scale is one number per family (§4.5), so a DUT whose logic packs - better or worse than the average moves by that much. `logic_gate` - compares a DUT with its own baseline, so the scale cancels there. +Logic LUTs are Vivado's logic LUTs (without distributed RAM and shift +registers) against Beacon's at the fitted scale; *before* is the estimator +before the rules of §4.5 above, at its earlier scale of 0.337 and IP costs. +`top` leaves out the command processor (`cp_core`) on both sides: CIRCT +elaborates it with LLHD signals Beacon does not read. + +| DUT | LUT ratio before | LUT (Beacon / Vivado) | FF | distributed RAM LUTs | block RAM | DSP | +|---|---|---|---|---|---|---| +| cache | 1.19 | 27,313 / 28,317 (0.96) | 34,922 / 25,280 (1.38) | 2,542 / 2,722 | 32 / 32 | 0 / 0 | +| core | 0.86 | 122,879 / 165,478 (0.74) | 158,903 / 146,008 (1.09) | 6,838 / 6,220 | 129 / 137 | 97 / 96 | +| dxa | 1.05 | 8,170 / 9,666 (0.85) | 9,215 / 7,132 (1.29) | 568 / 638 | 0 / 0 | 9 / 7 | +| gfx | 1.45 | 241,768 / 210,879 (1.15) | 379,837 / 236,308 (1.61) | 11,906 / 12,834 | 191.5 / 172.5 | 1,140 / 992 | +| lmem | 0.68 | 14,766 / 26,322 (0.56) | 14,361 / 13,623 (1.05) | 726 / 740 | 16 / 16 | 0 / 0 | +| mufu | 2.92 | 1,387 / 1,966 (0.71) | 1,806 / 1,105 (1.63) | 0 / 0 | 1 / 1 | 5 / 3 | +| om | 2.37 | 4,730 / 2,298 (2.06) | 4,456 / 3,496 (1.27) | 367 / 408 | 0 / 0 | 10 / 22 | +| raster | 1.00 | 5,044 / 7,981 (0.63) | 12,069 / 8,974 (1.34) | 496 / 606 | 8 / 8 | 14 / 14 | +| rtu | 2.01 | 99,325 / 63,721 (1.56) | 146,917 / 94,885 (1.55) | 1,904 / 1,840 | 16.5 / 18.5 | 140 / 140 | +| tcu | 1.04 | 121,388 / 115,727 (1.05) | 49,056 / 42,675 (1.15) | 915 / 1,360 | 0 / 0 | 769 / 768 | +| tensor | 1.12 | 220,251 / 225,309 (0.98) | 166,688 / 138,986 (1.20) | 8,962 / 7,842 | 116.5 / 133.5 | 267 / 265 | +| tex | 1.50 | 71,506 / 55,791 (1.28) | 96,949 / 34,582 (2.80) | 3,241 / 4,208 | 0 / 0 | 896 / 528 | +| top | 1.04 | 104,728 / 125,511 (0.83) | 157,573 / 144,522 (1.09) | 8,551 / 7,836 | 76 / 76 | 50 / 48 | +| vm | 1.55 | 109,070 / 116,552 (0.94) | 151,175 / 121,587 (1.24) | 10,276 / 10,366 | 168 / 172 | 0 / 0 | + +- **LUTs**: the geometric mean error over the 14 DUTs goes from 41 % to + 31 %, the range from 0.68–2.92 to 0.56–2.06. The L2 of `vm` came to + 105,071 LUTs before (311,783 mapped) and comes to 47,555 now (121,625 + mapped), against Vivado's 54,150; each bank's `VX_cache_amo` to 14,733 + before and 3,608 now, against 5,001. The scale is one number per family, + so a DUT whose logic Vivado trims or packs more or less than the average + moves by that much; `logic_gate` compares a DUT with its own baseline, so + the scale cancels there. The largest + residues are logic Vivado removes across module boundaries, which Beacon, + mapping module by module, keeps: `rtu`'s logic outside its 73 + floating-point cores is 19,536 LUTs and 18,088 flip-flops in Vivado against 53,970 LUTs + (138,032 mapped) and 68,357 flip-flops in Beacon; `tex` keeps 2.8× + Vivado's flip-flops; `om`'s `VX_om_dcr` and `VX_om_blend` come to 3,407 + LUTs against Vivado's 741. `lmem`, `raster` and `core` come out low. +- **Flip-flops** run 5–63 % high, and 2.8× on `tex`, for the same reason: + Vivado drops the bits a parent never reads or ties to a constant (the + cache DUT's stream buffers, 20,936 flip-flops in Beacon against 12,324), and moves pipeline registers into DSPs and block RAM + output registers, none of which a module-by-module count sees. +- **Distributed RAM** is within 15 % on 10 of the 13 DUTs that have any + (`raster` 0.82, `tex` 0.77, `tcu` 0.67). +- **Block RAM** is exact on 5 of the 10 DUTs that have any; the mask-lane + rule matches Vivado on every masked tag store. The RTU scheduler's + `window_store` (64 × 512 in eight 64-bit lanes) packs in 7.5 blocks, its + lanes being whole bytes, where Vivado builds a 36 Kb block per lane, 8. + `gfx` is 19 high: Beacon places four RTU scheduler memories without a + `ram_style` (`fbuf_ram`, `ray_store`, `xfres_ram`, `trires_ram`, 23.5 + blocks) in block RAM by its size and latency rule, and Vivado does not; + `core` and `tensor` are short by 8 and 17, not yet traced. +- **DSPs** are within two on 11 DUTs; `gfx` and `tex` are 148 and 368 high + and `om` 12 low, where Vivado chooses between DSPs and LUTs for narrow + products by a heuristic of its own. The repository's Vivado baselines (`ci/baselines/synthesis/xilinx/`) date from 5 October 2026 and predate later RTL changes, so they are not a From 508b00ffd33dc7a943b9e286b4a50075bf27bece Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 19:14:38 -0800 Subject: [PATCH 316/336] tcu: rebalance the TFR FEDP align, accumulate and multiply stages to 300 MHz The tcu DUT closed at 291.5 MHz (WNS -0.097 ns) on the U55C. Its worst paths ended in the FEDP's align stage, which in one cycle selected the maximum exponent, subtracted and saturated each lane's shift, then subtracted again to drive a 26-bit left shift for the sticky bit (Beacon: 15 levels); the aln->acc paths were next. - align: the sticky bit is "shift > trailing-zero count" of the registered magnitude, a count that resolves beside the exponent difference; the second subtraction, the left shifter and the 8-bit saturate go away (the shift's high bits flag an overshift). - acc: -(a+b) as ~(a+b-1), compressing the all-ones operand in a CSA first, so the negate is one adder with no increment behind it. - mul_f8: the reverse difference runs beside the forward one; the magnitude is a select, not a negate after the subtract. - mul_i4: the sign folds into the CSA (ones' complement plus a count of negated products). No stage is added and the latency is unchanged, so SimX's timing model stands. Moving the maximum-exponent select into stage 1 was tried and undone: it became stage 1's worst path, and with the DSP-input path the worst one left, Vivado's global retiming pulled pipe_mul/pipe_aln back into the product reduction (+40K flip-flops). Vivado (U55C, 2024.2.2): 291.5 -> 301.2 MHz (WNS +0.013 ns), LUT 117,087 -> 105,359, FF 42,675 -> 30,441 (retimed registers 18,720 -> 4,320). Beacon: pipe_mul 13 -> 12, pipe_aln 15 -> 9, pipe_acc 19 -> 15 levels, nothing deeper, LUT -2.0 %. The flip-flop count depends on the retimer: norm_round (25 levels, which Vivado fixes with ~2K registers) keeps it from chasing the pipe_fedp -> DSP input path; a change that makes that path the worst retimable one brings the growth back. Verified: FEDP bit-exact unit suites (run-tcu 19/19, run-tcu-dsp 10/10, run-tcu-unit 37/37, including the latency-5 FPGA variant); 48 TCU CI cases (sgemm_tcu, sgemm_tcu_wg, sgemm_tcu_sp on simx, every TFR rtlsim case, 5 model_parity); sgemm_tcu_sp on TFR rtlsim for fp16, tf32, int8, int4. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/tcu/tfr/VX_tcu_tfr_acc.sv | 27 ++++++++++---- hw/rtl/tcu/tfr/VX_tcu_tfr_align.sv | 56 +++++++++++++++++------------ hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv | 16 ++++++++- hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i4.sv | 17 ++++++--- 4 files changed, 80 insertions(+), 36 deletions(-) diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_acc.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_acc.sv index b42dfa5393..e9dfcfa2cf 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_acc.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_acc.sv @@ -77,19 +77,32 @@ module VX_tcu_tfr_acc import VX_tcu_pkg::*; #( `UNUSED_PIN (cout) ); - // -(a+b) = ~a + ~b + 2: cin=1 yields ~(a+b), the +1 completes the negate. - wire [WO-1:0] sig_neg_raw; + // -(a+b) = ~(a+b-1): compressing the all-ones (-1) operand in first keeps + // the negate to one adder, with no increment behind it. + wire [WO-1:0] dec_sum, dec_carry; + VX_csa_32 #( + .N (WO), + .WIDTH_O (WO) + ) dec_csa ( + .a (sum_vec), + .b (carry_vec), + .c ({WO{1'b1}}), + .sum (dec_sum), + .carry (dec_carry) + ); + + wire [WO-1:0] sig_dec; VX_ks_adder #( .N (WO), .BYPASS (`FORCE_BUILTIN_ADDER(WO)) ) neg_adder ( - .cin (1'b1), - .dataa (~sum_vec), - .datab (~carry_vec), - .sum (sig_neg_raw), + .cin (1'b0), + .dataa (dec_sum), + .datab (dec_carry), + .sum (sig_dec), `UNUSED_PIN (cout) ); - wire [WO-1:0] sig_neg = sig_neg_raw + WO'(1); + wire [WO-1:0] sig_neg = ~sig_dec; assign sign_out = sig_pos[WO-1]; assign sig_out = sign_out ? sig_neg : sig_pos; diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_align.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_align.sv index a62e5a3a12..3b36574d76 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_align.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_align.sv @@ -34,7 +34,7 @@ module VX_tcu_tfr_align import VX_tcu_pkg::*; #( input wire [N-1:0][WI-1:0] sigs_in, input wire is_int, - output logic [TCU_EXP_BITS-1:0] max_exp, + output wire [TCU_EXP_BITS-1:0] max_exp, output wire [N-1:0][WO-1:0] sigs_out, output wire [N-1:0] sticky_bits, output wire [N-1:0] fp_negs @@ -44,31 +44,30 @@ module VX_tcu_tfr_align import VX_tcu_pkg::*; #( localparam MAX_PRE_SHIFT = WI - 23; localparam SHIFT_MAG_W = (WI - 1) + MAX_PRE_SHIFT; + localparam SHIFT_W = `CLOG2(SHIFT_MAG_W + 1); + localparam TZ_W = `LOG2UP(SHIFT_MAG_W); - wire [TCU_EXP_BITS-1:0] or_red[N:0]; - wire [N-1:0][7:0] shift_amts; + `STATIC_ASSERT (TCU_EXP_BITS > SHIFT_W, ("invalid shift width: %0d", SHIFT_W)) - // Determine maximum exponent via OR-Reduction Tree - assign or_red[0] = {TCU_EXP_BITS{1'b0}}; - for (genvar i = 0; i < N; i++) begin : g_or_red - assign or_red[i+1] = or_red[i] | (sel_exp[i] ? exponents[i] : {TCU_EXP_BITS{1'b0}}); - end - assign max_exp = or_red[N]; + VX_onehot_mux #( + .DATAW (TCU_EXP_BITS), + .N (N) + ) max_exp_mux ( + .data_in (exponents), + .sel_in (sel_exp), + .data_out (max_exp) + ); // Per-lane right-shift distance from the maximum exponent (max_exp is the // largest input, so the difference is non-negative). - for (genvar i = 0; i < N; i++) begin : g_shift_amts - wire [TCU_EXP_BITS-1:0] shift_full = max_exp - exponents[i]; - if (TCU_EXP_BITS > 8) begin : g_sat - assign shift_amts[i] = (|shift_full[TCU_EXP_BITS-1:8]) ? 8'hFF : shift_full[7:0]; - end else begin : g_no_sat - assign shift_amts[i] = 8'(shift_full); - end + wire [N-1:0][TCU_EXP_BITS-1:0] shift_amts; + for (genvar i = 0; i < N; ++i) begin : g_shift_amts + assign shift_amts[i] = max_exp - exponents[i]; end - // Align significands and form the accumulator operands for (genvar i = 0; i < N; ++i) begin : g_align_lanes - wire [7:0] shift_amt = shift_amts[i]; + wire shift_hi = (| shift_amts[i][TCU_EXP_BITS-1:SHIFT_W]); + wire [SHIFT_W-1:0] shift_lo = shift_amts[i][SHIFT_W-1:0]; wire lane_en; if (i == N-1) begin : g_c_en @@ -90,13 +89,24 @@ module VX_tcu_tfr_align import VX_tcu_pkg::*; #( end // 3. Shift adjustment - wire is_overshift = (shift_amt >= 8'(SHIFT_MAG_W)); - wire [SHIFT_MAG_W-1:0] shift_res_full = mag_shifted >> shift_amt; + wire is_overshift = shift_hi || (shift_lo >= SHIFT_W'(SHIFT_MAG_W)); + wire [SHIFT_MAG_W-1:0] shift_res_full = mag_shifted >> shift_lo; wire [WI:0] adj_mag = is_overshift ? '0 : shift_res_full[WI:0]; - // 4. Sticky Calculation - wire [SHIFT_MAG_W-1:0] sticky_check_shift = mag_shifted << (8'(SHIFT_MAG_W) - shift_amt); - assign sticky_bits[i] = lane_en & (is_overshift ? (|mag_shifted) : (|sticky_check_shift)); + // 4. Sticky: a set bit shifts out iff the shift exceeds the trailing + // zero count, which comes straight from the registered magnitude and + // so resolves in parallel with the exponent difference. + wire [TZ_W-1:0] mag_tz; + wire mag_nz; + VX_lzc #( + .N (SHIFT_MAG_W), + .REVERSE (1) + ) mag_tzc ( + .data_in (mag_shifted), + .data_out (mag_tz), + .valid_out (mag_nz) + ); + assign sticky_bits[i] = lane_en && mag_nz && (shift_hi || (shift_lo > SHIFT_W'(mag_tz))); // 5. Operand formation: masked, negated (fp) or sign-extended (int). wire fp_neg = in_sign & lane_en & ~is_int; diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv index 1c8c3cbbc6..dedd90d8d7 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv @@ -314,8 +314,22 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; `UNUSED_PIN(cout) ); + // The reverse difference runs beside the forward one, so the magnitude + // is a select rather than a negate behind the subtract. + wire [24:0] sub_rev; + VX_ks_adder #( + .N(25), + .BYPASS(`FORCE_BUILTIN_ADDER(25)) + ) sub_rev_adder ( + .dataa(sig_min_ext), + .datab(~sig_max_ext), + .cin(1'b1), + .sum(sub_rev), + `UNUSED_PIN(cout) + ); + wire sub_neg = sub_raw[24]; - wire [24:0] sub_abs = sub_neg ? -sub_raw : sub_raw; + wire [24:0] sub_abs = sub_neg ? sub_rev : sub_raw; wire [24:0] sig_add_raw = do_sub ? sub_abs : add_raw; // Scaling by 1 avoids renormalization. diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i4.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i4.sv index 14715ff4a4..738145e053 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i4.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i4.sv @@ -55,6 +55,7 @@ module VX_tcu_tfr_mul_i4 import VX_tcu_pkg::*; #( for (genvar i = 0; i < TCK; ++i) begin : g_lane wire signed [3:0][9:0] y_prod_i4; + wire [9:0] y_corr; // Shared per-element extraction. wire [3:0][3:0] raw_a, raw_b; @@ -127,11 +128,16 @@ module VX_tcu_tfr_mul_i4 import VX_tcu_pkg::*; #( .data_in (psign), .data_out (psign_r) ); + // -m = ~m + 1: the negated products enter the tree as ones' + // complements and their +1s as one count operand, so no negate + // sits between the DSP and the tree. + wire [3:0] pneg = psign_r & pvalid_r; for (genvar j = 0; j < 4; ++j) begin : g_sign - wire signed [9:0] pf = psign_r[j] ? -$signed({2'b0, magp[j]}) - : $signed({2'b0, magp[j]}); - assign y_prod_i4[j] = pf & {10{pvalid_r[j]}}; + wire [9:0] pm = {2'b0, magp[j]} & {10{pvalid_r[j]}}; + assign y_prod_i4[j] = pneg[j] ? ~pm : pm; end + wire [2:0] pneg_cnt = 3'(pneg[0]) + 3'(pneg[1]) + 3'(pneg[2]) + 3'(pneg[3]); + assign y_corr = 10'(pneg_cnt); end else begin : g_lut for (genvar j = 0; j < 4; ++j) begin : g_mul wire signed [4:0] s_a = is_signed_int ? $signed({raw_a[j][3], raw_a[j]}) : $signed({1'b0, raw_a[j]}); @@ -150,15 +156,16 @@ module VX_tcu_tfr_mul_i4 import VX_tcu_pkg::*; #( ); assign y_prod_i4[j] = prod_r & {10{pvalid_r[j]}}; end + assign y_corr = '0; end wire [9:0] y_i4_sum, y_i4_carry; VX_csa_tree #( - .N (4), + .N (5), .W (10), .S (10) ) i4_csa ( - .operands (y_prod_i4), + .operands ({y_corr, y_prod_i4}), .sum (y_i4_sum), .carry (y_i4_carry) ); From c486a1395cfbf86865344af263947b9132ff2377 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 19:41:31 -0800 Subject: [PATCH 317/336] core: take lane-index muxes and the in-use reduction off four paths to 300 MHz The core DUT closed at 291.8 MHz on the U55C (WNS -0.094 ns). The worst 100 paths were placement-bound (about 2.5 ns of net delay against 0.8 ns of logic, 5-13 levels), in four families: an encoded lane index with a fanout of 128-136 driving a wide multiplexer in front of a decision (the wctl exit test, the CSR write data, the SHFL/WGATHER source), and the scoreboard's 64-wide in-use reduction. - wctl: one VX_find_first select tree picks the last active lane's rs1/rs2 and per-lane mask-nonzero flags, so the exit test no longer waits on the lane index and its multiplexer. - csr: rs1_first comes from VX_find_first over the data instead of an index and a multiplexer. - alu_int: WGATHER reads the fallback lane's operands once for all lanes, each lane's own source being a select within its group of 4; SHFL reads its source lane before the bound check, which then only picks between that read and the lane's own value. - scoreboard: each operand reads its in-use bit by index and compares its index with the cycle's release, instead of reducing the operand masks against the whole released set (same result). No stage is added and cycle behaviour is unchanged. The memory moves behind the core's LUTRAM +154 % / BRAM -45 % against the Sep-9 baseline are not a cause: a Sep-26 build with the same placement reached 300.8 MHz. Vivado (U55C, 2024.2.2): 291.8 -> 302.1 MHz (WNS +0.023 ns), LUT 174,006 -> 170,472. Beacon: wctl sh_state 11 -> 7, its result buffer and sh_phase 12 -> 10, the barrier phase 12 -> 9, scoreboard 9 -> 7, CSR write data 8 -> 6; registers at 8+ levels 319 -> 278; LUT -3.0 %. Remaining near-critical: scoreboard fu_pending (+0.023), the LSU req_ibuf allocator (+0.025), and the LSU-empty -> SFU dispatch buffer enable (1145 fanout, not addressed). Verified: tests/regression on simx and rtlsim (no FAILED); 34 CI cases (config1 diverge/dogfood on rtlsim, gbar, async gbarrier, riscv isa-2, rvc isa, dogfood model_parity); diverge, dogfood, wsync on rtlsim with 2 SFU and 2 ALU lanes (wgather fails there with and without this change: WGATHER works on 4-lane groups). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_alu_int.sv | 53 +++++++++++++++++++++--------------- hw/rtl/core/VX_csr_unit.sv | 12 ++++---- hw/rtl/core/VX_scoreboard.sv | 20 +++++++++++--- hw/rtl/core/VX_wctl_unit.sv | 40 +++++++++++++++------------ 4 files changed, 76 insertions(+), 49 deletions(-) diff --git a/hw/rtl/core/VX_alu_int.sv b/hw/rtl/core/VX_alu_int.sv index 52b03a96b8..998a12d00b 100644 --- a/hw/rtl/core/VX_alu_int.sv +++ b/hw/rtl/core/VX_alu_int.sv @@ -164,14 +164,24 @@ module VX_alu_int import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] wgather_result; if (NUM_LANES > 1) begin : g_wgather wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] alu_in3 = execute_if.data.rs3_data; + // The fallback lane is shared by all lanes, so its operands are read + // once; each lane's own source is then a group-local select. + wire [`VX_CFG_XLEN-1:0] last_in1 = alu_in1[last_tid]; + wire [`VX_CFG_XLEN-1:0] last_in2 = alu_in2[last_tid]; + wire [`VX_CFG_XLEN-1:0] last_in3 = alu_in3[last_tid]; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_i - wire [LANE_BITS-1:0] nominal = (LANE_BITS'(i) & ~LANE_BITS'(3)) | LANE_BITS'(wg_src_offset); - wire [LANE_BITS-1:0] src_lane = execute_if.data.header.tmask[nominal] ? nominal : last_tid; - wire [1:0] offset = 2'(i) - wg_src_offset; // (i - src) mod 4 - assign wgather_result[i] = (offset == 2'd1) ? alu_in1[src_lane] - : (offset == 2'd2) ? alu_in2[src_lane] - : (offset == 2'd3) ? alu_in3[src_lane] - : `VX_CFG_XLEN'(0); // offset 0 = source lane (write-suppressed) + wire [LANE_BITS-1:0] nominal = (LANE_BITS'(i) & ~LANE_BITS'(3)) | LANE_BITS'(wg_src_offset); + wire [1:0] offset = 2'(i) - wg_src_offset; // (i - src) mod 4 + // offset 0 = source lane (write-suppressed) + wire [`VX_CFG_XLEN-1:0] nominal_val = (offset == 2'd1) ? alu_in1[nominal] + : (offset == 2'd2) ? alu_in2[nominal] + : (offset == 2'd3) ? alu_in3[nominal] + : `VX_CFG_XLEN'(0); + wire [`VX_CFG_XLEN-1:0] last_val = (offset == 2'd1) ? last_in1 + : (offset == 2'd2) ? last_in2 + : (offset == 2'd3) ? last_in3 + : `VX_CFG_XLEN'(0); + assign wgather_result[i] = execute_if.data.header.tmask[nominal] ? nominal_val : last_val; end end else begin : g_wgather_0 assign wgather_result[0] = alu_in1[0]; @@ -193,33 +203,32 @@ module VX_alu_int import VX_gpu_pkg::*; #( wire [LANE_BITS-1:0] lane_bfly = LANE_BITS'(i) ^ bval; wire [LANE_BITS-1:0] lane_idx = minLane | (bval & ~mask); + // The source lane is read before its bound check resolves, which + // only picks between that read and the lane's own value. reg [LANE_BITS-1:0] lane; + reg in_bounds; always @(*) begin - lane = LANE_BITS'(i); case (alu_op[1:0]) INST_SHFL_UP: begin - if ($signed(lane_up) >= $signed({1'b0, minLane})) begin - lane = lane_up[LANE_BITS-1:0]; - end + lane = lane_up[LANE_BITS-1:0]; + in_bounds = ($signed(lane_up) >= $signed({1'b0, minLane})); end INST_SHFL_DOWN: begin - if (lane_down <= {1'b0, maxLane}) begin - lane = lane_down[LANE_BITS-1:0]; - end + lane = lane_down[LANE_BITS-1:0]; + in_bounds = (lane_down <= {1'b0, maxLane}); end INST_SHFL_BFLY: begin - if (lane_bfly <= maxLane) begin - lane = lane_bfly; - end + lane = lane_bfly; + in_bounds = (lane_bfly <= maxLane); end - INST_SHFL_IDX: begin - if (lane_idx <= maxLane) begin - lane = lane_idx; - end + default: begin // INST_SHFL_IDX + lane = lane_idx; + in_bounds = (lane_idx <= maxLane); end endcase end - assign shfl_result[i] = execute_if.data.header.tmask[lane] ? alu_in1[lane] : alu_in1[i]; + wire take_lane = in_bounds && execute_if.data.header.tmask[lane]; + assign shfl_result[i] = take_lane ? alu_in1[lane] : alu_in1[i]; end end else begin : g_shfl_0 assign shfl_result[0] = alu_in1[0]; diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index b53116a8a7..139967ec63 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -77,17 +77,17 @@ module VX_csr_unit import VX_gpu_pkg::*; #( wire [`VX_CFG_XLEN-1:0] rs1_first; if (NUM_LANES > 1) begin : g_rs1_first - wire [`LOG2UP(NUM_LANES)-1:0] first_lane; - VX_priority_encoder #( + // the data's own select tree, not an encoded lane index driving a wide mux + VX_find_first #( .N (NUM_LANES), + .DATAW (`VX_CFG_XLEN), .REVERSE (0) ) first_lane_sel ( - .data_in (execute_if.data.header.tmask), - .index_out (first_lane), - `UNUSED_PIN (onehot_out), + .valid_in (execute_if.data.header.tmask), + .data_in (rs1_data), + .data_out (rs1_first), `UNUSED_PIN (valid_out) ); - assign rs1_first = rs1_data[first_lane]; end else begin : g_rs1_first_0 assign rs1_first = rs1_data[0]; end diff --git a/hw/rtl/core/VX_scoreboard.sv b/hw/rtl/core/VX_scoreboard.sv index 5ccbfee990..65e94e6ccd 100644 --- a/hw/rtl/core/VX_scoreboard.sv +++ b/hw/rtl/core/VX_scoreboard.sv @@ -195,8 +195,8 @@ module VX_scoreboard import VX_gpu_pkg::*; #( // fire: staging hands off, its rd is reserved, the ibuffer instr moves in // Either way the flop gets the value a post-grant evaluation would. // - // Either outcome checks one of only two instrs, so each is reduced - // against the released in-use set once and the outcomes select bits. + // Either outcome checks one of only two instrs, so each is checked + // once and the outcomes select bits. wire [NUM_REGS-1:0] ibf_regs_mask = ibf_opd_mask[0] | ibf_opd_mask[1] | ibf_opd_mask[2] | ibf_opd_mask[3]; wire [NUM_REGS-1:0] stg_regs_mask = stg_opd_mask[0] | stg_opd_mask[1] | stg_opd_mask[2] | stg_opd_mask[3]; wire ibf_group_busy, stg_group_busy; @@ -221,8 +221,19 @@ module VX_scoreboard import VX_gpu_pkg::*; #( assign ibf_group_busy = 1'b0; assign stg_group_busy = 1'b0; end - wire ibf_busy = (| (wb_inuse_regs & ibf_regs_mask)) || ibf_group_busy; - wire stg_busy = (| (wb_inuse_regs & stg_regs_mask)) || stg_group_busy; + // Each operand reads its in-use bit by index and compares its index with + // this cycle's release, which is shallower than reducing the operand + // masks against the whole released set. + wire wb_release = (writeback_fire || writeback_rel) && writeback_if.data.wb; + wire [NUM_OPDS-1:0] ibf_opd_busy, stg_opd_busy; + for (genvar i = 0; i < NUM_OPDS; ++i) begin : g_opd_busy + assign ibf_opd_busy[i] = ibf_used_rs[i] && inuse_regs[ibf_opds[i]] + && ~(wb_release && (writeback_if.data.rd == ibf_opds[i])); + assign stg_opd_busy[i] = stg_used_rs[i] && inuse_regs[stg_opds[i]] + && ~(wb_release && (writeback_if.data.rd == stg_opds[i])); + end + wire ibf_busy = (| ibf_opd_busy) || ibf_group_busy; + wire stg_busy = (| stg_opd_busy) || stg_group_busy; wire ibf_xbusy = | (wb_inuse_xregs & ibf_xregs_mask); wire stg_xbusy = | (wb_inuse_xregs & stg_xregs_mask); @@ -275,6 +286,7 @@ module VX_scoreboard import VX_gpu_pkg::*; #( `UNUSED_VAR (xregs_busy) `else `UNUSED_VAR (ibuffer_fire) + `UNUSED_VAR ({ibf_regs_mask, stg_regs_mask}) `endif always @(posedge clk) begin diff --git a/hw/rtl/core/VX_wctl_unit.sv b/hw/rtl/core/VX_wctl_unit.sv index 49dbabb063..7b137d586a 100644 --- a/hw/rtl/core/VX_wctl_unit.sv +++ b/hw/rtl/core/VX_wctl_unit.sv @@ -29,7 +29,6 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( VX_result_if.master result_if ); `UNUSED_SPARAM (INSTANCE_ID) - localparam LANE_BITS = `CLOG2(NUM_LANES); localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); localparam WCTL_WIDTH = $bits(tmc_t) + $bits(split_t) + $bits(join_t); @@ -52,23 +51,29 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire wctl_valid; wire tmc_valid, split_valid, sjoin_valid, wsync_valid; - wire [`UP(LANE_BITS)-1:0] last_tid; - if (LANE_BITS != 0) begin : g_last_tid - VX_priority_encoder #( - .N (NUM_LANES), - .REVERSE (1) - ) last_tid_select ( - .data_in (execute_if.data.header.tmask), - .index_out (last_tid), - `UNUSED_PIN (onehot_out), - `UNUSED_PIN (valid_out) - ); - end else begin : g_no_tid - assign last_tid = 0; + // The last active lane's operands, and whether their thread masks are + // empty (reduced per lane, ahead of the select), are picked by the mask's + // own select tree rather than by an encoded lane index driving a wide mux. + wire [NUM_LANES-1:0][2*`VX_CFG_XLEN+1:0] lane_scalars; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lane_scalars + assign lane_scalars[i] = {(| execute_if.data.rs1_data[i][`VX_CFG_NUM_THREADS-1:0]), + (| execute_if.data.rs2_data[i][`VX_CFG_NUM_THREADS-1:0]), + execute_if.data.rs1_data[i], + execute_if.data.rs2_data[i]}; end - wire [`VX_CFG_XLEN-1:0] rs1_data = execute_if.data.rs1_data[last_tid]; - wire [`VX_CFG_XLEN-1:0] rs2_data = execute_if.data.rs2_data[last_tid]; + wire rs1_tmask_nz, rs2_tmask_nz; + wire [`VX_CFG_XLEN-1:0] rs1_data, rs2_data; + VX_find_first #( + .N (NUM_LANES), + .DATAW (2*`VX_CFG_XLEN+2), + .REVERSE (1) + ) last_lane_sel ( + .valid_in (execute_if.data.header.tmask), + .data_in (lane_scalars), + .data_out ({rs1_tmask_nz, rs2_tmask_nz, rs1_data, rs2_data}), + `UNUSED_PIN (valid_out) + ); `UNUSED_VAR (rs1_data) wire not_pred = execute_if.data.op_args.wctl.is_cond_neg; @@ -183,7 +188,8 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( // completes once the unit has served it; every other request is posted, // completing as it is accepted, and the warp resumes through its unlock path. wire bar_rsp_valid = is_bar && execute_if.data.op_args.wctl.is_bar_arrive; - wire is_exit = (is_tmc || is_pred) && (tmc.tmask == 0); + // tmc.tmask == 0; a predicate mask with active then-lanes is never empty + wire is_exit = (is_tmc && ~rs1_tmask_nz) || (is_pred && ~has_then && ~rs2_tmask_nz); wire is_shared = execute_if.data.header.eop && (is_bar || is_wspawn || is_exit); wire sh_posted = is_shared && ~bar_rsp_valid; From 5bc0b071ffe3e1b1a80a8debc89b34011e7ab909 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 19:59:59 -0800 Subject: [PATCH 318/336] lsu: rename the wide local loads to vx_flds.d/.q and erase scalar LDS/LDG The scalar explicit forms bought nothing: vx_lds/vx_sts measured +0.6% (noise) on the sgemm2 sweep and vx_ldg/vx_stg took the generic path, so they were a plain lw/sw with a smaller offset. They cost 42 LUTs of decode in the core (Beacon), against the rule that area needs a measured gain. The wide loads, which do pay (geomean -5.4%, up to -18.9% where the warp is issue-bound), take standard-looking names and the fld/flq layout: vx_flds.d (custom-3 funct3 3, 64 bits into an aligned FP pair) and vx_flds.q (funct3 4, 128 bits into an aligned quad), with a 12-bit signed byte offset. Global memory keeps the standard loads and stores. The other custom-3 codes and custom-0 funct7 0x08-0x17 are free again. The regression test becomes lds_wide; the CI ids follow. Verified: tests/regression run-simx (XLEN 32); lds_wide and the CI cases lds-wide-1..4, parity-lds-wide(-lanes) on simx/rtlsim (23 passed); XLEN 64 build. Beacon core: -42 LUT (decode 369 -> 327), FF/LUTRAM unchanged, no unit deeper; lmem unchanged. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 10 +- ci/testcases/regression.yaml | 20 +- .../custom_accelerator_isa_extensions.md | 34 +- hw/rtl/VX_gpu_pkg.sv | 6 +- hw/rtl/VX_trace_pkg.sv | 35 +- hw/rtl/core/VX_decode.sv | 84 +--- hw/rtl/core/VX_lsu_block.sv | 2 +- sim/simx/decode.cpp | 89 +--- sim/simx/types.h | 4 +- tests/regression/Makefile | 2 +- tests/regression/lds_ldg/common.h | 49 --- tests/regression/lds_ldg/kernel.cpp | 393 ------------------ .../regression/{lds_ldg => lds_wide}/Makefile | 2 +- tests/regression/lds_wide/common.h | 29 ++ tests/regression/lds_wide/kernel.cpp | 237 +++++++++++ .../regression/{lds_ldg => lds_wide}/main.cpp | 62 +-- 16 files changed, 368 insertions(+), 690 deletions(-) delete mode 100644 tests/regression/lds_ldg/common.h delete mode 100644 tests/regression/lds_ldg/kernel.cpp rename tests/regression/{lds_ldg => lds_wide}/Makefile (92%) create mode 100644 tests/regression/lds_wide/common.h create mode 100644 tests/regression/lds_wide/kernel.cpp rename tests/regression/{lds_ldg => lds_wide}/main.cpp (65%) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index cd41c45087..8159bbb485 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -265,17 +265,17 @@ tests: app: lmem_access args: -n16 -s configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 -# Explicit local and global loads and stores: SimX and RTL route them alike. -- id: parity-lds-ldg +# Wide local loads: SimX and RTL route and time them alike. +- id: parity-lds-wide check: model_parity via: blackbox - app: lds_ldg + app: lds_wide configs: -DVX_CFG_NUM_THREADS=16 # Fewer LSU lanes than SIMD lanes: a wide load's phases complete across packets. -- id: parity-lds-ldg-lanes +- id: parity-lds-wide-lanes check: model_parity via: blackbox - app: lds_ldg + app: lds_wide configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_LSU_LANES=4 - id: parity-sgemm2-lmem check: model_parity diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index ff04a92e95..e8196a0060 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -62,44 +62,44 @@ tests: configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 shape: threads: 16 -# Explicit local (vx_lds/vx_sts) and global (vx_ldg/vx_stg) loads and stores: -# every width, the offset limits, and local addresses given as a bare offset. -- id: lds-ldg-1 +# Wide local loads (vx_flds.d/.q): lane patterns, register groups, the offset +# limits, and local addresses given as a bare offset. +- id: lds-wide-1 via: blackbox drivers: - simx - rtlsim - app: lds_ldg + app: lds_wide shape: threads: 16 # Two LSU blocks: local requests on the second engine. -- id: lds-ldg-2 +- id: lds-wide-2 via: blackbox drivers: - simx - rtlsim - app: lds_ldg + app: lds_wide configs: -DVX_CFG_ISSUE_WIDTH=2 shape: threads: 32 warps: 8 # Fewer LSU lanes than SIMD lanes: a warp's request goes out in four packets. -- id: lds-ldg-3 +- id: lds-wide-3 via: blackbox drivers: - simx - rtlsim - app: lds_ldg + app: lds_wide configs: -DVX_CFG_NUM_LSU_LANES=4 shape: threads: 16 # One SIMD lane: the dispatcher splits every warp into single-lane packets. -- id: lds-ldg-4 +- id: lds-wide-4 via: blackbox drivers: - simx - rtlsim - app: lds_ldg + app: lds_wide configs: -DVX_CFG_SIMD_WIDTH=1 shape: threads: 4 diff --git a/docs/designs/custom_accelerator_isa_extensions.md b/docs/designs/custom_accelerator_isa_extensions.md index efc0a428b6..07f79c7adc 100644 --- a/docs/designs/custom_accelerator_isa_extensions.md +++ b/docs/designs/custom_accelerator_isa_extensions.md @@ -226,8 +226,7 @@ the table in the same change. | | | `0x02` | tensor core (WMMA/WGMMA) | R | | | | `0x03` | DXA async copy | R | | | | `0x04` | packed loads (`vx_packl*`) | R | -| | | `0x08`–`0x0F` | `vx_ldg`: `funct7[2]` FP, `[1:0]` cache hint; `rs2` = uimm5 byte offset | R | -| | | `0x10`–`0x17` | `vx_stg`: `funct7[2]` FP, `[1:0]` cache hint; `rd` = uimm5 byte offset | R | +| | | `0x08`–`0x17` | free | | | custom-1 `0x2B` | `funct3` | `0` | `vx_wgather` | R4 | | | | `3` | `vx_om_export` | R4 | | | | `5` | `vx_tex` | R4 | @@ -235,19 +234,24 @@ the table in the same change. | | | `7` | `vx_rt_trace`, `vx_rt_wait` | R4 | | | | `1`, `2`, `4` | free | | | custom-2 `0x5B` | `funct7` | `0x00` | MUFU special functions; `fn` in `rs2`, `fmt` in `funct3` | R | -| custom-3 `0x7B` | `funct3` | `0`–`6` | `vx_lds` (LOAD widths); `imm[11]` FP, `imm[10:0]` byte offset | I | -| | | `7` | `vx_sts`; `imm[11]` FP, `imm[10:9]` size, `imm[8:0]` byte offset | S | - -`vx_lds`/`vx_sts` address local memory by the low `LMEM_LOG_SIZE` bits of the -effective address, so a local pointer may hold the window address or a bare -offset. `vx_ldg`/`vx_stg` take the generic path: the local-memory window is -reserved, so no global pointer falls in it. Their hint is 0 (default), 1 -(`.cg`) or 2 (`.cs`); 3 is reserved. With `imm[11]` set, `funct3` 0 is -`vx_flds.128` and `funct3` 4 is `vx_flds.64` (XLEN 32): one request loads four -or two consecutive words into the quad- or pair-aligned FP registers starting at -`fd`. The other integer-only codes with `imm[11]` set and the FP store with -size 0 are reserved for wide local accesses, as are `vx_ldg` with `funct3` 7 -and `vx_stg` with `funct3` 4–7. Standard loads and stores stay generic. +| custom-3 `0x7B` | `funct3` | `3` | `vx_flds.d fd, imm(rs1)`: wide local load into `fd`, `fd+1` | I | +| | | `4` | `vx_flds.q fd, imm(rs1)`: wide local load into `fd`..`fd+3` | I | +| | | `0`–`2`, `5`–`7` | reserved | | + +`vx_flds.d` and `vx_flds.q` have the layout of the standard `fld`/`flq`: `rd` +is the first FP destination, `rs1` the base, and `imm[11:0]` a signed byte +offset. Each thread loads 8 or 16 bytes from local memory into the aligned +register pair or quad starting at `fd`; `fd` must be a multiple of 2 or 4, and +the address a multiple of 8 or 16. Local memory is addressed by the low +`LMEM_LOG_SIZE` bits of the effective address, so a local pointer may hold the +window address or a bare offset. Standard loads and stores serve every other +local and global access through the generic path. + +The two forms are defined only with FLEN 32 (XLEN 32 with F) and local memory; +elsewhere custom-3 is illegal. One FP register then holds one LSU word, so the +group fills one word per phase on the existing datapath and each phase +releases its register on its own. With FLEN 64 one register already holds 64 +bits and `fld` covers the `.d` case. --- diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index d05d41b24f..ada4c84a24 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1159,8 +1159,8 @@ package VX_gpu_pkg; // words, and VX_gfx_uops emits one uop per set bit. // Zero = not an export. Read ONLY by VX_gfx_uops; the LSU never sees it set // (the expander clears it), so the LSU stays a general-purpose unit. - // is_local marks vx_lds/vx_sts: the access goes to local memory at the - // low bits of its address, whatever the address window says. + // is_local marks a wide local load: the access goes to local memory at + // the low bits of its address, whatever the address window says. typedef struct packed { logic is_local; logic [1:0] export_mask; @@ -1673,7 +1673,7 @@ package VX_gpu_pkg; logic [LSU_CLIENT_TAG_WIDTH-1:0] tag; } lsu_req_data_t; - // Wide local loads (vx_flds.64/.128) fill an aligned FP register group, + // Wide local loads (vx_flds.d/.q) fill an aligned FP register group, // one word-phase at a time. localparam LDS_WIDE_ENABLE = `VX_CFG_LMEM_ENABLED && `VX_CFG_EXT_F_ENABLED && (`VX_CFG_XLEN == 32); diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index dcfd34d2fd..7fa1deb7ce 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -182,35 +182,14 @@ package VX_trace_pkg; end endcase end else if (op_args.lsu.is_local) begin - if (op_args.lsu.is_float) begin - `TRACE(level, ("VX_F")) + // only wide loads are local: a full-size bytesel with offset 3 + // is a quad, with offset 1 a pair + if (bytesel == BYTESEL_BITS'({XLENB_W'(XLENB-1), XLENB_W'(3)})) begin + `TRACE(level, ("VX_FLDS.Q")) + end else if (bytesel == BYTESEL_BITS'({XLENB_W'(XLENB-1), XLENB_W'(1)})) begin + `TRACE(level, ("VX_FLDS.D")) end else begin - `TRACE(level, ("VX_")) - end - if (op_args.lsu.is_store) begin - `TRACE(level, ("STS.")) - end else begin - `TRACE(level, ("LDS.")) - end - // a full-size bytesel with offset 1 or 3 is a wide load - if ((bytesel[BYTESEL_BITS-1 -: XLENB_W] == XLENB_W'(XLENB-1)) - && ((bytesel[XLENB_W-1:0] | XLENB_W'(2)) == XLENB_W'(3))) begin - if (bytesel[1]) begin - `TRACE(level, ("128")) - end else begin - `TRACE(level, ("64")) - end - end else begin - case (INST_LSU_BITS'(op_type)) - INST_LSU_LB, INST_LSU_SB: `TRACE(level, ("B")) - INST_LSU_LH, INST_LSU_SH: `TRACE(level, ("H")) - INST_LSU_LW, INST_LSU_SW: `TRACE(level, ("W")) - INST_LSU_LD, INST_LSU_SD: `TRACE(level, ("D")) - INST_LSU_LBU: `TRACE(level, ("BU")) - INST_LSU_LHU: `TRACE(level, ("HU")) - INST_LSU_LWU: `TRACE(level, ("WU")) - default: `TRACE(level, ("?")) - endcase + `TRACE(level, ("?")) end end else if (op_args.lsu.is_float) begin case (INST_LSU_BITS'(op_type)) diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index f2bfba8a3b..d146ca1bc3 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -107,11 +107,6 @@ module VX_decode import wire [11:0] i_imm = is_itype_sh ? {7'b0, instr[24:20]} : u_12; `endif wire [11:0] s_imm = {funct7, rd}; -`ifdef VX_CFG_LMEM_ENABLE -`ifdef VX_CFG_EXT_F_ENABLE - wire lds_wide = LDS_WIDE_ENABLE && instr[31] && (funct3[1:0] == 2'b00); -`endif -`endif wire [11:0] b_imm = {instr[31], instr[7], instr[30:25], instr[11:8]}; wire [19:0] jal_imm = {instr[31], instr[19:12], instr[20], instr[30:21]}; @@ -495,43 +490,27 @@ module VX_decode import `endif end `ifdef VX_CFG_LMEM_ENABLE - // vx_lds/vx_sts: imm[11] selects the FP register file; a store - // carries its size in imm[10:9]. + `ifdef VX_CFG_EXT_F_ENABLE + // vx_flds.d (funct3 3) and vx_flds.q (funct3 4) load 2 or 4 words + // into an aligned FP register group, one word per beat. A full-size + // bytesel with an odd offset, which no other instruction produces, + // carries the group size. INST_EXT4: begin - ex_type = EX_LSU; - op_args.lsu.is_float = instr[31]; - op_args.lsu.pack = 0; - op_args.lsu.is_local = 1; - `USED_IREG (rs1); - if (funct3 == 3'h7) begin - op_type = INST_OP_BITS'({2'b10, instr[30:29]}); - op_args.lsu.is_store = 1; - op_args.lsu.offset = {{3{s_imm[8]}}, s_imm[8:0]}; - `ifdef VX_CFG_EXT_F_ENABLE - `USED_REG (instr[31], rs2, 1'b1); - `else - `USED_IREG (rs2); - `endif - end else begin - op_type = INST_OP_BITS'({1'b0, funct3}); + if (LDS_WIDE_ENABLE && ((funct3 == 3'h3) || (funct3 == 3'h4))) begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'(INST_LSU_LW); op_args.lsu.is_store = 0; - op_args.lsu.offset = {u_12[10], u_12[10:0]}; - `ifdef VX_CFG_EXT_F_ENABLE - // vx_flds.128 (funct3 0) and vx_flds.64 (funct3 4) load 4 or 2 - // words into an aligned FP register group, one word per beat. - // A full-size bytesel with an odd offset, which no other - // instruction produces, carries the group size. - if (lds_wide) begin - op_type = INST_OP_BITS'(INST_LSU_LW); - bytesel = {XLENB_W'(XLENB-1), XLENB_W'({~funct3[2], 1'b1})}; - end - `USED_REG (instr[31], rd, 1'b1); - `else - `USED_IREG (rd); - `endif + op_args.lsu.is_float = 1; + op_args.lsu.pack = 0; + op_args.lsu.is_local = 1; + op_args.lsu.offset = u_12; + bytesel = {XLENB_W'(XLENB-1), XLENB_W'({funct3[2], 1'b1})}; + `USED_IREG (rs1); + `USED_FREG (rd); end end `endif + `endif `ifdef VX_CFG_EXT_F_ENABLE INST_FMADD, // 7'b1000011 INST_FMSUB, // 7'b1000111 @@ -859,37 +838,6 @@ module VX_decode import default:; endcase end - // vx_ldg/vx_stg: global accesses through the generic path, the - // window being reserved for local memory. funct7[2] selects the - // FP register file; the cache hint in funct7[1:0] is advisory. - 7'h08, 7'h09, 7'h0A, 7'h0B, 7'h0C, 7'h0D, 7'h0E, 7'h0F: begin - ex_type = EX_LSU; - op_type = INST_OP_BITS'({1'b0, funct3}); - op_args.lsu.is_store = 0; - op_args.lsu.is_float = funct7[2]; - op_args.lsu.pack = 0; - op_args.lsu.offset = 12'(rs2); - `USED_IREG (rs1); - `ifdef VX_CFG_EXT_F_ENABLE - `USED_REG (funct7[2], rd, 1'b1); - `else - `USED_IREG (rd); - `endif - end - 7'h10, 7'h11, 7'h12, 7'h13, 7'h14, 7'h15, 7'h16, 7'h17: begin - ex_type = EX_LSU; - op_type = INST_OP_BITS'({2'b10, funct3[1:0]}); - op_args.lsu.is_store = 1; - op_args.lsu.is_float = funct7[2]; - op_args.lsu.pack = 0; - op_args.lsu.offset = 12'(rd); - `USED_IREG (rs1); - `ifdef VX_CFG_EXT_F_ENABLE - `USED_REG (funct7[2], rs2, 1'b1); - `else - `USED_IREG (rs2); - `endif - end default:; endcase end diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 671d82608c..d3c7d3ef53 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -137,7 +137,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // carried through the scheduler's USER channel and out via the // lsu_mem_if's per-lane req_data.user field. - // vx_lds/vx_sts address local memory by the low bits alone, so a bare + // Wide local loads address local memory by the low bits alone, so a bare // offset that falls in another window is still local. `ifdef VX_CFG_LMEM_ENABLE wire req_local = execute_if.data.op_args.lsu.is_local; diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 94adeaa7b3..7580e009b5 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -268,11 +268,7 @@ static op_string_t op_string(const Instr &instr) { auto lsuArgs = std::get(instrArgs); bool dst_float = (instr.get_dest_reg().type == RegType::Float); if (lsuArgs.nregs > 1) { - return {(lsuArgs.nregs == 4) ? "VX_FLDS.128" : "VX_FLDS.64", to_hex_str(lsuArgs.offset)}; - } - if (lsuArgs.is_local) { - static const char* const widths[] = {"B", "H", "W", "D", "BU", "HU", "WU"}; - return {std::string(dst_float ? "VX_FLDS." : "VX_LDS.") + widths[lsuArgs.width], to_hex_str(lsuArgs.offset)}; + return {(lsuArgs.nregs == 4) ? "VX_FLDS.Q" : "VX_FLDS.D", to_hex_str(lsuArgs.offset)}; } if (instr.is_macro_op() && dst_float && lsuArgs.width == 0) { return {"PACKLB.F", ""}; @@ -306,10 +302,6 @@ static op_string_t op_string(const Instr &instr) { auto lsuArgs = std::get(instrArgs); // For stores, the data source (rs2) carries the type info. bool src_float = (instr.get_src_reg(1).type == RegType::Float); - if (lsuArgs.is_local) { - static const char* const widths[] = {"B", "H", "W", "D"}; - return {std::string(src_float ? "VX_FSTS." : "VX_STS.") + widths[lsuArgs.width], to_hex_str(lsuArgs.offset)}; - } if (src_float) { switch (lsuArgs.width) { case 1: return {"FSH", to_hex_str(lsuArgs.offset)}; @@ -1058,33 +1050,6 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { } instr->set_macro_op(); } break; - // vx_ldg/vx_stg: global accesses through the generic path, the window - // being reserved for local memory. funct7[2] selects the FP register - // file; the cache hint in funct7[1:0] is advisory. - case 0x08: case 0x09: case 0x0a: case 0x0b: - case 0x0c: case 0x0d: case 0x0e: case 0x0f: { - bool is_float = (funct7 >> 2) & 0x1; - if ((funct7 & 0x3) == 3 || funct3 == 7 || (is_float && (funct3 == 0 || funct3 > 3))) { - std::abort(); - } - instr->set_fu_type(FUType::LSU); - instr->set_op_type(LsuType::LOAD); - instr->set_dest_reg(rd, is_float ? RegType::Float : RegType::Integer); - instr->set_src_reg(0, rs1, RegType::Integer); - instr->set_args(IntrLsuArgs{funct3, /*stride*/ 0, int32_t(rs2)}); - } break; - case 0x10: case 0x11: case 0x12: case 0x13: - case 0x14: case 0x15: case 0x16: case 0x17: { - bool is_float = (funct7 >> 2) & 0x1; - if ((funct7 & 0x3) == 3 || funct3 > 3 || (is_float && funct3 == 0)) { - std::abort(); - } - instr->set_fu_type(FUType::LSU); - instr->set_op_type(LsuType::STORE); - instr->set_src_reg(0, rs1, RegType::Integer); - instr->set_src_reg(1, rs2, is_float ? RegType::Float : RegType::Integer); - instr->set_args(IntrLsuArgs{funct3, /*stride*/ 0, int32_t(rd)}); - } break; default: std::abort(); } @@ -1262,45 +1227,25 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_src_reg(0, rs1, RegType::Float); } break; #endif -#ifdef VX_CFG_LMEM_ENABLE +#if defined(VX_CFG_LMEM_ENABLE) && defined(VX_CFG_EXT_F_ENABLE) && defined(VX_CFG_XLEN_32) case Opcode::EXT4: { - // vx_lds/vx_sts: imm[11] selects the FP register file; a store carries - // its size in imm[10:9]. The remaining codes are reserved. + // vx_flds.d (funct3 3) and vx_flds.q (funct3 4) load words into an + // aligned pair or quad of FP registers. + if (funct3 != 3 && funct3 != 4) { + std::abort(); + } + uint32_t nregs = (funct3 == 4) ? 4 : 2; + if (rd % nregs != 0) { + std::cout << "Error: vx_flds." << ((nregs == 4) ? "q" : "d") << " destination f" << rd + << " is not aligned to " << nregs << " registers" << std::endl; + std::abort(); + } + uint32_t imm12 = code >> shift_rs2; instr->set_fu_type(FUType::LSU); + instr->set_op_type(LsuType::LOAD); + instr->set_dest_reg(rd, RegType::Float); instr->set_src_reg(0, rs1, RegType::Integer); - bool is_float = (funct7 >> 6) & 0x1; - if (funct3 == 7) { - uint32_t size = (funct7 >> 4) & 0x3; - if (is_float && size == 0) { - std::abort(); - } - uint32_t imm9 = ((funct7 & 0xf) << width_reg) | rd; - instr->set_src_reg(1, rs2, is_float ? RegType::Float : RegType::Integer); - instr->set_op_type(LsuType::STORE); - instr->set_args(IntrLsuArgs{size, /*stride*/ 0, (int32_t)sext(imm9, 9), /*is_local*/ true}); - } else { - // vx_flds.128 (funct3 0) and vx_flds.64 (funct3 4) load words into an - // aligned quad or pair of FP registers; defined on XLEN 32 only - uint32_t nregs = 1; - if (is_float && (funct3 == 0 || funct3 == 4)) { - #if defined(VX_CFG_XLEN_32) && defined(VX_CFG_EXT_F_ENABLE) - nregs = (funct3 == 0) ? 4 : 2; - #else - std::abort(); - #endif - } else if (is_float && funct3 > 3) { - std::abort(); - } - if (rd % nregs != 0) { - std::cout << "Error: vx_flds." << (32 * nregs) << " destination f" << rd - << " is not aligned to " << nregs << " registers" << std::endl; - std::abort(); - } - uint32_t imm11 = (code >> shift_rs2) & 0x7ff; - instr->set_dest_reg(rd, is_float ? RegType::Float : RegType::Integer); - instr->set_op_type(LsuType::LOAD); - instr->set_args(IntrLsuArgs{(nregs > 1) ? 2u : funct3, /*stride*/ 0, (int32_t)sext(imm11, 11), /*is_local*/ true, nregs}); - } + instr->set_args(IntrLsuArgs{/*width=LW*/ 2, /*stride*/ 0, (int32_t)sext(imm12, width_i_imm), /*is_local*/ true, nregs}); } break; #endif default: diff --git a/sim/simx/types.h b/sim/simx/types.h index 24d0b3689b..4042915be0 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -384,8 +384,8 @@ struct IntrLsuArgs { uint32_t width : 3; uint32_t stride : 4; // per-uop multiplier: 0 = regular LD/ST; 0..N-1 = packLD uop_idx int32_t offset; // immediate addr offset (signed); 0 for packLD uops - bool is_local = false; // vx_lds/vx_sts: local memory at the low address bits - uint32_t nregs = 1; // vx_flds.64/.128: registers fd.. loaded one word-phase each + bool is_local = false; // wide local load: local memory at the low address bits + uint32_t nregs = 1; // wide local load: registers fd.. loaded one word-phase each }; // A fence carries no access width; its width field says what it orders. diff --git a/tests/regression/Makefile b/tests/regression/Makefile index e9255f220c..df61d1eb34 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax uniform_ld lmem_access lds_ldg mufu zfhmin \ + softmax uniform_ld lmem_access lds_wide mufu zfhmin \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/lds_ldg/common.h b/tests/regression/lds_ldg/common.h deleted file mode 100644 index 0933e9f46b..0000000000 --- a/tests/regression/lds_ldg/common.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef _COMMON_H_ -#define _COMMON_H_ - -#include - -// Local memory per block: RO_WORDS read-only words, then SLOT_WORDS words per -// thread that it stores to, then, from a 16-byte boundary, WIDE_WORDS words per -// thread that it stores to and reads back with wide loads. -#define RO_WORDS 1024 -#define SLOT_WORDS 5 -#define WIDE_WORDS 4 - -static inline uint32_t wide_base(uint32_t block) { - return (RO_WORDS + block * SLOT_WORDS + 3) & ~3u; -} - -// Global input words read past each thread's index (the largest LDG offset). -#define IN_PAD 8 - -// Initial value of local word i and global input word i: bytes differ from -// word to word and about half of them have the sign bit set, so sub-word loads -// see both sign- and zero-extended values. -static inline uint32_t ro_word(uint32_t i) { - return (i * 0x9E3779B1u) ^ 0xA5C30000u; -} -static inline uint32_t in_word(uint32_t i) { - return (i * 0x85EBCA77u) ^ 0x5AC30000u; -} - -// The words a thread stores: local slot word k, and global output word k. -static inline uint32_t slot_word(uint32_t tid, uint32_t k) { - return 0xC0000000u | (tid << 4) | k; -} -static inline uint32_t out_word(uint32_t tid, uint32_t k) { - return 0x30000000u | (tid << 4) | k; -} -static inline uint32_t wide_word(uint32_t tid, uint32_t k) { - return 0x6C000000u | (tid << 4) | k; -} - -typedef struct { - uint32_t lmem_words; // local-memory words per block - uint64_t in_addr; // global input words, one per thread + IN_PAD - uint64_t out_addr; // global output, SLOT_WORDS words per thread - uint64_t err_addr; // out: per-thread error count - uint64_t dump_addr; // out: each block's final local memory -} kernel_arg_t; - -#endif diff --git a/tests/regression/lds_ldg/kernel.cpp b/tests/regression/lds_ldg/kernel.cpp deleted file mode 100644 index 600c40f445..0000000000 --- a/tests/regression/lds_ldg/kernel.cpp +++ /dev/null @@ -1,393 +0,0 @@ -#include -#include "common.h" - -// vx_lds/vx_sts (custom-3) and vx_ldg/vx_stg (custom-0), written as .insn so -// the encodings are tested independently of the compiler. -// vx_lds: I-type, funct3 = LOAD width, imm[11] = FP, imm[10:0] = offset -// vx_flds.128 / .64: vx_lds with imm[11] = 1 and funct3 = 0 / 4; fd is an -// aligned quad / pair, the base 16 / 8-byte aligned -// vx_sts: S-type, funct3 = 7, imm[11] = FP, imm[10:9] = size, imm[8:0] = offset -// vx_ldg: R-type, funct7 = 0b0001_F_HH, funct3 = LOAD width, rs2 = uimm5 offset -// vx_stg: R-type, funct7 = 0b0010_F_HH, funct3 = size, rd = uimm5 offset - -namespace { - -template -inline uint32_t lds(uintptr_t p) { - uint32_t v; - asm volatile (".insn i 0x7b, %2, %0, %1, %3" : "=r"(v) : "r"(p), "i"(F3), "i"(OFF & 0x7ff) : "memory"); - return v; -} - -template -inline uint32_t flds_w(uintptr_t p) { - float v; - asm volatile (".insn i 0x7b, 2, %0, %1, %2" : "=f"(v) : "r"(p), "i"((OFF & 0x7ff) - 2048) : "memory"); - uint32_t bits; - __builtin_memcpy(&bits, &v, sizeof(bits)); - return bits; -} - -constexpr int sts_imm(int fp, int size, int off) { - return ((fp << 11) | (size << 9) | (off & 0x1ff)) - (fp ? 4096 : 0); -} - -template -inline void sts(uintptr_t p, uint32_t v) { - asm volatile (".insn s 0x7b, 7, %0, %2(%1)" :: "r"(v), "r"(p), "i"(sts_imm(0, SIZE, OFF)) : "memory"); -} - -template -inline void fsts_w(uintptr_t p, uint32_t bits) { - float v; - __builtin_memcpy(&v, &bits, sizeof(v)); - asm volatile (".insn s 0x7b, 7, %0, %2(%1)" :: "f"(v), "r"(p), "i"(sts_imm(1, 2, OFF)) : "memory"); -} - -} // namespace - -#define VX_LDG(f3, f7, imm, p) ({ \ - uint32_t _v; \ - asm volatile (".insn r 0x0b, " #f3 ", " #f7 ", %0, %1, x" #imm : "=r"(_v) : "r"(p) : "memory"); \ - _v; }) - -#define VX_FLDG_W(f7, imm, p) ({ \ - float _f; \ - asm volatile (".insn r 0x0b, 2, " #f7 ", %0, %1, x" #imm : "=f"(_f) : "r"(p) : "memory"); \ - uint32_t _v; \ - __builtin_memcpy(&_v, &_f, sizeof(_v)); \ - _v; }) - -#define VX_STG(f3, f7, imm, p, v) \ - asm volatile (".insn r 0x0b, " #f3 ", " #f7 ", x" #imm ", %0, %1" :: "r"(p), "r"(v) : "memory") - -#define VX_FSTG_W(f7, imm, p, bits) do { \ - float _f; \ - uint32_t _b = (bits); \ - __builtin_memcpy(&_f, &_b, sizeof(_f)); \ - asm volatile (".insn r 0x0b, 2, " #f7 ", x" #imm ", %0, %1" :: "r"(p), "f"(_f) : "memory"); \ -} while (0) - -// vx_flds is defined on XLEN 32 with the F extension. -#if (__riscv_xlen == 32) && defined(__riscv_f) -#define HAS_FLDS 1 -#endif - -// A wide load and the reads of its registers sit in one asm block, so the -// compiler cannot reuse them in between. fd + 3 (fd + 1) is read first: it is -// the last phase's register. -#define VX_FLDS128(r0, r1, r2, r3, off, p, w) \ - asm volatile (".insn i 0x7b, 0, " r0 ", %4, %5\n\t" \ - "fmv.x.w %3, " r3 "\n\t" \ - "fmv.x.w %0, " r0 "\n\t" \ - "fmv.x.w %1, " r1 "\n\t" \ - "fmv.x.w %2, " r2 \ - : "=&r"((w)[0]), "=&r"((w)[1]), "=&r"((w)[2]), "=&r"((w)[3]) \ - : "r"(p), "i"(((off) & 0x7ff) - 2048) \ - : r0, r1, r2, r3, "memory") - -#define VX_FLDS64(r0, r1, off, p, w) \ - asm volatile (".insn i 0x7b, 4, " r0 ", %2, %3\n\t" \ - "fmv.x.w %1, " r1 "\n\t" \ - "fmv.x.w %0, " r0 \ - : "=&r"((w)[0]), "=&r"((w)[1]) \ - : "r"(p), "i"(((off) & 0x7ff) - 2048) \ - : r0, r1, "memory") - -#ifdef HAS_FLDS -static inline uint32_t ro_misses(const uint32_t* w, uint32_t n, uint32_t first) { - uint32_t misses = 0; - for (uint32_t k = 0; k < n; ++k) { - if (w[k] != ro_word(first + k)) { - ++misses; - } - } - return misses; -} -#endif - -static inline uint32_t sext8(uint32_t v) { return (uint32_t)(int32_t)(int8_t)v; } -static inline uint32_t sext16(uint32_t v) { return (uint32_t)(int32_t)(int16_t)v; } - -// Each thread checks, against the values the host and the generic path -// define: -// vx_lds - words at the offset limits, through the window address and -// through the bare offset (local memory decodes only the low -// bits); bytes and half-words, signed and unsigned; an FP word -// vx_sts - words at the offset limits, half-words through the bare -// offset, bytes, an FP word; each read back by a generic load -// ordering - a generic store, then a vx_lds of the same word -// vx_flds.128/.64 - a warp-uniform group (broadcast), a 16-byte lane stride -// (every lane on the same banks), a lane permutation, half the -// lanes under a divergent branch; fd = f0 and f28; two wide loads -// back to back; scalar writes to fd + 2 just before and just -// after; a wide load, an FP op on fd + 2 that waits for it, -// and a wide load of the same group again; offsets -1024 and 1008 through the window address and -// the bare offset; a generic store, then a wide load of the words -// vx_ldg - every width from offset 0 to the 31-byte limit, the FP form, -// every hint -// vx_stg - words, half-words, bytes up to the 31-byte limit, the FP form -__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - auto lmem = reinterpret_cast(__local_mem()); - auto in = reinterpret_cast(arg->in_addr); - auto out = reinterpret_cast(arg->out_addr); - auto err = reinterpret_cast(arg->err_addr); - auto dump = reinterpret_cast(arg->dump_addr); - const uint32_t lid = threadIdx.x; - const uint32_t bsize = blockDim.x; - const uint32_t tid = blockIdx.x * bsize + lid; - const uint32_t words = arg->lmem_words; - - for (uint32_t i = lid; i < RO_WORDS; i += bsize) { - lmem[i] = ro_word(i); - } - __syncthreads(); - - uint32_t errors = 0; - const uintptr_t window = reinterpret_cast(lmem); - const uintptr_t bare = window - VX_MEM_LMEM_BASE_ADDR; - - // reads reach qi - 256 .. qi + 255 words, inside the read-only words - const uint32_t qi = 384 + (lid * 7) % 256; - const uintptr_t q = window + qi * 4; - const uintptr_t qb = bare + qi * 4; - if (lds<2, 0>(q) != ro_word(qi)) { - ++errors; - } - if (lds<2, 1020>(q) != ro_word(qi + 255)) { - ++errors; - } - if (lds<2, -1024>(q) != ro_word(qi - 256)) { - ++errors; - } - if (lds<2, 0>(qb) != ro_word(qi)) { - ++errors; - } - if (lds<2, 1020>(qb) != ro_word(qi + 255)) { - ++errors; - } - if (lds<2, -1024>(qb) != ro_word(qi - 256)) { - ++errors; - } - - { - const uint32_t v = ro_word(qi); - const uint32_t bo = lid & 3; - const uint32_t ho = lid & 1; - if (lds<0, 0>(q + bo) != sext8(v >> (8 * bo))) { - ++errors; - } - if (lds<4, 0>(q + bo) != ((v >> (8 * bo)) & 0xff)) { - ++errors; - } - if (lds<1, 0>(q + 2 * ho) != sext16(v >> (16 * ho))) { - ++errors; - } - if (lds<5, 0>(q + 2 * ho) != ((v >> (16 * ho)) & 0xffff)) { - ++errors; - } - if (lds<4, 3>(q) != (v >> 24)) { - ++errors; - } - if (lds<0, -1>(q) != sext8(ro_word(qi - 1) >> 24)) { - ++errors; - } - if (lds<1, -2>(qb) != sext16(ro_word(qi - 1) >> 16)) { - ++errors; - } - if (flds_w<4>(q) != ro_word(qi + 1)) { - ++errors; - } - if (flds_w<-1024>(qb) != ro_word(qi - 256)) { - ++errors; - } - } - - { - const uint32_t si = RO_WORDS + lid * SLOT_WORDS; - const uintptr_t s = window + si * 4; - const uintptr_t sb = bare + si * 4; - sts<2, -256>(s + 256, slot_word(tid, 0)); - sts<2, 252>(s + 4 - 252, slot_word(tid, 1)); - sts<1, 0>(sb + 8, slot_word(tid, 2)); - sts<1, 2>(sb + 8, slot_word(tid, 2) >> 16); - fsts_w<12>(s, slot_word(tid, 3)); - sts<0, 16>(s, slot_word(tid, 4)); - sts<0, 17>(s, slot_word(tid, 4) >> 8); - sts<0, -238>(s + 256, slot_word(tid, 4) >> 16); - sts<0, 19>(sb, slot_word(tid, 4) >> 24); - for (uint32_t k = 0; k < SLOT_WORDS; ++k) { - if (lmem[si + k] != slot_word(tid, k)) { - ++errors; - } - } - lmem[si] = ~slot_word(tid, 0); - if (lds<2, 0>(s) != ~slot_word(tid, 0)) { - ++errors; - } - lmem[si] = slot_word(tid, 0); - } - - { - const uintptr_t g = reinterpret_cast(in + tid); - if (VX_LDG(2, 0x08, 0, g) != in_word(tid)) { - ++errors; - } - if (VX_LDG(2, 0x09, 28, g) != in_word(tid + 7)) { - ++errors; - } - if (VX_LDG(0, 0x0a, 1, g) != sext8(in_word(tid) >> 8)) { - ++errors; - } - if (VX_LDG(4, 0x09, 31, g) != (in_word(tid + 7) >> 24)) { - ++errors; - } - if (VX_LDG(1, 0x08, 30, g) != sext16(in_word(tid + 7) >> 16)) { - ++errors; - } - if (VX_LDG(5, 0x08, 0, g) != (in_word(tid) & 0xffff)) { - ++errors; - } - if (VX_FLDG_W(0x0c, 12, g) != in_word(tid + 3)) { - ++errors; - } - if (VX_FLDG_W(0x0e, 28, g) != in_word(tid + 7)) { - ++errors; - } - - const uintptr_t o = reinterpret_cast(out + tid * SLOT_WORDS); - VX_STG(2, 0x10, 0, o, out_word(tid, 0)); - VX_STG(2, 0x11, 28, o + 4 - 28, out_word(tid, 1)); - VX_STG(1, 0x12, 8, o, out_word(tid, 2)); - VX_STG(1, 0x10, 10, o, out_word(tid, 2) >> 16); - VX_FSTG_W(0x14, 12, o, out_word(tid, 3)); - VX_STG(0, 0x10, 16, o, out_word(tid, 4)); - VX_STG(0, 0x12, 31, o + 17 - 31, out_word(tid, 4) >> 8); - VX_STG(0, 0x10, 18, o, out_word(tid, 4) >> 16); - VX_STG(0, 0x10, 19, o, out_word(tid, 4) >> 24); - } - -#ifdef HAS_FLDS - { - uint32_t w[8]; - const uint32_t gu = 384; - VX_FLDS128("f8", "f9", "f10", "f11", 0, window + gu * 4, w); - errors += ro_misses(w, 4, gu); - VX_FLDS64("f8", "f9", 8, bare + gu * 4, w); - errors += ro_misses(w, 2, gu + 2); - - // a warp's lanes stay 16 bytes apart, and gs + 7 stays in the read-only - // words for any block size - const uint32_t gs = 256 + (lid % 128) * 4; - VX_FLDS128("f12", "f13", "f14", "f15", 0, window + gs * 4, w); - errors += ro_misses(w, 4, gs); - VX_FLDS64("f14", "f15", 0, window + gs * 4, w); - errors += ro_misses(w, 2, gs); - - const uint32_t gp = 512 + ((lid * 7 + 3) % 64) * 4; - VX_FLDS128("f16", "f17", "f18", "f19", 0, bare + gp * 4, w); - errors += ro_misses(w, 4, gp); - VX_FLDS64("f16", "f17", 8, window + gp * 4, w); - errors += ro_misses(w, 2, gp + 2); - - if (lid & 1) { - VX_FLDS128("f20", "f21", "f22", "f23", 0, window + gs * 4, w); - errors += ro_misses(w, 4, gs); - VX_FLDS64("f22", "f23", 8, window + gp * 4, w); - errors += ro_misses(w, 2, gp + 2); - } - - VX_FLDS128("f0", "f1", "f2", "f3", 0, window + gp * 4, w); - errors += ro_misses(w, 4, gp); - VX_FLDS128("f28", "f29", "f30", "f31", 0, bare + gs * 4, w); - errors += ro_misses(w, 4, gs); - VX_FLDS64("f30", "f31", 0, window + gu * 4, w); - errors += ro_misses(w, 2, gu); - - asm volatile (".insn i 0x7b, 0, f8, %8, %9\n\t" - ".insn i 0x7b, 0, f12, %8, %10\n\t" - "fmv.x.w %0, f8\n\t" - "fmv.x.w %1, f9\n\t" - "fmv.x.w %2, f10\n\t" - "fmv.x.w %3, f11\n\t" - "fmv.x.w %4, f12\n\t" - "fmv.x.w %5, f13\n\t" - "fmv.x.w %6, f14\n\t" - "fmv.x.w %7, f15" - : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]), - "=&r"(w[4]), "=&r"(w[5]), "=&r"(w[6]), "=&r"(w[7]) - : "r"(window + gs * 4), "i"(-2048), "i"((16 & 0x7ff) - 2048) - : "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "memory"); - errors += ro_misses(w, 4, gs); - errors += ro_misses(w + 4, 4, gs + 4); - - const uint32_t marker = 0x7F00D000u | lid; - asm volatile ("fmv.w.x f10, %1\n\t" - ".insn i 0x7b, 0, f8, %2, %3\n\t" - "fmv.x.w %0, f10" - : "=&r"(w[2]) - : "r"(marker), "r"(window + gp * 4), "i"(-2048) - : "f8", "f9", "f10", "f11", "memory"); - if (w[2] != ro_word(gp + 2)) { - ++errors; - } - asm volatile (".insn i 0x7b, 0, f8, %4, %5\n\t" - "fmv.w.x f10, %6\n\t" - "fmv.x.w %0, f8\n\t" - "fmv.x.w %1, f9\n\t" - "fmv.x.w %2, f10\n\t" - "fmv.x.w %3, f11" - : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) - : "r"(window + gp * 4), "i"(-2048), "r"(marker) - : "f8", "f9", "f10", "f11", "memory"); - if (w[0] != ro_word(gp) || w[1] != ro_word(gp + 1) || w[2] != marker || w[3] != ro_word(gp + 3)) { - ++errors; - } - asm volatile (".insn i 0x7b, 0, f8, %4, %5\n\t" - "fadd.s f10, f10, f10\n\t" - ".insn i 0x7b, 0, f8, %4, %6\n\t" - "fmv.x.w %0, f8\n\t" - "fmv.x.w %1, f9\n\t" - "fmv.x.w %2, f10\n\t" - "fmv.x.w %3, f11" - : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) - : "r"(window + gs * 4), "i"(-2048), "i"((16 & 0x7ff) - 2048) - : "f8", "f9", "f10", "f11", "memory"); - errors += ro_misses(w, 4, gs + 4); - - const uint32_t go = 512; - VX_FLDS128("f4", "f5", "f6", "f7", -1024, window + go * 4 + 1024, w); - errors += ro_misses(w, 4, go); - VX_FLDS128("f4", "f5", "f6", "f7", 1008, window + go * 4 - 1008, w); - errors += ro_misses(w, 4, go); - VX_FLDS128("f24", "f25", "f26", "f27", -1024, bare + gs * 4 + 1024, w); - errors += ro_misses(w, 4, gs); - VX_FLDS128("f24", "f25", "f26", "f27", 1008, bare + gs * 4 - 1008, w); - errors += ro_misses(w, 4, gs); - VX_FLDS64("f26", "f27", -1024, window + gp * 4 + 1024, w); - errors += ro_misses(w, 2, gp); - - const uint32_t wi = wide_base(bsize) + lid * WIDE_WORDS; - for (uint32_t k = 0; k < WIDE_WORDS; ++k) { - lmem[wi + k] = wide_word(tid, k); - } - VX_FLDS128("f16", "f17", "f18", "f19", 0, window + wi * 4, w); - VX_FLDS64("f20", "f21", 8, bare + wi * 4, w + 4); - for (uint32_t k = 0; k < WIDE_WORDS; ++k) { - if (w[k] != wide_word(tid, k)) { - ++errors; - } - } - if (w[4] != wide_word(tid, 2) || w[5] != wide_word(tid, 3)) { - ++errors; - } - } -#endif - - err[tid] = errors; - - __syncthreads(); - for (uint32_t i = lid; i < words; i += bsize) { - dump[blockIdx.x * words + i] = lmem[i]; - } -} diff --git a/tests/regression/lds_ldg/Makefile b/tests/regression/lds_wide/Makefile similarity index 92% rename from tests/regression/lds_ldg/Makefile rename to tests/regression/lds_wide/Makefile index d5465bb33c..5e27cb0b06 100644 --- a/tests/regression/lds_ldg/Makefile +++ b/tests/regression/lds_wide/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk -PROJECT := lds_ldg +PROJECT := lds_wide SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) diff --git a/tests/regression/lds_wide/common.h b/tests/regression/lds_wide/common.h new file mode 100644 index 0000000000..9a21bb4932 --- /dev/null +++ b/tests/regression/lds_wide/common.h @@ -0,0 +1,29 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Local memory per block: RO_WORDS read-only words, then WIDE_WORDS words per +// thread that it stores to and reads back. RO_WORDS keeps them 16-byte aligned. +#define RO_WORDS 1024 +#define WIDE_WORDS 4 + +// Initial value of local word i: words differ from each other and about half +// of them have the sign bit set. +static inline uint32_t ro_word(uint32_t i) { + return (i * 0x9E3779B1u) ^ 0xA5C30000u; +} + +// The words a thread stores to its wide slot. +static inline uint32_t wide_word(uint32_t tid, uint32_t k) { + return 0x6C000000u | (tid << 4) | k; +} + +typedef struct { + uint32_t lmem_words; // local-memory words per block + uint64_t err_addr; // out: per-thread error count + uint64_t dump_addr; // out: each block's final local memory + uint64_t wide_addr; // out: 1 if the kernel reads with wide loads +} kernel_arg_t; + +#endif diff --git a/tests/regression/lds_wide/kernel.cpp b/tests/regression/lds_wide/kernel.cpp new file mode 100644 index 0000000000..b57a771660 --- /dev/null +++ b/tests/regression/lds_wide/kernel.cpp @@ -0,0 +1,237 @@ +#include +#include "common.h" + +// vx_flds.d / vx_flds.q (custom-3), written as .insn so the encodings are +// tested independently of the compiler. Both are I-type like fld/flq: +// funct3 = 3 / 4, imm[11:0] = signed byte offset; fd is an aligned pair / +// quad, the address 8 / 16-byte aligned. +#if (__riscv_xlen == 32) && defined(__riscv_f) +#define HAS_FLDS 1 +#endif + +// A wide load and the reads of its registers sit in one asm block, so the +// compiler cannot reuse them in between. fd + 3 (fd + 1) is read first: it is +// the last phase's register. +#define VX_FLDS_Q(r0, r1, r2, r3, off, p, w) \ + asm volatile (".insn i 0x7b, 4, " r0 ", %4, %5\n\t" \ + "fmv.x.w %3, " r3 "\n\t" \ + "fmv.x.w %0, " r0 "\n\t" \ + "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %2, " r2 \ + : "=&r"((w)[0]), "=&r"((w)[1]), "=&r"((w)[2]), "=&r"((w)[3]) \ + : "r"(p), "i"(off) \ + : r0, r1, r2, r3, "memory") + +#define VX_FLDS_D(r0, r1, off, p, w) \ + asm volatile (".insn i 0x7b, 3, " r0 ", %2, %3\n\t" \ + "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %0, " r0 \ + : "=&r"((w)[0]), "=&r"((w)[1]) \ + : "r"(p), "i"(off) \ + : r0, r1, "memory") + +#ifdef HAS_FLDS +static inline uint32_t ro_misses(const uint32_t* w, uint32_t n, uint32_t first) { + uint32_t misses = 0; + for (uint32_t k = 0; k < n; ++k) { + if (w[k] != ro_word(first + k)) { + ++misses; + } + } + return misses; +} +#else +static inline uint32_t lmem_misses(volatile const uint32_t* lmem, uint32_t n, uint32_t first) { + uint32_t misses = 0; + for (uint32_t k = 0; k < n; ++k) { + if (lmem[first + k] != ro_word(first + k)) { + ++misses; + } + } + return misses; +} +#endif + +// Each thread checks the words its wide loads return against the values the +// host and the generic path define: +// - a warp-uniform group (broadcast), a 16-byte lane stride (every lane on +// the same banks), a lane permutation, half the lanes under a divergent +// branch; .d and .q through the window address and the bare offset (local +// memory decodes only the low bits) +// - fd = f0 and f28; two wide loads back to back +// - scalar writes to fd + 2 just before and just after +// - a wide load, an FP op on fd + 2 that waits for it, and a wide load of +// the same group again +// - the offset limits, -2048 and 2032 (.q) or 2040 (.d), through the window +// address and the bare offset +// - a generic store, then a wide load of the same words +// Without wide loads, generic loads read the same words. +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = reinterpret_cast(__local_mem()); + auto err = reinterpret_cast(arg->err_addr); + auto dump = reinterpret_cast(arg->dump_addr); + const uint32_t lid = threadIdx.x; + const uint32_t bsize = blockDim.x; + const uint32_t tid = blockIdx.x * bsize + lid; + const uint32_t words = arg->lmem_words; + + if (tid == 0) { + #ifdef HAS_FLDS + *reinterpret_cast(arg->wide_addr) = 1; + #else + *reinterpret_cast(arg->wide_addr) = 0; + #endif + } + + for (uint32_t i = lid; i < RO_WORDS; i += bsize) { + lmem[i] = ro_word(i); + } + __syncthreads(); + + uint32_t errors = 0; + + // uniform, lane stride and permuted groups; gs + 7 and gq + 3 stay in the + // read-only words for any block size, and gq * 4 exceeds the largest offset + const uint32_t gu = 384; + const uint32_t gs = 256 + (lid % 128) * 4; + const uint32_t gp = 512 + ((lid * 7 + 3) % 64) * 4; + const uint32_t go = 512; + const uint32_t gq = 512 + (lid % 128) * 4; + const uint32_t wi = RO_WORDS + lid * WIDE_WORDS; + +#ifdef HAS_FLDS + { + const uintptr_t window = reinterpret_cast(lmem); + const uintptr_t bare = window - VX_MEM_LMEM_BASE_ADDR; + uint32_t w[8]; + + VX_FLDS_Q("f8", "f9", "f10", "f11", 0, window + gu * 4, w); + errors += ro_misses(w, 4, gu); + VX_FLDS_D("f8", "f9", 8, bare + gu * 4, w); + errors += ro_misses(w, 2, gu + 2); + + VX_FLDS_Q("f12", "f13", "f14", "f15", 0, window + gs * 4, w); + errors += ro_misses(w, 4, gs); + VX_FLDS_D("f14", "f15", 0, window + gs * 4, w); + errors += ro_misses(w, 2, gs); + + VX_FLDS_Q("f16", "f17", "f18", "f19", 0, bare + gp * 4, w); + errors += ro_misses(w, 4, gp); + VX_FLDS_D("f16", "f17", 8, window + gp * 4, w); + errors += ro_misses(w, 2, gp + 2); + + if (lid & 1) { + VX_FLDS_Q("f20", "f21", "f22", "f23", 0, window + gs * 4, w); + errors += ro_misses(w, 4, gs); + VX_FLDS_D("f22", "f23", 8, window + gp * 4, w); + errors += ro_misses(w, 2, gp + 2); + } + + VX_FLDS_Q("f0", "f1", "f2", "f3", 0, window + gp * 4, w); + errors += ro_misses(w, 4, gp); + VX_FLDS_Q("f28", "f29", "f30", "f31", 0, bare + gs * 4, w); + errors += ro_misses(w, 4, gs); + VX_FLDS_D("f30", "f31", 0, window + gu * 4, w); + errors += ro_misses(w, 2, gu); + + asm volatile (".insn i 0x7b, 4, f8, %8, 0\n\t" + ".insn i 0x7b, 4, f12, %8, 16\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11\n\t" + "fmv.x.w %4, f12\n\t" + "fmv.x.w %5, f13\n\t" + "fmv.x.w %6, f14\n\t" + "fmv.x.w %7, f15" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]), + "=&r"(w[4]), "=&r"(w[5]), "=&r"(w[6]), "=&r"(w[7]) + : "r"(window + gs * 4) + : "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "memory"); + errors += ro_misses(w, 4, gs); + errors += ro_misses(w + 4, 4, gs + 4); + + const uint32_t marker = 0x7F00D000u | lid; + asm volatile ("fmv.w.x f10, %1\n\t" + ".insn i 0x7b, 4, f8, %2, 0\n\t" + "fmv.x.w %0, f10" + : "=&r"(w[2]) + : "r"(marker), "r"(window + gp * 4) + : "f8", "f9", "f10", "f11", "memory"); + if (w[2] != ro_word(gp + 2)) { + ++errors; + } + asm volatile (".insn i 0x7b, 4, f8, %4, 0\n\t" + "fmv.w.x f10, %5\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) + : "r"(window + gp * 4), "r"(marker) + : "f8", "f9", "f10", "f11", "memory"); + if (w[0] != ro_word(gp) || w[1] != ro_word(gp + 1) || w[2] != marker || w[3] != ro_word(gp + 3)) { + ++errors; + } + asm volatile (".insn i 0x7b, 4, f8, %4, 0\n\t" + "fadd.s f10, f10, f10\n\t" + ".insn i 0x7b, 4, f8, %4, 16\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) + : "r"(window + gs * 4) + : "f8", "f9", "f10", "f11", "memory"); + errors += ro_misses(w, 4, gs + 4); + + VX_FLDS_Q("f4", "f5", "f6", "f7", -2048, window + go * 4 + 2048, w); + errors += ro_misses(w, 4, go); + VX_FLDS_Q("f4", "f5", "f6", "f7", 2032, window + go * 4 - 2032, w); + errors += ro_misses(w, 4, go); + VX_FLDS_Q("f24", "f25", "f26", "f27", -2048, bare + gs * 4 + 2048, w); + errors += ro_misses(w, 4, gs); + VX_FLDS_Q("f24", "f25", "f26", "f27", 2032, bare + gq * 4 - 2032, w); + errors += ro_misses(w, 4, gq); + VX_FLDS_D("f26", "f27", -2048, window + gp * 4 + 2048, w); + errors += ro_misses(w, 2, gp); + VX_FLDS_D("f26", "f27", 2040, bare + gq * 4 - 2040, w); + errors += ro_misses(w, 2, gq); + + for (uint32_t k = 0; k < WIDE_WORDS; ++k) { + lmem[wi + k] = wide_word(tid, k); + } + VX_FLDS_Q("f16", "f17", "f18", "f19", 0, window + wi * 4, w); + VX_FLDS_D("f20", "f21", 8, bare + wi * 4, w + 4); + for (uint32_t k = 0; k < WIDE_WORDS; ++k) { + if (w[k] != wide_word(tid, k)) { + ++errors; + } + } + if (w[4] != wide_word(tid, 2) || w[5] != wide_word(tid, 3)) { + ++errors; + } + } +#else + errors += lmem_misses(lmem, 4, gu); + errors += lmem_misses(lmem, 8, gs); + errors += lmem_misses(lmem, 4, gp); + errors += lmem_misses(lmem, 4, go); + errors += lmem_misses(lmem, 4, gq); + for (uint32_t k = 0; k < WIDE_WORDS; ++k) { + lmem[wi + k] = wide_word(tid, k); + } + for (uint32_t k = 0; k < WIDE_WORDS; ++k) { + if (lmem[wi + k] != wide_word(tid, k)) { + ++errors; + } + } +#endif + + err[tid] = errors; + + __syncthreads(); + for (uint32_t i = lid; i < words; i += bsize) { + dump[blockIdx.x * words + i] = lmem[i]; + } +} diff --git a/tests/regression/lds_ldg/main.cpp b/tests/regression/lds_wide/main.cpp similarity index 65% rename from tests/regression/lds_ldg/main.cpp rename to tests/regression/lds_wide/main.cpp index 597bade802..6f1d10ac01 100644 --- a/tests/regression/lds_ldg/main.cpp +++ b/tests/regression/lds_wide/main.cpp @@ -1,6 +1,5 @@ -// lds_ldg — the explicit local (vx_lds/vx_sts, vx_flds.64/.128) and global -// (vx_ldg/vx_stg) loads and stores return and store what the generic path -// does, at every width and at the limits of their offsets. +// lds_wide — the wide local loads (vx_flds.d/.q) return what the generic path +// stores, across lane patterns, register groups and the limits of their offset. #include #include "common.h" @@ -54,15 +53,15 @@ int main(int argc, char** argv) { CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); CHECK(vx_device_query(dev, VX_CAPS_LOCAL_MEM_SIZE, &lmem_size)); CHECK(vx_device_query(dev, VX_CAPS_ISA_FLAGS, &isa)); - // the kernel's wide loads (vx_flds) exist on XLEN 32 with F + // the wide loads exist on XLEN 32 with F; elsewhere the kernel reads the + // same words with generic loads const bool wide = (VX_ISA_ARCH(isa) == 32) && (isa & VX_ISA_STD_F); const uint32_t block = static_cast(nt * nw); const uint32_t grid = static_cast(nc * 2); const uint32_t num_threads = block * grid; - const uint32_t slot_end = RO_WORDS + block * SLOT_WORDS; - const uint32_t wide_start = wide_base(block); - const uint32_t words = wide ? (wide_start + block * WIDE_WORDS) : slot_end; - std::cout << "lds_ldg: block=" << block << " grid=" << grid << " words=" << words << std::endl; + const uint32_t words = RO_WORDS + block * WIDE_WORDS; + std::cout << "lds_wide: block=" << block << " grid=" << grid << " words=" << words + << " wide=" << wide << std::endl; if (words * sizeof(uint32_t) > lmem_size) { std::cout << "local memory too small\nFAILED!" << std::endl; return 1; @@ -72,20 +71,12 @@ int main(int argc, char** argv) { vx_queue_h q = nullptr; CHECK(vx_queue_create(dev, &qi, &q)); - const uint32_t in_words = num_threads + IN_PAD; - std::vector h_in(in_words); - for (uint32_t i = 0; i < in_words; ++i) { - h_in[i] = in_word(i); - } - - vx_buffer_h in_buf = nullptr; - vx_buffer_h out_buf = nullptr; vx_buffer_h err_buf = nullptr; vx_buffer_h dump_buf = nullptr; - CHECK(vx_buffer_create(dev, in_words * sizeof(uint32_t), VX_MEM_READ, &in_buf)); - CHECK(vx_buffer_create(dev, num_threads * SLOT_WORDS * sizeof(uint32_t), VX_MEM_WRITE, &out_buf)); + vx_buffer_h wide_buf = nullptr; CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); CHECK(vx_buffer_create(dev, grid * words * sizeof(uint32_t), VX_MEM_WRITE, &dump_buf)); + CHECK(vx_buffer_create(dev, sizeof(uint32_t), VX_MEM_WRITE, &wide_buf)); vx_module_h mod = nullptr; vx_kernel_h kern = nullptr; @@ -94,10 +85,9 @@ int main(int argc, char** argv) { kernel_arg_t kernel_arg{}; kernel_arg.lmem_words = words; - CHECK(vx_buffer_address(in_buf, &kernel_arg.in_addr)); - CHECK(vx_buffer_address(out_buf, &kernel_arg.out_addr)); CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); CHECK(vx_buffer_address(dump_buf, &kernel_arg.dump_addr)); + CHECK(vx_buffer_address(wide_buf, &kernel_arg.wide_addr)); vx_launch_info_t li{}; li.struct_size = sizeof(li); @@ -109,20 +99,23 @@ int main(int argc, char** argv) { li.block_dim[0]= block; li.lmem_size = words * sizeof(uint32_t); - std::vector h_out(num_threads * SLOT_WORDS); std::vector h_err(num_threads); std::vector h_dump(grid * words); - vx_event_h write_ev = nullptr; + uint32_t h_wide = ~0u; vx_event_h launch_ev = nullptr; vx_event_h read_ev = nullptr; - CHECK(vx_enqueue_write(q, in_buf, 0, h_in.data(), in_words * sizeof(uint32_t), 0, nullptr, &write_ev)); - CHECK(vx_enqueue_launch(q, &li, 1, &write_ev, &launch_ev)); - CHECK(vx_enqueue_read(q, h_out.data(), out_buf, 0, h_out.size() * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, &h_wide, wide_buf, 0, sizeof(uint32_t), 1, &launch_ev, nullptr)); CHECK(vx_enqueue_read(q, h_dump.data(), dump_buf, 0, h_dump.size() * sizeof(uint32_t), 1, &launch_ev, &read_ev)); CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); int errors = 0; + // the kernel must take the wide path exactly where the ISA has it + if (h_wide != (wide ? 1u : 0u)) { + std::printf("*** kernel wide path=%u, expected %u\n", h_wide, wide ? 1u : 0u); + ++errors; + } for (uint32_t t = 0; t < num_threads; ++t) { if (h_err[t] != 0) { if (errors < 16) { @@ -130,24 +123,11 @@ int main(int argc, char** argv) { } errors += h_err[t]; } - for (uint32_t k = 0; k < SLOT_WORDS; ++k) { - uint32_t actual = h_out[t * SLOT_WORDS + k]; - if (actual != out_word(t, k)) { - if (errors < 16) { - std::printf("*** out[%u][%u] expected=0x%x actual=0x%x\n", t, k, out_word(t, k), actual); - } - ++errors; - } - } } for (uint32_t b = 0; b < grid; ++b) { for (uint32_t i = 0; i < words; ++i) { - if (i >= slot_end && i < wide_start) { - continue; // alignment padding, never written - } uint32_t expected = (i < RO_WORDS) ? ro_word(i) - : (i < slot_end) ? slot_word(b * block + (i - RO_WORDS) / SLOT_WORDS, (i - RO_WORDS) % SLOT_WORDS) - : wide_word(b * block + (i - wide_start) / WIDE_WORDS, (i - wide_start) % WIDE_WORDS); + : wide_word(b * block + (i - RO_WORDS) / WIDE_WORDS, (i - RO_WORDS) % WIDE_WORDS); uint32_t actual = h_dump[b * words + i]; if (actual != expected) { if (errors < 16) { @@ -160,11 +140,9 @@ int main(int argc, char** argv) { vx_event_release(read_ev); vx_event_release(launch_ev); - vx_event_release(write_ev); + vx_buffer_release(wide_buf); vx_buffer_release(dump_buf); vx_buffer_release(err_buf); - vx_buffer_release(out_buf); - vx_buffer_release(in_buf); vx_kernel_release(kern); vx_module_release(mod); vx_queue_release(q); From d24ec378ccd445fa94c78e918016eb2cca97a3d0 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 21:09:58 -0800 Subject: [PATCH 319/336] unittest: drive the full texture lod in the gfx and tex synthesis tops VX_gfx_top and VX_tex_core_top declared tex_req_lod as TEX_LOD_BITS per lane (4 bits), but the texture bus carries TEX_LODF_BITS (12, with the LOD fraction). The 64-bit port zero-extended into the 192-bit field, so lanes 6-15 saw a constant-zero lod and lanes 0-5 misaligned bits. Vivado folded that constant into a texture unit with lanes 6-15's blend arithmetic removed (528 tex_sampler DSPs) until ba381b07d changed the stream buffer's skid enable and the fold stopped; the gate then reported the full unit (768) as growth. The real GPU drives the whole lod (VX_tex_unit's req_sbuf onto tex_bus_if.req_data.lod), so the full unit is what these tops must build. The tex_core smoke bench also failed to compile: it assigned ~0ULL to the narrow cache_req_ready port (-Werror=overflow). It now converts to the port's own type. Verified: tex_core unittest builds and passes; Beacon elaborates and maps tex and gfx with the wider port (tex 896 DSP, gfx 1140, unchanged: Beacon never folded the constant). The gfx and tex FPGA baselines need a human re-record (expected about 992 and 768 DSPs). Co-Authored-By: Claude Opus 5.5 --- hw/unittest/gfx/VX_gfx_top.sv | 2 +- hw/unittest/tex_core/VX_tex_core_top.sv | 2 +- hw/unittest/tex_core/main.cpp | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index a48dcfbe40..db0c7c0166 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -93,7 +93,7 @@ module VX_gfx_top import VX_gpu_pkg::* input wire [NC-1:0] tex_req_valid, input wire [NC-1:0][SFU_LANES-1:0] tex_req_mask, input wire [NC-1:0][1:0][SFU_LANES-1:0][31:0] tex_req_coords, - input wire [NC-1:0][SFU_LANES-1:0][TEX_LOD_BITS-1:0] tex_req_lod, + input wire [NC-1:0][SFU_LANES-1:0][TEX_LODF_BITS-1:0] tex_req_lod, input wire [NC-1:0][TEX_STAGE_BITS-1:0] tex_req_stage, input wire [NC-1:0][TEX_REQ_TAG_WIDTH-1:0] tex_req_tag, output wire [NC-1:0] tex_req_ready, diff --git a/hw/unittest/tex_core/VX_tex_core_top.sv b/hw/unittest/tex_core/VX_tex_core_top.sv index ef2c474918..8457fd6053 100644 --- a/hw/unittest/tex_core/VX_tex_core_top.sv +++ b/hw/unittest/tex_core/VX_tex_core_top.sv @@ -28,7 +28,7 @@ module VX_tex_core_top import VX_gpu_pkg::*; import VX_tex_pkg::*; #( input wire tex_req_valid, input wire [NUM_LANES-1:0] tex_req_mask, input wire [1:0][NUM_LANES-1:0][31:0] tex_req_coords, - input wire [NUM_LANES-1:0][TEX_LOD_BITS-1:0] tex_req_lod, + input wire [NUM_LANES-1:0][TEX_LODF_BITS-1:0] tex_req_lod, input wire [TEX_STAGE_BITS-1:0] tex_req_stage, input wire [TAG_WIDTH-1:0] tex_req_tag, output wire tex_req_ready, diff --git a/hw/unittest/tex_core/main.cpp b/hw/unittest/tex_core/main.cpp index 90f094ba3b..ccec730071 100644 --- a/hw/unittest/tex_core/main.cpp +++ b/hw/unittest/tex_core/main.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "VVX_tex_core_top.h" #include "verilated.h" #ifdef VCD_OUTPUT @@ -49,7 +50,7 @@ int main(int argc, char** argv) { dut->dcr_write_valid = 0; dut->tex_req_valid = 0; dut->tex_rsp_ready = 1; - dut->cache_req_ready = ~0ULL; + dut->cache_req_ready = std::remove_reference_tcache_req_ready)>(~0ULL); dut->cache_rsp_valid = 0; for (int i = 0; i < 4; ++i) clock_cycle(); From 9f116ff7a21fd772696002cf71a47f2155b499f6 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 1 Nov 2026 22:47:37 -0800 Subject: [PATCH 320/336] lsu: wide loads and stores as uop macro-ops (vx_flw.d/.q, vx_fsw.d/.q) The previous wide local load (one issue, several writebacks) taught every stage between issue and writeback about multi-register results: group reservation and per-phase release in the scoreboard, early release in commit, phase counters and response rewriting in the LSU, a sideband and hold bit in the shared VX_mem_scheduler, word-phase replay in local memory, tie-offs in five other memory clients (+636 core LUT). It also worked on local memory only, which pushed address-space separation into the compiler, the kernel header and POCL's address numbering. VX_uop_sequencer exists for this kind of instruction. A wide access is now a macro-op that the packed-load expander turns into 2 or 4 plain flw/fsw uops, so scoreboard, dispatch, LSU, memory and commit see nothing new and each uop goes through the shared address window like any load: - vx_flw.d/.q fd, imm(rs1) (custom-3 funct3 3/4, I-type) and vx_fsw.d/.q fs2, imm(rs1) (funct3 5/6, S-type), defined with F and FLEN 32; any address, 4-byte word alignment. - Decode marks them pack=2'b11, the free pack code, so they share the packed-load sequencer slot instead of adding an ibuffer_t mux input per warp. Uop k ORs k into offset[3:2] and the register: fd/fs2 must be a multiple of 2/4 and imm of 8/16, so the expansion needs no adder (3 LUT per warp in Beacon; a select-based version cost 50). - SimX: the same macro-op expansion in LsuUopGen, in lockstep. - The previous design's pipeline changes and Phase 1's is_local bit are erased: those files are back to fc795c16e, VX_scoreboard.sv keeps c486a1395's per-operand busy check without the early-release term. - sw/kernel: vx_float2/vx_float4 and vx_flw_d/q, vx_fsw_d/q helpers as plain vector loads/stores (an LLVM without the instructions scalarizes them). - Benchmarks: a compile-time mode LDST_WIDE (default 0, today's source) in sgemm, sgemm_v1, sgemmx, sgemm2, sgemm2_v1, sgemm2_dxa and sgemm2_dxa_mcast; the TCU benchmarks' fragment registers are word-strided per lane, which the matrix load/store extension covers. - tests/regression/lds_wide becomes ldst_wide: loads and stores of both widths on local and global memory, lane patterns, offset limits, word- but not 16-byte-aligned addresses, scalar writes into a group, store then wide load. CI: ldst-wide-*, ldst-wide-sgemm*, the DXA wide cases, parity-ldst-wide(-lanes), parity-sgemm2-lmem-wide. Beacon (core): 126,192 LUT / 158,903 FF against 127,319 / 158,980 at the parent; the feature itself is decode +33 LUT and the expander +3 LUT per warp; commit, LSU, mem_scheduler and local memory match fc795c16e; no register deeper (deepest 15). SimX, hand-written sgemm2 over 16 shapes: fetched instructions -15.3%, cycles -1.3% (geomean); retired instructions are unchanged, since each uop still issues. With the new LLVM, LDST_WIDE=1 against pure-scalar mode 0 (cycles): sgemm -4.8%, sgemm2 -2.7%, sgemm_v1 -32%, sgemm2_dxa -36%, sgemm2_dxa_mcast -32% (these three mostly from stepping by 4), sgemm2_v1 +0.2%, sgemmx +3.1% (accumulator spills from hoisted wide loads, a compiler issue being fixed in LLVM). Verified: the CI cases above plus lmem-access, parity-sgemm2-lmem, parity-sgemmx and the packed-load sgemm_tcu cases on simx/rtlsim; make -C tests/regression run-simx; ldst_wide at 16 threads, 4 LSU lanes, SIMD width 1; XLEN 64 simx/rtlsim builds and ldst_wide on the fallback path; known-bad SimX/RTL variants (no +4k, no +k register, no offset OR) fail. Not run: an XLEN 64 build without D, the full rtlsim regression, FPGA/ASIC synthesis. packld_box fails on XLEN 64 rtlsim identically on the parent (pre-existing). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 17 +- ci/testcases/dxa.yaml | 15 + ci/testcases/regression.yaml | 62 +++- .../custom_accelerator_isa_extensions.md | 37 +-- docs/designs/simx_simulator_architecture.md | 2 +- hw/rtl/VX_gpu_pkg.sv | 21 +- hw/rtl/VX_trace_pkg.sv | 20 +- hw/rtl/core/VX_commit_slice.sv | 41 +-- hw/rtl/core/VX_core.sv | 7 +- hw/rtl/core/VX_decode.sv | 47 +-- hw/rtl/core/VX_dispatcher.sv | 2 +- hw/rtl/core/VX_lsu_block.sv | 127 +------- hw/rtl/core/VX_lsu_mem_path.sv | 9 +- hw/rtl/core/VX_lsu_scheduler.sv | 16 - hw/rtl/core/VX_mem_unit.sv | 3 +- hw/rtl/core/VX_scoreboard.sv | 85 +---- hw/rtl/core/VX_uop_packld.sv | 38 ++- hw/rtl/core/VX_uop_sequencer.sv | 2 +- hw/rtl/libs/VX_mem_scheduler.sv | 43 +-- hw/rtl/mem/VX_lmem_engine.sv | 113 +------ hw/rtl/mem/VX_lmem_switch.sv | 29 +- hw/rtl/mem/VX_lmem_unit.sv | 48 +-- hw/rtl/mem/VX_lsu_adapter.sv | 1 - hw/rtl/mem/VX_lsu_mem_arb.sv | 11 +- hw/rtl/mem/VX_lsu_mem_if.sv | 5 +- hw/rtl/om/VX_om_mem.sv | 3 - hw/rtl/raster/VX_raster_earlyz.sv | 3 - hw/rtl/raster/VX_raster_mem.sv | 3 - hw/rtl/tcu/VX_tcu_agu.sv | 1 - hw/rtl/tex/VX_tex_mem.sv | 3 - hw/unittest/issue/VX_issue_top.sv | 1 - hw/unittest/lmem_unit/VX_lmem_unit_top.sv | 12 +- hw/unittest/mem_unit/VX_mem_unit_top.sv | 7 +- hw/unittest/tensor/VX_tensor_top.sv | 7 +- sim/simx/core.cpp | 3 - sim/simx/decode.cpp | 62 ++-- sim/simx/instr_trace.h | 11 - sim/simx/lsu_unit.cpp | 92 ++---- sim/simx/mem/local_mem.cpp | 30 +- sim/simx/scoreboard.cpp | 65 +--- sim/simx/scoreboard.h | 6 - sim/simx/types.h | 9 - sw/kernel/include/vx_intrinsics.h | 30 ++ tests/regression/Makefile | 2 +- tests/regression/lds_wide/common.h | 29 -- tests/regression/lds_wide/kernel.cpp | 237 -------------- .../{lds_wide => ldst_wide}/Makefile | 2 +- tests/regression/ldst_wide/common.h | 47 +++ tests/regression/ldst_wide/kernel.cpp | 298 ++++++++++++++++++ .../{lds_wide => ldst_wide}/main.cpp | 55 +++- tests/regression/sgemm/common.h | 6 + tests/regression/sgemm/kernel.cpp | 25 ++ tests/regression/sgemm2/common.h | 6 + tests/regression/sgemm2/kernel.cpp | 13 + tests/regression/sgemm2_dxa/common.h | 6 + tests/regression/sgemm2_dxa/kernel.cpp | 15 + tests/regression/sgemm2_dxa_mcast/common.h | 6 + tests/regression/sgemm2_dxa_mcast/kernel.cpp | 15 + tests/regression/sgemm2_v1/common.h | 6 + tests/regression/sgemm2_v1/kernel.cpp | 15 + tests/regression/sgemm_v1/common.h | 6 + tests/regression/sgemm_v1/kernel.cpp | 16 + tests/regression/sgemmx/common.h | 6 + tests/regression/sgemmx/kernel.cpp | 69 ++++ 64 files changed, 960 insertions(+), 1069 deletions(-) delete mode 100644 tests/regression/lds_wide/common.h delete mode 100644 tests/regression/lds_wide/kernel.cpp rename tests/regression/{lds_wide => ldst_wide}/Makefile (91%) create mode 100644 tests/regression/ldst_wide/common.h create mode 100644 tests/regression/ldst_wide/kernel.cpp rename tests/regression/{lds_wide => ldst_wide}/main.cpp (66%) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 8159bbb485..8e65c169e3 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -265,17 +265,16 @@ tests: app: lmem_access args: -n16 -s configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 -# Wide local loads: SimX and RTL route and time them alike. -- id: parity-lds-wide +# Wide loads and stores: SimX and RTL expand and time them alike. +- id: parity-ldst-wide check: model_parity via: blackbox - app: lds_wide + app: ldst_wide configs: -DVX_CFG_NUM_THREADS=16 -# Fewer LSU lanes than SIMD lanes: a wide load's phases complete across packets. -- id: parity-lds-wide-lanes +- id: parity-ldst-wide-lanes check: model_parity via: blackbox - app: lds_wide + app: ldst_wide configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_LSU_LANES=4 - id: parity-sgemm2-lmem check: model_parity @@ -289,6 +288,12 @@ tests: app: sgemm2 args: -n64 -t16 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=2 +- id: parity-sgemm2-lmem-wide + check: model_parity + via: blackbox + app: sgemm2 + args: -n32 -t4 + configs: -DVX_CFG_NUM_THREADS=16 -DLDST_WIDE=1 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity diff --git a/ci/testcases/dxa.yaml b/ci/testcases/dxa.yaml index dad905a839..ef8c54d3b4 100644 --- a/ci/testcases/dxa.yaml +++ b/ci/testcases/dxa.yaml @@ -216,6 +216,21 @@ tests: - rtlsim app: sgemm2_dxa configs: -DVX_CFG_EXT_DXA_ENABLE +# The tile reads through the wide loads (LDST_WIDE=1). +- id: sgemm2_dxa-wide + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DLDST_WIDE=1 +- id: sgemm2_dxa_mcast-wide + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_dxa_mcast + configs: -DVX_CFG_EXT_DXA_ENABLE -DLDST_WIDE=1 - id: sgemm2_dxa_mcast-1 via: blackbox drivers: diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index e8196a0060..16cc964ab7 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -62,47 +62,85 @@ tests: configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 shape: threads: 16 -# Wide local loads (vx_flds.d/.q): lane patterns, register groups, the offset -# limits, and local addresses given as a bare offset. -- id: lds-wide-1 +# Wide loads and stores (vx_flw.d/.q, vx_fsw.d/.q) on local and global memory: +# lane patterns, register groups, word alignments and the offset limits. +- id: ldst-wide-1 via: blackbox drivers: - simx - rtlsim - app: lds_wide + app: ldst_wide shape: threads: 16 -# Two LSU blocks: local requests on the second engine. -- id: lds-wide-2 +# Two LSU blocks. +- id: ldst-wide-2 via: blackbox drivers: - simx - rtlsim - app: lds_wide + app: ldst_wide configs: -DVX_CFG_ISSUE_WIDTH=2 shape: threads: 32 warps: 8 -# Fewer LSU lanes than SIMD lanes: a warp's request goes out in four packets. -- id: lds-wide-3 +# Fewer LSU lanes than SIMD lanes: each uop's request goes out in four packets. +- id: ldst-wide-3 via: blackbox drivers: - simx - rtlsim - app: lds_wide + app: ldst_wide configs: -DVX_CFG_NUM_LSU_LANES=4 shape: threads: 16 # One SIMD lane: the dispatcher splits every warp into single-lane packets. -- id: lds-wide-4 +- id: ldst-wide-4 via: blackbox drivers: - simx - rtlsim - app: lds_wide + app: ldst_wide configs: -DVX_CFG_SIMD_WIDTH=1 shape: threads: 4 +# The sgemm benchmarks with each thread's consecutive words read and written +# through the wide forms (LDST_WIDE=1). +- id: ldst-wide-sgemm + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + configs: -DLDST_WIDE=1 +- id: ldst-wide-sgemm-v1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_v1 + configs: -DLDST_WIDE=1 +- id: ldst-wide-sgemmx + via: blackbox + drivers: + - simx + - rtlsim + app: sgemmx + configs: -DLDST_WIDE=1 +- id: ldst-wide-sgemm2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2 + args: -n32 -t4 + configs: -DLDST_WIDE=1 +- id: ldst-wide-sgemm2-v1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_v1 + configs: -DLDST_WIDE=1 - id: occupancy-1 via: blackbox drivers: diff --git a/docs/designs/custom_accelerator_isa_extensions.md b/docs/designs/custom_accelerator_isa_extensions.md index 07f79c7adc..1a84e546de 100644 --- a/docs/designs/custom_accelerator_isa_extensions.md +++ b/docs/designs/custom_accelerator_isa_extensions.md @@ -226,7 +226,6 @@ the table in the same change. | | | `0x02` | tensor core (WMMA/WGMMA) | R | | | | `0x03` | DXA async copy | R | | | | `0x04` | packed loads (`vx_packl*`) | R | -| | | `0x08`–`0x17` | free | | | custom-1 `0x2B` | `funct3` | `0` | `vx_wgather` | R4 | | | | `3` | `vx_om_export` | R4 | | | | `5` | `vx_tex` | R4 | @@ -234,24 +233,22 @@ the table in the same change. | | | `7` | `vx_rt_trace`, `vx_rt_wait` | R4 | | | | `1`, `2`, `4` | free | | | custom-2 `0x5B` | `funct7` | `0x00` | MUFU special functions; `fn` in `rs2`, `fmt` in `funct3` | R | -| custom-3 `0x7B` | `funct3` | `3` | `vx_flds.d fd, imm(rs1)`: wide local load into `fd`, `fd+1` | I | -| | | `4` | `vx_flds.q fd, imm(rs1)`: wide local load into `fd`..`fd+3` | I | -| | | `0`–`2`, `5`–`7` | reserved | | - -`vx_flds.d` and `vx_flds.q` have the layout of the standard `fld`/`flq`: `rd` -is the first FP destination, `rs1` the base, and `imm[11:0]` a signed byte -offset. Each thread loads 8 or 16 bytes from local memory into the aligned -register pair or quad starting at `fd`; `fd` must be a multiple of 2 or 4, and -the address a multiple of 8 or 16. Local memory is addressed by the low -`LMEM_LOG_SIZE` bits of the effective address, so a local pointer may hold the -window address or a bare offset. Standard loads and stores serve every other -local and global access through the generic path. - -The two forms are defined only with FLEN 32 (XLEN 32 with F) and local memory; -elsewhere custom-3 is illegal. One FP register then holds one LSU word, so the -group fills one word per phase on the existing datapath and each phase -releases its register on its own. With FLEN 64 one register already holds 64 -bits and `fld` covers the `.d` case. +| custom-3 `0x7B` | `funct3` | `3` | `vx_flw.d fd, imm(rs1)`: `fd`, `fd+1` ← two words | I | +| | | `4` | `vx_flw.q fd, imm(rs1)`: `fd`..`fd+3` ← four words | I | +| | | `5` | `vx_fsw.d fs2, imm(rs1)`: two words ← `fs2`, `fs2+1` | S | +| | | `6` | `vx_fsw.q fs2, imm(rs1)`: four words ← `fs2`..`fs2+3` | S | +| | | `0`–`2`, `7` | reserved | | + +The wide loads and stores have the layout of `fld`/`flq` and `fsd`/`fsq`, with a +signed 12-bit byte offset. Each is a macro-op of 2 or 4 plain `flw`/`fsw`: uop +*k* moves word *k* at `rs1 + imm + 4k` to or from register `fd + k` (`fs2 + k`), +so the result is exactly that of the scalar sequence, with the same address +windows, exceptions and lane mask, and no atomicity across the words. Each word +needs only 4-byte alignment. The register must be a multiple of 2 (`.d`) or 4 +(`.q`) and `imm` a multiple of 8 or 16, which lets the expander place *k* in the +low bits of both without an adder. An encoding that breaks either rule is +reserved and its result undefined; the assembler rejects it. They are defined +with F and FLEN 32 (no D); elsewhere custom-3 is illegal. --- @@ -279,7 +276,7 @@ a priority encoder in the sequencer: | slot | expander | macro-ops | |---|---|---| -| `UOP_PACKLD` | [`VX_uop_packld.sv`](../../hw/rtl/core/VX_uop_packld.sv) | packed loads | +| `UOP_PACKLD` | [`VX_uop_packld.sv`](../../hw/rtl/core/VX_uop_packld.sv) | packed loads, wide loads and stores | | `UOP_TCU` | [`VX_tcu_uops.sv`](../../hw/rtl/tcu/VX_tcu_uops.sv) | WMMA, WGMMA | | `UOP_GFX` | [`VX_gfx_uops.sv`](../../hw/rtl/core/VX_gfx_uops.sv) | ray-trace windows, fragment export | diff --git a/docs/designs/simx_simulator_architecture.md b/docs/designs/simx_simulator_architecture.md index 9da381bc39..81626299d0 100644 --- a/docs/designs/simx_simulator_architecture.md +++ b/docs/designs/simx_simulator_architecture.md @@ -105,7 +105,7 @@ atomics, threads, locks, or deferred-call plumbing — enforced by **Functional units** - `alu_unit`, `fpu_unit` — own private `execute()` (ISA semantics). -- `lsu_unit` — loads/stores/AMO; `LsuUopGen` for packed loads; pulls load +- `lsu_unit` — loads/stores/AMO; `LsuUopGen` for packed loads and wide loads/stores; pulls load data from `MemRsp::data`. - `sfu_unit` — special-function dispatcher routing to sub-units: WCTL, CSR, TEX, RASTER, DXA, OM. diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index ada4c84a24..a9f7ab8c1e 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1159,10 +1159,10 @@ package VX_gpu_pkg; // words, and VX_gfx_uops emits one uop per set bit. // Zero = not an export. Read ONLY by VX_gfx_uops; the LSU never sees it set // (the expander clears it), so the LSU stays a general-purpose unit. - // is_local marks a wide local load: the access goes to local memory at - // the low bits of its address, whatever the address window says. + // Spends two of the struct's spare padding bits -- INST_ARGS_BITS is + // unchanged (the PACKAGE_ASSERT below proves it). typedef struct packed { - logic is_local; + logic [(INST_ARGS_BITS-1-1-12-2-1-1-$bits(amo_op_e)-2-2)-1:0] __padding; logic [1:0] export_mask; logic amo_valid; logic amo_unsigned; @@ -1436,7 +1436,6 @@ package VX_gpu_pkg; logic [UUID_WIDTH-1:0] uuid; logic [ISSUE_WIS_W-1:0] wis; logic [PER_ISSUE_WARPS-1:0] eop_wis; // one-hot wis, set on eop - logic [PER_ISSUE_WARPS-1:0] rel_wis; // one-hot wis, releases rd ahead of eop logic [NCTA_WIDTH-1:0] cta_id; logic [SIMD_IDX_W-1:0] sid; logic [`VX_CFG_SIMD_WIDTH-1:0] tmask; @@ -1673,21 +1672,7 @@ package VX_gpu_pkg; logic [LSU_CLIENT_TAG_WIDTH-1:0] tag; } lsu_req_data_t; - // Wide local loads (vx_flds.d/.q) fill an aligned FP register group, - // one word-phase at a time. - localparam LDS_WIDE_ENABLE = `VX_CFG_LMEM_ENABLED && `VX_CFG_EXT_F_ENABLED && (`VX_CFG_XLEN == 32); - - // A local response's phase of a wide load. hold marks a phase before the - // last, whose lanes stay outstanding; phase_end marks its last pass. - typedef struct packed { - logic hold; - logic phase_end; - logic [1:0] phase; - } lsu_rsp_side_t; - localparam LSU_RSP_SIDE_W = LDS_WIDE_ENABLE ? $bits(lsu_rsp_side_t) : 0; - typedef struct packed { - logic [`UP(LSU_RSP_SIDE_W)-1:0] side; logic [`VX_CFG_NUM_LSU_LANES-1:0] mask; logic [`VX_CFG_NUM_LSU_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] data; logic [LSU_CLIENT_TAG_WIDTH-1:0] tag; diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index 7fa1deb7ce..f90d56d41b 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -177,20 +177,18 @@ package VX_trace_pkg; 2'd1: begin `TRACE(level, ("PACKLB.F")) end - default: begin + 2'd2: begin `TRACE(level, ("PACKLH.F")) end + default: begin + // a wide access's bytesel offset marks a quad + if (op_args.lsu.is_store) begin + `TRACE(level, ((bytesel != BYTESEL_DEFAULT) ? "VX_FSW.Q" : "VX_FSW.D")) + end else begin + `TRACE(level, ((bytesel != BYTESEL_DEFAULT) ? "VX_FLW.Q" : "VX_FLW.D")) + end + end endcase - end else if (op_args.lsu.is_local) begin - // only wide loads are local: a full-size bytesel with offset 3 - // is a quad, with offset 1 a pair - if (bytesel == BYTESEL_BITS'({XLENB_W'(XLENB-1), XLENB_W'(3)})) begin - `TRACE(level, ("VX_FLDS.Q")) - end else if (bytesel == BYTESEL_BITS'({XLENB_W'(XLENB-1), XLENB_W'(1)})) begin - `TRACE(level, ("VX_FLDS.D")) - end else begin - `TRACE(level, ("?")) - end end else if (op_args.lsu.is_float) begin case (INST_LSU_BITS'(op_type)) INST_LSU_LW: `TRACE(level, ("FLW")) diff --git a/hw/rtl/core/VX_commit_slice.sv b/hw/rtl/core/VX_commit_slice.sv index aa0fb9c200..b2a1133d24 100644 --- a/hw/rtl/core/VX_commit_slice.sv +++ b/hw/rtl/core/VX_commit_slice.sv @@ -30,50 +30,27 @@ module VX_commit_slice import VX_gpu_pkg::*; #( ); `UNUSED_SPARAM (INSTANCE_ID) localparam OUT_DATAW = $bits(commit_t); - localparam REL_W = LDS_WIDE_ENABLE ? PER_ISSUE_WARPS : 0; - localparam ARB_DATAW = PER_ISSUE_WARPS + REL_W + OUT_DATAW; VX_commit_if commit_arb_if(); wire [PER_ISSUE_WARPS-1:0] commit_eop_wis; - wire [PER_ISSUE_WARPS-1:0] commit_rel_wis; wire [NUM_EX_UNITS-1:0] valid_in; - wire [NUM_EX_UNITS-1:0][ARB_DATAW-1:0] data_in; - wire [ARB_DATAW-1:0] data_out; + wire [NUM_EX_UNITS-1:0][PER_ISSUE_WARPS+OUT_DATAW-1:0] data_in; wire [NUM_EX_UNITS-1:0] ready_in; // Decoded ahead of the output register so each warp's release reads - // its own flop. A wide load's beat (full-size bytesel with an odd offset) - // writes a whole register, and its offset bit 1 releases it. + // its own flop. for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_data_in wire [PER_ISSUE_WARPS-1:0] eop_wis = PER_ISSUE_WARPS'(commit_if[j].data.eop) << wid_to_wis(commit_if[j].data.wid); - if (LDS_WIDE_ENABLE && (j == EX_LSU)) begin : g_wide - wire [XLENB_W-1:0] size = commit_if[j].data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; - wire [XLENB_W-1:0] off = commit_if[j].data.bytesel[0 +: XLENB_W]; - wire wide = (size == XLENB_W'(XLENB-1)) && off[0]; - commit_t data; - always @(*) begin - data = commit_if[j].data; - if (wide) begin - data.bytesel = BYTESEL_DEFAULT; - end - end - wire [PER_ISSUE_WARPS-1:0] rel_wis = PER_ISSUE_WARPS'(wide && off[1]) - << wid_to_wis(commit_if[j].data.wid); - assign data_in[j] = {eop_wis, rel_wis, data}; - end else if (LDS_WIDE_ENABLE) begin : g_narrow - assign data_in[j] = {eop_wis, PER_ISSUE_WARPS'(0), commit_if[j].data}; - end else begin : g_no_wide - assign data_in[j] = {eop_wis, commit_if[j].data}; - end assign valid_in[j] = commit_if[j].valid; + assign data_in[j] = {eop_wis, commit_if[j].data}; assign commit_if[j].ready = ready_in[j]; end VX_stream_arb #( .NUM_INPUTS (NUM_EX_UNITS), - .DATAW (ARB_DATAW), + .DATAW (PER_ISSUE_WARPS + OUT_DATAW), .ARBITER ("P"), .OUT_BUF (1) ) commit_arb ( @@ -82,19 +59,12 @@ module VX_commit_slice import VX_gpu_pkg::*; #( .valid_in (valid_in), .ready_in (ready_in), .data_in (data_in), - .data_out (data_out), + .data_out ({commit_eop_wis, commit_arb_if.data}), .valid_out (commit_arb_if.valid), .ready_out (commit_arb_if.ready), `UNUSED_PIN (sel_out) ); - if (LDS_WIDE_ENABLE) begin : g_rel - assign {commit_eop_wis, commit_rel_wis, commit_arb_if.data} = data_out; - end else begin : g_no_rel - assign {commit_eop_wis, commit_arb_if.data} = data_out; - assign commit_rel_wis = '0; - end - // notify scheduler wire [PER_ISSUE_WARPS-1:0] committed_wis_n = commit_arb_if.valid ? commit_eop_wis : '0; `BUFFER(committed_wis, committed_wis_n); @@ -124,7 +94,6 @@ module VX_commit_slice import VX_gpu_pkg::*; #( assign writeback_if.data.uuid = commit_arb_if.data.uuid; assign writeback_if.data.wis = wid_to_wis(commit_arb_if.data.wid); assign writeback_if.data.eop_wis = commit_eop_wis; - assign writeback_if.data.rel_wis = commit_rel_wis; assign writeback_if.data.cta_id = commit_arb_if.data.cta_id; assign writeback_if.data.sid = commit_arb_if.data.sid; assign writeback_if.data.PC = commit_arb_if.data.PC; diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 78d400aca0..26891662aa 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -88,10 +88,9 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty; `ifdef VX_CFG_LMEM_ENABLE VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH), - .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); `endif VX_mem_bus_if #( diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index d146ca1bc3..3f083fe26d 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -222,7 +222,6 @@ module VX_decode import // address of every store in the program. op_args.lsu.export_mask = 2'b00; `endif - op_args.lsu.is_local = 1'b0; case (opcode) INST_I: begin @@ -489,25 +488,37 @@ module VX_decode import `USED_IREG (rs2); `endif end - `ifdef VX_CFG_LMEM_ENABLE `ifdef VX_CFG_EXT_F_ENABLE - // vx_flds.d (funct3 3) and vx_flds.q (funct3 4) load 2 or 4 words - // into an aligned FP register group, one word per beat. A full-size - // bytesel with an odd offset, which no other instruction produces, - // carries the group size. + `ifndef VX_CFG_FLEN_64 + // vx_flw.d/.q and vx_fsw.d/.q: 2 or 4 words of an aligned FP register + // group, expanded into flw/fsw uops; the bytesel offset carries the + // group size. INST_EXT4: begin - if (LDS_WIDE_ENABLE && ((funct3 == 3'h3) || (funct3 == 3'h4))) begin - ex_type = EX_LSU; - op_type = INST_OP_BITS'(INST_LSU_LW); - op_args.lsu.is_store = 0; - op_args.lsu.is_float = 1; - op_args.lsu.pack = 0; - op_args.lsu.is_local = 1; - op_args.lsu.offset = u_12; - bytesel = {XLENB_W'(XLENB-1), XLENB_W'({funct3[2], 1'b1})}; - `USED_IREG (rs1); - `USED_FREG (rd); - end + case (funct3) + 3'h3, 3'h4: begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'(INST_LSU_LW); + op_args.lsu.is_store = 0; + op_args.lsu.is_float = 1; + op_args.lsu.pack = 2'b11; + op_args.lsu.offset = u_12; + bytesel = {XLENB_W'(XLENB-1), XLENB_W'(!funct3[0])}; + `USED_IREG (rs1); + `USED_FREG (rd); + end + 3'h5, 3'h6: begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'(INST_LSU_SW); + op_args.lsu.is_store = 1; + op_args.lsu.is_float = 1; + op_args.lsu.pack = 2'b11; + op_args.lsu.offset = s_imm; + bytesel = {XLENB_W'(XLENB-1), XLENB_W'(!funct3[0])}; + `USED_IREG (rs1); + `USED_FREG (rs2); + end + default:; + endcase end `endif `endif diff --git a/hw/rtl/core/VX_dispatcher.sv b/hw/rtl/core/VX_dispatcher.sv index 148e11aeec..9c17114d77 100644 --- a/hw/rtl/core/VX_dispatcher.sv +++ b/hw/rtl/core/VX_dispatcher.sv @@ -113,7 +113,7 @@ module VX_dispatcher import VX_gpu_pkg::*; #( `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, wis_to_wid(dispatch_if[ex].data.wis, slice_id), dispatch_if[ex].data.cta_id, dispatch_if[ex].data.sid, to_fullPC(dispatch_if[ex].data.PC))) VX_trace_pkg::trace_ex_type(1, ex); `TRACE(1, (", op=")) - VX_trace_pkg::trace_ex_op(1, ex, dispatch_if[ex].data.op_type, dispatch_if[ex].data.op_args, dispatch_if[ex].data.bytesel); + VX_trace_pkg::trace_ex_op(1, ex, dispatch_if[ex].data.op_type, dispatch_if[ex].data.op_args); `TRACE(1, (", tmask=%b, wb=%b, wr_xregs=%b, rd=%0d, rs1_data=", dispatch_if[ex].data.tmask, dispatch_if[ex].data.wb, dispatch_if[ex].data.wr_xregs, dispatch_if[ex].data.rd)) `TRACE_ARRAY1D(1, "0x%0h", dispatch_if[ex].data.rs1_data, `VX_CFG_SIMD_WIDTH) `TRACE(1, (", rs2_data=")) diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index d3c7d3ef53..43f764107e 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -137,27 +137,6 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // carried through the scheduler's USER channel and out via the // lsu_mem_if's per-lane req_data.user field. - // Wide local loads address local memory by the low bits alone, so a bare - // offset that falls in another window is still local. -`ifdef VX_CFG_LMEM_ENABLE - wire req_local = execute_if.data.op_args.lsu.is_local; -`else - wire req_local = 1'b0; - `UNUSED_VAR (execute_if.data.op_args.lsu.is_local) -`endif - - // A wide local load names its group size, as log2 words, in the atomic op - // of a request that is not an atomic. - wire [1:0] req_wide; - if (LDS_WIDE_ENABLE) begin : g_req_wide - wire [XLENB_W-1:0] bytesel_size = execute_if.data.header.bytesel[BYTESEL_BITS-1 -: XLENB_W]; - wire [XLENB_W-1:0] bytesel_off = execute_if.data.header.bytesel[0 +: XLENB_W]; - wire is_wide = (bytesel_size == XLENB_W'(XLENB-1)) && bytesel_off[0]; - assign req_wide = is_wide ? (bytesel_off[1] ? 2'd2 : 2'd1) : 2'd0; - end else begin : g_no_req_wide - assign req_wide = 2'd0; - end - mem_bus_attr_t [NUM_LANES-1:0] mem_req_attr_struct; wire [NUM_LANES-1:0][MEM_ATTR_WIDTH-1:0] mem_req_attr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_mem_req_attr @@ -180,7 +159,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // the store off the L1->L2 trunk on that bit alone. wire [MEM_ADDRW-1:0] om_addr_start = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_OM_BASE_ADDR) >> MEM_ASHIFT); wire [MEM_ADDRW-1:0] om_addr_end = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_OM_END_ADDR) >> MEM_ASHIFT); - wire lane_is_om = (block_addr >= om_addr_start) && (block_addr < om_addr_end) && ~req_local; + wire lane_is_om = (block_addr >= om_addr_start) && (block_addr < om_addr_end); assign mem_req_attr_struct[i].is_addr_om = lane_is_om; `else wire lane_is_om = 1'b0; @@ -189,20 +168,19 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // a fence's token carries its kind: io for a fence ordering only device // accesses, and for an acquire fence's (byte enables set: no drain owed) assign mem_req_attr_struct[i].is_addr_io = req_is_fence ? (req_fence_io || req_fence_acq) - : (((block_addr >= io_addr_start) && (block_addr < io_addr_end) && ~req_local) + : (((block_addr >= io_addr_start) && (block_addr < io_addr_end)) || lane_is_om); `ifdef VX_CFG_LMEM_ENABLE // is local memory address - assign mem_req_attr_struct[i].is_addr_local = req_local - || (block_addr[MEM_ADDRW-1:LMEM_BLK_LSB] == LMEM_BLK_BASE[MEM_ADDRW-1:LMEM_BLK_LSB]); + assign mem_req_attr_struct[i].is_addr_local = + (block_addr[MEM_ADDRW-1:LMEM_BLK_LSB] == LMEM_BLK_BASE[MEM_ADDRW-1:LMEM_BLK_LSB]); `else assign mem_req_attr_struct[i].is_addr_local = 1'b0; `endif `ifdef VX_CFG_EXT_A_ENABLE amo_req_t lane_amo; assign lane_amo.amo_valid = execute_if.data.op_args.lsu.amo_valid; - assign lane_amo.amo_op = (LDS_WIDE_ENABLE && ~execute_if.data.op_args.lsu.amo_valid) - ? amo_op_e'($bits(amo_op_e)'(req_wide)) : execute_if.data.op_args.lsu.amo_op; + assign lane_amo.amo_op = execute_if.data.op_args.lsu.amo_op; // amo_unsigned distinguishes signed/unsigned AMOMIN/MAX variants // (decoder collapses MINU/MAXU into MIN/MAX + this bit). assign lane_amo.amo_unsigned = execute_if.data.op_args.lsu.amo_unsigned; @@ -221,12 +199,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `else // EXT_A disabled: tie the AMO sideband to zero so the bits don't // propagate as 'x under --x-assign unique. - amo_req_t lane_amo; - always @(*) begin - lane_amo = '0; - lane_amo.amo_op = amo_op_e'($bits(amo_op_e)'(req_wide)); - end - assign mem_req_attr_struct[i].amo = lane_amo; + assign mem_req_attr_struct[i].amo = '0; `endif assign mem_req_attr[i] = mem_req_attr_struct[i]; end @@ -250,13 +223,6 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire mem_rsp_eop; wire mem_rsp_ready; - // a wide load response's phase, and whether it completes that phase for - // the whole instruction, which releases the phase's register - wire mem_rsp_hold; - wire mem_rsp_phase_end; - wire [1:0] mem_rsp_phase; - wire mem_rsp_phase_rel; - wire mem_req_fire = mem_req_valid && mem_req_ready; wire mem_rsp_fire = mem_rsp_valid && mem_rsp_ready; @@ -358,20 +324,6 @@ module VX_lsu_block import VX_gpu_pkg::*; #( $time, execute_if.data.header.wid, to_fullPC(execute_if.data.header.PC), full_addr[i], inst_lsu_wsize(execute_if.data.op_type), execute_if.data.header.uuid)) end - // A wide load's lanes address whole groups and fill an aligned register group. - if (LDS_WIDE_ENABLE) begin : g_wide_align - wire lsu_req_fire = execute_if.valid && execute_if.ready; - wire [3:0] group_amask = req_wide[1] ? 4'hf : 4'h7; - for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lanes - `RUNTIME_ASSERT((~lsu_req_fire || (req_wide == 2'd0) || ~execute_if.data.header.tmask[i] || ((full_addr[i][3:0] & group_amask) == 4'h0)), - ("%t: misaligned wide local load, wid=%0d, PC=0x%0h, addr=0x%0h! (#%0d)", - $time, execute_if.data.header.wid, to_fullPC(execute_if.data.header.PC), full_addr[i], execute_if.data.header.uuid)) - end - `RUNTIME_ASSERT((~lsu_req_fire || (req_wide == 2'd0) || ((execute_if.data.header.rd[1:0] & {req_wide[1], 1'b1}) == 2'b00)), - ("%t: misaligned wide local load register group, wid=%0d, PC=0x%0h, rd=%0d! (#%0d)", - $time, execute_if.data.header.wid, to_fullPC(execute_if.data.header.PC), execute_if.data.header.rd, execute_if.data.header.uuid)) - end - // store data formatting for (genvar i = 0; i < NUM_LANES; ++i) begin : g_mem_req_data always @(*) begin @@ -451,35 +403,6 @@ module VX_lsu_block import VX_gpu_pkg::*; #( assign mem_rsp_sop_pkt = pkt_sop[pkt_raddr]; assign mem_rsp_eop_pkt = mem_rsp_eop && pkt_eop[pkt_raddr] && (pkt_ctr[pkt_raddr] == 1); - - // A wide load's packets serve their phases independently, so a phase - // before the last is complete once every packet has served it. - if (LDS_WIDE_ENABLE) begin : g_phase_ctr - reg [`VX_CFG_LSU_PENDING_SIZE-1:0][2:0][PID_BITS:0] phase_ctr; - wire phase_dec = mem_rsp_fire && mem_rsp_hold && mem_rsp_phase_end; - - always @(posedge clk) begin - for (integer s = 0; s < `VX_CFG_LSU_PENDING_SIZE; ++s) begin - for (integer k = 0; k < 3; ++k) begin - if (reset) begin - phase_ctr[s][k] <= '0; - end else if (mem_req_rd_sop_fire && (pkt_waddr == LSUQ_SIZEW'(s))) begin - phase_ctr[s][k] <= (PID_BITS+1)'(1); - end else begin - phase_ctr[s][k] <= phase_ctr[s][k] - + (PID_BITS+1)'(mem_req_rd_fire && (pkt_waddr == LSUQ_SIZEW'(s))) - - (PID_BITS+1)'(phase_dec && (pkt_raddr == LSUQ_SIZEW'(s)) && (mem_rsp_phase == 2'(k))); - end - end - end - end - - wire [PID_BITS:0] rsp_phase_ctr = phase_ctr[pkt_raddr][(mem_rsp_phase == 2'd3) ? 2'd0 : mem_rsp_phase]; - assign mem_rsp_phase_rel = mem_rsp_phase_end - && (mem_rsp_hold ? (pkt_eop[pkt_raddr] && (rsp_phase_ctr == 1)) : mem_rsp_eop_pkt); - end else begin : g_no_phase_ctr - assign mem_rsp_phase_rel = 1'b0; - end `RUNTIME_ASSERT(~(mem_req_rd_fire && full), ("allocator full!")) `RUNTIME_ASSERT(~(mem_req_rd_sop_fire && pkt_ctr[pkt_waddr] != 0), ("oops! broken sop request!")) // pkt_ctr is guarded against a same-cycle same-slot request/response by @@ -492,9 +415,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( assign pkt_waddr = 0; assign mem_rsp_sop_pkt = mem_rsp_sop; assign mem_rsp_eop_pkt = mem_rsp_eop; - assign mem_rsp_phase_rel = mem_rsp_phase_end; `UNUSED_VAR (pkt_raddr) - `UNUSED_VAR (mem_rsp_hold) end // pack memory request tag @@ -552,19 +473,6 @@ module VX_lsu_block import VX_gpu_pkg::*; #( assign mem_rsp_eop = client_if.rsp_data.eop; assign client_if.rsp_ready = mem_rsp_ready; - if (LDS_WIDE_ENABLE) begin : g_rsp_side - lsu_rsp_side_t rsp_side; - assign rsp_side = lsu_rsp_side_t'(client_if.rsp_data.side); - assign mem_rsp_hold = rsp_side.hold; - assign mem_rsp_phase_end = rsp_side.phase_end; - assign mem_rsp_phase = rsp_side.phase; - end else begin : g_no_rsp_side - assign mem_rsp_hold = 1'b0; - assign mem_rsp_phase_end = 1'b0; - assign mem_rsp_phase = 2'd0; - `UNUSED_VAR (client_if.rsp_data.side) - end - lsu_header_t rsp_hdr; wire [INST_LSU_BITS-1:0] rsp_op_type; wire [NUM_LANES-1:0][REQ_ASHIFT-1:0] rsp_align; @@ -629,35 +537,12 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // result - // A wide load's beat writes the register of its phase; its bytesel - // offset bit 1 says whether the beat releases that register. - wire rsp_wide; - wire rsp_quad; - if (LDS_WIDE_ENABLE) begin : g_rsp_wide - wire [XLENB_W-1:0] bytesel_size = rsp_hdr.bytesel[BYTESEL_BITS-1 -: XLENB_W]; - wire [XLENB_W-1:0] bytesel_off = rsp_hdr.bytesel[0 +: XLENB_W]; - assign rsp_wide = (bytesel_size == XLENB_W'(XLENB-1)) && bytesel_off[0]; - assign rsp_quad = bytesel_off[1]; - end else begin : g_no_rsp_wide - assign rsp_wide = 1'b0; - assign rsp_quad = 1'b0; - `UNUSED_VAR ({mem_rsp_hold, mem_rsp_phase_end, mem_rsp_phase_rel}) - end - lsu_header_t rsp_hdr2; always @(*) begin rsp_hdr2 = rsp_hdr; rsp_hdr2.tmask = mem_rsp_mask; rsp_hdr2.sop = mem_rsp_sop_pkt; rsp_hdr2.eop = mem_rsp_eop_pkt; - if (rsp_wide) begin - if (rsp_quad) begin - rsp_hdr2.rd[1:0] = mem_rsp_phase; - end else begin - rsp_hdr2.rd[0] = mem_rsp_phase[0]; - end - rsp_hdr2.bytesel = {XLENB_W'(XLENB-1), XLENB_W'({mem_rsp_phase_rel, 1'b1})}; - end end VX_elastic_buffer #( diff --git a/hw/rtl/core/VX_lsu_mem_path.sv b/hw/rtl/core/VX_lsu_mem_path.sv index c04a458dc8..6cef8767b3 100644 --- a/hw/rtl/core/VX_lsu_mem_path.sv +++ b/hw/rtl/core/VX_lsu_mem_path.sv @@ -49,10 +49,9 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH), - .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) ) lsu_mem_if(); // scheduler: client 0 is the block's lanes, client 1 the TCU metadata AGU @@ -207,8 +206,6 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( .out_rsp_tag (dcache_coalesced_if.rsp_data.tag), .out_rsp_ready (dcache_coalesced_if.rsp_ready) ); - assign lsu_dcache_if.rsp_data.side = '0; - `UNUSED_VAR (dcache_coalesced_if.rsp_data.side) end else begin : g_no_coalescing diff --git a/hw/rtl/core/VX_lsu_scheduler.sv b/hw/rtl/core/VX_lsu_scheduler.sv index bba1047cbe..f093e8cd7e 100644 --- a/hw/rtl/core/VX_lsu_scheduler.sv +++ b/hw/rtl/core/VX_lsu_scheduler.sv @@ -68,7 +68,6 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( wire [SCHED_TAG_WIDTH-1:0] sched_rsp_tag; wire sched_rsp_sop; wire sched_rsp_eop; - wire [`UP(LSU_RSP_SIDE_W)-1:0] sched_rsp_side; wire sched_rsp_ready; // Flatten interface array into per-signal vectors so the arbiter @@ -110,7 +109,6 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign cli_rsp_data[0].tag = sched_rsp_tag; assign cli_rsp_data[0].sop = sched_rsp_sop; assign cli_rsp_data[0].eop = sched_rsp_eop; - assign cli_rsp_data[0].side = sched_rsp_side; assign sched_rsp_ready = cli_rsp_ready[0]; end else begin : g_arb // Round-robin priority. rr_ptr rotates after every accepted request. @@ -188,7 +186,6 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign cli_rsp_data[i].tag = sched_rsp_tag[LSU_CLIENT_TAG_WIDTH-1:0]; assign cli_rsp_data[i].sop = sched_rsp_sop; assign cli_rsp_data[i].eop = sched_rsp_eop; - assign cli_rsp_data[i].side = sched_rsp_side; end // sched_rsp_ready is driven by the granted client's ready. logic sched_rsp_ready_l; @@ -218,7 +215,6 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0] lsu_mem_rsp_mask; wire [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] lsu_mem_rsp_data; wire [LSU_TAG_WIDTH-1:0] lsu_mem_rsp_tag; - wire lsu_mem_rsp_hold; wire lsu_mem_rsp_ready; wire sched_req_queue_empty; @@ -237,7 +233,6 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( .PENDING_SIZE (PENDING_SIZE), .UUID_WIDTH (UUID_WIDTH), .RSP_PARTIAL (1), - .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W), .MEM_OUT_BUF (0), .CORE_OUT_BUF(0), .LUTRAM (1) @@ -265,7 +260,6 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( .core_rsp_tag (sched_rsp_tag), .core_rsp_sop (sched_rsp_sop), .core_rsp_eop (sched_rsp_eop), - .core_rsp_side (sched_rsp_side), .core_rsp_ready (sched_rsp_ready), .mem_req_valid (lsu_mem_req_valid), @@ -282,8 +276,6 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( .mem_rsp_mask (lsu_mem_rsp_mask), .mem_rsp_data (lsu_mem_rsp_data), .mem_rsp_tag (lsu_mem_rsp_tag), - .mem_rsp_hold (lsu_mem_rsp_hold), - .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_rsp_ready) ); @@ -304,14 +296,6 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign lsu_mem_rsp_mask = lsu_mem_if.rsp_data.mask; assign lsu_mem_rsp_data = lsu_mem_if.rsp_data.data; assign lsu_mem_rsp_tag = lsu_mem_if.rsp_data.tag; - if (LSU_RSP_SIDE_W != 0) begin : g_rsp_hold - lsu_rsp_side_t rsp_side; - assign rsp_side = lsu_rsp_side_t'(lsu_mem_if.rsp_data.side); - assign lsu_mem_rsp_hold = rsp_side.hold; - `UNUSED_VAR (rsp_side) - end else begin : g_no_rsp_hold - assign lsu_mem_rsp_hold = 1'b0; - end assign lsu_mem_if.rsp_ready = lsu_mem_rsp_ready; assign empty = sched_req_queue_empty; diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 029ac0f27f..6c4f5d09b2 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -175,8 +175,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .TAG_WIDTH (LSU_TAG_WIDTH), .DMA_ENABLE (LMEM_DMA_EN), .DMA_TAG_WIDTH (LMEM_DMA_TAG_WIDTH), - .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), - .WIDE_ENABLE (LDS_WIDE_ENABLE) + .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) lmem_unit ( .clk (clk), .reset (reset), diff --git a/hw/rtl/core/VX_scoreboard.sv b/hw/rtl/core/VX_scoreboard.sv index 65e94e6ccd..9c1349c249 100644 --- a/hw/rtl/core/VX_scoreboard.sv +++ b/hw/rtl/core/VX_scoreboard.sv @@ -103,7 +103,6 @@ module VX_scoreboard import VX_gpu_pkg::*; #( wire staging_fire = staging_if[w].valid && staging_if[w].ready; wire writeback_fire = writeback_if.valid && writeback_if.data.eop_wis[w]; - wire writeback_rel = writeback_if.valid && writeback_if.data.rel_wis[w]; wire [NUM_OPDS-1:0] [NUM_REGS_BITS-1:0] ibf_opds, stg_opds; assign ibf_opds = {ibuffer_if[w].data.rs3, ibuffer_if[w].data.rs2, ibuffer_if[w].data.rs1, ibuffer_if[w].data.rd}; @@ -116,24 +115,6 @@ module VX_scoreboard import VX_gpu_pkg::*; #( wire [NUM_XREGS-1:0] ibf_xregs_mask = ibuffer_if[w].data.rd_xregs | ibuffer_if[w].data.wr_xregs; wire [NUM_XREGS-1:0] stg_xregs_mask = staging_if[w].data.rd_xregs | staging_if[w].data.wr_xregs; - // A wide load's rd names an aligned group of two or four registers. - wire ibf_wide, ibf_quad, stg_wide, stg_quad; - if (LDS_WIDE_ENABLE) begin : g_wide - wire [XLENB_W-1:0] ibf_size = ibuffer_if[w].data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; - wire [XLENB_W-1:0] ibf_off = ibuffer_if[w].data.bytesel[0 +: XLENB_W]; - wire [XLENB_W-1:0] stg_size = staging_if[w].data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; - wire [XLENB_W-1:0] stg_off = staging_if[w].data.bytesel[0 +: XLENB_W]; - assign ibf_wide = (ibf_size == XLENB_W'(XLENB-1)) && ibf_off[0]; - assign ibf_quad = ibf_wide && ibf_off[1]; - assign stg_wide = (stg_size == XLENB_W'(XLENB-1)) && stg_off[0]; - assign stg_quad = stg_wide && stg_off[1]; - end else begin : g_no_wide - assign ibf_wide = 1'b0; - assign ibf_quad = 1'b0; - assign stg_wide = 1'b0; - assign stg_quad = 1'b0; - end - wire [NUM_OPDS-1:0][REG_TYPES-1:0][RV_REGS-1:0] ibf_opd_mask, stg_opd_mask; for (genvar i = 0; i < NUM_OPDS; ++i) begin : g_opd_masks @@ -143,31 +124,17 @@ module VX_scoreboard import VX_gpu_pkg::*; #( end end - // the registers the staging instr reserves: its rd, or rd's group - wire [NUM_REGS-1:0] stg_rsv_mask; - if (LDS_WIDE_ENABLE) begin : g_rsv_group - for (genvar r = 0; r < NUM_REGS; ++r) begin : g_r - localparam [NUM_REGS_BITS-1:0] R = NUM_REGS_BITS'(r); - assign stg_rsv_mask[r] = stg_used_rs[0] - && (stg_opds[0][NUM_REGS_BITS-1:2] == R[NUM_REGS_BITS-1:2]) - && (stg_quad || (stg_opds[0][1] == R[1])) - && (stg_wide || (stg_opds[0][0] == R[0])); - end - end else begin : g_rsv_single - assign stg_rsv_mask = stg_opd_mask[0]; - end - - // Writeback release feeds wb_inuse_regs, which the busy check reads; the - // staging reserve is added on top to form the next state, inuse_regs_n. + // Writeback release feeds wb_inuse_regs; the staging reserve is added on + // top to form the next state, inuse_regs_n. reg [NUM_REGS-1:0] wb_inuse_regs; reg [NUM_XREGS-1:0] wb_inuse_xregs; always @(*) begin wb_inuse_regs = inuse_regs; wb_inuse_xregs = inuse_xregs; - if ((writeback_fire || writeback_rel) && writeback_if.data.wb) begin - wb_inuse_regs[writeback_if.data.rd] = 0; // release rd - end if (writeback_fire) begin + if (writeback_if.data.wb) begin + wb_inuse_regs[writeback_if.data.rd] = 0; // release rd + end wb_inuse_xregs &= ~writeback_if.data.wr_xregs; // release special regs end end @@ -177,7 +144,7 @@ module VX_scoreboard import VX_gpu_pkg::*; #( inuse_xregs_n = wb_inuse_xregs; if (staging_fire) begin if (staging_if[w].data.wb) begin - inuse_regs_n |= stg_rsv_mask; // reserve rd or its group + inuse_regs_n |= stg_opd_mask[0]; // reserve rd end inuse_xregs_n |= staging_if[w].data.wr_xregs; // reserve special regs end @@ -199,32 +166,10 @@ module VX_scoreboard import VX_gpu_pkg::*; #( // once and the outcomes select bits. wire [NUM_REGS-1:0] ibf_regs_mask = ibf_opd_mask[0] | ibf_opd_mask[1] | ibf_opd_mask[2] | ibf_opd_mask[3]; wire [NUM_REGS-1:0] stg_regs_mask = stg_opd_mask[0] | stg_opd_mask[1] | stg_opd_mask[2] | stg_opd_mask[3]; - wire ibf_group_busy, stg_group_busy; - if (LDS_WIDE_ENABLE) begin : g_group_busy - // A destination group's other registers are tested against the - // registered in-use set, reduced per pair and quad ahead, off the - // operand masks' reduction. Missing a release in this cycle only - // delays the group by a cycle. - wire [NUM_REGS/2-1:0] inuse_pairs; - wire [NUM_REGS/4-1:0] inuse_quads; - for (genvar g = 0; g < NUM_REGS/2; ++g) begin : g_pairs - assign inuse_pairs[g] = | inuse_regs[2*g +: 2]; - end - for (genvar g = 0; g < NUM_REGS/4; ++g) begin : g_quads - assign inuse_quads[g] = | inuse_regs[4*g +: 4]; - end - assign ibf_group_busy = ibf_quad ? inuse_quads[ibf_opds[0][NUM_REGS_BITS-1:2]] - : (ibf_wide && inuse_pairs[ibf_opds[0][NUM_REGS_BITS-1:1]]); - assign stg_group_busy = stg_quad ? inuse_quads[stg_opds[0][NUM_REGS_BITS-1:2]] - : (stg_wide && inuse_pairs[stg_opds[0][NUM_REGS_BITS-1:1]]); - end else begin : g_no_group_busy - assign ibf_group_busy = 1'b0; - assign stg_group_busy = 1'b0; - end // Each operand reads its in-use bit by index and compares its index with // this cycle's release, which is shallower than reducing the operand // masks against the whole released set. - wire wb_release = (writeback_fire || writeback_rel) && writeback_if.data.wb; + wire wb_release = writeback_fire && writeback_if.data.wb; wire [NUM_OPDS-1:0] ibf_opd_busy, stg_opd_busy; for (genvar i = 0; i < NUM_OPDS; ++i) begin : g_opd_busy assign ibf_opd_busy[i] = ibf_used_rs[i] && inuse_regs[ibf_opds[i]] @@ -232,23 +177,17 @@ module VX_scoreboard import VX_gpu_pkg::*; #( assign stg_opd_busy[i] = stg_used_rs[i] && inuse_regs[stg_opds[i]] && ~(wb_release && (writeback_if.data.rd == stg_opds[i])); end - wire ibf_busy = (| ibf_opd_busy) || ibf_group_busy; - wire stg_busy = (| stg_opd_busy) || stg_group_busy; + wire ibf_busy = | ibf_opd_busy; + wire stg_busy = | stg_opd_busy; wire ibf_xbusy = | (wb_inuse_xregs & ibf_xregs_mask); wire stg_xbusy = | (wb_inuse_xregs & stg_xregs_mask); // On fire the staging instr also reserves its rd and wr_xregs. Its // successor sees that reservation only through a register it names, - // and a lone staging instr always names its own rd. A register group - // matches any register in it. + // and a lone staging instr always names its own rd. wire [NUM_OPDS-1:0] ibf_reads_rd; for (genvar i = 0; i < NUM_OPDS; ++i) begin : g_ibf_reads_rd - wire any_bit1 = stg_quad || ((i == 0) && ibf_quad); - wire any_bit0 = stg_wide || ((i == 0) && ibf_wide); - assign ibf_reads_rd[i] = ibf_used_rs[i] - && (ibf_opds[i][NUM_REGS_BITS-1:2] == stg_opds[0][NUM_REGS_BITS-1:2]) - && (any_bit1 || (ibf_opds[i][1] == stg_opds[0][1])) - && (any_bit0 || (ibf_opds[i][0] == stg_opds[0][0])); + assign ibf_reads_rd[i] = ibf_used_rs[i] && (ibf_opds[i] == stg_opds[0]); end wire stg_rsv_rd = staging_if[w].data.wb; wire ibf_rsv = (stg_rsv_rd && (| ibf_reads_rd)) @@ -327,7 +266,7 @@ module VX_scoreboard import VX_gpu_pkg::*; #( w, to_fullPC(staging_if[w].data.PC), staging_if[w].data.tmask, timeout_ctr, operands_busy, staging_if[w].data.uuid)) - `RUNTIME_ASSERT(~((writeback_fire || writeback_rel) && writeback_if.data.wb) || inuse_regs[writeback_if.data.rd] != 0, + `RUNTIME_ASSERT(~(writeback_fire && writeback_if.data.wb) || inuse_regs[writeback_if.data.rd] != 0, ("invalid writeback register: wid=%0d, PC=0x%0h, tmask=%b, rd=%0d (#%0d)", w, to_fullPC(writeback_if.data.PC), writeback_if.data.tmask, writeback_if.data.rd, writeback_if.data.uuid)) diff --git a/hw/rtl/core/VX_uop_packld.sv b/hw/rtl/core/VX_uop_packld.sv index 9ec971e633..3f77222299 100644 --- a/hw/rtl/core/VX_uop_packld.sv +++ b/hw/rtl/core/VX_uop_packld.sv @@ -13,18 +13,20 @@ `include "VX_define.vh" -// VX_uop_packld: Micro-op sequencer for pack-load instructions. +// VX_uop_packld: Micro-op sequencer for pack-load and wide load/store instructions. // // Expands one vx_packlb_f / vx_packlh_f instruction into N sequential LSU uops: // pack==1 (PACKLB) → 4 uops of INST_LSU_LBU (unsigned byte loads) // pack==2 (PACKLH) → 2 uops of INST_LSU_LHU (unsigned halfword loads) +// pack==3 (vx_flw.d/.q, vx_fsw.d/.q) → 2 or 4 plain flw/fsw uops // -// Each uop encodes its index in op_args.lsu.offset[1:0]. -// The dispatcher reads pack (non-zero) to identify pack-load uops and computes: +// A pack-load uop encodes its index in op_args.lsu.offset[1:0]; the LSU AGU +// reads pack (non-zero) to identify it and computes // eff_rs1[lane] = rs1[lane] + rs2[lane] * uop_idx (shift-and-add) -// The pack field is cleared in op_args forwarded to the LSU slice so it sees a -// plain LBU or LHU. The NaN-boxing of the FLW result is handled automatically -// because is_float==1 and rd is a float register (triggers nan-box in lsu_block). +// A wide uop carries word k in offset[3:2] and register fd+k / fs2+k, and +// leaves with pack cleared, so the LSU sees a plain flw/fsw. The NaN-boxing of +// a loaded word is handled automatically because is_float==1 and rd is a float +// register (triggers nan-box in lsu_block). module VX_uop_packld import VX_gpu_pkg::*; ( input clk, @@ -41,18 +43,32 @@ module VX_uop_packld import VX_gpu_pkg::*; ( `UNUSED_VAR ({clk, reset, start, advance, uop_idx}) // pack==1 → PACKLB: 4 byte-load uops; - // pack==2 → PACKLH: 2 halfword-load uops + // pack==2 → PACKLH: 2 halfword-load uops; + // pack==3 → wide: 4 uops if the bytesel offset marks a quad, else 2 + wire is_packlb = (ibuf_in.op_args.lsu.pack == 2'b01); wire is_packlh = (ibuf_in.op_args.lsu.pack == 2'b10); - assign uop_count = is_packlh ? UOP_CTR_W'(2) : UOP_CTR_W'(4); + wire is_wide = (ibuf_in.op_args.lsu.pack == 2'b11); + wire is_pair = is_wide && ~ibuf_in.bytesel[0]; + assign uop_count = (is_packlh || is_pair) ? UOP_CTR_W'(2) : UOP_CTR_W'(4); wire [1:0] idx2 = uop_idx[1:0]; - wire [XLENB_W-1:0] byte_size = is_packlh ? '1 : '0; - wire [XLENB_W-1:0] byte_off = is_packlh ? XLENB_W'({idx2[0], 1'b0}) : XLENB_W'(idx2); + wire [XLENB_W-1:0] byte_size = ibuf_in.op_args.lsu.pack[1] ? '1 : '0; + wire [XLENB_W-1:0] byte_off = is_packlb ? XLENB_W'(idx2) : (is_packlh ? XLENB_W'({idx2[0], 1'b0}) : '0); + + // Decode leaves a packed load's offset zero, and a wide access's offset and + // register have their low group bits clear, so each uop ORs its index in: + // into offset[1:0] for a packed load; into offset[3:2] (word k) and the + // register (fd+k, fs2+k) for a wide access, which becomes a plain flw/fsw. + wire [1:0] wide_idx = is_wide ? idx2 : 2'b00; + wire [1:0] pack_idx = is_wide ? 2'b00 : idx2; ibuffer_t ibuf_r; always_comb begin ibuf_r = ibuf_in; - ibuf_r.op_args.lsu.offset = {10'd0, idx2}; + ibuf_r.op_args.lsu.pack = is_wide ? 2'b00 : ibuf_in.op_args.lsu.pack; + ibuf_r.op_args.lsu.offset = ibuf_in.op_args.lsu.offset | {8'd0, wide_idx, pack_idx}; + ibuf_r.rd = ibuf_in.rd | NUM_REGS_BITS'(wide_idx); + ibuf_r.rs2 = ibuf_in.rs2 | NUM_REGS_BITS'(wide_idx); ibuf_r.bytesel = {byte_size, byte_off}; end assign ibuf_out = ibuf_r; diff --git a/hw/rtl/core/VX_uop_sequencer.sv b/hw/rtl/core/VX_uop_sequencer.sv index f91d9ec6f0..2b888a8cce 100644 --- a/hw/rtl/core/VX_uop_sequencer.sv +++ b/hw/rtl/core/VX_uop_sequencer.sv @@ -234,7 +234,7 @@ module VX_uop_sequencer import `TRACE(1, ("%t: %s decode: wid=%0d, cta_id=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, WARP_ID, output_if.data.cta_id, to_fullPC(output_if.data.PC))) VX_trace_pkg::trace_ex_type(1, output_if.data.ex_type); `TRACE(1, (", op=")) - VX_trace_pkg::trace_ex_op(1, output_if.data.ex_type, output_if.data.op_type, output_if.data.op_args, output_if.data.bytesel); + VX_trace_pkg::trace_ex_op(1, output_if.data.ex_type, output_if.data.op_type, output_if.data.op_args); `TRACE(1, (", tmask=%b, wb=%b, rd_xregs=%b, wr_xregs=%b, used_rs=%b, rd=", output_if.data.tmask, output_if.data.wb, output_if.data.rd_xregs, output_if.data.wr_xregs, output_if.data.used_rs)) VX_trace_pkg::trace_reg_idx(1, output_if.data.rd); `TRACE(1, (", rs1=")) diff --git a/hw/rtl/libs/VX_mem_scheduler.sv b/hw/rtl/libs/VX_mem_scheduler.sv index 11b3d65b10..5ac1bca106 100644 --- a/hw/rtl/libs/VX_mem_scheduler.sv +++ b/hw/rtl/libs/VX_mem_scheduler.sv @@ -43,10 +43,6 @@ module VX_mem_scheduler #( // all feeding the one output register -- which then cannot be placed near // its own data. Distributed RAM lands in the slices that hold that register. parameter REQQ_LUTRAM = 0, - // Nonzero: a memory response carries a side word to its core response, and - // a hold bit that keeps the lanes it serves outstanding. Needs RSP_PARTIAL - // and no coalescing. - parameter RSP_SIDE_WIDTH = 0, parameter WORD_WIDTH = WORD_SIZE * 8, parameter LINE_WIDTH = LINE_SIZE * 8, @@ -89,7 +85,6 @@ module VX_mem_scheduler #( output wire [TAG_WIDTH-1:0] core_rsp_tag, output wire core_rsp_sop, output wire core_rsp_eop, - output wire [`UP(RSP_SIDE_WIDTH)-1:0] core_rsp_side, input wire core_rsp_ready, // Memory request @@ -108,8 +103,6 @@ module VX_mem_scheduler #( input wire [MEM_CHANNELS-1:0] mem_rsp_mask, input wire [MEM_CHANNELS-1:0][LINE_WIDTH-1:0] mem_rsp_data, input wire [MEM_TAG_WIDTH-1:0] mem_rsp_tag, - input wire mem_rsp_hold, - input wire [`UP(RSP_SIDE_WIDTH)-1:0] mem_rsp_side, output wire mem_rsp_ready ); localparam BATCH_SEL_WIDTH = `UP(MEM_BATCH_BITS); @@ -124,7 +117,6 @@ module VX_mem_scheduler #( `STATIC_ASSERT ((MEM_CHANNELS <= CORE_REQS), ("invalid parameter")) `STATIC_ASSERT (`IS_DIVISBLE(CORE_REQS * WORD_SIZE, LINE_SIZE), ("invalid parameter")) `STATIC_ASSERT ((TAG_WIDTH >= UUID_WIDTH), ("invalid parameter")) - `STATIC_ASSERT ((RSP_SIDE_WIDTH == 0) || ((RSP_PARTIAL != 0) && !COALESCE_ENABLE), ("invalid parameter")) `RUNTIME_ASSERT((~core_req_valid || core_req_mask != 0), ("invalid request mask")) wire ibuf_push; @@ -485,11 +477,7 @@ module VX_mem_scheduler #( assign crsp_valid = mem_rsp_valid_s; assign crsp_mask = mem_rsp_mask_s; assign crsp_sop = 1'b1; - if (RSP_SIDE_WIDTH != 0) begin : g_hold - assign crsp_eop = ~mem_rsp_hold; - end else begin : g_no_hold - assign crsp_eop = 1'b1; - end + assign crsp_eop = 1'b1; assign crsp_data = mem_rsp_data_s; assign mem_rsp_ready_s = crsp_ready; @@ -505,14 +493,7 @@ module VX_mem_scheduler #( assign curr_mask[r] = (BATCH_SEL_WIDTH'(i) == rsp_batch_idx) && mem_rsp_mask_s[j]; end - wire [CORE_REQS-1:0] rsp_retire_mask; - if (RSP_SIDE_WIDTH != 0) begin : g_hold - assign rsp_retire_mask = mem_rsp_hold ? '0 : curr_mask; - end else begin : g_no_hold - assign rsp_retire_mask = curr_mask; - end - - assign rsp_rem_mask_n = rsp_rem_mask[ibuf_raddr] & ~rsp_retire_mask; + assign rsp_rem_mask_n = rsp_rem_mask[ibuf_raddr] & ~curr_mask; wire mem_rsp_fire_s = mem_rsp_valid_s && mem_rsp_ready_s; @@ -616,22 +597,8 @@ module VX_mem_scheduler #( // Send response to caller - localparam CRSP_DATAW = CORE_REQS + 1 + 1 + (CORE_REQS * WORD_WIDTH) + TAG_WIDTH; - - wire [CRSP_DATAW+RSP_SIDE_WIDTH-1:0] crsp_buf_in, crsp_buf_out; - - if (RSP_SIDE_WIDTH != 0) begin : g_crsp_side - assign crsp_buf_in = {mem_rsp_side, crsp_mask, crsp_sop, crsp_eop, crsp_data, crsp_tag}; - assign {core_rsp_side, core_rsp_mask, core_rsp_sop, core_rsp_eop, core_rsp_data, core_rsp_tag} = crsp_buf_out; - end else begin : g_crsp_no_side - assign crsp_buf_in = {crsp_mask, crsp_sop, crsp_eop, crsp_data, crsp_tag}; - assign {core_rsp_mask, core_rsp_sop, core_rsp_eop, core_rsp_data, core_rsp_tag} = crsp_buf_out; - assign core_rsp_side = '0; - `UNUSED_VAR ({mem_rsp_hold, mem_rsp_side}) - end - VX_elastic_buffer #( - .DATAW (CRSP_DATAW + RSP_SIDE_WIDTH), + .DATAW (CORE_REQS + 1 + 1 + (CORE_REQS * WORD_WIDTH) + TAG_WIDTH), .SIZE (`TO_OUT_BUF_SIZE(CORE_OUT_BUF)), .OUT_REG (`TO_OUT_BUF_REG(CORE_OUT_BUF)) ) rsp_buf ( @@ -639,8 +606,8 @@ module VX_mem_scheduler #( .reset (reset), .valid_in (crsp_valid), .ready_in (crsp_ready), - .data_in (crsp_buf_in), - .data_out (crsp_buf_out), + .data_in ({crsp_mask, crsp_sop, crsp_eop, crsp_data, crsp_tag}), + .data_out ({core_rsp_mask, core_rsp_sop, core_rsp_eop, core_rsp_data, core_rsp_tag}), .valid_out (core_rsp_valid), .ready_out (core_rsp_ready) ); diff --git a/hw/rtl/mem/VX_lmem_engine.sv b/hw/rtl/mem/VX_lmem_engine.sv index 3b79fe9728..ef083fc8e0 100644 --- a/hw/rtl/mem/VX_lmem_engine.sv +++ b/hw/rtl/mem/VX_lmem_engine.sv @@ -35,13 +35,6 @@ // payload mux. The first pass's winners come from S0; a replay finds the next // winners in the cycle after a pass, so a conflicting request replays at one // pass per two cycles. The last pass frees S1 in its own cycle. -// -// A wide read takes two or four words per lane, its lanes' base words aligned -// to the group. It is served one word-phase at a time: a phase sets the low -// word-address bits of every lane to the phase number, which leaves each pair -// of lanes' bank and word equality unchanged, so every phase reuses the -// request's matrices and first winners, and starts in the cycle after the -// previous phase's last pass. module VX_lmem_engine import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter NUM_LANES = 4, @@ -56,14 +49,11 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( parameter ATTR_WIDTH = 1, parameter NUM_BLOCKS = 1, parameter RSP_QUEUE_SIZE = 8, - // serve wide reads - parameter WIDE_ENABLE = 0, parameter WORD_WIDTH = WORD_SIZE * 8, parameter BLOCK_WIDTH = `UP(`CLOG2(NUM_BLOCKS)), parameter DONE_WIDTH = `CLOG2(NUM_BLOCKS + 1), - parameter BANK_ADDR_WIDTH = ADDR_WIDTH - `CLOG2(NUM_BANKS), - localparam RSP_SIDE_WIDTH = WIDE_ENABLE ? $bits(lsu_rsp_side_t) : 0 + parameter BANK_ADDR_WIDTH = ADDR_WIDTH - `CLOG2(NUM_BANKS) ) ( input wire clk, input wire reset, @@ -81,8 +71,6 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( input wire [NUM_LANES-1:0][ADDR_WIDTH-1:0] req_addr, input wire [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] req_payload, input wire [TAG_WIDTH-1:0] req_tag, - // a read's words per lane, as log2 (up to 4 words) - input wire [1:0] req_wide, output wire req_ready, // the engine holds a request; the one it holds is a store or an atomic @@ -112,8 +100,6 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( output wire [NUM_LANES-1:0] rsp_mask, output wire [NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data, output wire [TAG_WIDTH-1:0] rsp_tag, - // the response's phase of a wide read (lsu_rsp_side_t) - output wire [`UP(RSP_SIDE_WIDTH)-1:0] rsp_side, input wire rsp_take, input wire [DONE_WIDTH-1:0] rsp_done, @@ -128,13 +114,9 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS); localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); localparam CREDIT_WIDTH = `CLOG2(RSP_QUEUE_SIZE + 1); - localparam WIDE_WIDTH = WIDE_ENABLE ? 2 : 0; - // the response's side word travels with its tag - localparam TAGX_WIDTH = TAG_WIDTH + RSP_SIDE_WIDTH; - localparam RSP_DATAW = BLOCK_WIDTH + NUM_LANES + NUM_LANES * WORD_WIDTH + TAGX_WIDTH; + localparam RSP_DATAW = BLOCK_WIDTH + NUM_LANES + NUM_LANES * WORD_WIDTH + TAG_WIDTH; `STATIC_ASSERT(ADDR_WIDTH > BANK_SEL_BITS, ("invalid parameter")) - `STATIC_ASSERT((WIDE_ENABLE == 0) || (ADDR_WIDTH >= 2), ("invalid parameter")) // S0: the engine's input queue. Holding two requests, it takes one in any // cycle that starts with at most one, so its ready is a register and the @@ -153,27 +135,10 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0][ADDR_WIDTH-1:0] s0_addr; wire [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] s0_payload; wire [TAG_WIDTH-1:0] s0_tag; - wire [1:0] s0_wide; wire s1_free; - wire [S0_DATAW-1:0] s0_base_in, s0_base_out; - wire [S0_DATAW+WIDE_WIDTH-1:0] s0_buf_in, s0_buf_out; - - assign s0_base_in = {req_block, req_rw, req_read, req_amo, req_attr, req_mask, req_addr, req_payload, req_tag}; - assign {s0_block, s0_rw, s0_read, s0_amo, s0_attr, s0_mask, s0_addr, s0_payload, s0_tag} = s0_base_out; - - if (WIDE_ENABLE != 0) begin : g_s0_wide - assign s0_buf_in = {req_wide, s0_base_in}; - assign {s0_wide, s0_base_out} = s0_buf_out; - end else begin : g_s0_no_wide - assign s0_buf_in = s0_base_in; - assign s0_base_out = s0_buf_out; - assign s0_wide = 2'b00; - `UNUSED_VAR (req_wide) - end - VX_elastic_buffer #( - .DATAW (S0_DATAW + WIDE_WIDTH), + .DATAW (S0_DATAW), .SIZE (2), .OUT_REG (1) ) s0_buf ( @@ -181,8 +146,8 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( .reset (reset), .valid_in (req_valid), .ready_in (req_ready), - .data_in (s0_buf_in), - .data_out (s0_buf_out), + .data_in ({req_block, req_rw, req_read, req_amo, req_attr, req_mask, req_addr, req_payload, req_tag}), + .data_out ({s0_block, s0_rw, s0_read, s0_amo, s0_attr, s0_mask, s0_addr, s0_payload, s0_tag}), .valid_out (s0_valid), .ready_out (s1_free) ); @@ -238,14 +203,6 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( reg [TAG_WIDTH-1:0] s1_tag; reg [NUM_LANES-1:0][NUM_LANES-1:0] s1_same_bank; reg [NUM_LANES-1:0][NUM_LANES-1:0] s1_same_word; - // a wide read's group size, its phase, whether the phase is its last, and - // the request's first pass, which every phase starts from - reg s1_quad; - reg [1:0] s1_phase; - reg s1_final; - reg [NUM_LANES-1:0] s1_mask0; - reg [NUM_LANES-1:0] s1_first0; - reg s1_one_pass0; // A pass serves the registered bank winners and the pending read lanes of // their words. After a pass that leaves lanes pending, the next cycle finds @@ -271,15 +228,10 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( reg [CREDIT_WIDTH-1:0] credits; wire has_rsp = ~s1_rw; wire pass_fire = s1_valid && s1_ready && pass_en && (~has_rsp || (credits != '0)); - wire last_phase = (WIDE_ENABLE == 0) || s1_final; - wire phase_end = pass_fire && s1_last; - wire s1_release = phase_end && last_phase; - wire s1_advance = phase_end && ~last_phase; + wire s1_release = pass_fire && s1_last; assign s1_free = ~s1_valid || s1_release; wire s1_load = s0_valid && s1_free; - wire [1:0] phase_n = s1_phase + 2'd1; - always @(posedge clk) begin if (reset) begin s1_valid <= 1'b0; @@ -303,25 +255,6 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( s1_tag <= s0_tag; s1_same_bank <= s0_same_bank; s1_same_word <= s0_same_word; - s1_quad <= s0_wide[1]; - s1_phase <= 2'd0; - s1_final <= (s0_wide == 2'd0); - s1_mask0 <= s0_mask; - s1_first0 <= s0_first; - s1_one_pass0 <= s0_one_pass; - end else if (s1_advance) begin - s1_ready <= 1'b1; - s1_last <= s1_one_pass0; - s1_winners <= s1_first0; - s1_pending <= s1_mask0; - s1_phase <= phase_n; - s1_final <= (phase_n == {s1_quad, 1'b1}); - for (integer i = 0; i < NUM_LANES; ++i) begin - s1_addr[i][0] <= phase_n[0]; - if (s1_quad) begin - s1_addr[i][1] <= phase_n[1]; - end - end end else if (pass_fire) begin s1_ready <= 1'b0; s1_pending <= s1_pending & ~served; @@ -332,18 +265,6 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( end end - wire [TAGX_WIDTH-1:0] s1_tagx; - if (WIDE_ENABLE != 0) begin : g_s1_side - lsu_rsp_side_t s1_side; - assign s1_side.hold = ~s1_final; - assign s1_side.phase_end = s1_last; - assign s1_side.phase = s1_phase; - assign s1_tagx = {s1_side, s1_tag}; - end else begin : g_s1_no_side - assign s1_tagx = s1_tag; - `UNUSED_VAR ({s1_quad, s1_phase, s1_final, s1_mask0, s1_first0, s1_one_pass0, phase_n}) - end - // stores and atomics held between S0's two entries and S1 reg [1:0] num_writes; wire write_in = req_valid && req_ready && ~req_read; @@ -428,17 +349,17 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( wire [BLOCK_WIDTH-1:0] m2_block; wire [NUM_LANES-1:0] m2_mask; wire [NUM_LANES-1:0][BANK_SEL_WIDTH-1:0] m2_bank; - wire [TAGX_WIDTH-1:0] m2_tag; + wire [TAG_WIDTH-1:0] m2_tag; VX_pipe_register #( - .DATAW (1 + BLOCK_WIDTH + NUM_LANES + NUM_LANES * BANK_SEL_WIDTH + TAGX_WIDTH), + .DATAW (1 + BLOCK_WIDTH + NUM_LANES + NUM_LANES * BANK_SEL_WIDTH + TAG_WIDTH), .RESETW (1), .DEPTH (2) ) rsp_meta_pipe ( .clk (clk), .reset (reset), .enable (1'b1), - .data_in ({pass_fire && has_rsp, s1_block, served, s1_bank, s1_tagx}), + .data_in ({pass_fire && has_rsp, s1_block, served, s1_bank, s1_tag}), .data_out ({m2_valid, m2_block, m2_mask, m2_bank, m2_tag}) ); @@ -453,7 +374,7 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( wire [BLOCK_WIDTH-1:0] m3_block; wire [NUM_LANES-1:0] m3_mask; wire [NUM_LANES-1:0][WORD_WIDTH-1:0] m3_data; - wire [TAGX_WIDTH-1:0] m3_tag; + wire [TAG_WIDTH-1:0] m3_tag; VX_pipe_register #( .DATAW (RSP_DATAW + 1), @@ -492,16 +413,8 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( `UNUSED_PIN (alm_full), `UNUSED_PIN (size) ); - wire [TAGX_WIDTH-1:0] rsp_tagx; assign rsp_valid = ~rsp_empty || m3_valid; - assign {rsp_block, rsp_mask, rsp_data, rsp_tagx} = rsp_empty ? {m3_block, m3_mask, m3_data, m3_tag} : rsp_head; - - if (WIDE_ENABLE != 0) begin : g_rsp_side - assign {rsp_side, rsp_tag} = rsp_tagx; - end else begin : g_rsp_no_side - assign rsp_tag = rsp_tagx; - assign rsp_side = '0; - end + assign {rsp_block, rsp_mask, rsp_data, rsp_tag} = rsp_empty ? {m3_block, m3_mask, m3_data, m3_tag} : rsp_head; // a response's slot is free again once its block has taken it on wire credit_take = pass_fire && has_rsp; @@ -520,8 +433,8 @@ module VX_lmem_engine import VX_gpu_pkg::*; #( `ifdef DBG_TRACE_MEM always @(posedge clk) begin if (pass_fire) begin - `TRACE(2, ("%t: %s pass: rw=%b, amo=%b, pending=%b, served=%b, phase=%0d, tag=0x%0h\n", - $time, INSTANCE_ID, s1_rw, s1_amo, s1_pending, served, s1_phase, s1_tag)) + `TRACE(2, ("%t: %s pass: rw=%b, amo=%b, pending=%b, served=%b, tag=0x%0h\n", + $time, INSTANCE_ID, s1_rw, s1_amo, s1_pending, served, s1_tag)) end end `endif diff --git a/hw/rtl/mem/VX_lmem_switch.sv b/hw/rtl/mem/VX_lmem_switch.sv index ae8aa381ec..c155fe903f 100644 --- a/hw/rtl/mem/VX_lmem_switch.sv +++ b/hw/rtl/mem/VX_lmem_switch.sv @@ -106,29 +106,9 @@ module VX_lmem_switch import VX_gpu_pkg::*; #( .ready_out (local_out_if.req_ready) ); - // The local responses carry a sideband; the global ones leave it clear. - wire [1:0][RSP_DATAW+LSU_RSP_SIDE_W-1:0] rsp_arb_in; - wire [RSP_DATAW+LSU_RSP_SIDE_W-1:0] rsp_arb_out; - - wire [RSP_DATAW-1:0] local_rsp = {local_out_if.rsp_data.mask, local_out_if.rsp_data.data, local_out_if.rsp_data.tag}; - wire [RSP_DATAW-1:0] global_rsp = {global_out_if.rsp_data.mask, global_out_if.rsp_data.data, global_out_if.rsp_data.tag}; - `UNUSED_VAR (global_out_if.rsp_data.side) - - if (LSU_RSP_SIDE_W != 0) begin : g_rsp_side - assign rsp_arb_in[1] = {local_out_if.rsp_data.side, local_rsp}; - assign rsp_arb_in[0] = {LSU_RSP_SIDE_W'(0), global_rsp}; - assign {lsu_in_if.rsp_data.side, lsu_in_if.rsp_data.mask, lsu_in_if.rsp_data.data, lsu_in_if.rsp_data.tag} = rsp_arb_out; - end else begin : g_no_rsp_side - assign rsp_arb_in[1] = local_rsp; - assign rsp_arb_in[0] = global_rsp; - assign {lsu_in_if.rsp_data.mask, lsu_in_if.rsp_data.data, lsu_in_if.rsp_data.tag} = rsp_arb_out; - assign lsu_in_if.rsp_data.side = '0; - `UNUSED_VAR (local_out_if.rsp_data.side) - end - VX_stream_arb #( .NUM_INPUTS (2), - .DATAW (RSP_DATAW + LSU_RSP_SIDE_W), + .DATAW (RSP_DATAW), .ARBITER (ARBITER), .OUT_BUF (RSP_OUT_BUF) ) rsp_arb ( @@ -142,8 +122,11 @@ module VX_lmem_switch import VX_gpu_pkg::*; #( local_out_if.rsp_ready, global_out_if.rsp_ready }), - .data_in (rsp_arb_in), - .data_out (rsp_arb_out), + .data_in ({ + local_out_if.rsp_data, + global_out_if.rsp_data + }), + .data_out (lsu_in_if.rsp_data), .valid_out (lsu_in_if.rsp_valid), .ready_out (lsu_in_if.rsp_ready), `UNUSED_PIN (sel_out) diff --git a/hw/rtl/mem/VX_lmem_unit.sv b/hw/rtl/mem/VX_lmem_unit.sv index f7cf4f4a80..a48400fb16 100644 --- a/hw/rtl/mem/VX_lmem_unit.sv +++ b/hw/rtl/mem/VX_lmem_unit.sv @@ -32,10 +32,6 @@ // request is seen, and it is accepted in the cycle that follows. An atomic owns // its bank for three cycles (read, capture, write back), so no pass or DMA access // is issued in the two cycles after an atomic pass. -// -// A wide read names its group size in the atomic op of a request that is not -// an atomic. Its lanes' addresses are aligned down to the group here, and the -// engine serves the group one word-phase at a time. module VX_lmem_unit import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", @@ -59,10 +55,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( parameter DMA_TAG_WIDTH = 1, // Synthesize atomic-op + LR/SC logic in the banks - parameter AMO_ENABLE = 0, - - // Serve wide reads - parameter WIDE_ENABLE = 0 + parameter AMO_ENABLE = 0 ) ( input wire clk, input wire reset, @@ -104,7 +97,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( localparam PAYLOAD_WIDTH = WORD_WIDTH + WORD_SIZE + LANE_SB_WIDTH; // common to all lanes: the atomic op and its signedness localparam ATTR_WIDTH = AMO_OP_WIDTH + 1; - localparam RSP_SIDE_WIDTH = WIDE_ENABLE ? $bits(lsu_rsp_side_t) : 0; `STATIC_ASSERT(ADDR_WIDTH == (BANK_ADDR_WIDTH + `CLOG2(NUM_BANKS)), ("invalid parameter")) `STATIC_ASSERT(`IS_POW2(NUM_BANKS), ("invalid parameter")) @@ -120,7 +112,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][ADDR_WIDTH-1:0] in_addr; wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] in_payload; wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] in_tag; - wire [NUM_BLOCKS-1:0][1:0] in_wide; wire [NUM_BLOCKS-1:0] in_ready; for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_in @@ -134,23 +125,9 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( assign in_attr[k] = AMO_ENABLE ? {attr0.amo.amo_op, attr0.amo.amo_unsigned} : '0; assign in_mask[k] = lsu_mem_if[k].req_data.mask; assign in_tag[k] = lsu_mem_if[k].req_data.tag; - if (WIDE_ENABLE != 0) begin : g_wide - wire [AMO_OP_WIDTH-1:0] attr0_op = attr0.amo.amo_op; - assign in_wide[k] = attr0.amo.amo_valid ? 2'b00 : attr0_op[1:0]; - `UNUSED_VAR (attr0_op) - end else begin : g_no_wide - assign in_wide[k] = 2'b00; - end `UNUSED_VAR (attr0) for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lanes - wire [ADDR_WIDTH-1:0] lane_addr = lsu_mem_if[k].req_data.addr[i][ADDR_WIDTH-1:0]; - if (WIDE_ENABLE != 0) begin : g_wide_addr - assign in_addr[k][i] = {lane_addr[ADDR_WIDTH-1:2], - lane_addr[1] && ~in_wide[k][1], - lane_addr[0] && (in_wide[k] == 2'b00)}; - end else begin : g_addr - assign in_addr[k][i] = lane_addr; - end + assign in_addr[k][i] = lsu_mem_if[k].req_data.addr[i][ADDR_WIDTH-1:0]; if (AMO_ENABLE != 0) begin : g_amo mem_bus_attr_t lane_attr; assign lane_attr = mem_bus_attr_t'(lsu_mem_if[k].req_data.user[i]); @@ -200,7 +177,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0] a_rsp_mask; wire [NUM_LANES-1:0][WORD_WIDTH-1:0] a_rsp_data; wire [TAG_WIDTH-1:0] a_rsp_tag; - wire [`UP(RSP_SIDE_WIDTH)-1:0] a_rsp_side; wire [NUM_LANES-1:0] a_perf_served; wire a_perf_store; @@ -218,7 +194,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0] b_rsp_mask; wire [NUM_LANES-1:0][WORD_WIDTH-1:0] b_rsp_data; wire [TAG_WIDTH-1:0] b_rsp_tag; - wire [`UP(RSP_SIDE_WIDTH)-1:0] b_rsp_side; wire [NUM_LANES-1:0] b_perf_served; wire b_perf_replay; @@ -336,8 +311,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .PAYLOAD_WIDTH (PAYLOAD_WIDTH), .ATTR_WIDTH (ATTR_WIDTH), .NUM_BLOCKS (NUM_BLOCKS), - .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE), - .WIDE_ENABLE (WIDE_ENABLE) + .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE) ) engine_a ( .clk (clk), .reset (reset), @@ -351,7 +325,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .req_addr (in_addr[a_req_block]), .req_payload (in_payload[a_req_block]), .req_tag (in_tag[a_req_block]), - .req_wide (in_wide[a_req_block]), .req_ready (a_req_ready), .busy (a_busy), .holds_write (a_holds_write), @@ -369,7 +342,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .rsp_mask (a_rsp_mask), .rsp_data (a_rsp_data), .rsp_tag (a_rsp_tag), - .rsp_side (a_rsp_side), .rsp_take (a_rsp_take), .rsp_done (a_rsp_done), .perf_served (a_perf_served), @@ -399,8 +371,7 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .PAYLOAD_WIDTH (1), .ATTR_WIDTH (ATTR_WIDTH), .NUM_BLOCKS (NUM_BLOCKS), - .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE), - .WIDE_ENABLE (WIDE_ENABLE) + .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE) ) engine_b ( .clk (clk), .reset (reset), @@ -414,7 +385,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .req_addr (in_addr[b_req_block]), .req_payload (b_req_payload), .req_tag (in_tag[b_req_block]), - .req_wide (in_wide[b_req_block]), .req_ready (b_req_ready), .busy (b_busy), .holds_write (b_holds_write), @@ -432,7 +402,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( .rsp_mask (b_rsp_mask), .rsp_data (b_rsp_data), .rsp_tag (b_rsp_tag), - .rsp_side (b_rsp_side), .rsp_take (b_rsp_take), .rsp_done (b_rsp_done), .perf_served (b_perf_served), @@ -451,7 +420,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( assign b_rsp_mask = '0; assign b_rsp_data = '0; assign b_rsp_tag = '0; - assign b_rsp_side = '0; assign b_perf_served = '0; assign b_perf_replay = 1'b0; `UNUSED_VAR (b_bank_rdata) @@ -480,7 +448,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( reg [NUM_LANES-1:0] rsp_mask_r; reg [NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data_r; reg [TAG_WIDTH-1:0] rsp_tag_r; - reg [`UP(RSP_SIDE_WIDTH)-1:0] rsp_side_r; wire rsp_fire = rsp_valid_r && lsu_mem_if[k].rsp_ready; wire rsp_free = ~rsp_valid_r || lsu_mem_if[k].rsp_ready; @@ -503,7 +470,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( rsp_mask_r <= a_hit ? a_rsp_mask : b_rsp_mask; rsp_data_r <= a_hit ? a_rsp_data : b_rsp_data; rsp_tag_r <= a_hit ? a_rsp_tag : b_rsp_tag; - rsp_side_r <= a_hit ? a_rsp_side : b_rsp_side; end end @@ -511,12 +477,6 @@ module VX_lmem_unit import VX_gpu_pkg::*; #( assign lsu_mem_if[k].rsp_data.mask = rsp_mask_r; assign lsu_mem_if[k].rsp_data.data = rsp_data_r; assign lsu_mem_if[k].rsp_data.tag = rsp_tag_r; - if (WIDE_ENABLE != 0) begin : g_rsp_side - assign lsu_mem_if[k].rsp_data.side = rsp_side_r; - end else begin : g_no_rsp_side - assign lsu_mem_if[k].rsp_data.side = '0; - `UNUSED_VAR (rsp_side_r) - end end `POP_COUNT(a_rsp_done, a_done); diff --git a/hw/rtl/mem/VX_lsu_adapter.sv b/hw/rtl/mem/VX_lsu_adapter.sv index 5504d31131..4b64b82b00 100644 --- a/hw/rtl/mem/VX_lsu_adapter.sv +++ b/hw/rtl/mem/VX_lsu_adapter.sv @@ -117,6 +117,5 @@ module VX_lsu_adapter import VX_gpu_pkg::*; #( .tag_out (lsu_mem_if.rsp_data.tag), .ready_out (lsu_mem_if.rsp_ready) ); - assign lsu_mem_if.rsp_data.side = '0; endmodule diff --git a/hw/rtl/mem/VX_lsu_mem_arb.sv b/hw/rtl/mem/VX_lsu_mem_arb.sv index 3b0817a7c3..90df8a233a 100644 --- a/hw/rtl/mem/VX_lsu_mem_arb.sv +++ b/hw/rtl/mem/VX_lsu_mem_arb.sv @@ -164,7 +164,6 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( bus_out_if[i].rsp_data.data, rsp_tag_out }; - `UNUSED_VAR (bus_out_if[i].rsp_data.side) assign bus_out_if[i].rsp_ready = rsp_ready_in[i]; end @@ -196,13 +195,8 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( TAG_WIDTH'(bus_out_if[i].rsp_data.tag) }; `else - assign rsp_data_in[i] = { - bus_out_if[i].rsp_data.mask, - bus_out_if[i].rsp_data.data, - bus_out_if[i].rsp_data.tag - }; + assign rsp_data_in[i] = bus_out_if[i].rsp_data; `endif - `UNUSED_VAR (bus_out_if[i].rsp_data.side) assign bus_out_if[i].rsp_ready = rsp_ready_in[i]; end @@ -228,8 +222,7 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_output assign bus_in_if[i].rsp_valid = rsp_valid_out[i]; - assign {bus_in_if[i].rsp_data.mask, bus_in_if[i].rsp_data.data, bus_in_if[i].rsp_data.tag} = rsp_data_out[i]; - assign bus_in_if[i].rsp_data.side = '0; + assign bus_in_if[i].rsp_data = rsp_data_out[i]; assign rsp_ready_out[i] = bus_in_if[i].rsp_ready; end diff --git a/hw/rtl/mem/VX_lsu_mem_if.sv b/hw/rtl/mem/VX_lsu_mem_if.sv index f335b4b8a6..31af0acb68 100644 --- a/hw/rtl/mem/VX_lsu_mem_if.sv +++ b/hw/rtl/mem/VX_lsu_mem_if.sv @@ -18,9 +18,7 @@ interface VX_lsu_mem_if import VX_gpu_pkg::*; #( parameter DATA_SIZE = 1, parameter TAG_WIDTH = 1, parameter USER_WIDTH = MEM_ATTR_WIDTH, - parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE), - // response sideband width; zero keeps a single bit that endpoints tie off - parameter RSP_SIDE_WIDTH = 0 + parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE) ) (); typedef struct packed { @@ -39,7 +37,6 @@ interface VX_lsu_mem_if import VX_gpu_pkg::*; #( } req_data_t; typedef struct packed { - logic [`UP(RSP_SIDE_WIDTH)-1:0] side; logic [NUM_LANES-1:0] mask; logic [NUM_LANES-1:0][DATA_SIZE*8-1:0] data; tag_t tag; diff --git a/hw/rtl/om/VX_om_mem.sv b/hw/rtl/om/VX_om_mem.sv index a3f56c89cd..f2c5e90097 100644 --- a/hw/rtl/om/VX_om_mem.sv +++ b/hw/rtl/om/VX_om_mem.sv @@ -283,7 +283,6 @@ module VX_om_mem import VX_gpu_pkg::*; import VX_om_pkg::*; #( .core_rsp_tag (mrsp_tag), `UNUSED_PIN (core_rsp_sop), `UNUSED_PIN (core_rsp_eop), - `UNUSED_PIN (core_rsp_side), .core_rsp_ready (mrsp_ready), // Memory request @@ -302,8 +301,6 @@ module VX_om_mem import VX_gpu_pkg::*; import VX_om_pkg::*; #( .mem_rsp_mask (lsu_mem_if.rsp_data.mask), .mem_rsp_data (lsu_mem_if.rsp_data.data), .mem_rsp_tag (lsu_mem_if.rsp_data.tag), - .mem_rsp_hold (1'b0), - .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_if.rsp_ready) ); diff --git a/hw/rtl/raster/VX_raster_earlyz.sv b/hw/rtl/raster/VX_raster_earlyz.sv index fe472bab08..c95061aa05 100644 --- a/hw/rtl/raster/VX_raster_earlyz.sv +++ b/hw/rtl/raster/VX_raster_earlyz.sv @@ -336,7 +336,6 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX `UNUSED_PIN (core_rsp_tag), `UNUSED_PIN (core_rsp_sop), `UNUSED_PIN (core_rsp_eop), - `UNUSED_PIN (core_rsp_side), .core_rsp_ready (mrsp_ready), // Memory request @@ -355,8 +354,6 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX .mem_rsp_mask (lsu_mem_if.rsp_data.mask), .mem_rsp_data (lsu_mem_if.rsp_data.data), .mem_rsp_tag (lsu_mem_if.rsp_data.tag), - .mem_rsp_hold (1'b0), - .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_if.rsp_ready) ); diff --git a/hw/rtl/raster/VX_raster_mem.sv b/hw/rtl/raster/VX_raster_mem.sv index df67986e36..b677ce96d5 100644 --- a/hw/rtl/raster/VX_raster_mem.sv +++ b/hw/rtl/raster/VX_raster_mem.sv @@ -385,7 +385,6 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( .core_rsp_tag (sched_rsp_tag), `UNUSED_PIN (core_rsp_sop), `UNUSED_PIN (core_rsp_eop), - `UNUSED_PIN (core_rsp_side), .core_rsp_ready (mem_rsp_ready), // Memory request @@ -404,8 +403,6 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( .mem_rsp_mask (lsu_mem_if.rsp_data.mask), .mem_rsp_data (lsu_mem_if.rsp_data.data), .mem_rsp_tag (lsu_mem_if.rsp_data.tag), - .mem_rsp_hold (1'b0), - .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_if.rsp_ready) ); diff --git a/hw/rtl/tcu/VX_tcu_agu.sv b/hw/rtl/tcu/VX_tcu_agu.sv index c5fb6dfe9f..315917365f 100644 --- a/hw/rtl/tcu/VX_tcu_agu.sv +++ b/hw/rtl/tcu/VX_tcu_agu.sv @@ -136,7 +136,6 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( assign client_if.req_data = req_data; assign client_if.rsp_ready = 1'b1; `UNUSED_VAR (client_if.rsp_data.sop) - `UNUSED_VAR (client_if.rsp_data.side) `UNUSED_VAR (client_if.rsp_data.tag) `RUNTIME_ASSERT(~client_if.rsp_valid || (issued_r && ~done_r), diff --git a/hw/rtl/tex/VX_tex_mem.sv b/hw/rtl/tex/VX_tex_mem.sv index 26f1464b73..597c45cd95 100644 --- a/hw/rtl/tex/VX_tex_mem.sv +++ b/hw/rtl/tex/VX_tex_mem.sv @@ -171,7 +171,6 @@ module VX_tex_mem import VX_gpu_pkg::*; import VX_tex_pkg::*; #( .core_rsp_tag (mem_rsp_tag), `UNUSED_PIN (core_rsp_sop), `UNUSED_PIN (core_rsp_eop), - `UNUSED_PIN (core_rsp_side), .core_rsp_ready (mem_rsp_ready), // Memory request @@ -190,8 +189,6 @@ module VX_tex_mem import VX_gpu_pkg::*; import VX_tex_pkg::*; #( .mem_rsp_mask (lsu_mem_if.rsp_data.mask), .mem_rsp_data (lsu_mem_if.rsp_data.data), .mem_rsp_tag (lsu_mem_if.rsp_data.tag), - .mem_rsp_hold (1'b0), - .mem_rsp_side (lsu_mem_if.rsp_data.side), .mem_rsp_ready (lsu_mem_if.rsp_ready) ); diff --git a/hw/unittest/issue/VX_issue_top.sv b/hw/unittest/issue/VX_issue_top.sv index eb78dccd84..01de117a5e 100644 --- a/hw/unittest/issue/VX_issue_top.sv +++ b/hw/unittest/issue/VX_issue_top.sv @@ -111,7 +111,6 @@ module VX_issue_top import VX_gpu_pkg::*; #( assign writeback_if[i].data.uuid = writeback_uuid[i]; assign writeback_if[i].data.wis = writeback_wis[i]; assign writeback_if[i].data.eop_wis = PER_ISSUE_WARPS'(writeback_eop[i]) << writeback_wis[i]; - assign writeback_if[i].data.rel_wis = '0; assign writeback_if[i].data.sid = writeback_sid[i]; assign writeback_if[i].data.tmask = writeback_tmask[i]; assign writeback_if[i].data.PC = writeback_PC[i]; diff --git a/hw/unittest/lmem_unit/VX_lmem_unit_top.sv b/hw/unittest/lmem_unit/VX_lmem_unit_top.sv index e75c0c0a49..0e6dabe6f2 100644 --- a/hw/unittest/lmem_unit/VX_lmem_unit_top.sv +++ b/hw/unittest/lmem_unit/VX_lmem_unit_top.sv @@ -52,7 +52,6 @@ module VX_lmem_unit_top import VX_gpu_pkg::*; #( output wire [NUM_BLOCKS-1:0][NUM_LANES-1:0] rsp_mask, output wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data, output wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] rsp_tag, - output wire [NUM_BLOCKS-1:0][`UP(LSU_RSP_SIDE_W)-1:0] rsp_side, input wire [NUM_BLOCKS-1:0] rsp_ready, // DMA requests @@ -74,10 +73,9 @@ module VX_lmem_unit_top import VX_gpu_pkg::*; #( localparam DMA_REQ_DATAW = 1 + DMA_ADDR_WIDTH + DMA_DATA_SIZE * 8 + DMA_DATA_SIZE + MEM_ATTR_WIDTH + DMA_TAG_WIDTH; VX_lsu_mem_if #( - .NUM_LANES (NUM_LANES), - .DATA_SIZE (WORD_SIZE), - .TAG_WIDTH (TAG_WIDTH), - .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) + .NUM_LANES (NUM_LANES), + .DATA_SIZE (WORD_SIZE), + .TAG_WIDTH (TAG_WIDTH) ) lsu_mem_if[NUM_BLOCKS](); VX_mem_bus_if #( @@ -110,7 +108,6 @@ module VX_lmem_unit_top import VX_gpu_pkg::*; #( assign rsp_mask[k] = lsu_mem_if[k].rsp_data.mask; assign rsp_data[k] = lsu_mem_if[k].rsp_data.data; assign rsp_tag[k] = lsu_mem_if[k].rsp_data.tag; - assign rsp_side[k] = lsu_mem_if[k].rsp_data.side; assign lsu_mem_if[k].rsp_ready = rsp_ready[k]; end @@ -144,8 +141,7 @@ module VX_lmem_unit_top import VX_gpu_pkg::*; #( .TAG_WIDTH (TAG_WIDTH), .DMA_ENABLE (1), .DMA_TAG_WIDTH (DMA_TAG_WIDTH), - .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), - .WIDE_ENABLE (LDS_WIDE_ENABLE) + .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) lmem_unit ( .clk (clk), .reset (reset), diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index c5a435cbaa..12d186a47d 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -47,10 +47,9 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( ); `ifdef VX_CFG_LMEM_ENABLE VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH), - .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); `endif diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 62fdb24f14..741521a4dc 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -223,10 +223,9 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( wire [NUM_LSU_BLKS-1:0] block_store_empty; VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH), - .RSP_SIDE_WIDTH (LSU_RSP_SIDE_W) + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); VX_mem_bus_if #( diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 9cd1d0c422..7479a64a02 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -632,9 +632,6 @@ class Core::Impl { trace->op_type = instr->get_op_type(); trace->dst_reg = instr->get_dest_reg(); trace->dst_bytesel = instr->get_dst_bytesel(); - if (auto lsu_args = std::get_if(&instr->get_args())) { - trace->dst_span = lsu_args->nregs; - } for (uint32_t i = 0; i < NUM_SRC_REGS; ++i) { trace->src_regs[i] = instr->get_src_reg(i); } diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 7580e009b5..cbcbc94e9a 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -267,15 +267,15 @@ static op_string_t op_string(const Instr &instr) { case LsuType::LOAD: { auto lsuArgs = std::get(instrArgs); bool dst_float = (instr.get_dest_reg().type == RegType::Float); - if (lsuArgs.nregs > 1) { - return {(lsuArgs.nregs == 4) ? "VX_FLDS.Q" : "VX_FLDS.D", to_hex_str(lsuArgs.offset)}; - } if (instr.is_macro_op() && dst_float && lsuArgs.width == 0) { return {"PACKLB.F", ""}; } if (instr.is_macro_op() && dst_float && lsuArgs.width == 1) { return {"PACKLH.F", ""}; } + if (instr.is_macro_op()) { + return {(lsuArgs.width == 4) ? "VX_FLW.Q" : "VX_FLW.D", to_hex_str(lsuArgs.offset)}; + } if (dst_float) { switch (lsuArgs.width) { case 1: return {"FLH", to_hex_str(lsuArgs.offset)}; @@ -300,6 +300,9 @@ static op_string_t op_string(const Instr &instr) { } case LsuType::STORE: { auto lsuArgs = std::get(instrArgs); + if (instr.is_macro_op()) { + return {(lsuArgs.width == 4) ? "VX_FSW.Q" : "VX_FSW.D", to_hex_str(lsuArgs.offset)}; + } // For stores, the data source (rs2) carries the type info. bool src_float = (instr.get_src_reg(1).type == RegType::Float); if (src_float) { @@ -1216,6 +1219,38 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { // known-violation handoff class, checked against its decoded destination. gfx_doctrine::check(*instr); } break; +#if defined(VX_CFG_EXT_F_ENABLE) && !defined(VX_CFG_FLEN_64) + case Opcode::EXT4: { + // vx_flw.d/.q (funct3 3/4) and vx_fsw.d/.q (funct3 5/6): a macro-op of 2 + // or 4 flw/fsw uops over an aligned FP register group. The width is the + // access's log2 bytes, as for fld/flq. + bool is_load = (funct3 == 3 || funct3 == 4); + if (!is_load && funct3 != 5 && funct3 != 6) { + std::abort(); + } + uint32_t width = is_load ? funct3 : (funct3 - 2); + uint32_t words = 1u << (width - 2); + uint32_t reg = is_load ? rd : rs2; + uint32_t imm12 = is_load ? (code >> shift_rs2) : ((funct7 << width_reg) | rd); + auto offset = (int32_t)sext(imm12, width_i_imm); + if ((reg % words) != 0 || (offset % int32_t(4 * words)) != 0) { + std::cout << "Error: " << (is_load ? "vx_flw." : "vx_fsw.") << ((words == 4) ? "q" : "d") + << " needs f" << reg << " aligned to " << words << " registers and offset " + << offset << " to " << (4 * words) << " bytes" << std::endl; + std::abort(); + } + instr->set_fu_type(FUType::LSU); + instr->set_op_type(is_load ? LsuType::LOAD : LsuType::STORE); + instr->set_src_reg(0, rs1, RegType::Integer); + if (is_load) { + instr->set_dest_reg(rd, RegType::Float); + } else { + instr->set_src_reg(1, rs2, RegType::Float); + } + instr->set_args(IntrLsuArgs{width, /*stride*/ 0, offset}); + instr->set_macro_op(); + } break; +#endif #ifdef VX_CFG_EXT_MUFU_ENABLE case Opcode::EXT3: { if (funct7 != 0 || funct3 != 0 || rs2 > uint32_t(MufuType::TANH)) { @@ -1226,27 +1261,6 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_dest_reg(rd, RegType::Float); instr->set_src_reg(0, rs1, RegType::Float); } break; -#endif -#if defined(VX_CFG_LMEM_ENABLE) && defined(VX_CFG_EXT_F_ENABLE) && defined(VX_CFG_XLEN_32) - case Opcode::EXT4: { - // vx_flds.d (funct3 3) and vx_flds.q (funct3 4) load words into an - // aligned pair or quad of FP registers. - if (funct3 != 3 && funct3 != 4) { - std::abort(); - } - uint32_t nregs = (funct3 == 4) ? 4 : 2; - if (rd % nregs != 0) { - std::cout << "Error: vx_flds." << ((nregs == 4) ? "q" : "d") << " destination f" << rd - << " is not aligned to " << nregs << " registers" << std::endl; - std::abort(); - } - uint32_t imm12 = code >> shift_rs2; - instr->set_fu_type(FUType::LSU); - instr->set_op_type(LsuType::LOAD); - instr->set_dest_reg(rd, RegType::Float); - instr->set_src_reg(0, rs1, RegType::Integer); - instr->set_args(IntrLsuArgs{/*width=LW*/ 2, /*stride*/ 0, (int32_t)sext(imm12, width_i_imm), /*is_local*/ true, nregs}); - } break; #endif default: std::abort(); diff --git a/sim/simx/instr_trace.h b/sim/simx/instr_trace.h index 8db44adbe2..63a275bba8 100644 --- a/sim/simx/instr_trace.h +++ b/sim/simx/instr_trace.h @@ -52,14 +52,6 @@ struct instr_trace_t { // Byte write enable uint8_t dst_bytesel; - // Registers from dst_reg up that the scoreboard reserves and releases: a - // wide local load reserves its quad or pair at issue, and each word-phase - // releases its own register. - uint32_t dst_span; - - // A wide local load's words 1..3, per thread, until their phase retires. - std::vector wide_data; - std::shared_ptr instr_ptr; int pid; @@ -98,7 +90,6 @@ struct instr_trace_t { , src_data(NUM_SRC_REGS, std::vector(VX_CFG_NUM_THREADS)) , dst_data(VX_CFG_NUM_THREADS) , dst_bytesel(0xFF) - , dst_span(1) , pid(-1) , sop(true) , eop(true) @@ -125,8 +116,6 @@ struct instr_trace_t { , src_data(rhs.src_data) , dst_data(rhs.dst_data) , dst_bytesel(rhs.dst_bytesel) - , dst_span(rhs.dst_span) - , wide_data(rhs.wide_data) , instr_ptr(rhs.instr_ptr) , pid(rhs.pid) , sop(rhs.sop) diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 8538c79669..d538e5ede7 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -36,7 +36,11 @@ uint32_t LsuUopGen::uop_count(const Instr& instr) { } // PACKLB.F: width=0 (LB), 4 byte-elements // PACKLH.F: width=1 (LH), 2 halfword-elements + // VX_FLW/VX_FSW.D: width=3, 2 words; .Q: width=4, 4 words auto args = std::get(instr.get_args()); + if (args.width >= 3) { + return 1u << (args.width - 2); + } return (args.width == 0) ? 4 : 2; } @@ -76,6 +80,22 @@ Instr::Ptr LsuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { } auto args = std::get(macro_instr.get_args()); + if (args.width >= 3) { + // a wide access's uop k moves word k: register fd/fs2 + k, offset + 4k + bool is_load = (std::get(macro_instr.get_op_type()) == LsuType::LOAD); + auto uop_instr = std::allocate_shared(pool_, uop_uuid, FUType::LSU); + uop_instr->set_parent_uuid(parent_uuid); + uop_instr->set_op_type(macro_instr.get_op_type()); + uop_instr->set_src_reg(0, macro_instr.get_src_reg(0).idx, RegType::Integer); + if (is_load) { + uop_instr->set_dest_reg(macro_instr.get_dest_reg().idx + uop_index, RegType::Float); + } else { + uop_instr->set_src_reg(1, macro_instr.get_src_reg(1).idx + uop_index, RegType::Float); + } + uop_instr->set_args(IntrLsuArgs{/*LW*/ 2, /*stride*/ 0, args.offset + int32_t(4 * uop_index)}); + return uop_instr; + } + // Macro encodes elem-size as width: 0=byte (PACKLB), 1=halfword (PACKLH). // The uop is a regular unsigned load (LBU/LHU = width + 4) so the LSU // doesn't sign-extend the loaded byte/halfword before the bytesel write. @@ -161,27 +181,21 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { const bool is_cas = is_amo && (*amo_tag == AmoType::AMOCAS); uint32_t num_threads = VX_CFG_NUM_THREADS; uint32_t data_bytes; - uint32_t align_bytes; bool is_write; int64_t offset; uint32_t stride; - [[maybe_unused]] bool is_local = false; if (is_amo) { auto amo_args = std::get(trace->instr_ptr->get_args()); data_bytes = 1u << (amo_args.width & 0x3); - align_bytes = data_bytes; is_write = false; // AMO never writes via the regular store path offset = 0; // AMO has no immediate offset stride = 0; // single-word access — addr = rs1[t] } else { auto lsu_args = std::get(trace->instr_ptr->get_args()); data_bytes = 1u << (lsu_args.width & 0x3); - // a wide local load reads its words from one aligned group per lane - align_bytes = data_bytes * lsu_args.nregs; is_write = (lsu_tag && *lsu_tag == LsuType::STORE); offset = lsu_args.offset; stride = lsu_args.stride; - is_local = lsu_args.is_local; if (is_write && lsu_args.width > 3) std::abort(); } @@ -205,14 +219,6 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { // upper bits into the 64-bit address field. e.addr = Word(rs1_data[t].i + (uint64_t)stride * rs2_data[t].u + offset); e.size = data_bytes; - #ifdef VX_CFG_LMEM_ENABLE - if (is_local) { - // Local memory decodes only the low bits; the window address - // carries the request to it. - constexpr uint64_t lmem_mask = (uint64_t(1) << VX_CFG_LMEM_LOG_SIZE) - 1; - e.addr = uint64_t(VX_MEM_LMEM_BASE_ADDR) | (e.addr & lmem_mask); - } else - #endif if (StackInterleave::contains(e.addr)) { // Interleaving is word-granular; a wider access would straddle two threads. if (e.size > StackInterleave::WORD_SIZE) { @@ -228,9 +234,9 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { // The hardware LSU enforces the same contract, so modelling // anything looser here would let a shader pass in simulation and fail // on the device. - if ((e.addr % align_bytes) != 0) { + if ((e.addr % e.size) != 0) { std::cout << "Error: misaligned memory access: addr=0x" << std::hex - << e.addr << std::dec << ", size=" << align_bytes << std::endl; + << e.addr << std::dec << ", size=" << e.size << std::endl; std::abort(); } if (is_write || is_amo) { @@ -313,18 +319,13 @@ void LsuUnit::process_response_step(uint32_t b) { : (uint32_t)__builtin_ctz(data_bs); bool nan_box = dst_float && (data_bytes < 8); bool half_float = VX_CFG_EXT_ZFHMIN_ENABLED && dst_float && (entry.lsu_args.width == 1); - // a wide local load's phase k returns word k of each lane's group - uint32_t phase = lsu_rsp.phase; - if (phase != 0 && trace->wide_data.empty()) { - trace->wide_data.resize(3 * VX_CFG_NUM_THREADS, 0); - } for (uint32_t lane = 0; lane < lsu_rsp.mask.size(); ++lane) { if (!lsu_rsp.mask.test(lane)) continue; const auto& lane_info = entry.lanes.at(lane); assert(lsu_rsp.data.at(lane) && "LOAD response must carry line payload"); - uint32_t off = (lane_info.addr & (VX_CFG_MEM_BLOCK_SIZE - 1)) + phase * data_bytes; + uint32_t off = lane_info.addr & (VX_CFG_MEM_BLOCK_SIZE - 1); uint64_t read_data = 0; std::memcpy(&read_data, lsu_rsp.data.at(lane)->data() + off, data_bytes); // Format the loaded value at low bits per RISC-V load semantics. @@ -351,14 +352,10 @@ void LsuUnit::process_response_step(uint32_t b) { std::abort(); } // Place at bytesel-specified position; OR-in NaN-box for Float dest. + auto& dst = entry.trace->dst_data.at(lane_info.tid); uint32_t box_bits = half_float ? 16 : 32; - uint64_t value = (formatted << (8 * byte_off)) - | (nan_box ? nanbox::box(0, box_bits) : 0); - if (phase != 0) { - trace->wide_data.at(lane_info.tid * 3 + phase - 1) = value; - } else { - trace->dst_data.at(lane_info.tid).u64 = value; - } + dst.u64 = (formatted << (8 * byte_off)) + | (nan_box ? nanbox::box(0, box_bits) : 0); } } entry.count -= lsu_rsp.mask.count(); // track remaining @@ -366,27 +363,7 @@ void LsuUnit::process_response_step(uint32_t b) { if (is_final) { state.pending_reqs.release(lsu_rsp.tag); } - if (entry.is_load && entry.lsu_args.nregs > 1 && lsu_rsp.phase_last) { - // A wide load's word-phase is complete: its beat writes register - // fd + phase and releases it, so a consumer of fd need not wait for - // fd + 3. The last phase's beat retires the instruction. - instr_trace_t* beat = trace; - if (!is_final) { - auto beat_alloc = core_->trace_pool().allocate(1); - beat = new (beat_alloc) instr_trace_t(*trace); - beat->eop = false; - } - uint32_t phase = lsu_rsp.phase; - if (phase != 0) { - for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { - beat->dst_data.at(t).u64 = trace->wide_data.at(t * 3 + phase - 1); - } - } - beat->dst_reg.idx += phase; - beat->dst_span = 1; - beat->wide_data.clear(); - output.send(beat, 2); - } else if (is_final && entry.eop) { + if (is_final && entry.eop) { // the response buffer, then the result arbiter's register output.send(trace, 2); } else { @@ -667,12 +644,8 @@ void LsuUnit::process_request_step(uint32_t b) { // signed/unsigned into one op + flag. lsu_req.op = is_amo ? amo_to_memop(*amo_tag) : (is_write ? MemOp::ST : MemOp::LD); - uint32_t nregs = 1; if (is_amo) { lsu_req.flags.amo_unsigned = amo_is_unsigned(*amo_tag); - } else { - nregs = std::get(trace->instr_ptr->get_args()).nregs; - lsu_req.flags.nwords_m1 = nregs - 1; } uint32_t t0 = state.addr_list.size() - state.remain_addrs; uint32_t beat_n = std::min(VX_CFG_NUM_LSU_LANES, state.remain_addrs); @@ -709,13 +682,6 @@ void LsuUnit::process_request_step(uint32_t b) { uint32_t count = lsu_req.mask.count(); bool is_eop = (state.remain_addrs == 0); - // A wide load's phases release its registers, so all its lanes must - // ride one request; the dispatcher's lane split guarantees it. - if (nregs > 1 && !is_eop) { - std::abort(); - } - // every word-phase answers each lane once - uint32_t rsp_count = count * nregs; uint32_t tag = 0; if (!is_write || is_amo) { @@ -736,7 +702,7 @@ void LsuUnit::process_request_step(uint32_t b) { --state.staged_reads; state.head_staged = false; } - tag = state.pending_reqs.allocate({trace, rsp_count, is_eop, std::move(lane_entries), entry_args, true}); + tag = state.pending_reqs.allocate({trace, count, is_eop, std::move(lane_entries), entry_args, true}); } lsu_req.tag = tag; lsu_req.cid = trace->cid; @@ -751,7 +717,7 @@ void LsuUnit::process_request_step(uint32_t b) { core_->perf_stats().stores += count; } else { core_->perf_stats().loads += count; - pending_loads_ += rsp_count; + pending_loads_ += count; } } diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index e568ab4a06..337b574605 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -80,7 +80,6 @@ class LocalMem::Impl { // the next pass's bank winners are known: a pass that leaves lanes // pending is followed by a cycle that finds the next ones bool ready = false; - uint32_t phase = 0; // a wide load's word-phase in service std::deque rsp_queue; }; @@ -145,9 +144,7 @@ class LocalMem::Impl { // a plain read also serves every pending lane reading the same word. A // store or an atomic takes its bank alone, so lanes writing one word land // in lane order and the highest lane's bytes are the ones left. - // A wide load's word-phase reads at a fixed byte offset from each lane's - // aligned group, so its conflicts are those of the group addresses. - BitVector<> select_pass(const LsuReq& req, const BitVector<>& pending, uint32_t offset) const { + BitVector<> select_pass(const LsuReq& req, const BitVector<>& pending) const { uint32_t num_banks = 1u << config_.B; std::vector winner(num_banks, -1); BitVector<> served(pending.size()); @@ -156,12 +153,11 @@ class LocalMem::Impl { if (!pending.test(i)) { continue; } - uint64_t addr = req.addrs.at(i) + offset; - uint32_t b = this->bank_of(addr); + uint32_t b = this->bank_of(req.addrs.at(i)); if (winner.at(b) == -1) { winner.at(b) = i; served.set(i); - } else if (join && this->bank_word(addr) == this->bank_word(req.addrs.at(winner.at(b)) + offset)) { + } else if (join && this->bank_word(req.addrs.at(i)) == this->bank_word(req.addrs.at(winner.at(b)))) { served.set(i); } } @@ -317,12 +313,11 @@ class LocalMem::Impl { if (has_rsp && rsp_count >= RSP_QUEUE_SIZE) { return false; } - auto served = this->select_pass(req, e.pending, e.phase * config_.word_size); + auto served = this->select_pass(req, e.pending); LsuRsp rsp(config_.num_lanes); - rsp.tag = req.tag; - rsp.cid = req.cid; - rsp.uuid = req.uuid; - rsp.phase = e.phase; + rsp.tag = req.tag; + rsp.cid = req.cid; + rsp.uuid = req.uuid; for (uint32_t i = 0; i < config_.num_lanes; ++i) { if (!served.test(i)) { continue; @@ -334,8 +329,7 @@ class LocalMem::Impl { } e.pending.reset(i); } - rsp.phase_last = e.pending.none(); - DT(4, simobject_->name() << " pass: phase=" << e.phase << ", served=" << served << ", " << req); + DT(4, simobject_->name() << " pass: served=" << served << ", " << req); if (has_rsp) { e.rsp_queue.push_back({SimPlatform::instance().cycles() + RSP_LATENCY, e.s1.block, std::move(rsp)}); } @@ -345,13 +339,6 @@ class LocalMem::Impl { } #endif if (e.pending.none()) { - if (e.phase < req.flags.nwords_m1) { - // the next word-phase serves every lane again, from the next cycle - ++e.phase; - e.pending = req.mask; - return false; - } - e.phase = 0; e.s1_valid = false; return true; } @@ -472,7 +459,6 @@ class LocalMem::Impl { e.s0.clear(); e.s1_valid = false; e.pending.reset(); - e.phase = 0; e.rsp_queue.clear(); } for (auto& r : rsp_regs_) { diff --git a/sim/simx/scoreboard.cpp b/sim/simx/scoreboard.cpp index 6c76af3e2b..7701aa15cb 100644 --- a/sim/simx/scoreboard.cpp +++ b/sim/simx/scoreboard.cpp @@ -19,14 +19,10 @@ using namespace vortex; Scoreboard::Scoreboard(const SimContext& ctx, const char* name) : SimObject(ctx, name) , in_use_regs_(VX_CFG_NUM_WARPS) - , in_use_groups_(VX_CFG_NUM_WARPS) , in_use_fcsr_(VX_CFG_NUM_WARPS, 0) { for (auto& in_use_reg : in_use_regs_) { in_use_reg.resize((int)RegType::Count); } - for (auto& in_use_group : in_use_groups_) { - in_use_group.resize((int)RegType::Count); - } this->on_reset(); } @@ -39,16 +35,10 @@ void Scoreboard::on_reset() { mask.reset(); } } - for (auto& in_use_group : in_use_groups_) { - for (auto& mask : in_use_group) { - mask.reset(); - } - } std::fill(in_use_fcsr_.begin(), in_use_fcsr_.end(), 0); owners_.clear(); commit_counts_.clear(); pending_releases_.clear(); - pending_group_releases_.clear(); } void Scoreboard::on_tick() { @@ -63,15 +53,7 @@ void Scoreboard::on_tick() { } pending_releases_.pop_front(); } - while (!pending_group_releases_.empty() && pending_group_releases_.front().due <= now) { - auto& r = pending_group_releases_.front(); - // a register reserved again since its release stays in use - if (!in_use_regs_.at(r.wid).at((int)r.reg.type).test(r.reg.idx)) { - in_use_groups_.at(r.wid).at((int)r.reg.type).reset(r.reg.idx); - } - pending_group_releases_.pop_front(); - } - if (pending_releases_.empty() && pending_group_releases_.empty()) { + if (pending_releases_.empty()) { this->tick_sleep(); } } @@ -83,11 +65,8 @@ bool Scoreboard::in_use(instr_trace_t* trace) const { } if (trace->wb) { assert(trace->dst_reg.type != RegType::None); - auto& dst_in_use = (trace->dst_span > 1) ? in_use_groups_ : in_use_regs_; - for (uint32_t i = 0; i < trace->dst_span; ++i) { - if (dst_in_use.at(trace->wid).at((int)trace->dst_reg.type).test(trace->dst_reg.idx + i)) { - return true; - } + if (in_use_regs_.at(trace->wid).at((int)trace->dst_reg.type).test(trace->dst_reg.idx)) { + return true; } } for (uint32_t i = 0; i < trace->src_regs.size(); ++i) { @@ -104,12 +83,10 @@ std::vector Scoreboard::get_uses(instr_trace_t* trace) co std::vector out; if (trace->wb) { assert(trace->dst_reg.type != RegType::None); - for (uint32_t i = 0; i < trace->dst_span; ++i) { - RegOpd reg{trace->dst_reg.type, trace->dst_reg.idx + i}; - if (in_use_regs_.at(trace->wid).at((int)reg.type).test(reg.idx)) { - auto owner = owners_.at(get_reg_id(reg, trace->wid)); - out.push_back({reg.type, reg.idx, owner->fu_type, owner->op_type, owner->uuid}); - } + if (in_use_regs_.at(trace->wid).at((int)trace->dst_reg.type).test(trace->dst_reg.idx)) { + uint32_t reg_id = get_reg_id(trace->dst_reg, trace->wid); + auto owner = owners_.at(reg_id); + out.push_back({trace->dst_reg.type, trace->dst_reg.idx, owner->fu_type, owner->op_type, owner->uuid}); } } for (uint32_t i = 0; i < trace->src_regs.size(); ++i) { @@ -125,29 +102,21 @@ std::vector Scoreboard::get_uses(instr_trace_t* trace) co } void Scoreboard::reserve(instr_trace_t* trace) { + uint32_t reg_id = get_reg_id(trace->dst_reg, trace->wid); assert(trace->wb); - for (uint32_t i = 0; i < trace->dst_span; ++i) { - RegOpd reg{trace->dst_reg.type, trace->dst_reg.idx + i}; - uint32_t reg_id = get_reg_id(reg, trace->wid); - in_use_regs_.at(trace->wid).at((int)reg.type).set(reg.idx); - in_use_groups_.at(trace->wid).at((int)reg.type).set(reg.idx); - assert(owners_.count(reg_id) == 0); - owners_[reg_id] = trace; - } + in_use_regs_.at(trace->wid).at((int)trace->dst_reg.type).set(trace->dst_reg.idx); + assert(owners_.count(reg_id) == 0); + owners_[reg_id] = trace; } void Scoreboard::release(instr_trace_t* trace) { + uint32_t reg_id = get_reg_id(trace->dst_reg, trace->wid); assert(trace->wb); - commit_counts_.erase(get_reg_id(trace->dst_reg, trace->wid)); - for (uint32_t i = 0; i < trace->dst_span; ++i) { - RegOpd reg{trace->dst_reg.type, trace->dst_reg.idx + i}; - assert(in_use_regs_.at(trace->wid).at((int)reg.type).test(reg.idx)); - assert(owners_.count(get_reg_id(reg, trace->wid)) != 0); - pending_releases_.push_back({trace->wid, reg, 0, - SimPlatform::instance().cycles() + kReleaseDelay}); - pending_group_releases_.push_back({trace->wid, reg, 0, - SimPlatform::instance().cycles() + kGroupReleaseDelay}); - } + assert(in_use_regs_.at(trace->wid).at((int)trace->dst_reg.type).test(trace->dst_reg.idx)); + assert(owners_.count(reg_id) != 0); + commit_counts_.erase(reg_id); + pending_releases_.push_back({trace->wid, trace->dst_reg, 0, + SimPlatform::instance().cycles() + kReleaseDelay}); this->tick_wake(); } diff --git a/sim/simx/scoreboard.h b/sim/simx/scoreboard.h index 377a019e32..d5514bc654 100644 --- a/sim/simx/scoreboard.h +++ b/sim/simx/scoreboard.h @@ -72,10 +72,6 @@ class Scoreboard : public SimObject { // A committed result is written back from the commit arbiter's output // register, and the issue stage reads a registered readiness. static constexpr uint32_t kReleaseDelay = 2; - // A wide load tests its destination group against the in-use state as - // registered at the start of the cycle, so it sees a release one cycle - // after other instructions do. - static constexpr uint32_t kGroupReleaseDelay = kReleaseDelay + 1; struct pending_release_t { uint32_t wid; @@ -85,10 +81,8 @@ class Scoreboard : public SimObject { }; std::vector> in_use_regs_; - std::vector> in_use_groups_; std::vector in_use_fcsr_; std::deque pending_releases_; - std::deque pending_group_releases_; std::unordered_map owners_; std::unordered_map commit_counts_; diff --git a/sim/simx/types.h b/sim/simx/types.h index 4042915be0..60fd7b531a 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -384,8 +384,6 @@ struct IntrLsuArgs { uint32_t width : 3; uint32_t stride : 4; // per-uop multiplier: 0 = regular LD/ST; 0..N-1 = packLD uop_idx int32_t offset; // immediate addr offset (signed); 0 for packLD uops - bool is_local = false; // wide local load: local memory at the low address bits - uint32_t nregs = 1; // wide local load: registers fd.. loaded one word-phase each }; // A fence carries no access width; its width field says what it orders. @@ -470,7 +468,6 @@ struct MemFlags { uint32_t dxa_notify_done : 1; // bit 7 uint32_t dxa_notify_bar_id : 16; // bits 8..23 #endif - uint32_t nwords_m1 : 2; // wide local load: words per lane - 1, served in word-phases }; }; @@ -1212,10 +1209,6 @@ struct LsuRsp { uint64_t tag; uint32_t cid; uint64_t uuid; - // Wide local load: the word-phase this pass served, and whether it was the - // phase's last pass. - uint32_t phase; - bool phase_last; // TLM data: per-lane block populated for loads/AMO returns. std::vector> data; @@ -1225,8 +1218,6 @@ struct LsuRsp { , tag (0) , cid(0) , uuid(0) - , phase(0) - , phase_last(true) , data(size) {} diff --git a/sw/kernel/include/vx_intrinsics.h b/sw/kernel/include/vx_intrinsics.h index b4ed86b144..61ce362a14 100644 --- a/sw/kernel/include/vx_intrinsics.h +++ b/sw/kernel/include/vx_intrinsics.h @@ -636,6 +636,36 @@ inline float vx_packlh_f(const void* base, size_t stride) { return out; } +// +// Wide loads and stores +// + +// Two or four consecutive words, each 4-byte aligned. A compiler with the +// wide load/store extension emits one vx_flw.d/.q or vx_fsw.d/.q for each +// access; any other splits it into flw/fsw. +typedef float vx_float2 __attribute__((vector_size(8), aligned(4))); +typedef float vx_float4 __attribute__((vector_size(16), aligned(4))); + +__attribute__((always_inline)) +inline vx_float2 vx_flw_d(const void* addr) { + return *(const vx_float2*)addr; +} + +__attribute__((always_inline)) +inline vx_float4 vx_flw_q(const void* addr) { + return *(const vx_float4*)addr; +} + +__attribute__((always_inline)) +inline void vx_fsw_d(void* addr, vx_float2 value) { + *(vx_float2*)addr = value; +} + +__attribute__((always_inline)) +inline void vx_fsw_q(void* addr, vx_float4 value) { + *(vx_float4*)addr = value; +} + #ifdef __cplusplus } #endif diff --git a/tests/regression/Makefile b/tests/regression/Makefile index df61d1eb34..56f2f10dc4 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax uniform_ld lmem_access lds_wide mufu zfhmin \ + softmax uniform_ld lmem_access ldst_wide mufu zfhmin \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/lds_wide/common.h b/tests/regression/lds_wide/common.h deleted file mode 100644 index 9a21bb4932..0000000000 --- a/tests/regression/lds_wide/common.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef _COMMON_H_ -#define _COMMON_H_ - -#include - -// Local memory per block: RO_WORDS read-only words, then WIDE_WORDS words per -// thread that it stores to and reads back. RO_WORDS keeps them 16-byte aligned. -#define RO_WORDS 1024 -#define WIDE_WORDS 4 - -// Initial value of local word i: words differ from each other and about half -// of them have the sign bit set. -static inline uint32_t ro_word(uint32_t i) { - return (i * 0x9E3779B1u) ^ 0xA5C30000u; -} - -// The words a thread stores to its wide slot. -static inline uint32_t wide_word(uint32_t tid, uint32_t k) { - return 0x6C000000u | (tid << 4) | k; -} - -typedef struct { - uint32_t lmem_words; // local-memory words per block - uint64_t err_addr; // out: per-thread error count - uint64_t dump_addr; // out: each block's final local memory - uint64_t wide_addr; // out: 1 if the kernel reads with wide loads -} kernel_arg_t; - -#endif diff --git a/tests/regression/lds_wide/kernel.cpp b/tests/regression/lds_wide/kernel.cpp deleted file mode 100644 index b57a771660..0000000000 --- a/tests/regression/lds_wide/kernel.cpp +++ /dev/null @@ -1,237 +0,0 @@ -#include -#include "common.h" - -// vx_flds.d / vx_flds.q (custom-3), written as .insn so the encodings are -// tested independently of the compiler. Both are I-type like fld/flq: -// funct3 = 3 / 4, imm[11:0] = signed byte offset; fd is an aligned pair / -// quad, the address 8 / 16-byte aligned. -#if (__riscv_xlen == 32) && defined(__riscv_f) -#define HAS_FLDS 1 -#endif - -// A wide load and the reads of its registers sit in one asm block, so the -// compiler cannot reuse them in between. fd + 3 (fd + 1) is read first: it is -// the last phase's register. -#define VX_FLDS_Q(r0, r1, r2, r3, off, p, w) \ - asm volatile (".insn i 0x7b, 4, " r0 ", %4, %5\n\t" \ - "fmv.x.w %3, " r3 "\n\t" \ - "fmv.x.w %0, " r0 "\n\t" \ - "fmv.x.w %1, " r1 "\n\t" \ - "fmv.x.w %2, " r2 \ - : "=&r"((w)[0]), "=&r"((w)[1]), "=&r"((w)[2]), "=&r"((w)[3]) \ - : "r"(p), "i"(off) \ - : r0, r1, r2, r3, "memory") - -#define VX_FLDS_D(r0, r1, off, p, w) \ - asm volatile (".insn i 0x7b, 3, " r0 ", %2, %3\n\t" \ - "fmv.x.w %1, " r1 "\n\t" \ - "fmv.x.w %0, " r0 \ - : "=&r"((w)[0]), "=&r"((w)[1]) \ - : "r"(p), "i"(off) \ - : r0, r1, "memory") - -#ifdef HAS_FLDS -static inline uint32_t ro_misses(const uint32_t* w, uint32_t n, uint32_t first) { - uint32_t misses = 0; - for (uint32_t k = 0; k < n; ++k) { - if (w[k] != ro_word(first + k)) { - ++misses; - } - } - return misses; -} -#else -static inline uint32_t lmem_misses(volatile const uint32_t* lmem, uint32_t n, uint32_t first) { - uint32_t misses = 0; - for (uint32_t k = 0; k < n; ++k) { - if (lmem[first + k] != ro_word(first + k)) { - ++misses; - } - } - return misses; -} -#endif - -// Each thread checks the words its wide loads return against the values the -// host and the generic path define: -// - a warp-uniform group (broadcast), a 16-byte lane stride (every lane on -// the same banks), a lane permutation, half the lanes under a divergent -// branch; .d and .q through the window address and the bare offset (local -// memory decodes only the low bits) -// - fd = f0 and f28; two wide loads back to back -// - scalar writes to fd + 2 just before and just after -// - a wide load, an FP op on fd + 2 that waits for it, and a wide load of -// the same group again -// - the offset limits, -2048 and 2032 (.q) or 2040 (.d), through the window -// address and the bare offset -// - a generic store, then a wide load of the same words -// Without wide loads, generic loads read the same words. -__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - auto lmem = reinterpret_cast(__local_mem()); - auto err = reinterpret_cast(arg->err_addr); - auto dump = reinterpret_cast(arg->dump_addr); - const uint32_t lid = threadIdx.x; - const uint32_t bsize = blockDim.x; - const uint32_t tid = blockIdx.x * bsize + lid; - const uint32_t words = arg->lmem_words; - - if (tid == 0) { - #ifdef HAS_FLDS - *reinterpret_cast(arg->wide_addr) = 1; - #else - *reinterpret_cast(arg->wide_addr) = 0; - #endif - } - - for (uint32_t i = lid; i < RO_WORDS; i += bsize) { - lmem[i] = ro_word(i); - } - __syncthreads(); - - uint32_t errors = 0; - - // uniform, lane stride and permuted groups; gs + 7 and gq + 3 stay in the - // read-only words for any block size, and gq * 4 exceeds the largest offset - const uint32_t gu = 384; - const uint32_t gs = 256 + (lid % 128) * 4; - const uint32_t gp = 512 + ((lid * 7 + 3) % 64) * 4; - const uint32_t go = 512; - const uint32_t gq = 512 + (lid % 128) * 4; - const uint32_t wi = RO_WORDS + lid * WIDE_WORDS; - -#ifdef HAS_FLDS - { - const uintptr_t window = reinterpret_cast(lmem); - const uintptr_t bare = window - VX_MEM_LMEM_BASE_ADDR; - uint32_t w[8]; - - VX_FLDS_Q("f8", "f9", "f10", "f11", 0, window + gu * 4, w); - errors += ro_misses(w, 4, gu); - VX_FLDS_D("f8", "f9", 8, bare + gu * 4, w); - errors += ro_misses(w, 2, gu + 2); - - VX_FLDS_Q("f12", "f13", "f14", "f15", 0, window + gs * 4, w); - errors += ro_misses(w, 4, gs); - VX_FLDS_D("f14", "f15", 0, window + gs * 4, w); - errors += ro_misses(w, 2, gs); - - VX_FLDS_Q("f16", "f17", "f18", "f19", 0, bare + gp * 4, w); - errors += ro_misses(w, 4, gp); - VX_FLDS_D("f16", "f17", 8, window + gp * 4, w); - errors += ro_misses(w, 2, gp + 2); - - if (lid & 1) { - VX_FLDS_Q("f20", "f21", "f22", "f23", 0, window + gs * 4, w); - errors += ro_misses(w, 4, gs); - VX_FLDS_D("f22", "f23", 8, window + gp * 4, w); - errors += ro_misses(w, 2, gp + 2); - } - - VX_FLDS_Q("f0", "f1", "f2", "f3", 0, window + gp * 4, w); - errors += ro_misses(w, 4, gp); - VX_FLDS_Q("f28", "f29", "f30", "f31", 0, bare + gs * 4, w); - errors += ro_misses(w, 4, gs); - VX_FLDS_D("f30", "f31", 0, window + gu * 4, w); - errors += ro_misses(w, 2, gu); - - asm volatile (".insn i 0x7b, 4, f8, %8, 0\n\t" - ".insn i 0x7b, 4, f12, %8, 16\n\t" - "fmv.x.w %0, f8\n\t" - "fmv.x.w %1, f9\n\t" - "fmv.x.w %2, f10\n\t" - "fmv.x.w %3, f11\n\t" - "fmv.x.w %4, f12\n\t" - "fmv.x.w %5, f13\n\t" - "fmv.x.w %6, f14\n\t" - "fmv.x.w %7, f15" - : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]), - "=&r"(w[4]), "=&r"(w[5]), "=&r"(w[6]), "=&r"(w[7]) - : "r"(window + gs * 4) - : "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "memory"); - errors += ro_misses(w, 4, gs); - errors += ro_misses(w + 4, 4, gs + 4); - - const uint32_t marker = 0x7F00D000u | lid; - asm volatile ("fmv.w.x f10, %1\n\t" - ".insn i 0x7b, 4, f8, %2, 0\n\t" - "fmv.x.w %0, f10" - : "=&r"(w[2]) - : "r"(marker), "r"(window + gp * 4) - : "f8", "f9", "f10", "f11", "memory"); - if (w[2] != ro_word(gp + 2)) { - ++errors; - } - asm volatile (".insn i 0x7b, 4, f8, %4, 0\n\t" - "fmv.w.x f10, %5\n\t" - "fmv.x.w %0, f8\n\t" - "fmv.x.w %1, f9\n\t" - "fmv.x.w %2, f10\n\t" - "fmv.x.w %3, f11" - : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) - : "r"(window + gp * 4), "r"(marker) - : "f8", "f9", "f10", "f11", "memory"); - if (w[0] != ro_word(gp) || w[1] != ro_word(gp + 1) || w[2] != marker || w[3] != ro_word(gp + 3)) { - ++errors; - } - asm volatile (".insn i 0x7b, 4, f8, %4, 0\n\t" - "fadd.s f10, f10, f10\n\t" - ".insn i 0x7b, 4, f8, %4, 16\n\t" - "fmv.x.w %0, f8\n\t" - "fmv.x.w %1, f9\n\t" - "fmv.x.w %2, f10\n\t" - "fmv.x.w %3, f11" - : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) - : "r"(window + gs * 4) - : "f8", "f9", "f10", "f11", "memory"); - errors += ro_misses(w, 4, gs + 4); - - VX_FLDS_Q("f4", "f5", "f6", "f7", -2048, window + go * 4 + 2048, w); - errors += ro_misses(w, 4, go); - VX_FLDS_Q("f4", "f5", "f6", "f7", 2032, window + go * 4 - 2032, w); - errors += ro_misses(w, 4, go); - VX_FLDS_Q("f24", "f25", "f26", "f27", -2048, bare + gs * 4 + 2048, w); - errors += ro_misses(w, 4, gs); - VX_FLDS_Q("f24", "f25", "f26", "f27", 2032, bare + gq * 4 - 2032, w); - errors += ro_misses(w, 4, gq); - VX_FLDS_D("f26", "f27", -2048, window + gp * 4 + 2048, w); - errors += ro_misses(w, 2, gp); - VX_FLDS_D("f26", "f27", 2040, bare + gq * 4 - 2040, w); - errors += ro_misses(w, 2, gq); - - for (uint32_t k = 0; k < WIDE_WORDS; ++k) { - lmem[wi + k] = wide_word(tid, k); - } - VX_FLDS_Q("f16", "f17", "f18", "f19", 0, window + wi * 4, w); - VX_FLDS_D("f20", "f21", 8, bare + wi * 4, w + 4); - for (uint32_t k = 0; k < WIDE_WORDS; ++k) { - if (w[k] != wide_word(tid, k)) { - ++errors; - } - } - if (w[4] != wide_word(tid, 2) || w[5] != wide_word(tid, 3)) { - ++errors; - } - } -#else - errors += lmem_misses(lmem, 4, gu); - errors += lmem_misses(lmem, 8, gs); - errors += lmem_misses(lmem, 4, gp); - errors += lmem_misses(lmem, 4, go); - errors += lmem_misses(lmem, 4, gq); - for (uint32_t k = 0; k < WIDE_WORDS; ++k) { - lmem[wi + k] = wide_word(tid, k); - } - for (uint32_t k = 0; k < WIDE_WORDS; ++k) { - if (lmem[wi + k] != wide_word(tid, k)) { - ++errors; - } - } -#endif - - err[tid] = errors; - - __syncthreads(); - for (uint32_t i = lid; i < words; i += bsize) { - dump[blockIdx.x * words + i] = lmem[i]; - } -} diff --git a/tests/regression/lds_wide/Makefile b/tests/regression/ldst_wide/Makefile similarity index 91% rename from tests/regression/lds_wide/Makefile rename to tests/regression/ldst_wide/Makefile index 5e27cb0b06..63ce26deef 100644 --- a/tests/regression/lds_wide/Makefile +++ b/tests/regression/ldst_wide/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk -PROJECT := lds_wide +PROJECT := ldst_wide SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) diff --git a/tests/regression/ldst_wide/common.h b/tests/regression/ldst_wide/common.h new file mode 100644 index 0000000000..b66d79a3f7 --- /dev/null +++ b/tests/regression/ldst_wide/common.h @@ -0,0 +1,47 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Read-only words the wide loads read, in local memory and in a global buffer, +// followed in local memory by a SLOT_WORDS-word slot per thread that the wide +// stores fill; each thread has a slot in a global buffer too. +#define RO_WORDS 1024 +#define SLOT_WORDS 16 + +// The global read-only words differ from the local ones. +#define GLOBAL_SALT 0x0F0F1234u + +// Read-only word i: words differ from each other and about half of them have +// the sign bit set. +static inline uint32_t ro_word(uint32_t i) { + return (i * 0x9E3779B1u) ^ 0xA5C30000u; +} + +static inline uint32_t wide_word(uint32_t tid, uint32_t k) { + return 0x6C000000u | (tid << 4) | k; +} + +static inline uint32_t alt_word(uint32_t tid, uint32_t k) { + return 0x3A000000u | (tid << 4) | k; +} + +// A slot's final word k: odd lanes overwrite words 0..3 with a second wide +// store, and a scalar store overwrites word 7 after the wide store of 5..8. +static inline uint32_t slot_word(uint32_t tid, uint32_t lid, uint32_t k) { + if ((k < 4 && (lid & 1)) || k == 7) { + return alt_word(tid, k); + } + return wide_word(tid, k); +} + +typedef struct { + uint32_t lmem_words; // local-memory words per block + uint64_t src_addr; // global read-only words + uint64_t dst_addr; // global slots + uint64_t err_addr; // out: per-thread error count + uint64_t dump_addr; // out: each block's final local memory + uint64_t wide_addr; // out: 1 if the kernel uses the wide forms +} kernel_arg_t; + +#endif diff --git a/tests/regression/ldst_wide/kernel.cpp b/tests/regression/ldst_wide/kernel.cpp new file mode 100644 index 0000000000..b98a43c176 --- /dev/null +++ b/tests/regression/ldst_wide/kernel.cpp @@ -0,0 +1,298 @@ +#include +#include "common.h" + +// vx_flw.d/.q (custom-3 funct3 3/4, I-type like fld/flq) and vx_fsw.d/.q +// (funct3 5/6, S-type like fsd/fsq), written as .insn so the encodings are +// tested independently of the compiler. fd/fs2 is an aligned pair or quad and +// imm a multiple of 8 or 16; each word needs only 4-byte alignment. +#if defined(__riscv_f) && (__riscv_flen == 32) +#define HAS_WIDE 1 +#endif + +#ifdef HAS_WIDE +// A wide load and the reads of its registers sit in one asm block, so the +// compiler cannot reuse them in between. The last uop's register is read first. +#define VX_FLW_Q(r0, r1, r2, r3, off, p, w) \ + asm volatile (".insn i 0x7b, 4, " r0 ", %4, %5\n\t" \ + "fmv.x.w %3, " r3 "\n\t" \ + "fmv.x.w %0, " r0 "\n\t" \ + "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %2, " r2 \ + : "=&r"((w)[0]), "=&r"((w)[1]), "=&r"((w)[2]), "=&r"((w)[3]) \ + : "r"(p), "i"(off) \ + : r0, r1, r2, r3, "memory") + +#define VX_FLW_D(r0, r1, off, p, w) \ + asm volatile (".insn i 0x7b, 3, " r0 ", %2, %3\n\t" \ + "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %0, " r0 \ + : "=&r"((w)[0]), "=&r"((w)[1]) \ + : "r"(p), "i"(off) \ + : r0, r1, "memory") + +#define VX_FSW_Q(r0, r1, r2, r3, off, p, w) \ + asm volatile ("fmv.w.x " r0 ", %0\n\t" \ + "fmv.w.x " r1 ", %1\n\t" \ + "fmv.w.x " r2 ", %2\n\t" \ + "fmv.w.x " r3 ", %3\n\t" \ + ".insn s 0x7b, 6, " r0 ", %5(%4)" \ + : \ + : "r"((w)[0]), "r"((w)[1]), "r"((w)[2]), "r"((w)[3]), "r"(p), "i"(off) \ + : r0, r1, r2, r3, "memory") + +#define VX_FSW_D(r0, r1, off, p, w) \ + asm volatile ("fmv.w.x " r0 ", %0\n\t" \ + "fmv.w.x " r1 ", %1\n\t" \ + ".insn s 0x7b, 5, " r0 ", %3(%2)" \ + : \ + : "r"((w)[0]), "r"((w)[1]), "r"(p), "i"(off) \ + : r0, r1, "memory") +#else +// Without the wide forms, generic accesses move the same words. +#define VX_GENERIC_LD(n, off, p, w) \ + do { \ + auto _a = reinterpret_cast((p) + (off)); \ + for (uint32_t _k = 0; _k < (n); ++_k) { \ + (w)[_k] = _a[_k]; \ + } \ + } while (0) +#define VX_GENERIC_ST(n, off, p, w) \ + do { \ + auto _a = reinterpret_cast((p) + (off)); \ + for (uint32_t _k = 0; _k < (n); ++_k) { \ + _a[_k] = (w)[_k]; \ + } \ + } while (0) +#define VX_FLW_Q(r0, r1, r2, r3, off, p, w) VX_GENERIC_LD(4, off, p, w) +#define VX_FLW_D(r0, r1, off, p, w) VX_GENERIC_LD(2, off, p, w) +#define VX_FSW_Q(r0, r1, r2, r3, off, p, w) VX_GENERIC_ST(4, off, p, w) +#define VX_FSW_D(r0, r1, off, p, w) VX_GENERIC_ST(2, off, p, w) +#endif + +static inline uint32_t ro_misses(const uint32_t* w, uint32_t n, uint32_t first, uint32_t salt) { + uint32_t misses = 0; + for (uint32_t k = 0; k < n; ++k) { + if (w[k] != (ro_word(first + k) ^ salt)) { + ++misses; + } + } + return misses; +} + +// Wide loads of the read-only words at ro (local or global), checked against +// the words the host defines: +// - a warp-uniform group (broadcast), a 16-byte lane stride (every lane on +// the same banks), a lane permutation, half the lanes under a divergent +// branch, and addresses 4- but not 8- or 16-byte aligned +// - fd = f0 and f28; two wide loads back to back +// - scalar writes to fd + 2 just before and just after a wide load +// - a wide load, an FP op on fd + 2 that waits for it, and a wide load of +// the same group again +// - the offset limits, -2048 and 2032 (.q) or 2040 (.d) +static inline uint32_t check_loads(uintptr_t ro, uint32_t salt, uint32_t lid) { + // gs + 8, gp + 5 and gq + 3 stay in the read-only words for any block + // size, and gq * 4 exceeds the largest offset + const uint32_t gu = 384; + const uint32_t gs = 256 + (lid % 128) * 4; + const uint32_t gp = 512 + ((lid * 7 + 3) % 64) * 4; + const uint32_t go = 512; + const uint32_t gq = 512 + (lid % 128) * 4; + uint32_t errors = 0; + uint32_t w[8]; + + VX_FLW_Q("f8", "f9", "f10", "f11", 0, ro + gu * 4, w); + errors += ro_misses(w, 4, gu, salt); + VX_FLW_D("f8", "f9", 8, ro + gu * 4, w); + errors += ro_misses(w, 2, gu + 2, salt); + + VX_FLW_Q("f12", "f13", "f14", "f15", 0, ro + gs * 4, w); + errors += ro_misses(w, 4, gs, salt); + VX_FLW_D("f14", "f15", 0, ro + gs * 4, w); + errors += ro_misses(w, 2, gs, salt); + VX_FLW_Q("f12", "f13", "f14", "f15", 16, ro + gs * 4, w); + errors += ro_misses(w, 4, gs + 4, salt); + + VX_FLW_Q("f16", "f17", "f18", "f19", 0, ro + gp * 4, w); + errors += ro_misses(w, 4, gp, salt); + VX_FLW_D("f16", "f17", 8, ro + gp * 4, w); + errors += ro_misses(w, 2, gp + 2, salt); + + if (lid & 1) { + VX_FLW_Q("f20", "f21", "f22", "f23", 0, ro + gs * 4, w); + errors += ro_misses(w, 4, gs, salt); + VX_FLW_D("f22", "f23", 8, ro + gp * 4, w); + errors += ro_misses(w, 2, gp + 2, salt); + } + + VX_FLW_Q("f20", "f21", "f22", "f23", 16, ro + (gs + 1) * 4, w); + errors += ro_misses(w, 4, gs + 5, salt); + VX_FLW_Q("f24", "f25", "f26", "f27", 0, ro + (gp + 2) * 4, w); + errors += ro_misses(w, 4, gp + 2, salt); + VX_FLW_D("f26", "f27", 8, ro + (gq - 1) * 4, w); + errors += ro_misses(w, 2, gq + 1, salt); + + VX_FLW_Q("f0", "f1", "f2", "f3", 0, ro + gp * 4, w); + errors += ro_misses(w, 4, gp, salt); + VX_FLW_Q("f28", "f29", "f30", "f31", 0, ro + gs * 4, w); + errors += ro_misses(w, 4, gs, salt); + VX_FLW_D("f30", "f31", 0, ro + gu * 4, w); + errors += ro_misses(w, 2, gu, salt); + + VX_FLW_Q("f4", "f5", "f6", "f7", -2048, ro + go * 4 + 2048, w); + errors += ro_misses(w, 4, go, salt); + VX_FLW_Q("f4", "f5", "f6", "f7", 2032, ro + go * 4 - 2032, w); + errors += ro_misses(w, 4, go, salt); + VX_FLW_Q("f24", "f25", "f26", "f27", -2048, ro + (gs + 1) * 4 + 2048, w); + errors += ro_misses(w, 4, gs + 1, salt); + VX_FLW_Q("f24", "f25", "f26", "f27", 2032, ro + gq * 4 - 2032, w); + errors += ro_misses(w, 4, gq, salt); + VX_FLW_D("f26", "f27", -2048, ro + gp * 4 + 2048, w); + errors += ro_misses(w, 2, gp, salt); + VX_FLW_D("f26", "f27", 2040, ro + (gq + 1) * 4 - 2040, w); + errors += ro_misses(w, 2, gq + 1, salt); + +#ifdef HAS_WIDE + asm volatile (".insn i 0x7b, 4, f8, %8, 0\n\t" + ".insn i 0x7b, 4, f12, %8, 16\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11\n\t" + "fmv.x.w %4, f12\n\t" + "fmv.x.w %5, f13\n\t" + "fmv.x.w %6, f14\n\t" + "fmv.x.w %7, f15" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]), + "=&r"(w[4]), "=&r"(w[5]), "=&r"(w[6]), "=&r"(w[7]) + : "r"(ro + gs * 4) + : "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "memory"); + errors += ro_misses(w, 8, gs, salt); + + const uint32_t marker = 0x7F00D000u | lid; + asm volatile ("fmv.w.x f10, %1\n\t" + ".insn i 0x7b, 4, f8, %2, 0\n\t" + "fmv.x.w %0, f10" + : "=&r"(w[2]) + : "r"(marker), "r"(ro + gp * 4) + : "f8", "f9", "f10", "f11", "memory"); + errors += (w[2] != (ro_word(gp + 2) ^ salt)); + asm volatile (".insn i 0x7b, 4, f8, %4, 0\n\t" + "fmv.w.x f10, %5\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) + : "r"(ro + gp * 4), "r"(marker) + : "f8", "f9", "f10", "f11", "memory"); + errors += (w[0] != (ro_word(gp) ^ salt)) + (w[1] != (ro_word(gp + 1) ^ salt)) + + (w[2] != marker) + (w[3] != (ro_word(gp + 3) ^ salt)); + asm volatile (".insn i 0x7b, 4, f8, %4, 0\n\t" + "fadd.s f10, f10, f10\n\t" + ".insn i 0x7b, 4, f8, %4, 16\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) + : "r"(ro + gs * 4) + : "f8", "f9", "f10", "f11", "memory"); + errors += ro_misses(w, 4, gs + 4, salt); +#endif + return errors; +} + +// Wide stores into the thread's slot (local or global), then wide loads of +// the same words: +// - .q at the slot, then on odd lanes (divergent) .q again over it at the +// -2048 offset limit +// - .q at a 4- but not 16-byte aligned word, then a scalar store into it +// - .d at the 2040 offset limit and at a 4- but not 8-byte aligned word +// - scalar stores between the wide ones +static inline uint32_t check_stores(uintptr_t slot, uint32_t tid, uint32_t lid) { + auto s = reinterpret_cast(slot); + uint32_t v[16]; + for (uint32_t k = 0; k < 16; ++k) { + v[k] = wide_word(tid, k); + } + uint32_t a[4]; + for (uint32_t k = 0; k < 4; ++k) { + a[k] = alt_word(tid, k); + } + + VX_FSW_Q("f8", "f9", "f10", "f11", 0, slot, v); + if (lid & 1) { + VX_FSW_Q("f12", "f13", "f14", "f15", -2048, slot + 2048, a); + } + s[4] = v[4]; + VX_FSW_Q("f16", "f17", "f18", "f19", 16, slot + 4, v + 5); + s[7] = alt_word(tid, 7); + s[9] = v[9]; + VX_FSW_D("f20", "f21", 2040, slot + 40 - 2040, v + 10); + s[12] = v[12]; + VX_FSW_D("f22", "f23", 0, slot + 52, v + 13); + s[15] = v[15]; + + uint32_t errors = 0; + uint32_t w[4]; + VX_FLW_Q("f24", "f25", "f26", "f27", 0, slot, w); + for (uint32_t k = 0; k < 4; ++k) { + errors += (w[k] != slot_word(tid, lid, k)); + } + VX_FLW_Q("f0", "f1", "f2", "f3", 0, slot + 20, w); + for (uint32_t k = 0; k < 4; ++k) { + errors += (w[k] != slot_word(tid, lid, 5 + k)); + } + VX_FLW_D("f4", "f5", 8, slot + 32, w); + for (uint32_t k = 0; k < 2; ++k) { + errors += (w[k] != slot_word(tid, lid, 10 + k)); + } + VX_FLW_D("f6", "f7", 0, slot + 52, w); + for (uint32_t k = 0; k < 2; ++k) { + errors += (w[k] != slot_word(tid, lid, 13 + k)); + } + VX_FLW_Q("f28", "f29", "f30", "f31", 32, slot + 16, w); + for (uint32_t k = 0; k < 4; ++k) { + errors += (w[k] != slot_word(tid, lid, 12 + k)); + } + return errors; +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = reinterpret_cast(__local_mem()); + auto err = reinterpret_cast(arg->err_addr); + auto dump = reinterpret_cast(arg->dump_addr); + const uint32_t lid = threadIdx.x; + const uint32_t bsize = blockDim.x; + const uint32_t tid = blockIdx.x * bsize + lid; + const uint32_t words = arg->lmem_words; + + if (tid == 0) { + #ifdef HAS_WIDE + *reinterpret_cast(arg->wide_addr) = 1; + #else + *reinterpret_cast(arg->wide_addr) = 0; + #endif + } + + for (uint32_t i = lid; i < RO_WORDS; i += bsize) { + lmem[i] = ro_word(i); + } + __syncthreads(); + + const uintptr_t lmem_ro = reinterpret_cast(lmem); + const uintptr_t lmem_slot = lmem_ro + (RO_WORDS + lid * SLOT_WORDS) * 4; + const uintptr_t glob_slot = static_cast(arg->dst_addr) + tid * SLOT_WORDS * 4; + + uint32_t errors = 0; + errors += check_loads(lmem_ro, 0, lid); + errors += check_loads(static_cast(arg->src_addr), GLOBAL_SALT, lid); + errors += check_stores(lmem_slot, tid, lid); + errors += check_stores(glob_slot, tid, lid); + err[tid] = errors; + + __syncthreads(); + for (uint32_t i = lid; i < words; i += bsize) { + dump[blockIdx.x * words + i] = lmem[i]; + } +} diff --git a/tests/regression/lds_wide/main.cpp b/tests/regression/ldst_wide/main.cpp similarity index 66% rename from tests/regression/lds_wide/main.cpp rename to tests/regression/ldst_wide/main.cpp index 6f1d10ac01..36281332ef 100644 --- a/tests/regression/lds_wide/main.cpp +++ b/tests/regression/ldst_wide/main.cpp @@ -1,5 +1,6 @@ -// lds_wide — the wide local loads (vx_flds.d/.q) return what the generic path -// stores, across lane patterns, register groups and the limits of their offset. +// ldst_wide — the wide loads and stores (vx_flw.d/.q, vx_fsw.d/.q) move what +// the scalar accesses would, in local and global memory, across lane patterns, +// register groups, word alignments and the limits of their offset. #include #include "common.h" @@ -53,14 +54,14 @@ int main(int argc, char** argv) { CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); CHECK(vx_device_query(dev, VX_CAPS_LOCAL_MEM_SIZE, &lmem_size)); CHECK(vx_device_query(dev, VX_CAPS_ISA_FLAGS, &isa)); - // the wide loads exist on XLEN 32 with F; elsewhere the kernel reads the - // same words with generic loads - const bool wide = (VX_ISA_ARCH(isa) == 32) && (isa & VX_ISA_STD_F); + // the wide forms exist with F and FLEN 32; elsewhere the kernel moves the + // same words with generic accesses + const bool wide = (isa & VX_ISA_STD_F) && !(isa & VX_ISA_STD_D); const uint32_t block = static_cast(nt * nw); const uint32_t grid = static_cast(nc * 2); const uint32_t num_threads = block * grid; - const uint32_t words = RO_WORDS + block * WIDE_WORDS; - std::cout << "lds_wide: block=" << block << " grid=" << grid << " words=" << words + const uint32_t words = RO_WORDS + block * SLOT_WORDS; + std::cout << "ldst_wide: block=" << block << " grid=" << grid << " words=" << words << " wide=" << wide << std::endl; if (words * sizeof(uint32_t) > lmem_size) { std::cout << "local memory too small\nFAILED!" << std::endl; @@ -71,9 +72,13 @@ int main(int argc, char** argv) { vx_queue_h q = nullptr; CHECK(vx_queue_create(dev, &qi, &q)); + vx_buffer_h src_buf = nullptr; + vx_buffer_h dst_buf = nullptr; vx_buffer_h err_buf = nullptr; vx_buffer_h dump_buf = nullptr; vx_buffer_h wide_buf = nullptr; + CHECK(vx_buffer_create(dev, RO_WORDS * sizeof(uint32_t), VX_MEM_READ, &src_buf)); + CHECK(vx_buffer_create(dev, num_threads * SLOT_WORDS * sizeof(uint32_t), VX_MEM_READ_WRITE, &dst_buf)); CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); CHECK(vx_buffer_create(dev, grid * words * sizeof(uint32_t), VX_MEM_WRITE, &dump_buf)); CHECK(vx_buffer_create(dev, sizeof(uint32_t), VX_MEM_WRITE, &wide_buf)); @@ -85,6 +90,8 @@ int main(int argc, char** argv) { kernel_arg_t kernel_arg{}; kernel_arg.lmem_words = words; + CHECK(vx_buffer_address(src_buf, &kernel_arg.src_addr)); + CHECK(vx_buffer_address(dst_buf, &kernel_arg.dst_addr)); CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); CHECK(vx_buffer_address(dump_buf, &kernel_arg.dump_addr)); CHECK(vx_buffer_address(wide_buf, &kernel_arg.wide_addr)); @@ -99,16 +106,27 @@ int main(int argc, char** argv) { li.block_dim[0]= block; li.lmem_size = words * sizeof(uint32_t); + std::vector h_src(RO_WORDS); + for (uint32_t i = 0; i < RO_WORDS; ++i) { + h_src[i] = ro_word(i) ^ GLOBAL_SALT; + } + std::vector h_dst(num_threads * SLOT_WORDS, 0); std::vector h_err(num_threads); std::vector h_dump(grid * words); uint32_t h_wide = ~0u; + vx_event_h write_ev[2] = {nullptr, nullptr}; vx_event_h launch_ev = nullptr; vx_event_h read_ev = nullptr; - CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + vx_event_h dst_ev = nullptr; + CHECK(vx_enqueue_write(q, src_buf, 0, h_src.data(), h_src.size() * sizeof(uint32_t), 0, nullptr, &write_ev[0])); + CHECK(vx_enqueue_write(q, dst_buf, 0, h_dst.data(), h_dst.size() * sizeof(uint32_t), 0, nullptr, &write_ev[1])); + CHECK(vx_enqueue_launch(q, &li, 2, write_ev, &launch_ev)); CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); CHECK(vx_enqueue_read(q, &h_wide, wide_buf, 0, sizeof(uint32_t), 1, &launch_ev, nullptr)); CHECK(vx_enqueue_read(q, h_dump.data(), dump_buf, 0, h_dump.size() * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_enqueue_read(q, h_dst.data(), dst_buf, 0, h_dst.size() * sizeof(uint32_t), 1, &launch_ev, &dst_ev)); CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + CHECK(vx_event_wait_value(dst_ev, 1, VX_TIMEOUT_INFINITE)); int errors = 0; // the kernel must take the wide path exactly where the ISA has it @@ -126,8 +144,9 @@ int main(int argc, char** argv) { } for (uint32_t b = 0; b < grid; ++b) { for (uint32_t i = 0; i < words; ++i) { + const uint32_t lid = (i - RO_WORDS) / SLOT_WORDS; uint32_t expected = (i < RO_WORDS) ? ro_word(i) - : wide_word(b * block + (i - RO_WORDS) / WIDE_WORDS, (i - RO_WORDS) % WIDE_WORDS); + : slot_word(b * block + lid, lid, (i - RO_WORDS) % SLOT_WORDS); uint32_t actual = h_dump[b * words + i]; if (actual != expected) { if (errors < 16) { @@ -138,11 +157,29 @@ int main(int argc, char** argv) { } } + for (uint32_t t = 0; t < num_threads; ++t) { + for (uint32_t k = 0; k < SLOT_WORDS; ++k) { + uint32_t expected = slot_word(t, t % block, k); + uint32_t actual = h_dst[t * SLOT_WORDS + k]; + if (actual != expected) { + if (errors < 16) { + std::printf("*** thread %u global slot[%u] expected=0x%x actual=0x%x\n", t, k, expected, actual); + } + ++errors; + } + } + } + + vx_event_release(dst_ev); vx_event_release(read_ev); vx_event_release(launch_ev); + vx_event_release(write_ev[1]); + vx_event_release(write_ev[0]); vx_buffer_release(wide_buf); vx_buffer_release(dump_buf); vx_buffer_release(err_buf); + vx_buffer_release(dst_buf); + vx_buffer_release(src_buf); vx_kernel_release(kern); vx_module_release(mod); vx_queue_release(q); diff --git a/tests/regression/sgemm/common.h b/tests/regression/sgemm/common.h index d3c6362d7d..64d1a6926d 100644 --- a/tests/regression/sgemm/common.h +++ b/tests/regression/sgemm/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t size; uint64_t A_addr; diff --git a/tests/regression/sgemm/kernel.cpp b/tests/regression/sgemm/kernel.cpp index 7b79bbfb11..2a0278aed7 100644 --- a/tests/regression/sgemm/kernel.cpp +++ b/tests/regression/sgemm/kernel.cpp @@ -11,10 +11,35 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { int row = blockIdx.y * blockDim.y + threadIdx.y; TYPE sum(0); +#if LDST_WIDE + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + const TYPE* a_row = A + row * size; + const TYPE* b_col = B + col; + size_t e = 0; + for (; e + 8 <= size; e += 8) { + vx_float4 a0 = vx_flw_q(a_row + e); + vx_float4 a1 = vx_flw_q(a_row + e + 4); + #pragma unroll + for (int k = 0; k < 4; ++k) { + sum += a0[k] * *b_col; + b_col += size; + } + #pragma unroll + for (int k = 0; k < 4; ++k) { + sum += a1[k] * *b_col; + b_col += size; + } + } + for (; e < size; ++e) { + sum += a_row[e] * *b_col; + b_col += size; + } +#else #pragma unroll 8 for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } +#endif C[row * size + col] = sum; } diff --git a/tests/regression/sgemm2/common.h b/tests/regression/sgemm2/common.h index 1f25306211..6957c0737b 100644 --- a/tests/regression/sgemm2/common.h +++ b/tests/regression/sgemm2/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t size; uint32_t block_size; diff --git a/tests/regression/sgemm2/kernel.cpp b/tests/regression/sgemm2/kernel.cpp index 3961f24ece..396cc01150 100644 --- a/tests/regression/sgemm2/kernel.cpp +++ b/tests/regression/sgemm2/kernel.cpp @@ -20,10 +20,23 @@ static void matrix_mul(const TYPE* A, const TYPE* B, TYPE* C, uint32_t size) { Bs[ty * BLOCK_SIZE + tx] = B[(t + ty) * size + col]; __syncthreads(); + #if LDST_WIDE + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + static_assert((BLOCK_SIZE % 4) == 0, "LDST_WIDE reads A in groups of 4"); + #pragma unroll + for (uint32_t k = 0; k < BLOCK_SIZE; k += 4) { + vx_float4 a = vx_flw_q(&As[ty * BLOCK_SIZE + k]); + sum += a[0] * Bs[k * BLOCK_SIZE + tx]; + sum += a[1] * Bs[(k + 1) * BLOCK_SIZE + tx]; + sum += a[2] * Bs[(k + 2) * BLOCK_SIZE + tx]; + sum += a[3] * Bs[(k + 3) * BLOCK_SIZE + tx]; + } + #else #pragma unroll for (uint32_t k = 0; k < BLOCK_SIZE; ++k) { sum += As[ty * BLOCK_SIZE + k] * Bs[k * BLOCK_SIZE + tx]; } + #endif __syncthreads(); } diff --git a/tests/regression/sgemm2_dxa/common.h b/tests/regression/sgemm2_dxa/common.h index c646688ddd..730db98ebe 100644 --- a/tests/regression/sgemm2_dxa/common.h +++ b/tests/regression/sgemm2_dxa/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t size; uint32_t tile_size; diff --git a/tests/regression/sgemm2_dxa/kernel.cpp b/tests/regression/sgemm2_dxa/kernel.cpp index 7693034844..53fe18fe58 100644 --- a/tests/regression/sgemm2_dxa/kernel.cpp +++ b/tests/regression/sgemm2_dxa/kernel.cpp @@ -13,9 +13,24 @@ constexpr uint32_t kDescB = 1; // shB layout: [chunk_k × tile_size], row-major (row = k, col = l_col) static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, size_t tile_size, size_t chunk_k, size_t l_row, size_t l_col) { +#if LDST_WIDE + size_t k = 0; + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + for (; k + 4 <= chunk_k; k += 4) { + vx_float4 a = vx_flw_q(&shA[l_row * chunk_k + k]); + sum += a[0] * shB[k * tile_size + l_col]; + sum += a[1] * shB[(k + 1) * tile_size + l_col]; + sum += a[2] * shB[(k + 2) * tile_size + l_col]; + sum += a[3] * shB[(k + 3) * tile_size + l_col]; + } + for (; k < chunk_k; ++k) { + sum += shA[l_row * chunk_k + k] * shB[k * tile_size + l_col]; + } +#else for (size_t k = 0; k < chunk_k; ++k) { sum += shA[l_row * chunk_k + k] * shB[k * tile_size + l_col]; } +#endif } __kernel void kernel_main(kernel_arg_t* arg) { diff --git a/tests/regression/sgemm2_dxa_mcast/common.h b/tests/regression/sgemm2_dxa_mcast/common.h index 40de5d8401..627d1bb294 100644 --- a/tests/regression/sgemm2_dxa_mcast/common.h +++ b/tests/regression/sgemm2_dxa_mcast/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + // Intra-core multicast (mw = "multicast warp-group", multiple CTAs on one core). // `mc_group_size` CTAs co-resident on one core share the same B column block. // Each CTA computes one row of the output; A is per-CTA, B is multicast. diff --git a/tests/regression/sgemm2_dxa_mcast/kernel.cpp b/tests/regression/sgemm2_dxa_mcast/kernel.cpp index 7552459835..34321ae0e1 100644 --- a/tests/regression/sgemm2_dxa_mcast/kernel.cpp +++ b/tests/regression/sgemm2_dxa_mcast/kernel.cpp @@ -70,9 +70,24 @@ __kernel void kernel_main(kernel_arg_t* arg) { // ── Compute: 1 row × tile_size cols ─────────────────────────────────────── const size_t l_col = threadIdx.x; // 0..tile_size-1 (single warp) TYPE sum(0); +#if LDST_WIDE + size_t k = 0; + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + for (; k + 4 <= chunk_k; k += 4) { + vx_float4 a = vx_flw_q(&shA[k]); + sum += a[0] * shB[k * tile_size + l_col]; + sum += a[1] * shB[(k + 1) * tile_size + l_col]; + sum += a[2] * shB[(k + 2) * tile_size + l_col]; + sum += a[3] * shB[(k + 3) * tile_size + l_col]; + } + for (; k < chunk_k; ++k) { + sum += shA[k] * shB[k * tile_size + l_col]; + } +#else for (size_t k = 0; k < chunk_k; ++k) { sum += shA[k] * shB[k * tile_size + l_col]; } +#endif // Store C[row, col_blk*tile_size + l_col] C[row * size + col_blk * tile_size + l_col] = sum; diff --git a/tests/regression/sgemm2_v1/common.h b/tests/regression/sgemm2_v1/common.h index fcc070f6d2..b5e7e49db3 100644 --- a/tests/regression/sgemm2_v1/common.h +++ b/tests/regression/sgemm2_v1/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t grid_dim[2]; uint32_t block_dim[2]; diff --git a/tests/regression/sgemm2_v1/kernel.cpp b/tests/regression/sgemm2_v1/kernel.cpp index 9f32b87f3d..5dd1060349 100644 --- a/tests/regression/sgemm2_v1/kernel.cpp +++ b/tests/regression/sgemm2_v1/kernel.cpp @@ -39,9 +39,24 @@ void kernel_body(kernel_arg_t *arg) { __syncthreads(); +#if LDST_WIDE + size_t j = 0; + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + for (; j + 4 <= chunk_k; j += 4) { + vx_float4 a = vx_flw_q(&local_A[l_row * chunk_k + j]); + sum += a[0] * local_B[j * tile_size + l_col]; + sum += a[1] * local_B[(j + 1) * tile_size + l_col]; + sum += a[2] * local_B[(j + 2) * tile_size + l_col]; + sum += a[3] * local_B[(j + 3) * tile_size + l_col]; + } + for (; j < chunk_k; ++j) { + sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; + } +#else for (size_t j = 0; j < chunk_k; ++j) { sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; } +#endif __syncthreads(); } diff --git a/tests/regression/sgemm_v1/common.h b/tests/regression/sgemm_v1/common.h index b58ec5f5e1..f244066363 100644 --- a/tests/regression/sgemm_v1/common.h +++ b/tests/regression/sgemm_v1/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t grid_dim[2]; uint32_t size; diff --git a/tests/regression/sgemm_v1/kernel.cpp b/tests/regression/sgemm_v1/kernel.cpp index 001651da5f..94ba29b4cd 100644 --- a/tests/regression/sgemm_v1/kernel.cpp +++ b/tests/regression/sgemm_v1/kernel.cpp @@ -11,9 +11,25 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { int row = blockIdx.y; TYPE sum(0); +#if LDST_WIDE + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + const TYPE* a_row = A + row * size; + size_t e = 0; + for (; e + 4 <= size; e += 4) { + vx_float4 a = vx_flw_q(a_row + e); + sum += a[0] * B[e * size + col]; + sum += a[1] * B[(e + 1) * size + col]; + sum += a[2] * B[(e + 2) * size + col]; + sum += a[3] * B[(e + 3) * size + col]; + } + for (; e < size; ++e) { + sum += a_row[e] * B[e * size + col]; + } +#else for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } +#endif C[row * size + col] = sum; } diff --git a/tests/regression/sgemmx/common.h b/tests/regression/sgemmx/common.h index b490d09f17..942fbc1e59 100644 --- a/tests/regression/sgemmx/common.h +++ b/tests/regression/sgemmx/common.h @@ -7,6 +7,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads and writes each thread's consecutive words with the wide +// loads and stores (vx_flw_d/q, vx_fsw_d/q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + // Per-thread output tile #ifndef THREAD_SIZE_Y #define THREAD_SIZE_Y 4 diff --git a/tests/regression/sgemmx/kernel.cpp b/tests/regression/sgemmx/kernel.cpp index 0f0b1698c3..eae70317ad 100644 --- a/tests/regression/sgemmx/kernel.cpp +++ b/tests/regression/sgemmx/kernel.cpp @@ -5,6 +5,51 @@ #define BLOCK_SIZE_M (BLOCK_DIM_Y * THREAD_SIZE_Y) #define BLOCK_SIZE_N (BLOCK_DIM_X * THREAD_SIZE_X) +#if LDST_WIDE +static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + +// N consecutive words, in groups of 4, then 2, then 1. +template +static inline void wide_load(TYPE* dst, const TYPE* src) { + int j = 0; + #pragma unroll + for (; j + 4 <= N; j += 4) { + vx_float4 v = vx_flw_q(src + j); + dst[j] = v[0]; + dst[j + 1] = v[1]; + dst[j + 2] = v[2]; + dst[j + 3] = v[3]; + } + #pragma unroll + for (; j + 2 <= N; j += 2) { + vx_float2 v = vx_flw_d(src + j); + dst[j] = v[0]; + dst[j + 1] = v[1]; + } + #pragma unroll + for (; j < N; ++j) { + dst[j] = src[j]; + } +} + +template +static inline void wide_store(TYPE* dst, const TYPE* src) { + int j = 0; + #pragma unroll + for (; j + 4 <= N; j += 4) { + vx_fsw_q(dst + j, vx_float4{src[j], src[j + 1], src[j + 2], src[j + 3]}); + } + #pragma unroll + for (; j + 2 <= N; j += 2) { + vx_fsw_d(dst + j, vx_float2{src[j], src[j + 1]}); + } + #pragma unroll + for (; j < N; ++j) { + dst[j] = src[j]; + } +} +#endif + __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { const TYPE* A = reinterpret_cast(arg->A_addr); const TYPE* B = reinterpret_cast(arg->B_addr); @@ -44,12 +89,21 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { } // Load B into shared memory + #if LDST_WIDE + { + TYPE b_row_data[THREAD_SIZE_X]; + int b_row = k_tile * BLOCK_SIZE_K + ty; + wide_load(b_row_data, &B[b_row * N + bx * BLOCK_SIZE_N + tx * THREAD_SIZE_X]); + wide_store(&Bs[ty * BLOCK_SIZE_N + tx * THREAD_SIZE_X], b_row_data); + } + #else #pragma unroll for (int j = 0; j < THREAD_SIZE_X; ++j) { int b_row = k_tile * BLOCK_SIZE_K + ty; int b_col = bx * BLOCK_SIZE_N + tx * THREAD_SIZE_X + j; Bs[ty * BLOCK_SIZE_N + tx * THREAD_SIZE_X + j] = B[b_row * N + b_col]; } + #endif __syncthreads(); @@ -60,10 +114,14 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { for (int i = 0; i < THREAD_SIZE_Y; ++i) { a_frag[i] = As[(ty * THREAD_SIZE_Y + i) * BLOCK_SIZE_K + k]; } + #if LDST_WIDE + wide_load(b_frag, &Bs[k * BLOCK_SIZE_N + tx * THREAD_SIZE_X]); + #else #pragma unroll for (int j = 0; j < THREAD_SIZE_X; ++j) { b_frag[j] = Bs[k * BLOCK_SIZE_N + tx * THREAD_SIZE_X + j]; } + #endif #pragma unroll for (int i = 0; i < THREAD_SIZE_Y; ++i) { @@ -80,11 +138,22 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // --- Write Back to Global Memory --- #pragma unroll for (int i = 0; i < THREAD_SIZE_Y; ++i) { + #if LDST_WIDE + TYPE c_data[THREAD_SIZE_X]; + TYPE* c_ptr = &C[(row + i) * N + col]; + wide_load(c_data, c_ptr); + #pragma unroll + for (int j = 0; j < THREAD_SIZE_X; ++j) { + c_data[j] = alpha * accum[i][j] + beta * c_data[j]; + } + wide_store(c_ptr, c_data); + #else #pragma unroll for (int j = 0; j < THREAD_SIZE_X; ++j) { int c_row = row + i; int c_col = col + j; C[c_row * N + c_col] = alpha * accum[i][j] + beta * C[c_row * N + c_col]; } + #endif } } From 5dce75a3ca3804197842532740ff7566fd69bf4f Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 05:21:17 -0800 Subject: [PATCH 321/336] sgemm2_tcu: keep B K-contiguous in shared memory sgemm2_tcu transposed B into a row-major shared tile, so every B fragment register was a gather across K rows: 8 vx_packlh plus 7 addi per K-tile. B is col-major in global memory, so the cooperative load now keeps it col-major (K-contiguous) in shared memory and the fragment load reads it with col_major, ldm = tileK. Each register becomes one plain flw with no integer work. The per-lane fragment offset is computed once before the K loop. The __syncthreads in the loop is an asm with side effects, so the compiler could not hoist the lane id read on its own. vx_tensor.h gains lane_offset(ldm) and a load_matrix_sync overload taking that offset, in the wmma and wgmma contexts. The three-argument loads share the same lane_coords/load_fragment code and compile to identical assembly for every kernel that does not use the new overload. SimX, CI LLVM, HEAD -> this commit (outputs bit-identical): CI case (-w 4) instrs -5.9% cycles -4.9% ifetches -4.4% parity 128x128x64 instrs -5.9% cycles -2.4% ifetches -4.4% fp8 NT8 instrs -4.9% cycles -5.9% ifetches -1.3% Verified: sgemm2_tcu-1 (simx), sgemm2_tcu-2 (rtlsim), model_parity-sgemm2-fp16 (gap 0.30%); every TCU kernel builds with -Werror at XLEN 32 and 64. The sgemm2-fp16 perf baseline moves and needs a re-record. Co-Authored-By: Claude Opus 5.5 --- sw/kernel/include/vx_tensor.h | 132 +++++++++++++++---------- tests/regression/sgemm2_tcu/kernel.cpp | 18 ++-- 2 files changed, 93 insertions(+), 57 deletions(-) diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index 277b6ecc8b..e3f4ed39d1 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -280,25 +280,71 @@ struct wmma_context { }); } + // Element offset of the calling lane's first fragment element in a tile with + // leading dimension ldm. A K loop computes it once and passes it to + // load_matrix_sync: an asm with side effects ahead of the load (a barrier, a + // metadata load) keeps the compiler from hoisting the lane id read. + template + static __attribute__((always_inline)) size_t lane_offset(size_t ldm) { + uint32_t row, col; + lane_coords(row, col); + return row * ldm + col; + } + template static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm) { + uint32_t row, col; + lane_coords(row, col); + load_fragment(dst, reinterpret_cast*>(src) + row * ldm + col, ldm); + } + + template + static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm, size_t lane_off) { + load_fragment(dst, reinterpret_cast*>(src) + lane_off, ldm); + } + +private: + + template + using frag_elem_t = std::conditional_t; + + // Row and column of the calling lane's first fragment element in its tile + template + static __attribute__((always_inline)) void lane_coords(uint32_t &row, uint32_t &col) { uint32_t lane = vx_thread_id(); if constexpr (Frag::Use == frag_use_t::matrix_a) { - // Load row-major matrix A uint32_t block_idx = (cfg::a_block_size == NT) ? 0 : (lane / cfg::a_block_size); uint32_t lane_in_blk = (cfg::a_block_size == NT) ? lane : (lane % cfg::a_block_size); - uint32_t block_row = (lane_in_blk / cfg::tcK) + (block_idx * cfg::tcM); - uint32_t block_col = (lane_in_blk % cfg::tcK) * i_ratio; + row = (lane_in_blk / cfg::tcK) + (block_idx * cfg::tcM); + col = (lane_in_blk % cfg::tcK) * i_ratio; + } else if constexpr (Frag::Use == frag_use_t::matrix_b) { + // Sparse B uses 2x tcK per block + constexpr uint32_t b_tcK = is_sparse ? (cfg::tcK * 2) : cfg::tcK; + constexpr uint32_t b_block_size = is_sparse ? cfg::b_block_size_sp : cfg::b_block_size; + uint32_t block_idx = (b_block_size == NT) ? 0 : (lane / b_block_size); + uint32_t lane_in_blk = (b_block_size == NT) ? lane : (lane % b_block_size); + row = (lane_in_blk % b_tcK) * i_ratio; + col = (lane_in_blk / b_tcK) + (block_idx * cfg::tcN); + } else { + row = lane / cfg::tcN; + col = lane % cfg::tcN; + } + if constexpr (layout == col_major) { + std::swap(row, col); + } + } + + // base already points at the calling lane's first element + template + static __attribute__((always_inline)) void load_fragment(Frag &dst, const frag_elem_t *base, size_t ldm) { + if constexpr (Frag::Use == frag_use_t::matrix_a) { + // Load row-major matrix A uint32_t m_stride = cfg::a_sub_blocks * cfg::tcM; uint32_t k_stride = cfg::tcK * i_ratio; - if constexpr (src_layout == col_major) { - std::swap(block_row, block_col); - } if constexpr (is_sparse) { // Sparse A load: only load half the K-steps (compressed A) constexpr uint32_t sparse_k_steps = cfg::k_steps / 2; constexpr uint32_t sparse_regs = cfg::m_steps * sparse_k_steps; - auto base = reinterpret_cast(src) + block_row * ldm + block_col; detail::unroll_for([&](auto r) { uint32_t block_m = r / sparse_k_steps; uint32_t block_k = r % sparse_k_steps; @@ -322,7 +368,6 @@ struct wmma_context { }); } else { // Dense A load: load all K-steps - auto base = reinterpret_cast(src) + block_row * ldm + block_col; detail::unroll_for([&](auto r) { uint32_t block_m = r / cfg::k_steps; uint32_t block_k = r % cfg::k_steps; @@ -350,18 +395,10 @@ struct wmma_context { if constexpr (is_sparse) { // Sparse B load: uses 2x tcK for B block constexpr uint32_t b_tcK = cfg::tcK * 2; - uint32_t block_idx = (cfg::b_block_size_sp == NT) ? 0 : (lane / cfg::b_block_size_sp); - uint32_t lane_in_blk = (cfg::b_block_size_sp == NT) ? lane : (lane % cfg::b_block_size_sp); - uint32_t block_col = (lane_in_blk / b_tcK) + (block_idx * cfg::tcN); - uint32_t block_row = (lane_in_blk % b_tcK) * i_ratio; // NT=16 sparse: each register = 2 columns × full K (n_stride=2, no K iteration) // NT=8/32 sparse: standard interleaved layout uint32_t n_stride = cfg::sym_sparse ? (cfg::tcN / 2) : (cfg::b_sub_blocks_sp * cfg::tcN); uint32_t k_stride = b_tcK * i_ratio; - if constexpr (src_layout == col_major) { - std::swap(block_row, block_col); - } - auto base = reinterpret_cast(src) + block_row * ldm + block_col; if constexpr (src_layout == row_major) { static_assert(input_is_subbyte == false, "row_major layout is not supported for sub-byte matrix_b"); @@ -413,16 +450,8 @@ struct wmma_context { } } else { // Dense B load - uint32_t block_idx = (cfg::b_block_size == NT) ? 0 : (lane / cfg::b_block_size); - uint32_t lane_in_blk = (cfg::b_block_size == NT) ? lane : (lane % cfg::b_block_size); - uint32_t block_col = (lane_in_blk / cfg::tcK) + (block_idx * cfg::tcN); - uint32_t block_row = (lane_in_blk % cfg::tcK) * i_ratio; uint32_t n_stride = cfg::b_sub_blocks * cfg::tcN; uint32_t k_stride = cfg::tcK * i_ratio; - if constexpr (src_layout == col_major) { - std::swap(block_row, block_col); - } - auto base = reinterpret_cast(src) + block_row * ldm + block_col; detail::unroll_for([&](auto r) { uint32_t block_k = r / cfg::b_sub_steps; uint32_t block_n = r % cfg::b_sub_steps; @@ -448,14 +477,8 @@ struct wmma_context { } else { // Load accumulator matrix C - uint32_t block_row = lane / cfg::tcN; - uint32_t block_col = lane % cfg::tcN; uint32_t m_stride = cfg::tcM; uint32_t n_stride = cfg::tcN; - if constexpr (src_layout == col_major) { - std::swap(block_row, block_col); - } - auto base = reinterpret_cast(src) + block_row * ldm + block_col; detail::unroll_for([&](auto r) { uint32_t block_m = r / cfg::n_steps; uint32_t block_n = r % cfg::n_steps; @@ -476,17 +499,15 @@ struct wmma_context { } } +public: + template static __attribute__((always_inline)) void store_matrix_sync(void *dst, const Frag &src, size_t ldm) { static_assert(Frag::Use == frag_use_t::accumulator, "only accumulator fragment can be stored"); - uint32_t lane = vx_thread_id(); - uint32_t block_row = lane / cfg::tcN; - uint32_t block_col = lane % cfg::tcN; + uint32_t block_row, block_col; + lane_coords(block_row, block_col); uint32_t m_stride = cfg::tcM; uint32_t n_stride = cfg::tcN; - if constexpr (dst_layout == col_major) { - std::swap(block_row, block_col); - } auto base = reinterpret_cast(dst) + block_row * ldm + block_col; detail::unroll_for([&](auto r) { uint32_t block_m = r / cfg::n_steps; @@ -846,6 +867,22 @@ struct wgmma_context { ctx_c::fill_fragment(dst, value); } + // Element offset of the calling lane's first A element (see wmma_context::lane_offset). + template + static __attribute__((always_inline)) size_t lane_offset(size_t ldm) { + static_assert(Frag::Use == frag_use_t::matrix_a, "only matrix_a fragment can be loaded from registers in wgmma context"); + static_assert(layout == row_major, "wgmma block-major load only accepts row_major caller hint"); + uint32_t lane = vx_thread_id(); + uint32_t i_in = lane / tcK; + uint32_t k_in_elem = (lane % tcK) * i_ratio; + if (ldm == 0) { + // Block-major: a single-K-step dense block holds fedpK elements per row + constexpr uint32_t row_elems = (k_steps == 1 && !is_sparse) ? (fedpK * i_ratio) : (tcK * i_ratio); + return i_in * row_elems + k_in_elem; + } + return i_in * ldm + k_in_elem; + } + // Load A fragment (NRA=4 config) from SMEM (RS path). // Layout selection via `ldm` (in elements): // ldm == 0 → block-major (cooperative-load layout): @@ -859,28 +896,27 @@ struct wgmma_context { // ctx_a::sparse_regs slot is reserved for the metadata word. template static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm) { + load_matrix_sync(dst, src, ldm, lane_offset(ldm)); + } + + template + static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm, size_t lane_off) { static_assert(Frag::Use == frag_use_t::matrix_a, "only matrix_a fragment can be loaded from registers in wgmma context"); static_assert(src_layout == row_major, "wgmma block-major load only accepts row_major caller hint"); static_assert(tcM * tcK == NT, "wgmma block-major load assumes canonical config (TC_M*TC_K == NT)"); constexpr uint32_t k_row_elems = tcK * i_ratio; constexpr uint32_t sparse_blk_elems = tcM * k_row_elems; - uint32_t lane = vx_thread_id(); - uint32_t i_in = lane / tcK; - uint32_t k_in_elem = (lane % tcK) * i_ratio; auto load_reg = [&](uint32_t m_blk, uint32_t k_blk) { size_t elem_off; if (ldm == 0) { // Block-major SMEM: blocks contiguous, k-block outer. constexpr uint32_t block_elems = is_sparse ? sparse_blk_elems : a_blk_elems; - elem_off = (k_blk * m_steps + m_blk) * block_elems - + i_in * k_row_elems - + k_in_elem; + elem_off = (k_blk * m_steps + m_blk) * block_elems + lane_off; } else { // Row-major SMEM: standard row*ldm + col. - elem_off = (m_blk * tcM + i_in) * ldm - + (k_blk * k_row_elems + k_in_elem); + elem_off = (m_blk * tcM) * ldm + k_blk * k_row_elems + lane_off; } return *reinterpret_cast( reinterpret_cast(src) + elem_off); @@ -901,13 +937,9 @@ struct wgmma_context { if constexpr (k_steps == 1) { size_t elem_off; if (ldm == 0) { - elem_off = m_blk * a_blk_elems - + i_in * (fedpK * i_ratio) - + k_blk * k_row_elems - + k_in_elem; + elem_off = m_blk * a_blk_elems + k_blk * k_row_elems + lane_off; } else { - elem_off = (m_blk * tcM + i_in) * ldm - + (k_blk * k_row_elems + k_in_elem); + elem_off = (m_blk * tcM) * ldm + k_blk * k_row_elems + lane_off; } dst.data[r] = *reinterpret_cast( reinterpret_cast(src) + elem_off); diff --git a/tests/regression/sgemm2_tcu/kernel.cpp b/tests/regression/sgemm2_tcu/kernel.cpp index 3cc808bf0d..a6c71e1643 100644 --- a/tests/regression/sgemm2_tcu/kernel.cpp +++ b/tests/regression/sgemm2_tcu/kernel.cpp @@ -22,7 +22,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t tile_row = blockIdx.y * cta_M; size_t tile_col = blockIdx.x * ctx::tileN; - // Shared memory: A [cta_M x tileK] row-major, B [tileK x tileN] row-major + // Shared memory: A [cta_M x tileK] row-major, B [tileK x tileN] col-major auto smem = reinterpret_cast(__local_mem()); auto A_smem = smem; auto B_smem = smem + cta_M * ctx::tileK; @@ -32,6 +32,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); + size_t a_lane_off = ctx::lane_offset(ctx::tileK); + size_t b_lane_off = ctx::lane_offset(ctx::tileK); + for (size_t k = 0; k < K; k += ctx::tileK) { // Cooperatively load A tile [cta_M x tileK] from DRAM (row-major). uint32_t a_size = cta_M * ctx::tileK; @@ -43,21 +46,22 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { } // Cooperatively load B tile [tileK x tileN] from DRAM (col-major source) - // and store row-major in shared memory. + // and keep it col-major in shared memory: K-contiguous, each B fragment + // register is one word rather than a gather across K rows. uint32_t b_size = ctx::tileK * ctx::tileN; for (size_t i = 0; i < b_size; i += num_threads) { size_t idx = i + tid; - size_t r = idx / ctx::tileN; - size_t c = idx % ctx::tileN; - B_smem[r * ctx::tileN + c] = pB[(tile_col + c) * K + (k + r)]; + size_t c = idx / ctx::tileK; + size_t r = idx % ctx::tileK; + B_smem[c * ctx::tileK + r] = pB[(tile_col + c) * K + (k + r)]; } __syncthreads(); // Per-warp WMMA from shared memory. auto A_warp = A_smem + warp_rank * ctx::tileM * ctx::tileK; - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK); - ctx::load_matrix_sync(fragB, B_smem, ctx::tileN); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK, a_lane_off); + ctx::load_matrix_sync(fragB, B_smem, ctx::tileK, b_lane_off); ctx::mma_sync(fragC, fragA, fragB, fragC); __syncthreads(); From a3f9907e2fa6b5cde1b336c44deea9e963716a78 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 10:44:28 -0800 Subject: [PATCH 322/336] simx: model the TCU metadata load's scoreboard bit and early AGU handoff SimX ran the mx tensor path up to 4% faster than the RTL (mx NT8 128^3: 185780 vs 193485 cycles) because it modeled TCU_LD differently in two ways: - The RTL scoreboard reuses the two FCSR bits as TCU metadata slots: a TCU_LD writes the MX (frm) or sparse (fflags) bit (VX_decode.sv, wr_xregs[rd[4] ? XREG_1 : XREG_0]) and an MX or sparse MMA reads it, so a warp's next metadata load waits for the MMA that consumes the current one. SimX had no such dependency. Decode now sets the same reads and writes, and each MMA micro-op inherits them from its macro-op. - The RTL hands a TCU_LD to the metadata AGU as soon as the AGU is free (execute_if.ready = agu_ld_ready in VX_tcu_block.sv), so MMAs queued behind it keep running while the metadata is fetched. SimX held the TCU_LD at the head of the TCU input until the load finished. It now pops it into the AGU and retires it from there with the same deposit-and-retire latency as before. Parity gaps (SimX vs RTL cycles), before -> after: mx NT8 128^3 4.00% -> 0.26% mxfp8 (k32) 0.55% -> 0.43% sparse-fp16 2.14% -> 1.66% wgmma-sp-dxa 2.18% -> 2.46% The other tensor parity cases (sgemm2-fp16, fp16, fp16-mc, wgmma-int8, wgmma-fedp2k-rs, wgmma-dxa, wgmma-dxa-mcast, sgemm2-dxa, sgemm2-dxa-mcast) are unchanged; all 12 pass. sp_mx, wg_mx, wg_sp and nvfp4 still pass on SimX. Retired-instruction counts do not change, and perf_gate runs RTL only, so no perf baseline moves. Co-Authored-By: Claude Opus 5.5 --- sim/simx/decode.cpp | 24 ++++++++++ sim/simx/tcu/tcu_unit.cpp | 96 ++++++++++++++++++--------------------- 2 files changed, 69 insertions(+), 51 deletions(-) diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index cbcbc94e9a..be5b6687a8 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -35,6 +35,27 @@ using namespace vortex; +#ifdef VX_CFG_EXT_TCU_ENABLE +// The scoreboard's two FCSR bits double as the TCU metadata slots, as in the +// RTL: a TCU_LD writes the sparse (fflags) or MX (frm) bit, and an MMA that +// consumes that metadata reads it. +static uint8_t tcu_meta_reads(uint32_t fmt_s, bool is_sparse) { + uint8_t mask = 0; + __unused(fmt_s, is_sparse); +#ifdef VX_CFG_TCU_SPARSE_ENABLE + if (is_sparse) { + mask |= Instr::FCSR_FFLAGS; + } +#endif +#ifdef VX_CFG_TCU_MX_ENABLE + if (fmt_s & 0x8) { + mask |= Instr::FCSR_FRM; + } +#endif + return mask; +} +#endif + static op_string_t op_string(const Instr &instr) { auto op_type = instr.get_op_type(); auto instrArgs = instr.get_args(); @@ -1007,6 +1028,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { bool is_sparse = (rs2 & 1) != 0; instr->set_op_type(is_sparse ? TcuType::WMMA_SP : TcuType::WMMA); instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, 0, 0, 0, 0, 0, 0}); + instr->set_fcsr_use(tcu_meta_reads(fmt_s, is_sparse), 0); instr->set_macro_op(); } break; #ifdef VX_CFG_TCU_WGMMA_ENABLE @@ -1017,6 +1039,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { bool is_a_smem = (rs2 >> 3) & 1; instr->set_op_type(is_sparse ? TcuType::WGMMA_SP : TcuType::WGMMA); instr->set_args(IntrTcuArgs{is_a_smem ? 1u : 0u, cd_nregs, fmt_s, fmt_d, 0, 0, 0, 0, 0, 0}); + instr->set_fcsr_use(tcu_meta_reads(fmt_s, is_sparse), 0); instr->set_macro_op(); } break; #endif // VX_CFG_TCU_WGMMA_ENABLE @@ -1028,6 +1051,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_args(IntrTcuArgs{0, 0, fmt_s, slot, 0, 0, 0, 0, 0, 0}); // rs1 holds the warp-broadcast base address (real I-reg read). instr->set_src_reg(0, rs1, RegType::Integer); + instr->set_fcsr_use(0, (rd & 0x10) ? Instr::FCSR_FRM : Instr::FCSR_FFLAGS); } break; #endif // TCU_META_ENABLE default: diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index e52cf83811..7302684141 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -493,6 +493,24 @@ class TcuUnit::Impl { } } + // A TCU_LD retires once its words are deposited into the metadata SRAM; + // the AGU takes the next load the cycle after. + void agu_retire(uint64_t now) { + for (uint32_t b = 0; b < VX_CFG_NUM_TCU_BLOCKS; ++b) { + auto& agu = agu_.at(b); + if (!agu.complete) + continue; + // The AGU round-trip models the memory latency; these cycles cover the + // metadata deposit and retire. + constexpr uint32_t kRetireDelay = 3; + if (simobject_->Outputs.at(b).try_send(agu.trace, kRetireDelay)) { + result_due_.at(b).push_back(now + kRetireDelay); + DT(3, simobject_->name() << " execute: op=" << TcuType::TCU_LD << ", " << *agu.trace); + agu = agu_state_t{}; + } + } + } + // Write lane T's fetched word into the selected metadata SRAM. void agu_write(const agu_state_t& agu, uint32_t T, uint32_t word) { uint32_t wid = agu.wid; @@ -558,13 +576,29 @@ class TcuUnit::Impl { auto& input = simobject_->Inputs.at(b); if (input.empty()) continue; - if (completing && backed_up) { - continue; - } auto trace = input.peek(); auto tcu_type = std::get(trace->op_type); auto tpuArgs = std::get(trace->instr_ptr->get_args()); + #ifdef TCU_META_ENABLE + // A TCU_LD leaves the input for the AGU once the AGU is free, so the ops + // behind it run while its metadata is in flight; it retires from the AGU. + if (tcu_type == TcuType::TCU_LD) { + if (!agu_.at(b).busy) { + // rs1 is a full-width address (.u64); use u64 to avoid truncation on XLEN=64. + auto& rs1_data = trace->src_data[0]; + uint64_t base_addr = rs1_data.empty() ? 0 : rs1_data.at(0).u64; + this->agu_start(b, trace->wid, tpuArgs.fmt_d, base_addr, trace); + input.pop(); + } + continue; + } + #endif + + if (completing && backed_up) { + continue; + } + #ifdef VX_CFG_TCU_WGMMA_ENABLE // A WGMMA compute uop fires once its tile-buffer operands are ready. if (tcu_is_wgmma(tcu_type) && !tpuArgs.is_setup_uop @@ -584,7 +618,7 @@ class TcuUnit::Impl { trace->dst_data.assign(num_threads, reg_data_t{}); auto& rd_data = trace->dst_data; - if (tcu_type != TcuType::TCU_LD && !vt::tcu_output_valid(tpuArgs.fmt_s, tpuArgs.fmt_d)) { + if (!vt::tcu_output_valid(tpuArgs.fmt_s, tpuArgs.fmt_d)) { std::cerr << "*** TCU output format " << tpuArgs.fmt_d << " for input format " << tpuArgs.fmt_s << " is not supported: the TCU returns fp32 (int32 for integer inputs)" << std::endl; std::abort(); @@ -638,16 +672,6 @@ class TcuUnit::Impl { } } } break; - #endif - #ifdef TCU_META_ENABLE - case TcuType::TCU_LD: { - if (agu_.at(b).busy) { - continue; - } - // rs1 is a full-width address (.u64); use u64 to avoid truncation on XLEN=64. - uint64_t base_addr = rs1_data.empty() ? 0 : rs1_data.at(0).u64; - this->agu_start(b, wid, tpuArgs.fmt_d, base_addr, trace); - } break; #endif default: std::abort(); @@ -655,49 +679,18 @@ class TcuUnit::Impl { exec_done_.at(b) = true; } - uint32_t delay = 0; - switch (tcu_type) { - case TcuType::WMMA: - case TcuType::WMMA_SP: - case TcuType::WGMMA: - case TcuType::WGMMA_SP: - delay = kMmaLatency; - break; - #ifdef TCU_META_ENABLE - case TcuType::TCU_LD: - // The AGU round-trip models the memory latency; the remaining - // cycles cover the metadata deposit and retire. - delay = 3; - break; - #endif - default: - std::abort(); - } - #ifdef TCU_META_ENABLE - // TCU_LD retires only after its words are deposited into the - // metadata SRAM. - if (tcu_type == TcuType::TCU_LD) { - auto& agu = agu_.at(b); - assert(agu.busy && agu.trace == trace); - if (!agu.complete) { - continue; // metadata still in flight — hold the block - } - } - #endif - if (simobject_->Outputs.at(b).try_send(trace, delay)) { - due.push_back(now + delay); + if (simobject_->Outputs.at(b).try_send(trace, kMmaLatency)) { + due.push_back(now + kMmaLatency); exec_done_.at(b) = false; - #ifdef TCU_META_ENABLE - // TCU_LD retired: free the AGU for the next metadata load. - if (tcu_type == TcuType::TCU_LD) { - agu_.at(b) = agu_state_t{}; - } - #endif DT(3, simobject_->name() << " execute: op=" << tcu_type << ", " << *trace); input.pop(); } } + #ifdef TCU_META_ENABLE + this->agu_retire(now); + #endif + #ifdef VX_CFG_TCU_WGMMA_ENABLE // A setup uop drops its block's A stripe and the shared B row at the end // of the cycle; uops firing alongside it still read the old contents. @@ -1690,6 +1683,7 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { auto uop_instr = std::allocate_shared(pool_, uop_uuid, FUType::TCU); uop_instr->set_parent_uuid(parent_uuid); + uop_instr->set_fcsr_use(macro_instr.fcsr_reads(), macro_instr.fcsr_writes()); if (tcu_is_wmma(tcu_type)) { using wmma = vt::wmma_config_t; From 7141af5e0a9092a3ae2eec863f36268236c90b00 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 11:54:09 -0800 Subject: [PATCH 323/336] sgemm_tcu_sp: bound the fp32 error by the uop chain, fail with exit 1 The sparse fp32 test failed at K=256 with a fixed 10-ULP tolerance even though the device result is correct. A host model that applies the TFR dot-product arithmetic once per TCU uop, chained over K, matches the device bit for bit at K=64, 128 and 256. Against the exact dot product the error is one-sided (mean -0.5, -2.5, -7.5 ULP; max 13 at K=256): each uop truncates product bits below the accumulator window, and with the test's non-negative inputs those errors add up instead of cancelling. The reference comment saying the device rounds to fp32 once was wrong; it rounds once per uop. The tolerance is now derived instead of fixed. A uop folds P = tcK * i_ratio products into the fp32 accumulator, each term and the sum rounded or truncated at most once by under half an ulp of the uop's result; with non-negative inputs no partial sum exceeds the final one, so the chain is bounded by uops * (P + 1) / 2 ulp of the result, plus one for the reference's rounding. NT8 fp16 gets 21 ULP at K=64 and 81 at K=256 against observed maxima of 3 and 13. A corrupted metadata word (64/4096 errors) and a flipped B exponent bit (28/4096) are still caught. main() returned the error count, so 4096 errors exited 0 (a multiple of 256) and a fully wrong run looked like a pass. It now returns 1. Verified on SimX: fp16 NT8 at K=32..512, bf16 at K=256 and 512, tf32 at K=256, fp16 K=256 at NT 4/16/32. New CI case sgemm_tcu_sp-k256 passes on SimX and on TFR rtlsim. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor_sp.yaml | 9 ++++++++ tests/regression/sgemm_tcu_sp/main.cpp | 31 ++++++++++++++++++-------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/ci/testcases/tensor_sp.yaml b/ci/testcases/tensor_sp.yaml index 8457299edb..fe787162e6 100644 --- a/ci/testcases/tensor_sp.yaml +++ b/ci/testcases/tensor_sp.yaml @@ -32,6 +32,15 @@ tests: - simx app: sgemm_tcu_sp configs: -DVX_CFG_NUM_THREADS=2 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_TF32_ENABLE -DITYPE=tf32 -DOTYPE=fp32 +# A long K: the fp32 tolerance grows with the accumulation chain. +- id: sgemm_tcu_sp-k256 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DITYPE=fp16 -DOTYPE=fp32 + args: -k256 - id: sgemm_tcu_sp-5 via: blackbox drivers: diff --git a/tests/regression/sgemm_tcu_sp/main.cpp b/tests/regression/sgemm_tcu_sp/main.cpp index dd357fe7a1..3fa3a674f8 100644 --- a/tests/regression/sgemm_tcu_sp/main.cpp +++ b/tests/regression/sgemm_tcu_sp/main.cpp @@ -12,9 +12,6 @@ #include #include -#ifndef FLOAT_ULP -#define FLOAT_ULP 10 -#endif #define MAX_ERRORS 100 #define RT_CHECK(_expr) \ @@ -30,6 +27,9 @@ using namespace vortex; namespace vt = tensor; +// fp32 result tolerance in ULPs; it grows with K (see fp32_ulp_bound). +static uint32_t fp32_max_ulp = 0; + /////////////////////////////////////////////////////////////////////////////// static void convert_row_to_col_major_4bit(uint8_t *dst, uint32_t width, uint32_t height, const uint8_t *src) { @@ -330,7 +330,7 @@ class Comparator { fa.f = a; fb.f = b; auto d = std::abs(fa.i - fb.i); - if (d > FLOAT_ULP) { + if (uint32_t(d) > fp32_max_ulp) { if (errors < MAX_ERRORS) { printf("*** error: [%d] expected=%f, actual=%f\n", index, fb.f, fa.f); } @@ -441,6 +441,18 @@ using otype_t = typename vt::OTYPE::dtype; static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), "TCU output type must be fp32 (int32 for integer inputs)"); +// Error bound of the device's fp32 result against the exact dot product. Each +// output accumulates through a chain of uops; a uop folds its products into +// the fp32 accumulator, rounding or truncating each of those terms and the sum +// once, every time by less than half an ulp of the uop's result. The inputs are +// non-negative, so no partial sum exceeds the final one: a uop of P products +// costs at most (P + 1) / 2 ulp of the result, and the chain adds them up. The +// reference's own rounding adds one more. +static uint32_t fp32_ulp_bound(uint32_t K) { + constexpr uint32_t products = cfg::tcK * cfg::i_ratio; + uint32_t uops = (K / cfg::tileK) * (cfg::k_steps / 2); + return (uops * (products + 1) + 1) / 2 + 1; +} // Dense CPU reference matmul (pruned A has zeros at masked positions) static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { @@ -449,10 +461,9 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t for (uint32_t m = 0; m < M; ++m) { for (uint32_t n = 0; n < N; ++n) { if constexpr (std::is_same::value) { - // fp32 output: the tensor core accumulates the K products in a wide - // accumulator and rounds to fp32 once; a per-step-rounded reference - // drifts by several ULP over K. Each product is exact in fp32, so a - // double accumulation reproduces the single-rounding dot product. + // fp32 output: the exact dot product, rounded once. Each product is + // exact in fp32 and their double sum is exact to far below an fp32 + // ulp; fp32_ulp_bound covers the device's per-uop rounding. double acc = 0.0; for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); @@ -675,6 +686,8 @@ int main(int argc, char *argv[]) { return -1; } + fp32_max_ulp = fp32_ulp_bound(K); + size_t sizeA_full = M * K; size_t sizeA = (M * K) / 2; size_t sizeB = K * N; @@ -1177,7 +1190,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return 1; } std::cout << "PASSED!" << std::endl; From 680bd8eef9ee5d810919d25b1cc02fa3e4bfdf6c Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 11:54:13 -0800 Subject: [PATCH 324/336] tcu: keep symmetric sparse in WGMMA builds VX_tcu_pkg forced SYM_SPARSE off in every WGMMA build so the WGMMA sparse path could share the folded column index. At NT 4 and 16 the non-symmetric WMMA_SP layout does not fit one register (b_sub_blocks_sp would be 0), and SimX and vx_tensor.h, which share cfg::sym_sparse in sw/common/tensor_cfg.h, keep it on. So WMMA_SP expanded differently in the two models: rtlsim with NT16 + WGMMA + sparse got 4096/4096 results wrong (and exited 0), while SimX passed. The design doc already scopes the symmetric geometry to WMMA, so the RTL follows SimX: SYM_SPARSE no longer depends on WGMMA, and the WGMMA sparse B read indexes by the unfolded column j. The WGMMA and WMMA paths already had separate formulas, so no mux is added. Beacon: tcu and tensor DUTs identical (also with sparse on). A core DUT with TCU + WGMMA + sparse at NT16 keeps its depth and grows 250 LUTs (+0.1%), all in tcu_uops: the symmetric expansion non-WGMMA builds already pay. Every WGMMA perf_gate case runs at NT8, where symmetric sparse is off before and after, so no baseline moves. New CI cases sgemm_tcu_sp-wgmma-1 (NT16 fp16) and -2 (NT4 int8) pass on SimX and rtlsim; sgemm_tcu_wg_sp-5 (NT16, rtlsim) still passes. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor_wg.yaml | 17 +++++++++++++++++ hw/rtl/tcu/VX_tcu_core.sv | 11 ++++++----- hw/rtl/tcu/VX_tcu_pkg.sv | 6 +++--- sw/common/tensor_cfg.h | 3 ++- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index 937d898bff..fd07f4ada8 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -132,6 +132,23 @@ tests: app: sgemm_tcu_wg_sp configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 +# WMMA_SP in a WGMMA build: at NT 4 and 16 it keeps the symmetric layout. +- id: sgemm_tcu_sp-wgmma-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DVX_CFG_TCU_SPARSE_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 +- id: sgemm_tcu_sp-wgmma-2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg_sp-7 via: blackbox drivers: diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index 035186b074..cee0c772dc 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -493,14 +493,15 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( for (genvar k_idx = 0; k_idx < TCU_TC_K; ++k_idx) begin : g_sparse_slice_assign localparam J_SP = SYM_SPARSE ? (j % (TCU_TC_N / 2)) : j; // rs2_data sparse-pair layout differs by op: - // WGMMA_SP: source is tbuf (shared mem), K-major → - // idx = k_idx*(TC_N*2) + J_SP*2 + lane - // WMMA_SP : source is the register file, J-major → + // WGMMA_SP: source is tbuf (shared mem), K-major, all columns → + // idx = k_idx*(TC_N*2) + j*2 + lane + // WMMA_SP : source is the register file, J-major, folded to + // column pairs when SYM_SPARSE → // idx = J_SP*(TC_K*2) + k_idx*2 + lane // The two layouts are incompatible; separate formulas are required. `ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [31:0] b_col_1_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + J_SP * 2]); - wire [31:0] b_col_2_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + J_SP * 2 + 1]); + wire [31:0] b_col_1_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + j * 2]); + wire [31:0] b_col_2_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + j * 2 + 1]); wire [31:0] b_col_1_wm = 32'(rs2_data[b_off_wm + J_SP * TCU_TC_K * 2 + k_idx * 2]); wire [31:0] b_col_2_wm = 32'(rs2_data[b_off_wm + J_SP * TCU_TC_K * 2 + k_idx * 2 + 1]); assign b_col_1[k_idx] = is_wgmma ? b_col_1_wg : b_col_1_wm; diff --git a/hw/rtl/tcu/VX_tcu_pkg.sv b/hw/rtl/tcu/VX_tcu_pkg.sv index 10d4940158..f16cf62297 100644 --- a/hw/rtl/tcu/VX_tcu_pkg.sv +++ b/hw/rtl/tcu/VX_tcu_pkg.sv @@ -113,9 +113,9 @@ package VX_tcu_pkg; localparam TCU_WG_B_BLOCK_SIZE = TCU_WG_FEDP_K * TCU_TC_N; localparam TCU_WG_B_SUB_BLOCKS = TCU_BLOCK_CAP / TCU_WG_B_BLOCK_SIZE; - // Symmetric sparse flag (NT=4, NT=16: block_em == block_en) - // WGMMA always uses full interleaved layout, so SYM_SPARSE is forced off. - localparam SYM_SPARSE = `VX_CFG_TCU_WGMMA_ENABLED ? 0 : (TCU_BLOCK_EM == TCU_BLOCK_EN); + // Symmetric sparse flag (NT=4, NT=16: block_em == block_en). It selects the + // WMMA_SP layout only: WGMMA_SP always reads the full interleaved layout. + localparam SYM_SPARSE = (TCU_BLOCK_EM == TCU_BLOCK_EN); // B micro-tiling (sparse 2:4) // NT=8/32: standard interleaved layout (tcK × tcN × 2 = NT lanes per block) diff --git a/sw/common/tensor_cfg.h b/sw/common/tensor_cfg.h index e980bc5c77..ecf3cef369 100644 --- a/sw/common/tensor_cfg.h +++ b/sw/common/tensor_cfg.h @@ -287,7 +287,8 @@ struct wmma_config_t { static constexpr uint32_t b_sub_blocks = block_cap / b_block_size; // number of B micro-tiles per register static constexpr uint32_t b_sub_steps = n_steps / b_sub_blocks; // number of B sub-steps per register - // Symmetric sparse flag (NT=4, NT=16: block_em == block_en) + // Symmetric sparse flag (NT=4, NT=16: block_em == block_en). It selects the + // WMMA_SP layout only, so a WGMMA build keeps it: WGMMA_SP has its own. static constexpr bool sym_sparse = (block_em == block_en); // Symmetric: column-pair layout (block_cap lanes per block) From d47fead9cb0b3c2aaaca2cac53596a82e463a3ff Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 11:55:08 -0800 Subject: [PATCH 325/336] tcu: issue TCU_LD in LSU-lane passes when the LSU is narrower than the warp A sparse or MX TCU build with fewer LSU lanes than threads did not build: VX_tcu_agu asserted NUM_TCU_LANES == NUM_LSU_LANES and was built around it, sending one full-warp request, finishing on the first eop and writing response lane i to metadata lane i. SimX already splits the load into NT / LSU_LANES beats, so only the RTL was wrong. The shared AGU port in VX_tcu_unit also overwrote the low tag bits with the block index, which would have destroyed any tag the AGU set. The AGU now issues NUM_TCU_LANES / NUM_LSU_LANES passes of consecutive words, one per cycle, like the LSU's own packets: the address advances by LSU_LANES words per pass, each request carries its pass in its tag so responses may return in any order, each response writes only its pass's lanes, and the load completes on the last pass's eop. With one pass none of this logic is generated. The shared port puts the block index in the tag's low bits and moves the block's own tag above it, and statically checks that the remaining bits still hold the pass number. Beacon (tcu and tensor DUTs against the parent): identical depth, LUT, FF, BRAM and DSP, so the default config is unchanged. perf_gate-sparse-fp16, -mxfp8 and -wgmma-sparse give the parent's exact cycles and instructions. New CI cases, all passing: sgemm_tcu_sp-lsu-lanes-1 (NT8, 4 lanes) and -lsu-lanes-2 (NT16, 4 lanes, two issue slices sharing one LSU block, so 4 passes through the shared port), sgemm_tcu_mx-lsu-lanes (NT16 mxfp8), each on SimX and rtlsim; model_parity-sparse-lsu-lanes (gap 1.62%) and model_parity-mxfp8-lsu-lanes (0.75%). lsu-lanes-1 and the MX case also pass at XLEN 64. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor_mx.yaml | 16 +++++ ci/testcases/tensor_sp.yaml | 22 +++++++ docs/designs/tensor_core_wgmma_engine.md | 5 +- hw/rtl/tcu/VX_tcu_agu.sv | 78 +++++++++++++++++++----- hw/rtl/tcu/VX_tcu_unit.sv | 18 ++++-- 5 files changed, 119 insertions(+), 20 deletions(-) diff --git a/ci/testcases/tensor_mx.yaml b/ci/testcases/tensor_mx.yaml index 9a90236244..197bcd60ec 100644 --- a/ci/testcases/tensor_mx.yaml +++ b/ci/testcases/tensor_mx.yaml @@ -247,6 +247,22 @@ tests: configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI -DTCU_MX_TLS args: "-k32" +# An LSU narrower than the warp: TCU_LD fetches the scale factors in several passes. +- id: sgemm_tcu_mx-lsu-lanes + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + args: "-k32" +- id: model_parity-mxfp8-lsu-lanes + check: model_parity + via: blackbox + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + args: "-k32" + # SimX<->rtlsim cycle parity on the MX datapath: same config on both drivers, # cycles must agree within tolerance (see core.yaml). - id: model_parity-mxfp8 diff --git a/ci/testcases/tensor_sp.yaml b/ci/testcases/tensor_sp.yaml index fe787162e6..bcaba14971 100644 --- a/ci/testcases/tensor_sp.yaml +++ b/ci/testcases/tensor_sp.yaml @@ -259,6 +259,28 @@ tests: app: sgemm_tcu_sp configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_TF32_ENABLE -DITYPE=tf32 -DOTYPE=fp32 +# An LSU narrower than the warp: TCU_LD fetches the metadata in several passes, +# through a shared AGU port in the second case. +- id: sgemm_tcu_sp-lsu-lanes-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 +- id: sgemm_tcu_sp-lsu-lanes-2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 +- id: model_parity-sparse-lsu-lanes + check: model_parity + via: blackbox + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE + # SimX<->rtlsim cycle parity on the native sparse fp16 datapath: same config on # both drivers, cycles must agree within tolerance (see core.yaml). - id: model_parity-sparse-fp16 diff --git a/docs/designs/tensor_core_wgmma_engine.md b/docs/designs/tensor_core_wgmma_engine.md index eefc1a8a15..dbd58a28e6 100644 --- a/docs/designs/tensor_core_wgmma_engine.md +++ b/docs/designs/tensor_core_wgmma_engine.md @@ -379,7 +379,10 @@ simulation assertion catches a forgotten mask. [`VX_tcu_agu.sv`](../../hw/rtl/tcu/VX_tcu_agu.sv) issues **one warp-wide fetch per `TCU_LD`**: lane `T` loads word `T` of the metadata tile at the base address in `rs1`. The kernel pre-advances the base per slot, so a -multi-slot load is several instructions. +multi-slot load is several instructions. An LSU narrower than the warp +takes the fetch in `NUM_THREADS / NUM_LSU_LANES` passes of consecutive +words, as the LSU splits a warp into packets; each request carries its pass +in its tag, so the responses can return in any order. The generator is **client 1 of the core's LSU scheduler** on block 0. It owns no memory port; it reuses the LSU's staging, outstanding pool and cache diff --git a/hw/rtl/tcu/VX_tcu_agu.sv b/hw/rtl/tcu/VX_tcu_agu.sv index 315917365f..79baf58dc8 100644 --- a/hw/rtl/tcu/VX_tcu_agu.sv +++ b/hw/rtl/tcu/VX_tcu_agu.sv @@ -16,10 +16,12 @@ `ifdef TCU_META_ENABLE // A TCU block's TCU_LD unit: one warp-wide load in flight through an LSU -// scheduler client port. Response fragments are written straight into the -// block's metadata RAM, lane by lane, so nothing is buffered here. The load -// cannot overwrite metadata an older MMA still needs: TCU_LD enters the block -// in order behind that MMA, which reads the RAM when it issues. +// scheduler client port, issued in NUM_TCU_LANES / NUM_LSU_LANES passes of +// consecutive words, like the LSU's own packets. Response fragments are +// written straight into the block's metadata RAM, lane by lane, so nothing is +// buffered here. The load cannot overwrite metadata an older MMA still needs: +// TCU_LD enters the block in order behind that MMA, which reads the RAM when +// it issues. module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", @@ -46,12 +48,20 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire result_ready ); `UNUSED_SPARAM (INSTANCE_ID) - `STATIC_ASSERT (NUM_LANES == `VX_CFG_NUM_LSU_LANES, ("TCU_LD requires NUM_TCU_LANES == NUM_LSU_LANES")) + + localparam LSU_LANES = `VX_CFG_NUM_LSU_LANES; + localparam NUM_PASSES = NUM_LANES / LSU_LANES; + localparam PASS_BITS = `CLOG2(NUM_PASSES); // An LSU word holds XLEN/32 metadata words; lane i reads word i. localparam LSU_W32_RATIO = LSU_WORD_SIZE / 4; localparam LG_W32_RATIO = (LSU_W32_RATIO > 1) ? $clog2(LSU_W32_RATIO) : 0; + `STATIC_ASSERT ((NUM_LANES % LSU_LANES) == 0, ("TCU_LD requires NUM_TCU_LANES to be a multiple of NUM_LSU_LANES")) + // each pass starts on an LSU word, so lane i keeps reading word i + `STATIC_ASSERT ((NUM_PASSES == 1) || ((LSU_LANES % LSU_W32_RATIO) == 0), ("TCU_LD passes must be LSU-word aligned")) + `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= PASS_BITS, ("LSU client tag cannot encode the TCU_LD pass")) + reg busy_r, issued_r, done_r; tcu_header_t header_r; reg [4:0] slot_r; @@ -62,6 +72,11 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire rsp_fire = client_if.rsp_valid && client_if.rsp_ready; wire result_fire = result_valid && result_ready; + // the pass being requested, and whether it and the response are the last + wire [`UP(PASS_BITS)-1:0] req_pass; + wire [`UP(PASS_BITS)-1:0] rsp_pass; + wire req_last, rsp_last; + always @(posedge clk) begin if (reset) begin busy_r <= 1'b0; @@ -71,10 +86,10 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( if (ld_fire) begin busy_r <= 1'b1; end - if (req_fire) begin + if (req_fire && req_last) begin issued_r <= 1'b1; end - if (rsp_fire && client_if.rsp_data.eop) begin + if (rsp_fire && client_if.rsp_data.eop && rsp_last) begin done_r <= 1'b1; end if (result_fire) begin @@ -87,14 +102,46 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( header_r <= ld_data.header; slot_r <= ld_data.op_args.tcu.fmt_d; addr_r <= ld_data.rs1_data[0]; + end else if (req_fire && (NUM_PASSES > 1)) begin + addr_r <= addr_r + `VX_CFG_XLEN'(LSU_LANES * 4); + end + end + + if (NUM_PASSES > 1) begin : g_passes + reg [PASS_BITS-1:0] req_pass_r; + reg [PASS_BITS-1:0] rsp_cnt_r; + always @(posedge clk) begin + if (reset || ld_fire) begin + req_pass_r <= '0; + rsp_cnt_r <= '0; + end else begin + if (req_fire) begin + req_pass_r <= req_pass_r + PASS_BITS'(1); + end + if (rsp_fire && client_if.rsp_data.eop) begin + rsp_cnt_r <= rsp_cnt_r + PASS_BITS'(1); + end + end end + assign req_pass = req_pass_r; + assign req_last = (req_pass_r == PASS_BITS'(NUM_PASSES-1)); + assign rsp_last = (rsp_cnt_r == PASS_BITS'(NUM_PASSES-1)); + // responses of different passes may return out of order + assign rsp_pass = client_if.rsp_data.tag[PASS_BITS-1:0]; + `UNUSED_VAR (client_if.rsp_data.tag[LSU_CLIENT_TAG_WIDTH-1:PASS_BITS]) + end else begin : g_no_passes + assign req_pass = '0; + assign req_last = 1'b1; + assign rsp_last = 1'b1; + assign rsp_pass = '0; + `UNUSED_VAR (client_if.rsp_data.tag) end assign ld_ready = ~busy_r; `UNUSED_VAR (ld_data) // ----------------------------------------------------------------------- - // Request: one warp-wide load of NUM_LANES consecutive 32-bit words. + // Request: one pass loads LSU_LANES consecutive 32-bit words. // ----------------------------------------------------------------------- wire [LSU_ADDR_WIDTH-1:0] base_word_addr = LSU_ADDR_WIDTH'(addr_r >> `CLOG2(LSU_WORD_SIZE)); `UNUSED_VAR (addr_r[`CLOG2(LSU_WORD_SIZE)-1:0]) @@ -124,8 +171,9 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( always_comb begin req_data = '0; req_data.rw = 1'b0; - req_data.mask = {NUM_LANES{1'b1}}; - for (int i = 0; i < NUM_LANES; ++i) begin + req_data.mask = {LSU_LANES{1'b1}}; + req_data.tag = LSU_CLIENT_TAG_WIDTH'(req_pass); + for (int i = 0; i < LSU_LANES; ++i) begin req_data.addr[i] = base_word_addr + (LSU_ADDR_WIDTH'(i) >> LG_W32_RATIO); req_data.byteen[i] = {LSU_WORD_SIZE{1'b1}}; req_data.attr[i] = req_attr; @@ -136,7 +184,6 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( assign client_if.req_data = req_data; assign client_if.rsp_ready = 1'b1; `UNUSED_VAR (client_if.rsp_data.sop) - `UNUSED_VAR (client_if.rsp_data.tag) `RUNTIME_ASSERT(~client_if.rsp_valid || (issued_r && ~done_r), ("%t: *** %s unexpected TCU_LD response", $time, INSTANCE_ID)) @@ -145,14 +192,14 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Metadata is read as a linear warp-wide tile, which an interleaved // per-thread stack cannot provide. if (`VX_CFG_NUM_THREADS > 1) begin : g_meta_stack_check - wire [`VX_CFG_XLEN-1:0] meta_end_addr = addr_r + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); + wire [`VX_CFG_XLEN-1:0] meta_end_addr = addr_r + `VX_CFG_XLEN'(LSU_LANES * 4 - 1); `RUNTIME_ASSERT(~req_fire || (meta_end_addr < STACK_WINDOW_BOTTOM) || (addr_r >= STACK_WINDOW_TOP), ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, addr_r)) end `endif // ----------------------------------------------------------------------- - // Metadata RAM write: each response fragment writes its own lanes. + // Metadata RAM write: each response fragment writes its own pass's lanes. // ----------------------------------------------------------------------- assign meta_wr_en = rsp_fire; assign meta_wr_wid = header_r.wid; @@ -160,8 +207,9 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_meta_wr if (i < NUM_LANES) begin : g_lane - wire [LSU_W32_RATIO-1:0][31:0] rsp_w32 = client_if.rsp_data.data[i]; - assign meta_wr_mask[i] = client_if.rsp_data.mask[i]; + wire [LSU_W32_RATIO-1:0][31:0] rsp_w32 = client_if.rsp_data.data[i % LSU_LANES]; + assign meta_wr_mask[i] = client_if.rsp_data.mask[i % LSU_LANES] + && (rsp_pass == `UP(PASS_BITS)'(i / LSU_LANES)); assign meta_wr_data[i] = rsp_w32[i % LSU_W32_RATIO]; `UNUSED_VAR (rsp_w32) end else begin : g_pad diff --git a/hw/rtl/tcu/VX_tcu_unit.sv b/hw/rtl/tcu/VX_tcu_unit.sv index 7d1a5250c2..b9547072a7 100644 --- a/hw/rtl/tcu/VX_tcu_unit.sv +++ b/hw/rtl/tcu/VX_tcu_unit.sv @@ -83,9 +83,14 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( assign tcu_mem_if[b].rsp_ready = block_mem_if[b].rsp_ready; end end else begin : g_mem_shared - // the TCU blocks share LSU block 0's client, tagged by block + // the TCU blocks share LSU block 0's client: the block index takes the + // tag's low bits and the block's own tag moves above it localparam BLOCK_IDX_BITS = `CLOG2(BLOCK_SIZE); - `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) + localparam BLOCK_TAG_BITS = LSU_CLIENT_TAG_WIDTH - BLOCK_IDX_BITS; + `STATIC_ASSERT (BLOCK_IDX_BITS > 0, ("invalid parameter")) + `STATIC_ASSERT (BLOCK_TAG_BITS > 0, ("LSU client tag cannot encode TCU block")) + // a block's TCU_LD pass number rides in its own tag bits + `STATIC_ASSERT (BLOCK_TAG_BITS >= `CLOG2(NUM_LANES / `VX_CFG_NUM_LSU_LANES), ("LSU client tag cannot encode TCU block and TCU_LD pass")) wire [BLOCK_SIZE-1:0] req_valids; wire [`UP(BLOCK_IDX_BITS)-1:0] req_idx; @@ -114,16 +119,21 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( lsu_req_data_t req_data; always_comb begin req_data = req_datas[req_idx]; - req_data.tag[`UP(BLOCK_IDX_BITS)-1:0] = req_idx; + req_data.tag = {req_datas[req_idx].tag[BLOCK_TAG_BITS-1:0], req_idx}; end assign tcu_mem_if[0].req_valid = req_valid; assign tcu_mem_if[0].req_data = req_data; wire [`UP(BLOCK_IDX_BITS)-1:0] rsp_idx = tcu_mem_if[0].rsp_data.tag[`UP(BLOCK_IDX_BITS)-1:0]; + lsu_rsp_data_t rsp_data; + always_comb begin + rsp_data = tcu_mem_if[0].rsp_data; + rsp_data.tag = LSU_CLIENT_TAG_WIDTH'(tcu_mem_if[0].rsp_data.tag[LSU_CLIENT_TAG_WIDTH-1:BLOCK_IDX_BITS]); + end wire [BLOCK_SIZE-1:0] rsp_readys; for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_rsp assign block_mem_if[b].rsp_valid = tcu_mem_if[0].rsp_valid && (rsp_idx == `UP(BLOCK_IDX_BITS)'(b)); - assign block_mem_if[b].rsp_data = tcu_mem_if[0].rsp_data; + assign block_mem_if[b].rsp_data = rsp_data; assign rsp_readys[b] = block_mem_if[b].rsp_ready; end assign tcu_mem_if[0].rsp_ready = rsp_readys[rsp_idx]; From 43850bea7897864e7c8df353a53736ceb1d706ff Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 14:24:56 -0800 Subject: [PATCH 326/336] Revert "cache_mshr: look up every dequeue candidate's address in parallel" This reverts commit 3288ba5196a597b01bfd18bcce6a8258cecfe680. The parallel lookups are cycle-exact and buy timing only, at +817 LUTs in the cache DUT (+2,660 in the full design), and Beacon shows them making the MSHR deeper (dequeue_rw_r 11 -> 12). The next commit takes the late dequeue pointer off the address with a register instead. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_mshr.sv | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index bac7c029f4..64ffe48898 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -571,28 +571,9 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // stalls and the replay never drains. reg [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_r; reg dequeue_rw_r; - - // The pointer's next entry, as the update above chooses it: a fill opens a - // chain, a dequeue walks to the chained successor or to a late joiner - // spliced this cycle. The dequeue handshake arrives late, so the address - // and the write flag of every candidate are looked up in parallel and the - // same choice selects among them, instead of the late pointer indexing the - // tables. - wire deq_to_next = ~dequeue_park_fire && dequeue_fire && next_table[dequeue_id]; - wire deq_to_fin = ~dequeue_park_fire && dequeue_fire && ~next_table[dequeue_id] - && finalize_valid && finalize_is_pending && (finalize_previd == dequeue_id); - wire [MSHR_ADDR_WIDTH-1:0] deq_next_id = next_index[dequeue_id]; - wire [MSHR_ADDR_WIDTH-1:0] deq_base_id = fill_valid ? fill_id : dequeue_id; - wire [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_n = deq_to_next ? addr_table[deq_next_id] - : (deq_to_fin ? addr_table[finalize_id] : addr_table[deq_base_id]); - wire dequeue_rw_n = deq_to_next ? write_table[deq_next_id] - : (deq_to_fin ? write_table[finalize_id] : write_table[deq_base_id]); - `RUNTIME_ASSERT((dequeue_addr_n == addr_table[dequeue_id_n]) && (dequeue_rw_n == write_table[dequeue_id_n]), - ("*** %s dequeue candidate select out of sync: id=%0d", INSTANCE_ID, dequeue_id_n)) - always @(posedge clk) begin - dequeue_addr_r <= dequeue_addr_n; - dequeue_rw_r <= dequeue_rw_n; + dequeue_addr_r <= addr_table[dequeue_id_n]; + dequeue_rw_r <= write_table[dequeue_id_n]; end assign dequeue_addr = dequeue_addr_r; From 8307a4512dddf9163ee8e9f79dfebdaf41c598d3 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 14:25:06 -0800 Subject: [PATCH 327/336] cache_mshr: capture the dequeue address at the fill The registered dequeue address was read as addr_table[dequeue_id_n]. The dequeue handshake picks that pointer late in the cycle (fill, chained successor or same-cycle late joiner), so it drove a 16:1 line-address mux in every bank. Coalescing keys on the line, so every entry a chain walks holds the line of the fill that opened it, which the existing "chain entry on foreign line" assert already checks. The address is now captured once from fill_addr when the fill arrives, and the late pointer no longer reaches it. Being a register, it still cannot follow a slot that is freed and reallocated while the pointer names it. The simulation-only chain_addr_r duplicate is removed; the assert checks dequeue_addr_r. Beacon, cache DUT, against the parent: dequeue_addr_r depth 10 -> 7, nothing deeper, LUTs 29,038 -> 28,750; vm: 14 registers shallower, LUTs 116,648 -> 115,640; FF, LUTRAM and BRAM unchanged. Vivado cache gate: PASS at 297.3 MHz (290.3 before 3288ba519), 9 failing endpoints vs 1,233, LUT 28,891. Cycle-exact: rtlsim cache CI cases (34) and regression (76) have the same instruction and cycle counts; a simulation shadow of the old register matched the new one on every live dequeue. SimX unchanged. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_mshr.sv | 47 +++++++++++------------------------ 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index 64ffe48898..2d1d8e02dd 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -357,19 +357,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( dequeue_id, next_table[dequeue_id])) `ifdef SIMULATION - // Every entry the dequeue pointer lands on must belong to the line whose - // fill opened the chain, whichever way the pointer got there (fill, - // next_index walk, or same-cycle late-joiner splice). Checked here rather - // than at the bank's forward port because only the MSHR can see all three. - reg [`CS_LINE_ADDR_WIDTH-1:0] chain_addr_r; - always @(posedge clk) begin - if (fill_valid) begin - chain_addr_r <= addr_table[fill_id]; - end - end - `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (addr_table[dequeue_id] != chain_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, - dequeue_id, `CS_BANK_TO_FULL_ADDR(addr_table[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(chain_addr_r, BANK_ID))) - // The slot the dequeue pointer names must not be handed to another line // while the pointer still rests on it: a combinational `dequeue_addr` on // the entry at `dequeue_id_r` would retarget an in-flight replay. @@ -552,29 +539,25 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( ("%t: *** %s order-hazard probe diverged from the full exclusion", $time, INSTANCE_ID)) assign dequeue_valid = dequeue_val && ~((LAZY_FETCH != 0) && dequeue_block); - // Sampled with dequeue_id_n, alongside the payload in mshr_store, rather - // than read live at dequeue_id_r a cycle later. The entry's slot can be - // freed and reallocated while the pointer still names it -- the free set - // offers a slot released by a dequeue or a finalize in the same cycle -- and - // a live read then retargets an in-flight replay onto the new occupant's - // line. The bank's fill-forward path answers that replay from the staged - // sector of the line it was actually issued for, so the requester silently - // receives another line's data; the guard for it is a RUNTIME_ASSERT, which - // synthesis compiles out. - // - // Taking the address from the same snapshot as the payload also removes an - // inconsistency that was there regardless: the payload was a snapshot and - // the address was not, so the two could describe different entries. - // - // Withholding the slot from the free set instead deadlocks: the pointer can - // park on a blocked replay, and if that slot is the last free one the core - // stalls and the replay never drains. + // Coalescing keys on the line, so every entry a chain walks (the fill's + // root, its next_index successors, a late joiner spliced in) holds the + // fill's line: the address is captured once, at the fill, and the late + // dequeue pointer stays off it. Being a register, it also cannot follow a + // slot that is freed and reallocated while the pointer still names it, + // which would retarget an in-flight replay onto the new occupant's line. + // Withholding that slot from the free set instead deadlocks: the pointer + // can park on a blocked replay, and if that slot is the last free one the + // core stalls and the replay never drains. reg [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_r; reg dequeue_rw_r; always @(posedge clk) begin - dequeue_addr_r <= addr_table[dequeue_id_n]; - dequeue_rw_r <= write_table[dequeue_id_n]; + if (fill_valid) begin + dequeue_addr_r <= fill_addr; + end + dequeue_rw_r <= write_table[dequeue_id_n]; end + `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (addr_table[dequeue_id] != dequeue_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, + dequeue_id, `CS_BANK_TO_FULL_ADDR(addr_table[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(dequeue_addr_r, BANK_ID))) assign dequeue_addr = dequeue_addr_r; assign dequeue_rw = dequeue_rw_r; From 194b8fc59fac43b45f21fdc39607095cfb2cab2c Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 15:45:13 -0800 Subject: [PATCH 328/336] simx: queue TCU metadata loads with the LSU's own and shorten the TCU result path Three SimX latencies on the TCU metadata load and result path differed from the RTL, and the fixed-priority arbiters turned the few cycles into a different warp order (mxfp8 with the hoisted lane offsets: 8.19% off). - The RTL's TCU metadata request enters the LSU block's request queue, arbitrated round-robin with the block's own requests at the queue input (VX_lsu_scheduler), and waits behind loads already queued. SimX sent it straight to the memory switch. It now takes a queue entry in order with the loads. - The RTL's TCU result reaches commit through the landing-queue bypass and one lane-gather register: 7 cycles minimum from dispatch to commit for an MMA uop. SimX added a second register (8). The TCU now uses the one-register hop and two-entry commit queue of the LSU and SFU; the 128^3 latency distributions then match (mean 15.91 vs 15.90). - The RTL commits a TCU_LD 2 cycles after its last metadata response (done_r, then the lane gather); SimX took 6. The retire delay is 0. model_parity (SimX vs rtlsim cycles), all within 5%: mxfp8 0.43 -> 0.21 (8.19 -> 0.84 with the hoisted lane offsets), fp16-mc 0.68 -> 0.09, wgmma-dxa-mcast 1.87 -> 0.48, fp16 0.04 -> 3.93 (SimX now faster), the other tensor and DXA cases within 0.5 points. Retired instructions are unchanged. RTL is unchanged, so perf baselines do not move. Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 15 ++++++++------- sim/simx/lsu_unit.cpp | 36 +++++++++++++++++++++++++++++------- sim/simx/lsu_unit.h | 7 +++++++ sim/simx/tcu/tcu_unit.cpp | 9 ++++----- 4 files changed, 48 insertions(+), 19 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 7479a64a02..c1a8590fac 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -363,10 +363,10 @@ class Core::Impl { for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { auto& queues = commit_queues_.emplace_back(); for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { - // A unit's lane gather holds two results. The LSU's and the SFU's - // cross one register into it; the others cross two and take a third - // slot so that they still retire one result per cycle. - uint32_t capacity = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU) ? 2 : 3; + // A unit's lane gather holds two results. The LSU's, the SFU's and the + // TCU's cross one register into it; the others cross two and take a + // third slot so that they still retire one result per cycle. + uint32_t capacity = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU || fu == (uint32_t)FUType::TCU) ? 2 : 3; queues.emplace_back(std::make_unique>(simobject_, capacity)); } } @@ -933,9 +933,10 @@ class Core::Impl { auto& arb_in = *commit_queues_.at(iw).at(fu); // The unit's output registers ahead of the commit arbiter that its // output channel does not already carry: the LSU's response buffer and - // result arbiter, and the SFU's result buffer and response switch, are - // in their channels, leaving each its lane gather. - uint32_t commit_delay = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU) ? 1 : 2; + // result arbiter, the SFU's result buffer and response switch, and the + // TCU's result landing queue are in their channels, leaving each its + // lane gather. + uint32_t commit_delay = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU || fu == (uint32_t)FUType::TCU) ? 1 : 2; if (arb_in.try_send(trace, commit_delay)) { // Release the warp as soon as its stalling instruction's result leaves // the functional unit — the branch target / fence / warp-control is diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index d538e5ede7..4c462ffa17 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -388,7 +388,24 @@ void LsuUnit::process_response_step(uint32_t b) { void LsuUnit::ingest_inputs(uint32_t b, bool queue_room, uint32_t reads_held) { auto& state = states_.at(b); auto& input = Inputs.at(b); - if (input.empty() || !queue_room) + if (!queue_room) + return; +#ifdef TCU_META_ENABLE + // The TCU metadata client and the block's own requests share the request + // queue, one entry per cycle, granted round-robin. + bool meta_req = (b < TCU_AGU_PORTS) + && !TcuReqIn.at(b).empty() + && (reads_held < VX_CFG_LSU_PENDING_SIZE); + if (meta_req && (input.empty() || !state.meta_last)) { + state.meta_reqs.push_back(TcuReqIn.at(b).peek()); + TcuReqIn.at(b).pop(); + state.req_queue.push(nullptr); + ++state.staged_reads; + state.meta_last = true; + return; + } +#endif + if (input.empty()) return; auto trace = input.peek(); // Hold a fence at the input head until req_queue is empty so the @@ -418,6 +435,9 @@ void LsuUnit::ingest_inputs(uint32_t b, bool queue_room, uint32_t reads_held) { } state.req_queue.push(trace); input.pop(); +#ifdef TCU_META_ENABLE + state.meta_last = false; +#endif } // The fence travels to the data cache as a flush token on the lanes of its @@ -484,14 +504,14 @@ void LsuUnit::process_request_step(uint32_t b) { } #ifdef TCU_META_ENABLE - // TCU metadata client (blocks below TCU_AGU_PORTS): forward one pending request per - // cycle into lmem_switch, ahead of the block's own dispatch. The wire tag + // TCU metadata client (blocks below TCU_AGU_PORTS): its request leaves the + // shared request queue in order with the block's own. The wire tag // is re-allocated from pending_reqs so it stays unique on the switch; the // client's own tag is restored on the response. An independent client, so // not gated by the fence lock — its entries still hold fence release via // pending_reqs.empty(). - if (b < TCU_AGU_PORTS && !TcuReqIn.at(b).empty()) { - const LsuReq& meta_req = TcuReqIn.at(b).peek(); + if (!state.req_queue.empty() && state.req_queue.front() == nullptr) { + const LsuReq& meta_req = state.meta_reqs.front(); bool to_dc = false, to_lmem = false; for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { if (meta_req.mask.test(i)) { @@ -525,9 +545,11 @@ void LsuUnit::process_request_step(uint32_t b) { lsu_req.tag = tag; core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); DT(3, this->name() << " tcu-meta-req: " << lsu_req); - TcuReqIn.at(b).pop(); - return; // one request per cycle into the switch + state.meta_reqs.pop_front(); + state.req_queue.pop(); + --state.staged_reads; } + return; // one request per cycle into the switch } #endif diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index 045ddc9880..e09feeb409 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -14,6 +14,7 @@ #pragma once #include +#include #include #include #include "func_unit.h" @@ -149,6 +150,10 @@ class LsuUnit : public FuncUnit { // Reads waiting in req_queue hold a pending_reqs slot from the moment // they enter it, so staged and outstanding reads share the pool. uint32_t staged_reads = 0; + // TCU metadata requests granted a req_queue slot (a null entry there), + // and whether the TCU client took the last queue entry. + std::deque meta_reqs; + bool meta_last = false; bool head_staged = false; FenceController fence; // the engaged fence's flush token is still to be sent @@ -160,6 +165,8 @@ class LsuUnit : public FuncUnit { this->req_queue.clear(); this->pending_reqs.clear(); this->staged_reads = 0; + this->meta_reqs.clear(); + this->meta_last = false; this->head_staged = false; this->fence.reset(); this->fence_token = false; diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index 7302684141..f456172aa4 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -500,11 +500,10 @@ class TcuUnit::Impl { auto& agu = agu_.at(b); if (!agu.complete) continue; - // The AGU round-trip models the memory latency; these cycles cover the - // metadata deposit and retire. - constexpr uint32_t kRetireDelay = 3; - if (simobject_->Outputs.at(b).try_send(agu.trace, kRetireDelay)) { - result_due_.at(b).push_back(now + kRetireDelay); + // The result is offered in the cycle its last fragment lands, as the + // AGU's done flag; the lane gather adds the one register to commit. + if (simobject_->Outputs.at(b).try_send(agu.trace, 0)) { + result_due_.at(b).push_back(now); DT(3, simobject_->name() << " execute: op=" << TcuType::TCU_LD << ", " << *agu.trace); agu = agu_state_t{}; } From 0321033cf17f1da1bde382be16859ee9c302b032 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 15:45:26 -0800 Subject: [PATCH 329/336] tcu tests: hoist the per-lane fragment offsets out of the K loop The lane offset of a fragment depends only on the leading dimension, but load_matrix_sync recomputed it from the lane id on every K tile. The eight TCU kernels that load A/B fragments from global memory (mx, sparse mx, and the WGMMA family) now compute it once per kernel with lane_offset(ldm) and pass it to the 4-argument load_matrix_sync, as sgemm2_tcu already does since 5dce75a3c. Retired-instruction and cycle counts drop in these kernels, so their perf baselines need a re-record. model_parity stays within 5% with the SimX metadata-load timing of 194b8fc59 (mxfp8 0.84%). Co-Authored-By: Claude Opus 5.5 --- tests/regression/sgemm_tcu_mx/kernel.cpp | 7 +++++-- tests/regression/sgemm_tcu_sp_mx/kernel.cpp | 7 +++++-- tests/regression/sgemm_tcu_wg/kernel.cpp | 16 ++++++++++------ tests/regression/sgemm_tcu_wg_dxa/kernel.cpp | 6 +++++- .../regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp | 6 +++++- tests/regression/sgemm_tcu_wg_mx/kernel.cpp | 6 +++++- tests/regression/sgemm_tcu_wg_sp/kernel.cpp | 6 +++++- tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp | 6 +++++- 8 files changed, 45 insertions(+), 15 deletions(-) diff --git a/tests/regression/sgemm_tcu_mx/kernel.cpp b/tests/regression/sgemm_tcu_mx/kernel.cpp index f091d62569..e065f357eb 100644 --- a/tests/regression/sgemm_tcu_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_mx/kernel.cpp @@ -30,6 +30,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fill_fragment(fragC, 0); + size_t a_lane_off = ctx::lane_offset(K); + size_t b_lane_off = ctx::lane_offset(K); + for (uint32_t i = 0; i < K; i += ctx::tileK) { uint32_t k_tile = i / ctx::tileK; @@ -40,8 +43,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::load_mx_metadata(fragA, pTileMxA); ctx::load_mx_metadata(fragB, pTileMxB); - ctx::load_matrix_sync(fragA, pTileA, K); - ctx::load_matrix_sync(fragB, pTileB, K); + ctx::load_matrix_sync(fragA, pTileA, K, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); ctx::mma_sync(fragC, fragA, fragB, fragC); } diff --git a/tests/regression/sgemm_tcu_sp_mx/kernel.cpp b/tests/regression/sgemm_tcu_sp_mx/kernel.cpp index f75f2e2be4..4d776b528f 100644 --- a/tests/regression/sgemm_tcu_sp_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_sp_mx/kernel.cpp @@ -31,6 +31,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fill_fragment(fragC, 0); + size_t a_lane_off = ctx::lane_offset(K / 2); + size_t b_lane_off = ctx::lane_offset(K); + for (uint32_t i = 0; i < K; i += ctx::tileK) { uint32_t k_tile = i / ctx::tileK; auto pTileA = pA + tile_row * (K / 2) + i / 2; @@ -42,8 +45,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::load_sp_metadata(fragA, pTileMetaSp); ctx::load_mx_metadata(fragA, pTileMxA); ctx::load_mx_metadata(fragB, pTileMxB); - ctx::load_matrix_sync(fragA, pTileA, K / 2); - ctx::load_matrix_sync(fragB, pTileB, K); + ctx::load_matrix_sync(fragA, pTileA, K / 2, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); ctx::mma_sync(fragC, fragA, fragB, fragC); } diff --git a/tests/regression/sgemm_tcu_wg/kernel.cpp b/tests/regression/sgemm_tcu_wg/kernel.cpp index 5d75739669..c395aeab13 100644 --- a/tests/regression/sgemm_tcu_wg/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg/kernel.cpp @@ -33,6 +33,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); +#ifdef WGMMA_RMAJOR_A + constexpr uint32_t a_ldm = ctx::tileK; // row-major A: ldm = tileK elements +#else + constexpr uint32_t a_ldm = 0; // block-major A +#endif +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(a_ldm); +#endif + for (uint32_t k = 0; k < K; k += ctx::tileK) { // Cooperatively load A [cta_M × tileK] into smem. // Default (block-major): A_smem[a_blockmajor_idx(r, c)]. @@ -74,15 +83,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #endif auto desc_b = vt::vx_make_smem_desc(B_smem, 0); // block-major: stride field unused - #ifdef WGMMA_RMAJOR_A - constexpr uint32_t a_ldm = ctx::tileK; // row-major A: ldm = tileK elements - #else - constexpr uint32_t a_ldm = 0; // block-major A - #endif #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) // RS: A from registers, B from smem (NRC <= 16 only). ctx::fragment_a fragA; - ctx::load_matrix_sync(fragA, A_warp, a_ldm); + ctx::load_matrix_sync(fragA, A_warp, a_ldm, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else // SS: both from smem diff --git a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp index 7b2596dabb..9a62b440a4 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp @@ -119,6 +119,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { (void)A_smem; (void)B_smem; (void)bar; (void)k; }; +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK); +#endif + // One WGMMA accumulation step out of the given buffer pair. auto compute_tile = [&](ctx::input_t* A_smem, ctx::input_t* B_smem) { auto A_warp = A_smem + warp_rank * ctx::xtileM * ctx::tileK; @@ -127,7 +131,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) // RS: A from registers, B from smem (NRC <= 16 only) ctx::fragment_a fragA; - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else // SS: both from smem diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp index 569b999df7..fedc34ccad 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp @@ -59,6 +59,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { vortex::group_barrier group_B(2, mc_group_size); const bool is_loader_warp = (get_sub_group_id() == 0); +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK); +#endif + for (uint32_t k = 0; k < K; k += ctx::tileK) { if (is_loader_warp) { local_A.expect_tx(1); // A: per-CTA, manual @@ -85,7 +89,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) ctx::fragment_a fragA; - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else auto desc_a = vt::vx_make_smem_desc(A_warp, ctx::tileK * sizeof(ctx::input_t)); diff --git a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp index 34bb9e5c9c..3ea0ced199 100644 --- a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp @@ -37,6 +37,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK); +#endif + for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; uint32_t a_bytes = cta_m * ctx::tileK; @@ -62,7 +66,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto A_warp = A_smem + warp_rank * ctx::xtileM * ctx::tileK; auto desc_b = vt::vx_make_smem_desc(B_smem, ctx::tileK); #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else auto desc_a = vt::vx_make_smem_desc(A_warp, ctx::tileK); diff --git a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp index f1872cbed6..f531b1cf2e 100644 --- a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp @@ -55,6 +55,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t num_k_tiles = K / ctx::tileK; uint32_t meta_words_per_tile = ctx::wg_meta_total_bytes / 4; +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK / 2); +#endif + for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; @@ -94,7 +98,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) // RS: A from registers, B from smem (NRC <= 16 only). - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK / 2); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK / 2, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else // SS: both A and B from smem descriptors diff --git a/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp index b02b4ea17c..ee7b27250d 100644 --- a/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp @@ -55,6 +55,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t num_k_tiles = K / ctx::tileK; uint32_t meta_words_per_tile = ctx::wg_meta_total_bytes / 4; +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK / 2); +#endif + // Loop over K tiles for (uint32_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; @@ -88,7 +92,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) // RS: A from registers, B from smem (NRC <= 16 only) - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK / 2); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK / 2, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else // SS: both A and B from smem descriptors From d72b25c0ae12a796a654b9ef38696beddf9a0a79 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 15:46:06 -0800 Subject: [PATCH 330/336] tcu: add matrix fragment load/store macro-ops (vx_ldm, vx_stm) A TCU fragment load or store was one scalar flw/fsw per register: 8 per A/B/C fragment, each fetched, decoded and issued on its own. vx_ldm and vx_stm name a whole fragment with one instruction: a 3-bit shape index (A, B, C, sparse A, symmetric sparse B, WGMMA C) plus a base register group, and VX_uop_sequencer expands it in the packld slot into the register-ordered uops the scalar code would have issued. Each uop carries its own destination/source register, so the scoreboard and writeback see the same per-register traffic as before. - RTL: the expansion is closed form (row group and column from the uop counter, the uop count carried in the bytesel offset); a simulation check in VX_uop_packld walks every shape and uop through the TCU's forward register permutations (tcu_ra/rb/rc_off) and asserts it lands back on the same register. Beacon against the parent: no register deeper in core, core_tcu or tcu; core_tcu +169 LUT, core -4. - SimX: the same expansion, searched from the forward permutations. - Kernel header: wmma_context/wgmma_context take a bool ldst_matrix template parameter; every TCU benchmark selects it with -DLDST_MATRIX, like -DITYPE and -DLDST_WIDE. Shapes the hardware does not cover, or that did not pay on rtlsim (sparse MX, asymmetric sparse B), keep the scalar loads; without the flag the code is identical to before. - Test: ldst_matrix checks every register holds the element the TCU reads from it (fails on a corrupted register map), on dense, sparse, WGMMA and 4-LSU-lane builds; model_parity and sgemm_tcu* -ldm cases. Mode 1 vs mode 0 on rtlsim: -3.4% to -10.1% cycles on the dense, mx and sparse TCU kernels; sgemm2_tcu +1.0% at the CI size (its two resident blocks fall into step and contend in the copy loop; -0.2%/-0.4% at 2 and 8 warps). Results are bit-identical to mode 0. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 6 + ci/testcases/dxa.yaml | 12 +- ci/testcases/regression.yaml | 34 ++ ci/testcases/tensor.yaml | 74 ++- ci/testcases/tensor_mx.yaml | 21 +- ci/testcases/tensor_sp.yaml | 25 +- ci/testcases/tensor_wg.yaml | 27 +- .../custom_accelerator_isa_extensions.md | 53 ++- docs/designs/simx_simulator_architecture.md | 4 +- hw/rtl/VX_trace_pkg.sv | 6 +- hw/rtl/core/VX_decode.sv | 45 +- hw/rtl/core/VX_uop_packld.sv | 64 ++- hw/rtl/tcu/VX_tcu_pkg.sv | 216 +++++++++ hw/rtl/tcu/VX_tcu_uops.sv | 30 +- sim/simx/decode.cpp | 39 +- sim/simx/lsu_unit.cpp | 28 ++ sim/simx/tcu/tcu_unit.cpp | 193 ++++++-- sim/simx/tcu/tcu_unit.h | 13 + sim/simx/types.h | 1 + sw/kernel/include/vx_tensor.h | 436 ++++++++++++++---- tests/regression/Makefile | 2 +- tests/regression/ldst_matrix/Makefile | 18 + tests/regression/ldst_matrix/common.h | 26 ++ tests/regression/ldst_matrix/kernel.cpp | 258 +++++++++++ tests/regression/ldst_matrix/main.cpp | 163 +++++++ tests/regression/sgemm2_tcu/common.h | 11 + tests/regression/sgemm2_tcu/kernel.cpp | 5 +- tests/regression/sgemm_tcu/common.h | 11 + tests/regression/sgemm_tcu/kernel.cpp | 33 +- tests/regression/sgemm_tcu_mx/common.h | 11 + tests/regression/sgemm_tcu_mx/kernel.cpp | 30 +- tests/regression/sgemm_tcu_sp/common.h | 11 + tests/regression/sgemm_tcu_sp/kernel.cpp | 36 +- tests/regression/sgemm_tcu_sp_mx/common.h | 11 + tests/regression/sgemm_tcu_sp_mx/kernel.cpp | 31 +- tests/regression/sgemm_tcu_wg/common.h | 11 + tests/regression/sgemm_tcu_wg/kernel.cpp | 5 +- tests/regression/sgemm_tcu_wg_dxa/common.h | 11 + tests/regression/sgemm_tcu_wg_dxa/kernel.cpp | 8 +- .../sgemm_tcu_wg_dxa_mcast/common.h | 11 + .../sgemm_tcu_wg_dxa_mcast/kernel.cpp | 5 +- tests/regression/sgemm_tcu_wg_mx/common.h | 11 + tests/regression/sgemm_tcu_wg_mx/kernel.cpp | 5 +- tests/regression/sgemm_tcu_wg_sp/common.h | 11 + tests/regression/sgemm_tcu_wg_sp/kernel.cpp | 5 +- tests/regression/sgemm_tcu_wg_sp_dxa/common.h | 11 + .../regression/sgemm_tcu_wg_sp_dxa/kernel.cpp | 5 +- 47 files changed, 1903 insertions(+), 180 deletions(-) create mode 100644 tests/regression/ldst_matrix/Makefile create mode 100644 tests/regression/ldst_matrix/common.h create mode 100644 tests/regression/ldst_matrix/kernel.cpp create mode 100644 tests/regression/ldst_matrix/main.cpp diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 8e65c169e3..51e8dcb36d 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -276,6 +276,12 @@ tests: via: blackbox app: ldst_wide configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_LSU_LANES=4 +# Matrix fragment loads and stores: SimX and RTL expand and time them alike. +- id: parity-ldst-matrix + check: model_parity + via: blackbox + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_SPARSE_ENABLE - id: parity-sgemm2-lmem check: model_parity via: blackbox diff --git a/ci/testcases/dxa.yaml b/ci/testcases/dxa.yaml index ef8c54d3b4..4e84687297 100644 --- a/ci/testcases/dxa.yaml +++ b/ci/testcases/dxa.yaml @@ -619,4 +619,14 @@ tests: configs: -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 shape: - threads: [4, 16] \ No newline at end of file + threads: [4, 16] + +# The DXA-staged WGMMA accumulator stored with vx_stm (LDST_MATRIX=1). +- id: sgemm_tcu_wg_dxa-ldm + via: blackbox + drivers: + - simx + app: sgemm_tcu_wg_dxa + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DLDST_MATRIX=1 + shape: {threads: 32, warps: 8} + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 16cc964ab7..3ad1590526 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -103,6 +103,40 @@ tests: configs: -DVX_CFG_SIMD_WIDTH=1 shape: threads: 4 +# Matrix fragment loads and stores (vx_ldm, vx_stm): every shape of the +# configuration, local and global memory, row strides that are not powers of +# two, zero/positive/negative offsets, aligned and unaligned register bases. +- id: ldst-matrix-1 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI +# Symmetric sparse geometry (4 lanes). +- id: ldst-matrix-2 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_SPARSE_ENABLE +# WGMMA accumulator chunks (16 lanes). +- id: ldst-matrix-3 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_WGMMA_ENABLE +# Each uop split into 4-lane packets, symmetric sparse shapes included. +- id: ldst-matrix-4 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_NUM_LSU_LANES=4 # The sgemm benchmarks with each thread's consecutive words read and written # through the wide forms (LDST_WIDE=1). - id: ldst-wide-sgemm diff --git a/ci/testcases/tensor.yaml b/ci/testcases/tensor.yaml index dbcba1d2f5..ff94ff8a3b 100644 --- a/ci/testcases/tensor.yaml +++ b/ci/testcases/tensor.yaml @@ -229,4 +229,76 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 # Not warp-width swept: the tensor-core fragment layout is defined over a fixed - # lane count, which the configs pin. \ No newline at end of file + # lane count, which the configs pin. + +# Fragments moved with vx_ldm/vx_stm (LDST_MATRIX=1), alone and with the K loop +# unrolled (KUNROLL), whose tile offsets the instructions take as immediates. +- id: sgemm_tcu-ldm-1 + via: blackbox + drivers: + - simx + app: sgemm_tcu + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu-ldm-2 + via: blackbox + drivers: + - simx + app: sgemm_tcu + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=bf16 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=2 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu-ldm-3 + via: blackbox + drivers: + - simx + app: sgemm_tcu + args: -m 64 -n 64 -k 128 + configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=fp8 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=4 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu-ldm-4 + via: blackbox + drivers: + - rtlsim + app: sgemm_tcu + configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu-ldm-5 + via: blackbox + drivers: + - rtlsim + app: sgemm_tcu + args: -m 32 -n 32 -k 96 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=4 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm2_tcu-ldm + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_tcu + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 + -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + args: -w 4 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-fp16-ldm + check: model_parity + via: blackbox + app: sgemm_tcu + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-fp16-ldm-u4 + check: model_parity + via: blackbox + app: sgemm_tcu + args: -m 64 -n 64 -k 128 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=4 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-sgemm2-fp16-ldm + check: model_parity + via: blackbox + app: sgemm2_tcu + args: -m 128 -n 128 -k 64 + # One execution block per unit: the four-block model ISSUE_WIDTH=4 implies is + # several times the size, and a hosted CI runner cannot build it in a job. + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/ci/testcases/tensor_mx.yaml b/ci/testcases/tensor_mx.yaml index 197bcd60ec..7662bacd36 100644 --- a/ci/testcases/tensor_mx.yaml +++ b/ci/testcases/tensor_mx.yaml @@ -281,4 +281,23 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI # Not warp-width swept: the tensor-core fragment layout is defined over a fixed - # lane count, which the configs pin. \ No newline at end of file + # lane count, which the configs pin. + +# MX fragments moved with vx_ldm/vx_stm (LDST_MATRIX=1). +- id: sgemm_tcu_mx-ldm-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DLDST_MATRIX=1 + args: -k64 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu_mx-ldm-2 + via: blackbox + drivers: + - simx + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp4 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=2 + args: -k64 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/ci/testcases/tensor_sp.yaml b/ci/testcases/tensor_sp.yaml index bcaba14971..1b4d69a159 100644 --- a/ci/testcases/tensor_sp.yaml +++ b/ci/testcases/tensor_sp.yaml @@ -298,4 +298,27 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE # Not warp-width swept: the tensor-core fragment layout is defined over a fixed - # lane count, which the configs pin. \ No newline at end of file + # lane count, which the configs pin. + +# Sparse fragments moved with vx_ldm (LDST_MATRIX=1). +- id: sgemm_tcu_sp-ldm-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_SPARSE_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu_sp-ldm-2 + via: blackbox + drivers: + - simx + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 -DLDST_MATRIX=1 -DKUNROLL=2 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-sp-ldm + check: model_parity + via: blackbox + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index fd07f4ada8..9b7787be1c 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -266,4 +266,29 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 # Not warp-width swept: the tensor-core fragment layout is defined over a fixed - # lane count, which the configs pin. \ No newline at end of file + # lane count, which the configs pin. + +# The WGMMA accumulator stored with vx_stm in 8-register chunks (LDST_MATRIX=1). +- id: sgemm_tcu_wg-ldm-1 + via: blackbox + drivers: + - simx + app: sgemm_tcu_wg + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu_wg-ldm-2 + via: blackbox + drivers: + - simx + app: sgemm_tcu_wg + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DLDST_MATRIX=1 -DKUNROLL=2 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-wgmma-ldm + check: model_parity + via: blackbox + app: sgemm_tcu_wg + args: -m64 -n64 -k64 + # One execution block per unit: the four-block model ISSUE_WIDTH=4 implies is + # several times the size, and a hosted CI runner cannot build it in a job. + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=16 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/docs/designs/custom_accelerator_isa_extensions.md b/docs/designs/custom_accelerator_isa_extensions.md index 1a84e546de..d6c611a042 100644 --- a/docs/designs/custom_accelerator_isa_extensions.md +++ b/docs/designs/custom_accelerator_isa_extensions.md @@ -233,11 +233,13 @@ the table in the same change. | | | `7` | `vx_rt_trace`, `vx_rt_wait` | R4 | | | | `1`, `2`, `4` | free | | | custom-2 `0x5B` | `funct7` | `0x00` | MUFU special functions; `fn` in `rs2`, `fmt` in `funct3` | R | -| custom-3 `0x7B` | `funct3` | `3` | `vx_flw.d fd, imm(rs1)`: `fd`, `fd+1` ← two words | I | +| custom-3 `0x7B` | `funct3` | `0` | `vx_ldm fd, off(rs1), shape`: a TCU matrix fragment ← memory | I | +| | | `2` | `vx_stm fs2, off(rs1), shape`: memory ← a TCU matrix fragment | S | +| | | `3` | `vx_flw.d fd, imm(rs1)`: `fd`, `fd+1` ← two words | I | | | | `4` | `vx_flw.q fd, imm(rs1)`: `fd`..`fd+3` ← four words | I | | | | `5` | `vx_fsw.d fs2, imm(rs1)`: two words ← `fs2`, `fs2+1` | S | | | | `6` | `vx_fsw.q fs2, imm(rs1)`: four words ← `fs2`..`fs2+3` | S | -| | | `0`–`2`, `7` | reserved | | +| | | `1`, `7` | reserved | | The wide loads and stores have the layout of `fld`/`flq` and `fsd`/`fsq`, with a signed 12-bit byte offset. Each is a macro-op of 2 or 4 plain `flw`/`fsw`: uop @@ -248,7 +250,50 @@ needs only 4-byte alignment. The register must be a multiple of 2 (`.d`) or 4 (`.q`) and `imm` a multiple of 8 or 16, which lets the expander place *k* in the low bits of both without an adder. An encoding that breaks either rule is reserved and its result undefined; the assembler rejects it. They are defined -with F and FLEN 32 (no D); elsewhere custom-3 is illegal. +with F and FLEN 32 (no D). + +The matrix fragment load and store move one tensor-core fragment (the +registers `vx_tensor.h`'s `load_matrix_sync`/`store_matrix_sync` fill) with one +instruction. `imm[4:0]` is the fragment **shape**, `imm[11:5]` a byte offset +`off` (a multiple of 32), and `rs1` the first of a **base group** of J integer +registers, a multiple of J, where register `rs1 + j` holds the lane's address +of the fragment's row group *j*. Each is a macro-op of one plain `flw`/`fsw` per +fragment register: uop *e* moves register `fd + e` (`fs2 + e`) to or from the +word at `x[rs1 | j(r)] + (off | c(r)·CS)`, where *r* = π⁻¹(*e*) is the fragment +element the WMMA/WGMMA expansion reads from that register (π is its +bank-conflict-free register order), `(j, c)` a bit split of *r*, and CS the byte +stride between a lane's words in a row. The register add takes the uop counter +directly; the shape only selects the row group and column. So neither the LSU +nor the address path does anything new: the +row step and the lane's own offset live in the base group, computed by +software once per tile, and the column offsets are bit selects. `off` must be +zero in the bits the column offsets use, which makes the OR an add; unrolled K +tiles then share one base group and differ only in `off`. The shapes are +`{ctx, role[1:0], trans, sparse}`; those a configuration has are + +| shape | fragment | J | per row group | registers | +|---|---|---|---|---| +| `0x00` | WMMA A, row-major | M steps | K steps, `4·tcK` B apart | 8 | +| `0x01` | WMMA sparse A, row-major | M steps | K/2 steps, `4·tcK` B apart | 4 | +| `0x06` | WMMA B, col-major | N steps per register | K steps, `4·tcK` B apart | 4 or 8 | +| `0x07` | WMMA symmetric sparse B, col-major | one N step per register | 1 word | 4 or 8 | +| `0x08` | WMMA C, row-major | M steps | N steps, `4·tcN` B apart | 8 | +| `0x18` | WGMMA C, row-major, one 8-register chunk | 2 | 4 columns, `4·tcN` B apart | 8 | + +with the sparse shapes under `VX_CFG_TCU_SPARSE_ENABLE` (sparse B only in the +symmetric layouts, NT=4 and NT=16) and the WGMMA one under +`VX_CFG_TCU_WGMMA_ENABLE`; every other shape is reserved (SimX stops on it). +In the RTL, [`VX_tcu_pkg.sv`](../../hw/rtl/tcu/VX_tcu_pkg.sv) (`tcu_ldm_uop`) +holds the shapes' geometry next to the operand permutations `VX_tcu_uops` +reads with, and `VX_uop_packld` asserts at elaboration that every uop of +every shape maps back through those permutations to its own register; SimX +derives its order from the same permutations by search. So the two cannot +drift apart. The words of a fragment register span its element type, +so one shape serves every ITYPE. A layout whose register gathers words from +several rows (a sub-word B row-major, A col-major) has no shape: software stages +such a tile transposed. A 16- or 32-register WGMMA accumulator takes 2 or 4 +`vx_stm`, one per chunk (`fs2 = f0, f8, f16, f24`), each chunk's column offset +in `off`. --- @@ -276,7 +321,7 @@ a priority encoder in the sequencer: | slot | expander | macro-ops | |---|---|---| -| `UOP_PACKLD` | [`VX_uop_packld.sv`](../../hw/rtl/core/VX_uop_packld.sv) | packed loads, wide loads and stores | +| `UOP_PACKLD` | [`VX_uop_packld.sv`](../../hw/rtl/core/VX_uop_packld.sv) | packed loads, wide loads and stores, matrix fragment loads and stores | | `UOP_TCU` | [`VX_tcu_uops.sv`](../../hw/rtl/tcu/VX_tcu_uops.sv) | WMMA, WGMMA | | `UOP_GFX` | [`VX_gfx_uops.sv`](../../hw/rtl/core/VX_gfx_uops.sv) | ray-trace windows, fragment export | diff --git a/docs/designs/simx_simulator_architecture.md b/docs/designs/simx_simulator_architecture.md index 81626299d0..a37cfe2425 100644 --- a/docs/designs/simx_simulator_architecture.md +++ b/docs/designs/simx_simulator_architecture.md @@ -105,7 +105,9 @@ atomics, threads, locks, or deferred-call plumbing — enforced by **Functional units** - `alu_unit`, `fpu_unit` — own private `execute()` (ISA semantics). -- `lsu_unit` — loads/stores/AMO; `LsuUopGen` for packed loads and wide loads/stores; pulls load +- `lsu_unit` — loads/stores/AMO; `LsuUopGen` for packed loads, wide loads/stores and + matrix fragment loads/stores (`vx_ldm`/`vx_stm`, whose geometry and register order + `TcuUopGen::ldm_uop` derives with the WMMA expansion's permutations); pulls load data from `MemRsp::data`. - `sfu_unit` — special-function dispatcher routing to sub-units: WCTL, CSR, TEX, RASTER, DXA, OM. diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index f90d56d41b..0842b8b26d 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -181,8 +181,10 @@ package VX_trace_pkg; `TRACE(level, ("PACKLH.F")) end default: begin - // a wide access's bytesel offset marks a quad - if (op_args.lsu.is_store) begin + // the bytesel offset marks a quad (1) or a matrix fragment (2, 3) + if (bytesel[1]) begin + `TRACE(level, (op_args.lsu.is_store ? "VX_STM" : "VX_LDM")) + end else if (op_args.lsu.is_store) begin `TRACE(level, ((bytesel != BYTESEL_DEFAULT) ? "VX_FSW.Q" : "VX_FSW.D")) end else begin `TRACE(level, ((bytesel != BYTESEL_DEFAULT) ? "VX_FLW.Q" : "VX_FLW.D")) diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 3f083fe26d..7e2e746664 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -489,12 +489,44 @@ module VX_decode import `endif end `ifdef VX_CFG_EXT_F_ENABLE - `ifndef VX_CFG_FLEN_64 - // vx_flw.d/.q and vx_fsw.d/.q: 2 or 4 words of an aligned FP register - // group, expanded into flw/fsw uops; the bytesel offset carries the - // group size. INST_EXT4: begin case (funct3) + `ifdef VX_CFG_EXT_TCU_ENABLE + // vx_ldm/vx_stm: a TCU matrix fragment, expanded into one flw/fsw + // per register. imm[4:0] is the fragment shape, replaced by its + // expander code; a bytesel offset of 2 (4 registers) or 3 (8) + // marks the access as a fragment. + 3'h0: begin + if (VX_tcu_pkg::tcu_ldm_valid(u_12[4:0])) begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'(INST_LSU_LW); + op_args.lsu.is_store = 0; + op_args.lsu.is_float = 1; + op_args.lsu.pack = 2'b11; + op_args.lsu.offset = {u_12[11:5], VX_tcu_pkg::tcu_ldm_code(u_12[4:0])}; + bytesel = {XLENB_W'(XLENB-1), XLENB_W'({1'b1, VX_tcu_pkg::tcu_ldm_is8(u_12[4:0])})}; + `USED_IREG (rs1); + `USED_FREG (rd); + end + end + 3'h2: begin + if (VX_tcu_pkg::tcu_ldm_valid(rd)) begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'(INST_LSU_SW); + op_args.lsu.is_store = 1; + op_args.lsu.is_float = 1; + op_args.lsu.pack = 2'b11; + op_args.lsu.offset = {s_imm[11:5], VX_tcu_pkg::tcu_ldm_code(rd)}; + bytesel = {XLENB_W'(XLENB-1), XLENB_W'({1'b1, VX_tcu_pkg::tcu_ldm_is8(rd)})}; + `USED_IREG (rs1); + `USED_FREG (rs2); + end + end + `endif + `ifndef VX_CFG_FLEN_64 + // vx_flw.d/.q and vx_fsw.d/.q: 2 or 4 words of an aligned FP register + // group, expanded into flw/fsw uops; the bytesel offset carries the + // group size. 3'h3, 3'h4: begin ex_type = EX_LSU; op_type = INST_OP_BITS'(INST_LSU_LW); @@ -517,11 +549,11 @@ module VX_decode import `USED_IREG (rs1); `USED_FREG (rs2); end + `endif default:; endcase end `endif - `endif `ifdef VX_CFG_EXT_F_ENABLE INST_FMADD, // 7'b1000011 INST_FMSUB, // 7'b1000111 @@ -831,6 +863,9 @@ module VX_decode import op_args.lsu.is_float = 1; op_args.lsu.pack = 2'b01; op_args.lsu.offset = '0; + // the bytesel offset of a pack/wide/fragment access + // carries its uop count: 0 → 2, 1 or 2 → 4, 3 → 8 + bytesel = {XLENB_W'(XLENB-1), XLENB_W'(1)}; `USED_FREG (rd); `USED_IREG (rs1); `USED_IREG (rs2); diff --git a/hw/rtl/core/VX_uop_packld.sv b/hw/rtl/core/VX_uop_packld.sv index 3f77222299..f745fce632 100644 --- a/hw/rtl/core/VX_uop_packld.sv +++ b/hw/rtl/core/VX_uop_packld.sv @@ -19,6 +19,8 @@ // pack==1 (PACKLB) → 4 uops of INST_LSU_LBU (unsigned byte loads) // pack==2 (PACKLH) → 2 uops of INST_LSU_LHU (unsigned halfword loads) // pack==3 (vx_flw.d/.q, vx_fsw.d/.q) → 2 or 4 plain flw/fsw uops +// pack==3 with bytesel offset 2/3 (vx_ldm, vx_stm) → 4/8 flw/fsw, one per +// register of a TCU matrix fragment // // A pack-load uop encodes its index in op_args.lsu.offset[1:0]; the LSU AGU // reads pack (non-zero) to identify it and computes @@ -27,8 +29,16 @@ // leaves with pack cleared, so the LSU sees a plain flw/fsw. The NaN-boxing of // a loaded word is handled automatically because is_float==1 and rd is a float // register (triggers nan-box in lsu_block). +// A fragment uop e moves register fd/fs2 + e: it reads its base from +// rs1 | grp(e) and ORs its column into the immediate (whose low 5 bits decode +// set to the shape's code), for the element the TCU expansion reads from that +// register. -module VX_uop_packld import VX_gpu_pkg::*; ( +module VX_uop_packld import +`ifdef VX_CFG_EXT_TCU_ENABLE + VX_tcu_pkg::*, +`endif + VX_gpu_pkg::*; ( input clk, input reset, @@ -44,12 +54,40 @@ module VX_uop_packld import VX_gpu_pkg::*; ( // pack==1 → PACKLB: 4 byte-load uops; // pack==2 → PACKLH: 2 halfword-load uops; - // pack==3 → wide: 4 uops if the bytesel offset marks a quad, else 2 + // pack==3 → bytesel offset 0/1: a wide pair/quad, 2 or 4 uops; + // bytesel offset 2/3: a 4/8-register fragment wire is_packlb = (ibuf_in.op_args.lsu.pack == 2'b01); wire is_packlh = (ibuf_in.op_args.lsu.pack == 2'b10); wire is_wide = (ibuf_in.op_args.lsu.pack == 2'b11); - wire is_pair = is_wide && ~ibuf_in.bytesel[0]; - assign uop_count = (is_packlh || is_pair) ? UOP_CTR_W'(2) : UOP_CTR_W'(4); +`ifdef VX_CFG_EXT_TCU_ENABLE + wire is_matrix = is_wide && ibuf_in.bytesel[1]; + tcu_ldm_uop_t ldm_uop; + assign ldm_uop = tcu_ldm_uop(ibuf_in.op_args.lsu.offset[2:0], uop_idx[2:0]); + + // uop e of every valid shape reads or writes the register its element + // occupies in the WMMA/WGMMA expansion + for (genvar s = 0; s < 6; ++s) begin : g_ldm_check + localparam bit VALID = (s == TCU_LDM_IDX_A_SP) ? `VX_CFG_TCU_SPARSE_ENABLED + : (s == TCU_LDM_IDX_B_SP) ? (`VX_CFG_TCU_SPARSE_ENABLED && SYM_SPARSE) + : (s == TCU_LDM_IDX_WG_C) ? `VX_CFG_TCU_WGMMA_ENABLED : 1; + if (VALID) begin : g_valid + for (genvar e = 0; e < tcu_ldm_count(3'(s)); ++e) begin : g_uops + `STATIC_ASSERT (tcu_ldm_reg(3'(s), tcu_ldm_uop(3'(s), 3'(e))) == e, + ("vx_ldm shape %0d uop %0d does not invert the TCU register permutation", s, e)) + end + end + end +`endif + // Decode sets the bytesel offset to the uop count: 0 → 2 (PACKLH, a wide + // pair), 1 → 4 (PACKLB, a wide quad), 2 → 4 and 3 → 8 (fragments) +`ifdef VX_CFG_EXT_TCU_ENABLE + wire cnt8 = ibuf_in.bytesel[1] && ibuf_in.bytesel[0]; + wire cnt4 = ibuf_in.bytesel[1] ^ ibuf_in.bytesel[0]; +`else + wire cnt8 = 1'b0; + wire cnt4 = ibuf_in.bytesel[0]; +`endif + assign uop_count = UOP_CTR_W'({cnt8, cnt4, ~(cnt8 || cnt4), 1'b0}); wire [1:0] idx2 = uop_idx[1:0]; wire [XLENB_W-1:0] byte_size = ibuf_in.op_args.lsu.pack[1] ? '1 : '0; @@ -59,16 +97,32 @@ module VX_uop_packld import VX_gpu_pkg::*; ( // register have their low group bits clear, so each uop ORs its index in: // into offset[1:0] for a packed load; into offset[3:2] (word k) and the // register (fd+k, fs2+k) for a wide access, which becomes a plain flw/fsw. + // A fragment uop replaces offset[4:0] (the shape code) with its column and + // adds its register offset, through the same adder the wide uops use. wire [1:0] wide_idx = is_wide ? idx2 : 2'b00; wire [1:0] pack_idx = is_wide ? 2'b00 : idx2; +`ifdef VX_CFG_EXT_TCU_ENABLE + wire [11:0] off_base = is_matrix ? {ibuf_in.op_args.lsu.offset[11:5], 5'd0} : ibuf_in.op_args.lsu.offset; + wire [11:0] off_add = is_matrix ? 12'(ldm_uop.col) : {8'd0, wide_idx, pack_idx}; + wire [4:0] reg_add = is_matrix ? 5'(uop_idx[2:0]) : 5'(wide_idx); + wire [1:0] grp = is_matrix ? ldm_uop.grp : 2'b00; +`endif + ibuffer_t ibuf_r; always_comb begin ibuf_r = ibuf_in; - ibuf_r.op_args.lsu.pack = is_wide ? 2'b00 : ibuf_in.op_args.lsu.pack; + ibuf_r.op_args.lsu.pack = is_wide ? 2'b00 : ibuf_in.op_args.lsu.pack; + `ifdef VX_CFG_EXT_TCU_ENABLE + ibuf_r.op_args.lsu.offset = off_base | off_add; + ibuf_r.rs1 = ibuf_in.rs1 | NUM_REGS_BITS'(grp); + ibuf_r.rd = {ibuf_in.rd[NUM_REGS_BITS-1:5], ibuf_in.rd[4:0] + reg_add}; + ibuf_r.rs2 = {ibuf_in.rs2[NUM_REGS_BITS-1:5], ibuf_in.rs2[4:0] + reg_add}; + `else ibuf_r.op_args.lsu.offset = ibuf_in.op_args.lsu.offset | {8'd0, wide_idx, pack_idx}; ibuf_r.rd = ibuf_in.rd | NUM_REGS_BITS'(wide_idx); ibuf_r.rs2 = ibuf_in.rs2 | NUM_REGS_BITS'(wide_idx); + `endif ibuf_r.bytesel = {byte_size, byte_off}; end assign ibuf_out = ibuf_r; diff --git a/hw/rtl/tcu/VX_tcu_pkg.sv b/hw/rtl/tcu/VX_tcu_pkg.sv index f16cf62297..5ae866aefc 100644 --- a/hw/rtl/tcu/VX_tcu_pkg.sv +++ b/hw/rtl/tcu/VX_tcu_pkg.sv @@ -172,6 +172,222 @@ package VX_tcu_pkg; localparam TCU_MAX_INPUTS = TCU_TC_K * TCU_MAX_ELT_RATIO; + localparam TCU_LG_M = $clog2(TCU_M_STEPS); + localparam TCU_LG_N = $clog2(TCU_N_STEPS); + localparam TCU_LG_K = $clog2(TCU_K_STEPS); + localparam TCU_LG_B_SB = $clog2(TCU_B_SUB_BLOCKS); + + // Register offsets of the WMMA operands of step (m, n, k), permuted so the + // three operands of a step never share a register bank. + // one B sub-block: A={m[0],~m[hi],k}, B={n^k,~k}, C={m[0],~m[hi],XNOR(m[hi],n)} + // two B sub-blocks: A={k[0],~m,m^k[hi]}, B={k[0],k[hi]^np,~np}, C={n[0],~m,n[hi]} + // asymmetric sparse: B={n[hi],~(n[0]^k),~k}, C={n[hi],m,~(m^n[0])} + function automatic logic [2:0] tcu_ra_off(input logic [`UP(TCU_LG_M)-1:0] m, + input logic [`UP(TCU_LG_K)-1:0] k); + if (TCU_LG_B_SB == 0) begin + return 3'({m[0], ~m[`UP(TCU_LG_M)-1], k[0]}); + end else begin + return 3'({k[0], ~m[0], m[0] ^ k[`UP(TCU_LG_K)-1]}); + end + endfunction + + function automatic logic [2:0] tcu_rb_off(input logic [`UP(TCU_LG_N)-1:0] n, + input logic [`UP(TCU_LG_K)-1:0] k); + if (TCU_LG_B_SB == 0) begin + return 3'({n[0] ^ k[0], ~k[0]}); + end else begin + return 3'({k[0], k[`UP(TCU_LG_K)-1] ^ n[`UP(TCU_LG_N)-1], ~n[`UP(TCU_LG_N)-1]}); + end + endfunction + + function automatic logic [2:0] tcu_rc_off(input logic [`UP(TCU_LG_M)-1:0] m, + input logic [`UP(TCU_LG_N)-1:0] n); + if (TCU_LG_B_SB == 0) begin + return 3'({m[0], ~m[`UP(TCU_LG_M)-1], ~(m[`UP(TCU_LG_M)-1] ^ n[0])}); + end else begin + return 3'({n[0], ~m[0], n[`UP(TCU_LG_N)-1]}); + end + endfunction + + function automatic logic [2:0] tcu_rb_off_sp(input logic [`UP(TCU_LG_N)-1:0] n, + input logic [`UP(TCU_LG_K)-1:0] k); + return 3'({n[`UP(TCU_LG_N)-1], ~(n[0] ^ k[0]), ~k[0]}); + endfunction + + function automatic logic [2:0] tcu_rc_off_sp(input logic [`UP(TCU_LG_M)-1:0] m, + input logic [`UP(TCU_LG_N)-1:0] n); + return 3'({n[`UP(TCU_LG_N)-1], m[0], ~(m[0] ^ n[0])}); + endfunction + + // Matrix fragment load/store (vx_ldm/vx_stm) shapes: {ctx, role[1:0], trans, sparse}. + // Uop e moves fragment register fd + e between it and x[rs1 | grp] + + // (imm[11:5] | col), the element the WMMA/WGMMA expansion reads from it. + localparam TCU_LDM_A = 5'b00000; // WMMA A, row-major + localparam TCU_LDM_A_SP = 5'b00001; // WMMA sparse A, row-major + localparam TCU_LDM_B = 5'b00110; // WMMA B, col-major + localparam TCU_LDM_B_SP = 5'b00111; // WMMA symmetric sparse B, col-major + localparam TCU_LDM_C = 5'b01000; // WMMA C, row-major + localparam TCU_LDM_WG_C = 5'b11000; // WGMMA C, row-major, one 8-register chunk + + // Decode replaces the shape in imm[4:0] with its index, so the per-warp + // expander decodes 3 bits instead of 5, and marks an 8-register fragment + // in bytesel. + localparam TCU_LDM_IDX_A = 3'd0; + localparam TCU_LDM_IDX_A_SP = 3'd1; + localparam TCU_LDM_IDX_B = 3'd2; + localparam TCU_LDM_IDX_B_SP = 3'd3; + localparam TCU_LDM_IDX_C = 3'd4; + localparam TCU_LDM_IDX_WG_C = 3'd5; + + // Byte strides between the words a lane holds along a fragment row + localparam TCU_LDM_LG_CS_K = $clog2(TCU_TC_K * 4); + localparam TCU_LDM_LG_CS_N = $clog2(TCU_TC_N * 4); + localparam TCU_LDM_LG_K_SP = (TCU_LG_K > 0) ? (TCU_LG_K - 1) : 0; + localparam TCU_LDM_COL_W = TCU_LDM_LG_CS_K + 4; + + typedef struct packed { + logic [1:0] grp; + logic [TCU_LDM_COL_W-1:0] col; + } tcu_ldm_uop_t; + + function automatic logic tcu_ldm_valid(input logic [4:0] shape); + case (shape) + TCU_LDM_A, + TCU_LDM_B, + TCU_LDM_C: return 1'b1; + TCU_LDM_A_SP: return 1'(`VX_CFG_TCU_SPARSE_ENABLED); + TCU_LDM_B_SP: return 1'(`VX_CFG_TCU_SPARSE_ENABLED && SYM_SPARSE); + TCU_LDM_WG_C: return 1'(`VX_CFG_TCU_WGMMA_ENABLED); + default: return 1'b0; + endcase + endfunction + + function automatic logic [4:0] tcu_ldm_code(input logic [4:0] shape); + case (shape) + TCU_LDM_A: return 5'(TCU_LDM_IDX_A); + TCU_LDM_A_SP: return 5'(TCU_LDM_IDX_A_SP); + TCU_LDM_B: return 5'(TCU_LDM_IDX_B); + TCU_LDM_B_SP: return 5'(TCU_LDM_IDX_B_SP); + TCU_LDM_C: return 5'(TCU_LDM_IDX_C); + default: return 5'(TCU_LDM_IDX_WG_C); + endcase + endfunction + + function automatic int unsigned tcu_ldm_count(input logic [2:0] idx); + case (idx) + TCU_LDM_IDX_A_SP: return 4; + TCU_LDM_IDX_B, + TCU_LDM_IDX_B_SP: return TCU_NRB; + default: return 8; + endcase + endfunction + + function automatic logic tcu_ldm_is8(input logic [4:0] shape); + return (tcu_ldm_count(3'(tcu_ldm_code(shape))) == 8); + endfunction + + // Uop e moves register fd + e, so the counter feeds the register add + // directly; the shape only maps e to the row group and column of the element + // the WMMA/WGMMA expansion reads from that register, by inverting + // tcu_ra_off/tcu_rb_off/tcu_rc_off (a few inverted and XORed bits of e). + // VX_uop_packld checks the inverse against them at elaboration. + function automatic tcu_ldm_uop_t tcu_ldm_uop(input logic [2:0] idx, input logic [2:0] e); + tcu_ldm_uop_t u; + logic [2:0] m, n, k; + u = '0; + m = '0; + n = '0; + k = '0; + case (idx) + TCU_LDM_IDX_A: begin + if (TCU_LG_B_SB == 0) begin // A={m[0],~m[1],k} + m = {1'b0, ~e[1], e[2]}; + k = {2'b0, e[0]}; + end else begin // A={k[0],~m,m^k[1]} + m = {2'b0, ~e[1]}; + k = {1'b0, e[0] ^ ~e[1], e[2]}; + end + u.grp = 2'(m); + u.col = TCU_LDM_COL_W'(k) << TCU_LDM_LG_CS_K; + end + // registers in element order {m, k} over half the K steps + TCU_LDM_IDX_A_SP: begin + u.grp = 2'(e >> TCU_LDM_LG_K_SP); + u.col = TCU_LDM_COL_W'(e & 3'((1 << TCU_LDM_LG_K_SP) - 1)) << TCU_LDM_LG_CS_K; + end + // a col-major B row is one of its columns (a pair with two sub-blocks) + TCU_LDM_IDX_B: begin + if (TCU_LG_B_SB == 0) begin // B={n^k,~k} + k = {2'b0, ~e[0]}; + n = {2'b0, e[1] ^ ~e[0]}; + end else begin // B={k[0],k[1]^np,~np} + n = {2'b0, ~e[0]}; + k = {1'b0, e[1] ^ ~e[0], e[2]}; + end + u.grp = 2'(n); + u.col = TCU_LDM_COL_W'(k) << TCU_LDM_LG_CS_K; + end + // symmetric sparse: one row per register + TCU_LDM_IDX_B_SP: begin + u.grp = 2'(e); + end + TCU_LDM_IDX_C: begin + if (TCU_LG_B_SB == 0) begin // C={m[0],~m[1],~(m[1]^n)} + m = {1'b0, ~e[1], e[2]}; + n = {2'b0, e[0] ^ e[1]}; + end else begin // C={n[0],~m,n[1]} + m = {2'b0, ~e[1]}; + n = {1'b0, e[0], e[2]}; + end + u.grp = 2'(m); + u.col = TCU_LDM_COL_W'(n) << TCU_LDM_LG_CS_N; + end + // n-major registers {n, m}, two row groups + default: begin + u.grp = 2'(e[0]); + u.col = TCU_LDM_COL_W'(e >> 1) << TCU_LDM_LG_CS_N; + end + endcase + return u; + endfunction + + // The register the WMMA/WGMMA expansion reads element (grp, col) of a + // fragment from, through the forward permutations; -1 when (grp, col) is not + // an element of the fragment. + function automatic int tcu_ldm_reg(input logic [2:0] idx, input tcu_ldm_uop_t u); + int unsigned g, c, ck, cn; + g = int'(u.grp); + c = int'(u.col); + ck = c >> TCU_LDM_LG_CS_K; + cn = c >> TCU_LDM_LG_CS_N; + case (idx) + TCU_LDM_IDX_A: begin + if ((ck << TCU_LDM_LG_CS_K) != c || g >= TCU_M_STEPS || ck >= TCU_K_STEPS) return -1; + return int'(tcu_ra_off(`UP(TCU_LG_M)'(g), `UP(TCU_LG_K)'(ck))); + end + TCU_LDM_IDX_A_SP: begin + if ((ck << TCU_LDM_LG_CS_K) != c || g >= TCU_M_STEPS || ck >= (1 << TCU_LDM_LG_K_SP)) return -1; + return int'((g << TCU_LDM_LG_K_SP) | ck); + end + TCU_LDM_IDX_B: begin + if ((ck << TCU_LDM_LG_CS_K) != c || (g << TCU_LG_B_SB) >= `MAX(TCU_N_STEPS, 1 << TCU_LG_B_SB) || ck >= TCU_K_STEPS) return -1; + return int'(tcu_rb_off(`UP(TCU_LG_N)'(g << TCU_LG_B_SB), `UP(TCU_LG_K)'(ck))); + end + TCU_LDM_IDX_B_SP: begin + if (c != 0 || g >= TCU_NRB) return -1; + return int'(g); + end + TCU_LDM_IDX_C: begin + if ((cn << TCU_LDM_LG_CS_N) != c || g >= TCU_M_STEPS || cn >= TCU_N_STEPS) return -1; + return int'(tcu_rc_off(`UP(TCU_LG_M)'(g), `UP(TCU_LG_N)'(cn))); + end + default: begin + if ((cn << TCU_LDM_LG_CS_N) != c || g >= 2 || cn >= 4) return -1; + return int'((cn << 1) | g); + end + endcase + endfunction + function automatic int unsigned mx_fedp_sf_count( input int unsigned data_bits, input int unsigned block_elems diff --git a/hw/rtl/tcu/VX_tcu_uops.sv b/hw/rtl/tcu/VX_tcu_uops.sv index 974866a235..437c89c4ad 100644 --- a/hw/rtl/tcu/VX_tcu_uops.sv +++ b/hw/rtl/tcu/VX_tcu_uops.sv @@ -75,7 +75,6 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( `ifdef VX_CFG_TCU_SPARSE_ENABLE localparam LG_A_SB = $clog2(TCU_A_SUB_BLOCKS); `endif - localparam LG_B_SB = $clog2(TCU_B_SUB_BLOCKS); `UNUSED_VAR ({clk, reset, start, advance, uop_idx}) @@ -277,21 +276,11 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( logic [`UP(CTR_W)-1:0] rs2_offset; logic [`UP(CTR_W)-1:0] rs3_offset; - // Bank-conflict-free register offset formulas. - // Permutes A, B, C offsets so all three operands always land in - // different GPR banks, eliminating all RF read-port stalls. - // - Pattern A (LG_B_SB==0): A={m[0],~m[hi],k}, B={n^k,~k}, C={m[0],~m[hi],XNOR(m[hi],n)} - // - Pattern B (LG_B_SB>0): A={k[0],~m,m^k[hi]}, B={k[0],k[hi]^np,~np}, C={n[0],~m,n[hi]} - wire [`UP(CTR_W)-1:0] ra_off, rb_off, rc_off; - if (LG_B_SB == 0) begin : g_bcfree_bsub1 - assign ra_off = `UP(CTR_W)'({m_index[0], ~m_index[`UP(LG_M)-1], k_index[0]}); - assign rb_off = `UP(CTR_W)'({n_index[0] ^ k_index[0], ~k_index[0]}); - assign rc_off = `UP(CTR_W)'({m_index[0], ~m_index[`UP(LG_M)-1], ~(m_index[`UP(LG_M)-1] ^ n_index[0])}); - end else begin : g_bcfree_bsub2 - assign ra_off = `UP(CTR_W)'({k_index[0], ~m_index[0], m_index[0] ^ k_index[`UP(LG_K)-1]}); - assign rb_off = `UP(CTR_W)'({k_index[0], k_index[`UP(LG_K)-1] ^ n_index[`UP(LG_N)-1], ~n_index[`UP(LG_N)-1]}); - assign rc_off = `UP(CTR_W)'({n_index[0], ~m_index[0], n_index[`UP(LG_N)-1]}); - end + // Bank-conflict-free register offsets: all three operands of a step land + // in different GPR banks, so the RF read ports never stall. + wire [`UP(CTR_W)-1:0] ra_off = `UP(CTR_W)'(tcu_ra_off(m_index, k_index)); + wire [`UP(CTR_W)-1:0] rb_off = `UP(CTR_W)'(tcu_rb_off(n_index, k_index)); + wire [`UP(CTR_W)-1:0] rc_off = `UP(CTR_W)'(tcu_rc_off(m_index, n_index)); `ifdef VX_CFG_TCU_SPARSE_ENABLE if (SYM_SPARSE) begin : g_sym_off @@ -301,12 +290,9 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( assign rs2_offset = is_sparse ? `UP(CTR_W)'(n_sp) : rb_off; assign rs3_offset = is_sparse ? (`UP(CTR_W)'(eff_ctr) >> 1) : rc_off; end else begin : g_def_off - // Bank-conflict-free sparse register offsets (non-sym sparse). - // A stays identity; B and C are permuted for 0 stalls. - // B_off_sp = {n[hi], ~(n[0]^k[0]), ~k[0]} - // C_off_sp = {n[hi], m[0], ~(m[0]^n[0])} - wire [`UP(CTR_W)-1:0] rb_off_sp = `UP(CTR_W)'({n_index[`UP(LG_N)-1], ~(n_index[0] ^ k_index[0]), ~k_index[0]}); - wire [`UP(CTR_W)-1:0] rc_off_sp = `UP(CTR_W)'({n_index[`UP(LG_N)-1], m_index[0], ~(m_index[0] ^ n_index[0])}); + // Non-symmetric sparse: A stays identity; B and C are permuted. + wire [`UP(CTR_W)-1:0] rb_off_sp = `UP(CTR_W)'(tcu_rb_off_sp(n_index, k_index)); + wire [`UP(CTR_W)-1:0] rc_off_sp = `UP(CTR_W)'(tcu_rc_off_sp(m_index, n_index)); assign rs1_offset = is_sparse ? ((`UP(CTR_W)'(m_index) >> LG_A_SB) << (LG_K / 2)) | `UP(CTR_W)'(k_index) : ra_off; diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index be5b6687a8..a9fccab17e 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -288,6 +288,9 @@ static op_string_t op_string(const Instr &instr) { case LsuType::LOAD: { auto lsuArgs = std::get(instrArgs); bool dst_float = (instr.get_dest_reg().type == RegType::Float); + if (lsuArgs.matrix) { + return {"VX_LDM", to_hex_str(lsuArgs.offset)}; + } if (instr.is_macro_op() && dst_float && lsuArgs.width == 0) { return {"PACKLB.F", ""}; } @@ -321,6 +324,9 @@ static op_string_t op_string(const Instr &instr) { } case LsuType::STORE: { auto lsuArgs = std::get(instrArgs); + if (lsuArgs.matrix) { + return {"VX_STM", to_hex_str(lsuArgs.offset)}; + } if (instr.is_macro_op()) { return {(lsuArgs.width == 4) ? "VX_FSW.Q" : "VX_FSW.D", to_hex_str(lsuArgs.offset)}; } @@ -1243,8 +1249,36 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { // known-violation handoff class, checked against its decoded destination. gfx_doctrine::check(*instr); } break; -#if defined(VX_CFG_EXT_F_ENABLE) && !defined(VX_CFG_FLEN_64) +#if defined(VX_CFG_EXT_F_ENABLE) && (defined(VX_CFG_EXT_TCU_ENABLE) || !defined(VX_CFG_FLEN_64)) case Opcode::EXT4: { + #ifdef VX_CFG_EXT_TCU_ENABLE + // vx_ldm (funct3 0) and vx_stm (funct3 2): a macro-op of one flw/fsw per + // register of a TCU matrix fragment; imm[4:0] is the fragment shape. + if (funct3 == 0 || funct3 == 2) { + bool is_load = (funct3 == 0); + uint32_t imm12 = is_load ? (code >> shift_rs2) : ((funct7 << width_reg) | rd); + uint32_t shape = imm12 & 0x1f; + if (!TcuUopGen::ldm_valid(shape)) { + std::cout << "Error: " << (is_load ? "vx_ldm" : "vx_stm") << " shape " << shape + << " is not supported by this configuration" << std::endl; + std::abort(); + } + instr->set_fu_type(FUType::LSU); + instr->set_op_type(is_load ? LsuType::LOAD : LsuType::STORE); + instr->set_src_reg(0, rs1, RegType::Integer); + if (is_load) { + instr->set_dest_reg(rd, RegType::Float); + } else { + instr->set_src_reg(1, rs2, RegType::Float); + } + IntrLsuArgs args{/*LW*/ 2, /*stride*/ 0, (int32_t)sext(imm12, width_i_imm)}; + args.matrix = true; + instr->set_args(args); + instr->set_macro_op(); + break; + } + #endif + #ifndef VX_CFG_FLEN_64 // vx_flw.d/.q (funct3 3/4) and vx_fsw.d/.q (funct3 5/6): a macro-op of 2 // or 4 flw/fsw uops over an aligned FP register group. The width is the // access's log2 bytes, as for fld/flq. @@ -1273,6 +1307,9 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { } instr->set_args(IntrLsuArgs{width, /*stride*/ 0, offset}); instr->set_macro_op(); + #else + std::abort(); + #endif } break; #endif #ifdef VX_CFG_EXT_MUFU_ENABLE diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 4c462ffa17..135d2f43d6 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -26,6 +26,9 @@ #include "stack_interleave.h" #include "VX_types.h" #include "VX_config.h" +#ifdef VX_CFG_EXT_TCU_ENABLE +#include "tcu_unit.h" +#endif using namespace vortex; @@ -37,7 +40,13 @@ uint32_t LsuUopGen::uop_count(const Instr& instr) { // PACKLB.F: width=0 (LB), 4 byte-elements // PACKLH.F: width=1 (LH), 2 halfword-elements // VX_FLW/VX_FSW.D: width=3, 2 words; .Q: width=4, 4 words + // VX_LDM/VX_STM: one word per register of the fragment shape auto args = std::get(instr.get_args()); +#ifdef VX_CFG_EXT_TCU_ENABLE + if (args.matrix) { + return TcuUopGen::ldm_count(args.offset & 0x1f); + } +#endif if (args.width >= 3) { return 1u << (args.width - 2); } @@ -80,6 +89,25 @@ Instr::Ptr LsuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { } auto args = std::get(macro_instr.get_args()); +#ifdef VX_CFG_EXT_TCU_ENABLE + if (args.matrix) { + // a fragment uop reads its base from rs1 | grp, ORs its column into the + // offset (whose low 5 bits held the shape) and moves register fd/fs2 + reg_off + bool is_load = (std::get(macro_instr.get_op_type()) == LsuType::LOAD); + auto u = TcuUopGen::ldm_uop(args.offset & 0x1f, uop_index); + auto uop_instr = std::allocate_shared(pool_, uop_uuid, FUType::LSU); + uop_instr->set_parent_uuid(parent_uuid); + uop_instr->set_op_type(macro_instr.get_op_type()); + uop_instr->set_src_reg(0, macro_instr.get_src_reg(0).idx | u.grp, RegType::Integer); + if (is_load) { + uop_instr->set_dest_reg((macro_instr.get_dest_reg().idx + u.reg_off) & 0x1f, RegType::Float); + } else { + uop_instr->set_src_reg(1, (macro_instr.get_src_reg(1).idx + u.reg_off) & 0x1f, RegType::Float); + } + uop_instr->set_args(IntrLsuArgs{/*LW*/ 2, /*stride*/ 0, int32_t((args.offset & ~0x1f) | u.col)}); + return uop_instr; + } +#endif if (args.width >= 3) { // a wide access's uop k moves word k: register fd/fs2 + k, offset + 4k bool is_load = (std::get(macro_instr.get_op_type()) == LsuType::LOAD); diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index f456172aa4..7d958215ce 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -1632,6 +1632,158 @@ op_string_t TcuUnit::op_string(TcuType tcu_type, IntrTcuArgs args) { /////////////////////////////////////////////////////////////////////////////// +namespace { + +constexpr uint32_t clog2(uint32_t x) { + return (x < 2) ? 0 : (1 + clog2(x / 2)); +} + +constexpr uint32_t kLgM = clog2(cfg::m_steps); +constexpr uint32_t kLgN = clog2(cfg::n_steps); +constexpr uint32_t kLgK = clog2(cfg::k_steps); +constexpr uint32_t kLgBSb = clog2(cfg::b_sub_blocks); + +constexpr uint32_t hi_bit(uint32_t x, uint32_t lg) { + return (x >> (lg ? (lg - 1) : 0)) & 1; +} + +// Register offsets of the WMMA operands of step (m, n, k), permuted so the +// three operands of a step never share a register bank. +// one B sub-block: A={m[0],~m[hi],k}, B={n^k,~k}, C={m[0],~m[hi],XNOR(m[hi],n)} +// two B sub-blocks: A={k[0],~m,m^k[hi]}, B={k[0],k[hi]^np,~np}, C={n[0],~m,n[hi]} +// asymmetric sparse: B={n[hi],~(n[0]^k),~k}, C={n[hi],m,~(m^n[0])} +uint32_t ra_off(uint32_t m, uint32_t k) { + if (kLgBSb == 0) { + return ((m & 1) << 2) | ((1 - hi_bit(m, kLgM)) << 1) | (k & 1); + } + return ((k & 1) << 2) | ((1 - (m & 1)) << 1) | ((m & 1) ^ hi_bit(k, kLgK)); +} + +uint32_t rb_off(uint32_t n, uint32_t k) { + if (kLgBSb == 0) { + return (((n ^ k) & 1) << 1) | (1 - (k & 1)); + } + uint32_t np = hi_bit(n, kLgN); + return ((k & 1) << 2) | ((hi_bit(k, kLgK) ^ np) << 1) | (1 - np); +} + +uint32_t rc_off(uint32_t m, uint32_t n) { + if (kLgBSb == 0) { + uint32_t m_hi = hi_bit(m, kLgM); + return ((m & 1) << 2) | ((1 - m_hi) << 1) | (1 - (m_hi ^ (n & 1))); + } + return ((n & 1) << 2) | ((1 - (m & 1)) << 1) | hi_bit(n, kLgN); +} + +uint32_t rb_off_sp(uint32_t n, uint32_t k) { + return (hi_bit(n, kLgN) << 2) | ((1 - ((n ^ k) & 1)) << 1) | (1 - (k & 1)); +} + +uint32_t rc_off_sp(uint32_t m, uint32_t n) { + return (hi_bit(n, kLgN) << 2) | ((m & 1) << 1) | (1 - ((m ^ n) & 1)); +} + +// vx_ldm/vx_stm shapes +constexpr uint32_t kLdmA = 0x00; // WMMA A, row-major +constexpr uint32_t kLdmASp = 0x01; // WMMA sparse A, row-major +constexpr uint32_t kLdmB = 0x06; // WMMA B, col-major +constexpr uint32_t kLdmBSp = 0x07; // WMMA symmetric sparse B, col-major +constexpr uint32_t kLdmC = 0x08; // WMMA C, row-major +constexpr uint32_t kLdmWgC = 0x18; // WGMMA C, row-major, one 8-register chunk + +// Byte strides between the words a lane holds along a fragment row +constexpr uint32_t kLdmLgCsK = clog2(cfg::tcK * 4); +constexpr uint32_t kLdmLgCsN = clog2(cfg::tcN * 4); +// Row groups of a col-major B fragment +constexpr uint32_t kLdmLgBRows = kLgN - kLgBSb; +constexpr uint32_t kLdmLgKSp = (kLgK > 0) ? (kLgK - 1) : 0; + +uint32_t ldm_lg_count(uint32_t shape) { + switch (shape) { + case kLdmA: return kLgM + kLgK; + case kLdmASp: return kLgM + kLdmLgKSp; + case kLdmB: + case kLdmBSp: return clog2(cfg::NRB); + default: return 3; // C: 8 registers + } +} + +} // namespace + +bool TcuUopGen::ldm_valid(uint32_t shape) { + switch (shape) { + case kLdmA: + case kLdmB: + case kLdmC: + return true; +#ifdef VX_CFG_TCU_SPARSE_ENABLE + case kLdmASp: + return true; + case kLdmBSp: + return cfg::sym_sparse; +#endif +#ifdef VX_CFG_TCU_WGMMA_ENABLE + case kLdmWgC: + return true; +#endif + default: + return false; + } +} + +uint32_t TcuUopGen::ldm_count(uint32_t shape) { + return 1u << ldm_lg_count(shape); +} + +// Element e of a fragment: its register offset, row group and column +static TcuUopGen::LdmUop ldm_elem(uint32_t shape, uint32_t e) { + TcuUopGen::LdmUop u{0, 0, 0}; + switch (shape) { + case kLdmA: // e = {m, k}: row group m, word column k + u.grp = e >> kLgK; + u.col = (e & ((1u << kLgK) - 1)) << kLdmLgCsK; + u.reg_off = ra_off(e >> kLgK, e); + break; + case kLdmASp: // e = {m, k} over half the K steps; registers in order + u.grp = e >> kLdmLgKSp; + u.col = (e & ((1u << kLdmLgKSp) - 1)) << kLdmLgCsK; + u.reg_off = e; + break; + case kLdmB: // e = {k, n}: a col-major B row is one of its columns (a pair with two sub-blocks) + u.grp = e & ((1u << kLdmLgBRows) - 1); + u.col = (e >> kLdmLgBRows) << kLdmLgCsK; + u.reg_off = rb_off(u.grp << kLgBSb, e >> kLdmLgBRows); + break; + case kLdmBSp: // symmetric sparse: one row per register + u.grp = e; + u.reg_off = e; + break; + case kLdmC: // e = {m, n} + u.grp = e >> kLgN; + u.col = (e & ((1u << kLgN) - 1)) << kLdmLgCsN; + u.reg_off = rc_off(e >> kLgN, e); + break; + default: // WGMMA C, e = {n, m}: n-major registers, two row groups + u.grp = e & 1; + u.col = (e >> 1) << kLdmLgCsN; + u.reg_off = e; + break; + } + return u; +} + +// Uop e moves register fd/fs2 + e, holding the element whose register offset is e +TcuUopGen::LdmUop TcuUopGen::ldm_uop(uint32_t shape, uint32_t e) { + uint32_t count = ldm_count(shape); + for (uint32_t r = 0; r < count; ++r) { + auto u = ldm_elem(shape, r); + if (u.reg_off == e) { + return u; + } + } + std::abort(); +} + uint32_t TcuUopGen::uop_count(const Instr& instr) { if (instr.get_fu_type() != FUType::TCU) return 1; @@ -1727,51 +1879,22 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { uop_instr->set_tmask(ThreadMask(VX_CFG_NUM_THREADS, (mma_idx & 1) ? (all_lanes & ~sym_mask_lo) : sym_mask_lo)); } else { // Standard k-major triple loop (dense or non-sym sparse) - uint32_t b_sub = is_sparse ? wmma::b_sub_blocks_sp : wmma::b_sub_blocks; uint32_t mn = wmma::m_steps * wmma::n_steps; uint32_t k = mma_idx / mn; uint32_t rem = mma_idx % mn; uint32_t m = rem / wmma::n_steps; uint32_t n = rem % wmma::n_steps; - // Bank-conflict-free register offset formulas (0 stalls). - // Permutes A, B, C offsets so all three operands always land in - // different GPR banks for every uop. uint32_t reg_rs1, reg_rs2, reg_rs3; if (is_sparse) { - // Sparse non-sym: A=identity, B and C permuted for 0 stalls. - // B={n[hi], ~(n[0]^k), ~k}, C={n[hi], m, ~(m^n[0])} - uint32_t n_hi = n >> 1; - uint32_t n_lo = n & 1; + // Sparse non-sym: A=identity, B and C permuted. uint32_t a_off = (m / wmma::a_sub_blocks) * k_count + k; - uint32_t b_off = (n_hi << 2) | ((1 - (n_lo ^ k)) << 1) | (1 - k); - uint32_t c_off = (n_hi << 2) | (m << 1) | (1 - (m ^ n_lo)); reg_rs1 = ra_base + a_off; - reg_rs2 = rb_base + b_off; - reg_rs3 = rc_base + c_off; - } else if (b_sub == 1) { - // Dense Pattern A (NT=4,16,64): m_steps=4, k_steps=2, n_steps=2 - // A={m[0], ~m[hi], k}, B={n^k, ~k}, C={m[0], ~m[hi], XNOR(m[hi],n)} - uint32_t m_hi = m >> 1; - uint32_t m_lo = m & 1; - uint32_t a_off = (m_lo << 2) | ((1 - m_hi) << 1) | k; - uint32_t b_off = ((n ^ k) << 1) | (1 - k); - uint32_t c_off = (m_lo << 2) | ((1 - m_hi) << 1) | (1 - (m_hi ^ n)); - reg_rs1 = ra_base + a_off; - reg_rs2 = rb_base + b_off; - reg_rs3 = rc_base + c_off; + reg_rs2 = rb_base + rb_off_sp(n, k); + reg_rs3 = rc_base + rc_off_sp(m, n); } else { - // Dense Pattern B (NT=8,32): m_steps=2, k_steps=4, n_steps=4, b_sub=2 - // A={k[0], ~m, m^k[hi]}, B={k[0], k[hi]^np, ~np}, C={n[0], ~m, n[hi]} - uint32_t k_hi = k >> 1; - uint32_t k_lo = k & 1; - uint32_t n_pair = n >> 1; - uint32_t n_lo = n & 1; - uint32_t a_off = (k_lo << 2) | ((1 - m) << 1) | (m ^ k_hi); - uint32_t b_off = (k_lo << 2) | ((k_hi ^ n_pair) << 1) | (1 - n_pair); - uint32_t c_off = (n_lo << 2) | ((1 - m) << 1) | n_pair; - reg_rs1 = ra_base + a_off; - reg_rs2 = rb_base + b_off; - reg_rs3 = rc_base + c_off; + reg_rs1 = ra_base + ra_off(m, k); + reg_rs2 = rb_base + rb_off(n, k); + reg_rs3 = rc_base + rc_off(m, n); } uop_instr->set_op_type(is_sparse ? TcuType::WMMA_SP : TcuType::WMMA); uop_instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, m, n, k, 0, 0, 0}); diff --git a/sim/simx/tcu/tcu_unit.h b/sim/simx/tcu/tcu_unit.h index 6a4e0e7215..d36b5a181d 100644 --- a/sim/simx/tcu/tcu_unit.h +++ b/sim/simx/tcu/tcu_unit.h @@ -53,6 +53,19 @@ class TcuUopGen { // Generate micro-op Instr at uop_index for the given macro instruction. Instr::Ptr get(const Instr& macro_instr, uint32_t uop_index); + // Matrix fragment load/store (vx_ldm/vx_stm) shapes, {ctx, role[1:0], trans, + // sparse}. Uop e moves register fd/fs2 + e (reg_off = e) to or from + // x[rs1 | grp] + (offset | col), the address of the fragment element the + // WMMA/WGMMA expansion reads from that register. + struct LdmUop { + uint32_t reg_off; + uint32_t grp; + uint32_t col; + }; + static bool ldm_valid(uint32_t shape); + static uint32_t ldm_count(uint32_t shape); + static LdmUop ldm_uop(uint32_t shape, uint32_t e); + private: PoolAllocator& pool_; }; diff --git a/sim/simx/types.h b/sim/simx/types.h index 60fd7b531a..b307fa5963 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -384,6 +384,7 @@ struct IntrLsuArgs { uint32_t width : 3; uint32_t stride : 4; // per-uop multiplier: 0 = regular LD/ST; 0..N-1 = packLD uop_idx int32_t offset; // immediate addr offset (signed); 0 for packLD uops + bool matrix = false; // vx_ldm/vx_stm: offset[4:0] is the TCU fragment shape }; // A fence carries no access width; its width field says what it orders. diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index e3f4ed39d1..21daf265f0 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -145,17 +145,143 @@ namespace detail { return *reinterpret_cast(&result_u); } }; + + // vx_ldm fd, imm(rs1) / vx_stm fs2, imm(rs1): imm[4:0] is the fragment + // shape, imm[11:5] a byte offset; rs1 is the first of J registers, a multiple + // of J, holding the lane's address in each of the fragment's row groups. + enum ldm_shape_t : int32_t { + LDM_NONE = -1, + LDM_A = 0x00, // WMMA A, row-major + LDM_A_SP = 0x01, // WMMA sparse A, row-major + LDM_B = 0x06, // WMMA B, col-major + LDM_B_SP = 0x07, // WMMA sparse B, col-major + LDM_C = 0x08, // WMMA C, row-major + LDM_WG_C = 0x18 // WGMMA C, row-major, one 8-register chunk + }; + +#if __riscv_xlen == 64 + using gpr_pair_t = unsigned __int128; +#else + using gpr_pair_t = uint64_t; +#endif + +#define VX_TENSOR_FREGS_0 "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7" +#define VX_TENSOR_FREGS_8 "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15" +#define VX_TENSOR_FREGS_10 "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17" +#define VX_TENSOR_FREGS_16 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23" +#define VX_TENSOR_FREGS_24 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31" +#define VX_TENSOR_FREGS4_10 "f10", "f11", "f12", "f13" +#define VX_TENSOR_FREGS4_28 "f28", "f29", "f30", "f31" + +// The register lists expand before the inner macros split them into names. +#define VX_TENSOR_LDM8(ival, regs, ...) VX_TENSOR_LDM8_(ival, regs, __VA_ARGS__) +#define VX_TENSOR_LDM8_(ival, r0, r1, r2, r3, r4, r5, r6, r7, ...) { \ + register float x0 __asm__(r0); register float x1 __asm__(r1); \ + register float x2 __asm__(r2); register float x3 __asm__(r3); \ + register float x4 __asm__(r4); register float x5 __asm__(r5); \ + register float x6 __asm__(r6); register float x7 __asm__(r7); \ + __asm__ volatile (".insn i %[op], 0, %[x0], %[g], %[imm]" \ + : [x0]"=f"(x0), "=f"(x1), "=f"(x2), "=f"(x3), "=f"(x4), "=f"(x5), "=f"(x6), "=f"(x7)\ + : [op]"i"(RISCV_CUSTOM3), [imm]"i"(ival), __VA_ARGS__ : "memory"); \ + v[0] = x0; v[1] = x1; v[2] = x2; v[3] = x3; v[4] = x4; v[5] = x5; v[6] = x6; v[7] = x7; } +#define VX_TENSOR_LDM4(ival, regs, ...) VX_TENSOR_LDM4_(ival, regs, __VA_ARGS__) +#define VX_TENSOR_LDM4_(ival, r0, r1, r2, r3, ...) { \ + register float x0 __asm__(r0); register float x1 __asm__(r1); \ + register float x2 __asm__(r2); register float x3 __asm__(r3); \ + __asm__ volatile (".insn i %[op], 0, %[x0], %[g], %[imm]" \ + : [x0]"=f"(x0), "=f"(x1), "=f"(x2), "=f"(x3) \ + : [op]"i"(RISCV_CUSTOM3), [imm]"i"(ival), __VA_ARGS__ : "memory"); \ + v[0] = x0; v[1] = x1; v[2] = x2; v[3] = x3; } +#define VX_TENSOR_STM8(ival, regs, ...) VX_TENSOR_STM8_(ival, regs, __VA_ARGS__) +#define VX_TENSOR_STM8_(ival, r0, r1, r2, r3, r4, r5, r6, r7, ...) { \ + register float x0 __asm__(r0) = v[0]; register float x1 __asm__(r1) = v[1]; \ + register float x2 __asm__(r2) = v[2]; register float x3 __asm__(r3) = v[3]; \ + register float x4 __asm__(r4) = v[4]; register float x5 __asm__(r5) = v[5]; \ + register float x6 __asm__(r6) = v[6]; register float x7 __asm__(r7) = v[7]; \ + __asm__ volatile (".insn s %[op], 2, %[x0], %[imm](%[g])" \ + :: [op]"i"(RISCV_CUSTOM3), [imm]"i"(ival), [x0]"f"(x0), "f"(x1), "f"(x2), "f"(x3), \ + "f"(x4), "f"(x5), "f"(x6), "f"(x7), __VA_ARGS__ : "memory"); } + + // A J=2 group rides in a register pair; a J=4 group is pinned to x28..x31 + // or x12..x15. +#define VX_TENSOR_LDM_GROUP(EMIT) \ + if constexpr (J == 2) { \ + gpr_pair_t p = gpr_pair_t(g[0]) | (gpr_pair_t(g[1]) << (8 * sizeof(uintptr_t))); \ + EMIT([g]"R"(p)); \ + } else if constexpr (Q == 28) { \ + register uintptr_t q0 __asm__("x28") = g[0]; register uintptr_t q1 __asm__("x29") = g[1]; \ + register uintptr_t q2 __asm__("x30") = g[2]; register uintptr_t q3 __asm__("x31") = g[3]; \ + EMIT([g]"r"(q0), "r"(q1), "r"(q2), "r"(q3)); \ + } else { \ + register uintptr_t q0 __asm__("x12") = g[0]; register uintptr_t q1 __asm__("x13") = g[1]; \ + register uintptr_t q2 __asm__("x14") = g[2]; register uintptr_t q3 __asm__("x15") = g[3]; \ + EMIT([g]"r"(q0), "r"(q1), "r"(q2), "r"(q3)); \ + } + + // vx_ldm of NR registers from fd = FD; Q picks a J=4 group's registers. + template + __attribute__((always_inline)) inline void vx_ldm(float (&v)[NR], const uintptr_t (&g)[J]) { + static_assert(J == 2 || J == 4, "a fragment has 2 or 4 row groups"); + #define VX_TENSOR_LDM_EMIT(...) \ + if constexpr (FD == 10 && NR == 8) VX_TENSOR_LDM8(IMM, VX_TENSOR_FREGS_10, __VA_ARGS__) \ + else if constexpr (FD == 10) VX_TENSOR_LDM4(IMM, VX_TENSOR_FREGS4_10, __VA_ARGS__) \ + else if constexpr (FD == 24) VX_TENSOR_LDM8(IMM, VX_TENSOR_FREGS_24, __VA_ARGS__) \ + else if constexpr (FD == 28) VX_TENSOR_LDM4(IMM, VX_TENSOR_FREGS4_28, __VA_ARGS__) \ + else { static_assert(FD == 0 && NR == 8, "unsupported fragment registers"); \ + VX_TENSOR_LDM8(IMM, VX_TENSOR_FREGS_0, __VA_ARGS__) } + VX_TENSOR_LDM_GROUP(VX_TENSOR_LDM_EMIT) + #undef VX_TENSOR_LDM_EMIT + } + + // vx_stm of 8 registers from fs2 = FD + template + __attribute__((always_inline)) inline void vx_stm(const float (&v)[8], const uintptr_t (&g)[J]) { + static_assert(J == 2 || J == 4, "a fragment has 2 or 4 row groups"); + #define VX_TENSOR_STM_EMIT(...) \ + if constexpr (FD == 0) VX_TENSOR_STM8(IMM, VX_TENSOR_FREGS_0, __VA_ARGS__) \ + else if constexpr (FD == 8) VX_TENSOR_STM8(IMM, VX_TENSOR_FREGS_8, __VA_ARGS__) \ + else if constexpr (FD == 16) VX_TENSOR_STM8(IMM, VX_TENSOR_FREGS_16, __VA_ARGS__) \ + else { static_assert(FD == 24, "unsupported fragment registers"); \ + VX_TENSOR_STM8(IMM, VX_TENSOR_FREGS_24, __VA_ARGS__) } + VX_TENSOR_LDM_GROUP(VX_TENSOR_STM_EMIT) + #undef VX_TENSOR_STM_EMIT + } + +#undef VX_TENSOR_FREGS_0 +#undef VX_TENSOR_FREGS_8 +#undef VX_TENSOR_FREGS_10 +#undef VX_TENSOR_FREGS_16 +#undef VX_TENSOR_FREGS_24 +#undef VX_TENSOR_FREGS4_10 +#undef VX_TENSOR_FREGS4_28 +#undef VX_TENSOR_LDM8 +#undef VX_TENSOR_LDM8_ +#undef VX_TENSOR_LDM4 +#undef VX_TENSOR_LDM4_ +#undef VX_TENSOR_STM8 +#undef VX_TENSOR_STM8_ +#undef VX_TENSOR_LDM_GROUP + + // The immediate of a fragment access at byte offset off from the group: off + // folds in when it is a multiple of 32 and of the column span (the OR with + // the uops' column offsets is then an add). + template + inline constexpr bool ldm_folds = (off % 32 == 0) && (off % int32_t(span) == 0) + && (off >= -2048) && (off <= 2016); } // Hoist detail::frag_use_t into the enclosing tensor namespace using detail::frag_use_t; +// Compile-time loop, e.g. over the K tiles of an unrolled K loop +using detail::unroll_for; template struct wmma_context { private: @@ -295,12 +421,22 @@ struct wmma_context { static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm) { uint32_t row, col; lane_coords(row, col); - load_fragment(dst, reinterpret_cast*>(src) + row * ldm + col, ldm); + if constexpr (ldst_matrix && ldm_shape() != detail::LDM_NONE) { + ldm_load(dst, src, row * ldm + col, ldm); + } else { + load_fragment(dst, reinterpret_cast*>(src) + row * ldm + col, ldm); + } } - template + // k_off: a constant element offset from src, e.g. the next K tile of an + // unrolled loop, which a vx_ldm takes in its immediate. + template static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm, size_t lane_off) { - load_fragment(dst, reinterpret_cast*>(src) + lane_off, ldm); + if constexpr (ldst_matrix && ldm_shape() != detail::LDM_NONE) { + ldm_load(dst, src, lane_off, ldm); + } else { + load_fragment(dst, reinterpret_cast*>(src) + lane_off + k_off, ldm); + } } private: @@ -334,6 +470,182 @@ struct wmma_context { } } + // The TCU reads its operands in a bank-conflict-free register order, so a + // fragment must sit in registers in that order. These map a register offset + // back to the logical fragment index it holds. + static constexpr uint32_t b_sub_eff = is_sparse ? cfg::b_sub_blocks_sp : cfg::b_sub_blocks; + static constexpr bool bcfree_sp = is_sparse && !cfg::sym_sparse; // sparse non-sym + static constexpr bool bcfree_a = !is_sparse && (b_sub_eff == 1); // dense pattern A + static constexpr bool bcfree_b = !is_sparse && (b_sub_eff > 1); // dense pattern B + + // A inverse: physical offset → logical fragment index + static constexpr uint32_t ra_idx(uint32_t off) { + if constexpr (bcfree_sp) { + // Sparse non-sym: A is identity (off = m*k_count+k) + return off; + } else if constexpr (bcfree_a) { + // Dense Pattern A (b_sub==1): forward A={m[0],~m[hi],k} + uint32_t m = ((1 - ((off >> 1) & 1)) << 1) | (off >> 2); + uint32_t k = off & 1; + return m * cfg::k_steps + k; + } else if constexpr (bcfree_b) { + // Dense Pattern B (b_sub==2): forward A={k[0],~m,m^k[hi]} + uint32_t m = 1 - ((off >> 1) & 1); + uint32_t k_hi = m ^ (off & 1); + uint32_t k_lo = off >> 2; + return m * cfg::k_steps + (k_hi << 1 | k_lo); + } else { + return off; + } + } + + // B inverse: physical offset → logical fragment index + static constexpr uint32_t rb_idx(uint32_t off) { + if constexpr (bcfree_sp) { + // Sparse non-sym: forward B={n[hi], ~(n[0]^k), ~k} + // Inverse: k = 1-off[0], n[0] = (1-off[1])^k, n[hi] = off[2] + uint32_t k = 1 - (off & 1); + uint32_t n_lo = (1 - ((off >> 1) & 1)) ^ k; + uint32_t n_hi = off >> 2; + return k * cfg::n_steps + (n_hi << 1 | n_lo); + } else if constexpr (bcfree_b) { + // Dense Pattern B (b_sub==2): forward B={k[0],k[hi]^np,~np} + uint32_t n_pair = 1 - (off & 1); + uint32_t k_hi = ((off >> 1) & 1) ^ n_pair; + uint32_t k_lo = off >> 2; + return (k_hi << 1 | k_lo) * cfg::b_sub_blocks + n_pair; + } else if constexpr (bcfree_a) { + // Dense Pattern A (b_sub==1): forward B={n^k,~k} + uint32_t k = 1 - (off & 1); + uint32_t n = ((off >> 1) & 1) ^ k; + return k * cfg::n_steps + n; + } else { + return off; + } + } + + // C inverse: physical offset → logical fragment index + static constexpr uint32_t rc_idx(uint32_t off) { + if constexpr (bcfree_sp) { + // Sparse non-sym: forward C={n[hi], m, ~(m^n[0])} + // Inverse: m = off[1], n[0] = (1-off[0])^m, n[hi] = off[2] + uint32_t m = (off >> 1) & 1; + uint32_t n_lo = (1 - (off & 1)) ^ m; + uint32_t n_hi = off >> 2; + return m * cfg::n_steps + (n_hi << 1 | n_lo); + } else if constexpr (bcfree_a) { + // Dense Pattern A (b_sub==1): forward C={m[0],~m[hi],XNOR(m[hi],n)} + uint32_t m_hi = 1 - ((off >> 1) & 1); + uint32_t m = (m_hi << 1) | (off >> 2); + uint32_t n = 1 - (m_hi ^ (off & 1)); + return m * cfg::n_steps + n; + } else if constexpr (bcfree_b) { + // Dense Pattern B (b_sub==2): forward C={n[0],~m,n[hi]} + uint32_t m = 1 - ((off >> 1) & 1); + uint32_t n = ((off & 1) << 1) | (off >> 2); + return m * cfg::n_steps + n; + } else { + return off; + } + } + + // vx_ldm/vx_stm geometry of a fragment: NR registers from FD in J row groups + // row_step * ldm elements apart, span bytes of columns per row group, and Q + // the registers of a J=4 group. + template + struct ldm_geom { + static constexpr bool is_a = (Frag::Use == frag_use_t::matrix_a); + static constexpr bool is_b = (Frag::Use == frag_use_t::matrix_b); + static constexpr uint32_t NR = (is_a && is_sparse) ? sparse_regs : Frag::NR; + static constexpr uint32_t J = is_a ? cfg::m_steps + : is_b ? (is_sparse ? Frag::NR : cfg::b_sub_steps) + : cfg::m_steps; + static constexpr uint32_t row_step = is_a ? (cfg::a_sub_blocks * cfg::tcM) + : is_b ? (is_sparse ? (cfg::tcN / 2) : (cfg::b_sub_blocks * cfg::tcN)) + : cfg::tcM; + static constexpr uint32_t col_bytes = (is_b && is_sparse) ? 4 + : (is_a || is_b) ? (cfg::tcK * 4) : (cfg::tcN * 4); + static constexpr uint32_t span = (NR / J) * col_bytes; + static constexpr uint32_t FD = is_a ? 10 : is_b ? ((Frag::NR == 8) ? 24 : 28) : 0; + static constexpr uint32_t Q = is_b ? 12 : 28; + }; + + // The fragment's vx_ldm/vx_stm shape in a memory layout; LDM_NONE when a + // fragment register is not one word of the tile, the order is not covered, + // the context is not the 8-register C/D tile the shapes are defined on, or + // the macro-op does not pay: a sparse MX K tile is bound by its three + // serialized metadata loads, which the fragment macro-ops cannot shorten and + // their base-group setup delays; an asymmetric sparse B, one burst of eight + // uops, stalls the MMA on its operands longer than the scalar loads do (it + // has no shape). + template + static constexpr int32_t ldm_shape() { + if constexpr (NR_ != 8 || DK_ != 0 || (is_sparse && input_is_mx)) { + return detail::LDM_NONE; + } else if constexpr (Frag::Use == frag_use_t::matrix_a) { + return (layout == row_major) ? (is_sparse ? detail::LDM_A_SP : detail::LDM_A) : detail::LDM_NONE; + } else if constexpr (Frag::Use == frag_use_t::matrix_b) { + if constexpr (is_sparse && !cfg::sym_sparse) { + return detail::LDM_NONE; + } + return (layout == col_major) ? (is_sparse ? detail::LDM_B_SP : detail::LDM_B) : detail::LDM_NONE; + } else { + return (layout == row_major && !is_sparse) ? detail::LDM_C : detail::LDM_NONE; + } + } + + template + static constexpr uint32_t ldm_idx(uint32_t off) { + if constexpr (Frag::Use == frag_use_t::matrix_a) { + return ra_idx(off); + } else if constexpr (Frag::Use == frag_use_t::matrix_b) { + return rb_idx(off); + } else { + return rc_idx(off); + } + } + + // Lane addresses of the fragment's row groups; off_bytes is added unless it + // folds into the instruction's immediate. Each is the lane's tile address + // plus a loop-invariant row offset. + template + static __attribute__((always_inline)) void ldm_group(uintptr_t (&g)[ldm_geom::J], + const void *src, size_t lane_off, size_t ldm) { + using geom = ldm_geom; + constexpr bool fold = detail::ldm_folds; + size_t lane_bytes = lane_off * sizeof(frag_elem_t) + (fold ? 0 : off_bytes); + size_t row_bytes = geom::row_step * ldm * sizeof(frag_elem_t); + uintptr_t base = reinterpret_cast(src) + lane_bytes; + if constexpr (geom::J == 4) { + // opaque, so a K loop steps one induction variable and derives the other + // three registers by adds instead of stepping one per register; volatile, + // so those adds stay behind the tile's metadata loads instead of delaying them + __asm__ volatile("" : "+r"(base)); + } + detail::unroll_for([&](auto j) { + g[j] = base + j * row_bytes; + }); + } + + template + static constexpr int32_t ldm_imm() { + return (detail::ldm_folds::span> ? off_bytes : 0) | shape; + } + + template + static __attribute__((always_inline)) void ldm_load(Frag &dst, const void *src, size_t lane_off, size_t ldm) { + using geom = ldm_geom; + constexpr int32_t off_bytes = k_off * int32_t(sizeof(frag_elem_t)); + uintptr_t g[geom::J]; + ldm_group(g, src, lane_off, ldm); + float v[geom::NR]; + detail::vx_ldm()>()>(v, g); + detail::unroll_for([&](auto r) { + dst.data[ldm_idx(r)] = v[r]; + }); + } + // base already points at the calling lane's first element template static __attribute__((always_inline)) void load_fragment(Frag &dst, const frag_elem_t *base, size_t ldm) { @@ -509,6 +821,17 @@ struct wmma_context { uint32_t m_stride = cfg::tcM; uint32_t n_stride = cfg::tcN; auto base = reinterpret_cast(dst) + block_row * ldm + block_col; + if constexpr (ldst_matrix && ldm_shape() != detail::LDM_NONE) { + using geom = ldm_geom; + uintptr_t g[geom::J]; + ldm_group(g, dst, block_row * ldm + block_col, ldm); + float v[8]; + detail::unroll_for<8>([&](auto r) { + v[r] = src.data[rc_idx(r)]; + }); + detail::vx_stm(v, g); + return; + } detail::unroll_for([&](auto r) { uint32_t block_m = r / cfg::n_steps; uint32_t block_n = r % cfg::n_steps; @@ -535,87 +858,6 @@ struct wmma_context { static_assert(FragC::Use == frag_use_t::accumulator, "C must be accumulator"); static_assert(FragD::Use == frag_use_t::accumulator, "D must be accumulator"); - // Bank-conflict-free register offset permutations (0 stalls). - // SW must place fragment data into registers matching the HW's - // permuted offset order. These are the INVERSE of the forward - // formulas in VX_tcu_uops.sv, mapping physical register offset - // back to the logical fragment index. - constexpr uint32_t b_sub_eff = is_sparse ? cfg::b_sub_blocks_sp : cfg::b_sub_blocks; - constexpr bool bcfree_sp = is_sparse && !cfg::sym_sparse; // sparse non-sym - constexpr bool bcfree_a = !is_sparse && (b_sub_eff == 1); // dense pattern A - constexpr bool bcfree_b = !is_sparse && (b_sub_eff > 1); // dense pattern B - - // A inverse: physical offset → logical fragment index - constexpr auto ra_idx = [](uint32_t off) constexpr -> uint32_t { - if constexpr (bcfree_sp) { - // Sparse non-sym: A is identity (off = m*k_count+k) - return off; - } else if constexpr (bcfree_a) { - // Dense Pattern A (b_sub==1): forward A={m[0],~m[hi],k} - uint32_t m = ((1 - ((off >> 1) & 1)) << 1) | (off >> 2); - uint32_t k = off & 1; - return m * cfg::k_steps + k; - } else if constexpr (bcfree_b) { - // Dense Pattern B (b_sub==2): forward A={k[0],~m,m^k[hi]} - uint32_t m = 1 - ((off >> 1) & 1); - uint32_t k_hi = m ^ (off & 1); - uint32_t k_lo = off >> 2; - return m * cfg::k_steps + (k_hi << 1 | k_lo); - } else { - return off; - } - }; - - // B inverse: physical offset → logical fragment index - constexpr auto rb_idx = [](uint32_t off) constexpr -> uint32_t { - if constexpr (bcfree_sp) { - // Sparse non-sym: forward B={n[hi], ~(n[0]^k), ~k} - // Inverse: k = 1-off[0], n[0] = (1-off[1])^k, n[hi] = off[2] - uint32_t k = 1 - (off & 1); - uint32_t n_lo = (1 - ((off >> 1) & 1)) ^ k; - uint32_t n_hi = off >> 2; - return k * cfg::n_steps + (n_hi << 1 | n_lo); - } else if constexpr (bcfree_b) { - // Dense Pattern B (b_sub==2): forward B={k[0],k[hi]^np,~np} - uint32_t n_pair = 1 - (off & 1); - uint32_t k_hi = ((off >> 1) & 1) ^ n_pair; - uint32_t k_lo = off >> 2; - return (k_hi << 1 | k_lo) * cfg::b_sub_blocks + n_pair; - } else if constexpr (bcfree_a) { - // Dense Pattern A (b_sub==1): forward B={n^k,~k} - uint32_t k = 1 - (off & 1); - uint32_t n = ((off >> 1) & 1) ^ k; - return k * cfg::n_steps + n; - } else { - return off; - } - }; - - // C inverse: physical offset → logical fragment index - constexpr auto rc_idx = [](uint32_t off) constexpr -> uint32_t { - if constexpr (bcfree_sp) { - // Sparse non-sym: forward C={n[hi], m, ~(m^n[0])} - // Inverse: m = off[1], n[0] = (1-off[0])^m, n[hi] = off[2] - uint32_t m = (off >> 1) & 1; - uint32_t n_lo = (1 - (off & 1)) ^ m; - uint32_t n_hi = off >> 2; - return m * cfg::n_steps + (n_hi << 1 | n_lo); - } else if constexpr (bcfree_a) { - // Dense Pattern A (b_sub==1): forward C={m[0],~m[hi],XNOR(m[hi],n)} - uint32_t m_hi = 1 - ((off >> 1) & 1); - uint32_t m = (m_hi << 1) | (off >> 2); - uint32_t n = 1 - (m_hi ^ (off & 1)); - return m * cfg::n_steps + n; - } else if constexpr (bcfree_b) { - // Dense Pattern B (b_sub==2): forward C={n[0],~m,n[hi]} - uint32_t m = 1 - ((off >> 1) & 1); - uint32_t n = ((off & 1) << 1) | (off >> 2); - return m * cfg::n_steps + n; - } else { - return off; - } - }; - // frag_c initialized into accumulator registers (f0-f7) register float fd0 __asm__("f0") = frag_c.data[rc_idx(0)]; register float fd1 __asm__("f1") = frag_c.data[rc_idx(1)]; @@ -697,7 +939,8 @@ template + uint32_t NRC_ = 8, + bool ldst_matrix = false> // store the accumulator with vx_stm struct wgmma_context { static_assert(tcu_output_valid(It::id, Ot::id), "TCU output type must be fp32 (int32 for integer inputs)"); @@ -705,8 +948,8 @@ struct wgmma_context { private: static constexpr uint32_t NRA = 4; - using ctx_c = wmma_context; - using ctx_a = wmma_context; + using ctx_c = wmma_context; + using ctx_a = wmma_context; // Direct geometry from NT (no wmma_config_t dependency) static constexpr uint32_t clog2(uint32_t x) { @@ -986,6 +1229,23 @@ struct wgmma_context { uint32_t base_row = lane / tcN; uint32_t base_col = lane % tcN; auto base = reinterpret_cast(dst) + base_row * ldm + base_col; + if constexpr (ldst_matrix && dst_layout == row_major) { + // one vx_stm per 8-register chunk; chunk q holds columns 4q..4q+3 + constexpr uint32_t chunk_bytes = 4 * tcN * sizeof(output_t); + static_assert((chunk_bytes % 32) == 0, "a chunk's column offset must fold into the immediate"); + uintptr_t g[2]; + g[0] = reinterpret_cast(base); + g[1] = g[0] + tcM * ldm * sizeof(output_t); + detail::unroll_for([&](auto q) { + constexpr uint32_t chunk = decltype(q)::value; + float v[8]; + detail::unroll_for<8>([&](auto r) { + v[r] = src.data[chunk * 8 + r]; + }); + detail::vx_stm(v, g); + }); + return; + } detail::unroll_for([&](auto r) { uint32_t block_m = r % m_steps; // n-major: m is inner uint32_t block_n = r / m_steps; // n-major: n is outer diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 56f2f10dc4..6d3221dea4 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax uniform_ld lmem_access ldst_wide mufu zfhmin \ + softmax uniform_ld lmem_access ldst_wide ldst_matrix mufu zfhmin \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/ldst_matrix/Makefile b/tests/regression/ldst_matrix/Makefile new file mode 100644 index 0000000000..03dd696dd3 --- /dev/null +++ b/tests/regression/ldst_matrix/Makefile @@ -0,0 +1,18 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_TCU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_TCU_ENABLE) + +PROJECT := ldst_matrix + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/ldst_matrix/common.h b/tests/regression/ldst_matrix/common.h new file mode 100644 index 0000000000..c9de0c0003 --- /dev/null +++ b/tests/regression/ldst_matrix/common.h @@ -0,0 +1,26 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Each thread owns a source and a destination region in global memory and in +// local memory; a fragment's row groups sit inside them. +#define GLOBAL_REGION_WORDS 512 +#define LOCAL_REGION_WORDS 128 + +// Source word i of a thread's region; local and global words differ. +static inline uint32_t src_word(uint32_t tid, uint32_t i, uint32_t salt) { + return ((tid << 12) | i | 0x40000000u) ^ salt; +} + +#define GLOBAL_SALT 0x00800000u +#define LOCAL_SALT 0x01000000u + +typedef struct { + uint64_t src_addr; // global source regions, filled by the host + uint64_t dst_addr; // global destination regions, zeroed by the host + uint64_t err_addr; // out: per-thread error count + uint64_t shape_addr; // out: bit s set when shape s ran +} kernel_arg_t; + +#endif diff --git a/tests/regression/ldst_matrix/kernel.cpp b/tests/regression/ldst_matrix/kernel.cpp new file mode 100644 index 0000000000..12aa3c3b2c --- /dev/null +++ b/tests/regression/ldst_matrix/kernel.cpp @@ -0,0 +1,258 @@ +#include +#include +#include "common.h" + +namespace vt = vortex::tensor; +using cfg = vt::wmma_config_t; + +// vx_ldm fd, imm(rs1) and vx_stm fs2, imm(rs1) (custom-3 funct3 0/2, I- and +// S-type), written as .insn so the encodings are tested independently of the +// compiler. imm[4:0] is the fragment shape, imm[11:5] a byte offset; rs1 is +// the first of J registers, a multiple of J, holding one address per row group. +enum : uint32_t { + SHAPE_A = 0x00, + SHAPE_A_SP = 0x01, + SHAPE_B = 0x06, + SHAPE_B_SP = 0x07, // symmetric sparse only + SHAPE_C = 0x08, + SHAPE_WG_C = 0x18 +}; + +// A fragment of a shape: J row groups of C words, CS bytes apart in a row. +struct geom_t { + uint32_t J, C, CS; +}; + +constexpr geom_t geom(uint32_t shape) { + switch (shape) { + case SHAPE_A: return {cfg::m_steps, cfg::k_steps, cfg::tcK * 4}; + case SHAPE_A_SP: return {cfg::m_steps, cfg::k_steps / 2, cfg::tcK * 4}; + case SHAPE_B: return {cfg::b_sub_steps, cfg::k_steps, cfg::tcK * 4}; + case SHAPE_B_SP: return {cfg::NRB, 1, 4}; + case SHAPE_C: return {cfg::m_steps, cfg::n_steps, cfg::tcN * 4}; + default: return {2, 4, cfg::tcN * 4}; + } +} + +constexpr uint32_t lg2(uint32_t x) { + return (x < 2) ? 0 : 1 + lg2(x / 2); +} + +constexpr uint32_t bit(uint32_t x, uint32_t i) { + return (x >> i) & 1; +} + +// the top bit of a step index of lg bits +constexpr uint32_t hi(uint32_t x, uint32_t lg) { + return bit(x, (lg > 0) ? (lg - 1) : 0); +} + +// The register the WMMA/WGMMA expansion reads element c of row group j of a +// fragment from: the TCU's bank-conflict-free operand permutations, written +// out independently of the header, SimX and the RTL. +constexpr uint32_t frag_reg(uint32_t shape, uint32_t j, uint32_t c) { + constexpr uint32_t lgM = lg2(cfg::m_steps), lgN = lg2(cfg::n_steps), lgK = lg2(cfg::k_steps); + constexpr bool two_sb = (cfg::b_sub_blocks > 1); + switch (shape) { + case SHAPE_A: // A={m[0],~m[hi],k} or {k[0],~m,m^k[hi]} + return two_sb ? ((bit(c, 0) << 2) | ((1 - bit(j, 0)) << 1) | (bit(j, 0) ^ hi(c, lgK))) + : ((bit(j, 0) << 2) | ((1 - hi(j, lgM)) << 1) | bit(c, 0)); + case SHAPE_B: { // B={n^k,~k} or {k[0],k[hi]^n[hi],~n[hi]}, n the row group's first step + uint32_t n = j * cfg::b_sub_blocks; + return two_sb ? ((bit(c, 0) << 2) | ((hi(c, lgK) ^ hi(n, lgN)) << 1) | (1 - hi(n, lgN))) + : (((bit(n, 0) ^ bit(c, 0)) << 1) | (1 - bit(c, 0))); + } + case SHAPE_C: // C={m[0],~m[hi],~(m[hi]^n)} or {n[0],~m,n[hi]} + return two_sb ? ((bit(c, 0) << 2) | ((1 - bit(j, 0)) << 1) | hi(c, lgN)) + : ((bit(j, 0) << 2) | ((1 - hi(j, lgM)) << 1) | (1 - (hi(j, lgM) ^ bit(c, 0)))); + case SHAPE_WG_C: // n-major + return (c << 1) | j; + default: // sparse A and B: element order + return j * geom(shape).C + c; + } +} + +template +constexpr bool frag_reg_is_permutation() { + constexpr geom_t g = geom(SHAPE); + uint32_t seen = 0; + for (uint32_t j = 0; j < g.J; ++j) { + for (uint32_t c = 0; c < g.C; ++c) { + uint32_t r = frag_reg(SHAPE, j, c); + if (r >= g.J * g.C || (seen & (1u << r))) { + return false; + } + seen |= 1u << r; + } + } + return true; +} + +constexpr uint32_t pow2ceil(uint32_t x) { + uint32_t p = 1; + while (p < x) { + p <<= 1; + } + return p; +} + +// One asm block: vx_ldm from the source group, read every register of the +// fragment, vx_stm the same registers to the destination group. +#define FRAG_MOVE8(imm, r0, r1, r2, r3, r4, r5, r6, r7) \ + asm volatile (".insn i 0x7b, 0, " r0 ", %[gs], %[im]\n\t" \ + "fmv.x.w %0, " r0 "\n\t" "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %2, " r2 "\n\t" "fmv.x.w %3, " r3 "\n\t" \ + "fmv.x.w %4, " r4 "\n\t" "fmv.x.w %5, " r5 "\n\t" \ + "fmv.x.w %6, " r6 "\n\t" "fmv.x.w %7, " r7 "\n\t" \ + ".insn s 0x7b, 2, " r0 ", %[im](%[gd])" \ + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]), \ + "=&r"(w[4]), "=&r"(w[5]), "=&r"(w[6]), "=&r"(w[7]) \ + : [gs]"r"(s0), "r"(s1), "r"(s2), "r"(s3), \ + [gd]"r"(d0), "r"(d1), "r"(d2), "r"(d3), [im]"i"(imm) \ + : r0, r1, r2, r3, r4, r5, r6, r7, "memory") + +#define FRAG_MOVE4(imm, r0, r1, r2, r3) \ + asm volatile (".insn i 0x7b, 0, " r0 ", %[gs], %[im]\n\t" \ + "fmv.x.w %0, " r0 "\n\t" "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %2, " r2 "\n\t" "fmv.x.w %3, " r3 "\n\t" \ + ".insn s 0x7b, 2, " r0 ", %[im](%[gd])" \ + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) \ + : [gs]"r"(s0), "r"(s1), "r"(s2), "r"(s3), \ + [gd]"r"(d0), "r"(d1), "r"(d2), "r"(d3), [im]"i"(imm) \ + : r0, r1, r2, r3, "memory") + +// Moves the fragment of SHAPE at byte offset OFF, with fd = FD, between the +// thread's source and destination regions, whose row groups are row bytes +// apart from byte S on; returns the number of wrong words. +template +uint32_t check(volatile uint32_t* src, volatile uint32_t* dst, uint32_t region_words, uint32_t row) { + constexpr geom_t g = geom(SHAPE); + constexpr uint32_t NR = g.J * g.C; + constexpr uint32_t S = pow2ceil((g.C * g.CS > 32) ? (g.C * g.CS) : 32); + static_assert((g.J == 2 || g.J == 4) && (NR == 4 || NR == 8), "unexpected fragment geometry"); + static_assert(frag_reg_is_permutation(), "fragment register map is not a permutation"); + + uintptr_t gs[4] = {0, 0, 0, 0}; + uintptr_t gd[4] = {0, 0, 0, 0}; + for (uint32_t j = 0; j < g.J; ++j) { + gs[j] = reinterpret_cast(src) + S + j * row; + gd[j] = reinterpret_cast(dst) + S + j * row; + } + register uintptr_t s0 __asm__("x28") = gs[0]; + register uintptr_t s1 __asm__("x29") = gs[1]; + register uintptr_t s2 __asm__("x30") = gs[2]; + register uintptr_t s3 __asm__("x31") = gs[3]; + register uintptr_t d0 __asm__("x12") = gd[0]; + register uintptr_t d1 __asm__("x13") = gd[1]; + register uintptr_t d2 __asm__("x14") = gd[2]; + register uintptr_t d3 __asm__("x15") = gd[3]; + + constexpr int32_t imm = OFF | int32_t(SHAPE); + uint32_t w[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + if constexpr (NR == 8 && FD == 0) { + FRAG_MOVE8(imm, "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7"); + } else if constexpr (NR == 8 && FD == 10) { + FRAG_MOVE8(imm, "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); + } else if constexpr (NR == 8 && FD == 13) { + FRAG_MOVE8(imm, "f13", "f14", "f15", "f16", "f17", "f18", "f19", "f20"); + } else if constexpr (NR == 8 && FD == 24) { + FRAG_MOVE8(imm, "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"); + } else if constexpr (NR == 4 && FD == 10) { + FRAG_MOVE4(imm, "f10", "f11", "f12", "f13"); + } else { + static_assert(NR == 4 && FD == 28, "unsupported fragment registers"); + FRAG_MOVE4(imm, "f28", "f29", "f30", "f31"); + } + + // the footprint, as word indices into the regions + uint32_t foot[8]; + uint32_t errors = 0; + for (uint32_t j = 0, e = 0; j < g.J; ++j) { + for (uint32_t c = 0; c < g.C; ++c, ++e) { + foot[e] = (S + j * row + OFF + c * g.CS) / 4; + // each register holds the element the TCU reads from it + errors += (w[frag_reg(SHAPE, j, c)] != src[foot[e]]); + } + } + // the store wrote the footprint; clearing it lets the final scans of the + // destination regions catch a word written anywhere else + for (uint32_t e = 0; e < NR; ++e) { + errors += (foot[e] >= region_words) || (dst[foot[e]] != src[foot[e]]); + } + for (uint32_t e = 0; e < NR; ++e) { + dst[foot[e]] = 0; + } + return errors; +} + +// Every offset form on one shape and register base: zero, positive, negative. +template +uint32_t check_offsets(volatile uint32_t* src, volatile uint32_t* dst, uint32_t region_words, uint32_t row) { + constexpr geom_t g = geom(SHAPE); + constexpr int32_t S = int32_t(pow2ceil((g.C * g.CS > 32) ? (g.C * g.CS) : 32)); + uint32_t errors = 0; + errors += check(src, dst, region_words, row); + errors += check(src, dst, region_words, row); + errors += check(src, dst, region_words, row); + errors += check(src, dst, region_words, row); + return errors; +} + +constexpr uint32_t kFdB = (cfg::NRB == 8) ? 24 : 28; + +uint32_t check_shapes(volatile uint32_t* src, volatile uint32_t* dst, uint32_t region_words, uint32_t row) { + uint32_t errors = 0; + errors += check_offsets(src, dst, region_words, row); + errors += check_offsets(src, dst, region_words, row); + errors += check_offsets(src, dst, region_words, row); + errors += check_offsets(src, dst, region_words, row); +#ifdef VX_CFG_TCU_SPARSE_ENABLE + errors += check_offsets(src, dst, region_words, row); + if constexpr (cfg::sym_sparse) { + errors += check_offsets(src, dst, region_words, row); + } +#endif +#ifdef VX_CFG_TCU_WGMMA_ENABLE + errors += check_offsets(src, dst, region_words, row); + errors += check_offsets(src, dst, region_words, row); +#endif + return errors; +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = reinterpret_cast(__local_mem()); + const uint32_t lid = threadIdx.x; + const uint32_t tid = blockIdx.x * blockDim.x + lid; + + if (tid == 0) { + uint32_t shapes = (1u << SHAPE_A) | (1u << SHAPE_B) | (1u << SHAPE_C); + #ifdef VX_CFG_TCU_SPARSE_ENABLE + shapes |= (1u << SHAPE_A_SP) | (cfg::sym_sparse ? (1u << SHAPE_B_SP) : 0); + #endif + #ifdef VX_CFG_TCU_WGMMA_ENABLE + shapes |= (1u << SHAPE_WG_C); + #endif + *reinterpret_cast(arg->shape_addr) = shapes; + } + + // local regions: this thread's source, then its destination + auto lsrc = lmem + lid * 2 * LOCAL_REGION_WORDS; + auto ldst = lsrc + LOCAL_REGION_WORDS; + for (uint32_t i = 0; i < LOCAL_REGION_WORDS; ++i) { + lsrc[i] = src_word(tid, i, LOCAL_SALT); + ldst[i] = 0; + } + auto gsrc = reinterpret_cast(arg->src_addr) + tid * GLOBAL_REGION_WORDS; + auto gdst = reinterpret_cast(arg->dst_addr) + tid * GLOBAL_REGION_WORDS; + + // row strides that are not powers of two; the global one exceeds 256 bytes + uint32_t errors = 0; + errors += check_shapes(lsrc, ldst, LOCAL_REGION_WORDS, 68); + errors += check_shapes(gsrc, gdst, GLOBAL_REGION_WORDS, 68); + errors += check_shapes(gsrc, gdst, GLOBAL_REGION_WORDS, 260); + // the host scans the global destination regions + for (uint32_t i = 0; i < LOCAL_REGION_WORDS; ++i) { + errors += (ldst[i] != 0); + } + reinterpret_cast(arg->err_addr)[tid] = errors; +} diff --git a/tests/regression/ldst_matrix/main.cpp b/tests/regression/ldst_matrix/main.cpp new file mode 100644 index 0000000000..444025711c --- /dev/null +++ b/tests/regression/ldst_matrix/main.cpp @@ -0,0 +1,163 @@ +// ldst_matrix — the matrix fragment loads and stores (vx_ldm, vx_stm) move the +// words of each fragment shape the configuration has, from and to local and +// global memory, with row strides that are not powers of two, zero, positive +// and negative offsets, and aligned and unaligned register bases. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "k:h")) != -1) { + switch (c) { + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + uint64_t nt = 0; + uint64_t nc = 0; + uint64_t lmem_size = 0; + uint64_t isa = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + CHECK(vx_device_query(dev, VX_CAPS_LOCAL_MEM_SIZE, &lmem_size)); + CHECK(vx_device_query(dev, VX_CAPS_ISA_FLAGS, &isa)); + if (!(isa & VX_ISA_EXT_TCU)) { + std::cout << "TCU extension not supported!\nFAILED!" << std::endl; + return 1; + } + + // a block is one warp, as far as each thread's two local regions fit + const uint32_t lmem_per_thread = 2 * LOCAL_REGION_WORDS * sizeof(uint32_t); + const uint32_t block = static_cast(std::min(nt, lmem_size / lmem_per_thread)); + const uint32_t grid = static_cast(nc * 2); + const uint32_t num_threads = block * grid; + const uint32_t global_words = num_threads * GLOBAL_REGION_WORDS; + std::cout << "ldst_matrix: block=" << block << " grid=" << grid << std::endl; + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + vx_buffer_h src_buf = nullptr; + vx_buffer_h dst_buf = nullptr; + vx_buffer_h err_buf = nullptr; + vx_buffer_h shape_buf = nullptr; + CHECK(vx_buffer_create(dev, global_words * sizeof(uint32_t), VX_MEM_READ, &src_buf)); + CHECK(vx_buffer_create(dev, global_words * sizeof(uint32_t), VX_MEM_READ_WRITE, &dst_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); + CHECK(vx_buffer_create(dev, sizeof(uint32_t), VX_MEM_WRITE, &shape_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + CHECK(vx_buffer_address(src_buf, &kernel_arg.src_addr)); + CHECK(vx_buffer_address(dst_buf, &kernel_arg.dst_addr)); + CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); + CHECK(vx_buffer_address(shape_buf, &kernel_arg.shape_addr)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid; + li.block_dim[0]= block; + li.lmem_size = block * lmem_per_thread; + + std::vector h_src(global_words); + for (uint32_t t = 0; t < num_threads; ++t) { + for (uint32_t i = 0; i < GLOBAL_REGION_WORDS; ++i) { + h_src[t * GLOBAL_REGION_WORDS + i] = src_word(t, i, GLOBAL_SALT); + } + } + std::vector h_dst(global_words, 0); + std::vector h_err(num_threads); + uint32_t h_shapes = 0; + vx_event_h write_ev[2] = {nullptr, nullptr}; + vx_event_h launch_ev = nullptr; + vx_event_h read_ev = nullptr; + CHECK(vx_enqueue_write(q, src_buf, 0, h_src.data(), h_src.size() * sizeof(uint32_t), 0, nullptr, &write_ev[0])); + CHECK(vx_enqueue_write(q, dst_buf, 0, h_dst.data(), h_dst.size() * sizeof(uint32_t), 0, nullptr, &write_ev[1])); + CHECK(vx_enqueue_launch(q, &li, 2, write_ev, &launch_ev)); + CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, &h_shapes, shape_buf, 0, sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_dst.data(), dst_buf, 0, h_dst.size() * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + std::cout << "shapes=0x" << std::hex << h_shapes << std::dec << std::endl; + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::printf("*** thread %u: %u wrong words\n", t, h_err[t]); + } + errors += h_err[t]; + } + } + // each store's footprint is cleared after its check + for (uint32_t i = 0; i < global_words; ++i) { + if (h_dst[i] != 0) { + if (errors < 16) { + std::printf("*** global destination word %u = 0x%x\n", i, h_dst[i]); + } + ++errors; + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_event_release(write_ev[1]); + vx_event_release(write_ev[0]); + vx_buffer_release(shape_buf); + vx_buffer_release(err_buf); + vx_buffer_release(dst_buf); + vx_buffer_release(src_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/sgemm2_tcu/common.h b/tests/regression/sgemm2_tcu/common.h index fde4cc31de..453bc225b1 100644 --- a/tests/regression/sgemm2_tcu/common.h +++ b/tests/regression/sgemm2_tcu/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint32_t cta_M; diff --git a/tests/regression/sgemm2_tcu/kernel.cpp b/tests/regression/sgemm2_tcu/kernel.cpp index a6c71e1643..5a69f64a09 100644 --- a/tests/regression/sgemm2_tcu/kernel.cpp +++ b/tests/regression/sgemm2_tcu/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -35,6 +35,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t a_lane_off = ctx::lane_offset(ctx::tileK); size_t b_lane_off = ctx::lane_offset(ctx::tileK); +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (size_t k = 0; k < K; k += ctx::tileK) { // Cooperatively load A tile [cta_M x tileK] from DRAM (row-major). uint32_t a_size = cta_M * ctx::tileK; diff --git a/tests/regression/sgemm_tcu/common.h b/tests/regression/sgemm_tcu/common.h index 5e79c82a4a..8c3c3f8941 100644 --- a/tests/regression/sgemm_tcu/common.h +++ b/tests/regression/sgemm_tcu/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu/kernel.cpp b/tests/regression/sgemm_tcu/kernel.cpp index b3f8c8d2d9..b5f68054e0 100644 --- a/tests/regression/sgemm_tcu/kernel.cpp +++ b/tests/regression/sgemm_tcu/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -29,6 +29,36 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t cycles = 0; #endif +#if KUNROLL > 1 + // KUNROLL tiles per iteration: each tile's load takes its K offset as a + // constant, which a vx_ldm folds into its immediate. + size_t a_lane_off = ctx::lane_offset(K); + size_t b_lane_off = ctx::lane_offset(K); + auto k_tiles = [&](size_t i, auto count) { + auto pTileA = pA + tile_row * K + i; + auto pTileB = pB + tile_col * K + i; +#ifdef PROFILE_ENABLE + __rdcycle_time t0 = vx_rdcycle_sync_begin(); +#endif + vt::unroll_for([&](auto u) { + constexpr int32_t k_off = decltype(u)::value * ctx::tileK; + ctx::load_matrix_sync(fragA, pTileA, K, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); + ctx::mma_sync(fragC, fragA, fragB, fragC); + }); +#ifdef PROFILE_ENABLE + __rdcycle_time t1 = vx_rdcycle_sync_end(); + cycles += vx_rdcycle_sync_diff(t0, t1); +#endif + }; + size_t i = 0; + for (; i + KUNROLL * ctx::tileK <= K; i += KUNROLL * ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } + for (; i < K; i += ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } +#else for (size_t i = 0; i < K; i += ctx::tileK) { auto pTileA = pA + tile_row * K + i; auto pTileB = pB + tile_col * K + i; @@ -44,6 +74,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { cycles += vx_rdcycle_sync_diff(t0, t1); #endif } +#endif // Store the computed C tile auto pTileC = pC + tile_row * N + tile_col; diff --git a/tests/regression/sgemm_tcu_mx/common.h b/tests/regression/sgemm_tcu_mx/common.h index f490685e4d..673ec252d5 100644 --- a/tests/regression/sgemm_tcu_mx/common.h +++ b/tests/regression/sgemm_tcu_mx/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_mx/kernel.cpp b/tests/regression/sgemm_tcu_mx/kernel.cpp index e065f357eb..45252cc78f 100644 --- a/tests/regression/sgemm_tcu_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_mx/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -33,6 +33,33 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t a_lane_off = ctx::lane_offset(K); size_t b_lane_off = ctx::lane_offset(K); +#if KUNROLL > 1 + // KUNROLL tiles per iteration: each tile's load takes its K offset as a + // constant, which a vx_ldm folds into its immediate. + auto k_tiles = [&](uint32_t i, auto count) { + uint32_t k_tile = i / ctx::tileK; + auto pTileA = pA + tile_row * K + i; + auto pTileB = pB + tile_col * K + i; + auto pTileMxA = pMxA + (blockIdx.y * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + auto pTileMxB = pMxB + (blockIdx.x * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + vt::unroll_for([&](auto u) { + constexpr uint32_t t = decltype(u)::value; + constexpr int32_t k_off = t * ctx::tileK; + ctx::load_mx_metadata(fragA, pTileMxA + t * VX_CFG_NUM_THREADS); + ctx::load_mx_metadata(fragB, pTileMxB + t * VX_CFG_NUM_THREADS); + ctx::load_matrix_sync(fragA, pTileA, K, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); + ctx::mma_sync(fragC, fragA, fragB, fragC); + }); + }; + uint32_t i = 0; + for (; i + KUNROLL * ctx::tileK <= K; i += KUNROLL * ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } + for (; i < K; i += ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } +#else for (uint32_t i = 0; i < K; i += ctx::tileK) { uint32_t k_tile = i / ctx::tileK; @@ -47,6 +74,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); ctx::mma_sync(fragC, fragA, fragB, fragC); } +#endif #ifdef TCU_MX_TLS if constexpr (std::is_same::value) { diff --git a/tests/regression/sgemm_tcu_sp/common.h b/tests/regression/sgemm_tcu_sp/common.h index 1d4f38e744..9f82306c12 100644 --- a/tests/regression/sgemm_tcu_sp/common.h +++ b/tests/regression/sgemm_tcu_sp/common.h @@ -11,6 +11,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_sp/kernel.cpp b/tests/regression/sgemm_tcu_sp/kernel.cpp index a71f5a298e..38544d128b 100644 --- a/tests/regression/sgemm_tcu_sp/kernel.cpp +++ b/tests/regression/sgemm_tcu_sp/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; // is_sparse=true +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t *__UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -45,6 +45,39 @@ __kernel void kernel_main(kernel_arg_t *__UNIFORM__ arg) { constexpr uint32_t a_k_stride = ctx::tileK / 2; auto pTileB = pB + tile_col * K; +#if KUNROLL > 1 + // KUNROLL tiles per iteration: each tile's load takes its K offset as a + // constant, which a vx_ldm folds into its immediate. + size_t a_lane_off = ctx::lane_offset(stride_A); + size_t b_lane_off = ctx::lane_offset(K); + auto k_tiles = [&](auto count) { + constexpr uint32_t n = decltype(count)::value; +#ifdef PROFILE_ENABLE + __rdcycle_time t0 = vx_rdcycle_sync_begin(); +#endif + vt::unroll_for([&](auto u) { + constexpr uint32_t t = decltype(u)::value; + ctx::load_matrix_sync(fragA, pTileA, stride_A, a_lane_off); + ctx::load_sp_metadata(fragA, pMetaSp + t * per_k_tile_words); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); + ctx::mma_sync(fragC, fragA, fragB, fragC); + }); +#ifdef PROFILE_ENABLE + __rdcycle_time t1 = vx_rdcycle_sync_end(); + cycles += vx_rdcycle_sync_diff(t0, t1); +#endif + pMetaSp += n * per_k_tile_words; + pTileA += n * a_k_stride; + pTileB += n * ctx::tileK; + }; + uint32_t i = 0; + for (; i + KUNROLL * ctx::tileK <= K; i += KUNROLL * ctx::tileK) { + k_tiles(std::integral_constant{}); + } + for (; i < K; i += ctx::tileK) { + k_tiles(std::integral_constant{}); + } +#else for (int i = 0; i < (int)K; i += (int)ctx::tileK) { #ifdef PROFILE_ENABLE __rdcycle_time t0 = vx_rdcycle_sync_begin(); @@ -61,6 +94,7 @@ __kernel void kernel_main(kernel_arg_t *__UNIFORM__ arg) { pTileA += a_k_stride; pTileB += ctx::tileK; } +#endif auto pTileC = pC + tile_row * N + tile_col; ctx::store_matrix_sync(pTileC, fragC, N); diff --git a/tests/regression/sgemm_tcu_sp_mx/common.h b/tests/regression/sgemm_tcu_sp_mx/common.h index 0c9f792d30..7b941e5d38 100644 --- a/tests/regression/sgemm_tcu_sp_mx/common.h +++ b/tests/regression/sgemm_tcu_sp_mx/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_sp_mx/kernel.cpp b/tests/regression/sgemm_tcu_sp_mx/kernel.cpp index 4d776b528f..1ca07382f8 100644 --- a/tests/regression/sgemm_tcu_sp_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_sp_mx/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -34,6 +34,34 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t a_lane_off = ctx::lane_offset(K / 2); size_t b_lane_off = ctx::lane_offset(K); +#if KUNROLL > 1 + // KUNROLL tiles per iteration: each tile's load takes its K offset as a + // constant, which a vx_ldm folds into its immediate. + auto k_tiles = [&](uint32_t i, auto count) { + uint32_t k_tile = i / ctx::tileK; + auto pTileA = pA + tile_row * (K / 2) + i / 2; + auto pTileB = pB + tile_col * K + i; + auto pTileMxA = pMxA + (blockIdx.y * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + auto pTileMxB = pMxB + (blockIdx.x * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + auto pTileMetaSp = pMetaSp + (blockIdx.y * num_k_tiles + k_tile) * ctx::meta_stride; + vt::unroll_for([&](auto u) { + constexpr uint32_t t = decltype(u)::value; + ctx::load_sp_metadata(fragA, pTileMetaSp + t * ctx::meta_stride); + ctx::load_mx_metadata(fragA, pTileMxA + t * VX_CFG_NUM_THREADS); + ctx::load_mx_metadata(fragB, pTileMxB + t * VX_CFG_NUM_THREADS); + ctx::load_matrix_sync(fragA, pTileA, K / 2, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); + ctx::mma_sync(fragC, fragA, fragB, fragC); + }); + }; + uint32_t i = 0; + for (; i + KUNROLL * ctx::tileK <= K; i += KUNROLL * ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } + for (; i < K; i += ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } +#else for (uint32_t i = 0; i < K; i += ctx::tileK) { uint32_t k_tile = i / ctx::tileK; auto pTileA = pA + tile_row * (K / 2) + i / 2; @@ -49,6 +77,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); ctx::mma_sync(fragC, fragA, fragB, fragC); } +#endif #ifdef TCU_MX_TLS if constexpr (std::is_same::value) { diff --git a/tests/regression/sgemm_tcu_wg/common.h b/tests/regression/sgemm_tcu_wg/common.h index 7671ea6464..f0072eab37 100644 --- a/tests/regression/sgemm_tcu_wg/common.h +++ b/tests/regression/sgemm_tcu_wg/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_wg/kernel.cpp b/tests/regression/sgemm_tcu_wg/kernel.cpp index c395aeab13..235fcebe00 100644 --- a/tests/regression/sgemm_tcu_wg/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg/kernel.cpp @@ -5,7 +5,7 @@ namespace vt = vortex::tensor; -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -42,6 +42,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t a_lane_off = ctx::lane_offset(a_ldm); #endif +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t k = 0; k < K; k += ctx::tileK) { // Cooperatively load A [cta_M × tileK] into smem. // Default (block-major): A_smem[a_blockmajor_idx(r, c)]. diff --git a/tests/regression/sgemm_tcu_wg_dxa/common.h b/tests/regression/sgemm_tcu_wg_dxa/common.h index c2eee71301..6b23f832cb 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/common.h +++ b/tests/regression/sgemm_tcu_wg_dxa/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp index 9a62b440a4..01a7d58e68 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp @@ -6,7 +6,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; // DXA descriptor slots (programmed by host in main.cpp). [[maybe_unused]] constexpr uint32_t kDescA = 0; @@ -150,6 +150,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // the buffer may be re-staged. Loop-carried state is {A_cur, B_cur, // bar_cur} mutated by XOR — no arrays, no unrolling, no object reloads. load_tile(A_cur, B_cur, bar_cur, 0); +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t i = 0; i < num_k_tiles; ++i) { if (i + 1 < num_k_tiles) { // Prefetch tile i+1 into the other buffer while tile i is computed. @@ -175,6 +178,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #else // Serial single-buffer schedule: stage and compute alternately in buf0. load_tile(A_cur, B_cur, bar_cur, 0); +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t i = 0; i < num_k_tiles; ++i) { if (i != 0) { load_tile(A_cur, B_cur, bar_cur, i * ctx::tileK); diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/common.h b/tests/regression/sgemm_tcu_wg_dxa_mcast/common.h index b69e7ef5e8..5083f575e6 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/common.h +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/common.h @@ -17,6 +17,17 @@ // Intra-core multicast variant of sgemm_tcu_wg_dxa. `mc_group_size` CTAs // co-resident on one core share the same B tile via DXA multicast. +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint32_t mc_group_size; diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp index fedc34ccad..43d4f615fe 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp @@ -15,7 +15,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; constexpr uint32_t kDescA = 0; constexpr uint32_t kDescB = 1; @@ -63,6 +63,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t a_lane_off = ctx::lane_offset(ctx::tileK); #endif +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t k = 0; k < K; k += ctx::tileK) { if (is_loader_warp) { local_A.expect_tx(1); // A: per-CTA, manual diff --git a/tests/regression/sgemm_tcu_wg_mx/common.h b/tests/regression/sgemm_tcu_wg_mx/common.h index f6cc3275d0..c8fe174ac4 100644 --- a/tests/regression/sgemm_tcu_wg_mx/common.h +++ b/tests/regression/sgemm_tcu_wg_mx/common.h @@ -13,6 +13,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr, B_addr, C_addr; diff --git a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp index 3ea0ced199..4a507b86d2 100644 --- a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp @@ -5,7 +5,7 @@ namespace vt = vortex::tensor; using ctx = vt::wgmma_context; + false, WGMMA_NRC, LDST_MATRIX>; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { static_assert(sizeof(ctx::input_t) == 1, "MX WGMMA inputs use byte storage"); @@ -41,6 +41,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t a_lane_off = ctx::lane_offset(ctx::tileK); #endif +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; uint32_t a_bytes = cta_m * ctx::tileK; diff --git a/tests/regression/sgemm_tcu_wg_sp/common.h b/tests/regression/sgemm_tcu_wg_sp/common.h index 7a2301fc08..6c62a5a380 100644 --- a/tests/regression/sgemm_tcu_wg_sp/common.h +++ b/tests/regression/sgemm_tcu_wg_sp/common.h @@ -19,6 +19,17 @@ #define CTA_SIZE VX_CFG_NUM_THREADS #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; // compressed A in global memory (M × K/2 elements) diff --git a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp index f531b1cf2e..ea3efe2ec9 100644 --- a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp @@ -5,7 +5,7 @@ namespace vt = vortex::tensor; -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; // Per-warp smem layout: [A_compressed][reserved metadata row], then shared B. static constexpr uint32_t smem_a_elems = ctx::xtileM * (ctx::tileK / 2); @@ -59,6 +59,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t a_lane_off = ctx::lane_offset(ctx::tileK / 2); #endif +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; diff --git a/tests/regression/sgemm_tcu_wg_sp_dxa/common.h b/tests/regression/sgemm_tcu_wg_sp_dxa/common.h index b556c4fc6c..add5468cb7 100644 --- a/tests/regression/sgemm_tcu_wg_sp_dxa/common.h +++ b/tests/regression/sgemm_tcu_wg_sp_dxa/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; // compressed A in global memory (M x K/2 elements) diff --git a/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp index ee7b27250d..982cd35319 100644 --- a/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp @@ -7,7 +7,7 @@ namespace vt = vortex::tensor; // WGMMA accumulator; is_sparse=true (A is 2:4 compressed in smem) -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; // Per-warp smem layout: [A_compressed][reserved metadata row], then shared B. static constexpr uint32_t smem_a_elems = ctx::xtileM * (ctx::tileK / 2); @@ -60,6 +60,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #endif // Loop over K tiles +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; From 3f48e52e27de561b815567a98dbc4e478c54e58f Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 16:24:55 -0800 Subject: [PATCH 331/336] simx: build the commit path without the TCU extension 194b8fc59 named FUType::TCU in the commit-queue capacity and delay, but the enumerator exists only with VX_CFG_EXT_TCU_ENABLE, so the default SimX build failed to compile. The single-hop units are now listed in one helper with the TCU case guarded. Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index c1a8590fac..d1d1baaf5c 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -54,6 +54,21 @@ using namespace vortex; +// Units whose output channel already holds every register ahead of the commit +// arbiter except the lane gather's. +static bool commit_single_hop(uint32_t fu) { + switch ((FUType)fu) { + case FUType::LSU: + case FUType::SFU: +#ifdef VX_CFG_EXT_TCU_ENABLE + case FUType::TCU: +#endif + return true; + default: + return false; + } +} + class Core::Impl { public: Impl(const SimContext& ctx, Core* simobject) @@ -366,7 +381,7 @@ class Core::Impl { // A unit's lane gather holds two results. The LSU's, the SFU's and the // TCU's cross one register into it; the others cross two and take a // third slot so that they still retire one result per cycle. - uint32_t capacity = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU || fu == (uint32_t)FUType::TCU) ? 2 : 3; + uint32_t capacity = commit_single_hop(fu) ? 2 : 3; queues.emplace_back(std::make_unique>(simobject_, capacity)); } } @@ -936,7 +951,7 @@ class Core::Impl { // result arbiter, the SFU's result buffer and response switch, and the // TCU's result landing queue are in their channels, leaving each its // lane gather. - uint32_t commit_delay = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU || fu == (uint32_t)FUType::TCU) ? 1 : 2; + uint32_t commit_delay = commit_single_hop(fu) ? 1 : 2; if (arb_in.try_send(trace, commit_delay)) { // Release the warp as soon as its stalling instruction's result leaves // the functional unit — the branch target / fence / warp-control is From 3bf9734cb73af99e136de5fcfd854f0956769f49 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 19:59:31 -0800 Subject: [PATCH 332/336] dxa: require a whole WGATHER group per ALU packet The DXA descriptor is packed by vx_wgather, which gathers within groups of 4 lanes. With fewer than 4 ALU lanes a group spans several packets, so the RTL built a wrong descriptor while SimX, which gathers across the whole warp, still passed. Such builds are not a supported configuration: the RTU (full-warp ALU) and raster (whole quads) already reject theirs. DXA now does the same, with a STATIC_ASSERT in VX_dxa_unit and the matching static_assert in SimX, so both models refuse the configuration instead of disagreeing on it. Checked: DXA with the default 4 ALU lanes builds on SimX and rtlsim; with NUM_ALU_LANES=2 SimX fails to compile and rtlsim stops at start with the message. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/dxa/VX_dxa_unit.sv | 2 ++ sim/simx/dxa/dxa_unit.cpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/hw/rtl/dxa/VX_dxa_unit.sv b/hw/rtl/dxa/VX_dxa_unit.sv index 385d56db7f..61f5a72d2e 100644 --- a/hw/rtl/dxa/VX_dxa_unit.sv +++ b/hw/rtl/dxa/VX_dxa_unit.sv @@ -28,6 +28,8 @@ module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_VAR (execute_if.data.rs3_data) + `STATIC_ASSERT((`VX_CFG_NUM_ALU_LANES >= 4), ("invalid parameter: NUM_ALU_LANES=%0d: the DXA descriptor is packed by WGATHER, which gathers within groups of 4 lanes, so the ALU must hold a whole group in one packet", `VX_CFG_NUM_ALU_LANES)) + // Wgather-based layout (lane index = thread_id & 3): // Lane 0: rs1=smem_addr, rs2=coord2 // Lane 1: rs1=meta, rs2=coord3 diff --git a/sim/simx/dxa/dxa_unit.cpp b/sim/simx/dxa/dxa_unit.cpp index e84f2ea0d2..e398b84ffb 100644 --- a/sim/simx/dxa/dxa_unit.cpp +++ b/sim/simx/dxa/dxa_unit.cpp @@ -18,6 +18,10 @@ using namespace vortex; +static_assert(VX_CFG_NUM_ALU_LANES >= 4, + "the DXA descriptor is packed by WGATHER, which gathers within groups of 4 lanes, " + "so the ALU must hold a whole group in one packet"); + instr_trace_t* DxaUnit::process(instr_trace_t* trace) { if (req_out_.full()) { return nullptr; From ad11347e108bc32f084b7177f12a880775f61fc5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 20:03:27 -0800 Subject: [PATCH 333/336] lsu: count only the block's own requests in the load/store perf counters The perf taps sat on the scheduler's output, which also carries the TCU metadata client, so the RTL counted every metadata read as a program load (and its response in the load latency). SimX counts the LSU's own requests only. The taps now read the block's client port. sgemm_tcu_sp (NT8, sparse, --perf=1): rtlsim loads 29,696 -> 27,648, equal to SimX's 27,648; stores unchanged; PASSED on both. Perf logic only (PERF_ENABLE), so cycles and retired instructions do not move. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_lsu_mem_path.sv | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hw/rtl/core/VX_lsu_mem_path.sv b/hw/rtl/core/VX_lsu_mem_path.sv index 6cef8767b3..68b6ce98a9 100644 --- a/hw/rtl/core/VX_lsu_mem_path.sv +++ b/hw/rtl/core/VX_lsu_mem_path.sv @@ -96,10 +96,12 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( ); `ifdef PERF_ENABLE + // The block's own client only: the TCU metadata loads are the TCU's, not + // the program's loads. for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_perf - assign perf_rd_req_fire[j] = lsu_mem_if.req_valid && lsu_mem_if.req_data.mask[j] && lsu_mem_if.req_ready && ~lsu_mem_if.req_data.rw; - assign perf_wr_req_fire[j] = lsu_mem_if.req_valid && lsu_mem_if.req_data.mask[j] && lsu_mem_if.req_ready && lsu_mem_if.req_data.rw; - assign perf_rsp_fire[j] = lsu_mem_if.rsp_valid && lsu_mem_if.rsp_data.mask[j] && lsu_mem_if.rsp_ready; + assign perf_rd_req_fire[j] = client_if.req_valid && client_if.req_data.mask[j] && client_if.req_ready && ~client_if.req_data.rw; + assign perf_wr_req_fire[j] = client_if.req_valid && client_if.req_data.mask[j] && client_if.req_ready && client_if.req_data.rw; + assign perf_rsp_fire[j] = client_if.rsp_valid && client_if.rsp_data.mask[j] && client_if.rsp_ready; end `endif From 29e641b3386102baeebcbd82f018271fdf64ac80 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 20:11:25 -0800 Subject: [PATCH 334/336] profiles: per-unit lanes and memory bandwidth to the parts; add gtx1060, rtx3060, rx6600, rx9070xt Checked every GPU profile against its part per SM/CU, compute and memory. - Per-unit lanes: NUM_ALU/FPU/LSU_LANES now carry the part's INT32, FP32 and LD/ST lanes per processing block (v100/a100 16/16/8, h100 16/32/8, gtx1080 32/32/8, rtx5080 32/32/4); a warp instruction takes NUM_THREADS/lanes cycles, as on the part. Before, every unit was full-warp, 2-8x the part's LD/ST and up to 2x its INT32. - Shared memory: 16 banks. A Vortex bank is XLEN wide (8 B), so the old 32 banks gave 256 B/clk against the parts' 128; reads of one word by several lanes still share a bank access. - a100: L1 at 128 B/clk (DCACHE_NUM_BANKS 4 x 32 B; the default gave 8 banks) and FDIV/FSQRT/FCVT_PE_RATIO 4 (16 results/clk/SM, CUDA throughput table, cc 8.0). - New: gtx1060 (on gtx1080), rtx3060 (RTU off: it needs full-warp ALU lanes), rx6600 and rx9070xt (Infinity Cache as the L3, ray accelerators as RTU, rx9070xt matrix units as the TCU). Every profile passes gen_config validation; SimX sgemm, sgemm2 and sgemm_tcu pass on each (2 cores). The 64-thread AMD profiles land with the NUM_THREADS=64 build fix. Co-Authored-By: Claude Opus 5.5 --- docs/simulation.md | 11 +++++-- profiles/a100.json | 15 +++++++-- profiles/gtx1060.json | 24 ++++++++++++++ profiles/gtx1080.json | 10 ++++-- profiles/h100.json | 10 ++++-- profiles/rtx3060.json | 63 ++++++++++++++++++++++++++++++++++++ profiles/rtx5080.json | 10 ++++-- profiles/rx6600.json | 68 +++++++++++++++++++++++++++++++++++++++ profiles/rx9070xt.json | 73 ++++++++++++++++++++++++++++++++++++++++++ profiles/v100.json | 9 ++++-- 10 files changed, 276 insertions(+), 17 deletions(-) create mode 100644 profiles/gtx1060.json create mode 100644 profiles/rtx3060.json create mode 100644 profiles/rx6600.json create mode 100644 profiles/rx9070xt.json diff --git a/docs/simulation.md b/docs/simulation.md index 0c4ea4271a..aef657e962 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -232,14 +232,21 @@ machine from it: SimX, rtlsim, the FPGA and ASIC flows, and the kernels. |---|---| | `tinygpu` | 1 core, 2 warps x 2 threads, shared memory | | `gtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 96 KB shared memory (64 KB modelled), 2 MB L2, GDDR5X as 16 GDDR6 channels; no tensor cores | +| `gtx1060` | GeForce GTX 1060 6 GB (Pascal), on `gtx1080`: 10 cores, 1 MB L2, 192 GB/s as 8 GDDR6 channels | +| `rtx3060` | GeForce RTX 3060 12 GB (Ampere GA106): 28 cores x 48 warps, 64 KB shared memory, 2 MB L2, GDDR6 at 360 GB/s; FP16/TF32/INT8/INT4 tensor cores with sparsity; no RTU (it needs full-warp ALU lanes, GA106 has 16 INT32 per block) | +| `rx6600` | Radeon RX 6600 (RDNA 2): 28 CUs x 40 waves x 32 lanes, 64 KB LDS, 2 MB L2, 32 MB Infinity Cache as the L3, GDDR6 at 224 GB/s; ray accelerators as RTU | +| `rx9070xt` | Radeon RX 9070 XT (RDNA 4): 64 CUs x 32 waves x 32 lanes, 64 KB LDS, 8 MB L2, 64 MB Infinity Cache as the L3, GDDR6 at 640 GB/s; FP16/FP8/INT8/INT4 matrix units with sparsity, ray accelerators as RTU | | `v100` | Tesla V100 (Volta): 80 cores x 64 warps, 64 KB shared memory, 4 MB L2, 32 HBM2 channels at 1752 MT/s; FP16 tensor cores | | `a100` | A100 (Ampere): 108 cores x 64 warps, 128 KB shared memory, 32 MB L2, HBM2 at 1555 GB/s; FP16/TF32/INT8/INT4 tensor cores with 2:4 sparsity | | `h100` | H100 SXM5 (Hopper): 132 cores x 64 warps, 128 KB shared memory, 32 MB L2, HBM3 at 3.35 TB/s; FP16/TF32/FP8/INT8 tensor cores with sparsity and wgmma, TMA as DXA | | `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 KB shared memory, 64 MB L2, GDDR7 bandwidth on the GDDR6 model; FP16/TF32/FP8/FP4/MX/INT8 tensor cores with sparsity, TMA as DXA, RT cores as RTU | Every GPU profile also turns on virtual memory, the A extension with Zacas, -Zfhmin, MUFU at the part's special-function throughput, and 32 shared-memory -banks. The real parts are 64-bit machines, so these profiles set `"xlen": 64`: +Zfhmin, MUFU at the part's special-function throughput, the part's INT32, +FP32 and load/store lanes per processing block (`NUM_ALU_LANES`, +`NUM_FPU_LANES`, `NUM_LSU_LANES`; a warp instruction takes `NUM_THREADS` / +lanes cycles, as on the part), and shared memory at the part's 128 B per clock +(16 banks: a Vortex bank is XLEN wide). The real parts are 64-bit machines, so these profiles set `"xlen": 64`: `configure --profile=` picks XLEN from the profile (an explicit, conflicting `--xlen` is an error), and a `build32` tree rejects them. A profile without an `xlen` field, like `tinygpu`, builds at either width. diff --git a/profiles/a100.json b/profiles/a100.json index bdf6a6f8f2..62770583bd 100644 --- a/profiles/a100.json +++ b/profiles/a100.json @@ -6,10 +6,17 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_NUM_ALU_LANES": 16, + "VX_CFG_NUM_FPU_LANES": 16, + "VX_CFG_NUM_LSU_LANES": 8, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4, + "VX_CFG_FCVT_PE_RATIO": 4, "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_DCACHE_NUM_BANKS": 4, "VX_CFG_FETCH_WIDTH": 4, "VX_CFG_LMEM_LOG_SIZE": 17, - "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_LMEM_NUM_BANKS": 16, "VX_CFG_VM_ENABLE": true, "VX_CFG_EXT_A_ENABLE": true, "VX_CFG_EXT_ZACAS_ENABLE": true, @@ -44,6 +51,8 @@ }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "108 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA A100 Tensor Core GPU Architecture whitepaper, Table 3)", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and all type conversions not to or from 8/16-bit integers (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 8.0); 16 FPU lanes / 4", + "DCACHE_NUM_BANKS": "128 B per clock of L1 data per SM (A100 whitepaper): 4 banks of the 32 B dcache word. The default (one bank per LSU request channel) gives 8, twice the part", "DCACHE_SIZE": "192 KB combined L1/shared per SM (whitepaper), rounded down to the power of two the cache accepts", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM2 stacks x 8 128-bit channels = 40 channels at 2430 MT/s, modelled as 32 channels at 3038 MT/s for a power-of-two interleave: 32 x 16 B x 3038 MT/s = 1555 GB/s, the card's bandwidth; the array timing is in nanoseconds, so latency is unchanged", @@ -52,7 +61,7 @@ "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA A100 Tensor Core GPU Architecture whitepaper)", "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", "LMEM_LOG_SIZE, DCACHE_SIZE": "164 KB maximum shared memory per SM (CUDA Programming Guide, compute capability 8.0), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the 192 KB L1/shared pool rounded down: the driver gives a kernel the L1 its shared-memory use leaves", - "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Third-generation tensor cores: FP16/BF16, TF32, INT8 and INT4, with 2:4 structured sparsity (A100 whitepaper). A100 has cp.async but no TMA, so no DXA", "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", @@ -60,6 +69,6 @@ "EXT_TEX_ENABLE": "Texture units in every SM", "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Off: the datacenter part is built for compute (no display outputs; the raster and ROP hardware is reduced or absent), so the profile models its compute machine", "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 8.0)", - "SIMD_WIDTH": "Left at NUM_THREADS (32) although the part has 16 FP32 / 16 INT32 lanes per SM processing block, so a warp instruction issues over two cycles (A100 whitepaper): the RTL TCU needs full-warp operands (NUM_TCU_LANES == NUM_THREADS) and VX_lane_dispatch cannot widen a 16-lane dispatch to 32 lanes, so SIMD_WIDTH=16 with EXT_TCU_ENABLE does not build in rtlsim. The tensor cores are kept; the ALU/FPU/LSU issue width is 2x the part's" + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GA100 SM (A100 whitepaper, Figure 7): 4 processing blocks, each with 16 FP32, 16 INT32, 8 FP64 cores, 8 LD/ST units, 1 SFU and 1 tensor core. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 64 FP64 lanes per SM against the part's 32" } } diff --git a/profiles/gtx1060.json b/profiles/gtx1060.json new file mode 100644 index 0000000000..419eabbf20 --- /dev/null +++ b/profiles/gtx1060.json @@ -0,0 +1,24 @@ +{ + "name": "gtx1060", + "extends": "gtx1080", + "description": "NVIDIA GeForce GTX 1060 6GB (GP106, Pascal, sm_61) at its 1506 MHz base clock: one Vortex core per SM; the SM is GP104's (gtx1080).", + "chip": { + "VX_CFG_NUM_CORES": 10, + "VX_CFG_L2_SIZE": 1048576, + "VX_CFG_L2_NUM_BANKS": 8 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 8, + "VX_CFG_PLATFORM_CLOCK_RATE": 1506, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 192000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 12000 + }, + "sources": { + "SM (all core knobs)": "Inherited from gtx1080: GP106 uses the same Pascal SM as GP104 (128 CUDA cores, 32 LD/ST, 32 SFU, 4 FP64, 96 KB shared memory, 48 KB L1/texture)", + "NUM_CORES": "10 SMs, 1280 CUDA cores (GTX 1060 6GB)", + "L2_SIZE, L2_NUM_BANKS": "1.5 MB L2 in 12 slices (6 x 32-bit controllers, 2 slices each), rounded down to the power-of-two 1 MB and 8 banks", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE, PLATFORM_MEMORY_PEAK_BW": "192-bit GDDR5 at 8 Gbps (192 GB/s) modelled as 8 GDDR6 x16 channels at 12000 MT/s for a power-of-two interleave: 8 x 2 B x 12 GT/s = 192 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE": "1506 MHz base clock (1708 MHz boost), base as in gtx1080" + } +} diff --git a/profiles/gtx1080.json b/profiles/gtx1080.json index 9834525c10..e1961ff102 100644 --- a/profiles/gtx1080.json +++ b/profiles/gtx1080.json @@ -6,10 +6,13 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_NUM_ALU_LANES": 32, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 8, "VX_CFG_DCACHE_SIZE": 32768, "VX_CFG_FETCH_WIDTH": 4, "VX_CFG_LMEM_LOG_SIZE": 16, - "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_LMEM_NUM_BANKS": 16, "VX_CFG_VM_ENABLE": true, "VX_CFG_EXT_A_ENABLE": true, "VX_CFG_EXT_ZACAS_ENABLE": true, @@ -46,13 +49,14 @@ "FETCH_WIDTH": "4 SM processing blocks, each with its own instruction buffer and warp scheduler (NVIDIA GeForce GTX 1080 (GP104) whitepaper)", "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", "LMEM_LOG_SIZE, DCACHE_SIZE": "96 KB dedicated shared memory per SM, separate from the 48 KB L1/texture (GTX 1080 whitepaper), rounded down to the power of two the LMEM accepts", - "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", "EXT_TCU_ENABLE": "Pascal has no tensor cores and no RT cores", "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", "EXT_TEX_ENABLE": "Texture units in every SM", "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", - "NUM_MUFU_LANES": "32 special-function results per clock per SM (8 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 6.1)" + "NUM_MUFU_LANES": "32 special-function results per clock per SM (8 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 6.1)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GP104 SM (GTX 1080 whitepaper): 4 processing blocks, each with 32 CUDA cores (FP32 and INT32 on the same cores), 8 LD/ST units and 8 SFUs; 4 FP64 cores per SM. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 4" } } diff --git a/profiles/h100.json b/profiles/h100.json index 00eab71817..56f0e658e7 100644 --- a/profiles/h100.json +++ b/profiles/h100.json @@ -6,10 +6,13 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_NUM_ALU_LANES": 16, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 8, "VX_CFG_DCACHE_SIZE": 262144, "VX_CFG_FETCH_WIDTH": 4, "VX_CFG_LMEM_LOG_SIZE": 17, - "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_LMEM_NUM_BANKS": 16, "VX_CFG_VM_ENABLE": true, "VX_CFG_EXT_A_ENABLE": true, "VX_CFG_EXT_ZACAS_ENABLE": true, @@ -51,13 +54,14 @@ "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", "LMEM_LOG_SIZE, DCACHE_SIZE": "228 KB maximum shared memory per SM (CUDA Programming Guide, compute capability 9.0), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the whole 256 KB L1/shared pool: the driver gives a kernel the L1 its shared-memory use leaves", - "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Fourth-generation tensor cores: FP16/BF16, TF32, FP8 and INT8 (no INT4), 2:4 structured sparsity, warpgroup MMA (wgmma); the Tensor Memory Accelerator is the DXA (H100 whitepaper)", "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", "EXT_TEX_ENABLE": "Texture units in every SM", "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Off: the datacenter part is built for compute (no display outputs; the raster and ROP hardware is reduced or absent), so the profile models its compute machine", - "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 9.0)" + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 9.0)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GH100 SM (H100 whitepaper, Figure 7): 4 processing blocks, each with 32 FP32, 16 INT32, 16 FP64 cores, 8 LD/ST units, 1 SFU and 1 tensor core. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 64" } } diff --git a/profiles/rtx3060.json b/profiles/rtx3060.json new file mode 100644 index 0000000000..37f3c001e7 --- /dev/null +++ b/profiles/rtx3060.json @@ -0,0 +1,63 @@ +{ + "name": "rtx3060", + "description": "NVIDIA GeForce RTX 3060 12GB (GA106, Ampere, sm_86) at its 1320 MHz base clock: one Vortex core per SM.", + "xlen": 64, + "core": { + "VX_CFG_NUM_WARPS": 48, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_NUM_ALU_LANES": 16, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 4, + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_TF32_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_INT4_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 28, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 2097152, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 16, + "VX_CFG_CP_LLC_CLIENT": 1 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, + "VX_CFG_PLATFORM_CLOCK_RATE": 1320, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 360000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 11250 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH, FETCH_WIDTH": "28 SMs (3584 CUDA cores), 48 resident warps per SM (compute capability 8.6), 4 processing blocks each with its own scheduler and L0 instruction cache (NVIDIA Ampere GA102 whitepaper, SM)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GA10x SM processing block: 16 FP32 cores plus 16 cores that run FP32 or INT32, 4 LD/ST units, 1 SFU, 1 tensor core (GA102 whitepaper): 32 FP32, 16 INT32 and 4 LD/ST lanes per block. The shared FP32/INT32 half is modelled as FPU lanes, so an INT and an FP32 instruction can overlap where the part would share those 16 cores. FP64 (2 per SM on the part) is not modelled separately: with XLEN=64 all 128 FPU lanes execute FP64", + "DCACHE_SIZE": "128 KB L1/shared per SM (GA102 whitepaper)", + "LMEM_LOG_SIZE": "100 KB maximum shared memory per SM (compute capability 8.6), rounded down to the power of two the LMEM accepts", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "3 MB L2 (GA106), rounded down to 2 MB; one bank per memory channel; 16 ways as in GPGPU-Sim's SM86_RTX3070", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE, PLATFORM_MEMORY_PEAK_BW": "192-bit GDDR6 at 15 Gbps: 6 devices x 2 x16 channels = 12 channels, modelled as 16 at 11250 MT/s for a power-of-two interleave: 16 x 2 B x 11.25 GT/s = 360 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE": "1320 MHz base clock (1777 MHz boost), base as in rtx5080", + "EXT_TCU_ENABLE, TCU_*": "Third-generation tensor cores, one per processing block: FP16/BF16, TF32, INT8, INT4, 2:4 sparsity", + "EXT_RTU_ENABLE": "Off although the part has one RT core per SM: the RTU needs full-warp ALU lanes (NUM_ALU_LANES == NUM_THREADS), and the part's 16 INT32 lanes per block take precedence", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (compute capability 8.6)", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, NUM_BARRIERS, EXT_TEX_ENABLE, EXT_RASTER_ENABLE, EXT_OM_ENABLE": "As in the other NVIDIA profiles: demand-paged MMU, atomics with CAS, FP16, 16 named barriers, and the graphics pipeline of a GeForce part", + "xlen": "64-bit machine: 64-bit pointers and FP64" + } +} diff --git a/profiles/rtx5080.json b/profiles/rtx5080.json index 4f250455ef..f42359b9bc 100644 --- a/profiles/rtx5080.json +++ b/profiles/rtx5080.json @@ -6,11 +6,14 @@ "VX_CFG_NUM_WARPS": 48, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_NUM_ALU_LANES": 32, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 4, "VX_CFG_DCACHE_SIZE": 131072, "VX_CFG_DCACHE_MSHR_SIZE": 24, "VX_CFG_FETCH_WIDTH": 4, "VX_CFG_LMEM_LOG_SIZE": 16, - "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_LMEM_NUM_BANKS": 16, "VX_CFG_VM_ENABLE": true, "VX_CFG_EXT_A_ENABLE": true, "VX_CFG_EXT_ZACAS_ENABLE": true, @@ -89,7 +92,7 @@ "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA RTX Blackwell GPU Architecture whitepaper)", "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", "LMEM_LOG_SIZE, DCACHE_SIZE": "100 KB maximum shared memory per SM (compute capability 12.0 device query: 102400 B per SM), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the whole 128 KB L1/shared pool: the driver gives a kernel the L1 its shared-memory use leaves", - "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Fifth-generation tensor cores: FP16/BF16, TF32, FP8, FP4 (NVFP4, MXFP4) and the MX formats, INT8, with 2:4 structured sparsity; no wgmma on sm_120. TMA is the DXA; the fourth-generation RT cores are the RTU (RTX Blackwell whitepaper)", "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", @@ -97,6 +100,7 @@ "EXT_TEX_ENABLE": "Texture units in every SM", "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", "RTU_TLAS_ENABLE": "RT cores traverse two-level acceleration structures (instance TLAS over BLAS), as DXR/Vulkan ray tracing requires", - "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 12.0)" + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 12.0)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GB203 SM (RTX Blackwell whitepaper, Figure 5 and App. B): 4 processing blocks, each with 32 unified FP32/INT32 CUDA cores (INT32 TOPS = FP32 TFLOPS, Table 4), 4 LD/ST units and 1 SFU, 1 tensor core; 2 FP64 cores per SM. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 2" } } diff --git a/profiles/rx6600.json b/profiles/rx6600.json new file mode 100644 index 0000000000..06e8f9e927 --- /dev/null +++ b/profiles/rx6600.json @@ -0,0 +1,68 @@ +{ + "name": "rx6600", + "description": "AMD Radeon RX 6600 (Navi 23, RDNA 2, gfx1032) at its 2044 MHz game clock: one Vortex core per compute unit (CU).", + "xlen": 64, + "core": { + "VX_CFG_NUM_WARPS": 40, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 2, + "VX_CFG_FETCH_WIDTH": 2, + "VX_CFG_NUM_ALU_LANES": 32, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 16, + "VX_CFG_NUM_LSU_BLOCKS": 1, + "VX_CFG_DCACHE_SIZE": 16384, + "VX_CFG_ICACHE_SIZE": 16384, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_RTU_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 28, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 2097152, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 8, + "VX_CFG_L3_ENABLE": true, + "VX_CFG_L3_SIZE": 33554432, + "VX_CFG_L3_NUM_WAYS": 16, + "VX_CFG_L3_NUM_BANKS": 8, + "VX_CFG_CP_LLC_CLIENT": 1 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 8, + "VX_CFG_PLATFORM_CLOCK_RATE": 2044, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 224000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 14000 + }, + "sources": { + "NUM_CORES": "28 CUs (14 WGPs), 1792 stream processors (RX 6600)", + "NUM_THREADS, ISSUE_WIDTH, FETCH_WIDTH, NUM_WARPS": "2 SIMD32 units per CU, each with its own scheduler, running wave32 at one instruction per clock; 20 waves per SIMD (AMD RDNA architecture whitepaper)", + "NUM_ALU_LANES, NUM_FPU_LANES": "32 VALU lanes per SIMD for FP32 and INT32 (no dual-issue on RDNA 2)", + "NUM_LSU_LANES, NUM_LSU_BLOCKS": "One vector-memory path per CU behind the L0 vector cache, 64 B per clock (16 x 4 B lanes)", + "DCACHE_SIZE": "16 KB L0 vector cache per CU, 128 B lines. The 128 KB graphics L1 per shader array, shared by its CUs, has no Vortex level: cores in a socket would share the L0-sized dcache, so the L1 is left out", + "ICACHE_SIZE": "32 KB instruction cache per WGP (two CUs), i.e. 16 KB per CU", + "LMEM_LOG_SIZE, LMEM_NUM_BANKS": "64 KB LDS per CU in CU mode (128 KB per WGP), 32 banks of 4 B = 128 B per clock per CU; Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2, one slice per 16-bit memory channel, 16-way", + "L3_ENABLE, L3_SIZE, L3_NUM_WAYS, L3_NUM_BANKS": "32 MB Infinity Cache (first generation) behind the L2, modelled as the Vortex L3, one bank per memory channel", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE, PLATFORM_MEMORY_PEAK_BW": "128-bit GDDR6 at 14 Gbps: 4 devices x 2 x16 channels = 8 channels, 8 x 2 B x 14 GT/s = 224 GB/s; latency not calibrated", + "PLATFORM_CLOCK_RATE": "2044 MHz game clock (2491 MHz boost)", + "NUM_MUFU_LANES": "Transcendentals at a quarter rate per SIMD: 16 per CU per clock", + "EXT_TCU_ENABLE": "Off: RDNA 2 has no matrix units (only packed dot-product instructions)", + "EXT_RTU_ENABLE": "One ray accelerator per CU", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, NUM_BARRIERS, EXT_TEX_ENABLE, EXT_RASTER_ENABLE, EXT_OM_ENABLE": "GPUVM paging, global and LDS atomics with compare-swap, FP16 instructions and workgroup barriers, and the graphics pipeline (4 texture units per CU, rasterizers, ROPs) of a gaming part", + "xlen": "64-bit machine: 64-bit pointers and FP64. FP64 runs at 1/16 of FP32 (4 per CU per clock) on the part; with XLEN=64 every FPU lane executes FP64, so the profile has 64 FP64 lanes per CU" + } +} diff --git a/profiles/rx9070xt.json b/profiles/rx9070xt.json new file mode 100644 index 0000000000..55f397cf6e --- /dev/null +++ b/profiles/rx9070xt.json @@ -0,0 +1,73 @@ +{ + "name": "rx9070xt", + "description": "AMD Radeon RX 9070 XT (Navi 48, RDNA 4, gfx1201) at its 2400 MHz game clock: one Vortex core per compute unit (CU).", + "xlen": 64, + "core": { + "VX_CFG_NUM_WARPS": 32, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 2, + "VX_CFG_FETCH_WIDTH": 2, + "VX_CFG_NUM_ALU_LANES": 32, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 16, + "VX_CFG_NUM_LSU_BLOCKS": 1, + "VX_CFG_DCACHE_SIZE": 32768, + "VX_CFG_ICACHE_SIZE": 16384, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_FP8_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_INT4_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_EXT_RTU_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 64, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 8388608, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 16, + "VX_CFG_L3_ENABLE": true, + "VX_CFG_L3_SIZE": 67108864, + "VX_CFG_L3_NUM_WAYS": 16, + "VX_CFG_L3_NUM_BANKS": 16, + "VX_CFG_CP_LLC_CLIENT": 1 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, + "VX_CFG_PLATFORM_CLOCK_RATE": 2400, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 640000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 20000 + }, + "sources": { + "NUM_CORES": "64 CUs in 4 shader engines (AMD RX 9070 XT specifications; Chips and Cheese, RDNA 4 architecture)", + "NUM_THREADS, ISSUE_WIDTH, FETCH_WIDTH, NUM_WARPS": "2 SIMD32 units per CU, each with its own scheduler, running wave32 at one instruction per clock; 16 waves per SIMD (AMD RDNA 4 ISA; zolotukhin.ai RDNA3/RDNA4 reference)", + "NUM_ALU_LANES, NUM_FPU_LANES": "32 lanes per SIMD for FP32 and INT32 (VALU). RDNA 4's dual-issue (VOPD) and the paired FP32 rate behind its 48.7 TFLOPS are not modelled: the profile issues one VALU instruction per SIMD per clock", + "NUM_LSU_LANES, NUM_LSU_BLOCKS": "One vector-memory path per CU behind the 32 KB L0 vector cache, which returns 64 B per clock (16 x 4 B lanes)", + "DCACHE_SIZE": "32 KB L0 vector cache per CU, 64 B lines. RDNA 4 turned the shader-array L1 into a read/write coalescing buffer, so there is no L1 level to model (Chips and Cheese, RDNA 4)", + "ICACHE_SIZE": "32 KB instruction cache per WGP (two CUs), i.e. 16 KB per CU", + "LMEM_LOG_SIZE, LMEM_NUM_BANKS": "64 KB LDS per CU in CU mode (128 KB per WGP), 32 banks of 4 B = 128 B per clock per CU; Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "8 MB L2, 2 MB per shader engine, 128 B lines (Chips and Cheese, RDNA 4); one bank per memory channel; 16 ways as GCN/RDNA L2s", + "L3_ENABLE, L3_SIZE, L3_NUM_WAYS, L3_NUM_BANKS": "64 MB third-generation Infinity Cache (MALL) behind the L2, modelled as the Vortex L3, one bank per memory channel", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE, PLATFORM_MEMORY_PEAK_BW": "256-bit GDDR6 at 20 Gbps: 8 devices x 2 x16 channels = 16 channels, 16 x 2 B x 20 GT/s = 640 GB/s; latency not calibrated", + "PLATFORM_CLOCK_RATE": "2400 MHz game clock (2970 MHz boost)", + "EXT_MUFU_ENABLE, NUM_MUFU_LANES": "Transcendentals (exp, log, rcp, rsq, sqrt) at a quarter rate per SIMD: 8 per SIMD, 16 per CU per clock", + "EXT_TCU_ENABLE, TCU_*": "Two AI accelerators (WMMA) per CU, one per SIMD as NUM_TCU_BLOCKS=ISSUE_WIDTH: FP16/BF16, FP8/BF8, INT8 and INT4 with 4:2 sparsity; no TF32", + "EXT_RTU_ENABLE": "One ray accelerator per CU", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, NUM_BARRIERS, EXT_TEX_ENABLE, EXT_RASTER_ENABLE, EXT_OM_ENABLE": "GPUVM paging, global and LDS atomics with compare-swap, FP16, workgroup barriers, and the graphics pipeline (4 texture units per CU, rasterizers, ROPs) of a gaming part", + "xlen": "64-bit machine: 64-bit pointers and FP64. FP64 runs at 1/32 of the dual-issue FP32 rate (4 per CU per clock) on the part; with XLEN=64 every FPU lane executes FP64, so the profile has 64 FP64 lanes per CU" + } +} diff --git a/profiles/v100.json b/profiles/v100.json index cf8a79ab7a..c75c2d48ae 100644 --- a/profiles/v100.json +++ b/profiles/v100.json @@ -6,10 +6,13 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_NUM_ALU_LANES": 16, + "VX_CFG_NUM_FPU_LANES": 16, + "VX_CFG_NUM_LSU_LANES": 8, "VX_CFG_DCACHE_SIZE": 131072, "VX_CFG_FETCH_WIDTH": 4, "VX_CFG_LMEM_LOG_SIZE": 16, - "VX_CFG_LMEM_NUM_BANKS": 32, + "VX_CFG_LMEM_NUM_BANKS": 16, "VX_CFG_VM_ENABLE": true, "VX_CFG_EXT_A_ENABLE": true, "VX_CFG_EXT_ZACAS_ENABLE": true, @@ -47,7 +50,7 @@ "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA Tesla V100 GPU Architecture whitepaper)", "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", "LMEM_LOG_SIZE, DCACHE_SIZE": "96 KB maximum shared memory per SM (CUDA Programming Guide, compute capability 7.0), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the whole 128 KB L1/shared pool: the driver gives a kernel the L1 its shared-memory use leaves, so a kernel without shared memory sees the full pool", - "LMEM_NUM_BANKS": "32 shared-memory banks of 4 B, as on every NVIDIA GPU since Maxwell (CUDA C++ Programming Guide, shared memory)", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Volta tensor cores: FP16 inputs with FP32 accumulation only (Volta whitepaper). The TCU's FP16 format group also carries BF16, which Volta lacks: no knob separates them", "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", @@ -55,6 +58,6 @@ "EXT_TEX_ENABLE": "Texture units in every SM", "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 7.0)", - "SIMD_WIDTH": "Left at NUM_THREADS (32) although the part has 16 FP32 / 16 INT32 lanes per SM processing block, so a warp instruction issues over two cycles (Volta whitepaper): the RTL TCU needs full-warp operands (NUM_TCU_LANES == NUM_THREADS) and VX_lane_dispatch cannot widen a 16-lane dispatch to 32 lanes, so SIMD_WIDTH=16 with EXT_TCU_ENABLE does not build in rtlsim. The tensor cores are kept; the ALU/FPU/LSU issue width is 2x the part's" + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GV100 SM (Volta whitepaper, Figure 5): 4 processing blocks, each with 16 FP32, 16 INT32, 8 FP64 cores, 8 LD/ST units, 1 SFU and 2 tensor cores. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 64 FP64 lanes per SM against the part's 32" } } From 070b7e697d37459f4363412f197260f0a6414e61 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 20:13:11 -0800 Subject: [PATCH 335/336] tests: exit 1 on any error instead of the error count 75 test mains returned their error count, and the exit status keeps only its low 8 bits, so a test with a multiple of 256 errors exited 0 and the runner saw a pass. They now return errors != 0. Helpers that return a count to their caller are unchanged. Built: tests/regression, opencl, kernel and hip. Co-Authored-By: Claude Opus 5.5 --- tests/hip/atomicreduce/main.cpp | 2 +- tests/hip/histogram/main.cpp | 2 +- tests/hip/sgemm/main.cpp | 2 +- tests/hip/vecadd/main.cpp | 2 +- tests/kernel/conform/main.cpp | 2 +- tests/kernel/fibonacci/main.cpp | 2 +- tests/opencl/atomicreduce/main.cc | 2 +- tests/opencl/backprop/main.cc | 2 +- tests/opencl/bfs/main.cc | 2 +- tests/opencl/btree/main.cc | 2 +- tests/opencl/cfd/main.cc | 2 +- tests/opencl/conv3/main.cc | 2 +- tests/opencl/dwt2d/main.cc | 2 +- tests/opencl/guassian/main.cc | 2 +- tests/opencl/heartwall/main.cc | 2 +- tests/opencl/histogram/main.cc | 2 +- tests/opencl/hotspot/main.cc | 2 +- tests/opencl/hotspot3D/main.cc | 2 +- tests/opencl/hybridsort/main.cc | 2 +- tests/opencl/lavaMD/main.cc | 2 +- tests/opencl/lbm/main.cc | 2 +- tests/opencl/lud/main.cc | 2 +- tests/opencl/myocyte/main.cc | 2 +- tests/opencl/nearn/main.cc | 2 +- tests/opencl/nw/main.cc | 2 +- tests/opencl/pathfinder/main.cc | 2 +- tests/opencl/psort/main.cc | 2 +- tests/opencl/psum/main.cc | 2 +- tests/opencl/saxpy/main.cc | 2 +- tests/opencl/sfilter/main.cc | 2 +- tests/opencl/sgemm/main.cc | 2 +- tests/opencl/sgemm2/main.cc | 2 +- tests/opencl/sgemm3/main.cc | 2 +- tests/opencl/spmv/main.cc | 2 +- tests/opencl/srad/main.cc | 2 +- tests/opencl/stencil/main.cc | 2 +- tests/opencl/streamcluster/main.cc | 2 +- tests/opencl/vecadd/main.cc | 2 +- tests/regression/amo/main.cpp | 2 +- tests/regression/async_barrier/main.cpp | 2 +- tests/regression/async_gbarrier/main.cpp | 2 +- tests/regression/basic/main.cpp | 2 +- tests/regression/conv3/main.cpp | 2 +- tests/regression/cta_cluster/main.cpp | 2 +- tests/regression/demo/main.cpp | 2 +- tests/regression/diverge/main.cpp | 2 +- tests/regression/dogfood/main.cpp | 2 +- tests/regression/dxa_copy_mcast/main.cpp | 2 +- tests/regression/fence/main.cpp | 2 +- tests/regression/gbar_phase/main.cpp | 2 +- tests/regression/io_addr/main.cpp | 2 +- tests/regression/madmax/main.cpp | 2 +- tests/regression/module_reload/main.cpp | 2 +- tests/regression/reopen/main.cpp | 2 +- tests/regression/sgemm/main.cpp | 2 +- tests/regression/sgemm2_dxa/main.cpp | 2 +- tests/regression/sgemm2_dxa_mcast/main.cpp | 2 +- tests/regression/sgemm2_tcu/main.cpp | 2 +- tests/regression/sgemm2_v1/main.cpp | 2 +- tests/regression/sgemm_tcu/main.cpp | 2 +- tests/regression/sgemm_tcu_mx/main.cpp | 2 +- tests/regression/sgemm_tcu_sp_mx/main.cpp | 2 +- tests/regression/sgemm_tcu_wg/main.cpp | 2 +- tests/regression/sgemm_tcu_wg_dxa/main.cpp | 2 +- tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp | 2 +- tests/regression/sgemm_tcu_wg_mx/main.cpp | 2 +- tests/regression/sgemm_tcu_wg_sp/main.cpp | 2 +- tests/regression/sgemm_tcu_wg_sp_dxa/main.cpp | 2 +- tests/regression/sgemm_v1/main.cpp | 2 +- tests/regression/sgemmx/main.cpp | 2 +- tests/regression/sort/main.cpp | 2 +- tests/regression/stencil3d/main.cpp | 2 +- tests/regression/vm_stress/main.cpp | 2 +- tests/regression/vm_test/main.cpp | 2 +- tests/regression/wsync/main.cpp | 2 +- 75 files changed, 75 insertions(+), 75 deletions(-) diff --git a/tests/hip/atomicreduce/main.cpp b/tests/hip/atomicreduce/main.cpp index 9d863b4ecb..9a9b0fd83e 100644 --- a/tests/hip/atomicreduce/main.cpp +++ b/tests/hip/atomicreduce/main.cpp @@ -108,5 +108,5 @@ int main(int argc, char** argv) { } else { printf("FAILED! - %d errors\n", errors); } - return errors; + return errors != 0; } diff --git a/tests/hip/histogram/main.cpp b/tests/hip/histogram/main.cpp index 3155a05760..9aaf7494f1 100644 --- a/tests/hip/histogram/main.cpp +++ b/tests/hip/histogram/main.cpp @@ -111,5 +111,5 @@ int main(int argc, char** argv) { } else { printf("FAILED! - %d errors\n", errors); } - return errors; + return errors != 0; } diff --git a/tests/hip/sgemm/main.cpp b/tests/hip/sgemm/main.cpp index 99b63c0da8..383f9748e9 100644 --- a/tests/hip/sgemm/main.cpp +++ b/tests/hip/sgemm/main.cpp @@ -151,5 +151,5 @@ int main(int argc, char** argv) { } else { printf("FAILED! - %d errors\n", errors); } - return errors; + return errors != 0; } diff --git a/tests/hip/vecadd/main.cpp b/tests/hip/vecadd/main.cpp index 5e8e8b7e4b..7af6a5c871 100644 --- a/tests/hip/vecadd/main.cpp +++ b/tests/hip/vecadd/main.cpp @@ -131,5 +131,5 @@ int main(int argc, char** argv) { } else { printf("FAILED! - %d errors\n", errors); } - return errors; + return errors != 0; } diff --git a/tests/kernel/conform/main.cpp b/tests/kernel/conform/main.cpp index b34997ad6d..4790a0de34 100644 --- a/tests/kernel/conform/main.cpp +++ b/tests/kernel/conform/main.cpp @@ -41,5 +41,5 @@ int main() { PRINTF("Failed!\n"); } - return errors; + return errors != 0; } diff --git a/tests/kernel/fibonacci/main.cpp b/tests/kernel/fibonacci/main.cpp index c6fc036aa9..b083db6d1b 100644 --- a/tests/kernel/fibonacci/main.cpp +++ b/tests/kernel/fibonacci/main.cpp @@ -24,7 +24,7 @@ int main() { errors = 1; } - return errors; + return errors != 0; } diff --git a/tests/opencl/atomicreduce/main.cc b/tests/opencl/atomicreduce/main.cc index 7ff0f74fab..cce4c9f7ce 100644 --- a/tests/opencl/atomicreduce/main.cc +++ b/tests/opencl/atomicreduce/main.cc @@ -170,5 +170,5 @@ int main (int argc, char **argv) { cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/backprop/main.cc b/tests/opencl/backprop/main.cc index 4823ecea7f..47dd70fc9f 100644 --- a/tests/opencl/backprop/main.cc +++ b/tests/opencl/backprop/main.cc @@ -302,7 +302,7 @@ int main(int argc, char** argv) { if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/bfs/main.cc b/tests/opencl/bfs/main.cc index 490f6f4b24..325bafb204 100755 --- a/tests/opencl/bfs/main.cc +++ b/tests/opencl/bfs/main.cc @@ -307,7 +307,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { printf("Failed!\n"); - return errors; + return errors != 0; } printf("Passed!\n"); diff --git a/tests/opencl/btree/main.cc b/tests/opencl/btree/main.cc index a4a2d1e940..d72fdf86fe 100644 --- a/tests/opencl/btree/main.cc +++ b/tests/opencl/btree/main.cc @@ -504,7 +504,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/cfd/main.cc b/tests/opencl/cfd/main.cc index 1d8d11c741..ca00e4f18d 100644 --- a/tests/opencl/cfd/main.cc +++ b/tests/opencl/cfd/main.cc @@ -519,7 +519,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/conv3/main.cc b/tests/opencl/conv3/main.cc index 44bfe64886..34b083837e 100644 --- a/tests/opencl/conv3/main.cc +++ b/tests/opencl/conv3/main.cc @@ -247,5 +247,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/dwt2d/main.cc b/tests/opencl/dwt2d/main.cc index 92e223b744..edf8e4f5df 100644 --- a/tests/opencl/dwt2d/main.cc +++ b/tests/opencl/dwt2d/main.cc @@ -222,7 +222,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/guassian/main.cc b/tests/opencl/guassian/main.cc index a06ebbbdcd..b2ee50de55 100755 --- a/tests/opencl/guassian/main.cc +++ b/tests/opencl/guassian/main.cc @@ -123,7 +123,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/heartwall/main.cc b/tests/opencl/heartwall/main.cc index 0770c4e43c..11d922fcdd 100644 --- a/tests/opencl/heartwall/main.cc +++ b/tests/opencl/heartwall/main.cc @@ -357,7 +357,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/histogram/main.cc b/tests/opencl/histogram/main.cc index d6b1d3b879..b8131f4417 100644 --- a/tests/opencl/histogram/main.cc +++ b/tests/opencl/histogram/main.cc @@ -174,5 +174,5 @@ int main (int argc, char **argv) { cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/hotspot/main.cc b/tests/opencl/hotspot/main.cc index a399f93616..7e519d39b7 100644 --- a/tests/opencl/hotspot/main.cc +++ b/tests/opencl/hotspot/main.cc @@ -279,7 +279,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/hotspot3D/main.cc b/tests/opencl/hotspot3D/main.cc index af729c56b7..c2a30a0d17 100644 --- a/tests/opencl/hotspot3D/main.cc +++ b/tests/opencl/hotspot3D/main.cc @@ -273,7 +273,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/hybridsort/main.cc b/tests/opencl/hybridsort/main.cc index 60b2349c5d..0b891363ee 100644 --- a/tests/opencl/hybridsort/main.cc +++ b/tests/opencl/hybridsort/main.cc @@ -310,7 +310,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/lavaMD/main.cc b/tests/opencl/lavaMD/main.cc index 4da28826f2..a6d713b531 100644 --- a/tests/opencl/lavaMD/main.cc +++ b/tests/opencl/lavaMD/main.cc @@ -355,7 +355,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/lbm/main.cc b/tests/opencl/lbm/main.cc index 2da7435108..c8a917242b 100644 --- a/tests/opencl/lbm/main.cc +++ b/tests/opencl/lbm/main.cc @@ -150,7 +150,7 @@ int main(int nArgs, char *arg[]) { pb_SwitchToTimer(&timers, pb_TimerID_NONE); pb_PrintTimerSet(&timers); pb_FreeParameters(params); - return errors; + return errors != 0; } /*############################################################################*/ diff --git a/tests/opencl/lud/main.cc b/tests/opencl/lud/main.cc index c1f00fa916..c40e63f95f 100644 --- a/tests/opencl/lud/main.cc +++ b/tests/opencl/lud/main.cc @@ -265,7 +265,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/myocyte/main.cc b/tests/opencl/myocyte/main.cc index e359e8c033..117ea15f37 100644 --- a/tests/opencl/myocyte/main.cc +++ b/tests/opencl/myocyte/main.cc @@ -474,7 +474,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/nearn/main.cc b/tests/opencl/nearn/main.cc index ea8238e167..fa73fbfbef 100755 --- a/tests/opencl/nearn/main.cc +++ b/tests/opencl/nearn/main.cc @@ -68,7 +68,7 @@ int main(int argc, char *argv[]) { printf("Passed!\n"); } - return errors; + return errors != 0; } float *OpenClFindNearestNeighbors(cl_context context, int numRecords, diff --git a/tests/opencl/nw/main.cc b/tests/opencl/nw/main.cc index 38375c3485..bfbff5cdec 100644 --- a/tests/opencl/nw/main.cc +++ b/tests/opencl/nw/main.cc @@ -279,7 +279,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/pathfinder/main.cc b/tests/opencl/pathfinder/main.cc index a2b9d67696..69c868643d 100644 --- a/tests/opencl/pathfinder/main.cc +++ b/tests/opencl/pathfinder/main.cc @@ -223,7 +223,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/psort/main.cc b/tests/opencl/psort/main.cc index 0e72599ece..d7ca45dc7a 100644 --- a/tests/opencl/psort/main.cc +++ b/tests/opencl/psort/main.cc @@ -229,5 +229,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/psum/main.cc b/tests/opencl/psum/main.cc index 5606de8c5b..4e37212cae 100644 --- a/tests/opencl/psum/main.cc +++ b/tests/opencl/psum/main.cc @@ -225,5 +225,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/saxpy/main.cc b/tests/opencl/saxpy/main.cc index 2cb4b71ba8..b208755051 100644 --- a/tests/opencl/saxpy/main.cc +++ b/tests/opencl/saxpy/main.cc @@ -257,5 +257,5 @@ int main(int argc, char **argv) { Cleanup(device_id, context, queue, program, kernel, memObjects); - return errors; + return errors != 0; } diff --git a/tests/opencl/sfilter/main.cc b/tests/opencl/sfilter/main.cc index c56d26ffdc..7cf4f6e126 100644 --- a/tests/opencl/sfilter/main.cc +++ b/tests/opencl/sfilter/main.cc @@ -298,5 +298,5 @@ int main(int argc, char **argv) { Cleanup(kernel_bin, device_id, context, queue, program, kernel, memObjects); - return errors; + return errors != 0; } diff --git a/tests/opencl/sgemm/main.cc b/tests/opencl/sgemm/main.cc index f8a7350b6a..95a4d707cd 100644 --- a/tests/opencl/sgemm/main.cc +++ b/tests/opencl/sgemm/main.cc @@ -262,5 +262,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/sgemm2/main.cc b/tests/opencl/sgemm2/main.cc index 43c016514b..3cc02fddb6 100644 --- a/tests/opencl/sgemm2/main.cc +++ b/tests/opencl/sgemm2/main.cc @@ -267,5 +267,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/sgemm3/main.cc b/tests/opencl/sgemm3/main.cc index 21bec0c71e..f567d4795f 100644 --- a/tests/opencl/sgemm3/main.cc +++ b/tests/opencl/sgemm3/main.cc @@ -273,5 +273,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/spmv/main.cc b/tests/opencl/spmv/main.cc index fada19c156..0e72c6c70b 100644 --- a/tests/opencl/spmv/main.cc +++ b/tests/opencl/spmv/main.cc @@ -379,5 +379,5 @@ int main(int argc, char **argv) { pb_PrintTimerSet(&timers); pb_FreeParameters(parameters); - return errors; + return errors != 0; } diff --git a/tests/opencl/srad/main.cc b/tests/opencl/srad/main.cc index c15d1deb8f..516ee78e1c 100644 --- a/tests/opencl/srad/main.cc +++ b/tests/opencl/srad/main.cc @@ -361,7 +361,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/stencil/main.cc b/tests/opencl/stencil/main.cc index 30dee81bd0..a1994acca6 100644 --- a/tests/opencl/stencil/main.cc +++ b/tests/opencl/stencil/main.cc @@ -354,5 +354,5 @@ int main(int argc, char** argv) { pb_PrintTimerSet(&timers); pb_FreeParameters(parameters); - return errors; + return errors != 0; } diff --git a/tests/opencl/streamcluster/main.cc b/tests/opencl/streamcluster/main.cc index 97039e1c5b..0f549a1983 100644 --- a/tests/opencl/streamcluster/main.cc +++ b/tests/opencl/streamcluster/main.cc @@ -298,7 +298,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/vecadd/main.cc b/tests/opencl/vecadd/main.cc index 2334e0b4e9..d66d486d20 100644 --- a/tests/opencl/vecadd/main.cc +++ b/tests/opencl/vecadd/main.cc @@ -239,5 +239,5 @@ int main (int argc, char **argv) { cleanup(); - return errors; + return errors != 0; } diff --git a/tests/regression/amo/main.cpp b/tests/regression/amo/main.cpp index 74bf4265c9..7b7d49a306 100644 --- a/tests/regression/amo/main.cpp +++ b/tests/regression/amo/main.cpp @@ -176,5 +176,5 @@ int main(int argc, char *argv[]) { } cleanup(); - return errors; + return errors != 0; } diff --git a/tests/regression/async_barrier/main.cpp b/tests/regression/async_barrier/main.cpp index 9f62c03e8e..e775e1040f 100644 --- a/tests/regression/async_barrier/main.cpp +++ b/tests/regression/async_barrier/main.cpp @@ -263,7 +263,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/async_gbarrier/main.cpp b/tests/regression/async_gbarrier/main.cpp index 845212dc83..6bb42eb3b5 100644 --- a/tests/regression/async_gbarrier/main.cpp +++ b/tests/regression/async_gbarrier/main.cpp @@ -171,7 +171,7 @@ int main(int argc, char* argv[]) { if (errors != 0) { std::cout << "Found " << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/basic/main.cpp b/tests/regression/basic/main.cpp index 0967e27ad4..c628d2d1ae 100755 --- a/tests/regression/basic/main.cpp +++ b/tests/regression/basic/main.cpp @@ -266,7 +266,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "Test PASSED" << std::endl; diff --git a/tests/regression/conv3/main.cpp b/tests/regression/conv3/main.cpp index 93e67b278b..4a3f0b8f6d 100644 --- a/tests/regression/conv3/main.cpp +++ b/tests/regression/conv3/main.cpp @@ -291,7 +291,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/cta_cluster/main.cpp b/tests/regression/cta_cluster/main.cpp index 5c97e6bb3c..8cbdab5326 100644 --- a/tests/regression/cta_cluster/main.cpp +++ b/tests/regression/cta_cluster/main.cpp @@ -217,7 +217,7 @@ int main(int argc, char* argv[]) { } cleanup(); - if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; return errors; } + if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; return errors != 0; } std::cout << "PASSED\n"; return 0; } diff --git a/tests/regression/demo/main.cpp b/tests/regression/demo/main.cpp index 73b0387f42..b59869a27e 100644 --- a/tests/regression/demo/main.cpp +++ b/tests/regression/demo/main.cpp @@ -254,7 +254,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/diverge/main.cpp b/tests/regression/diverge/main.cpp index 41f77fab4d..ea092746a4 100644 --- a/tests/regression/diverge/main.cpp +++ b/tests/regression/diverge/main.cpp @@ -277,7 +277,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!\nFAILED!\n"; - return errors; + return errors != 0; } std::cout << "PASSED!\n"; diff --git a/tests/regression/dogfood/main.cpp b/tests/regression/dogfood/main.cpp index e2f6bda765..7c52206262 100644 --- a/tests/regression/dogfood/main.cpp +++ b/tests/regression/dogfood/main.cpp @@ -237,5 +237,5 @@ int main(int argc, char *argv[]) { std::cout << "cleanup" << std::endl; cleanup(); - return errors; + return errors != 0; } \ No newline at end of file diff --git a/tests/regression/dxa_copy_mcast/main.cpp b/tests/regression/dxa_copy_mcast/main.cpp index 16134186e1..0316d6c51f 100644 --- a/tests/regression/dxa_copy_mcast/main.cpp +++ b/tests/regression/dxa_copy_mcast/main.cpp @@ -195,7 +195,7 @@ int main(int argc, char* argv[]) { cleanup(); if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; - return errors; + return errors != 0; } std::cout << "PASSED\n"; return 0; diff --git a/tests/regression/fence/main.cpp b/tests/regression/fence/main.cpp index 30ce2da511..98c701057b 100644 --- a/tests/regression/fence/main.cpp +++ b/tests/regression/fence/main.cpp @@ -184,7 +184,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/gbar_phase/main.cpp b/tests/regression/gbar_phase/main.cpp index 9a0a77adbc..7c947a2de3 100644 --- a/tests/regression/gbar_phase/main.cpp +++ b/tests/regression/gbar_phase/main.cpp @@ -168,7 +168,7 @@ int main(int argc, char* argv[]) { if (errors != 0) { std::cout << "Found " << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/io_addr/main.cpp b/tests/regression/io_addr/main.cpp index c3334ef377..1907980435 100644 --- a/tests/regression/io_addr/main.cpp +++ b/tests/regression/io_addr/main.cpp @@ -226,7 +226,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/madmax/main.cpp b/tests/regression/madmax/main.cpp index 2af9a86614..c2b7c45dd9 100755 --- a/tests/regression/madmax/main.cpp +++ b/tests/regression/madmax/main.cpp @@ -162,7 +162,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; return 0; diff --git a/tests/regression/module_reload/main.cpp b/tests/regression/module_reload/main.cpp index 6378d093e4..7ec08adc10 100644 --- a/tests/regression/module_reload/main.cpp +++ b/tests/regression/module_reload/main.cpp @@ -233,7 +233,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "Test PASSED" << std::endl; diff --git a/tests/regression/reopen/main.cpp b/tests/regression/reopen/main.cpp index abd691f4b0..f873a5dcaf 100644 --- a/tests/regression/reopen/main.cpp +++ b/tests/regression/reopen/main.cpp @@ -187,7 +187,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "Test PASSED" << std::endl; diff --git a/tests/regression/sgemm/main.cpp b/tests/regression/sgemm/main.cpp index 585de0d841..0f6cf0900d 100644 --- a/tests/regression/sgemm/main.cpp +++ b/tests/regression/sgemm/main.cpp @@ -158,7 +158,7 @@ int main(int argc, char** argv) { if (errors) { std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; return 0; diff --git a/tests/regression/sgemm2_dxa/main.cpp b/tests/regression/sgemm2_dxa/main.cpp index 3f5849209d..27837c320c 100644 --- a/tests/regression/sgemm2_dxa/main.cpp +++ b/tests/regression/sgemm2_dxa/main.cpp @@ -304,7 +304,7 @@ int main(int argc, char* argv[]) { if (errors != 0) { std::cout << "Found " << errors << " errors\n"; std::cout << "FAILED\n"; - return errors; + return errors != 0; } std::cout << "PASSED\n"; diff --git a/tests/regression/sgemm2_dxa_mcast/main.cpp b/tests/regression/sgemm2_dxa_mcast/main.cpp index e238e943b8..c3c11ce607 100644 --- a/tests/regression/sgemm2_dxa_mcast/main.cpp +++ b/tests/regression/sgemm2_dxa_mcast/main.cpp @@ -223,7 +223,7 @@ int main(int argc, char* argv[]) { } cleanup(); - if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; return errors; } + if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; return errors != 0; } std::cout << "PASSED\n"; return 0; } diff --git a/tests/regression/sgemm2_tcu/main.cpp b/tests/regression/sgemm2_tcu/main.cpp index a38b6c40f4..505efb20e1 100644 --- a/tests/regression/sgemm2_tcu/main.cpp +++ b/tests/regression/sgemm2_tcu/main.cpp @@ -588,7 +588,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm2_v1/main.cpp b/tests/regression/sgemm2_v1/main.cpp index cf70a37949..e01d37ba26 100644 --- a/tests/regression/sgemm2_v1/main.cpp +++ b/tests/regression/sgemm2_v1/main.cpp @@ -282,7 +282,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu/main.cpp b/tests/regression/sgemm_tcu/main.cpp index 0c9446a561..565f39530d 100644 --- a/tests/regression/sgemm_tcu/main.cpp +++ b/tests/regression/sgemm_tcu/main.cpp @@ -764,7 +764,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_mx/main.cpp b/tests/regression/sgemm_tcu_mx/main.cpp index f3d10d3b77..83a566e5b3 100644 --- a/tests/regression/sgemm_tcu_mx/main.cpp +++ b/tests/regression/sgemm_tcu_mx/main.cpp @@ -424,7 +424,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_sp_mx/main.cpp b/tests/regression/sgemm_tcu_sp_mx/main.cpp index b7b4e9048c..1033b7df89 100644 --- a/tests/regression/sgemm_tcu_sp_mx/main.cpp +++ b/tests/regression/sgemm_tcu_sp_mx/main.cpp @@ -504,7 +504,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg/main.cpp b/tests/regression/sgemm_tcu_wg/main.cpp index 2085b13249..9fe0860e2b 100644 --- a/tests/regression/sgemm_tcu_wg/main.cpp +++ b/tests/regression/sgemm_tcu_wg/main.cpp @@ -762,7 +762,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_dxa/main.cpp b/tests/regression/sgemm_tcu_wg_dxa/main.cpp index f9175cdf33..e29f62f7c9 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/main.cpp @@ -481,7 +481,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp index aca8b0962f..c72114c769 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp @@ -507,7 +507,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_mx/main.cpp b/tests/regression/sgemm_tcu_wg_mx/main.cpp index ac8b594628..a54056ddc2 100644 --- a/tests/regression/sgemm_tcu_wg_mx/main.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/main.cpp @@ -430,7 +430,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_sp/main.cpp b/tests/regression/sgemm_tcu_wg_sp/main.cpp index b96e99740e..d1d388f265 100644 --- a/tests/regression/sgemm_tcu_wg_sp/main.cpp +++ b/tests/regression/sgemm_tcu_wg_sp/main.cpp @@ -482,7 +482,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_sp_dxa/main.cpp b/tests/regression/sgemm_tcu_wg_sp_dxa/main.cpp index 95c3b1277f..58ef247e40 100644 --- a/tests/regression/sgemm_tcu_wg_sp_dxa/main.cpp +++ b/tests/regression/sgemm_tcu_wg_sp_dxa/main.cpp @@ -473,7 +473,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_v1/main.cpp b/tests/regression/sgemm_v1/main.cpp index 88326e9b27..d47a686a11 100644 --- a/tests/regression/sgemm_v1/main.cpp +++ b/tests/regression/sgemm_v1/main.cpp @@ -231,7 +231,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemmx/main.cpp b/tests/regression/sgemmx/main.cpp index 105d4c62da..528022d23b 100644 --- a/tests/regression/sgemmx/main.cpp +++ b/tests/regression/sgemmx/main.cpp @@ -252,7 +252,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sort/main.cpp b/tests/regression/sort/main.cpp index bfe56aa562..8a69a6bf18 100644 --- a/tests/regression/sort/main.cpp +++ b/tests/regression/sort/main.cpp @@ -197,7 +197,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/stencil3d/main.cpp b/tests/regression/stencil3d/main.cpp index e333438615..6a8d481f57 100644 --- a/tests/regression/stencil3d/main.cpp +++ b/tests/regression/stencil3d/main.cpp @@ -321,7 +321,7 @@ int main(int argc, char *argv[]) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/vm_stress/main.cpp b/tests/regression/vm_stress/main.cpp index cbb3a98939..9efa622346 100644 --- a/tests/regression/vm_stress/main.cpp +++ b/tests/regression/vm_stress/main.cpp @@ -177,7 +177,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/vm_test/main.cpp b/tests/regression/vm_test/main.cpp index 20f9b4b123..89bd2c400e 100644 --- a/tests/regression/vm_test/main.cpp +++ b/tests/regression/vm_test/main.cpp @@ -236,7 +236,7 @@ int main(int argc, char* argv[]) { if (errors != 0) { std::cout << "FAILED! - " << errors << " errors" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; return 0; diff --git a/tests/regression/wsync/main.cpp b/tests/regression/wsync/main.cpp index c1433659a6..c17611968b 100644 --- a/tests/regression/wsync/main.cpp +++ b/tests/regression/wsync/main.cpp @@ -157,7 +157,7 @@ int main(int argc, char* argv[]) { if (0 != errors) { std::cout << "WSYNC timing mismatches: " << errors << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; From 9226859262eb7b91cc3e39548190921d1810dfba Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 2 Nov 2026 20:30:19 -0800 Subject: [PATCH 336/336] profiles: divide/sqrt/convert rates, L1 banks and L2 banks to the parts - FDIV/FSQRT_PE_RATIO (and FCVT on NVIDIA cc 7.0+): division and square root are built on the reciprocal and reciprocal square root, which the parts run at 16/clk/SM on NVIDIA cc 7.0-12.0 (32 on cc 6.1) and at a quarter of the FP32 rate on RDNA; the ratio is FPU lanes / that rate. - DCACHE_NUM_BANKS 4 on v100 and h100 (128 B/clk L1; the default gives 8 banks of 32 B at 8 LSU lanes, as a100 already set). - L2_NUM_BANKS 64 on a100, h100 and rtx5080: 64 x 64 B is the power of two nearest the parts' L2 bandwidth per clock (A100 80 slices, about 5 KB/clk; RTX 5080 48 slices, about 3.5 KB/clk); 32 gave half. All profiles pass gen_config validation. Integration testing runs once the phase is code-complete. Co-Authored-By: Claude Opus 5.5 --- profiles/a100.json | 4 ++-- profiles/gtx1080.json | 7 +++++-- profiles/h100.json | 14 ++++++++++---- profiles/rtx3060.json | 8 ++++++-- profiles/rtx5080.json | 12 ++++++++---- profiles/rx6600.json | 7 +++++-- profiles/rx9070xt.json | 7 +++++-- profiles/v100.json | 10 ++++++++-- 8 files changed, 49 insertions(+), 20 deletions(-) diff --git a/profiles/a100.json b/profiles/a100.json index 62770583bd..c000279319 100644 --- a/profiles/a100.json +++ b/profiles/a100.json @@ -37,7 +37,7 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 33554432, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_L2_NUM_BANKS": 64, "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { @@ -54,7 +54,7 @@ "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and all type conversions not to or from 8/16-bit integers (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 8.0); 16 FPU lanes / 4", "DCACHE_NUM_BANKS": "128 B per clock of L1 data per SM (A100 whitepaper): 4 banks of the 32 B dcache word. The default (one bank per LSU request channel) gives 8, twice the part", "DCACHE_SIZE": "192 KB combined L1/shared per SM (whitepaper), rounded down to the power of two the cache accepts", - "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB; 16 ways as in GPGPU-Sim's SM80_A100 dl2; 64 banks of 64 B (4 KB/clk), the power of two nearest the 80 slices' 5 KB/clk", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM2 stacks x 8 128-bit channels = 40 channels at 2430 MT/s, modelled as 32 channels at 3038 MT/s for a power-of-two interleave: 32 x 16 B x 3038 MT/s = 1555 GB/s, the card's bandwidth; the array timing is in nanoseconds, so latency is unchanged", "PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_QUEUE_SIZE": "Fixed DRAM latency and per-channel request queue of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)", diff --git a/profiles/gtx1080.json b/profiles/gtx1080.json index e1961ff102..443dd54561 100644 --- a/profiles/gtx1080.json +++ b/profiles/gtx1080.json @@ -22,7 +22,9 @@ "VX_CFG_NUM_BARRIERS": 16, "VX_CFG_EXT_TEX_ENABLE": true, "VX_CFG_EXT_RASTER_ENABLE": true, - "VX_CFG_EXT_OM_ENABLE": true + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -57,6 +59,7 @@ "EXT_TEX_ENABLE": "Texture units in every SM", "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", "NUM_MUFU_LANES": "32 special-function results per clock per SM (8 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 6.1)", - "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GP104 SM (GTX 1080 whitepaper): 4 processing blocks, each with 32 CUDA cores (FP32 and INT32 on the same cores), 8 LD/ST units and 8 SFUs; 4 FP64 cores per SM. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 4" + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GP104 SM (GTX 1080 whitepaper): 4 processing blocks, each with 32 CUDA cores (FP32 and INT32 on the same cores), 8 LD/ST units and 8 SFUs; 4 FP64 cores per SM. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 4", + "FDIV_PE_RATIO, FSQRT_PE_RATIO": "32 results per clock per SM, 8 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 6.1); 32 FPU lanes / 4" } } diff --git a/profiles/h100.json b/profiles/h100.json index 56f0e658e7..fb5c6a2f94 100644 --- a/profiles/h100.json +++ b/profiles/h100.json @@ -27,7 +27,11 @@ "VX_CFG_TCU_WGMMA_ENABLE": true, "VX_CFG_EXT_DXA_ENABLE": true, "VX_CFG_NUM_BARRIERS": 16, - "VX_CFG_EXT_TEX_ENABLE": true + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 8, + "VX_CFG_FSQRT_PE_RATIO": 8, + "VX_CFG_FCVT_PE_RATIO": 8, + "VX_CFG_DCACHE_NUM_BANKS": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -35,7 +39,7 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 33554432, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_L2_NUM_BANKS": 64, "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { @@ -48,7 +52,7 @@ "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "132 SMs (SXM5), 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", "DCACHE_SIZE": "256 KB combined L1/shared per SM (whitepaper)", - "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways and 32 banks as in the a100 profile", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways; 64 banks of 64 B, the power of two nearest the part's L2 bandwidth per clock (as a100)", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM3 stacks x 16 64-bit channels at 5236 MT/s = 40 128-bit channel pairs, modelled as 32 at 6545 MT/s for a power-of-two interleave: 32 x 16 B x 6545 MT/s = 3.35 TB/s, the card's bandwidth; latency not calibrated", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)", "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", @@ -62,6 +66,8 @@ "EXT_TEX_ENABLE": "Texture units in every SM", "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Off: the datacenter part is built for compute (no display outputs; the raster and ROP hardware is reduced or absent), so the profile models its compute machine", "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 9.0)", - "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GH100 SM (H100 whitepaper, Figure 7): 4 processing blocks, each with 32 FP32, 16 INT32, 16 FP64 cores, 8 LD/ST units, 1 SFU and 1 tensor core. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 64" + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GH100 SM (H100 whitepaper, Figure 7): 4 processing blocks, each with 32 FP32, 16 INT32, 16 FP64 cores, 8 LD/ST units, 1 SFU and 1 tensor core. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 64", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and the 32-bit type conversions (CUDA C++ Programming Guide, arithmetic instruction throughput); 32 FPU lanes / 8, compute capability 9.0", + "DCACHE_NUM_BANKS": "128 B per clock of L1 data per SM: 4 banks of the 32 B dcache word. The default (one bank per LSU request channel) gives 8, twice the part" } } diff --git a/profiles/rtx3060.json b/profiles/rtx3060.json index 37f3c001e7..3f8d9dcb14 100644 --- a/profiles/rtx3060.json +++ b/profiles/rtx3060.json @@ -27,7 +27,10 @@ "VX_CFG_NUM_BARRIERS": 16, "VX_CFG_EXT_TEX_ENABLE": true, "VX_CFG_EXT_RASTER_ENABLE": true, - "VX_CFG_EXT_OM_ENABLE": true + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 8, + "VX_CFG_FSQRT_PE_RATIO": 8, + "VX_CFG_FCVT_PE_RATIO": 8 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -58,6 +61,7 @@ "EXT_RTU_ENABLE": "Off although the part has one RT core per SM: the RTU needs full-warp ALU lanes (NUM_ALU_LANES == NUM_THREADS), and the part's 16 INT32 lanes per block take precedence", "NUM_MUFU_LANES": "16 special-function results per clock per SM (compute capability 8.6)", "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, NUM_BARRIERS, EXT_TEX_ENABLE, EXT_RASTER_ENABLE, EXT_OM_ENABLE": "As in the other NVIDIA profiles: demand-paged MMU, atomics with CAS, FP16, 16 named barriers, and the graphics pipeline of a GeForce part", - "xlen": "64-bit machine: 64-bit pointers and FP64" + "xlen": "64-bit machine: 64-bit pointers and FP64", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and the 32-bit type conversions (CUDA C++ Programming Guide, arithmetic instruction throughput); 32 FPU lanes / 8, compute capability 8.6" } } diff --git a/profiles/rtx5080.json b/profiles/rtx5080.json index f42359b9bc..fbba2f3bf0 100644 --- a/profiles/rtx5080.json +++ b/profiles/rtx5080.json @@ -35,7 +35,10 @@ "VX_CFG_EXT_TEX_ENABLE": true, "VX_CFG_EXT_RASTER_ENABLE": true, "VX_CFG_EXT_OM_ENABLE": true, - "VX_CFG_RTU_TLAS_ENABLE": true + "VX_CFG_RTU_TLAS_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 8, + "VX_CFG_FSQRT_PE_RATIO": 8, + "VX_CFG_FCVT_PE_RATIO": 8 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -43,7 +46,7 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 67108864, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_L2_NUM_BANKS": 64, "VX_CFG_L2_MSHR_SIZE": 192, "VX_CFG_CP_LLC_CLIENT": 1 }, @@ -82,7 +85,7 @@ "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "84 SMs, 48 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA RTX Blackwell GPU Architecture whitepaper, App. B; cudaGetDeviceProperties on an RTX 5080)", "DCACHE_SIZE": "128 KB L1/shared per SM (whitepaper App. B)", "L2_SIZE, L2_NUM_WAYS": "64 MB L2 (whitepaper App. B); 16 ways as in GPGPU-Sim's Ampere/Hopper L2 configs", - "L2_NUM_BANKS": "32 banks as the nearest power of two to the 48 L2 slices measured by arXiv 2507.10789", + "L2_NUM_BANKS": "64 banks of 64 B (4 KB/clk): the 48 L2 slices measured by arXiv 2507.10789 deliver about 3.5 KB/clk, nearer 64 banks than 32", "DCACHE_MSHR_SIZE, L2_MSHR_SIZE": "384 L1 MSHR entries per SM (24 x 16 banks) and 192 per L2 slice, GPGPU-Sim SM86_RTX3070 (-gpgpu_cache:dl1 A:384:48, dl2 A:192:4)", "PLATFORM_MEMORY_NUM_BANKS": "8 x 32-bit GDDR7 devices = 32 independent channels (whitepaper App. B)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", @@ -101,6 +104,7 @@ "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", "RTU_TLAS_ENABLE": "RT cores traverse two-level acceleration structures (instance TLAS over BLAS), as DXR/Vulkan ray tracing requires", "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 12.0)", - "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GB203 SM (RTX Blackwell whitepaper, Figure 5 and App. B): 4 processing blocks, each with 32 unified FP32/INT32 CUDA cores (INT32 TOPS = FP32 TFLOPS, Table 4), 4 LD/ST units and 1 SFU, 1 tensor core; 2 FP64 cores per SM. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 2" + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GB203 SM (RTX Blackwell whitepaper, Figure 5 and App. B): 4 processing blocks, each with 32 unified FP32/INT32 CUDA cores (INT32 TOPS = FP32 TFLOPS, Table 4), 4 LD/ST units and 1 SFU, 1 tensor core; 2 FP64 cores per SM. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 2", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and the 32-bit type conversions (CUDA C++ Programming Guide, arithmetic instruction throughput); 32 FPU lanes / 8, compute capability 12.0" } } diff --git a/profiles/rx6600.json b/profiles/rx6600.json index 06e8f9e927..1e343cac94 100644 --- a/profiles/rx6600.json +++ b/profiles/rx6600.json @@ -25,7 +25,9 @@ "VX_CFG_NUM_BARRIERS": 16, "VX_CFG_EXT_TEX_ENABLE": true, "VX_CFG_EXT_RASTER_ENABLE": true, - "VX_CFG_EXT_OM_ENABLE": true + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -63,6 +65,7 @@ "EXT_TCU_ENABLE": "Off: RDNA 2 has no matrix units (only packed dot-product instructions)", "EXT_RTU_ENABLE": "One ray accelerator per CU", "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, NUM_BARRIERS, EXT_TEX_ENABLE, EXT_RASTER_ENABLE, EXT_OM_ENABLE": "GPUVM paging, global and LDS atomics with compare-swap, FP16 instructions and workgroup barriers, and the graphics pipeline (4 texture units per CU, rasterizers, ROPs) of a gaming part", - "xlen": "64-bit machine: 64-bit pointers and FP64. FP64 runs at 1/16 of FP32 (4 per CU per clock) on the part; with XLEN=64 every FPU lane executes FP64, so the profile has 64 FP64 lanes per CU" + "xlen": "64-bit machine: 64-bit pointers and FP64. FP64 runs at 1/16 of FP32 (4 per CU per clock) on the part; with XLEN=64 every FPU lane executes FP64, so the profile has 64 FP64 lanes per CU", + "FDIV_PE_RATIO, FSQRT_PE_RATIO": "Transcendentals (reciprocal, reciprocal square root, on which division and square root are built) run at a quarter of the FP32 rate on GCN, RDNA and CDNA: FPU lanes / 4" } } diff --git a/profiles/rx9070xt.json b/profiles/rx9070xt.json index 55f397cf6e..6f59395d8d 100644 --- a/profiles/rx9070xt.json +++ b/profiles/rx9070xt.json @@ -30,7 +30,9 @@ "VX_CFG_NUM_BARRIERS": 16, "VX_CFG_EXT_TEX_ENABLE": true, "VX_CFG_EXT_RASTER_ENABLE": true, - "VX_CFG_EXT_OM_ENABLE": true + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -68,6 +70,7 @@ "EXT_TCU_ENABLE, TCU_*": "Two AI accelerators (WMMA) per CU, one per SIMD as NUM_TCU_BLOCKS=ISSUE_WIDTH: FP16/BF16, FP8/BF8, INT8 and INT4 with 4:2 sparsity; no TF32", "EXT_RTU_ENABLE": "One ray accelerator per CU", "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, NUM_BARRIERS, EXT_TEX_ENABLE, EXT_RASTER_ENABLE, EXT_OM_ENABLE": "GPUVM paging, global and LDS atomics with compare-swap, FP16, workgroup barriers, and the graphics pipeline (4 texture units per CU, rasterizers, ROPs) of a gaming part", - "xlen": "64-bit machine: 64-bit pointers and FP64. FP64 runs at 1/32 of the dual-issue FP32 rate (4 per CU per clock) on the part; with XLEN=64 every FPU lane executes FP64, so the profile has 64 FP64 lanes per CU" + "xlen": "64-bit machine: 64-bit pointers and FP64. FP64 runs at 1/32 of the dual-issue FP32 rate (4 per CU per clock) on the part; with XLEN=64 every FPU lane executes FP64, so the profile has 64 FP64 lanes per CU", + "FDIV_PE_RATIO, FSQRT_PE_RATIO": "Transcendentals (reciprocal, reciprocal square root, on which division and square root are built) run at a quarter of the FP32 rate on GCN, RDNA and CDNA: FPU lanes / 4" } } diff --git a/profiles/v100.json b/profiles/v100.json index c75c2d48ae..dc1007341f 100644 --- a/profiles/v100.json +++ b/profiles/v100.json @@ -23,7 +23,11 @@ "VX_CFG_NUM_BARRIERS": 16, "VX_CFG_EXT_TEX_ENABLE": true, "VX_CFG_EXT_RASTER_ENABLE": true, - "VX_CFG_EXT_OM_ENABLE": true + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4, + "VX_CFG_FCVT_PE_RATIO": 4, + "VX_CFG_DCACHE_NUM_BANKS": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -58,6 +62,8 @@ "EXT_TEX_ENABLE": "Texture units in every SM", "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 7.0)", - "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GV100 SM (Volta whitepaper, Figure 5): 4 processing blocks, each with 16 FP32, 16 INT32, 8 FP64 cores, 8 LD/ST units, 1 SFU and 2 tensor cores. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 64 FP64 lanes per SM against the part's 32" + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GV100 SM (Volta whitepaper, Figure 5): 4 processing blocks, each with 16 FP32, 16 INT32, 8 FP64 cores, 8 LD/ST units, 1 SFU and 2 tensor cores. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 64 FP64 lanes per SM against the part's 32", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and the 32-bit type conversions (CUDA C++ Programming Guide, arithmetic instruction throughput); 16 FPU lanes / 4, compute capability 7.0", + "DCACHE_NUM_BANKS": "128 B per clock of L1 data per SM: 4 banks of the 32 B dcache word. The default (one bank per LSU request channel) gives 8, twice the part (Volta whitepaper: 128 B/clk L1)" } }